From 36e66158a20d6a0174e720fb880abfd6bf7489dc Mon Sep 17 00:00:00 2001 From: Joe-Blount <73478756+Joe-Blount@users.noreply.github.com> Date: Thu, 24 Aug 2023 09:13:23 -0500 Subject: [PATCH 1/5] chore: add log message before compacting partition (#8569) --- compactor/src/driver.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compactor/src/driver.rs b/compactor/src/driver.rs index 4f0d6726b6..51628679e5 100644 --- a/compactor/src/driver.rs +++ b/compactor/src/driver.rs @@ -73,6 +73,8 @@ async fn compact_partition( span.set_metadata("partition_id", partition_id.get().to_string()); let scratchpad = components.scratchpad_gen.pad(); + info!(partition_id = partition_id.get(), "compaction job starting"); + let res = timeout_with_progress_checking(partition_timeout, |transmit_progress_signal| { let components = Arc::clone(&components); let scratchpad = Arc::clone(&scratchpad); From 246918feb6c15e3d21c305b34ef223d9fafcab17 Mon Sep 17 00:00:00 2001 From: Nga Tran Date: Thu, 24 Aug 2023 12:16:12 -0400 Subject: [PATCH 2/5] feat: teach compactor to use sort_key_ids instead of sort_key (#8560) * feat: teach compactor to use sort_key_ids instead of sort_key * test: update the test output after chatting with Joe and know the reason of the chnanges --- .../src/components/columns_source/catalog.rs | 46 + .../src/components/columns_source/mock.rs | 71 ++ .../src/components/columns_source/mod.rs | 15 + compactor/src/components/hardcoded.rs | 2 + compactor/src/components/mod.rs | 1 + .../partition_info_source/sub_sources.rs | 65 +- compactor/tests/layouts/backfill.rs | 12 +- compactor/tests/layouts/stuck.rs | 978 +++++++++--------- data_types/src/partition.rs | 15 + iox_tests/src/builders.rs | 50 +- iox_tests/src/lib.rs | 4 +- 11 files changed, 753 insertions(+), 506 deletions(-) create mode 100644 compactor/src/components/columns_source/catalog.rs create mode 100644 compactor/src/components/columns_source/mock.rs create mode 100644 compactor/src/components/columns_source/mod.rs diff --git a/compactor/src/components/columns_source/catalog.rs b/compactor/src/components/columns_source/catalog.rs new file mode 100644 index 0000000000..87ed6c0e38 --- /dev/null +++ b/compactor/src/components/columns_source/catalog.rs @@ -0,0 +1,46 @@ +use std::{fmt::Display, sync::Arc}; + +use async_trait::async_trait; +use backoff::{Backoff, BackoffConfig}; +use data_types::{Column, TableId}; +use iox_catalog::interface::Catalog; + +use super::ColumnsSource; + +#[derive(Debug)] +pub struct CatalogColumnsSource { + backoff_config: BackoffConfig, + catalog: Arc, +} + +impl CatalogColumnsSource { + pub fn new(backoff_config: BackoffConfig, catalog: Arc) -> Self { + Self { + backoff_config, + catalog, + } + } +} + +impl Display for CatalogColumnsSource { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "catalog") + } +} + +#[async_trait] +impl ColumnsSource for CatalogColumnsSource { + async fn fetch(&self, table: TableId) -> Vec { + Backoff::new(&self.backoff_config) + .retry_all_errors("table_of_given_table_id", || async { + self.catalog + .repositories() + .await + .columns() + .list_by_table_id(table) + .await + }) + .await + .expect("retry forever") + } +} diff --git a/compactor/src/components/columns_source/mock.rs b/compactor/src/components/columns_source/mock.rs new file mode 100644 index 0000000000..8fe1128ecf --- /dev/null +++ b/compactor/src/components/columns_source/mock.rs @@ -0,0 +1,71 @@ +use std::{collections::HashMap, fmt::Display}; + +use async_trait::async_trait; +use data_types::{Column, TableId}; + +use super::ColumnsSource; + +#[derive(Debug)] +pub struct MockColumnsSource { + tables: HashMap>, +} + +impl MockColumnsSource { + #[allow(dead_code)] // not used anywhere + pub fn new(tables: HashMap>) -> Self { + Self { tables } + } +} + +impl Display for MockColumnsSource { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "mock") + } +} + +#[async_trait] +impl ColumnsSource for MockColumnsSource { + async fn fetch(&self, table: TableId) -> Vec { + self.tables.get(&table).cloned().unwrap_or_default() + } +} + +#[cfg(test)] +mod tests { + use data_types::ColumnType; + use iox_tests::{ColumnBuilder, TableBuilder}; + + use super::*; + + #[test] + fn test_display() { + assert_eq!( + MockColumnsSource::new(HashMap::default()).to_string(), + "mock", + ) + } + + #[tokio::test] + async fn test_fetch() { + // // t_1 has one column and t_2 has no column + let t1 = TableBuilder::new(1).with_name("table1").build(); + let t1_c1 = ColumnBuilder::new(1, t1.id.get()) + .with_name("time") + .with_column_type(ColumnType::Time) + .build(); + let t2 = TableBuilder::new(2).with_name("table2").build(); + + let tables = HashMap::from([(t1.id, vec![t1_c1.clone()]), (t2.id, vec![])]); + let source = MockColumnsSource::new(tables); + + // different tables + assert_eq!(source.fetch(t1.id).await, vec![t1_c1.clone()],); + assert_eq!(source.fetch(t2.id).await, vec![]); + + // fetching does not drain + assert_eq!(source.fetch(t1.id).await, vec![t1_c1],); + + // unknown table => empty result + assert_eq!(source.fetch(TableId::new(3)).await, vec![]); + } +} diff --git a/compactor/src/components/columns_source/mod.rs b/compactor/src/components/columns_source/mod.rs new file mode 100644 index 0000000000..eeac48db26 --- /dev/null +++ b/compactor/src/components/columns_source/mod.rs @@ -0,0 +1,15 @@ +use std::fmt::{Debug, Display}; + +use async_trait::async_trait; +use data_types::{Column, TableId}; + +pub mod catalog; +pub mod mock; + +#[async_trait] +pub trait ColumnsSource: Debug + Display + Send + Sync { + /// Get Columns for a given table + /// + /// This method performs retries. + async fn fetch(&self, table: TableId) -> Vec; +} diff --git a/compactor/src/components/hardcoded.rs b/compactor/src/components/hardcoded.rs index faa78c83d0..bce0bacc3a 100644 --- a/compactor/src/components/hardcoded.rs +++ b/compactor/src/components/hardcoded.rs @@ -12,6 +12,7 @@ use crate::{config::Config, error::ErrorKind, object_store::ignore_writes::Ignor use super::{ changed_files_filter::logging::LoggingChangedFiles, + columns_source::catalog::CatalogColumnsSource, commit::CommitToScheduler, compaction_job_done_sink::{ error_kind::ErrorKindCompactionJobDoneSinkWrapper, @@ -192,6 +193,7 @@ fn make_compaction_job_stream( fn make_partition_info_source(config: &Config) -> Arc { Arc::new(SubSourcePartitionInfoSource::new( + CatalogColumnsSource::new(config.backoff_config.clone(), Arc::clone(&config.catalog)), LoggingPartitionSourceWrapper::new(MetricsPartitionSourceWrapper::new( CatalogPartitionSource::new(config.backoff_config.clone(), Arc::clone(&config.catalog)), &config.metric_registry, diff --git a/compactor/src/components/mod.rs b/compactor/src/components/mod.rs index 04902a593d..98fba7d9a0 100644 --- a/compactor/src/components/mod.rs +++ b/compactor/src/components/mod.rs @@ -12,6 +12,7 @@ use self::{ }; pub mod changed_files_filter; +pub mod columns_source; pub(crate) mod commit; pub mod compaction_job_done_sink; pub mod compaction_job_stream; diff --git a/compactor/src/components/partition_info_source/sub_sources.rs b/compactor/src/components/partition_info_source/sub_sources.rs index f2bd7e51c5..f671c79c76 100644 --- a/compactor/src/components/partition_info_source/sub_sources.rs +++ b/compactor/src/components/partition_info_source/sub_sources.rs @@ -2,11 +2,12 @@ use std::{fmt::Display, sync::Arc}; use async_trait::async_trait; use data_types::PartitionId; +use schema::sort::SortKey; use crate::{ components::{ - namespaces_source::NamespacesSource, partition_source::PartitionSource, - tables_source::TablesSource, + columns_source::ColumnsSource, namespaces_source::NamespacesSource, + partition_source::PartitionSource, tables_source::TablesSource, }, error::DynError, partition_info::PartitionInfo, @@ -15,25 +16,34 @@ use crate::{ use super::PartitionInfoSource; #[derive(Debug)] -pub struct SubSourcePartitionInfoSource +pub struct SubSourcePartitionInfoSource where + C: ColumnsSource, P: PartitionSource, T: TablesSource, N: NamespacesSource, { + columns_source: C, partition_source: P, tables_source: T, namespaces_source: N, } -impl SubSourcePartitionInfoSource +impl SubSourcePartitionInfoSource where + C: ColumnsSource, P: PartitionSource, T: TablesSource, N: NamespacesSource, { - pub fn new(partition_source: P, tables_source: T, namespaces_source: N) -> Self { + pub fn new( + columns_source: C, + partition_source: P, + tables_source: T, + namespaces_source: N, + ) -> Self { Self { + columns_source, partition_source, tables_source, namespaces_source, @@ -41,8 +51,9 @@ where } } -impl Display for SubSourcePartitionInfoSource +impl Display for SubSourcePartitionInfoSource where + C: ColumnsSource, P: PartitionSource, T: TablesSource, N: NamespacesSource, @@ -57,8 +68,9 @@ where } #[async_trait] -impl PartitionInfoSource for SubSourcePartitionInfoSource +impl PartitionInfoSource for SubSourcePartitionInfoSource where + C: ColumnsSource, P: PartitionSource, T: TablesSource, N: NamespacesSource, @@ -96,6 +108,43 @@ where .get(&table.name) .ok_or_else::(|| String::from("Cannot find table schema").into())?; + // fetch table columns to get column names for the partition's sort_key_ids + let columns = self.columns_source.fetch(table.id).await; + + // sort_key_ids of the partition + let sort_key_ids = partition.sort_key_ids_none_if_empty(); + // sort_key of the partition. This will be removed but until then, use it to validate the + // sort_key computed by mapping sort_key_ids to column names + let p_sort_key = partition.sort_key(); + + // convert column ids to column names + let sort_key = sort_key_ids.as_ref().map(|ids| { + let names = ids + .iter() + .map(|id| { + columns + .iter() + .find(|c| c.id == *id) + .map(|c| c.name.clone()) + .ok_or_else::(|| { + format!( + "Cannot find column with id {} for table {}", + id.get(), + table.name + ) + .into() + }) + }) + .collect::, _>>() + .expect("Cannot find column names for sort key ids"); + + SortKey::from_columns(names.iter().map(|s| &**s)) + }); + + // This is here to catch bugs if any while mapping sort_key_ids to column names + // This wil be removed once sort_key is removed from partition + assert_eq!(sort_key, p_sort_key); + Ok(Arc::new(PartitionInfo { partition_id, partition_hash_id: partition.hash_id().cloned(), @@ -103,7 +152,7 @@ where namespace_name: namespace.name, table: Arc::new(table), table_schema: Arc::new(table_schema.clone()), - sort_key: partition.sort_key(), + sort_key, partition_key: partition.partition_key, })) } diff --git a/compactor/tests/layouts/backfill.rs b/compactor/tests/layouts/backfill.rs index 35f5760d56..7140198942 100644 --- a/compactor/tests/layouts/backfill.rs +++ b/compactor/tests/layouts/backfill.rs @@ -738,7 +738,7 @@ async fn random_backfill_empty_partition() { - "L0 " - "L0.190[0,355] 1.02us 76mb|------------L0.190------------| " - "L0.193[356,668] 1.02us 79mb |----------L0.193----------| " - - "L0.195[669,986] 1.02us 67mb |----------L0.195----------| " + - "L0.194[669,986] 1.02us 67mb |----------L0.194----------| " - "L0.191[42,355] 1.04us 71mb |----------L0.191----------| " - "**** 3 Output Files (parquet_file_id not yet assigned), 292mb total:" - "L1 " @@ -746,11 +746,11 @@ async fn random_backfill_empty_partition() { - "L1.?[339,676] 1.04us 100mb |------------L1.?------------| " - "L1.?[677,986] 1.04us 92mb |-----------L1.?-----------| " - "Committing partition 1:" - - " Soft Deleting 4 files: L0.190, L0.191, L0.193, L0.195" + - " Soft Deleting 4 files: L0.190, L0.191, L0.193, L0.194" - " Creating 3 files" - "**** Simulation run 59, type=split(ReduceOverlap)(split_times=[676]). 1 Input Files, 60mb total:" - "L0, all files 60mb " - - "L0.196[669,986] 1.05us |-----------------------------------------L0.196-----------------------------------------|" + - "L0.195[669,986] 1.05us |-----------------------------------------L0.195-----------------------------------------|" - "**** 2 Output Files (parquet_file_id not yet assigned), 60mb total:" - "L0 " - "L0.?[669,676] 1.05us 2mb |L0.?| " @@ -763,11 +763,11 @@ async fn random_backfill_empty_partition() { - "L0.?[42,338] 1.05us 38mb |---------------------------------------L0.?----------------------------------------| " - "L0.?[339,355] 1.05us 2mb |L0.?|" - "Committing partition 1:" - - " Soft Deleting 2 files: L0.192, L0.196" + - " Soft Deleting 2 files: L0.192, L0.195" - " Creating 4 files" - "**** Simulation run 61, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[528]). 2 Input Files, 179mb total:" - "L0 " - - "L0.194[356,668] 1.04us 79mb |-------------------------------------L0.194--------------------------------------| " + - "L0.196[356,668] 1.04us 79mb |-------------------------------------L0.196--------------------------------------| " - "L1 " - "L1.199[339,676] 1.04us 100mb|-----------------------------------------L1.199-----------------------------------------|" - "**** 2 Output Files (parquet_file_id not yet assigned), 179mb total:" @@ -775,7 +775,7 @@ async fn random_backfill_empty_partition() { - "L1.?[339,528] 1.04us 100mb|----------------------L1.?----------------------| " - "L1.?[529,676] 1.04us 78mb |----------------L1.?-----------------| " - "Committing partition 1:" - - " Soft Deleting 2 files: L0.194, L1.199" + - " Soft Deleting 2 files: L0.196, L1.199" - " Creating 2 files" - "**** Simulation run 62, type=split(ReduceOverlap)(split_times=[528]). 1 Input Files, 38mb total:" - "L0, all files 38mb " diff --git a/compactor/tests/layouts/stuck.rs b/compactor/tests/layouts/stuck.rs index 3a256c2cdd..8ac51d2af6 100644 --- a/compactor/tests/layouts/stuck.rs +++ b/compactor/tests/layouts/stuck.rs @@ -2044,24 +2044,25 @@ async fn stuck_l0_large_l0s() { - "L0.?[1602,1761] 17ns 8mb |L0.?-| " - "L0.?[1762,1921] 17ns 8mb |L0.?-| " - "L0.?[1922,2000] 17ns 4mb |L0.?|" - - "**** Simulation run 18, type=split(VerticalSplit)(split_times=[161, 321, 481, 641, 801, 961, 1121, 1281, 1441, 1601, 1761, 1921]). 1 Input Files, 100mb total:" - - "L0, all files 100mb " - - "L0.19[1,2000] 18ns |-----------------------------------------L0.19------------------------------------------|" - - "**** 13 Output Files (parquet_file_id not yet assigned), 100mb total:" + - "**** Simulation run 18, type=split(VerticalSplit)(split_times=[161, 321, 481, 641, 801, 961, 1121, 1281, 1441, 1601, 1761, 1921, 2086]). 1 Input Files, 10b total:" + - "L0, all files 10b " + - "L0.21[20,200000] 20ns |-----------------------------------------L0.21------------------------------------------|" + - "**** 14 Output Files (parquet_file_id not yet assigned), 10b total:" - "L0 " - - "L0.?[1,161] 18ns 8mb |L0.?-| " - - "L0.?[162,321] 18ns 8mb |L0.?-| " - - "L0.?[322,481] 18ns 8mb |L0.?-| " - - "L0.?[482,641] 18ns 8mb |L0.?-| " - - "L0.?[642,801] 18ns 8mb |L0.?-| " - - "L0.?[802,961] 18ns 8mb |L0.?-| " - - "L0.?[962,1121] 18ns 8mb |L0.?-| " - - "L0.?[1122,1281] 18ns 8mb |L0.?-| " - - "L0.?[1282,1441] 18ns 8mb |L0.?-| " - - "L0.?[1442,1601] 18ns 8mb |L0.?-| " - - "L0.?[1602,1761] 18ns 8mb |L0.?-| " - - "L0.?[1762,1921] 18ns 8mb |L0.?-| " - - "L0.?[1922,2000] 18ns 4mb |L0.?|" + - "L0.?[20,161] 20ns 0b |L0.?| " + - "L0.?[162,321] 20ns 0b |L0.?| " + - "L0.?[322,481] 20ns 0b |L0.?| " + - "L0.?[482,641] 20ns 0b |L0.?| " + - "L0.?[642,801] 20ns 0b |L0.?| " + - "L0.?[802,961] 20ns 0b |L0.?| " + - "L0.?[962,1121] 20ns 0b |L0.?| " + - "L0.?[1122,1281] 20ns 0b |L0.?| " + - "L0.?[1282,1441] 20ns 0b |L0.?| " + - "L0.?[1442,1601] 20ns 0b |L0.?| " + - "L0.?[1602,1761] 20ns 0b |L0.?| " + - "L0.?[1762,1921] 20ns 0b |L0.?| " + - "L0.?[1922,2086] 20ns 0b |L0.?| " + - "L0.?[2087,200000] 20ns 10b|-----------------------------------------L0.?------------------------------------------| " - "**** Simulation run 19, type=split(VerticalSplit)(split_times=[161, 321, 481, 641, 801, 961, 1121, 1281, 1441, 1601, 1761, 1921, 2086]). 1 Input Files, 10b total:" - "L0, all files 10b " - "L0.22[21,210000] 21ns |-----------------------------------------L0.22------------------------------------------|" @@ -2195,25 +2196,24 @@ async fn stuck_l0_large_l0s() { - "L0.?[1762,1921] 27ns 0b |L0.?| " - "L0.?[1922,2086] 27ns 0b |L0.?| " - "L0.?[2087,270000] 27ns 10b|-----------------------------------------L0.?------------------------------------------| " - - "**** Simulation run 26, type=split(VerticalSplit)(split_times=[161, 321, 481, 641, 801, 961, 1121, 1281, 1441, 1601, 1761, 1921, 2086]). 1 Input Files, 10b total:" - - "L0, all files 10b " - - "L0.29[28,280000] 28ns |-----------------------------------------L0.29------------------------------------------|" - - "**** 14 Output Files (parquet_file_id not yet assigned), 10b total:" + - "**** Simulation run 26, type=split(VerticalSplit)(split_times=[161, 321, 481, 641, 801, 961, 1121, 1281, 1441, 1601, 1761, 1921]). 1 Input Files, 100mb total:" + - "L0, all files 100mb " + - "L0.19[1,2000] 18ns |-----------------------------------------L0.19------------------------------------------|" + - "**** 13 Output Files (parquet_file_id not yet assigned), 100mb total:" - "L0 " - - "L0.?[28,161] 28ns 0b |L0.?| " - - "L0.?[162,321] 28ns 0b |L0.?| " - - "L0.?[322,481] 28ns 0b |L0.?| " - - "L0.?[482,641] 28ns 0b |L0.?| " - - "L0.?[642,801] 28ns 0b |L0.?| " - - "L0.?[802,961] 28ns 0b |L0.?| " - - "L0.?[962,1121] 28ns 0b |L0.?| " - - "L0.?[1122,1281] 28ns 0b |L0.?| " - - "L0.?[1282,1441] 28ns 0b |L0.?| " - - "L0.?[1442,1601] 28ns 0b |L0.?| " - - "L0.?[1602,1761] 28ns 0b |L0.?| " - - "L0.?[1762,1921] 28ns 0b |L0.?| " - - "L0.?[1922,2086] 28ns 0b |L0.?| " - - "L0.?[2087,280000] 28ns 10b|-----------------------------------------L0.?------------------------------------------| " + - "L0.?[1,161] 18ns 8mb |L0.?-| " + - "L0.?[162,321] 18ns 8mb |L0.?-| " + - "L0.?[322,481] 18ns 8mb |L0.?-| " + - "L0.?[482,641] 18ns 8mb |L0.?-| " + - "L0.?[642,801] 18ns 8mb |L0.?-| " + - "L0.?[802,961] 18ns 8mb |L0.?-| " + - "L0.?[962,1121] 18ns 8mb |L0.?-| " + - "L0.?[1122,1281] 18ns 8mb |L0.?-| " + - "L0.?[1282,1441] 18ns 8mb |L0.?-| " + - "L0.?[1442,1601] 18ns 8mb |L0.?-| " + - "L0.?[1602,1761] 18ns 8mb |L0.?-| " + - "L0.?[1762,1921] 18ns 8mb |L0.?-| " + - "L0.?[1922,2000] 18ns 4mb |L0.?|" - "**** Simulation run 27, type=split(VerticalSplit)(split_times=[161, 321, 481, 641, 801, 961, 1121, 1281, 1441, 1601, 1761, 1921]). 1 Input Files, 100mb total:" - "L0, all files 100mb " - "L0.20[1,2000] 19ns |-----------------------------------------L0.20------------------------------------------|" @@ -2234,23 +2234,23 @@ async fn stuck_l0_large_l0s() { - "L0.?[1922,2000] 19ns 4mb |L0.?|" - "**** Simulation run 28, type=split(VerticalSplit)(split_times=[161, 321, 481, 641, 801, 961, 1121, 1281, 1441, 1601, 1761, 1921, 2086]). 1 Input Files, 10b total:" - "L0, all files 10b " - - "L0.21[20,200000] 20ns |-----------------------------------------L0.21------------------------------------------|" + - "L0.29[28,280000] 28ns |-----------------------------------------L0.29------------------------------------------|" - "**** 14 Output Files (parquet_file_id not yet assigned), 10b total:" - "L0 " - - "L0.?[20,161] 20ns 0b |L0.?| " - - "L0.?[162,321] 20ns 0b |L0.?| " - - "L0.?[322,481] 20ns 0b |L0.?| " - - "L0.?[482,641] 20ns 0b |L0.?| " - - "L0.?[642,801] 20ns 0b |L0.?| " - - "L0.?[802,961] 20ns 0b |L0.?| " - - "L0.?[962,1121] 20ns 0b |L0.?| " - - "L0.?[1122,1281] 20ns 0b |L0.?| " - - "L0.?[1282,1441] 20ns 0b |L0.?| " - - "L0.?[1442,1601] 20ns 0b |L0.?| " - - "L0.?[1602,1761] 20ns 0b |L0.?| " - - "L0.?[1762,1921] 20ns 0b |L0.?| " - - "L0.?[1922,2086] 20ns 0b |L0.?| " - - "L0.?[2087,200000] 20ns 10b|-----------------------------------------L0.?------------------------------------------| " + - "L0.?[28,161] 28ns 0b |L0.?| " + - "L0.?[162,321] 28ns 0b |L0.?| " + - "L0.?[322,481] 28ns 0b |L0.?| " + - "L0.?[482,641] 28ns 0b |L0.?| " + - "L0.?[642,801] 28ns 0b |L0.?| " + - "L0.?[802,961] 28ns 0b |L0.?| " + - "L0.?[962,1121] 28ns 0b |L0.?| " + - "L0.?[1122,1281] 28ns 0b |L0.?| " + - "L0.?[1282,1441] 28ns 0b |L0.?| " + - "L0.?[1442,1601] 28ns 0b |L0.?| " + - "L0.?[1602,1761] 28ns 0b |L0.?| " + - "L0.?[1762,1921] 28ns 0b |L0.?| " + - "L0.?[1922,2086] 28ns 0b |L0.?| " + - "L0.?[2087,280000] 28ns 10b|-----------------------------------------L0.?------------------------------------------| " - "**** Simulation run 29, type=split(VerticalSplit)(split_times=[161, 321, 481, 641, 801, 961, 1121, 1281, 1441, 1601, 1761, 1921, 2086]). 1 Input Files, 10b total:" - "L0, all files 10b " - "L0.30[29,290000] 29ns |-----------------------------------------L0.30------------------------------------------|" @@ -4257,23 +4257,23 @@ async fn stuck_l0_large_l0s() { - "L0.?[2087,1330000] 133ns 10b|-----------------------------------------L0.?------------------------------------------| " - "**** Simulation run 134, type=split(VerticalSplit)(split_times=[161, 321, 481, 641, 801, 961, 1121, 1281, 1441, 1601, 1761, 1921, 2086]). 1 Input Files, 10b total:" - "L0, all files 10b " - - "L0.135[134,1340000] 134ns|-----------------------------------------L0.135-----------------------------------------|" + - "L0.137[136,1360000] 136ns|-----------------------------------------L0.137-----------------------------------------|" - "**** 14 Output Files (parquet_file_id not yet assigned), 10b total:" - "L0 " - - "L0.?[134,161] 134ns 0b |L0.?| " - - "L0.?[162,321] 134ns 0b |L0.?| " - - "L0.?[322,481] 134ns 0b |L0.?| " - - "L0.?[482,641] 134ns 0b |L0.?| " - - "L0.?[642,801] 134ns 0b |L0.?| " - - "L0.?[802,961] 134ns 0b |L0.?| " - - "L0.?[962,1121] 134ns 0b |L0.?| " - - "L0.?[1122,1281] 134ns 0b |L0.?| " - - "L0.?[1282,1441] 134ns 0b |L0.?| " - - "L0.?[1442,1601] 134ns 0b |L0.?| " - - "L0.?[1602,1761] 134ns 0b |L0.?| " - - "L0.?[1762,1921] 134ns 0b |L0.?| " - - "L0.?[1922,2086] 134ns 0b |L0.?| " - - "L0.?[2087,1340000] 134ns 10b|-----------------------------------------L0.?------------------------------------------| " + - "L0.?[136,161] 136ns 0b |L0.?| " + - "L0.?[162,321] 136ns 0b |L0.?| " + - "L0.?[322,481] 136ns 0b |L0.?| " + - "L0.?[482,641] 136ns 0b |L0.?| " + - "L0.?[642,801] 136ns 0b |L0.?| " + - "L0.?[802,961] 136ns 0b |L0.?| " + - "L0.?[962,1121] 136ns 0b |L0.?| " + - "L0.?[1122,1281] 136ns 0b |L0.?| " + - "L0.?[1282,1441] 136ns 0b |L0.?| " + - "L0.?[1442,1601] 136ns 0b |L0.?| " + - "L0.?[1602,1761] 136ns 0b |L0.?| " + - "L0.?[1762,1921] 136ns 0b |L0.?| " + - "L0.?[1922,2086] 136ns 0b |L0.?| " + - "L0.?[2087,1360000] 136ns 10b|-----------------------------------------L0.?------------------------------------------| " - "**** Simulation run 135, type=split(VerticalSplit)(split_times=[161, 321, 481, 641, 801, 961, 1121, 1281, 1441, 1601, 1761, 1921, 2086]). 1 Input Files, 10b total:" - "L0, all files 10b " - "L0.138[137,1370000] 137ns|-----------------------------------------L0.138-----------------------------------------|" @@ -4409,23 +4409,23 @@ async fn stuck_l0_large_l0s() { - "L0.?[2087,1430000] 143ns 10b|-----------------------------------------L0.?------------------------------------------| " - "**** Simulation run 142, type=split(VerticalSplit)(split_times=[161, 321, 481, 641, 801, 961, 1121, 1281, 1441, 1601, 1761, 1921, 2086]). 1 Input Files, 10b total:" - "L0, all files 10b " - - "L0.145[144,1440000] 144ns|-----------------------------------------L0.145-----------------------------------------|" + - "L0.135[134,1340000] 134ns|-----------------------------------------L0.135-----------------------------------------|" - "**** 14 Output Files (parquet_file_id not yet assigned), 10b total:" - "L0 " - - "L0.?[144,161] 144ns 0b |L0.?| " - - "L0.?[162,321] 144ns 0b |L0.?| " - - "L0.?[322,481] 144ns 0b |L0.?| " - - "L0.?[482,641] 144ns 0b |L0.?| " - - "L0.?[642,801] 144ns 0b |L0.?| " - - "L0.?[802,961] 144ns 0b |L0.?| " - - "L0.?[962,1121] 144ns 0b |L0.?| " - - "L0.?[1122,1281] 144ns 0b |L0.?| " - - "L0.?[1282,1441] 144ns 0b |L0.?| " - - "L0.?[1442,1601] 144ns 0b |L0.?| " - - "L0.?[1602,1761] 144ns 0b |L0.?| " - - "L0.?[1762,1921] 144ns 0b |L0.?| " - - "L0.?[1922,2086] 144ns 0b |L0.?| " - - "L0.?[2087,1440000] 144ns 10b|-----------------------------------------L0.?------------------------------------------| " + - "L0.?[134,161] 134ns 0b |L0.?| " + - "L0.?[162,321] 134ns 0b |L0.?| " + - "L0.?[322,481] 134ns 0b |L0.?| " + - "L0.?[482,641] 134ns 0b |L0.?| " + - "L0.?[642,801] 134ns 0b |L0.?| " + - "L0.?[802,961] 134ns 0b |L0.?| " + - "L0.?[962,1121] 134ns 0b |L0.?| " + - "L0.?[1122,1281] 134ns 0b |L0.?| " + - "L0.?[1282,1441] 134ns 0b |L0.?| " + - "L0.?[1442,1601] 134ns 0b |L0.?| " + - "L0.?[1602,1761] 134ns 0b |L0.?| " + - "L0.?[1762,1921] 134ns 0b |L0.?| " + - "L0.?[1922,2086] 134ns 0b |L0.?| " + - "L0.?[2087,1340000] 134ns 10b|-----------------------------------------L0.?------------------------------------------| " - "**** Simulation run 143, type=split(VerticalSplit)(split_times=[161, 321, 481, 641, 801, 961, 1121, 1281, 1441, 1601, 1761, 1921, 2086]). 1 Input Files, 10b total:" - "L0, all files 10b " - "L0.136[135,1350000] 135ns|-----------------------------------------L0.136-----------------------------------------|" @@ -4447,23 +4447,23 @@ async fn stuck_l0_large_l0s() { - "L0.?[2087,1350000] 135ns 10b|-----------------------------------------L0.?------------------------------------------| " - "**** Simulation run 144, type=split(VerticalSplit)(split_times=[161, 321, 481, 641, 801, 961, 1121, 1281, 1441, 1601, 1761, 1921, 2086]). 1 Input Files, 10b total:" - "L0, all files 10b " - - "L0.137[136,1360000] 136ns|-----------------------------------------L0.137-----------------------------------------|" + - "L0.145[144,1440000] 144ns|-----------------------------------------L0.145-----------------------------------------|" - "**** 14 Output Files (parquet_file_id not yet assigned), 10b total:" - "L0 " - - "L0.?[136,161] 136ns 0b |L0.?| " - - "L0.?[162,321] 136ns 0b |L0.?| " - - "L0.?[322,481] 136ns 0b |L0.?| " - - "L0.?[482,641] 136ns 0b |L0.?| " - - "L0.?[642,801] 136ns 0b |L0.?| " - - "L0.?[802,961] 136ns 0b |L0.?| " - - "L0.?[962,1121] 136ns 0b |L0.?| " - - "L0.?[1122,1281] 136ns 0b |L0.?| " - - "L0.?[1282,1441] 136ns 0b |L0.?| " - - "L0.?[1442,1601] 136ns 0b |L0.?| " - - "L0.?[1602,1761] 136ns 0b |L0.?| " - - "L0.?[1762,1921] 136ns 0b |L0.?| " - - "L0.?[1922,2086] 136ns 0b |L0.?| " - - "L0.?[2087,1360000] 136ns 10b|-----------------------------------------L0.?------------------------------------------| " + - "L0.?[144,161] 144ns 0b |L0.?| " + - "L0.?[162,321] 144ns 0b |L0.?| " + - "L0.?[322,481] 144ns 0b |L0.?| " + - "L0.?[482,641] 144ns 0b |L0.?| " + - "L0.?[642,801] 144ns 0b |L0.?| " + - "L0.?[802,961] 144ns 0b |L0.?| " + - "L0.?[962,1121] 144ns 0b |L0.?| " + - "L0.?[1122,1281] 144ns 0b |L0.?| " + - "L0.?[1282,1441] 144ns 0b |L0.?| " + - "L0.?[1442,1601] 144ns 0b |L0.?| " + - "L0.?[1602,1761] 144ns 0b |L0.?| " + - "L0.?[1762,1921] 144ns 0b |L0.?| " + - "L0.?[1922,2086] 144ns 0b |L0.?| " + - "L0.?[2087,1440000] 144ns 10b|-----------------------------------------L0.?------------------------------------------| " - "**** Simulation run 145, type=split(VerticalSplit)(split_times=[161, 321, 481, 641, 801, 961, 1121, 1281, 1441, 1601, 1761, 1921, 2086]). 1 Input Files, 10b total:" - "L0, all files 10b " - "L0.146[145,1450000] 145ns|-----------------------------------------L0.146-----------------------------------------|" @@ -5497,25 +5497,25 @@ async fn stuck_l0_large_l0s() { - "L0.396[1,161] 15ns |-----------------------------------------L0.396-----------------------------------------|" - "L0.409[1,161] 16ns |-----------------------------------------L0.409-----------------------------------------|" - "L0.422[1,161] 17ns |-----------------------------------------L0.422-----------------------------------------|" - - "L0.435[1,161] 18ns |-----------------------------------------L0.435-----------------------------------------|" + - "L0.547[1,161] 18ns |-----------------------------------------L0.547-----------------------------------------|" - "L0.560[1,161] 19ns |-----------------------------------------L0.560-----------------------------------------|" - "**** 1 Output Files (parquet_file_id not yet assigned), 161mb total:" - "L0, all files 161mb " - "L0.?[1,161] 19ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.201, L0.214, L0.227, L0.240, L0.253, L0.266, L0.279, L0.292, L0.305, L0.318, L0.331, L0.344, L0.357, L0.370, L0.383, L0.396, L0.409, L0.422, L0.435, L0.560" + - " Soft Deleting 20 files: L0.201, L0.214, L0.227, L0.240, L0.253, L0.266, L0.279, L0.292, L0.305, L0.318, L0.331, L0.344, L0.357, L0.370, L0.383, L0.396, L0.409, L0.422, L0.547, L0.560" - " Creating 1 files" - "**** Simulation run 201, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " - - "L0.573[20,161] 20ns |-----------------------------------------L0.573-----------------------------------------|" - - "L0.448[21,161] 21ns |----------------------------------------L0.448-----------------------------------------| " - - "L0.462[22,161] 22ns |----------------------------------------L0.462----------------------------------------| " - - "L0.476[23,161] 23ns |----------------------------------------L0.476----------------------------------------| " - - "L0.490[24,161] 24ns |---------------------------------------L0.490----------------------------------------| " - - "L0.504[25,161] 25ns |---------------------------------------L0.504---------------------------------------| " - - "L0.518[26,161] 26ns |---------------------------------------L0.518---------------------------------------| " - - "L0.532[27,161] 27ns |--------------------------------------L0.532---------------------------------------| " - - "L0.546[28,161] 28ns |--------------------------------------L0.546--------------------------------------| " + - "L0.435[20,161] 20ns |-----------------------------------------L0.435-----------------------------------------|" + - "L0.449[21,161] 21ns |----------------------------------------L0.449-----------------------------------------| " + - "L0.463[22,161] 22ns |----------------------------------------L0.463----------------------------------------| " + - "L0.477[23,161] 23ns |----------------------------------------L0.477----------------------------------------| " + - "L0.491[24,161] 24ns |---------------------------------------L0.491----------------------------------------| " + - "L0.505[25,161] 25ns |---------------------------------------L0.505---------------------------------------| " + - "L0.519[26,161] 26ns |---------------------------------------L0.519---------------------------------------| " + - "L0.533[27,161] 27ns |--------------------------------------L0.533---------------------------------------| " + - "L0.573[28,161] 28ns |--------------------------------------L0.573--------------------------------------| " - "L0.587[29,161] 29ns |--------------------------------------L0.587--------------------------------------| " - "L0.601[30,161] 30ns |-------------------------------------L0.601--------------------------------------| " - "L0.615[31,161] 31ns |-------------------------------------L0.615-------------------------------------| " @@ -5531,7 +5531,7 @@ async fn stuck_l0_large_l0s() { - "L0, all files 0b " - "L0.?[20,161] 39ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.448, L0.462, L0.476, L0.490, L0.504, L0.518, L0.532, L0.546, L0.573, L0.587, L0.601, L0.615, L0.629, L0.643, L0.657, L0.671, L0.685, L0.699, L0.713, L0.727" + - " Soft Deleting 20 files: L0.435, L0.449, L0.463, L0.477, L0.491, L0.505, L0.519, L0.533, L0.573, L0.587, L0.601, L0.615, L0.629, L0.643, L0.657, L0.671, L0.685, L0.699, L0.713, L0.727" - " Creating 1 files" - "**** Simulation run 202, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " @@ -5661,9 +5661,9 @@ async fn stuck_l0_large_l0s() { - "L0.2015[131,161] 131ns |----------------------------L0.2015----------------------------| " - "L0.2029[132,161] 132ns |---------------------------L0.2029---------------------------| " - "L0.2043[133,161] 133ns |--------------------------L0.2043--------------------------| " - - "L0.2057[134,161] 134ns |-------------------------L0.2057-------------------------| " + - "L0.2169[134,161] 134ns |-------------------------L0.2169-------------------------| " - "L0.2183[135,161] 135ns |------------------------L0.2183------------------------| " - - "L0.2197[136,161] 136ns |----------------------L0.2197-----------------------| " + - "L0.2057[136,161] 136ns |----------------------L0.2057-----------------------| " - "L0.2071[137,161] 137ns |---------------------L0.2071----------------------| " - "L0.2085[138,161] 138ns |--------------------L0.2085---------------------| " - "L0.2099[139,161] 139ns |-------------------L0.2099--------------------| " @@ -5671,7 +5671,7 @@ async fn stuck_l0_large_l0s() { - "L0, all files 0b " - "L0.?[120,161] 139ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.1861, L0.1875, L0.1889, L0.1903, L0.1917, L0.1931, L0.1945, L0.1959, L0.1973, L0.1987, L0.2001, L0.2015, L0.2029, L0.2043, L0.2057, L0.2071, L0.2085, L0.2099, L0.2183, L0.2197" + - " Soft Deleting 20 files: L0.1861, L0.1875, L0.1889, L0.1903, L0.1917, L0.1931, L0.1945, L0.1959, L0.1973, L0.1987, L0.2001, L0.2015, L0.2029, L0.2043, L0.2057, L0.2071, L0.2085, L0.2099, L0.2169, L0.2183" - " Creating 1 files" - "**** Simulation run 207, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " @@ -5679,7 +5679,7 @@ async fn stuck_l0_large_l0s() { - "L0.2127[141,161] 141ns |--------------------------------------L0.2127--------------------------------------| " - "L0.2141[142,161] 142ns |------------------------------------L0.2141------------------------------------| " - "L0.2155[143,161] 143ns |----------------------------------L0.2155----------------------------------| " - - "L0.2169[144,161] 144ns |-------------------------------L0.2169--------------------------------| " + - "L0.2197[144,161] 144ns |-------------------------------L0.2197--------------------------------| " - "L0.2211[145,161] 145ns |-----------------------------L0.2211------------------------------| " - "L0.2225[146,161] 146ns |---------------------------L0.2225----------------------------| " - "L0.2239[147,161] 147ns |-------------------------L0.2239--------------------------|" @@ -5699,7 +5699,7 @@ async fn stuck_l0_large_l0s() { - "L0, all files 0b " - "L0.?[140,161] 159ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.2113, L0.2127, L0.2141, L0.2155, L0.2169, L0.2211, L0.2225, L0.2239, L0.2253, L0.2267, L0.2281, L0.2295, L0.2309, L0.2323, L0.2337, L0.2351, L0.2365, L0.2379, L0.2393, L0.2407" + - " Soft Deleting 20 files: L0.2113, L0.2127, L0.2141, L0.2155, L0.2197, L0.2211, L0.2225, L0.2239, L0.2253, L0.2267, L0.2281, L0.2295, L0.2309, L0.2323, L0.2337, L0.2351, L0.2365, L0.2379, L0.2393, L0.2407" - " Creating 1 files" - "**** Simulation run 208, type=compact(ManySmallFiles). 2 Input Files, 0b total:" - "L0, all files 0b " @@ -5731,25 +5731,25 @@ async fn stuck_l0_large_l0s() { - "L0.397[162,321] 15ns |-----------------------------------------L0.397-----------------------------------------|" - "L0.410[162,321] 16ns |-----------------------------------------L0.410-----------------------------------------|" - "L0.423[162,321] 17ns |-----------------------------------------L0.423-----------------------------------------|" - - "L0.436[162,321] 18ns |-----------------------------------------L0.436-----------------------------------------|" + - "L0.548[162,321] 18ns |-----------------------------------------L0.548-----------------------------------------|" - "L0.561[162,321] 19ns |-----------------------------------------L0.561-----------------------------------------|" - "**** 1 Output Files (parquet_file_id not yet assigned), 160mb total:" - "L0, all files 160mb " - "L0.?[162,321] 19ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.202, L0.215, L0.228, L0.241, L0.254, L0.267, L0.280, L0.293, L0.306, L0.319, L0.332, L0.345, L0.358, L0.371, L0.384, L0.397, L0.410, L0.423, L0.436, L0.561" + - " Soft Deleting 20 files: L0.202, L0.215, L0.228, L0.241, L0.254, L0.267, L0.280, L0.293, L0.306, L0.319, L0.332, L0.345, L0.358, L0.371, L0.384, L0.397, L0.410, L0.423, L0.548, L0.561" - " Creating 1 files" - "**** Simulation run 210, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " - - "L0.574[162,321] 20ns |-----------------------------------------L0.574-----------------------------------------|" - - "L0.449[162,321] 21ns |-----------------------------------------L0.449-----------------------------------------|" - - "L0.463[162,321] 22ns |-----------------------------------------L0.463-----------------------------------------|" - - "L0.477[162,321] 23ns |-----------------------------------------L0.477-----------------------------------------|" - - "L0.491[162,321] 24ns |-----------------------------------------L0.491-----------------------------------------|" - - "L0.505[162,321] 25ns |-----------------------------------------L0.505-----------------------------------------|" - - "L0.519[162,321] 26ns |-----------------------------------------L0.519-----------------------------------------|" - - "L0.533[162,321] 27ns |-----------------------------------------L0.533-----------------------------------------|" - - "L0.547[162,321] 28ns |-----------------------------------------L0.547-----------------------------------------|" + - "L0.436[162,321] 20ns |-----------------------------------------L0.436-----------------------------------------|" + - "L0.450[162,321] 21ns |-----------------------------------------L0.450-----------------------------------------|" + - "L0.464[162,321] 22ns |-----------------------------------------L0.464-----------------------------------------|" + - "L0.478[162,321] 23ns |-----------------------------------------L0.478-----------------------------------------|" + - "L0.492[162,321] 24ns |-----------------------------------------L0.492-----------------------------------------|" + - "L0.506[162,321] 25ns |-----------------------------------------L0.506-----------------------------------------|" + - "L0.520[162,321] 26ns |-----------------------------------------L0.520-----------------------------------------|" + - "L0.534[162,321] 27ns |-----------------------------------------L0.534-----------------------------------------|" + - "L0.574[162,321] 28ns |-----------------------------------------L0.574-----------------------------------------|" - "L0.588[162,321] 29ns |-----------------------------------------L0.588-----------------------------------------|" - "L0.602[162,321] 30ns |-----------------------------------------L0.602-----------------------------------------|" - "L0.616[162,321] 31ns |-----------------------------------------L0.616-----------------------------------------|" @@ -5765,10 +5765,38 @@ async fn stuck_l0_large_l0s() { - "L0, all files 0b " - "L0.?[162,321] 39ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.449, L0.463, L0.477, L0.491, L0.505, L0.519, L0.533, L0.547, L0.574, L0.588, L0.602, L0.616, L0.630, L0.644, L0.658, L0.672, L0.686, L0.700, L0.714, L0.728" + - " Soft Deleting 20 files: L0.436, L0.450, L0.464, L0.478, L0.492, L0.506, L0.520, L0.534, L0.574, L0.588, L0.602, L0.616, L0.630, L0.644, L0.658, L0.672, L0.686, L0.700, L0.714, L0.728" - " Creating 1 files" - "**** Simulation run 211, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " + - "L0.1022[162,321] 60ns |----------------------------------------L0.1022-----------------------------------------|" + - "L0.1036[162,321] 61ns |----------------------------------------L0.1036-----------------------------------------|" + - "L0.1050[162,321] 62ns |----------------------------------------L0.1050-----------------------------------------|" + - "L0.1064[162,321] 63ns |----------------------------------------L0.1064-----------------------------------------|" + - "L0.1078[162,321] 64ns |----------------------------------------L0.1078-----------------------------------------|" + - "L0.1092[162,321] 65ns |----------------------------------------L0.1092-----------------------------------------|" + - "L0.1106[162,321] 66ns |----------------------------------------L0.1106-----------------------------------------|" + - "L0.1120[162,321] 67ns |----------------------------------------L0.1120-----------------------------------------|" + - "L0.1134[162,321] 68ns |----------------------------------------L0.1134-----------------------------------------|" + - "L0.1148[162,321] 69ns |----------------------------------------L0.1148-----------------------------------------|" + - "L0.1162[162,321] 70ns |----------------------------------------L0.1162-----------------------------------------|" + - "L0.1176[162,321] 71ns |----------------------------------------L0.1176-----------------------------------------|" + - "L0.1190[162,321] 72ns |----------------------------------------L0.1190-----------------------------------------|" + - "L0.1204[162,321] 73ns |----------------------------------------L0.1204-----------------------------------------|" + - "L0.1218[162,321] 74ns |----------------------------------------L0.1218-----------------------------------------|" + - "L0.1232[162,321] 75ns |----------------------------------------L0.1232-----------------------------------------|" + - "L0.1246[162,321] 76ns |----------------------------------------L0.1246-----------------------------------------|" + - "L0.1260[162,321] 77ns |----------------------------------------L0.1260-----------------------------------------|" + - "L0.1274[162,321] 78ns |----------------------------------------L0.1274-----------------------------------------|" + - "L0.1288[162,321] 79ns |----------------------------------------L0.1288-----------------------------------------|" + - "**** 1 Output Files (parquet_file_id not yet assigned), 0b total:" + - "L0, all files 0b " + - "L0.?[162,321] 79ns |------------------------------------------L0.?------------------------------------------|" + - "Committing partition 1:" + - " Soft Deleting 20 files: L0.1022, L0.1036, L0.1050, L0.1064, L0.1078, L0.1092, L0.1106, L0.1120, L0.1134, L0.1148, L0.1162, L0.1176, L0.1190, L0.1204, L0.1218, L0.1232, L0.1246, L0.1260, L0.1274, L0.1288" + - " Creating 1 files" + - "**** Simulation run 212, type=compact(ManySmallFiles). 20 Input Files, 0b total:" + - "L0, all files 0b " - "L0.1302[162,321] 80ns |----------------------------------------L0.1302-----------------------------------------|" - "L0.1316[162,321] 81ns |----------------------------------------L0.1316-----------------------------------------|" - "L0.1330[162,321] 82ns |----------------------------------------L0.1330-----------------------------------------|" @@ -5795,7 +5823,7 @@ async fn stuck_l0_large_l0s() { - "Committing partition 1:" - " Soft Deleting 20 files: L0.1302, L0.1316, L0.1330, L0.1344, L0.1358, L0.1372, L0.1386, L0.1400, L0.1414, L0.1428, L0.1442, L0.1456, L0.1470, L0.1484, L0.1498, L0.1512, L0.1526, L0.1540, L0.1554, L0.1568" - " Creating 1 files" - - "**** Simulation run 212, type=compact(ManySmallFiles). 20 Input Files, 0b total:" + - "**** Simulation run 213, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " - "L0.1582[162,321] 100ns |----------------------------------------L0.1582-----------------------------------------|" - "L0.1596[162,321] 101ns |----------------------------------------L0.1596-----------------------------------------|" @@ -5823,7 +5851,7 @@ async fn stuck_l0_large_l0s() { - "Committing partition 1:" - " Soft Deleting 20 files: L0.1582, L0.1596, L0.1610, L0.1624, L0.1638, L0.1652, L0.1666, L0.1680, L0.1694, L0.1708, L0.1722, L0.1736, L0.1750, L0.1764, L0.1778, L0.1792, L0.1806, L0.1820, L0.1834, L0.1848" - " Creating 1 files" - - "**** Simulation run 213, type=compact(ManySmallFiles). 20 Input Files, 0b total:" + - "**** Simulation run 214, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " - "L0.1862[162,321] 120ns |----------------------------------------L0.1862-----------------------------------------|" - "L0.1876[162,321] 121ns |----------------------------------------L0.1876-----------------------------------------|" @@ -5839,9 +5867,9 @@ async fn stuck_l0_large_l0s() { - "L0.2016[162,321] 131ns |----------------------------------------L0.2016-----------------------------------------|" - "L0.2030[162,321] 132ns |----------------------------------------L0.2030-----------------------------------------|" - "L0.2044[162,321] 133ns |----------------------------------------L0.2044-----------------------------------------|" - - "L0.2058[162,321] 134ns |----------------------------------------L0.2058-----------------------------------------|" + - "L0.2170[162,321] 134ns |----------------------------------------L0.2170-----------------------------------------|" - "L0.2184[162,321] 135ns |----------------------------------------L0.2184-----------------------------------------|" - - "L0.2198[162,321] 136ns |----------------------------------------L0.2198-----------------------------------------|" + - "L0.2058[162,321] 136ns |----------------------------------------L0.2058-----------------------------------------|" - "L0.2072[162,321] 137ns |----------------------------------------L0.2072-----------------------------------------|" - "L0.2086[162,321] 138ns |----------------------------------------L0.2086-----------------------------------------|" - "L0.2100[162,321] 139ns |----------------------------------------L0.2100-----------------------------------------|" @@ -5849,15 +5877,15 @@ async fn stuck_l0_large_l0s() { - "L0, all files 0b " - "L0.?[162,321] 139ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.1862, L0.1876, L0.1890, L0.1904, L0.1918, L0.1932, L0.1946, L0.1960, L0.1974, L0.1988, L0.2002, L0.2016, L0.2030, L0.2044, L0.2058, L0.2072, L0.2086, L0.2100, L0.2184, L0.2198" + - " Soft Deleting 20 files: L0.1862, L0.1876, L0.1890, L0.1904, L0.1918, L0.1932, L0.1946, L0.1960, L0.1974, L0.1988, L0.2002, L0.2016, L0.2030, L0.2044, L0.2058, L0.2072, L0.2086, L0.2100, L0.2170, L0.2184" - " Creating 1 files" - - "**** Simulation run 214, type=compact(ManySmallFiles). 20 Input Files, 0b total:" + - "**** Simulation run 215, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " - "L0.2114[162,321] 140ns |----------------------------------------L0.2114-----------------------------------------|" - "L0.2128[162,321] 141ns |----------------------------------------L0.2128-----------------------------------------|" - "L0.2142[162,321] 142ns |----------------------------------------L0.2142-----------------------------------------|" - "L0.2156[162,321] 143ns |----------------------------------------L0.2156-----------------------------------------|" - - "L0.2170[162,321] 144ns |----------------------------------------L0.2170-----------------------------------------|" + - "L0.2198[162,321] 144ns |----------------------------------------L0.2198-----------------------------------------|" - "L0.2212[162,321] 145ns |----------------------------------------L0.2212-----------------------------------------|" - "L0.2226[162,321] 146ns |----------------------------------------L0.2226-----------------------------------------|" - "L0.2240[162,321] 147ns |----------------------------------------L0.2240-----------------------------------------|" @@ -5877,9 +5905,9 @@ async fn stuck_l0_large_l0s() { - "L0, all files 0b " - "L0.?[162,321] 159ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.2114, L0.2128, L0.2142, L0.2156, L0.2170, L0.2212, L0.2226, L0.2240, L0.2254, L0.2268, L0.2282, L0.2296, L0.2310, L0.2324, L0.2338, L0.2352, L0.2366, L0.2380, L0.2394, L0.2408" + - " Soft Deleting 20 files: L0.2114, L0.2128, L0.2142, L0.2156, L0.2198, L0.2212, L0.2226, L0.2240, L0.2254, L0.2268, L0.2282, L0.2296, L0.2310, L0.2324, L0.2338, L0.2352, L0.2366, L0.2380, L0.2394, L0.2408" - " Creating 1 files" - - "**** Simulation run 215, type=compact(ManySmallFiles). 20 Input Files, 0b total:" + - "**** Simulation run 216, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " - "L0.2422[162,321] 160ns |----------------------------------------L0.2422-----------------------------------------|" - "L0.2436[162,321] 161ns |----------------------------------------L0.2436-----------------------------------------|" @@ -5907,7 +5935,7 @@ async fn stuck_l0_large_l0s() { - "Committing partition 1:" - " Soft Deleting 20 files: L0.2422, L0.2436, L0.2449, L0.2462, L0.2475, L0.2488, L0.2501, L0.2514, L0.2527, L0.2540, L0.2553, L0.2566, L0.2579, L0.2592, L0.2605, L0.2618, L0.2631, L0.2644, L0.2657, L0.2670" - " Creating 1 files" - - "**** Simulation run 216, type=compact(ManySmallFiles). 20 Input Files, 0b total:" + - "**** Simulation run 217, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " - "L0.2683[180,321] 180ns |----------------------------------------L0.2683-----------------------------------------|" - "L0.2696[181,321] 181ns |----------------------------------------L0.2696----------------------------------------| " @@ -5935,7 +5963,7 @@ async fn stuck_l0_large_l0s() { - "Committing partition 1:" - " Soft Deleting 20 files: L0.2683, L0.2696, L0.2709, L0.2722, L0.2735, L0.2748, L0.2761, L0.2774, L0.2787, L0.2800, L0.2813, L0.2826, L0.2839, L0.2852, L0.2865, L0.2878, L0.2891, L0.2904, L0.2917, L0.2930" - " Creating 1 files" - - "**** Simulation run 217, type=compact(ManySmallFiles). 20 Input Files, 160mb total:" + - "**** Simulation run 218, type=compact(ManySmallFiles). 20 Input Files, 160mb total:" - "L0, all files 8mb " - "L0.203[322,481] 0ns |-----------------------------------------L0.203-----------------------------------------|" - "L0.216[322,481] 1ns |-----------------------------------------L0.216-----------------------------------------|" @@ -5955,41 +5983,13 @@ async fn stuck_l0_large_l0s() { - "L0.398[322,481] 15ns |-----------------------------------------L0.398-----------------------------------------|" - "L0.411[322,481] 16ns |-----------------------------------------L0.411-----------------------------------------|" - "L0.424[322,481] 17ns |-----------------------------------------L0.424-----------------------------------------|" - - "L0.437[322,481] 18ns |-----------------------------------------L0.437-----------------------------------------|" + - "L0.549[322,481] 18ns |-----------------------------------------L0.549-----------------------------------------|" - "L0.562[322,481] 19ns |-----------------------------------------L0.562-----------------------------------------|" - "**** 1 Output Files (parquet_file_id not yet assigned), 160mb total:" - "L0, all files 160mb " - "L0.?[322,481] 19ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.203, L0.216, L0.229, L0.242, L0.255, L0.268, L0.281, L0.294, L0.307, L0.320, L0.333, L0.346, L0.359, L0.372, L0.385, L0.398, L0.411, L0.424, L0.437, L0.562" - - " Creating 1 files" - - "**** Simulation run 218, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - - "L0, all files 0b " - - "L0.575[322,481] 20ns |-----------------------------------------L0.575-----------------------------------------|" - - "L0.450[322,481] 21ns |-----------------------------------------L0.450-----------------------------------------|" - - "L0.464[322,481] 22ns |-----------------------------------------L0.464-----------------------------------------|" - - "L0.478[322,481] 23ns |-----------------------------------------L0.478-----------------------------------------|" - - "L0.492[322,481] 24ns |-----------------------------------------L0.492-----------------------------------------|" - - "L0.506[322,481] 25ns |-----------------------------------------L0.506-----------------------------------------|" - - "L0.520[322,481] 26ns |-----------------------------------------L0.520-----------------------------------------|" - - "L0.534[322,481] 27ns |-----------------------------------------L0.534-----------------------------------------|" - - "L0.548[322,481] 28ns |-----------------------------------------L0.548-----------------------------------------|" - - "L0.589[322,481] 29ns |-----------------------------------------L0.589-----------------------------------------|" - - "L0.603[322,481] 30ns |-----------------------------------------L0.603-----------------------------------------|" - - "L0.617[322,481] 31ns |-----------------------------------------L0.617-----------------------------------------|" - - "L0.631[322,481] 32ns |-----------------------------------------L0.631-----------------------------------------|" - - "L0.645[322,481] 33ns |-----------------------------------------L0.645-----------------------------------------|" - - "L0.659[322,481] 34ns |-----------------------------------------L0.659-----------------------------------------|" - - "L0.673[322,481] 35ns |-----------------------------------------L0.673-----------------------------------------|" - - "L0.687[322,481] 36ns |-----------------------------------------L0.687-----------------------------------------|" - - "L0.701[322,481] 37ns |-----------------------------------------L0.701-----------------------------------------|" - - "L0.715[322,481] 38ns |-----------------------------------------L0.715-----------------------------------------|" - - "L0.729[322,481] 39ns |-----------------------------------------L0.729-----------------------------------------|" - - "**** 1 Output Files (parquet_file_id not yet assigned), 0b total:" - - "L0, all files 0b " - - "L0.?[322,481] 39ns |------------------------------------------L0.?------------------------------------------|" - - "Committing partition 1:" - - " Soft Deleting 20 files: L0.450, L0.464, L0.478, L0.492, L0.506, L0.520, L0.534, L0.548, L0.575, L0.589, L0.603, L0.617, L0.631, L0.645, L0.659, L0.673, L0.687, L0.701, L0.715, L0.729" + - " Soft Deleting 20 files: L0.203, L0.216, L0.229, L0.242, L0.255, L0.268, L0.281, L0.294, L0.307, L0.320, L0.333, L0.346, L0.359, L0.372, L0.385, L0.398, L0.411, L0.424, L0.549, L0.562" - " Creating 1 files" - "**** Simulation run 219, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " @@ -6021,31 +6021,31 @@ async fn stuck_l0_large_l0s() { - " Creating 1 files" - "**** Simulation run 220, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " - - "L0.1022[162,321] 60ns |----------------------------------------L0.1022-----------------------------------------|" - - "L0.1036[162,321] 61ns |----------------------------------------L0.1036-----------------------------------------|" - - "L0.1050[162,321] 62ns |----------------------------------------L0.1050-----------------------------------------|" - - "L0.1064[162,321] 63ns |----------------------------------------L0.1064-----------------------------------------|" - - "L0.1078[162,321] 64ns |----------------------------------------L0.1078-----------------------------------------|" - - "L0.1092[162,321] 65ns |----------------------------------------L0.1092-----------------------------------------|" - - "L0.1106[162,321] 66ns |----------------------------------------L0.1106-----------------------------------------|" - - "L0.1120[162,321] 67ns |----------------------------------------L0.1120-----------------------------------------|" - - "L0.1134[162,321] 68ns |----------------------------------------L0.1134-----------------------------------------|" - - "L0.1148[162,321] 69ns |----------------------------------------L0.1148-----------------------------------------|" - - "L0.1162[162,321] 70ns |----------------------------------------L0.1162-----------------------------------------|" - - "L0.1176[162,321] 71ns |----------------------------------------L0.1176-----------------------------------------|" - - "L0.1190[162,321] 72ns |----------------------------------------L0.1190-----------------------------------------|" - - "L0.1204[162,321] 73ns |----------------------------------------L0.1204-----------------------------------------|" - - "L0.1218[162,321] 74ns |----------------------------------------L0.1218-----------------------------------------|" - - "L0.1232[162,321] 75ns |----------------------------------------L0.1232-----------------------------------------|" - - "L0.1246[162,321] 76ns |----------------------------------------L0.1246-----------------------------------------|" - - "L0.1260[162,321] 77ns |----------------------------------------L0.1260-----------------------------------------|" - - "L0.1274[162,321] 78ns |----------------------------------------L0.1274-----------------------------------------|" - - "L0.1288[162,321] 79ns |----------------------------------------L0.1288-----------------------------------------|" + - "L0.437[322,481] 20ns |-----------------------------------------L0.437-----------------------------------------|" + - "L0.451[322,481] 21ns |-----------------------------------------L0.451-----------------------------------------|" + - "L0.465[322,481] 22ns |-----------------------------------------L0.465-----------------------------------------|" + - "L0.479[322,481] 23ns |-----------------------------------------L0.479-----------------------------------------|" + - "L0.493[322,481] 24ns |-----------------------------------------L0.493-----------------------------------------|" + - "L0.507[322,481] 25ns |-----------------------------------------L0.507-----------------------------------------|" + - "L0.521[322,481] 26ns |-----------------------------------------L0.521-----------------------------------------|" + - "L0.535[322,481] 27ns |-----------------------------------------L0.535-----------------------------------------|" + - "L0.575[322,481] 28ns |-----------------------------------------L0.575-----------------------------------------|" + - "L0.589[322,481] 29ns |-----------------------------------------L0.589-----------------------------------------|" + - "L0.603[322,481] 30ns |-----------------------------------------L0.603-----------------------------------------|" + - "L0.617[322,481] 31ns |-----------------------------------------L0.617-----------------------------------------|" + - "L0.631[322,481] 32ns |-----------------------------------------L0.631-----------------------------------------|" + - "L0.645[322,481] 33ns |-----------------------------------------L0.645-----------------------------------------|" + - "L0.659[322,481] 34ns |-----------------------------------------L0.659-----------------------------------------|" + - "L0.673[322,481] 35ns |-----------------------------------------L0.673-----------------------------------------|" + - "L0.687[322,481] 36ns |-----------------------------------------L0.687-----------------------------------------|" + - "L0.701[322,481] 37ns |-----------------------------------------L0.701-----------------------------------------|" + - "L0.715[322,481] 38ns |-----------------------------------------L0.715-----------------------------------------|" + - "L0.729[322,481] 39ns |-----------------------------------------L0.729-----------------------------------------|" - "**** 1 Output Files (parquet_file_id not yet assigned), 0b total:" - "L0, all files 0b " - - "L0.?[162,321] 79ns |------------------------------------------L0.?------------------------------------------|" + - "L0.?[322,481] 39ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.1022, L0.1036, L0.1050, L0.1064, L0.1078, L0.1092, L0.1106, L0.1120, L0.1134, L0.1148, L0.1162, L0.1176, L0.1190, L0.1204, L0.1218, L0.1232, L0.1246, L0.1260, L0.1274, L0.1288" + - " Soft Deleting 20 files: L0.437, L0.451, L0.465, L0.479, L0.493, L0.507, L0.521, L0.535, L0.575, L0.589, L0.603, L0.617, L0.631, L0.645, L0.659, L0.673, L0.687, L0.701, L0.715, L0.729" - " Creating 1 files" - "**** Simulation run 221, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " @@ -6175,9 +6175,9 @@ async fn stuck_l0_large_l0s() { - "L0.2017[322,481] 131ns |----------------------------------------L0.2017-----------------------------------------|" - "L0.2031[322,481] 132ns |----------------------------------------L0.2031-----------------------------------------|" - "L0.2045[322,481] 133ns |----------------------------------------L0.2045-----------------------------------------|" - - "L0.2059[322,481] 134ns |----------------------------------------L0.2059-----------------------------------------|" + - "L0.2171[322,481] 134ns |----------------------------------------L0.2171-----------------------------------------|" - "L0.2185[322,481] 135ns |----------------------------------------L0.2185-----------------------------------------|" - - "L0.2199[322,481] 136ns |----------------------------------------L0.2199-----------------------------------------|" + - "L0.2059[322,481] 136ns |----------------------------------------L0.2059-----------------------------------------|" - "L0.2073[322,481] 137ns |----------------------------------------L0.2073-----------------------------------------|" - "L0.2087[322,481] 138ns |----------------------------------------L0.2087-----------------------------------------|" - "L0.2101[322,481] 139ns |----------------------------------------L0.2101-----------------------------------------|" @@ -6185,7 +6185,7 @@ async fn stuck_l0_large_l0s() { - "L0, all files 0b " - "L0.?[322,481] 139ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.1863, L0.1877, L0.1891, L0.1905, L0.1919, L0.1933, L0.1947, L0.1961, L0.1975, L0.1989, L0.2003, L0.2017, L0.2031, L0.2045, L0.2059, L0.2073, L0.2087, L0.2101, L0.2185, L0.2199" + - " Soft Deleting 20 files: L0.1863, L0.1877, L0.1891, L0.1905, L0.1919, L0.1933, L0.1947, L0.1961, L0.1975, L0.1989, L0.2003, L0.2017, L0.2031, L0.2045, L0.2059, L0.2073, L0.2087, L0.2101, L0.2171, L0.2185" - " Creating 1 files" - "**** Simulation run 226, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " @@ -6193,7 +6193,7 @@ async fn stuck_l0_large_l0s() { - "L0.2129[322,481] 141ns |----------------------------------------L0.2129-----------------------------------------|" - "L0.2143[322,481] 142ns |----------------------------------------L0.2143-----------------------------------------|" - "L0.2157[322,481] 143ns |----------------------------------------L0.2157-----------------------------------------|" - - "L0.2171[322,481] 144ns |----------------------------------------L0.2171-----------------------------------------|" + - "L0.2199[322,481] 144ns |----------------------------------------L0.2199-----------------------------------------|" - "L0.2213[322,481] 145ns |----------------------------------------L0.2213-----------------------------------------|" - "L0.2227[322,481] 146ns |----------------------------------------L0.2227-----------------------------------------|" - "L0.2241[322,481] 147ns |----------------------------------------L0.2241-----------------------------------------|" @@ -6213,7 +6213,7 @@ async fn stuck_l0_large_l0s() { - "L0, all files 0b " - "L0.?[322,481] 159ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.2115, L0.2129, L0.2143, L0.2157, L0.2171, L0.2213, L0.2227, L0.2241, L0.2255, L0.2269, L0.2283, L0.2297, L0.2311, L0.2325, L0.2339, L0.2353, L0.2367, L0.2381, L0.2395, L0.2409" + - " Soft Deleting 20 files: L0.2115, L0.2129, L0.2143, L0.2157, L0.2199, L0.2213, L0.2227, L0.2241, L0.2255, L0.2269, L0.2283, L0.2297, L0.2311, L0.2325, L0.2339, L0.2353, L0.2367, L0.2381, L0.2395, L0.2409" - " Creating 1 files" - "**** Simulation run 227, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " @@ -6291,25 +6291,25 @@ async fn stuck_l0_large_l0s() { - "L0.399[482,641] 15ns |-----------------------------------------L0.399-----------------------------------------|" - "L0.412[482,641] 16ns |-----------------------------------------L0.412-----------------------------------------|" - "L0.425[482,641] 17ns |-----------------------------------------L0.425-----------------------------------------|" - - "L0.438[482,641] 18ns |-----------------------------------------L0.438-----------------------------------------|" + - "L0.550[482,641] 18ns |-----------------------------------------L0.550-----------------------------------------|" - "L0.563[482,641] 19ns |-----------------------------------------L0.563-----------------------------------------|" - "**** 1 Output Files (parquet_file_id not yet assigned), 160mb total:" - "L0, all files 160mb " - "L0.?[482,641] 19ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.204, L0.217, L0.230, L0.243, L0.256, L0.269, L0.282, L0.295, L0.308, L0.321, L0.334, L0.347, L0.360, L0.373, L0.386, L0.399, L0.412, L0.425, L0.438, L0.563" + - " Soft Deleting 20 files: L0.204, L0.217, L0.230, L0.243, L0.256, L0.269, L0.282, L0.295, L0.308, L0.321, L0.334, L0.347, L0.360, L0.373, L0.386, L0.399, L0.412, L0.425, L0.550, L0.563" - " Creating 1 files" - "**** Simulation run 230, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " - - "L0.576[482,641] 20ns |-----------------------------------------L0.576-----------------------------------------|" - - "L0.451[482,641] 21ns |-----------------------------------------L0.451-----------------------------------------|" - - "L0.465[482,641] 22ns |-----------------------------------------L0.465-----------------------------------------|" - - "L0.479[482,641] 23ns |-----------------------------------------L0.479-----------------------------------------|" - - "L0.493[482,641] 24ns |-----------------------------------------L0.493-----------------------------------------|" - - "L0.507[482,641] 25ns |-----------------------------------------L0.507-----------------------------------------|" - - "L0.521[482,641] 26ns |-----------------------------------------L0.521-----------------------------------------|" - - "L0.535[482,641] 27ns |-----------------------------------------L0.535-----------------------------------------|" - - "L0.549[482,641] 28ns |-----------------------------------------L0.549-----------------------------------------|" + - "L0.438[482,641] 20ns |-----------------------------------------L0.438-----------------------------------------|" + - "L0.452[482,641] 21ns |-----------------------------------------L0.452-----------------------------------------|" + - "L0.466[482,641] 22ns |-----------------------------------------L0.466-----------------------------------------|" + - "L0.480[482,641] 23ns |-----------------------------------------L0.480-----------------------------------------|" + - "L0.494[482,641] 24ns |-----------------------------------------L0.494-----------------------------------------|" + - "L0.508[482,641] 25ns |-----------------------------------------L0.508-----------------------------------------|" + - "L0.522[482,641] 26ns |-----------------------------------------L0.522-----------------------------------------|" + - "L0.536[482,641] 27ns |-----------------------------------------L0.536-----------------------------------------|" + - "L0.576[482,641] 28ns |-----------------------------------------L0.576-----------------------------------------|" - "L0.590[482,641] 29ns |-----------------------------------------L0.590-----------------------------------------|" - "L0.604[482,641] 30ns |-----------------------------------------L0.604-----------------------------------------|" - "L0.618[482,641] 31ns |-----------------------------------------L0.618-----------------------------------------|" @@ -6325,7 +6325,7 @@ async fn stuck_l0_large_l0s() { - "L0, all files 0b " - "L0.?[482,641] 39ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.451, L0.465, L0.479, L0.493, L0.507, L0.521, L0.535, L0.549, L0.576, L0.590, L0.604, L0.618, L0.632, L0.646, L0.660, L0.674, L0.688, L0.702, L0.716, L0.730" + - " Soft Deleting 20 files: L0.438, L0.452, L0.466, L0.480, L0.494, L0.508, L0.522, L0.536, L0.576, L0.590, L0.604, L0.618, L0.632, L0.646, L0.660, L0.674, L0.688, L0.702, L0.716, L0.730" - " Creating 1 files" - "**** Simulation run 231, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " @@ -6455,9 +6455,9 @@ async fn stuck_l0_large_l0s() { - "L0.2018[482,641] 131ns |----------------------------------------L0.2018-----------------------------------------|" - "L0.2032[482,641] 132ns |----------------------------------------L0.2032-----------------------------------------|" - "L0.2046[482,641] 133ns |----------------------------------------L0.2046-----------------------------------------|" - - "L0.2060[482,641] 134ns |----------------------------------------L0.2060-----------------------------------------|" + - "L0.2172[482,641] 134ns |----------------------------------------L0.2172-----------------------------------------|" - "L0.2186[482,641] 135ns |----------------------------------------L0.2186-----------------------------------------|" - - "L0.2200[482,641] 136ns |----------------------------------------L0.2200-----------------------------------------|" + - "L0.2060[482,641] 136ns |----------------------------------------L0.2060-----------------------------------------|" - "L0.2074[482,641] 137ns |----------------------------------------L0.2074-----------------------------------------|" - "L0.2088[482,641] 138ns |----------------------------------------L0.2088-----------------------------------------|" - "L0.2102[482,641] 139ns |----------------------------------------L0.2102-----------------------------------------|" @@ -6465,7 +6465,7 @@ async fn stuck_l0_large_l0s() { - "L0, all files 0b " - "L0.?[482,641] 139ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.1864, L0.1878, L0.1892, L0.1906, L0.1920, L0.1934, L0.1948, L0.1962, L0.1976, L0.1990, L0.2004, L0.2018, L0.2032, L0.2046, L0.2060, L0.2074, L0.2088, L0.2102, L0.2186, L0.2200" + - " Soft Deleting 20 files: L0.1864, L0.1878, L0.1892, L0.1906, L0.1920, L0.1934, L0.1948, L0.1962, L0.1976, L0.1990, L0.2004, L0.2018, L0.2032, L0.2046, L0.2060, L0.2074, L0.2088, L0.2102, L0.2172, L0.2186" - " Creating 1 files" - "**** Simulation run 236, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " @@ -6473,7 +6473,7 @@ async fn stuck_l0_large_l0s() { - "L0.2130[482,641] 141ns |----------------------------------------L0.2130-----------------------------------------|" - "L0.2144[482,641] 142ns |----------------------------------------L0.2144-----------------------------------------|" - "L0.2158[482,641] 143ns |----------------------------------------L0.2158-----------------------------------------|" - - "L0.2172[482,641] 144ns |----------------------------------------L0.2172-----------------------------------------|" + - "L0.2200[482,641] 144ns |----------------------------------------L0.2200-----------------------------------------|" - "L0.2214[482,641] 145ns |----------------------------------------L0.2214-----------------------------------------|" - "L0.2228[482,641] 146ns |----------------------------------------L0.2228-----------------------------------------|" - "L0.2242[482,641] 147ns |----------------------------------------L0.2242-----------------------------------------|" @@ -6492,9 +6492,6 @@ async fn stuck_l0_large_l0s() { - "**** 1 Output Files (parquet_file_id not yet assigned), 0b total:" - "L0, all files 0b " - "L0.?[482,641] 159ns |------------------------------------------L0.?------------------------------------------|" - - "Committing partition 1:" - - " Soft Deleting 20 files: L0.2116, L0.2130, L0.2144, L0.2158, L0.2172, L0.2214, L0.2228, L0.2242, L0.2256, L0.2270, L0.2284, L0.2298, L0.2312, L0.2326, L0.2340, L0.2354, L0.2368, L0.2382, L0.2396, L0.2410" - - " Creating 1 files" - "**** Simulation run 237, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " - "L0.2685[482,641] 180ns |----------------------------------------L0.2685-----------------------------------------|" @@ -6543,25 +6540,25 @@ async fn stuck_l0_large_l0s() { - "L0.400[642,801] 15ns |-----------------------------------------L0.400-----------------------------------------|" - "L0.413[642,801] 16ns |-----------------------------------------L0.413-----------------------------------------|" - "L0.426[642,801] 17ns |-----------------------------------------L0.426-----------------------------------------|" - - "L0.439[642,801] 18ns |-----------------------------------------L0.439-----------------------------------------|" + - "L0.551[642,801] 18ns |-----------------------------------------L0.551-----------------------------------------|" - "L0.564[642,801] 19ns |-----------------------------------------L0.564-----------------------------------------|" - "**** 1 Output Files (parquet_file_id not yet assigned), 160mb total:" - "L0, all files 160mb " - "L0.?[642,801] 19ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.205, L0.218, L0.231, L0.244, L0.257, L0.270, L0.283, L0.296, L0.309, L0.322, L0.335, L0.348, L0.361, L0.374, L0.387, L0.400, L0.413, L0.426, L0.439, L0.564" + - " Soft Deleting 20 files: L0.205, L0.218, L0.231, L0.244, L0.257, L0.270, L0.283, L0.296, L0.309, L0.322, L0.335, L0.348, L0.361, L0.374, L0.387, L0.400, L0.413, L0.426, L0.551, L0.564" - " Creating 1 files" - "**** Simulation run 239, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " - - "L0.577[642,801] 20ns |-----------------------------------------L0.577-----------------------------------------|" - - "L0.452[642,801] 21ns |-----------------------------------------L0.452-----------------------------------------|" - - "L0.466[642,801] 22ns |-----------------------------------------L0.466-----------------------------------------|" - - "L0.480[642,801] 23ns |-----------------------------------------L0.480-----------------------------------------|" - - "L0.494[642,801] 24ns |-----------------------------------------L0.494-----------------------------------------|" - - "L0.508[642,801] 25ns |-----------------------------------------L0.508-----------------------------------------|" - - "L0.522[642,801] 26ns |-----------------------------------------L0.522-----------------------------------------|" - - "L0.536[642,801] 27ns |-----------------------------------------L0.536-----------------------------------------|" - - "L0.550[642,801] 28ns |-----------------------------------------L0.550-----------------------------------------|" + - "L0.439[642,801] 20ns |-----------------------------------------L0.439-----------------------------------------|" + - "L0.453[642,801] 21ns |-----------------------------------------L0.453-----------------------------------------|" + - "L0.467[642,801] 22ns |-----------------------------------------L0.467-----------------------------------------|" + - "L0.481[642,801] 23ns |-----------------------------------------L0.481-----------------------------------------|" + - "L0.495[642,801] 24ns |-----------------------------------------L0.495-----------------------------------------|" + - "L0.509[642,801] 25ns |-----------------------------------------L0.509-----------------------------------------|" + - "L0.523[642,801] 26ns |-----------------------------------------L0.523-----------------------------------------|" + - "L0.537[642,801] 27ns |-----------------------------------------L0.537-----------------------------------------|" + - "L0.577[642,801] 28ns |-----------------------------------------L0.577-----------------------------------------|" - "L0.591[642,801] 29ns |-----------------------------------------L0.591-----------------------------------------|" - "L0.605[642,801] 30ns |-----------------------------------------L0.605-----------------------------------------|" - "L0.619[642,801] 31ns |-----------------------------------------L0.619-----------------------------------------|" @@ -6577,7 +6574,7 @@ async fn stuck_l0_large_l0s() { - "L0, all files 0b " - "L0.?[642,801] 39ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.452, L0.466, L0.480, L0.494, L0.508, L0.522, L0.536, L0.550, L0.577, L0.591, L0.605, L0.619, L0.633, L0.647, L0.661, L0.675, L0.689, L0.703, L0.717, L0.731" + - " Soft Deleting 20 files: L0.439, L0.453, L0.467, L0.481, L0.495, L0.509, L0.523, L0.537, L0.577, L0.591, L0.605, L0.619, L0.633, L0.647, L0.661, L0.675, L0.689, L0.703, L0.717, L0.731" - " Creating 1 files" - "**** Simulation run 240, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " @@ -6707,9 +6704,9 @@ async fn stuck_l0_large_l0s() { - "L0.2019[642,801] 131ns |----------------------------------------L0.2019-----------------------------------------|" - "L0.2033[642,801] 132ns |----------------------------------------L0.2033-----------------------------------------|" - "L0.2047[642,801] 133ns |----------------------------------------L0.2047-----------------------------------------|" - - "L0.2061[642,801] 134ns |----------------------------------------L0.2061-----------------------------------------|" + - "L0.2173[642,801] 134ns |----------------------------------------L0.2173-----------------------------------------|" - "L0.2187[642,801] 135ns |----------------------------------------L0.2187-----------------------------------------|" - - "L0.2201[642,801] 136ns |----------------------------------------L0.2201-----------------------------------------|" + - "L0.2061[642,801] 136ns |----------------------------------------L0.2061-----------------------------------------|" - "L0.2075[642,801] 137ns |----------------------------------------L0.2075-----------------------------------------|" - "L0.2089[642,801] 138ns |----------------------------------------L0.2089-----------------------------------------|" - "L0.2103[642,801] 139ns |----------------------------------------L0.2103-----------------------------------------|" @@ -6717,7 +6714,10 @@ async fn stuck_l0_large_l0s() { - "L0, all files 0b " - "L0.?[642,801] 139ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.1865, L0.1879, L0.1893, L0.1907, L0.1921, L0.1935, L0.1949, L0.1963, L0.1977, L0.1991, L0.2005, L0.2019, L0.2033, L0.2047, L0.2061, L0.2075, L0.2089, L0.2103, L0.2187, L0.2201" + - " Soft Deleting 20 files: L0.1865, L0.1879, L0.1893, L0.1907, L0.1921, L0.1935, L0.1949, L0.1963, L0.1977, L0.1991, L0.2005, L0.2019, L0.2033, L0.2047, L0.2061, L0.2075, L0.2089, L0.2103, L0.2173, L0.2187" + - " Creating 1 files" + - "Committing partition 1:" + - " Soft Deleting 20 files: L0.2116, L0.2130, L0.2144, L0.2158, L0.2200, L0.2214, L0.2228, L0.2242, L0.2256, L0.2270, L0.2284, L0.2298, L0.2312, L0.2326, L0.2340, L0.2354, L0.2368, L0.2382, L0.2396, L0.2410" - " Creating 1 files" - "**** Simulation run 245, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " @@ -6753,7 +6753,7 @@ async fn stuck_l0_large_l0s() { - "L0.2131[642,801] 141ns |----------------------------------------L0.2131-----------------------------------------|" - "L0.2145[642,801] 142ns |----------------------------------------L0.2145-----------------------------------------|" - "L0.2159[642,801] 143ns |----------------------------------------L0.2159-----------------------------------------|" - - "L0.2173[642,801] 144ns |----------------------------------------L0.2173-----------------------------------------|" + - "L0.2201[642,801] 144ns |----------------------------------------L0.2201-----------------------------------------|" - "L0.2215[642,801] 145ns |----------------------------------------L0.2215-----------------------------------------|" - "L0.2229[642,801] 146ns |----------------------------------------L0.2229-----------------------------------------|" - "L0.2243[642,801] 147ns |----------------------------------------L0.2243-----------------------------------------|" @@ -6773,7 +6773,7 @@ async fn stuck_l0_large_l0s() { - "L0, all files 0b " - "L0.?[642,801] 159ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.2117, L0.2131, L0.2145, L0.2159, L0.2173, L0.2215, L0.2229, L0.2243, L0.2257, L0.2271, L0.2285, L0.2299, L0.2313, L0.2327, L0.2341, L0.2355, L0.2369, L0.2383, L0.2397, L0.2411" + - " Soft Deleting 20 files: L0.2117, L0.2131, L0.2145, L0.2159, L0.2201, L0.2215, L0.2229, L0.2243, L0.2257, L0.2271, L0.2285, L0.2299, L0.2313, L0.2327, L0.2341, L0.2355, L0.2369, L0.2383, L0.2397, L0.2411" - " Creating 1 files" - "**** Simulation run 247, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " @@ -6851,25 +6851,25 @@ async fn stuck_l0_large_l0s() { - "L0.401[802,961] 15ns |-----------------------------------------L0.401-----------------------------------------|" - "L0.414[802,961] 16ns |-----------------------------------------L0.414-----------------------------------------|" - "L0.427[802,961] 17ns |-----------------------------------------L0.427-----------------------------------------|" - - "L0.440[802,961] 18ns |-----------------------------------------L0.440-----------------------------------------|" + - "L0.552[802,961] 18ns |-----------------------------------------L0.552-----------------------------------------|" - "L0.565[802,961] 19ns |-----------------------------------------L0.565-----------------------------------------|" - "**** 1 Output Files (parquet_file_id not yet assigned), 160mb total:" - "L0, all files 160mb " - "L0.?[802,961] 19ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.206, L0.219, L0.232, L0.245, L0.258, L0.271, L0.284, L0.297, L0.310, L0.323, L0.336, L0.349, L0.362, L0.375, L0.388, L0.401, L0.414, L0.427, L0.440, L0.565" + - " Soft Deleting 20 files: L0.206, L0.219, L0.232, L0.245, L0.258, L0.271, L0.284, L0.297, L0.310, L0.323, L0.336, L0.349, L0.362, L0.375, L0.388, L0.401, L0.414, L0.427, L0.552, L0.565" - " Creating 1 files" - "**** Simulation run 250, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " - - "L0.578[802,961] 20ns |-----------------------------------------L0.578-----------------------------------------|" - - "L0.453[802,961] 21ns |-----------------------------------------L0.453-----------------------------------------|" - - "L0.467[802,961] 22ns |-----------------------------------------L0.467-----------------------------------------|" - - "L0.481[802,961] 23ns |-----------------------------------------L0.481-----------------------------------------|" - - "L0.495[802,961] 24ns |-----------------------------------------L0.495-----------------------------------------|" - - "L0.509[802,961] 25ns |-----------------------------------------L0.509-----------------------------------------|" - - "L0.523[802,961] 26ns |-----------------------------------------L0.523-----------------------------------------|" - - "L0.537[802,961] 27ns |-----------------------------------------L0.537-----------------------------------------|" - - "L0.551[802,961] 28ns |-----------------------------------------L0.551-----------------------------------------|" + - "L0.440[802,961] 20ns |-----------------------------------------L0.440-----------------------------------------|" + - "L0.454[802,961] 21ns |-----------------------------------------L0.454-----------------------------------------|" + - "L0.468[802,961] 22ns |-----------------------------------------L0.468-----------------------------------------|" + - "L0.482[802,961] 23ns |-----------------------------------------L0.482-----------------------------------------|" + - "L0.496[802,961] 24ns |-----------------------------------------L0.496-----------------------------------------|" + - "L0.510[802,961] 25ns |-----------------------------------------L0.510-----------------------------------------|" + - "L0.524[802,961] 26ns |-----------------------------------------L0.524-----------------------------------------|" + - "L0.538[802,961] 27ns |-----------------------------------------L0.538-----------------------------------------|" + - "L0.578[802,961] 28ns |-----------------------------------------L0.578-----------------------------------------|" - "L0.592[802,961] 29ns |-----------------------------------------L0.592-----------------------------------------|" - "L0.606[802,961] 30ns |-----------------------------------------L0.606-----------------------------------------|" - "L0.620[802,961] 31ns |-----------------------------------------L0.620-----------------------------------------|" @@ -6885,7 +6885,7 @@ async fn stuck_l0_large_l0s() { - "L0, all files 0b " - "L0.?[802,961] 39ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.453, L0.467, L0.481, L0.495, L0.509, L0.523, L0.537, L0.551, L0.578, L0.592, L0.606, L0.620, L0.634, L0.648, L0.662, L0.676, L0.690, L0.704, L0.718, L0.732" + - " Soft Deleting 20 files: L0.440, L0.454, L0.468, L0.482, L0.496, L0.510, L0.524, L0.538, L0.578, L0.592, L0.606, L0.620, L0.634, L0.648, L0.662, L0.676, L0.690, L0.704, L0.718, L0.732" - " Creating 1 files" - "**** Simulation run 251, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " @@ -7015,9 +7015,9 @@ async fn stuck_l0_large_l0s() { - "L0.2020[802,961] 131ns |----------------------------------------L0.2020-----------------------------------------|" - "L0.2034[802,961] 132ns |----------------------------------------L0.2034-----------------------------------------|" - "L0.2048[802,961] 133ns |----------------------------------------L0.2048-----------------------------------------|" - - "L0.2062[802,961] 134ns |----------------------------------------L0.2062-----------------------------------------|" + - "L0.2174[802,961] 134ns |----------------------------------------L0.2174-----------------------------------------|" - "L0.2188[802,961] 135ns |----------------------------------------L0.2188-----------------------------------------|" - - "L0.2202[802,961] 136ns |----------------------------------------L0.2202-----------------------------------------|" + - "L0.2062[802,961] 136ns |----------------------------------------L0.2062-----------------------------------------|" - "L0.2076[802,961] 137ns |----------------------------------------L0.2076-----------------------------------------|" - "L0.2090[802,961] 138ns |----------------------------------------L0.2090-----------------------------------------|" - "L0.2104[802,961] 139ns |----------------------------------------L0.2104-----------------------------------------|" @@ -7025,7 +7025,7 @@ async fn stuck_l0_large_l0s() { - "L0, all files 0b " - "L0.?[802,961] 139ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.1866, L0.1880, L0.1894, L0.1908, L0.1922, L0.1936, L0.1950, L0.1964, L0.1978, L0.1992, L0.2006, L0.2020, L0.2034, L0.2048, L0.2062, L0.2076, L0.2090, L0.2104, L0.2188, L0.2202" + - " Soft Deleting 20 files: L0.1866, L0.1880, L0.1894, L0.1908, L0.1922, L0.1936, L0.1950, L0.1964, L0.1978, L0.1992, L0.2006, L0.2020, L0.2034, L0.2048, L0.2062, L0.2076, L0.2090, L0.2104, L0.2174, L0.2188" - " Creating 1 files" - "**** Simulation run 256, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " @@ -7033,7 +7033,7 @@ async fn stuck_l0_large_l0s() { - "L0.2132[802,961] 141ns |----------------------------------------L0.2132-----------------------------------------|" - "L0.2146[802,961] 142ns |----------------------------------------L0.2146-----------------------------------------|" - "L0.2160[802,961] 143ns |----------------------------------------L0.2160-----------------------------------------|" - - "L0.2174[802,961] 144ns |----------------------------------------L0.2174-----------------------------------------|" + - "L0.2202[802,961] 144ns |----------------------------------------L0.2202-----------------------------------------|" - "L0.2216[802,961] 145ns |----------------------------------------L0.2216-----------------------------------------|" - "L0.2230[802,961] 146ns |----------------------------------------L0.2230-----------------------------------------|" - "L0.2244[802,961] 147ns |----------------------------------------L0.2244-----------------------------------------|" @@ -7053,7 +7053,7 @@ async fn stuck_l0_large_l0s() { - "L0, all files 0b " - "L0.?[802,961] 159ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.2118, L0.2132, L0.2146, L0.2160, L0.2174, L0.2216, L0.2230, L0.2244, L0.2258, L0.2272, L0.2286, L0.2300, L0.2314, L0.2328, L0.2342, L0.2356, L0.2370, L0.2384, L0.2398, L0.2412" + - " Soft Deleting 20 files: L0.2118, L0.2132, L0.2146, L0.2160, L0.2202, L0.2216, L0.2230, L0.2244, L0.2258, L0.2272, L0.2286, L0.2300, L0.2314, L0.2328, L0.2342, L0.2356, L0.2370, L0.2384, L0.2398, L0.2412" - " Creating 1 files" - "**** Simulation run 257, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " @@ -7131,25 +7131,25 @@ async fn stuck_l0_large_l0s() { - "L0.402[962,1121] 15ns |-----------------------------------------L0.402-----------------------------------------|" - "L0.415[962,1121] 16ns |-----------------------------------------L0.415-----------------------------------------|" - "L0.428[962,1121] 17ns |-----------------------------------------L0.428-----------------------------------------|" - - "L0.441[962,1121] 18ns |-----------------------------------------L0.441-----------------------------------------|" + - "L0.553[962,1121] 18ns |-----------------------------------------L0.553-----------------------------------------|" - "L0.566[962,1121] 19ns |-----------------------------------------L0.566-----------------------------------------|" - "**** 1 Output Files (parquet_file_id not yet assigned), 160mb total:" - "L0, all files 160mb " - "L0.?[962,1121] 19ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.207, L0.220, L0.233, L0.246, L0.259, L0.272, L0.285, L0.298, L0.311, L0.324, L0.337, L0.350, L0.363, L0.376, L0.389, L0.402, L0.415, L0.428, L0.441, L0.566" + - " Soft Deleting 20 files: L0.207, L0.220, L0.233, L0.246, L0.259, L0.272, L0.285, L0.298, L0.311, L0.324, L0.337, L0.350, L0.363, L0.376, L0.389, L0.402, L0.415, L0.428, L0.553, L0.566" - " Creating 1 files" - "**** Simulation run 260, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " - - "L0.579[962,1121] 20ns |-----------------------------------------L0.579-----------------------------------------|" - - "L0.454[962,1121] 21ns |-----------------------------------------L0.454-----------------------------------------|" - - "L0.468[962,1121] 22ns |-----------------------------------------L0.468-----------------------------------------|" - - "L0.482[962,1121] 23ns |-----------------------------------------L0.482-----------------------------------------|" - - "L0.496[962,1121] 24ns |-----------------------------------------L0.496-----------------------------------------|" - - "L0.510[962,1121] 25ns |-----------------------------------------L0.510-----------------------------------------|" - - "L0.524[962,1121] 26ns |-----------------------------------------L0.524-----------------------------------------|" - - "L0.538[962,1121] 27ns |-----------------------------------------L0.538-----------------------------------------|" - - "L0.552[962,1121] 28ns |-----------------------------------------L0.552-----------------------------------------|" + - "L0.441[962,1121] 20ns |-----------------------------------------L0.441-----------------------------------------|" + - "L0.455[962,1121] 21ns |-----------------------------------------L0.455-----------------------------------------|" + - "L0.469[962,1121] 22ns |-----------------------------------------L0.469-----------------------------------------|" + - "L0.483[962,1121] 23ns |-----------------------------------------L0.483-----------------------------------------|" + - "L0.497[962,1121] 24ns |-----------------------------------------L0.497-----------------------------------------|" + - "L0.511[962,1121] 25ns |-----------------------------------------L0.511-----------------------------------------|" + - "L0.525[962,1121] 26ns |-----------------------------------------L0.525-----------------------------------------|" + - "L0.539[962,1121] 27ns |-----------------------------------------L0.539-----------------------------------------|" + - "L0.579[962,1121] 28ns |-----------------------------------------L0.579-----------------------------------------|" - "L0.593[962,1121] 29ns |-----------------------------------------L0.593-----------------------------------------|" - "L0.607[962,1121] 30ns |-----------------------------------------L0.607-----------------------------------------|" - "L0.621[962,1121] 31ns |-----------------------------------------L0.621-----------------------------------------|" @@ -7165,7 +7165,7 @@ async fn stuck_l0_large_l0s() { - "L0, all files 0b " - "L0.?[962,1121] 39ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.454, L0.468, L0.482, L0.496, L0.510, L0.524, L0.538, L0.552, L0.579, L0.593, L0.607, L0.621, L0.635, L0.649, L0.663, L0.677, L0.691, L0.705, L0.719, L0.733" + - " Soft Deleting 20 files: L0.441, L0.455, L0.469, L0.483, L0.497, L0.511, L0.525, L0.539, L0.579, L0.593, L0.607, L0.621, L0.635, L0.649, L0.663, L0.677, L0.691, L0.705, L0.719, L0.733" - " Creating 1 files" - "**** Simulation run 261, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " @@ -7197,31 +7197,6 @@ async fn stuck_l0_large_l0s() { - " Creating 1 files" - "**** Simulation run 262, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " - - "L0.1027[962,1121] 60ns |----------------------------------------L0.1027-----------------------------------------|" - - "L0.1041[962,1121] 61ns |----------------------------------------L0.1041-----------------------------------------|" - - "L0.1055[962,1121] 62ns |----------------------------------------L0.1055-----------------------------------------|" - - "L0.1069[962,1121] 63ns |----------------------------------------L0.1069-----------------------------------------|" - - "L0.1083[962,1121] 64ns |----------------------------------------L0.1083-----------------------------------------|" - - "L0.1097[962,1121] 65ns |----------------------------------------L0.1097-----------------------------------------|" - - "L0.1111[962,1121] 66ns |----------------------------------------L0.1111-----------------------------------------|" - - "L0.1125[962,1121] 67ns |----------------------------------------L0.1125-----------------------------------------|" - - "L0.1139[962,1121] 68ns |----------------------------------------L0.1139-----------------------------------------|" - - "L0.1153[962,1121] 69ns |----------------------------------------L0.1153-----------------------------------------|" - - "L0.1167[962,1121] 70ns |----------------------------------------L0.1167-----------------------------------------|" - - "L0.1181[962,1121] 71ns |----------------------------------------L0.1181-----------------------------------------|" - - "L0.1195[962,1121] 72ns |----------------------------------------L0.1195-----------------------------------------|" - - "L0.1209[962,1121] 73ns |----------------------------------------L0.1209-----------------------------------------|" - - "L0.1223[962,1121] 74ns |----------------------------------------L0.1223-----------------------------------------|" - - "L0.1237[962,1121] 75ns |----------------------------------------L0.1237-----------------------------------------|" - - "L0.1251[962,1121] 76ns |----------------------------------------L0.1251-----------------------------------------|" - - "L0.1265[962,1121] 77ns |----------------------------------------L0.1265-----------------------------------------|" - - "L0.1279[962,1121] 78ns |----------------------------------------L0.1279-----------------------------------------|" - - "L0.1293[962,1121] 79ns |----------------------------------------L0.1293-----------------------------------------|" - - "**** 1 Output Files (parquet_file_id not yet assigned), 0b total:" - - "L0, all files 0b " - - "L0.?[962,1121] 79ns |------------------------------------------L0.?------------------------------------------|" - - "**** Simulation run 263, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - - "L0, all files 0b " - "L0.1587[962,1121] 100ns |----------------------------------------L0.1587-----------------------------------------|" - "L0.1601[962,1121] 101ns |----------------------------------------L0.1601-----------------------------------------|" - "L0.1615[962,1121] 102ns |----------------------------------------L0.1615-----------------------------------------|" @@ -7248,7 +7223,7 @@ async fn stuck_l0_large_l0s() { - "Committing partition 1:" - " Soft Deleting 20 files: L0.1587, L0.1601, L0.1615, L0.1629, L0.1643, L0.1657, L0.1671, L0.1685, L0.1699, L0.1713, L0.1727, L0.1741, L0.1755, L0.1769, L0.1783, L0.1797, L0.1811, L0.1825, L0.1839, L0.1853" - " Creating 1 files" - - "**** Simulation run 264, type=compact(ManySmallFiles). 20 Input Files, 0b total:" + - "**** Simulation run 263, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " - "L0.1867[962,1121] 120ns |----------------------------------------L0.1867-----------------------------------------|" - "L0.1881[962,1121] 121ns |----------------------------------------L0.1881-----------------------------------------|" @@ -7264,9 +7239,9 @@ async fn stuck_l0_large_l0s() { - "L0.2021[962,1121] 131ns |----------------------------------------L0.2021-----------------------------------------|" - "L0.2035[962,1121] 132ns |----------------------------------------L0.2035-----------------------------------------|" - "L0.2049[962,1121] 133ns |----------------------------------------L0.2049-----------------------------------------|" - - "L0.2063[962,1121] 134ns |----------------------------------------L0.2063-----------------------------------------|" + - "L0.2175[962,1121] 134ns |----------------------------------------L0.2175-----------------------------------------|" - "L0.2189[962,1121] 135ns |----------------------------------------L0.2189-----------------------------------------|" - - "L0.2203[962,1121] 136ns |----------------------------------------L0.2203-----------------------------------------|" + - "L0.2063[962,1121] 136ns |----------------------------------------L0.2063-----------------------------------------|" - "L0.2077[962,1121] 137ns |----------------------------------------L0.2077-----------------------------------------|" - "L0.2091[962,1121] 138ns |----------------------------------------L0.2091-----------------------------------------|" - "L0.2105[962,1121] 139ns |----------------------------------------L0.2105-----------------------------------------|" @@ -7274,15 +7249,15 @@ async fn stuck_l0_large_l0s() { - "L0, all files 0b " - "L0.?[962,1121] 139ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.1867, L0.1881, L0.1895, L0.1909, L0.1923, L0.1937, L0.1951, L0.1965, L0.1979, L0.1993, L0.2007, L0.2021, L0.2035, L0.2049, L0.2063, L0.2077, L0.2091, L0.2105, L0.2189, L0.2203" + - " Soft Deleting 20 files: L0.1867, L0.1881, L0.1895, L0.1909, L0.1923, L0.1937, L0.1951, L0.1965, L0.1979, L0.1993, L0.2007, L0.2021, L0.2035, L0.2049, L0.2063, L0.2077, L0.2091, L0.2105, L0.2175, L0.2189" - " Creating 1 files" - - "**** Simulation run 265, type=compact(ManySmallFiles). 20 Input Files, 0b total:" + - "**** Simulation run 264, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " - "L0.2119[962,1121] 140ns |----------------------------------------L0.2119-----------------------------------------|" - "L0.2133[962,1121] 141ns |----------------------------------------L0.2133-----------------------------------------|" - "L0.2147[962,1121] 142ns |----------------------------------------L0.2147-----------------------------------------|" - "L0.2161[962,1121] 143ns |----------------------------------------L0.2161-----------------------------------------|" - - "L0.2175[962,1121] 144ns |----------------------------------------L0.2175-----------------------------------------|" + - "L0.2203[962,1121] 144ns |----------------------------------------L0.2203-----------------------------------------|" - "L0.2217[962,1121] 145ns |----------------------------------------L0.2217-----------------------------------------|" - "L0.2231[962,1121] 146ns |----------------------------------------L0.2231-----------------------------------------|" - "L0.2245[962,1121] 147ns |----------------------------------------L0.2245-----------------------------------------|" @@ -7302,9 +7277,9 @@ async fn stuck_l0_large_l0s() { - "L0, all files 0b " - "L0.?[962,1121] 159ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.2119, L0.2133, L0.2147, L0.2161, L0.2175, L0.2217, L0.2231, L0.2245, L0.2259, L0.2273, L0.2287, L0.2301, L0.2315, L0.2329, L0.2343, L0.2357, L0.2371, L0.2385, L0.2399, L0.2413" + - " Soft Deleting 20 files: L0.2119, L0.2133, L0.2147, L0.2161, L0.2203, L0.2217, L0.2231, L0.2245, L0.2259, L0.2273, L0.2287, L0.2301, L0.2315, L0.2329, L0.2343, L0.2357, L0.2371, L0.2385, L0.2399, L0.2413" - " Creating 1 files" - - "**** Simulation run 266, type=compact(ManySmallFiles). 20 Input Files, 0b total:" + - "**** Simulation run 265, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " - "L0.2427[962,1121] 160ns |----------------------------------------L0.2427-----------------------------------------|" - "L0.2441[962,1121] 161ns |----------------------------------------L0.2441-----------------------------------------|" @@ -7332,7 +7307,7 @@ async fn stuck_l0_large_l0s() { - "Committing partition 1:" - " Soft Deleting 20 files: L0.2427, L0.2441, L0.2454, L0.2467, L0.2480, L0.2493, L0.2506, L0.2519, L0.2532, L0.2545, L0.2558, L0.2571, L0.2584, L0.2597, L0.2610, L0.2623, L0.2636, L0.2649, L0.2662, L0.2675" - " Creating 1 files" - - "**** Simulation run 267, type=compact(ManySmallFiles). 20 Input Files, 0b total:" + - "**** Simulation run 266, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " - "L0.2688[962,1121] 180ns |----------------------------------------L0.2688-----------------------------------------|" - "L0.2701[962,1121] 181ns |----------------------------------------L0.2701-----------------------------------------|" @@ -7360,7 +7335,7 @@ async fn stuck_l0_large_l0s() { - "Committing partition 1:" - " Soft Deleting 20 files: L0.2688, L0.2701, L0.2714, L0.2727, L0.2740, L0.2753, L0.2766, L0.2779, L0.2792, L0.2805, L0.2818, L0.2831, L0.2844, L0.2857, L0.2870, L0.2883, L0.2896, L0.2909, L0.2922, L0.2935" - " Creating 1 files" - - "**** Simulation run 268, type=compact(ManySmallFiles). 20 Input Files, 160mb total:" + - "**** Simulation run 267, type=compact(ManySmallFiles). 20 Input Files, 160mb total:" - "L0, all files 8mb " - "L0.208[1122,1281] 0ns |-----------------------------------------L0.208-----------------------------------------|" - "L0.221[1122,1281] 1ns |-----------------------------------------L0.221-----------------------------------------|" @@ -7380,25 +7355,25 @@ async fn stuck_l0_large_l0s() { - "L0.403[1122,1281] 15ns |-----------------------------------------L0.403-----------------------------------------|" - "L0.416[1122,1281] 16ns |-----------------------------------------L0.416-----------------------------------------|" - "L0.429[1122,1281] 17ns |-----------------------------------------L0.429-----------------------------------------|" - - "L0.442[1122,1281] 18ns |-----------------------------------------L0.442-----------------------------------------|" + - "L0.554[1122,1281] 18ns |-----------------------------------------L0.554-----------------------------------------|" - "L0.567[1122,1281] 19ns |-----------------------------------------L0.567-----------------------------------------|" - "**** 1 Output Files (parquet_file_id not yet assigned), 160mb total:" - "L0, all files 160mb " - "L0.?[1122,1281] 19ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.208, L0.221, L0.234, L0.247, L0.260, L0.273, L0.286, L0.299, L0.312, L0.325, L0.338, L0.351, L0.364, L0.377, L0.390, L0.403, L0.416, L0.429, L0.442, L0.567" + - " Soft Deleting 20 files: L0.208, L0.221, L0.234, L0.247, L0.260, L0.273, L0.286, L0.299, L0.312, L0.325, L0.338, L0.351, L0.364, L0.377, L0.390, L0.403, L0.416, L0.429, L0.554, L0.567" - " Creating 1 files" - - "**** Simulation run 269, type=compact(ManySmallFiles). 20 Input Files, 0b total:" + - "**** Simulation run 268, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " - - "L0.580[1122,1281] 20ns |-----------------------------------------L0.580-----------------------------------------|" - - "L0.455[1122,1281] 21ns |-----------------------------------------L0.455-----------------------------------------|" - - "L0.469[1122,1281] 22ns |-----------------------------------------L0.469-----------------------------------------|" - - "L0.483[1122,1281] 23ns |-----------------------------------------L0.483-----------------------------------------|" - - "L0.497[1122,1281] 24ns |-----------------------------------------L0.497-----------------------------------------|" - - "L0.511[1122,1281] 25ns |-----------------------------------------L0.511-----------------------------------------|" - - "L0.525[1122,1281] 26ns |-----------------------------------------L0.525-----------------------------------------|" - - "L0.539[1122,1281] 27ns |-----------------------------------------L0.539-----------------------------------------|" - - "L0.553[1122,1281] 28ns |-----------------------------------------L0.553-----------------------------------------|" + - "L0.442[1122,1281] 20ns |-----------------------------------------L0.442-----------------------------------------|" + - "L0.456[1122,1281] 21ns |-----------------------------------------L0.456-----------------------------------------|" + - "L0.470[1122,1281] 22ns |-----------------------------------------L0.470-----------------------------------------|" + - "L0.484[1122,1281] 23ns |-----------------------------------------L0.484-----------------------------------------|" + - "L0.498[1122,1281] 24ns |-----------------------------------------L0.498-----------------------------------------|" + - "L0.512[1122,1281] 25ns |-----------------------------------------L0.512-----------------------------------------|" + - "L0.526[1122,1281] 26ns |-----------------------------------------L0.526-----------------------------------------|" + - "L0.540[1122,1281] 27ns |-----------------------------------------L0.540-----------------------------------------|" + - "L0.580[1122,1281] 28ns |-----------------------------------------L0.580-----------------------------------------|" - "L0.594[1122,1281] 29ns |-----------------------------------------L0.594-----------------------------------------|" - "L0.608[1122,1281] 30ns |-----------------------------------------L0.608-----------------------------------------|" - "L0.622[1122,1281] 31ns |-----------------------------------------L0.622-----------------------------------------|" @@ -7414,9 +7389,9 @@ async fn stuck_l0_large_l0s() { - "L0, all files 0b " - "L0.?[1122,1281] 39ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.455, L0.469, L0.483, L0.497, L0.511, L0.525, L0.539, L0.553, L0.580, L0.594, L0.608, L0.622, L0.636, L0.650, L0.664, L0.678, L0.692, L0.706, L0.720, L0.734" + - " Soft Deleting 20 files: L0.442, L0.456, L0.470, L0.484, L0.498, L0.512, L0.526, L0.540, L0.580, L0.594, L0.608, L0.622, L0.636, L0.650, L0.664, L0.678, L0.692, L0.706, L0.720, L0.734" - " Creating 1 files" - - "**** Simulation run 270, type=compact(ManySmallFiles). 20 Input Files, 0b total:" + - "**** Simulation run 269, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " - "L0.748[1122,1281] 40ns |-----------------------------------------L0.748-----------------------------------------|" - "L0.762[1122,1281] 41ns |-----------------------------------------L0.762-----------------------------------------|" @@ -7444,6 +7419,31 @@ async fn stuck_l0_large_l0s() { - "Committing partition 1:" - " Soft Deleting 20 files: L0.748, L0.762, L0.776, L0.790, L0.804, L0.818, L0.832, L0.846, L0.860, L0.874, L0.888, L0.902, L0.916, L0.930, L0.944, L0.958, L0.972, L0.986, L0.1000, L0.1014" - " Creating 1 files" + - "**** Simulation run 270, type=compact(ManySmallFiles). 20 Input Files, 0b total:" + - "L0, all files 0b " + - "L0.1027[962,1121] 60ns |----------------------------------------L0.1027-----------------------------------------|" + - "L0.1041[962,1121] 61ns |----------------------------------------L0.1041-----------------------------------------|" + - "L0.1055[962,1121] 62ns |----------------------------------------L0.1055-----------------------------------------|" + - "L0.1069[962,1121] 63ns |----------------------------------------L0.1069-----------------------------------------|" + - "L0.1083[962,1121] 64ns |----------------------------------------L0.1083-----------------------------------------|" + - "L0.1097[962,1121] 65ns |----------------------------------------L0.1097-----------------------------------------|" + - "L0.1111[962,1121] 66ns |----------------------------------------L0.1111-----------------------------------------|" + - "L0.1125[962,1121] 67ns |----------------------------------------L0.1125-----------------------------------------|" + - "L0.1139[962,1121] 68ns |----------------------------------------L0.1139-----------------------------------------|" + - "L0.1153[962,1121] 69ns |----------------------------------------L0.1153-----------------------------------------|" + - "L0.1167[962,1121] 70ns |----------------------------------------L0.1167-----------------------------------------|" + - "L0.1181[962,1121] 71ns |----------------------------------------L0.1181-----------------------------------------|" + - "L0.1195[962,1121] 72ns |----------------------------------------L0.1195-----------------------------------------|" + - "L0.1209[962,1121] 73ns |----------------------------------------L0.1209-----------------------------------------|" + - "L0.1223[962,1121] 74ns |----------------------------------------L0.1223-----------------------------------------|" + - "L0.1237[962,1121] 75ns |----------------------------------------L0.1237-----------------------------------------|" + - "L0.1251[962,1121] 76ns |----------------------------------------L0.1251-----------------------------------------|" + - "L0.1265[962,1121] 77ns |----------------------------------------L0.1265-----------------------------------------|" + - "L0.1279[962,1121] 78ns |----------------------------------------L0.1279-----------------------------------------|" + - "L0.1293[962,1121] 79ns |----------------------------------------L0.1293-----------------------------------------|" + - "**** 1 Output Files (parquet_file_id not yet assigned), 0b total:" + - "L0, all files 0b " + - "L0.?[962,1121] 79ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - " Soft Deleting 20 files: L0.1027, L0.1041, L0.1055, L0.1069, L0.1083, L0.1097, L0.1111, L0.1125, L0.1139, L0.1153, L0.1167, L0.1181, L0.1195, L0.1209, L0.1223, L0.1237, L0.1251, L0.1265, L0.1279, L0.1293" - " Creating 1 files" @@ -7575,9 +7575,9 @@ async fn stuck_l0_large_l0s() { - "L0.2022[1122,1281] 131ns |----------------------------------------L0.2022-----------------------------------------|" - "L0.2036[1122,1281] 132ns |----------------------------------------L0.2036-----------------------------------------|" - "L0.2050[1122,1281] 133ns |----------------------------------------L0.2050-----------------------------------------|" - - "L0.2064[1122,1281] 134ns |----------------------------------------L0.2064-----------------------------------------|" + - "L0.2176[1122,1281] 134ns |----------------------------------------L0.2176-----------------------------------------|" - "L0.2190[1122,1281] 135ns |----------------------------------------L0.2190-----------------------------------------|" - - "L0.2204[1122,1281] 136ns |----------------------------------------L0.2204-----------------------------------------|" + - "L0.2064[1122,1281] 136ns |----------------------------------------L0.2064-----------------------------------------|" - "L0.2078[1122,1281] 137ns |----------------------------------------L0.2078-----------------------------------------|" - "L0.2092[1122,1281] 138ns |----------------------------------------L0.2092-----------------------------------------|" - "L0.2106[1122,1281] 139ns |----------------------------------------L0.2106-----------------------------------------|" @@ -7585,7 +7585,7 @@ async fn stuck_l0_large_l0s() { - "L0, all files 0b " - "L0.?[1122,1281] 139ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.1868, L0.1882, L0.1896, L0.1910, L0.1924, L0.1938, L0.1952, L0.1966, L0.1980, L0.1994, L0.2008, L0.2022, L0.2036, L0.2050, L0.2064, L0.2078, L0.2092, L0.2106, L0.2190, L0.2204" + - " Soft Deleting 20 files: L0.1868, L0.1882, L0.1896, L0.1910, L0.1924, L0.1938, L0.1952, L0.1966, L0.1980, L0.1994, L0.2008, L0.2022, L0.2036, L0.2050, L0.2064, L0.2078, L0.2092, L0.2106, L0.2176, L0.2190" - " Creating 1 files" - "**** Simulation run 276, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " @@ -7593,7 +7593,7 @@ async fn stuck_l0_large_l0s() { - "L0.2134[1122,1281] 141ns |----------------------------------------L0.2134-----------------------------------------|" - "L0.2148[1122,1281] 142ns |----------------------------------------L0.2148-----------------------------------------|" - "L0.2162[1122,1281] 143ns |----------------------------------------L0.2162-----------------------------------------|" - - "L0.2176[1122,1281] 144ns |----------------------------------------L0.2176-----------------------------------------|" + - "L0.2204[1122,1281] 144ns |----------------------------------------L0.2204-----------------------------------------|" - "L0.2218[1122,1281] 145ns |----------------------------------------L0.2218-----------------------------------------|" - "L0.2232[1122,1281] 146ns |----------------------------------------L0.2232-----------------------------------------|" - "L0.2246[1122,1281] 147ns |----------------------------------------L0.2246-----------------------------------------|" @@ -7613,7 +7613,7 @@ async fn stuck_l0_large_l0s() { - "L0, all files 0b " - "L0.?[1122,1281] 159ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.2120, L0.2134, L0.2148, L0.2162, L0.2176, L0.2218, L0.2232, L0.2246, L0.2260, L0.2274, L0.2288, L0.2302, L0.2316, L0.2330, L0.2344, L0.2358, L0.2372, L0.2386, L0.2400, L0.2414" + - " Soft Deleting 20 files: L0.2120, L0.2134, L0.2148, L0.2162, L0.2204, L0.2218, L0.2232, L0.2246, L0.2260, L0.2274, L0.2288, L0.2302, L0.2316, L0.2330, L0.2344, L0.2358, L0.2372, L0.2386, L0.2400, L0.2414" - " Creating 1 files" - "**** Simulation run 277, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " @@ -7691,25 +7691,25 @@ async fn stuck_l0_large_l0s() { - "L0.404[1282,1441] 15ns |-----------------------------------------L0.404-----------------------------------------|" - "L0.417[1282,1441] 16ns |-----------------------------------------L0.417-----------------------------------------|" - "L0.430[1282,1441] 17ns |-----------------------------------------L0.430-----------------------------------------|" - - "L0.443[1282,1441] 18ns |-----------------------------------------L0.443-----------------------------------------|" + - "L0.555[1282,1441] 18ns |-----------------------------------------L0.555-----------------------------------------|" - "L0.568[1282,1441] 19ns |-----------------------------------------L0.568-----------------------------------------|" - "**** 1 Output Files (parquet_file_id not yet assigned), 160mb total:" - "L0, all files 160mb " - "L0.?[1282,1441] 19ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.209, L0.222, L0.235, L0.248, L0.261, L0.274, L0.287, L0.300, L0.313, L0.326, L0.339, L0.352, L0.365, L0.378, L0.391, L0.404, L0.417, L0.430, L0.443, L0.568" + - " Soft Deleting 20 files: L0.209, L0.222, L0.235, L0.248, L0.261, L0.274, L0.287, L0.300, L0.313, L0.326, L0.339, L0.352, L0.365, L0.378, L0.391, L0.404, L0.417, L0.430, L0.555, L0.568" - " Creating 1 files" - "**** Simulation run 280, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " - - "L0.581[1282,1441] 20ns |-----------------------------------------L0.581-----------------------------------------|" - - "L0.456[1282,1441] 21ns |-----------------------------------------L0.456-----------------------------------------|" - - "L0.470[1282,1441] 22ns |-----------------------------------------L0.470-----------------------------------------|" - - "L0.484[1282,1441] 23ns |-----------------------------------------L0.484-----------------------------------------|" - - "L0.498[1282,1441] 24ns |-----------------------------------------L0.498-----------------------------------------|" - - "L0.512[1282,1441] 25ns |-----------------------------------------L0.512-----------------------------------------|" - - "L0.526[1282,1441] 26ns |-----------------------------------------L0.526-----------------------------------------|" - - "L0.540[1282,1441] 27ns |-----------------------------------------L0.540-----------------------------------------|" - - "L0.554[1282,1441] 28ns |-----------------------------------------L0.554-----------------------------------------|" + - "L0.443[1282,1441] 20ns |-----------------------------------------L0.443-----------------------------------------|" + - "L0.457[1282,1441] 21ns |-----------------------------------------L0.457-----------------------------------------|" + - "L0.471[1282,1441] 22ns |-----------------------------------------L0.471-----------------------------------------|" + - "L0.485[1282,1441] 23ns |-----------------------------------------L0.485-----------------------------------------|" + - "L0.499[1282,1441] 24ns |-----------------------------------------L0.499-----------------------------------------|" + - "L0.513[1282,1441] 25ns |-----------------------------------------L0.513-----------------------------------------|" + - "L0.527[1282,1441] 26ns |-----------------------------------------L0.527-----------------------------------------|" + - "L0.541[1282,1441] 27ns |-----------------------------------------L0.541-----------------------------------------|" + - "L0.581[1282,1441] 28ns |-----------------------------------------L0.581-----------------------------------------|" - "L0.595[1282,1441] 29ns |-----------------------------------------L0.595-----------------------------------------|" - "L0.609[1282,1441] 30ns |-----------------------------------------L0.609-----------------------------------------|" - "L0.623[1282,1441] 31ns |-----------------------------------------L0.623-----------------------------------------|" @@ -7725,7 +7725,7 @@ async fn stuck_l0_large_l0s() { - "L0, all files 0b " - "L0.?[1282,1441] 39ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.456, L0.470, L0.484, L0.498, L0.512, L0.526, L0.540, L0.554, L0.581, L0.595, L0.609, L0.623, L0.637, L0.651, L0.665, L0.679, L0.693, L0.707, L0.721, L0.735" + - " Soft Deleting 20 files: L0.443, L0.457, L0.471, L0.485, L0.499, L0.513, L0.527, L0.541, L0.581, L0.595, L0.609, L0.623, L0.637, L0.651, L0.665, L0.679, L0.693, L0.707, L0.721, L0.735" - " Creating 1 files" - "**** Simulation run 281, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " @@ -7855,9 +7855,9 @@ async fn stuck_l0_large_l0s() { - "L0.2023[1282,1441] 131ns |----------------------------------------L0.2023-----------------------------------------|" - "L0.2037[1282,1441] 132ns |----------------------------------------L0.2037-----------------------------------------|" - "L0.2051[1282,1441] 133ns |----------------------------------------L0.2051-----------------------------------------|" - - "L0.2065[1282,1441] 134ns |----------------------------------------L0.2065-----------------------------------------|" + - "L0.2177[1282,1441] 134ns |----------------------------------------L0.2177-----------------------------------------|" - "L0.2191[1282,1441] 135ns |----------------------------------------L0.2191-----------------------------------------|" - - "L0.2205[1282,1441] 136ns |----------------------------------------L0.2205-----------------------------------------|" + - "L0.2065[1282,1441] 136ns |----------------------------------------L0.2065-----------------------------------------|" - "L0.2079[1282,1441] 137ns |----------------------------------------L0.2079-----------------------------------------|" - "L0.2093[1282,1441] 138ns |----------------------------------------L0.2093-----------------------------------------|" - "L0.2107[1282,1441] 139ns |----------------------------------------L0.2107-----------------------------------------|" @@ -7865,7 +7865,7 @@ async fn stuck_l0_large_l0s() { - "L0, all files 0b " - "L0.?[1282,1441] 139ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.1869, L0.1883, L0.1897, L0.1911, L0.1925, L0.1939, L0.1953, L0.1967, L0.1981, L0.1995, L0.2009, L0.2023, L0.2037, L0.2051, L0.2065, L0.2079, L0.2093, L0.2107, L0.2191, L0.2205" + - " Soft Deleting 20 files: L0.1869, L0.1883, L0.1897, L0.1911, L0.1925, L0.1939, L0.1953, L0.1967, L0.1981, L0.1995, L0.2009, L0.2023, L0.2037, L0.2051, L0.2065, L0.2079, L0.2093, L0.2107, L0.2177, L0.2191" - " Creating 1 files" - "**** Simulation run 286, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " @@ -7873,7 +7873,7 @@ async fn stuck_l0_large_l0s() { - "L0.2135[1282,1441] 141ns |----------------------------------------L0.2135-----------------------------------------|" - "L0.2149[1282,1441] 142ns |----------------------------------------L0.2149-----------------------------------------|" - "L0.2163[1282,1441] 143ns |----------------------------------------L0.2163-----------------------------------------|" - - "L0.2177[1282,1441] 144ns |----------------------------------------L0.2177-----------------------------------------|" + - "L0.2205[1282,1441] 144ns |----------------------------------------L0.2205-----------------------------------------|" - "L0.2219[1282,1441] 145ns |----------------------------------------L0.2219-----------------------------------------|" - "L0.2233[1282,1441] 146ns |----------------------------------------L0.2233-----------------------------------------|" - "L0.2247[1282,1441] 147ns |----------------------------------------L0.2247-----------------------------------------|" @@ -7893,7 +7893,7 @@ async fn stuck_l0_large_l0s() { - "L0, all files 0b " - "L0.?[1282,1441] 159ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.2121, L0.2135, L0.2149, L0.2163, L0.2177, L0.2219, L0.2233, L0.2247, L0.2261, L0.2275, L0.2289, L0.2303, L0.2317, L0.2331, L0.2345, L0.2359, L0.2373, L0.2387, L0.2401, L0.2415" + - " Soft Deleting 20 files: L0.2121, L0.2135, L0.2149, L0.2163, L0.2205, L0.2219, L0.2233, L0.2247, L0.2261, L0.2275, L0.2289, L0.2303, L0.2317, L0.2331, L0.2345, L0.2359, L0.2373, L0.2387, L0.2401, L0.2415" - " Creating 1 files" - "**** Simulation run 287, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " @@ -7943,25 +7943,25 @@ async fn stuck_l0_large_l0s() { - "L0.405[1442,1601] 15ns |-----------------------------------------L0.405-----------------------------------------|" - "L0.418[1442,1601] 16ns |-----------------------------------------L0.418-----------------------------------------|" - "L0.431[1442,1601] 17ns |-----------------------------------------L0.431-----------------------------------------|" - - "L0.444[1442,1601] 18ns |-----------------------------------------L0.444-----------------------------------------|" + - "L0.556[1442,1601] 18ns |-----------------------------------------L0.556-----------------------------------------|" - "L0.569[1442,1601] 19ns |-----------------------------------------L0.569-----------------------------------------|" - "**** 1 Output Files (parquet_file_id not yet assigned), 160mb total:" - "L0, all files 160mb " - "L0.?[1442,1601] 19ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.210, L0.223, L0.236, L0.249, L0.262, L0.275, L0.288, L0.301, L0.314, L0.327, L0.340, L0.353, L0.366, L0.379, L0.392, L0.405, L0.418, L0.431, L0.444, L0.569" + - " Soft Deleting 20 files: L0.210, L0.223, L0.236, L0.249, L0.262, L0.275, L0.288, L0.301, L0.314, L0.327, L0.340, L0.353, L0.366, L0.379, L0.392, L0.405, L0.418, L0.431, L0.556, L0.569" - " Creating 1 files" - "**** Simulation run 289, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " - - "L0.582[1442,1601] 20ns |-----------------------------------------L0.582-----------------------------------------|" - - "L0.457[1442,1601] 21ns |-----------------------------------------L0.457-----------------------------------------|" - - "L0.471[1442,1601] 22ns |-----------------------------------------L0.471-----------------------------------------|" - - "L0.485[1442,1601] 23ns |-----------------------------------------L0.485-----------------------------------------|" - - "L0.499[1442,1601] 24ns |-----------------------------------------L0.499-----------------------------------------|" - - "L0.513[1442,1601] 25ns |-----------------------------------------L0.513-----------------------------------------|" - - "L0.527[1442,1601] 26ns |-----------------------------------------L0.527-----------------------------------------|" - - "L0.541[1442,1601] 27ns |-----------------------------------------L0.541-----------------------------------------|" - - "L0.555[1442,1601] 28ns |-----------------------------------------L0.555-----------------------------------------|" + - "L0.444[1442,1601] 20ns |-----------------------------------------L0.444-----------------------------------------|" + - "L0.458[1442,1601] 21ns |-----------------------------------------L0.458-----------------------------------------|" + - "L0.472[1442,1601] 22ns |-----------------------------------------L0.472-----------------------------------------|" + - "L0.486[1442,1601] 23ns |-----------------------------------------L0.486-----------------------------------------|" + - "L0.500[1442,1601] 24ns |-----------------------------------------L0.500-----------------------------------------|" + - "L0.514[1442,1601] 25ns |-----------------------------------------L0.514-----------------------------------------|" + - "L0.528[1442,1601] 26ns |-----------------------------------------L0.528-----------------------------------------|" + - "L0.542[1442,1601] 27ns |-----------------------------------------L0.542-----------------------------------------|" + - "L0.582[1442,1601] 28ns |-----------------------------------------L0.582-----------------------------------------|" - "L0.596[1442,1601] 29ns |-----------------------------------------L0.596-----------------------------------------|" - "L0.610[1442,1601] 30ns |-----------------------------------------L0.610-----------------------------------------|" - "L0.624[1442,1601] 31ns |-----------------------------------------L0.624-----------------------------------------|" @@ -7977,7 +7977,7 @@ async fn stuck_l0_large_l0s() { - "L0, all files 0b " - "L0.?[1442,1601] 39ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.457, L0.471, L0.485, L0.499, L0.513, L0.527, L0.541, L0.555, L0.582, L0.596, L0.610, L0.624, L0.638, L0.652, L0.666, L0.680, L0.694, L0.708, L0.722, L0.736" + - " Soft Deleting 20 files: L0.444, L0.458, L0.472, L0.486, L0.500, L0.514, L0.528, L0.542, L0.582, L0.596, L0.610, L0.624, L0.638, L0.652, L0.666, L0.680, L0.694, L0.708, L0.722, L0.736" - " Creating 1 files" - "**** Simulation run 290, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " @@ -8107,9 +8107,9 @@ async fn stuck_l0_large_l0s() { - "L0.2024[1442,1601] 131ns |----------------------------------------L0.2024-----------------------------------------|" - "L0.2038[1442,1601] 132ns |----------------------------------------L0.2038-----------------------------------------|" - "L0.2052[1442,1601] 133ns |----------------------------------------L0.2052-----------------------------------------|" - - "L0.2066[1442,1601] 134ns |----------------------------------------L0.2066-----------------------------------------|" + - "L0.2178[1442,1601] 134ns |----------------------------------------L0.2178-----------------------------------------|" - "L0.2192[1442,1601] 135ns |----------------------------------------L0.2192-----------------------------------------|" - - "L0.2206[1442,1601] 136ns |----------------------------------------L0.2206-----------------------------------------|" + - "L0.2066[1442,1601] 136ns |----------------------------------------L0.2066-----------------------------------------|" - "L0.2080[1442,1601] 137ns |----------------------------------------L0.2080-----------------------------------------|" - "L0.2094[1442,1601] 138ns |----------------------------------------L0.2094-----------------------------------------|" - "L0.2108[1442,1601] 139ns |----------------------------------------L0.2108-----------------------------------------|" @@ -8117,7 +8117,7 @@ async fn stuck_l0_large_l0s() { - "L0, all files 0b " - "L0.?[1442,1601] 139ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.1870, L0.1884, L0.1898, L0.1912, L0.1926, L0.1940, L0.1954, L0.1968, L0.1982, L0.1996, L0.2010, L0.2024, L0.2038, L0.2052, L0.2066, L0.2080, L0.2094, L0.2108, L0.2192, L0.2206" + - " Soft Deleting 20 files: L0.1870, L0.1884, L0.1898, L0.1912, L0.1926, L0.1940, L0.1954, L0.1968, L0.1982, L0.1996, L0.2010, L0.2024, L0.2038, L0.2052, L0.2066, L0.2080, L0.2094, L0.2108, L0.2178, L0.2192" - " Creating 1 files" - "**** Simulation run 295, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " @@ -8125,7 +8125,7 @@ async fn stuck_l0_large_l0s() { - "L0.2136[1442,1601] 141ns |----------------------------------------L0.2136-----------------------------------------|" - "L0.2150[1442,1601] 142ns |----------------------------------------L0.2150-----------------------------------------|" - "L0.2164[1442,1601] 143ns |----------------------------------------L0.2164-----------------------------------------|" - - "L0.2178[1442,1601] 144ns |----------------------------------------L0.2178-----------------------------------------|" + - "L0.2206[1442,1601] 144ns |----------------------------------------L0.2206-----------------------------------------|" - "L0.2220[1442,1601] 145ns |----------------------------------------L0.2220-----------------------------------------|" - "L0.2234[1442,1601] 146ns |----------------------------------------L0.2234-----------------------------------------|" - "L0.2248[1442,1601] 147ns |----------------------------------------L0.2248-----------------------------------------|" @@ -8145,7 +8145,7 @@ async fn stuck_l0_large_l0s() { - "L0, all files 0b " - "L0.?[1442,1601] 159ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.2122, L0.2136, L0.2150, L0.2164, L0.2178, L0.2220, L0.2234, L0.2248, L0.2262, L0.2276, L0.2290, L0.2304, L0.2318, L0.2332, L0.2346, L0.2360, L0.2374, L0.2388, L0.2402, L0.2416" + - " Soft Deleting 20 files: L0.2122, L0.2136, L0.2150, L0.2164, L0.2206, L0.2220, L0.2234, L0.2248, L0.2262, L0.2276, L0.2290, L0.2304, L0.2318, L0.2332, L0.2346, L0.2360, L0.2374, L0.2388, L0.2402, L0.2416" - " Creating 1 files" - "**** Simulation run 296, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " @@ -8251,25 +8251,25 @@ async fn stuck_l0_large_l0s() { - "L0.406[1602,1761] 15ns |-----------------------------------------L0.406-----------------------------------------|" - "L0.419[1602,1761] 16ns |-----------------------------------------L0.419-----------------------------------------|" - "L0.432[1602,1761] 17ns |-----------------------------------------L0.432-----------------------------------------|" - - "L0.445[1602,1761] 18ns |-----------------------------------------L0.445-----------------------------------------|" + - "L0.557[1602,1761] 18ns |-----------------------------------------L0.557-----------------------------------------|" - "L0.570[1602,1761] 19ns |-----------------------------------------L0.570-----------------------------------------|" - "**** 1 Output Files (parquet_file_id not yet assigned), 160mb total:" - "L0, all files 160mb " - "L0.?[1602,1761] 19ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.211, L0.224, L0.237, L0.250, L0.263, L0.276, L0.289, L0.302, L0.315, L0.328, L0.341, L0.354, L0.367, L0.380, L0.393, L0.406, L0.419, L0.432, L0.445, L0.570" + - " Soft Deleting 20 files: L0.211, L0.224, L0.237, L0.250, L0.263, L0.276, L0.289, L0.302, L0.315, L0.328, L0.341, L0.354, L0.367, L0.380, L0.393, L0.406, L0.419, L0.432, L0.557, L0.570" - " Creating 1 files" - "**** Simulation run 300, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " - - "L0.583[1602,1761] 20ns |-----------------------------------------L0.583-----------------------------------------|" - - "L0.458[1602,1761] 21ns |-----------------------------------------L0.458-----------------------------------------|" - - "L0.472[1602,1761] 22ns |-----------------------------------------L0.472-----------------------------------------|" - - "L0.486[1602,1761] 23ns |-----------------------------------------L0.486-----------------------------------------|" - - "L0.500[1602,1761] 24ns |-----------------------------------------L0.500-----------------------------------------|" - - "L0.514[1602,1761] 25ns |-----------------------------------------L0.514-----------------------------------------|" - - "L0.528[1602,1761] 26ns |-----------------------------------------L0.528-----------------------------------------|" - - "L0.542[1602,1761] 27ns |-----------------------------------------L0.542-----------------------------------------|" - - "L0.556[1602,1761] 28ns |-----------------------------------------L0.556-----------------------------------------|" + - "L0.445[1602,1761] 20ns |-----------------------------------------L0.445-----------------------------------------|" + - "L0.459[1602,1761] 21ns |-----------------------------------------L0.459-----------------------------------------|" + - "L0.473[1602,1761] 22ns |-----------------------------------------L0.473-----------------------------------------|" + - "L0.487[1602,1761] 23ns |-----------------------------------------L0.487-----------------------------------------|" + - "L0.501[1602,1761] 24ns |-----------------------------------------L0.501-----------------------------------------|" + - "L0.515[1602,1761] 25ns |-----------------------------------------L0.515-----------------------------------------|" + - "L0.529[1602,1761] 26ns |-----------------------------------------L0.529-----------------------------------------|" + - "L0.543[1602,1761] 27ns |-----------------------------------------L0.543-----------------------------------------|" + - "L0.583[1602,1761] 28ns |-----------------------------------------L0.583-----------------------------------------|" - "L0.597[1602,1761] 29ns |-----------------------------------------L0.597-----------------------------------------|" - "L0.611[1602,1761] 30ns |-----------------------------------------L0.611-----------------------------------------|" - "L0.625[1602,1761] 31ns |-----------------------------------------L0.625-----------------------------------------|" @@ -8285,7 +8285,7 @@ async fn stuck_l0_large_l0s() { - "L0, all files 0b " - "L0.?[1602,1761] 39ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.458, L0.472, L0.486, L0.500, L0.514, L0.528, L0.542, L0.556, L0.583, L0.597, L0.611, L0.625, L0.639, L0.653, L0.667, L0.681, L0.695, L0.709, L0.723, L0.737" + - " Soft Deleting 20 files: L0.445, L0.459, L0.473, L0.487, L0.501, L0.515, L0.529, L0.543, L0.583, L0.597, L0.611, L0.625, L0.639, L0.653, L0.667, L0.681, L0.695, L0.709, L0.723, L0.737" - " Creating 1 files" - "**** Simulation run 301, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " @@ -8415,9 +8415,9 @@ async fn stuck_l0_large_l0s() { - "L0.2025[1602,1761] 131ns |----------------------------------------L0.2025-----------------------------------------|" - "L0.2039[1602,1761] 132ns |----------------------------------------L0.2039-----------------------------------------|" - "L0.2053[1602,1761] 133ns |----------------------------------------L0.2053-----------------------------------------|" - - "L0.2067[1602,1761] 134ns |----------------------------------------L0.2067-----------------------------------------|" + - "L0.2179[1602,1761] 134ns |----------------------------------------L0.2179-----------------------------------------|" - "L0.2193[1602,1761] 135ns |----------------------------------------L0.2193-----------------------------------------|" - - "L0.2207[1602,1761] 136ns |----------------------------------------L0.2207-----------------------------------------|" + - "L0.2067[1602,1761] 136ns |----------------------------------------L0.2067-----------------------------------------|" - "L0.2081[1602,1761] 137ns |----------------------------------------L0.2081-----------------------------------------|" - "L0.2095[1602,1761] 138ns |----------------------------------------L0.2095-----------------------------------------|" - "L0.2109[1602,1761] 139ns |----------------------------------------L0.2109-----------------------------------------|" @@ -8425,7 +8425,7 @@ async fn stuck_l0_large_l0s() { - "L0, all files 0b " - "L0.?[1602,1761] 139ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.1871, L0.1885, L0.1899, L0.1913, L0.1927, L0.1941, L0.1955, L0.1969, L0.1983, L0.1997, L0.2011, L0.2025, L0.2039, L0.2053, L0.2067, L0.2081, L0.2095, L0.2109, L0.2193, L0.2207" + - " Soft Deleting 20 files: L0.1871, L0.1885, L0.1899, L0.1913, L0.1927, L0.1941, L0.1955, L0.1969, L0.1983, L0.1997, L0.2011, L0.2025, L0.2039, L0.2053, L0.2067, L0.2081, L0.2095, L0.2109, L0.2179, L0.2193" - " Creating 1 files" - "**** Simulation run 306, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " @@ -8433,7 +8433,7 @@ async fn stuck_l0_large_l0s() { - "L0.2137[1602,1761] 141ns |----------------------------------------L0.2137-----------------------------------------|" - "L0.2151[1602,1761] 142ns |----------------------------------------L0.2151-----------------------------------------|" - "L0.2165[1602,1761] 143ns |----------------------------------------L0.2165-----------------------------------------|" - - "L0.2179[1602,1761] 144ns |----------------------------------------L0.2179-----------------------------------------|" + - "L0.2207[1602,1761] 144ns |----------------------------------------L0.2207-----------------------------------------|" - "L0.2221[1602,1761] 145ns |----------------------------------------L0.2221-----------------------------------------|" - "L0.2235[1602,1761] 146ns |----------------------------------------L0.2235-----------------------------------------|" - "L0.2249[1602,1761] 147ns |----------------------------------------L0.2249-----------------------------------------|" @@ -8453,7 +8453,7 @@ async fn stuck_l0_large_l0s() { - "L0, all files 0b " - "L0.?[1602,1761] 159ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.2123, L0.2137, L0.2151, L0.2165, L0.2179, L0.2221, L0.2235, L0.2249, L0.2263, L0.2277, L0.2291, L0.2305, L0.2319, L0.2333, L0.2347, L0.2361, L0.2375, L0.2389, L0.2403, L0.2417" + - " Soft Deleting 20 files: L0.2123, L0.2137, L0.2151, L0.2165, L0.2207, L0.2221, L0.2235, L0.2249, L0.2263, L0.2277, L0.2291, L0.2305, L0.2319, L0.2333, L0.2347, L0.2361, L0.2375, L0.2389, L0.2403, L0.2417" - " Creating 1 files" - "**** Simulation run 307, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " @@ -8531,25 +8531,25 @@ async fn stuck_l0_large_l0s() { - "L0.407[1762,1921] 15ns |-----------------------------------------L0.407-----------------------------------------|" - "L0.420[1762,1921] 16ns |-----------------------------------------L0.420-----------------------------------------|" - "L0.433[1762,1921] 17ns |-----------------------------------------L0.433-----------------------------------------|" - - "L0.446[1762,1921] 18ns |-----------------------------------------L0.446-----------------------------------------|" + - "L0.558[1762,1921] 18ns |-----------------------------------------L0.558-----------------------------------------|" - "L0.571[1762,1921] 19ns |-----------------------------------------L0.571-----------------------------------------|" - "**** 1 Output Files (parquet_file_id not yet assigned), 160mb total:" - "L0, all files 160mb " - "L0.?[1762,1921] 19ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.212, L0.225, L0.238, L0.251, L0.264, L0.277, L0.290, L0.303, L0.316, L0.329, L0.342, L0.355, L0.368, L0.381, L0.394, L0.407, L0.420, L0.433, L0.446, L0.571" + - " Soft Deleting 20 files: L0.212, L0.225, L0.238, L0.251, L0.264, L0.277, L0.290, L0.303, L0.316, L0.329, L0.342, L0.355, L0.368, L0.381, L0.394, L0.407, L0.420, L0.433, L0.558, L0.571" - " Creating 1 files" - "**** Simulation run 310, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " - - "L0.584[1762,1921] 20ns |-----------------------------------------L0.584-----------------------------------------|" - - "L0.459[1762,1921] 21ns |-----------------------------------------L0.459-----------------------------------------|" - - "L0.473[1762,1921] 22ns |-----------------------------------------L0.473-----------------------------------------|" - - "L0.487[1762,1921] 23ns |-----------------------------------------L0.487-----------------------------------------|" - - "L0.501[1762,1921] 24ns |-----------------------------------------L0.501-----------------------------------------|" - - "L0.515[1762,1921] 25ns |-----------------------------------------L0.515-----------------------------------------|" - - "L0.529[1762,1921] 26ns |-----------------------------------------L0.529-----------------------------------------|" - - "L0.543[1762,1921] 27ns |-----------------------------------------L0.543-----------------------------------------|" - - "L0.557[1762,1921] 28ns |-----------------------------------------L0.557-----------------------------------------|" + - "L0.446[1762,1921] 20ns |-----------------------------------------L0.446-----------------------------------------|" + - "L0.460[1762,1921] 21ns |-----------------------------------------L0.460-----------------------------------------|" + - "L0.474[1762,1921] 22ns |-----------------------------------------L0.474-----------------------------------------|" + - "L0.488[1762,1921] 23ns |-----------------------------------------L0.488-----------------------------------------|" + - "L0.502[1762,1921] 24ns |-----------------------------------------L0.502-----------------------------------------|" + - "L0.516[1762,1921] 25ns |-----------------------------------------L0.516-----------------------------------------|" + - "L0.530[1762,1921] 26ns |-----------------------------------------L0.530-----------------------------------------|" + - "L0.544[1762,1921] 27ns |-----------------------------------------L0.544-----------------------------------------|" + - "L0.584[1762,1921] 28ns |-----------------------------------------L0.584-----------------------------------------|" - "L0.598[1762,1921] 29ns |-----------------------------------------L0.598-----------------------------------------|" - "L0.612[1762,1921] 30ns |-----------------------------------------L0.612-----------------------------------------|" - "L0.626[1762,1921] 31ns |-----------------------------------------L0.626-----------------------------------------|" @@ -8565,7 +8565,7 @@ async fn stuck_l0_large_l0s() { - "L0, all files 0b " - "L0.?[1762,1921] 39ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.459, L0.473, L0.487, L0.501, L0.515, L0.529, L0.543, L0.557, L0.584, L0.598, L0.612, L0.626, L0.640, L0.654, L0.668, L0.682, L0.696, L0.710, L0.724, L0.738" + - " Soft Deleting 20 files: L0.446, L0.460, L0.474, L0.488, L0.502, L0.516, L0.530, L0.544, L0.584, L0.598, L0.612, L0.626, L0.640, L0.654, L0.668, L0.682, L0.696, L0.710, L0.724, L0.738" - " Creating 1 files" - "**** Simulation run 311, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " @@ -8664,9 +8664,9 @@ async fn stuck_l0_large_l0s() { - "L0.2026[1762,1921] 131ns |----------------------------------------L0.2026-----------------------------------------|" - "L0.2040[1762,1921] 132ns |----------------------------------------L0.2040-----------------------------------------|" - "L0.2054[1762,1921] 133ns |----------------------------------------L0.2054-----------------------------------------|" - - "L0.2068[1762,1921] 134ns |----------------------------------------L0.2068-----------------------------------------|" + - "L0.2180[1762,1921] 134ns |----------------------------------------L0.2180-----------------------------------------|" - "L0.2194[1762,1921] 135ns |----------------------------------------L0.2194-----------------------------------------|" - - "L0.2208[1762,1921] 136ns |----------------------------------------L0.2208-----------------------------------------|" + - "L0.2068[1762,1921] 136ns |----------------------------------------L0.2068-----------------------------------------|" - "L0.2082[1762,1921] 137ns |----------------------------------------L0.2082-----------------------------------------|" - "L0.2096[1762,1921] 138ns |----------------------------------------L0.2096-----------------------------------------|" - "L0.2110[1762,1921] 139ns |----------------------------------------L0.2110-----------------------------------------|" @@ -8674,7 +8674,7 @@ async fn stuck_l0_large_l0s() { - "L0, all files 0b " - "L0.?[1762,1921] 139ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.1872, L0.1886, L0.1900, L0.1914, L0.1928, L0.1942, L0.1956, L0.1970, L0.1984, L0.1998, L0.2012, L0.2026, L0.2040, L0.2054, L0.2068, L0.2082, L0.2096, L0.2110, L0.2194, L0.2208" + - " Soft Deleting 20 files: L0.1872, L0.1886, L0.1900, L0.1914, L0.1928, L0.1942, L0.1956, L0.1970, L0.1984, L0.1998, L0.2012, L0.2026, L0.2040, L0.2054, L0.2068, L0.2082, L0.2096, L0.2110, L0.2180, L0.2194" - " Creating 1 files" - "**** Simulation run 315, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " @@ -8682,7 +8682,7 @@ async fn stuck_l0_large_l0s() { - "L0.2138[1762,1921] 141ns |----------------------------------------L0.2138-----------------------------------------|" - "L0.2152[1762,1921] 142ns |----------------------------------------L0.2152-----------------------------------------|" - "L0.2166[1762,1921] 143ns |----------------------------------------L0.2166-----------------------------------------|" - - "L0.2180[1762,1921] 144ns |----------------------------------------L0.2180-----------------------------------------|" + - "L0.2208[1762,1921] 144ns |----------------------------------------L0.2208-----------------------------------------|" - "L0.2222[1762,1921] 145ns |----------------------------------------L0.2222-----------------------------------------|" - "L0.2236[1762,1921] 146ns |----------------------------------------L0.2236-----------------------------------------|" - "L0.2250[1762,1921] 147ns |----------------------------------------L0.2250-----------------------------------------|" @@ -8702,7 +8702,7 @@ async fn stuck_l0_large_l0s() { - "L0, all files 0b " - "L0.?[1762,1921] 159ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.2124, L0.2138, L0.2152, L0.2166, L0.2180, L0.2222, L0.2236, L0.2250, L0.2264, L0.2278, L0.2292, L0.2306, L0.2320, L0.2334, L0.2348, L0.2362, L0.2376, L0.2390, L0.2404, L0.2418" + - " Soft Deleting 20 files: L0.2124, L0.2138, L0.2152, L0.2166, L0.2208, L0.2222, L0.2236, L0.2250, L0.2264, L0.2278, L0.2292, L0.2306, L0.2320, L0.2334, L0.2348, L0.2362, L0.2376, L0.2390, L0.2404, L0.2418" - " Creating 1 files" - "**** Simulation run 316, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " @@ -8780,25 +8780,25 @@ async fn stuck_l0_large_l0s() { - "L0.408[1922,2000] 15ns |-----------------------------------------L0.408-----------------------------------------|" - "L0.421[1922,2000] 16ns |-----------------------------------------L0.421-----------------------------------------|" - "L0.434[1922,2000] 17ns |-----------------------------------------L0.434-----------------------------------------|" - - "L0.447[1922,2000] 18ns |-----------------------------------------L0.447-----------------------------------------|" + - "L0.559[1922,2000] 18ns |-----------------------------------------L0.559-----------------------------------------|" - "L0.572[1922,2000] 19ns |-----------------------------------------L0.572-----------------------------------------|" - "**** 1 Output Files (parquet_file_id not yet assigned), 79mb total:" - "L0, all files 79mb " - "L0.?[1922,2000] 19ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.213, L0.226, L0.239, L0.252, L0.265, L0.278, L0.291, L0.304, L0.317, L0.330, L0.343, L0.356, L0.369, L0.382, L0.395, L0.408, L0.421, L0.434, L0.447, L0.572" + - " Soft Deleting 20 files: L0.213, L0.226, L0.239, L0.252, L0.265, L0.278, L0.291, L0.304, L0.317, L0.330, L0.343, L0.356, L0.369, L0.382, L0.395, L0.408, L0.421, L0.434, L0.559, L0.572" - " Creating 1 files" - "**** Simulation run 319, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " - - "L0.585[1922,2086] 20ns |-----------------------------------------L0.585-----------------------------------------|" - - "L0.460[1922,2086] 21ns |-----------------------------------------L0.460-----------------------------------------|" - - "L0.474[1922,2086] 22ns |-----------------------------------------L0.474-----------------------------------------|" - - "L0.488[1922,2086] 23ns |-----------------------------------------L0.488-----------------------------------------|" - - "L0.502[1922,2086] 24ns |-----------------------------------------L0.502-----------------------------------------|" - - "L0.516[1922,2086] 25ns |-----------------------------------------L0.516-----------------------------------------|" - - "L0.530[1922,2086] 26ns |-----------------------------------------L0.530-----------------------------------------|" - - "L0.544[1922,2086] 27ns |-----------------------------------------L0.544-----------------------------------------|" - - "L0.558[1922,2086] 28ns |-----------------------------------------L0.558-----------------------------------------|" + - "L0.447[1922,2086] 20ns |-----------------------------------------L0.447-----------------------------------------|" + - "L0.461[1922,2086] 21ns |-----------------------------------------L0.461-----------------------------------------|" + - "L0.475[1922,2086] 22ns |-----------------------------------------L0.475-----------------------------------------|" + - "L0.489[1922,2086] 23ns |-----------------------------------------L0.489-----------------------------------------|" + - "L0.503[1922,2086] 24ns |-----------------------------------------L0.503-----------------------------------------|" + - "L0.517[1922,2086] 25ns |-----------------------------------------L0.517-----------------------------------------|" + - "L0.531[1922,2086] 26ns |-----------------------------------------L0.531-----------------------------------------|" + - "L0.545[1922,2086] 27ns |-----------------------------------------L0.545-----------------------------------------|" + - "L0.585[1922,2086] 28ns |-----------------------------------------L0.585-----------------------------------------|" - "L0.599[1922,2086] 29ns |-----------------------------------------L0.599-----------------------------------------|" - "L0.613[1922,2086] 30ns |-----------------------------------------L0.613-----------------------------------------|" - "L0.627[1922,2086] 31ns |-----------------------------------------L0.627-----------------------------------------|" @@ -8814,7 +8814,7 @@ async fn stuck_l0_large_l0s() { - "L0, all files 0b " - "L0.?[1922,2086] 39ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.460, L0.474, L0.488, L0.502, L0.516, L0.530, L0.544, L0.558, L0.585, L0.599, L0.613, L0.627, L0.641, L0.655, L0.669, L0.683, L0.697, L0.711, L0.725, L0.739" + - " Soft Deleting 20 files: L0.447, L0.461, L0.475, L0.489, L0.503, L0.517, L0.531, L0.545, L0.585, L0.599, L0.613, L0.627, L0.641, L0.655, L0.669, L0.683, L0.697, L0.711, L0.725, L0.739" - " Creating 1 files" - "**** Simulation run 320, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " @@ -8975,9 +8975,9 @@ async fn stuck_l0_large_l0s() { - "L0.2027[1922,2086] 131ns |----------------------------------------L0.2027-----------------------------------------|" - "L0.2041[1922,2086] 132ns |----------------------------------------L0.2041-----------------------------------------|" - "L0.2055[1922,2086] 133ns |----------------------------------------L0.2055-----------------------------------------|" - - "L0.2069[1922,2086] 134ns |----------------------------------------L0.2069-----------------------------------------|" + - "L0.2181[1922,2086] 134ns |----------------------------------------L0.2181-----------------------------------------|" - "L0.2195[1922,2086] 135ns |----------------------------------------L0.2195-----------------------------------------|" - - "L0.2209[1922,2086] 136ns |----------------------------------------L0.2209-----------------------------------------|" + - "L0.2069[1922,2086] 136ns |----------------------------------------L0.2069-----------------------------------------|" - "L0.2083[1922,2086] 137ns |----------------------------------------L0.2083-----------------------------------------|" - "L0.2097[1922,2086] 138ns |----------------------------------------L0.2097-----------------------------------------|" - "L0.2111[1922,2086] 139ns |----------------------------------------L0.2111-----------------------------------------|" @@ -8985,7 +8985,7 @@ async fn stuck_l0_large_l0s() { - "L0, all files 0b " - "L0.?[1922,2086] 139ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.1873, L0.1887, L0.1901, L0.1915, L0.1929, L0.1943, L0.1957, L0.1971, L0.1985, L0.1999, L0.2013, L0.2027, L0.2041, L0.2055, L0.2069, L0.2083, L0.2097, L0.2111, L0.2195, L0.2209" + - " Soft Deleting 20 files: L0.1873, L0.1887, L0.1901, L0.1915, L0.1929, L0.1943, L0.1957, L0.1971, L0.1985, L0.1999, L0.2013, L0.2027, L0.2041, L0.2055, L0.2069, L0.2083, L0.2097, L0.2111, L0.2181, L0.2195" - " Creating 1 files" - "**** Simulation run 326, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " @@ -8993,7 +8993,7 @@ async fn stuck_l0_large_l0s() { - "L0.2139[1922,2086] 141ns |----------------------------------------L0.2139-----------------------------------------|" - "L0.2153[1922,2086] 142ns |----------------------------------------L0.2153-----------------------------------------|" - "L0.2167[1922,2086] 143ns |----------------------------------------L0.2167-----------------------------------------|" - - "L0.2181[1922,2086] 144ns |----------------------------------------L0.2181-----------------------------------------|" + - "L0.2209[1922,2086] 144ns |----------------------------------------L0.2209-----------------------------------------|" - "L0.2223[1922,2086] 145ns |----------------------------------------L0.2223-----------------------------------------|" - "L0.2237[1922,2086] 146ns |----------------------------------------L0.2237-----------------------------------------|" - "L0.2251[1922,2086] 147ns |----------------------------------------L0.2251-----------------------------------------|" @@ -9013,7 +9013,7 @@ async fn stuck_l0_large_l0s() { - "L0, all files 0b " - "L0.?[1922,2086] 159ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.2125, L0.2139, L0.2153, L0.2167, L0.2181, L0.2223, L0.2237, L0.2251, L0.2265, L0.2279, L0.2293, L0.2307, L0.2321, L0.2335, L0.2349, L0.2363, L0.2377, L0.2391, L0.2405, L0.2419" + - " Soft Deleting 20 files: L0.2125, L0.2139, L0.2153, L0.2167, L0.2209, L0.2223, L0.2237, L0.2251, L0.2265, L0.2279, L0.2293, L0.2307, L0.2321, L0.2335, L0.2349, L0.2363, L0.2377, L0.2391, L0.2405, L0.2419" - " Creating 1 files" - "**** Simulation run 327, type=compact(ManySmallFiles). 20 Input Files, 0b total:" - "L0, all files 0b " @@ -9073,15 +9073,15 @@ async fn stuck_l0_large_l0s() { - " Creating 1 files" - "**** Simulation run 329, type=compact(ManySmallFiles). 20 Input Files, 200b total:" - "L0, all files 10b " - - "L0.586[2087,200000] 20ns |------------------L0.586-------------------| " - - "L0.461[2087,210000] 21ns |--------------------L0.461--------------------| " - - "L0.475[2087,220000] 22ns |---------------------L0.475---------------------| " - - "L0.489[2087,230000] 23ns |----------------------L0.489----------------------| " - - "L0.503[2087,240000] 24ns |-----------------------L0.503------------------------| " - - "L0.517[2087,250000] 25ns |------------------------L0.517-------------------------| " - - "L0.531[2087,260000] 26ns |-------------------------L0.531--------------------------| " - - "L0.545[2087,270000] 27ns |---------------------------L0.545---------------------------| " - - "L0.559[2087,280000] 28ns |----------------------------L0.559----------------------------| " + - "L0.448[2087,200000] 20ns |------------------L0.448-------------------| " + - "L0.462[2087,210000] 21ns |--------------------L0.462--------------------| " + - "L0.476[2087,220000] 22ns |---------------------L0.476---------------------| " + - "L0.490[2087,230000] 23ns |----------------------L0.490----------------------| " + - "L0.504[2087,240000] 24ns |-----------------------L0.504------------------------| " + - "L0.518[2087,250000] 25ns |------------------------L0.518-------------------------| " + - "L0.532[2087,260000] 26ns |-------------------------L0.532--------------------------| " + - "L0.546[2087,270000] 27ns |---------------------------L0.546---------------------------| " + - "L0.586[2087,280000] 28ns |----------------------------L0.586----------------------------| " - "L0.600[2087,290000] 29ns |-----------------------------L0.600-----------------------------| " - "L0.614[2087,300000] 30ns |------------------------------L0.614-------------------------------| " - "L0.628[2087,310000] 31ns |-------------------------------L0.628--------------------------------| " @@ -9097,7 +9097,7 @@ async fn stuck_l0_large_l0s() { - "L0, all files 200b " - "L0.?[2087,390000] 39ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.461, L0.475, L0.489, L0.503, L0.517, L0.531, L0.545, L0.559, L0.586, L0.600, L0.614, L0.628, L0.642, L0.656, L0.670, L0.684, L0.698, L0.712, L0.726, L0.740" + - " Soft Deleting 20 files: L0.448, L0.462, L0.476, L0.490, L0.504, L0.518, L0.532, L0.546, L0.586, L0.600, L0.614, L0.628, L0.642, L0.656, L0.670, L0.684, L0.698, L0.712, L0.726, L0.740" - " Creating 1 files" - "**** Simulation run 330, type=compact(ManySmallFiles). 20 Input Files, 200b total:" - "L0, all files 10b " @@ -9227,9 +9227,9 @@ async fn stuck_l0_large_l0s() { - "L0.2028[2087,1310000] 131ns|-------------------------------------L0.2028--------------------------------------| " - "L0.2042[2087,1320000] 132ns|--------------------------------------L0.2042--------------------------------------| " - "L0.2056[2087,1330000] 133ns|--------------------------------------L0.2056---------------------------------------| " - - "L0.2070[2087,1340000] 134ns|--------------------------------------L0.2070---------------------------------------| " + - "L0.2182[2087,1340000] 134ns|--------------------------------------L0.2182---------------------------------------| " - "L0.2196[2087,1350000] 135ns|---------------------------------------L0.2196---------------------------------------| " - - "L0.2210[2087,1360000] 136ns|---------------------------------------L0.2210----------------------------------------| " + - "L0.2070[2087,1360000] 136ns|---------------------------------------L0.2070----------------------------------------| " - "L0.2084[2087,1370000] 137ns|---------------------------------------L0.2084----------------------------------------| " - "L0.2098[2087,1380000] 138ns|----------------------------------------L0.2098----------------------------------------| " - "L0.2112[2087,1390000] 139ns|----------------------------------------L0.2112-----------------------------------------|" @@ -9237,7 +9237,7 @@ async fn stuck_l0_large_l0s() { - "L0, all files 200b " - "L0.?[2087,1390000] 139ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.1874, L0.1888, L0.1902, L0.1916, L0.1930, L0.1944, L0.1958, L0.1972, L0.1986, L0.2000, L0.2014, L0.2028, L0.2042, L0.2056, L0.2070, L0.2084, L0.2098, L0.2112, L0.2196, L0.2210" + - " Soft Deleting 20 files: L0.1874, L0.1888, L0.1902, L0.1916, L0.1930, L0.1944, L0.1958, L0.1972, L0.1986, L0.2000, L0.2014, L0.2028, L0.2042, L0.2056, L0.2070, L0.2084, L0.2098, L0.2112, L0.2182, L0.2196" - " Creating 1 files" - "**** Simulation run 335, type=compact(ManySmallFiles). 20 Input Files, 200b total:" - "L0, all files 10b " @@ -9245,7 +9245,7 @@ async fn stuck_l0_large_l0s() { - "L0.2140[2087,1410000] 141ns|-----------------------------------L0.2140-----------------------------------| " - "L0.2154[2087,1420000] 142ns|-----------------------------------L0.2154------------------------------------| " - "L0.2168[2087,1430000] 143ns|-----------------------------------L0.2168------------------------------------| " - - "L0.2182[2087,1440000] 144ns|------------------------------------L0.2182------------------------------------| " + - "L0.2210[2087,1440000] 144ns|------------------------------------L0.2210------------------------------------| " - "L0.2224[2087,1450000] 145ns|------------------------------------L0.2224-------------------------------------| " - "L0.2238[2087,1460000] 146ns|------------------------------------L0.2238-------------------------------------| " - "L0.2252[2087,1470000] 147ns|-------------------------------------L0.2252-------------------------------------| " @@ -9265,7 +9265,7 @@ async fn stuck_l0_large_l0s() { - "L0, all files 200b " - "L0.?[2087,1590000] 159ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 20 files: L0.2126, L0.2140, L0.2154, L0.2168, L0.2182, L0.2224, L0.2238, L0.2252, L0.2266, L0.2280, L0.2294, L0.2308, L0.2322, L0.2336, L0.2350, L0.2364, L0.2378, L0.2392, L0.2406, L0.2420" + - " Soft Deleting 20 files: L0.2126, L0.2140, L0.2154, L0.2168, L0.2210, L0.2224, L0.2238, L0.2252, L0.2266, L0.2280, L0.2294, L0.2308, L0.2322, L0.2336, L0.2350, L0.2364, L0.2378, L0.2392, L0.2406, L0.2420" - " Creating 1 files" - "**** Simulation run 336, type=compact(ManySmallFiles). 20 Input Files, 200b total:" - "L0, all files 10b " @@ -9333,10 +9333,44 @@ async fn stuck_l0_large_l0s() { - "Committing partition 1:" - " Soft Deleting 1 files: L0.2943" - " Creating 2 files" - - "**** Simulation run 339, type=compact(ManySmallFiles). 10 Input Files, 160mb total:" + - "**** Simulation run 339, type=compact(ManySmallFiles). 8 Input Files, 0b total:" + - "L0, all files 0b " + - "L0.2944[20,161] 39ns |----------------------------------------L0.2944-----------------------------------------|" + - "L0.2945[40,161] 59ns |----------------------------------L0.2945----------------------------------| " + - "L0.2946[60,161] 79ns |---------------------------L0.2946----------------------------| " + - "L0.2947[80,161] 99ns |---------------------L0.2947---------------------| " + - "L0.2948[100,161] 119ns |--------------L0.2948---------------| " + - "L0.2949[120,161] 139ns |--------L0.2949---------| " + - "L0.2950[140,161] 159ns |--L0.2950--| " + - "L0.2951[160,161] 161ns |L0.2951|" + - "**** 1 Output Files (parquet_file_id not yet assigned), 0b total:" + - "L0, all files 0b " + - "L0.?[20,161] 161ns |------------------------------------------L0.?------------------------------------------|" + - "Committing partition 1:" + - " Soft Deleting 8 files: L0.2944, L0.2945, L0.2946, L0.2947, L0.2948, L0.2949, L0.2950, L0.2951" + - " Creating 1 files" + - "**** Simulation run 340, type=compact(ManySmallFiles). 10 Input Files, 160mb total:" - "L0 " - - "L0.2960[322,481] 19ns 160mb|----------------------------------------L0.2960-----------------------------------------|" - - "L0.2961[322,481] 39ns 0b |----------------------------------------L0.2961-----------------------------------------|" + - "L0.2952[162,321] 19ns 160mb|----------------------------------------L0.2952-----------------------------------------|" + - "L0.2953[162,321] 39ns 0b |----------------------------------------L0.2953-----------------------------------------|" + - "L0.2962[162,321] 59ns 0b |----------------------------------------L0.2962-----------------------------------------|" + - "L0.2954[162,321] 79ns 0b |----------------------------------------L0.2954-----------------------------------------|" + - "L0.2955[162,321] 99ns 0b |----------------------------------------L0.2955-----------------------------------------|" + - "L0.2956[162,321] 119ns 0b|----------------------------------------L0.2956-----------------------------------------|" + - "L0.2957[162,321] 139ns 0b|----------------------------------------L0.2957-----------------------------------------|" + - "L0.2958[162,321] 159ns 0b|----------------------------------------L0.2958-----------------------------------------|" + - "L0.2959[162,321] 179ns 0b|----------------------------------------L0.2959-----------------------------------------|" + - "L0.2960[180,321] 199ns 0b |-----------------------------------L0.2960-----------------------------------| " + - "**** 1 Output Files (parquet_file_id not yet assigned), 160mb total:" + - "L0, all files 160mb " + - "L0.?[162,321] 199ns |------------------------------------------L0.?------------------------------------------|" + - "Committing partition 1:" + - " Soft Deleting 10 files: L0.2952, L0.2953, L0.2954, L0.2955, L0.2956, L0.2957, L0.2958, L0.2959, L0.2960, L0.2962" + - " Creating 1 files" + - "**** Simulation run 341, type=compact(ManySmallFiles). 10 Input Files, 160mb total:" + - "L0 " + - "L0.2961[322,481] 19ns 160mb|----------------------------------------L0.2961-----------------------------------------|" + - "L0.2963[322,481] 39ns 0b |----------------------------------------L0.2963-----------------------------------------|" - "L0.2964[322,481] 59ns 0b |----------------------------------------L0.2964-----------------------------------------|" - "L0.2965[322,481] 79ns 0b |----------------------------------------L0.2965-----------------------------------------|" - "L0.2966[322,481] 99ns 0b |----------------------------------------L0.2966-----------------------------------------|" @@ -9349,9 +9383,9 @@ async fn stuck_l0_large_l0s() { - "L0, all files 160mb " - "L0.?[322,481] 199ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 10 files: L0.2960, L0.2961, L0.2964, L0.2965, L0.2966, L0.2967, L0.2968, L0.2969, L0.2970, L0.2971" + - " Soft Deleting 10 files: L0.2961, L0.2963, L0.2964, L0.2965, L0.2966, L0.2967, L0.2968, L0.2969, L0.2970, L0.2971" - " Creating 1 files" - - "**** Simulation run 340, type=compact(ManySmallFiles). 10 Input Files, 160mb total:" + - "**** Simulation run 342, type=compact(ManySmallFiles). 10 Input Files, 160mb total:" - "L0 " - "L0.2972[482,641] 19ns 160mb|----------------------------------------L0.2972-----------------------------------------|" - "L0.2973[482,641] 39ns 0b |----------------------------------------L0.2973-----------------------------------------|" @@ -9369,7 +9403,7 @@ async fn stuck_l0_large_l0s() { - "Committing partition 1:" - " Soft Deleting 10 files: L0.2972, L0.2973, L0.2974, L0.2975, L0.2976, L0.2977, L0.2978, L0.2979, L0.2987, L0.2988" - " Creating 1 files" - - "**** Simulation run 341, type=compact(ManySmallFiles). 10 Input Files, 160mb total:" + - "**** Simulation run 343, type=compact(ManySmallFiles). 10 Input Files, 160mb total:" - "L0 " - "L0.2980[642,801] 19ns 160mb|----------------------------------------L0.2980-----------------------------------------|" - "L0.2981[642,801] 39ns 0b |----------------------------------------L0.2981-----------------------------------------|" @@ -9387,7 +9421,7 @@ async fn stuck_l0_large_l0s() { - "Committing partition 1:" - " Soft Deleting 10 files: L0.2980, L0.2981, L0.2982, L0.2983, L0.2984, L0.2985, L0.2986, L0.2989, L0.2990, L0.2991" - " Creating 1 files" - - "**** Simulation run 342, type=compact(ManySmallFiles). 10 Input Files, 160mb total:" + - "**** Simulation run 344, type=compact(ManySmallFiles). 10 Input Files, 160mb total:" - "L0 " - "L0.2992[802,961] 19ns 160mb|----------------------------------------L0.2992-----------------------------------------|" - "L0.2993[802,961] 39ns 0b |----------------------------------------L0.2993-----------------------------------------|" @@ -9405,7 +9439,7 @@ async fn stuck_l0_large_l0s() { - "Committing partition 1:" - " Soft Deleting 10 files: L0.2992, L0.2993, L0.2994, L0.2995, L0.2996, L0.2997, L0.2998, L0.2999, L0.3000, L0.3001" - " Creating 1 files" - - "**** Simulation run 343, type=compact(ManySmallFiles). 10 Input Files, 160mb total:" + - "**** Simulation run 345, type=compact(ManySmallFiles). 10 Input Files, 160mb total:" - "L0 " - "L0.3002[962,1121] 19ns 160mb|----------------------------------------L0.3002-----------------------------------------|" - "L0.3003[962,1121] 39ns 0b|----------------------------------------L0.3003-----------------------------------------|" @@ -9423,7 +9457,7 @@ async fn stuck_l0_large_l0s() { - "Committing partition 1:" - " Soft Deleting 10 files: L0.3002, L0.3003, L0.3004, L0.3005, L0.3006, L0.3007, L0.3008, L0.3009, L0.3013, L0.3014" - " Creating 1 files" - - "**** Simulation run 344, type=compact(ManySmallFiles). 10 Input Files, 160mb total:" + - "**** Simulation run 346, type=compact(ManySmallFiles). 10 Input Files, 160mb total:" - "L0 " - "L0.3010[1122,1281] 19ns 160mb|----------------------------------------L0.3010-----------------------------------------|" - "L0.3011[1122,1281] 39ns 0b|----------------------------------------L0.3011-----------------------------------------|" @@ -9441,7 +9475,7 @@ async fn stuck_l0_large_l0s() { - "Committing partition 1:" - " Soft Deleting 10 files: L0.3010, L0.3011, L0.3012, L0.3015, L0.3016, L0.3017, L0.3018, L0.3019, L0.3020, L0.3021" - " Creating 1 files" - - "**** Simulation run 345, type=compact(ManySmallFiles). 10 Input Files, 160mb total:" + - "**** Simulation run 347, type=compact(ManySmallFiles). 10 Input Files, 160mb total:" - "L0 " - "L0.3022[1282,1441] 19ns 160mb|----------------------------------------L0.3022-----------------------------------------|" - "L0.3023[1282,1441] 39ns 0b|----------------------------------------L0.3023-----------------------------------------|" @@ -9451,65 +9485,31 @@ async fn stuck_l0_large_l0s() { - "L0.3027[1282,1441] 119ns 0b|----------------------------------------L0.3027-----------------------------------------|" - "L0.3028[1282,1441] 139ns 0b|----------------------------------------L0.3028-----------------------------------------|" - "L0.3029[1282,1441] 159ns 0b|----------------------------------------L0.3029-----------------------------------------|" - - "L0.3030[1282,1441] 179ns 0b|----------------------------------------L0.3030-----------------------------------------|" + - "L0.3038[1282,1441] 179ns 0b|----------------------------------------L0.3038-----------------------------------------|" - "L0.3039[1282,1441] 199ns 0b|----------------------------------------L0.3039-----------------------------------------|" - "**** 1 Output Files (parquet_file_id not yet assigned), 160mb total:" - "L0, all files 160mb " - "L0.?[1282,1441] 199ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 10 files: L0.3022, L0.3023, L0.3024, L0.3025, L0.3026, L0.3027, L0.3028, L0.3029, L0.3030, L0.3039" + - " Soft Deleting 10 files: L0.3022, L0.3023, L0.3024, L0.3025, L0.3026, L0.3027, L0.3028, L0.3029, L0.3038, L0.3039" - " Creating 1 files" - - "**** Simulation run 346, type=compact(ManySmallFiles). 10 Input Files, 160mb total:" + - "**** Simulation run 348, type=compact(ManySmallFiles). 10 Input Files, 160mb total:" - "L0 " - - "L0.3031[1442,1601] 19ns 160mb|----------------------------------------L0.3031-----------------------------------------|" - - "L0.3032[1442,1601] 39ns 0b|----------------------------------------L0.3032-----------------------------------------|" - - "L0.3033[1442,1601] 59ns 0b|----------------------------------------L0.3033-----------------------------------------|" - - "L0.3034[1442,1601] 79ns 0b|----------------------------------------L0.3034-----------------------------------------|" - - "L0.3035[1442,1601] 99ns 0b|----------------------------------------L0.3035-----------------------------------------|" - - "L0.3036[1442,1601] 119ns 0b|----------------------------------------L0.3036-----------------------------------------|" - - "L0.3037[1442,1601] 139ns 0b|----------------------------------------L0.3037-----------------------------------------|" - - "L0.3038[1442,1601] 159ns 0b|----------------------------------------L0.3038-----------------------------------------|" + - "L0.3030[1442,1601] 19ns 160mb|----------------------------------------L0.3030-----------------------------------------|" + - "L0.3031[1442,1601] 39ns 0b|----------------------------------------L0.3031-----------------------------------------|" + - "L0.3032[1442,1601] 59ns 0b|----------------------------------------L0.3032-----------------------------------------|" + - "L0.3033[1442,1601] 79ns 0b|----------------------------------------L0.3033-----------------------------------------|" + - "L0.3034[1442,1601] 99ns 0b|----------------------------------------L0.3034-----------------------------------------|" + - "L0.3035[1442,1601] 119ns 0b|----------------------------------------L0.3035-----------------------------------------|" + - "L0.3036[1442,1601] 139ns 0b|----------------------------------------L0.3036-----------------------------------------|" + - "L0.3037[1442,1601] 159ns 0b|----------------------------------------L0.3037-----------------------------------------|" - "L0.3040[1442,1601] 179ns 0b|----------------------------------------L0.3040-----------------------------------------|" - "L0.3041[1442,1601] 199ns 0b|----------------------------------------L0.3041-----------------------------------------|" - "**** 1 Output Files (parquet_file_id not yet assigned), 160mb total:" - "L0, all files 160mb " - "L0.?[1442,1601] 199ns |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 10 files: L0.3031, L0.3032, L0.3033, L0.3034, L0.3035, L0.3036, L0.3037, L0.3038, L0.3040, L0.3041" - - " Creating 1 files" - - "**** Simulation run 347, type=compact(ManySmallFiles). 8 Input Files, 0b total:" - - "L0, all files 0b " - - "L0.2944[20,161] 39ns |----------------------------------------L0.2944-----------------------------------------|" - - "L0.2945[40,161] 59ns |----------------------------------L0.2945----------------------------------| " - - "L0.2946[60,161] 79ns |---------------------------L0.2946----------------------------| " - - "L0.2947[80,161] 99ns |---------------------L0.2947---------------------| " - - "L0.2948[100,161] 119ns |--------------L0.2948---------------| " - - "L0.2949[120,161] 139ns |--------L0.2949---------| " - - "L0.2950[140,161] 159ns |--L0.2950--| " - - "L0.2951[160,161] 161ns |L0.2951|" - - "**** 1 Output Files (parquet_file_id not yet assigned), 0b total:" - - "L0, all files 0b " - - "L0.?[20,161] 161ns |------------------------------------------L0.?------------------------------------------|" - - "Committing partition 1:" - - " Soft Deleting 8 files: L0.2944, L0.2945, L0.2946, L0.2947, L0.2948, L0.2949, L0.2950, L0.2951" - - " Creating 1 files" - - "**** Simulation run 348, type=compact(ManySmallFiles). 10 Input Files, 160mb total:" - - "L0 " - - "L0.2952[162,321] 19ns 160mb|----------------------------------------L0.2952-----------------------------------------|" - - "L0.2953[162,321] 39ns 0b |----------------------------------------L0.2953-----------------------------------------|" - - "L0.2962[162,321] 59ns 0b |----------------------------------------L0.2962-----------------------------------------|" - - "L0.2963[162,321] 79ns 0b |----------------------------------------L0.2963-----------------------------------------|" - - "L0.2954[162,321] 99ns 0b |----------------------------------------L0.2954-----------------------------------------|" - - "L0.2955[162,321] 119ns 0b|----------------------------------------L0.2955-----------------------------------------|" - - "L0.2956[162,321] 139ns 0b|----------------------------------------L0.2956-----------------------------------------|" - - "L0.2957[162,321] 159ns 0b|----------------------------------------L0.2957-----------------------------------------|" - - "L0.2958[162,321] 179ns 0b|----------------------------------------L0.2958-----------------------------------------|" - - "L0.2959[180,321] 199ns 0b |-----------------------------------L0.2959-----------------------------------| " - - "**** 1 Output Files (parquet_file_id not yet assigned), 160mb total:" - - "L0, all files 160mb " - - "L0.?[162,321] 199ns |------------------------------------------L0.?------------------------------------------|" - - "Committing partition 1:" - - " Soft Deleting 10 files: L0.2952, L0.2953, L0.2954, L0.2955, L0.2956, L0.2957, L0.2958, L0.2959, L0.2962, L0.2963" + - " Soft Deleting 10 files: L0.3030, L0.3031, L0.3032, L0.3033, L0.3034, L0.3035, L0.3036, L0.3037, L0.3040, L0.3041" - " Creating 1 files" - "**** Simulation run 349, type=compact(ManySmallFiles). 10 Input Files, 160mb total:" - "L0 " @@ -9584,13 +9584,13 @@ async fn stuck_l0_large_l0s() { - " Creating 1 files" - "**** Simulation run 353, type=split(ReduceOverlap)(split_times=[101]). 1 Input Files, 0b total:" - "L0, all files 0b " - - "L0.3091[20,161] 161ns |----------------------------------------L0.3091-----------------------------------------|" + - "L0.3083[20,161] 161ns |----------------------------------------L0.3083-----------------------------------------|" - "**** 2 Output Files (parquet_file_id not yet assigned), 0b total:" - "L0, all files 0b " - "L0.?[20,101] 161ns |----------------------L0.?-----------------------| " - "L0.?[102,161] 161ns |---------------L0.?----------------| " - "Committing partition 1:" - - " Soft Deleting 1 files: L0.3091" + - " Soft Deleting 1 files: L0.3083" - " Creating 2 files" - "**** Simulation run 354, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[101]). 4 Input Files, 161mb total:" - "L0 " @@ -9616,19 +9616,19 @@ async fn stuck_l0_large_l0s() { - "L1.?[1592385,1990000] 199ns 16mb |-----L1.?------| " - "Committing partition 1:" - " Soft Deleting 2 files: L0.3095, L0.3096" - - " Upgrading 11 files level to CompactionLevel::L1: L0.3083, L0.3084, L0.3085, L0.3086, L0.3087, L0.3088, L0.3089, L0.3090, L0.3092, L0.3093, L0.3094" + - " Upgrading 11 files level to CompactionLevel::L1: L0.3084, L0.3085, L0.3086, L0.3087, L0.3088, L0.3089, L0.3090, L0.3091, L0.3092, L0.3093, L0.3094" - " Creating 2 files" - "**** Simulation run 356, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[202, 302]). 2 Input Files, 220mb total:" - "L1 " - "L1.3100[102,161] 161ns 60mb|-------L1.3100--------| " - - "L1.3092[162,321] 199ns 160mb |----------------------------L1.3092----------------------------| " + - "L1.3084[162,321] 199ns 160mb |----------------------------L1.3084----------------------------| " - "**** 3 Output Files (parquet_file_id not yet assigned), 220mb total:" - "L2 " - "L2.?[102,202] 199ns 101mb|-----------------L2.?------------------| " - "L2.?[203,302] 199ns 100mb |-----------------L2.?-----------------| " - "L2.?[303,321] 199ns 19mb |L2.?-| " - "Committing partition 1:" - - " Soft Deleting 2 files: L1.3092, L1.3100" + - " Soft Deleting 2 files: L1.3084, L1.3100" - " Upgrading 1 files level to CompactionLevel::L2: L1.3099" - " Creating 3 files" - "**** Simulation run 357, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[1592384]). 2 Input Files, 79mb total:" @@ -9641,18 +9641,18 @@ async fn stuck_l0_large_l0s() { - "L2.?[1592385,1990000] 199ns 16mb |-----L2.?------| " - "Committing partition 1:" - " Soft Deleting 2 files: L1.3101, L1.3102" - - " Upgrading 10 files level to CompactionLevel::L2: L1.3083, L1.3084, L1.3085, L1.3086, L1.3087, L1.3088, L1.3089, L1.3090, L1.3093, L1.3094" + - " Upgrading 10 files level to CompactionLevel::L2: L1.3085, L1.3086, L1.3087, L1.3088, L1.3089, L1.3090, L1.3091, L1.3092, L1.3093, L1.3094" - " Creating 2 files" - "**** Final Output Files (6.39gb written)" - "L2 " - - "L2.3083[322,481] 199ns 160mb|L2.3083| " - - "L2.3084[482,641] 199ns 160mb|L2.3084| " - - "L2.3085[642,801] 199ns 160mb|L2.3085| " - - "L2.3086[802,961] 199ns 160mb|L2.3086| " - - "L2.3087[962,1121] 199ns 160mb|L2.3087| " - - "L2.3088[1122,1281] 199ns 160mb|L2.3088| " - - "L2.3089[1282,1441] 199ns 160mb|L2.3089| " - - "L2.3090[1442,1601] 199ns 160mb|L2.3090| " + - "L2.3085[322,481] 199ns 160mb|L2.3085| " + - "L2.3086[482,641] 199ns 160mb|L2.3086| " + - "L2.3087[642,801] 199ns 160mb|L2.3087| " + - "L2.3088[802,961] 199ns 160mb|L2.3088| " + - "L2.3089[962,1121] 199ns 160mb|L2.3089| " + - "L2.3090[1122,1281] 199ns 160mb|L2.3090| " + - "L2.3091[1282,1441] 199ns 160mb|L2.3091| " + - "L2.3092[1442,1601] 199ns 160mb|L2.3092| " - "L2.3093[1602,1761] 199ns 160mb|L2.3093| " - "L2.3094[1762,1921] 199ns 160mb|L2.3094| " - "L2.3099[1,101] 161ns 101mb|L2.3099| " @@ -9667,14 +9667,14 @@ async fn stuck_l0_large_l0s() { - 158mb written by split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize)) - 3.75gb written by compact(ManySmallFiles) - 542mb written by split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize)) - - "WARNING: file L2.3083[322,481] 199ns 160mb exceeds soft limit 100mb by more than 50%" - - "WARNING: file L2.3084[482,641] 199ns 160mb exceeds soft limit 100mb by more than 50%" - - "WARNING: file L2.3085[642,801] 199ns 160mb exceeds soft limit 100mb by more than 50%" - - "WARNING: file L2.3086[802,961] 199ns 160mb exceeds soft limit 100mb by more than 50%" - - "WARNING: file L2.3087[962,1121] 199ns 160mb exceeds soft limit 100mb by more than 50%" - - "WARNING: file L2.3088[1122,1281] 199ns 160mb exceeds soft limit 100mb by more than 50%" - - "WARNING: file L2.3089[1282,1441] 199ns 160mb exceeds soft limit 100mb by more than 50%" - - "WARNING: file L2.3090[1442,1601] 199ns 160mb exceeds soft limit 100mb by more than 50%" + - "WARNING: file L2.3085[322,481] 199ns 160mb exceeds soft limit 100mb by more than 50%" + - "WARNING: file L2.3086[482,641] 199ns 160mb exceeds soft limit 100mb by more than 50%" + - "WARNING: file L2.3087[642,801] 199ns 160mb exceeds soft limit 100mb by more than 50%" + - "WARNING: file L2.3088[802,961] 199ns 160mb exceeds soft limit 100mb by more than 50%" + - "WARNING: file L2.3089[962,1121] 199ns 160mb exceeds soft limit 100mb by more than 50%" + - "WARNING: file L2.3090[1122,1281] 199ns 160mb exceeds soft limit 100mb by more than 50%" + - "WARNING: file L2.3091[1282,1441] 199ns 160mb exceeds soft limit 100mb by more than 50%" + - "WARNING: file L2.3092[1442,1601] 199ns 160mb exceeds soft limit 100mb by more than 50%" - "WARNING: file L2.3093[1602,1761] 199ns 160mb exceeds soft limit 100mb by more than 50%" - "WARNING: file L2.3094[1762,1921] 199ns 160mb exceeds soft limit 100mb by more than 50%" "### diff --git a/data_types/src/partition.rs b/data_types/src/partition.rs index f1e383aaba..a1b5d245fd 100644 --- a/data_types/src/partition.rs +++ b/data_types/src/partition.rs @@ -527,6 +527,21 @@ impl Partition { pub fn sort_key_ids(&self) -> Option<&SortedColumnSet> { self.sort_key_ids.as_ref() } + + // todo: resue the same function in https://github.com/influxdata/influxdb_iox/pull/8556/ + /// The sort_key_ids if present and not empty + pub fn sort_key_ids_none_if_empty(&self) -> Option { + match self.sort_key_ids.as_ref() { + None => None, + Some(sort_key_ids) => { + if sort_key_ids.is_empty() { + None + } else { + Some(sort_key_ids.clone()) + } + } + } + } } #[cfg(test)] diff --git a/iox_tests/src/builders.rs b/iox_tests/src/builders.rs index f0ba1621ed..0293e89909 100644 --- a/iox_tests/src/builders.rs +++ b/iox_tests/src/builders.rs @@ -1,6 +1,7 @@ use data_types::{ - ColumnSet, CompactionLevel, NamespaceId, ParquetFile, ParquetFileId, Partition, PartitionId, - PartitionKey, SkippedCompaction, Table, TableId, Timestamp, TransitionPartitionId, + Column, ColumnId, ColumnSet, ColumnType, CompactionLevel, NamespaceId, ParquetFile, + ParquetFileId, Partition, PartitionId, PartitionKey, SkippedCompaction, Table, TableId, + Timestamp, TransitionPartitionId, }; use uuid::Uuid; @@ -111,6 +112,51 @@ impl From for ParquetFileBuilder { } } +#[derive(Debug)] +/// Build [`Column`]s for testing +pub struct ColumnBuilder { + column: Column, +} + +impl ColumnBuilder { + /// Create a builder to create a column with `table_id` `id` + pub fn new(id: i64, table_id: i64) -> Self { + Self { + column: Column { + id: ColumnId::new(id), + table_id: TableId::new(table_id), + name: "column".to_string(), + column_type: ColumnType::Tag, + }, + } + } + + /// Set the column name + pub fn with_name(self, name: &str) -> Self { + Self { + column: Column { + name: name.to_string(), + ..self.column + }, + } + } + + /// Set column type + pub fn with_column_type(self, column_type: ColumnType) -> Self { + Self { + column: Column { + column_type, + ..self.column + }, + } + } + + /// Create the table + pub fn build(self) -> Column { + self.column + } +} + #[derive(Debug)] /// Build [`Table`]s for testing pub struct TableBuilder { diff --git a/iox_tests/src/lib.rs b/iox_tests/src/lib.rs index 0da20014c9..98f65aefa1 100644 --- a/iox_tests/src/lib.rs +++ b/iox_tests/src/lib.rs @@ -25,7 +25,9 @@ pub use catalog::{ }; mod builders; -pub use builders::{ParquetFileBuilder, PartitionBuilder, SkippedCompactionBuilder, TableBuilder}; +pub use builders::{ + ColumnBuilder, ParquetFileBuilder, PartitionBuilder, SkippedCompactionBuilder, TableBuilder, +}; /// Create a partition identifier from an int (which gets used as the table ID) and a partition key /// with the string "arbitrary". Most useful in cases where there isn't any actual catalog From e4505912a1f36e6a8472ab970b0ab425297571f5 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Thu, 24 Aug 2023 14:31:33 -0400 Subject: [PATCH 3/5] chore: Update DataFusion pin (#8544) * chore: Update DataFusion pin * refactor: Use upstream check --------- Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com> --- Cargo.lock | 16 +++++++------- Cargo.toml | 4 ++-- iox_query/src/exec/context.rs | 39 ++++++----------------------------- workspace-hack/Cargo.toml | 6 +++--- 4 files changed, 19 insertions(+), 46 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8cafc86e31..2b31e2a0a2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1390,7 +1390,7 @@ dependencies = [ [[package]] name = "datafusion" version = "29.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=f2c0100a5a10bf3ea166a1a590d94b8c9b6cf673#f2c0100a5a10bf3ea166a1a590d94b8c9b6cf673" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=f3722c0af8418bcb19cf9dc5f7e458a3aa5f0f32#f3722c0af8418bcb19cf9dc5f7e458a3aa5f0f32" dependencies = [ "ahash", "arrow", @@ -1438,7 +1438,7 @@ dependencies = [ [[package]] name = "datafusion-common" version = "29.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=f2c0100a5a10bf3ea166a1a590d94b8c9b6cf673#f2c0100a5a10bf3ea166a1a590d94b8c9b6cf673" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=f3722c0af8418bcb19cf9dc5f7e458a3aa5f0f32#f3722c0af8418bcb19cf9dc5f7e458a3aa5f0f32" dependencies = [ "arrow", "arrow-array", @@ -1452,7 +1452,7 @@ dependencies = [ [[package]] name = "datafusion-execution" version = "29.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=f2c0100a5a10bf3ea166a1a590d94b8c9b6cf673#f2c0100a5a10bf3ea166a1a590d94b8c9b6cf673" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=f3722c0af8418bcb19cf9dc5f7e458a3aa5f0f32#f3722c0af8418bcb19cf9dc5f7e458a3aa5f0f32" dependencies = [ "arrow", "dashmap", @@ -1471,7 +1471,7 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "29.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=f2c0100a5a10bf3ea166a1a590d94b8c9b6cf673#f2c0100a5a10bf3ea166a1a590d94b8c9b6cf673" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=f3722c0af8418bcb19cf9dc5f7e458a3aa5f0f32#f3722c0af8418bcb19cf9dc5f7e458a3aa5f0f32" dependencies = [ "ahash", "arrow", @@ -1485,7 +1485,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "29.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=f2c0100a5a10bf3ea166a1a590d94b8c9b6cf673#f2c0100a5a10bf3ea166a1a590d94b8c9b6cf673" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=f3722c0af8418bcb19cf9dc5f7e458a3aa5f0f32#f3722c0af8418bcb19cf9dc5f7e458a3aa5f0f32" dependencies = [ "arrow", "async-trait", @@ -1502,7 +1502,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "29.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=f2c0100a5a10bf3ea166a1a590d94b8c9b6cf673#f2c0100a5a10bf3ea166a1a590d94b8c9b6cf673" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=f3722c0af8418bcb19cf9dc5f7e458a3aa5f0f32#f3722c0af8418bcb19cf9dc5f7e458a3aa5f0f32" dependencies = [ "ahash", "arrow", @@ -1536,7 +1536,7 @@ dependencies = [ [[package]] name = "datafusion-proto" version = "29.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=f2c0100a5a10bf3ea166a1a590d94b8c9b6cf673#f2c0100a5a10bf3ea166a1a590d94b8c9b6cf673" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=f3722c0af8418bcb19cf9dc5f7e458a3aa5f0f32#f3722c0af8418bcb19cf9dc5f7e458a3aa5f0f32" dependencies = [ "arrow", "chrono", @@ -1550,7 +1550,7 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "29.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=f2c0100a5a10bf3ea166a1a590d94b8c9b6cf673#f2c0100a5a10bf3ea166a1a590d94b8c9b6cf673" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=f3722c0af8418bcb19cf9dc5f7e458a3aa5f0f32#f3722c0af8418bcb19cf9dc5f7e458a3aa5f0f32" dependencies = [ "arrow", "arrow-schema", diff --git a/Cargo.toml b/Cargo.toml index bf14b407fb..aa91492ffe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -122,8 +122,8 @@ license = "MIT OR Apache-2.0" [workspace.dependencies] arrow = { version = "45.0.0" } arrow-flight = { version = "45.0.0" } -datafusion = { git = "https://github.com/apache/arrow-datafusion.git", rev = "f2c0100a5a10bf3ea166a1a590d94b8c9b6cf673", default-features = false } -datafusion-proto = { git = "https://github.com/apache/arrow-datafusion.git", rev = "f2c0100a5a10bf3ea166a1a590d94b8c9b6cf673" } +datafusion = { git = "https://github.com/apache/arrow-datafusion.git", rev = "f3722c0af8418bcb19cf9dc5f7e458a3aa5f0f32", default-features = false } +datafusion-proto = { git = "https://github.com/apache/arrow-datafusion.git", rev = "f3722c0af8418bcb19cf9dc5f7e458a3aa5f0f32" } hashbrown = { version = "0.14.0" } object_store = { version = "0.6.0" } diff --git a/iox_query/src/exec/context.rs b/iox_query/src/exec/context.rs index e5b4be57f9..4060eb48c8 100644 --- a/iox_query/src/exec/context.rs +++ b/iox_query/src/exec/context.rs @@ -34,10 +34,6 @@ use arrow::record_batch::RecordBatch; use async_trait::async_trait; use datafusion::{ catalog::CatalogProvider, - common::{ - plan_err, - tree_node::{TreeNode, TreeNodeVisitor, VisitRecursion}, - }, execution::{ context::{QueryPlanner, SessionState, TaskContext}, memory_pool::MemoryPool, @@ -346,10 +342,13 @@ impl IOxSessionContext { pub async fn sql_to_logical_plan(&self, sql: &str) -> Result { let ctx = self.child_ctx("sql_to_logical_plan"); debug!(text=%sql, "planning SQL query"); - // NOTE can not use ctx.inner.sql() here as it also interprets DDL let plan = ctx.inner.state().create_logical_plan(sql).await?; - // TODO use better API: https://github.com/apache/arrow-datafusion/issues/7328 - verify_plan(&plan)?; + // ensure the plan does not contain unwanted statements + let verifier = SQLOptions::new() + .with_allow_ddl(false) // no CREATE ... + .with_allow_dml(false) // no INSERT or COPY + .with_allow_statements(false); // no SET VARIABLE, etc + verifier.verify_plan(&plan)?; Ok(plan) } @@ -696,32 +695,6 @@ impl IOxSessionContext { } } -/// Returns an error if this plan contains any unsupported statements: -/// -/// * DDL (`CREATE TABLE`) - creates state in a context that is dropped at the end of the request -/// * Statements (`SET VARIABLE`) - can cause denial of service by using more memory or cput -/// * DML (`INSERT`, `COPY`) - can write local files so is a security risk on servers -fn verify_plan(plan: &LogicalPlan) -> Result<()> { - plan.visit(&mut BadPlanVisitor {})?; - Ok(()) -} - -struct BadPlanVisitor {} - -impl TreeNodeVisitor for BadPlanVisitor { - type N = LogicalPlan; - - fn pre_visit(&mut self, node: &Self::N) -> Result { - match node { - LogicalPlan::Ddl(ddl) => plan_err!("DDL not supported: {}", ddl.name()), - LogicalPlan::Dml(dml) => plan_err!("DML not supported: {}", dml.op), - LogicalPlan::Copy(_) => plan_err!("DML not supported: COPY"), - LogicalPlan::Statement(stmt) => plan_err!("Statement not supported: {}", stmt.name()), - _ => Ok(VisitRecursion::Continue), - } - } -} - /// Extension trait to pull IOx spans out of DataFusion contexts. pub trait SessionContextIOxExt { /// Get child span of the current context. diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index fc362da05d..7cbb5bb485 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -28,9 +28,9 @@ bytes = { version = "1" } chrono = { version = "0.4", default-features = false, features = ["alloc", "clock", "serde"] } crossbeam-utils = { version = "0.8" } crypto-common = { version = "0.1", default-features = false, features = ["std"] } -datafusion = { git = "https://github.com/apache/arrow-datafusion.git", rev = "f2c0100a5a10bf3ea166a1a590d94b8c9b6cf673" } -datafusion-optimizer = { git = "https://github.com/apache/arrow-datafusion.git", rev = "f2c0100a5a10bf3ea166a1a590d94b8c9b6cf673", default-features = false, features = ["crypto_expressions", "regex_expressions", "unicode_expressions"] } -datafusion-physical-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "f2c0100a5a10bf3ea166a1a590d94b8c9b6cf673", default-features = false, features = ["crypto_expressions", "encoding_expressions", "regex_expressions", "unicode_expressions"] } +datafusion = { git = "https://github.com/apache/arrow-datafusion.git", rev = "f3722c0af8418bcb19cf9dc5f7e458a3aa5f0f32" } +datafusion-optimizer = { git = "https://github.com/apache/arrow-datafusion.git", rev = "f3722c0af8418bcb19cf9dc5f7e458a3aa5f0f32", default-features = false, features = ["crypto_expressions", "regex_expressions", "unicode_expressions"] } +datafusion-physical-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "f3722c0af8418bcb19cf9dc5f7e458a3aa5f0f32", default-features = false, features = ["crypto_expressions", "encoding_expressions", "regex_expressions", "unicode_expressions"] } digest = { version = "0.10", features = ["mac", "std"] } either = { version = "1", features = ["serde"] } fixedbitset = { version = "0.4" } From af8f98e35f8171ab718332aa04630fd772499851 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 25 Aug 2023 07:54:14 +0000 Subject: [PATCH 4/5] chore(deps): Bump clap from 4.3.24 to 4.4.0 (#8571) Bumps [clap](https://github.com/clap-rs/clap) from 4.3.24 to 4.4.0. - [Release notes](https://github.com/clap-rs/clap/releases) - [Changelog](https://github.com/clap-rs/clap/blob/master/CHANGELOG.md) - [Commits](https://github.com/clap-rs/clap/compare/v4.3.24...clap_complete-v4.4.0) --- updated-dependencies: - dependency-name: clap dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Cargo.lock | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2b31e2a0a2..6e4ef03f8d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -89,16 +89,15 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.3.2" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" +checksum = "b1f58811cfac344940f1a400b6e6231ce35171f614f26439e80f8c1465c5cc0c" dependencies = [ "anstyle", "anstyle-parse", "anstyle-query", "anstyle-wincon", "colorchoice", - "is-terminal", "utf8parse", ] @@ -128,9 +127,9 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "1.0.2" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c677ab05e09154296dd37acecd46420c17b9713e8366facafa8fc0885167cf4c" +checksum = "58f54d10c6dfa51283a066ceab3ec1ab78d13fae00aa49243a45e4571fb79dfd" dependencies = [ "anstyle", "windows-sys 0.48.0", @@ -869,9 +868,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.3.24" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb690e81c7840c0d7aade59f242ea3b41b9bc27bcd5997890e7702ae4b32e487" +checksum = "1d5f1946157a96594eb2d2c10eb7ad9a2b27518cb3000209dec700c35df9197d" dependencies = [ "clap_builder", "clap_derive", @@ -902,9 +901,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.3.24" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ed2e96bc16d8d740f6f48d663eddf4b8a0983e79210fd55479b7bcd0a69860e" +checksum = "78116e32a042dd73c2901f0dc30790d20ff3447f3e3472fad359e8c3d282bcd6" dependencies = [ "anstream", "anstyle", @@ -915,9 +914,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.3.12" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54a9bb5758fc5dfe728d1019941681eccaf0cf8a4189b692a0ee2f2ecf90a050" +checksum = "c9fd1a5729c4548118d7d70ff234a44868d00489a4b6597b0b020918a0e91a1a" dependencies = [ "heck", "proc-macro2", From 196c589ef64f73677eb3e89e60b219f862bde19a Mon Sep 17 00:00:00 2001 From: Marco Neumann Date: Fri, 25 Aug 2023 11:01:36 +0200 Subject: [PATCH 5/5] refactor: `is_terminal` no longer requires an external crate (#8572) * refactor: `is_terminal` no longer requires an external crate This is now part of the stdlib. * chore: Run cargo hakari tasks --------- Co-authored-by: CircleCI[bot] --- Cargo.lock | 2 -- trogging/Cargo.toml | 1 - trogging/src/lib.rs | 8 ++++---- workspace-hack/Cargo.toml | 6 ------ 4 files changed, 4 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6e4ef03f8d..1987b9fc0b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6366,7 +6366,6 @@ name = "trogging" version = "0.1.0" dependencies = [ "clap", - "is-terminal", "logfmt", "observability_deps", "regex", @@ -6961,7 +6960,6 @@ dependencies = [ "regex-syntax 0.7.4", "reqwest", "ring", - "rustix", "rustls", "scopeguard", "serde", diff --git a/trogging/Cargo.toml b/trogging/Cargo.toml index 4ace2280ae..2da6d5a363 100644 --- a/trogging/Cargo.toml +++ b/trogging/Cargo.toml @@ -8,7 +8,6 @@ license.workspace = true [dependencies] clap = { version = "4", features = ["derive", "env"], optional = true } -is-terminal = "0.4.9" logfmt = { path = "../logfmt" } observability_deps = { path = "../observability_deps" } thiserror = "1.0.47" diff --git a/trogging/src/lib.rs b/trogging/src/lib.rs index 01487abf93..47b4131a26 100644 --- a/trogging/src/lib.rs +++ b/trogging/src/lib.rs @@ -20,14 +20,14 @@ pub mod config; pub use config::*; -use is_terminal::IsTerminal; // Re-export tracing_subscriber pub use tracing_subscriber; use observability_deps::tracing::{self, Subscriber}; -use std::cmp::min; -use std::io; -use std::io::Write; +use std::{ + cmp::min, + io::{self, IsTerminal, Write}, +}; use thiserror::Error; use tracing_subscriber::{ fmt::{self, writer::BoxMakeWriter, MakeWriter}, diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index 7cbb5bb485..ef188a0998 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -166,39 +166,33 @@ uuid = { version = "1", features = ["v4"] } bitflags = { version = "2", default-features = false, features = ["std"] } nix = { version = "0.26" } once_cell = { version = "1", default-features = false, features = ["unstable"] } -rustix = { version = "0.38", features = ["fs", "termios"] } rustls = { version = "0.21" } [target.x86_64-unknown-linux-gnu.build-dependencies] bitflags = { version = "2", default-features = false, features = ["std"] } once_cell = { version = "1", default-features = false, features = ["unstable"] } -rustix = { version = "0.38", features = ["fs", "termios"] } rustls = { version = "0.21" } [target.x86_64-apple-darwin.dependencies] bitflags = { version = "2", default-features = false, features = ["std"] } nix = { version = "0.26" } once_cell = { version = "1", default-features = false, features = ["unstable"] } -rustix = { version = "0.38", features = ["fs", "termios"] } rustls = { version = "0.21" } [target.x86_64-apple-darwin.build-dependencies] bitflags = { version = "2", default-features = false, features = ["std"] } once_cell = { version = "1", default-features = false, features = ["unstable"] } -rustix = { version = "0.38", features = ["fs", "termios"] } rustls = { version = "0.21" } [target.aarch64-apple-darwin.dependencies] bitflags = { version = "2", default-features = false, features = ["std"] } nix = { version = "0.26" } once_cell = { version = "1", default-features = false, features = ["unstable"] } -rustix = { version = "0.38", features = ["fs", "termios"] } rustls = { version = "0.21" } [target.aarch64-apple-darwin.build-dependencies] bitflags = { version = "2", default-features = false, features = ["std"] } once_cell = { version = "1", default-features = false, features = ["unstable"] } -rustix = { version = "0.38", features = ["fs", "termios"] } rustls = { version = "0.21" } [target.x86_64-pc-windows-msvc.dependencies]