fix: Remove max number of level 0 files configuration option

clap_blocks/src/compactor.rs

@@ -27,18 +27,6 @@ pub struct CompactorConfig {
     )]
     pub write_buffer_partition_range_end: i32,
 
-    /// Max number of level-0 files (written by ingester) we want to compact with level-1 each time
-    /// Default: 3
-    #[clap(
-        // TODO: verify the theory that if the streaming works as expected,
-        // we do not need to limit this number
-        long = "--compaction-max-number-level-0-files",
-        env = "INFLUXDB_IOX_COMPACTION_MAX_NUMBER_LEVEL_0_FILES",
-        default_value = "3",
-        action
-    )]
-    pub max_number_level_0_files: u32,
-
     /// Desired max size of compacted parquet files.
     /// It is a target desired value, rather than a guarantee.
     /// Default is 1024 * 1024 * 100 = 104,857,600 bytes (100MB)
@@ -91,7 +79,7 @@ pub struct CompactorConfig {
         default_value = "1073741824",
         action
     )]
-    pub max_concurrent_size_bytes: i64,
+    pub max_concurrent_size_bytes: u64,
 
     /// Max number of partitions per sequencer we want to compact per cycle
     /// Default: 1

compactor/src/compact.rs

@@ -2588,19 +2588,17 @@ mod tests {
     }
 
     fn make_compactor_config() -> CompactorConfig {
-        let max_number_level_0_files = 3;
         let max_desired_file_size_bytes = 10_000;
         let percentage_max_file_size = 30;
         let split_percentage = 80;
-        let max_concurrent_compaction_size_bytes = 100_000;
+        let max_concurrent_size_bytes = 100_000;
         let max_number_partitions_per_sequencer = 1;
         let min_number_recent_ingested_per_partition = 1;
         CompactorConfig::new(
-            max_number_level_0_files,
             max_desired_file_size_bytes,
             percentage_max_file_size,
             split_percentage,
-            max_concurrent_compaction_size_bytes,
+            max_concurrent_size_bytes,
             max_number_partitions_per_sequencer,
             min_number_recent_ingested_per_partition,
         )

compactor/src/handler.rs

@@ -98,9 +98,6 @@ impl CompactorHandlerImpl {
 /// The configuration options for the compactor.
 #[derive(Debug, Clone, Copy)]
 pub struct CompactorConfig {
-    /// Max number of level-0 files (written by ingester) we want to compact with level-1 each time
-    max_number_level_0_files: u32,
-
     /// Desired max size of compacted parquet files
     /// It is a target desired value than a guarantee
     max_desired_file_size_bytes: u64,
@@ -137,7 +134,6 @@ pub struct CompactorConfig {
 impl CompactorConfig {
     /// Initialize a valid config
     pub fn new(
-        max_number_level_0_files: u32,
         max_desired_file_size_bytes: u64,
         percentage_max_file_size: u16,
         split_percentage: u16,
@@ -148,7 +144,6 @@ impl CompactorConfig {
         assert!(split_percentage > 0 && split_percentage <= 100);
 
         Self {
-            max_number_level_0_files,
             max_desired_file_size_bytes,
             percentage_max_file_size,
             split_percentage,
@@ -158,11 +153,6 @@ impl CompactorConfig {
         }
     }
 
-    /// Max number of level-0 files we want to compact with level-1 each time
-    pub fn max_number_level_0_files(&self) -> u32 {
-        self.max_number_level_0_files
-    }
-
     /// Desired max file of a compacted file
     pub fn max_desired_file_size_bytes(&self) -> u64 {
         self.max_desired_file_size_bytes

influxdb_iox/src/commands/run/all_in_one.rs

@@ -392,7 +392,6 @@ impl Config {
             topic: QUERY_POOL_NAME.to_string(),
             write_buffer_partition_range_start,
             write_buffer_partition_range_end,
-            max_number_level_0_files: 3,
             max_desired_file_size_bytes: 30000,
             percentage_max_file_size: 30,
             split_percentage: 80,

ioxd_compactor/src/lib.rs

@@ -164,7 +164,6 @@ pub async fn create_compactor_server_type(
     let parquet_store = ParquetStorage::new(object_store);
 
     let compactor_config = compactor::handler::CompactorConfig::new(
-        compactor_config.max_number_level_0_files,
         compactor_config.max_desired_file_size_bytes,
         compactor_config.percentage_max_file_size,
         compactor_config.split_percentage,