Merge pull request #5860 from influxdata/dom/reduce-column-limit

refactor(iox_catalog): reduce default column limit

iox_catalog/migrations/20221014122742_lower-default-per-table-column-limit.sql

@@ -0,0 +1,7 @@
+-- Lower the defualt per-table column limit.
+--
+-- https://github.com/influxdata/influxdb_iox/issues/5858
+ALTER TABLE
+    namespace ALTER max_columns_per_table
+SET
+    DEFAULT 200;

iox_catalog/src/interface.rs

@@ -864,7 +864,7 @@ pub async fn list_schemas(
 
 #[cfg(test)]
 pub(crate) mod test_helpers {
-    use crate::validate_or_insert_schema;
+    use crate::{validate_or_insert_schema, DEFAULT_MAX_COLUMNS_PER_TABLE, DEFAULT_MAX_TABLES};
 
     use super::*;
     use ::test_helpers::{assert_contains, tracing::TracingCapture};
@@ -956,6 +956,13 @@ pub(crate) mod test_helpers {
         assert!(namespace.id > NamespaceId::new(0));
         assert_eq!(namespace.name, namespace_name);
 
+        // Assert default values for service protection limits.
+        assert_eq!(namespace.max_tables, DEFAULT_MAX_TABLES);
+        assert_eq!(
+            namespace.max_columns_per_table,
+            DEFAULT_MAX_COLUMNS_PER_TABLE
+        );
+
         let conflict = repos
             .namespaces()
             .create(namespace_name, "inf", topic.id, pool.id)
@@ -4021,7 +4028,12 @@ pub(crate) mod test_helpers {
 
         let batches = mutable_batch_lp::lines_to_batches(lines, 42).unwrap();
         let batches = batches.iter().map(|(table, batch)| (table.as_str(), batch));
-        let ns = NamespaceSchema::new(namespace.id, topic.id, pool.id, 1000);
+        let ns = NamespaceSchema::new(
+            namespace.id,
+            topic.id,
+            pool.id,
+            namespace.max_columns_per_table,
+        );
 
         let schema = validate_or_insert_schema(batches, &ns, repos)
             .await

iox_catalog/src/lib.rs

@@ -27,6 +27,11 @@ const SHARED_TOPIC_NAME: &str = "iox-shared";
 const SHARED_QUERY_POOL: &str = SHARED_TOPIC_NAME;
 const TIME_COLUMN: &str = "time";
 
+/// Default per-namespace table count service protection limit.
+pub const DEFAULT_MAX_TABLES: i32 = 10_000;
+/// Default per-table column count service protection limit.
+pub const DEFAULT_MAX_COLUMNS_PER_TABLE: i32 = 200;
+
 /// A string value representing an infinite retention policy.
 pub const INFINITE_RETENTION_POLICY: &str = "inf";
 

iox_catalog/src/mem.rs

@@ -9,6 +9,7 @@ use crate::{
         TombstoneRepo, TopicMetadataRepo, Transaction,
     },
     metrics::MetricDecorator,
+    DEFAULT_MAX_COLUMNS_PER_TABLE, DEFAULT_MAX_TABLES,
 };
 use async_trait::async_trait;
 use data_types::{
@@ -302,8 +303,8 @@ impl NamespaceRepo for MemTxn {
             topic_id,
             query_pool_id,
             retention_duration: Some(retention_duration.to_string()),
-            max_tables: 10000,
-            max_columns_per_table: 1000,
+            max_tables: DEFAULT_MAX_TABLES,
+            max_columns_per_table: DEFAULT_MAX_COLUMNS_PER_TABLE,
         };
         stage.namespaces.push(namespace);
         Ok(stage.namespaces.last().unwrap().clone())

iox_catalog/src/postgres.rs

@@ -8,6 +8,7 @@ use crate::{
         TombstoneRepo, TopicMetadataRepo, Transaction,
     },
     metrics::MetricDecorator,
+    DEFAULT_MAX_COLUMNS_PER_TABLE, DEFAULT_MAX_TABLES,
 };
 use async_trait::async_trait;
 use data_types::{
@@ -618,6 +619,10 @@ RETURNING *;
             }
         })?;
 
+        // Ensure the column default values match the code values.
+        debug_assert_eq!(rec.max_tables, DEFAULT_MAX_TABLES);
+        debug_assert_eq!(rec.max_columns_per_table, DEFAULT_MAX_COLUMNS_PER_TABLE);
+
         Ok(rec)
     }
 

router/src/dml_handlers/ns_autocreation.rs

@@ -223,8 +223,8 @@ mod tests {
                 retention_duration: Some("inf".to_owned()),
                 topic_id: TopicId::new(42),
                 query_pool_id: QueryPoolId::new(42),
-                max_tables: 10000,
-                max_columns_per_table: 1000,
+                max_tables: iox_catalog::DEFAULT_MAX_TABLES,
+                max_columns_per_table: iox_catalog::DEFAULT_MAX_COLUMNS_PER_TABLE,
             }
         );
     }