diff --git a/iox_catalog/migrations/20221014122742_lower-default-per-table-column-limit.sql b/iox_catalog/migrations/20221014122742_lower-default-per-table-column-limit.sql new file mode 100644 index 0000000000..69ed514488 --- /dev/null +++ b/iox_catalog/migrations/20221014122742_lower-default-per-table-column-limit.sql @@ -0,0 +1,7 @@ +-- Lower the defualt per-table column limit. +-- +-- https://github.com/influxdata/influxdb_iox/issues/5858 +ALTER TABLE + namespace ALTER max_columns_per_table +SET + DEFAULT 200; diff --git a/iox_catalog/src/interface.rs b/iox_catalog/src/interface.rs index 8d78d33cfc..d76698ba2b 100644 --- a/iox_catalog/src/interface.rs +++ b/iox_catalog/src/interface.rs @@ -864,7 +864,7 @@ pub async fn list_schemas( #[cfg(test)] pub(crate) mod test_helpers { - use crate::validate_or_insert_schema; + use crate::{validate_or_insert_schema, DEFAULT_MAX_COLUMNS_PER_TABLE, DEFAULT_MAX_TABLES}; use super::*; use ::test_helpers::{assert_contains, tracing::TracingCapture}; @@ -956,6 +956,13 @@ pub(crate) mod test_helpers { assert!(namespace.id > NamespaceId::new(0)); assert_eq!(namespace.name, namespace_name); + // Assert default values for service protection limits. + assert_eq!(namespace.max_tables, DEFAULT_MAX_TABLES); + assert_eq!( + namespace.max_columns_per_table, + DEFAULT_MAX_COLUMNS_PER_TABLE + ); + let conflict = repos .namespaces() .create(namespace_name, "inf", topic.id, pool.id) @@ -4021,7 +4028,12 @@ pub(crate) mod test_helpers { let batches = mutable_batch_lp::lines_to_batches(lines, 42).unwrap(); let batches = batches.iter().map(|(table, batch)| (table.as_str(), batch)); - let ns = NamespaceSchema::new(namespace.id, topic.id, pool.id, 1000); + let ns = NamespaceSchema::new( + namespace.id, + topic.id, + pool.id, + namespace.max_columns_per_table, + ); let schema = validate_or_insert_schema(batches, &ns, repos) .await diff --git a/iox_catalog/src/lib.rs b/iox_catalog/src/lib.rs index 13e595a46b..471ef23314 100644 --- a/iox_catalog/src/lib.rs +++ b/iox_catalog/src/lib.rs @@ -27,6 +27,11 @@ const SHARED_TOPIC_NAME: &str = "iox-shared"; const SHARED_QUERY_POOL: &str = SHARED_TOPIC_NAME; const TIME_COLUMN: &str = "time"; +/// Default per-namespace table count service protection limit. +pub const DEFAULT_MAX_TABLES: i32 = 10_000; +/// Default per-table column count service protection limit. +pub const DEFAULT_MAX_COLUMNS_PER_TABLE: i32 = 200; + /// A string value representing an infinite retention policy. pub const INFINITE_RETENTION_POLICY: &str = "inf"; diff --git a/iox_catalog/src/mem.rs b/iox_catalog/src/mem.rs index 45be0e8838..bdba0ccd6b 100644 --- a/iox_catalog/src/mem.rs +++ b/iox_catalog/src/mem.rs @@ -9,6 +9,7 @@ use crate::{ TombstoneRepo, TopicMetadataRepo, Transaction, }, metrics::MetricDecorator, + DEFAULT_MAX_COLUMNS_PER_TABLE, DEFAULT_MAX_TABLES, }; use async_trait::async_trait; use data_types::{ @@ -302,8 +303,8 @@ impl NamespaceRepo for MemTxn { topic_id, query_pool_id, retention_duration: Some(retention_duration.to_string()), - max_tables: 10000, - max_columns_per_table: 1000, + max_tables: DEFAULT_MAX_TABLES, + max_columns_per_table: DEFAULT_MAX_COLUMNS_PER_TABLE, }; stage.namespaces.push(namespace); Ok(stage.namespaces.last().unwrap().clone()) diff --git a/iox_catalog/src/postgres.rs b/iox_catalog/src/postgres.rs index 7561303d0e..2c02e2fb45 100644 --- a/iox_catalog/src/postgres.rs +++ b/iox_catalog/src/postgres.rs @@ -8,6 +8,7 @@ use crate::{ TombstoneRepo, TopicMetadataRepo, Transaction, }, metrics::MetricDecorator, + DEFAULT_MAX_COLUMNS_PER_TABLE, DEFAULT_MAX_TABLES, }; use async_trait::async_trait; use data_types::{ @@ -618,6 +619,10 @@ RETURNING *; } })?; + // Ensure the column default values match the code values. + debug_assert_eq!(rec.max_tables, DEFAULT_MAX_TABLES); + debug_assert_eq!(rec.max_columns_per_table, DEFAULT_MAX_COLUMNS_PER_TABLE); + Ok(rec) } diff --git a/router/src/dml_handlers/ns_autocreation.rs b/router/src/dml_handlers/ns_autocreation.rs index 81e5ebb608..4e5c2d6343 100644 --- a/router/src/dml_handlers/ns_autocreation.rs +++ b/router/src/dml_handlers/ns_autocreation.rs @@ -223,8 +223,8 @@ mod tests { retention_duration: Some("inf".to_owned()), topic_id: TopicId::new(42), query_pool_id: QueryPoolId::new(42), - max_tables: 10000, - max_columns_per_table: 1000, + max_tables: iox_catalog::DEFAULT_MAX_TABLES, + max_columns_per_table: iox_catalog::DEFAULT_MAX_COLUMNS_PER_TABLE, } ); }