diff --git a/data_types/src/service_limits.rs b/data_types/src/service_limits.rs index a26b67417d..1fbd625a24 100644 --- a/data_types/src/service_limits.rs +++ b/data_types/src/service_limits.rs @@ -16,6 +16,17 @@ impl MaxTables { pub fn get(&self) -> i32 { self.0 } + + /// Default per-namespace table count service protection limit. + pub const fn const_default() -> Self { + Self(500) + } +} + +impl Default for MaxTables { + fn default() -> Self { + Self::const_default() + } } impl std::fmt::Display for MaxTables { @@ -38,6 +49,17 @@ impl MaxColumnsPerTable { pub fn get(&self) -> i32 { self.0 } + + /// Default per-table column count service protection limit. + pub const fn const_default() -> Self { + Self(200) + } +} + +impl Default for MaxColumnsPerTable { + fn default() -> Self { + Self::const_default() + } } impl std::fmt::Display for MaxColumnsPerTable { diff --git a/ingester_test_ctx/src/lib.rs b/ingester_test_ctx/src/lib.rs index 557be032e8..8b39bf8f81 100644 --- a/ingester_test_ctx/src/lib.rs +++ b/ingester_test_ctx/src/lib.rs @@ -23,8 +23,7 @@ use arrow::record_batch::RecordBatch; use arrow_flight::{decode::FlightRecordBatchStream, flight_service_server::FlightService, Ticket}; use data_types::{ partition_template::{NamespacePartitionTemplateOverride, TablePartitionTemplateOverride}, - MaxColumnsPerTable, MaxTables, Namespace, NamespaceId, NamespaceSchema, ParquetFile, - PartitionKey, SequenceNumber, TableId, + Namespace, NamespaceId, NamespaceSchema, ParquetFile, PartitionKey, SequenceNumber, TableId, }; use dml::{DmlMeta, DmlWrite}; use futures::{stream::FuturesUnordered, FutureExt, StreamExt, TryStreamExt}; @@ -237,10 +236,8 @@ where NamespaceSchema { id: ns.id, tables: Default::default(), - max_columns_per_table: MaxColumnsPerTable::new( - iox_catalog::DEFAULT_MAX_COLUMNS_PER_TABLE - ), - max_tables: MaxTables::new(iox_catalog::DEFAULT_MAX_TABLES), + max_columns_per_table: Default::default(), + max_tables: Default::default(), retention_period_ns, partition_template: partition_template.unwrap_or_default(), }, diff --git a/iox_catalog/src/interface.rs b/iox_catalog/src/interface.rs index dfe8c2c4a6..5c7b3317f9 100644 --- a/iox_catalog/src/interface.rs +++ b/iox_catalog/src/interface.rs @@ -764,13 +764,13 @@ pub async fn list_schemas( pub(crate) mod test_helpers { use crate::{ test_helpers::{arbitrary_namespace, arbitrary_parquet_file_params, arbitrary_table}, - validate_or_insert_schema, DEFAULT_MAX_COLUMNS_PER_TABLE, DEFAULT_MAX_TABLES, + validate_or_insert_schema, }; use super::*; use ::test_helpers::assert_error; use assert_matches::assert_matches; - use data_types::{ColumnId, CompactionLevel}; + use data_types::{ColumnId, CompactionLevel, MaxColumnsPerTable, MaxTables}; use futures::Future; use generated_types::influxdata::iox::partition_template::v1 as proto; use metric::{Attributes, DurationHistogram, Metric}; @@ -843,10 +843,10 @@ pub(crate) mod test_helpers { assert_eq!(namespace, lookup_namespace); // Assert default values for service protection limits. - assert_eq!(namespace.max_tables.get(), DEFAULT_MAX_TABLES); + assert_eq!(namespace.max_tables, MaxTables::default()); assert_eq!( - namespace.max_columns_per_table.get(), - DEFAULT_MAX_COLUMNS_PER_TABLE + namespace.max_columns_per_table, + MaxColumnsPerTable::default() ); let conflict = repos diff --git a/iox_catalog/src/lib.rs b/iox_catalog/src/lib.rs index ff28396f5b..e6ebe99a61 100644 --- a/iox_catalog/src/lib.rs +++ b/iox_catalog/src/lib.rs @@ -30,10 +30,6 @@ use thiserror::Error; const TIME_COLUMN: &str = "time"; -/// Default per-namespace table count service protection limit. -pub const DEFAULT_MAX_TABLES: i32 = 500; -/// Default per-table column count service protection limit. -pub const DEFAULT_MAX_COLUMNS_PER_TABLE: i32 = 200; /// Default retention period for data in the catalog. pub const DEFAULT_RETENTION_PERIOD: Option = None; diff --git a/iox_catalog/src/mem.rs b/iox_catalog/src/mem.rs index 7e87017cb0..fb50c48576 100644 --- a/iox_catalog/src/mem.rs +++ b/iox_catalog/src/mem.rs @@ -9,7 +9,6 @@ use crate::{ MAX_PARQUET_FILES_SELECTED_ONCE_FOR_RETENTION, }, metrics::MetricDecorator, - DEFAULT_MAX_COLUMNS_PER_TABLE, DEFAULT_MAX_TABLES, }; use async_trait::async_trait; use data_types::SortedColumnSet; @@ -166,10 +165,10 @@ impl NamespaceRepo for MemTxn { let namespace = Namespace { id: NamespaceId::new(stage.namespaces.len() as i64 + 1), name: name.to_string(), - max_tables: MaxTables::new(max_tables.unwrap_or(DEFAULT_MAX_TABLES)), - max_columns_per_table: MaxColumnsPerTable::new( - max_columns_per_table.unwrap_or(DEFAULT_MAX_COLUMNS_PER_TABLE), - ), + max_tables: max_tables.map(MaxTables::new).unwrap_or_default(), + max_columns_per_table: max_columns_per_table + .map(MaxColumnsPerTable::new) + .unwrap_or_default(), retention_period_ns, deleted_at: None, partition_template: partition_template.unwrap_or_default(), diff --git a/iox_catalog/src/postgres.rs b/iox_catalog/src/postgres.rs index 5192b0e1dc..3bf8ba1086 100644 --- a/iox_catalog/src/postgres.rs +++ b/iox_catalog/src/postgres.rs @@ -13,7 +13,6 @@ use crate::{ }, metrics::MetricDecorator, migrate::IOxMigrator, - DEFAULT_MAX_COLUMNS_PER_TABLE, DEFAULT_MAX_TABLES, }; use async_trait::async_trait; use data_types::SortedColumnSet; @@ -714,8 +713,8 @@ RETURNING id, name, retention_period_ns, max_tables, max_columns_per_table, dele .bind(SHARED_TOPIC_ID) // $2 .bind(SHARED_QUERY_POOL_ID) // $3 .bind(retention_period_ns) // $4 - .bind(max_tables.unwrap_or(DEFAULT_MAX_TABLES)) // $5 - .bind(max_columns_per_table.unwrap_or(DEFAULT_MAX_COLUMNS_PER_TABLE)) // $6 + .bind(max_tables.unwrap_or_default()) // $5 + .bind(max_columns_per_table.unwrap_or_default()) // $6 .bind(partition_template); // $7 let rec = rec.fetch_one(&mut self.inner).await.map_err(|e| { @@ -2640,9 +2639,9 @@ RETURNING id, hash_id, table_id, partition_key, sort_key, sort_key_ids, new_file let insert_null_partition_template_namespace = sqlx::query( r#" INSERT INTO namespace ( - name, topic_id, query_pool_id, retention_period_ns, max_tables, partition_template + name, topic_id, query_pool_id, retention_period_ns, partition_template ) -VALUES ( $1, $2, $3, $4, $5, NULL ) +VALUES ( $1, $2, $3, $4, NULL ) RETURNING id, name, retention_period_ns, max_tables, max_columns_per_table, deleted_at, partition_template; "#, @@ -2650,8 +2649,7 @@ RETURNING id, name, retention_period_ns, max_tables, max_columns_per_table, dele .bind(namespace_name) // $1 .bind(SHARED_TOPIC_ID) // $2 .bind(SHARED_QUERY_POOL_ID) // $3 - .bind(None::>) // $4 - .bind(DEFAULT_MAX_TABLES); // $5 + .bind(None::>); // $4 insert_null_partition_template_namespace .fetch_one(&pool) diff --git a/iox_catalog/src/sqlite.rs b/iox_catalog/src/sqlite.rs index 37e539d075..bac053d3a6 100644 --- a/iox_catalog/src/sqlite.rs +++ b/iox_catalog/src/sqlite.rs @@ -11,7 +11,6 @@ use crate::{ TRANSITION_SHARD_ID, TRANSITION_SHARD_INDEX, }, metrics::MetricDecorator, - DEFAULT_MAX_COLUMNS_PER_TABLE, DEFAULT_MAX_TABLES, }; use async_trait::async_trait; use data_types::{ @@ -286,8 +285,8 @@ RETURNING id, name, retention_period_ns, max_tables, max_columns_per_table, dele .bind(SHARED_TOPIC_ID) // $2 .bind(SHARED_QUERY_POOL_ID) // $3 .bind(retention_period_ns) // $4 - .bind(max_tables.unwrap_or(DEFAULT_MAX_TABLES)) // $5 - .bind(max_columns_per_table.unwrap_or(DEFAULT_MAX_COLUMNS_PER_TABLE)) // $6 + .bind(max_tables.unwrap_or_default()) // $5 + .bind(max_columns_per_table.unwrap_or_default()) // $6 .bind(partition_template); // $7 let rec = rec.fetch_one(self.inner.get_mut()).await.map_err(|e| { @@ -2121,9 +2120,9 @@ RETURNING id, hash_id, table_id, partition_key, sort_key, sort_key_ids, new_file let insert_null_partition_template_namespace = sqlx::query( r#" INSERT INTO namespace ( - name, topic_id, query_pool_id, retention_period_ns, max_tables, partition_template + name, topic_id, query_pool_id, retention_period_ns, partition_template ) -VALUES ( $1, $2, $3, $4, $5, NULL ) +VALUES ( $1, $2, $3, $4, NULL ) RETURNING id, name, retention_period_ns, max_tables, max_columns_per_table, deleted_at, partition_template; "#, @@ -2131,8 +2130,7 @@ RETURNING id, name, retention_period_ns, max_tables, max_columns_per_table, dele .bind(namespace_name) // $1 .bind(SHARED_TOPIC_ID) // $2 .bind(SHARED_QUERY_POOL_ID) // $3 - .bind(None::>) // $4 - .bind(DEFAULT_MAX_TABLES); // $5 + .bind(None::>); // $4 insert_null_partition_template_namespace .fetch_one(&pool) diff --git a/ioxd_querier/src/rpc/namespace.rs b/ioxd_querier/src/rpc/namespace.rs index c6ed2479ba..08cfb32146 100644 --- a/ioxd_querier/src/rpc/namespace.rs +++ b/ioxd_querier/src/rpc/namespace.rs @@ -102,16 +102,12 @@ mod tests { use std::collections::HashMap; use super::*; + use data_types::{MaxColumnsPerTable, MaxTables}; use generated_types::influxdata::iox::namespace::v1::namespace_service_server::NamespaceService; use iox_tests::TestCatalog; use querier::{create_ingester_connection_for_testing, QuerierCatalogCache}; use tokio::runtime::Handle; - use iox_catalog::{ - DEFAULT_MAX_COLUMNS_PER_TABLE as TEST_MAX_COLUMNS_PER_TABLE, - DEFAULT_MAX_TABLES as TEST_MAX_TABLES, - }; - /// Common retention period value we'll use in tests const TEST_RETENTION_PERIOD_NS: Option = Some(3_600 * 1_000_000_000); @@ -185,16 +181,16 @@ mod tests { id: 1, name: "namespace2".to_string(), retention_period_ns: TEST_RETENTION_PERIOD_NS, - max_tables: TEST_MAX_TABLES, - max_columns_per_table: TEST_MAX_COLUMNS_PER_TABLE, + max_tables: MaxTables::default().get(), + max_columns_per_table: MaxColumnsPerTable::default().get(), partition_template: None, }, proto::Namespace { id: 2, name: "namespace1".to_string(), retention_period_ns: TEST_RETENTION_PERIOD_NS, - max_tables: TEST_MAX_TABLES, - max_columns_per_table: TEST_MAX_COLUMNS_PER_TABLE, + max_tables: MaxTables::default().get(), + max_columns_per_table: MaxColumnsPerTable::default().get(), partition_template: None, }, ] diff --git a/router/src/lib.rs b/router/src/lib.rs index adc0d57d1f..4d9417fe5d 100644 --- a/router/src/lib.rs +++ b/router/src/lib.rs @@ -160,10 +160,8 @@ pub(crate) mod test_helpers { NamespaceSchema { id: NamespaceId::new(id), tables: BTreeMap::new(), - max_columns_per_table: MaxColumnsPerTable::new( - iox_catalog::DEFAULT_MAX_COLUMNS_PER_TABLE, - ), - max_tables: MaxTables::new(iox_catalog::DEFAULT_MAX_TABLES), + max_columns_per_table: MaxColumnsPerTable::const_default(), + max_tables: MaxTables::const_default(), retention_period_ns: None, partition_template: DEFAULT_NAMESPACE_PARTITION_TEMPLATE, } diff --git a/router/src/namespace_resolver/ns_autocreation.rs b/router/src/namespace_resolver/ns_autocreation.rs index eb4e15ec6f..e3860874c7 100644 --- a/router/src/namespace_resolver/ns_autocreation.rs +++ b/router/src/namespace_resolver/ns_autocreation.rs @@ -137,7 +137,7 @@ mod tests { use std::sync::Arc; use assert_matches::assert_matches; - use data_types::{MaxColumnsPerTable, MaxTables, Namespace, NamespaceId}; + use data_types::{Namespace, NamespaceId}; use iox_catalog::{interface::SoftDeletedRows, mem::MemCatalog}; use super::*; @@ -228,10 +228,8 @@ mod tests { Namespace { id: NamespaceId::new(1), name: ns.to_string(), - max_tables: MaxTables::new(iox_catalog::DEFAULT_MAX_TABLES), - max_columns_per_table: MaxColumnsPerTable::new( - iox_catalog::DEFAULT_MAX_COLUMNS_PER_TABLE - ), + max_tables: Default::default(), + max_columns_per_table: Default::default(), retention_period_ns: TEST_RETENTION_PERIOD_NS, deleted_at: None, partition_template: Default::default(), diff --git a/router/tests/grpc.rs b/router/tests/grpc.rs index cf357c5050..f87e5ae02e 100644 --- a/router/tests/grpc.rs +++ b/router/tests/grpc.rs @@ -1,7 +1,7 @@ use std::time::Duration; use assert_matches::assert_matches; -use data_types::NamespaceId; +use data_types::{MaxColumnsPerTable, MaxTables, NamespaceId}; use generated_types::influxdata::{ iox::{ ingester::v1::WriteRequest, @@ -624,7 +624,7 @@ async fn test_update_namespace_limit_max_tables() { assert_eq!(got.max_tables, 1); assert_eq!( got.max_columns_per_table, - iox_catalog::DEFAULT_MAX_COLUMNS_PER_TABLE + MaxColumnsPerTable::default().get() ); // The list namespace RPC should show the updated namespace @@ -706,7 +706,7 @@ async fn test_update_namespace_limit_max_columns_per_table() { assert_eq!(got.name, "bananas_test"); assert_eq!(got.id, 1); - assert_eq!(got.max_tables, iox_catalog::DEFAULT_MAX_TABLES); + assert_eq!(got.max_tables, MaxTables::default().get()); assert_eq!(got.max_columns_per_table, 1); // The list namespace RPC should show the updated namespace