Influxdata
/
influxdb
mirror of https://github.com/influxdata/influxdb.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

refactor: Define the default limits on the types, not the catalog

pull/24376/head
Carol (Nichols || Goulding) 2023-09-15 13:52:29 -04:00
parent 94b86e317b
commit d56217a8f6
No known key found for this signature in database
GPG Key ID: E907EE5A736F87D4
11 changed files with 57 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
data_types/src/service_limits.rs
View File

@ -16,6 +16,17 @@ impl MaxTables {
pub fn get(&self) -> i32 {
self.0
}
/// Default per-namespace table count service protection limit.
pub const fn const_default() -> Self {
Self(500)
}
}
impl Default for MaxTables {
fn default() -> Self {
Self::const_default()
}
}
impl std::fmt::Display for MaxTables {
@ -38,6 +49,17 @@ impl MaxColumnsPerTable {
pub fn get(&self) -> i32 {
self.0
}
/// Default per-table column count service protection limit.
pub const fn const_default() -> Self {
Self(200)
}
}
impl Default for MaxColumnsPerTable {
fn default() -> Self {
Self::const_default()
}
}
impl std::fmt::Display for MaxColumnsPerTable {

9
ingester_test_ctx/src/lib.rs
View File

@ -23,8 +23,7 @@ use arrow::record_batch::RecordBatch;
use arrow_flight::{decode::FlightRecordBatchStream, flight_service_server::FlightService, Ticket};
use data_types::{
partition_template::{NamespacePartitionTemplateOverride, TablePartitionTemplateOverride},
MaxColumnsPerTable, MaxTables, Namespace, NamespaceId, NamespaceSchema, ParquetFile,
PartitionKey, SequenceNumber, TableId,
Namespace, NamespaceId, NamespaceSchema, ParquetFile, PartitionKey, SequenceNumber, TableId,
};
use dml::{DmlMeta, DmlWrite};
use futures::{stream::FuturesUnordered, FutureExt, StreamExt, TryStreamExt};
@ -237,10 +236,8 @@ where
NamespaceSchema {
id: ns.id,
tables: Default::default(),
max_columns_per_table: MaxColumnsPerTable::new(
iox_catalog::DEFAULT_MAX_COLUMNS_PER_TABLE
),
max_tables: MaxTables::new(iox_catalog::DEFAULT_MAX_TABLES),
max_columns_per_table: Default::default(),
max_tables: Default::default(),
retention_period_ns,
partition_template: partition_template.unwrap_or_default(),
},

10
iox_catalog/src/interface.rs
View File

@ -764,13 +764,13 @@ pub async fn list_schemas(
pub(crate) mod test_helpers {
use crate::{
test_helpers::{arbitrary_namespace, arbitrary_parquet_file_params, arbitrary_table},
validate_or_insert_schema, DEFAULT_MAX_COLUMNS_PER_TABLE, DEFAULT_MAX_TABLES,
validate_or_insert_schema,
};
use super::*;
use ::test_helpers::assert_error;
use assert_matches::assert_matches;
use data_types::{ColumnId, CompactionLevel};
use data_types::{ColumnId, CompactionLevel, MaxColumnsPerTable, MaxTables};
use futures::Future;
use generated_types::influxdata::iox::partition_template::v1 as proto;
use metric::{Attributes, DurationHistogram, Metric};
@ -843,10 +843,10 @@ pub(crate) mod test_helpers {
assert_eq!(namespace, lookup_namespace);
// Assert default values for service protection limits.
assert_eq!(namespace.max_tables.get(), DEFAULT_MAX_TABLES);
assert_eq!(namespace.max_tables, MaxTables::default());
assert_eq!(
namespace.max_columns_per_table.get(),
DEFAULT_MAX_COLUMNS_PER_TABLE
namespace.max_columns_per_table,
MaxColumnsPerTable::default()
);
let conflict = repos

4
iox_catalog/src/lib.rs
View File

@ -30,10 +30,6 @@ use thiserror::Error;
const TIME_COLUMN: &str = "time";
/// Default per-namespace table count service protection limit.
pub const DEFAULT_MAX_TABLES: i32 = 500;
/// Default per-table column count service protection limit.
pub const DEFAULT_MAX_COLUMNS_PER_TABLE: i32 = 200;
/// Default retention period for data in the catalog.
pub const DEFAULT_RETENTION_PERIOD: Option<i64> = None;

9
iox_catalog/src/mem.rs
View File

@ -9,7 +9,6 @@ use crate::{
MAX_PARQUET_FILES_SELECTED_ONCE_FOR_RETENTION,
},
metrics::MetricDecorator,
DEFAULT_MAX_COLUMNS_PER_TABLE, DEFAULT_MAX_TABLES,
};
use async_trait::async_trait;
use data_types::SortedColumnSet;
@ -166,10 +165,10 @@ impl NamespaceRepo for MemTxn {
let namespace = Namespace {
id: NamespaceId::new(stage.namespaces.len() as i64 + 1),
name: name.to_string(),
max_tables: MaxTables::new(max_tables.unwrap_or(DEFAULT_MAX_TABLES)),
max_columns_per_table: MaxColumnsPerTable::new(
max_columns_per_table.unwrap_or(DEFAULT_MAX_COLUMNS_PER_TABLE),
),
max_tables: max_tables.map(MaxTables::new).unwrap_or_default(),
max_columns_per_table: max_columns_per_table
.map(MaxColumnsPerTable::new)
.unwrap_or_default(),
retention_period_ns,
deleted_at: None,
partition_template: partition_template.unwrap_or_default(),

12
iox_catalog/src/postgres.rs
View File

@ -13,7 +13,6 @@ use crate::{
},
metrics::MetricDecorator,
migrate::IOxMigrator,
DEFAULT_MAX_COLUMNS_PER_TABLE, DEFAULT_MAX_TABLES,
};
use async_trait::async_trait;
use data_types::SortedColumnSet;
@ -714,8 +713,8 @@ RETURNING id, name, retention_period_ns, max_tables, max_columns_per_table, dele
.bind(SHARED_TOPIC_ID) // $2
.bind(SHARED_QUERY_POOL_ID) // $3
.bind(retention_period_ns) // $4
.bind(max_tables.unwrap_or(DEFAULT_MAX_TABLES)) // $5
.bind(max_columns_per_table.unwrap_or(DEFAULT_MAX_COLUMNS_PER_TABLE)) // $6
.bind(max_tables.unwrap_or_default()) // $5
.bind(max_columns_per_table.unwrap_or_default()) // $6
.bind(partition_template); // $7
let rec = rec.fetch_one(&mut self.inner).await.map_err(|e| {
@ -2640,9 +2639,9 @@ RETURNING id, hash_id, table_id, partition_key, sort_key, sort_key_ids, new_file
let insert_null_partition_template_namespace = sqlx::query(
r#"
INSERT INTO namespace (
name, topic_id, query_pool_id, retention_period_ns, max_tables, partition_template
name, topic_id, query_pool_id, retention_period_ns, partition_template
)
VALUES ( $1, $2, $3, $4, $5, NULL )
VALUES ( $1, $2, $3, $4, NULL )
RETURNING id, name, retention_period_ns, max_tables, max_columns_per_table, deleted_at,
partition_template;
"#,
@ -2650,8 +2649,7 @@ RETURNING id, name, retention_period_ns, max_tables, max_columns_per_table, dele
.bind(namespace_name) // $1
.bind(SHARED_TOPIC_ID) // $2
.bind(SHARED_QUERY_POOL_ID) // $3
.bind(None::<Option<i64>>) // $4
.bind(DEFAULT_MAX_TABLES); // $5
.bind(None::<Option<i64>>); // $4
insert_null_partition_template_namespace
.fetch_one(&pool)

12
iox_catalog/src/sqlite.rs
View File

@ -11,7 +11,6 @@ use crate::{
TRANSITION_SHARD_ID, TRANSITION_SHARD_INDEX,
},
metrics::MetricDecorator,
DEFAULT_MAX_COLUMNS_PER_TABLE, DEFAULT_MAX_TABLES,
};
use async_trait::async_trait;
use data_types::{
@ -286,8 +285,8 @@ RETURNING id, name, retention_period_ns, max_tables, max_columns_per_table, dele
.bind(SHARED_TOPIC_ID) // $2
.bind(SHARED_QUERY_POOL_ID) // $3
.bind(retention_period_ns) // $4
.bind(max_tables.unwrap_or(DEFAULT_MAX_TABLES)) // $5
.bind(max_columns_per_table.unwrap_or(DEFAULT_MAX_COLUMNS_PER_TABLE)) // $6
.bind(max_tables.unwrap_or_default()) // $5
.bind(max_columns_per_table.unwrap_or_default()) // $6
.bind(partition_template); // $7
let rec = rec.fetch_one(self.inner.get_mut()).await.map_err(|e| {
@ -2121,9 +2120,9 @@ RETURNING id, hash_id, table_id, partition_key, sort_key, sort_key_ids, new_file
let insert_null_partition_template_namespace = sqlx::query(
r#"
INSERT INTO namespace (
name, topic_id, query_pool_id, retention_period_ns, max_tables, partition_template
name, topic_id, query_pool_id, retention_period_ns, partition_template
)
VALUES ( $1, $2, $3, $4, $5, NULL )
VALUES ( $1, $2, $3, $4, NULL )
RETURNING id, name, retention_period_ns, max_tables, max_columns_per_table, deleted_at,
partition_template;
"#,
@ -2131,8 +2130,7 @@ RETURNING id, name, retention_period_ns, max_tables, max_columns_per_table, dele
.bind(namespace_name) // $1
.bind(SHARED_TOPIC_ID) // $2
.bind(SHARED_QUERY_POOL_ID) // $3
.bind(None::<Option<i64>>) // $4
.bind(DEFAULT_MAX_TABLES); // $5
.bind(None::<Option<i64>>); // $4
insert_null_partition_template_namespace
.fetch_one(&pool)

14
ioxd_querier/src/rpc/namespace.rs
View File

@ -102,16 +102,12 @@ mod tests {
use std::collections::HashMap;
use super::*;
use data_types::{MaxColumnsPerTable, MaxTables};
use generated_types::influxdata::iox::namespace::v1::namespace_service_server::NamespaceService;
use iox_tests::TestCatalog;
use querier::{create_ingester_connection_for_testing, QuerierCatalogCache};
use tokio::runtime::Handle;
use iox_catalog::{
DEFAULT_MAX_COLUMNS_PER_TABLE as TEST_MAX_COLUMNS_PER_TABLE,
DEFAULT_MAX_TABLES as TEST_MAX_TABLES,
};
/// Common retention period value we'll use in tests
const TEST_RETENTION_PERIOD_NS: Option<i64> = Some(3_600 * 1_000_000_000);
@ -185,16 +181,16 @@ mod tests {
id: 1,
name: "namespace2".to_string(),
retention_period_ns: TEST_RETENTION_PERIOD_NS,
max_tables: TEST_MAX_TABLES,
max_columns_per_table: TEST_MAX_COLUMNS_PER_TABLE,
max_tables: MaxTables::default().get(),
max_columns_per_table: MaxColumnsPerTable::default().get(),
partition_template: None,
},
proto::Namespace {
id: 2,
name: "namespace1".to_string(),
retention_period_ns: TEST_RETENTION_PERIOD_NS,
max_tables: TEST_MAX_TABLES,
max_columns_per_table: TEST_MAX_COLUMNS_PER_TABLE,
max_tables: MaxTables::default().get(),
max_columns_per_table: MaxColumnsPerTable::default().get(),
partition_template: None,
},
]

6
router/src/lib.rs
View File

@ -160,10 +160,8 @@ pub(crate) mod test_helpers {
NamespaceSchema {
id: NamespaceId::new(id),
tables: BTreeMap::new(),
max_columns_per_table: MaxColumnsPerTable::new(
iox_catalog::DEFAULT_MAX_COLUMNS_PER_TABLE,
),
max_tables: MaxTables::new(iox_catalog::DEFAULT_MAX_TABLES),
max_columns_per_table: MaxColumnsPerTable::const_default(),
max_tables: MaxTables::const_default(),
retention_period_ns: None,
partition_template: DEFAULT_NAMESPACE_PARTITION_TEMPLATE,
}

8
router/src/namespace_resolver/ns_autocreation.rs
View File

@ -137,7 +137,7 @@ mod tests {
use std::sync::Arc;
use assert_matches::assert_matches;
use data_types::{MaxColumnsPerTable, MaxTables, Namespace, NamespaceId};
use data_types::{Namespace, NamespaceId};
use iox_catalog::{interface::SoftDeletedRows, mem::MemCatalog};
use super::*;
@ -228,10 +228,8 @@ mod tests {
Namespace {
id: NamespaceId::new(1),
name: ns.to_string(),
max_tables: MaxTables::new(iox_catalog::DEFAULT_MAX_TABLES),
max_columns_per_table: MaxColumnsPerTable::new(
iox_catalog::DEFAULT_MAX_COLUMNS_PER_TABLE
),
max_tables: Default::default(),
max_columns_per_table: Default::default(),
retention_period_ns: TEST_RETENTION_PERIOD_NS,
deleted_at: None,
partition_template: Default::default(),

6
router/tests/grpc.rs
View File

@ -1,7 +1,7 @@
use std::time::Duration;
use assert_matches::assert_matches;
use data_types::NamespaceId;
use data_types::{MaxColumnsPerTable, MaxTables, NamespaceId};
use generated_types::influxdata::{
iox::{
ingester::v1::WriteRequest,
@ -624,7 +624,7 @@ async fn test_update_namespace_limit_max_tables() {
assert_eq!(got.max_tables, 1);
assert_eq!(
got.max_columns_per_table,
iox_catalog::DEFAULT_MAX_COLUMNS_PER_TABLE
MaxColumnsPerTable::default().get()
);
// The list namespace RPC should show the updated namespace
@ -706,7 +706,7 @@ async fn test_update_namespace_limit_max_columns_per_table() {
assert_eq!(got.name, "bananas_test");
assert_eq!(got.id, 1);
assert_eq!(got.max_tables, iox_catalog::DEFAULT_MAX_TABLES);
assert_eq!(got.max_tables, MaxTables::default().get());
assert_eq!(got.max_columns_per_table, 1);
// The list namespace RPC should show the updated namespace