From a4704dd165a816aa5beb9c78a1cb5ea6a3684981 Mon Sep 17 00:00:00 2001 From: Paul Dix Date: Tue, 20 Jul 2021 15:40:50 -0400 Subject: [PATCH] chore: update parquet_cache_limit to u64 and 0 for default --- data_types/src/database_rules.rs | 5 +++-- .../iox/management/v1/database_rules.proto | 9 +++------ .../src/database_rules/lifecycle.rs | 19 +++++++------------ src/commands/database.rs | 10 +++++----- 4 files changed, 18 insertions(+), 25 deletions(-) diff --git a/data_types/src/database_rules.rs b/data_types/src/database_rules.rs index ddde203c93..86a71be778 100644 --- a/data_types/src/database_rules.rs +++ b/data_types/src/database_rules.rs @@ -167,8 +167,9 @@ pub struct LifecycleRules { /// Maximum number of rows to buffer in a MUB chunk before compacting it pub mub_row_threshold: NonZeroUsize, - /// Use up to this amount of space in bytes for caching Parquet files - pub parquet_cache_limit: Option, + /// Use up to this amount of space in bytes for caching Parquet files. None + /// will disable Parquet file caching. + pub parquet_cache_limit: Option, } impl LifecycleRules { diff --git a/generated_types/protos/influxdata/iox/management/v1/database_rules.proto b/generated_types/protos/influxdata/iox/management/v1/database_rules.proto index 68d9cbf6f9..b1ad761dbe 100644 --- a/generated_types/protos/influxdata/iox/management/v1/database_rules.proto +++ b/generated_types/protos/influxdata/iox/management/v1/database_rules.proto @@ -83,12 +83,9 @@ message LifecycleRules { // See data_types::database_rules::DEFAULT_MAX_ACTIVE_COMPACTIONS uint32 max_active_compactions = 16; - // Use up to this amount of space in bytes for caching Parquet files - ParquetCacheLimit parquet_cache_limit = 17; -} - -message ParquetCacheLimit { - uint64 value = 1; + // Use up to this amount of space in bytes for caching Parquet files. + // A value of 0 disables Parquet caching + uint64 parquet_cache_limit = 17; } message DatabaseRules { diff --git a/generated_types/src/database_rules/lifecycle.rs b/generated_types/src/database_rules/lifecycle.rs index 9a62c54df6..ab71e38de5 100644 --- a/generated_types/src/database_rules/lifecycle.rs +++ b/generated_types/src/database_rules/lifecycle.rs @@ -10,7 +10,6 @@ use data_types::database_rules::{ use crate::google::FieldViolation; use crate::influxdata::iox::management::v1 as management; -use crate::influxdata::iox::management::v1::ParquetCacheLimit; impl From for management::LifecycleRules { fn from(config: LifecycleRules) -> Self { @@ -36,9 +35,10 @@ impl From for management::LifecycleRules { persist_row_threshold: config.persist_row_threshold.get() as u64, persist_age_threshold_seconds: config.persist_age_threshold_seconds.get(), mub_row_threshold: config.mub_row_threshold.get() as u64, - parquet_cache_limit: config.parquet_cache_limit.map(|x| ParquetCacheLimit { - value: x.get() as u64, - }), + parquet_cache_limit: config + .parquet_cache_limit + .map(|v| v.get()) + .unwrap_or_default(), } } } @@ -47,11 +47,6 @@ impl TryFrom for LifecycleRules { type Error = FieldViolation; fn try_from(proto: management::LifecycleRules) -> Result { - let parquet_cache_limit = match proto.parquet_cache_limit { - Some(l) => (l.value as usize).try_into().ok(), - None => None, - }; - Ok(Self { buffer_size_soft: (proto.buffer_size_soft as usize).try_into().ok(), buffer_size_hard: (proto.buffer_size_hard as usize).try_into().ok(), @@ -78,7 +73,7 @@ impl TryFrom for LifecycleRules { .unwrap_or_else(|| NonZeroU32::new(DEFAULT_PERSIST_AGE_THRESHOLD_SECONDS).unwrap()), mub_row_threshold: NonZeroUsize::new(proto.mub_row_threshold as usize) .unwrap_or_else(|| NonZeroUsize::new(DEFAULT_MUB_ROW_THRESHOLD).unwrap()), - parquet_cache_limit, + parquet_cache_limit: NonZeroU64::new(proto.parquet_cache_limit), }) } } @@ -103,7 +98,7 @@ mod tests { persist_row_threshold: 57, persist_age_threshold_seconds: 23, mub_row_threshold: 3454, - parquet_cache_limit: Some(ParquetCacheLimit { value: 10 }), + parquet_cache_limit: 10, }; let config: LifecycleRules = protobuf.clone().try_into().unwrap(); @@ -138,7 +133,7 @@ mod tests { assert_eq!(back.mub_row_threshold, protobuf.mub_row_threshold); assert_eq!( config.parquet_cache_limit.unwrap().get(), - protobuf.parquet_cache_limit.as_ref().unwrap().value as usize + protobuf.parquet_cache_limit ); assert_eq!(back.parquet_cache_limit, protobuf.parquet_cache_limit); } diff --git a/src/commands/database.rs b/src/commands/database.rs index 77f25471a5..3ff2c2bbf9 100644 --- a/src/commands/database.rs +++ b/src/commands/database.rs @@ -13,7 +13,6 @@ use influxdb_iox_client::{ }, write::{self, WriteError}, }; -use std::num::NonZeroUsize; mod catalog; mod chunk; @@ -121,9 +120,10 @@ struct Create { #[structopt(long, default_value = "100000")] mub_row_threshold: u64, - /// Use up to this amount of space in bytes for caching Parquet files - #[structopt(long, parse(try_from_str))] - pub parquet_cache_limit: Option, + /// Use up to this amount of space in bytes for caching Parquet files. A + /// value of zero disables Parquet file caching. + #[structopt(long, default_value = "0")] + parquet_cache_limit: u64, } /// Get list of databases @@ -198,7 +198,7 @@ pub async fn command(url: String, config: Config) -> Result<()> { persist_row_threshold: command.persist_row_threshold, persist_age_threshold_seconds: command.persist_age_threshold_seconds, mub_row_threshold: command.mub_row_threshold, - parquet_cache_limit: command.parquet_cache_limit.map(|l| ParquetCacheLimit{value: l.get() as u64}), + parquet_cache_limit: command.parquet_cache_limit, }), // Default to hourly partitions