From b870242ec7f51360d22be4336eb061689a853c7c Mon Sep 17 00:00:00 2001
From: wiedld <dlw405@gmail.com>
Date: Fri, 21 Apr 2023 16:24:46 -0700
Subject: [PATCH 01/25] chore(idpe-17434): remove utf8-percent encoding on v2
 write path, such that it matches v1 writes and onCreate

---
 data_types/src/namespace_name.rs             | 23 ++++++++------------
 router/src/server/http/write/multi_tenant.rs |  6 ++---
 2 files changed, 12 insertions(+), 17 deletions(-)

diff --git a/data_types/src/namespace_name.rs b/data_types/src/namespace_name.rs
index a89230e9b5..e86dcfb216 100644
--- a/data_types/src/namespace_name.rs
+++ b/data_types/src/namespace_name.rs
@@ -1,6 +1,5 @@
 use std::{borrow::Cow, ops::RangeInclusive};
 
-use percent_encoding::{utf8_percent_encode, NON_ALPHANUMERIC};
 use thiserror::Error;
 
 /// Length constraints for a [`NamespaceName`] name.
@@ -123,11 +122,7 @@ impl<'a> NamespaceName<'a> {
             return Err(OrgBucketMappingError::NoOrgBucketSpecified);
         }
 
-        let prefix: Cow<'_, str> = utf8_percent_encode(org, NON_ALPHANUMERIC).into();
-        let suffix: Cow<'_, str> = utf8_percent_encode(bucket, NON_ALPHANUMERIC).into();
-
-        let db_name = format!("{}_{}", prefix, suffix);
-        Ok(Self::new(db_name)?)
+        Ok(Self::new(format!("{}_{}", org, bucket))?)
     }
 }
 
@@ -188,34 +183,34 @@ mod tests {
     #[test]
     fn test_org_bucket_map_db_contains_underscore() {
         let got = NamespaceName::from_org_and_bucket("my_org", "bucket").unwrap();
-        assert_eq!(got.as_str(), "my%5Forg_bucket");
+        assert_eq!(got.as_str(), "my_org_bucket");
 
         let got = NamespaceName::from_org_and_bucket("org", "my_bucket").unwrap();
-        assert_eq!(got.as_str(), "org_my%5Fbucket");
+        assert_eq!(got.as_str(), "org_my_bucket");
 
         let got = NamespaceName::from_org_and_bucket("org", "my__bucket").unwrap();
-        assert_eq!(got.as_str(), "org_my%5F%5Fbucket");
+        assert_eq!(got.as_str(), "org_my__bucket");
 
         let got = NamespaceName::from_org_and_bucket("my_org", "my_bucket").unwrap();
-        assert_eq!(got.as_str(), "my%5Forg_my%5Fbucket");
+        assert_eq!(got.as_str(), "my_org_my_bucket");
     }
 
     #[test]
     fn test_org_bucket_map_db_contains_underscore_and_percent() {
         let got = NamespaceName::from_org_and_bucket("my%5Forg", "bucket").unwrap();
-        assert_eq!(got.as_str(), "my%255Forg_bucket");
+        assert_eq!(got.as_str(), "my%5Forg_bucket");
 
         let got = NamespaceName::from_org_and_bucket("my%5Forg_", "bucket").unwrap();
-        assert_eq!(got.as_str(), "my%255Forg%5F_bucket");
+        assert_eq!(got.as_str(), "my%5Forg__bucket");
     }
 
     #[test]
     fn test_bad_namespace_name_is_encoded() {
         let got = NamespaceName::from_org_and_bucket("org", "bucket?").unwrap();
-        assert_eq!(got.as_str(), "org_bucket%3F");
+        assert_eq!(got.as_str(), "org_bucket?");
 
         let got = NamespaceName::from_org_and_bucket("org!", "bucket").unwrap();
-        assert_eq!(got.as_str(), "org%21_bucket");
+        assert_eq!(got.as_str(), "org!_bucket");
     }
 
     #[test]
diff --git a/router/src/server/http/write/multi_tenant.rs b/router/src/server/http/write/multi_tenant.rs
index f89cc869f1..4cc218ec97 100644
--- a/router/src/server/http/write/multi_tenant.rs
+++ b/router/src/server/http/write/multi_tenant.rs
@@ -171,7 +171,7 @@ mod tests {
             namespace,
             ..
         }) => {
-            assert_eq!(namespace.as_str(), "cool%5Fconfusing_bucket");
+            assert_eq!(namespace.as_str(), "cool_confusing_bucket");
         }
     );
 
@@ -193,7 +193,7 @@ mod tests {
             namespace,
             ..
         }) => {
-            assert_eq!(namespace.as_str(), "cool%27confusing_bucket");
+            assert_eq!(namespace.as_str(), "cool'confusing_bucket");
         }
     );
 
@@ -204,7 +204,7 @@ mod tests {
             namespace,
             ..
         }) => {
-            assert_eq!(namespace.as_str(), "%5Fcoolconfusing_bucket");
+            assert_eq!(namespace.as_str(), "_coolconfusing_bucket");
         }
     );
 

From daabe9663cb197ea5e6a788536b6492f949ee6c7 Mon Sep 17 00:00:00 2001
From: wiedld <dlw405@gmail.com>
Date: Fri, 21 Apr 2023 16:36:00 -0700
Subject: [PATCH 02/25] chore(idpe-17434): make restrictive whitelist of chars
 accepted, for any NamespaceName

---
 data_types/src/namespace_name.rs              | 73 ++++++++++++++-----
 router/src/server/http/write/multi_tenant.rs  | 12 +--
 .../server/http/write/single_tenant/mod.rs    | 32 ++++----
 service_grpc_namespace/src/lib.rs             | 11 ++-
 4 files changed, 87 insertions(+), 41 deletions(-)

diff --git a/data_types/src/namespace_name.rs b/data_types/src/namespace_name.rs
index e86dcfb216..040e8ee9d4 100644
--- a/data_types/src/namespace_name.rs
+++ b/data_types/src/namespace_name.rs
@@ -7,6 +7,13 @@ use thiserror::Error;
 /// A `RangeInclusive` is a closed interval, covering [1, 64]
 const LENGTH_CONSTRAINT: RangeInclusive<usize> = 1..=64;
 
+/// Whitelisted chars for a [`NamespaceName`] name.
+///
+/// '/' | '_' | '-' are utilized by the platforms.
+fn is_whitelisted(c: char) -> bool {
+    c.is_alphanumeric() || matches!(c, '/' | '_' | '-')
+}
+
 /// Errors returned when attempting to construct a [`NamespaceName`] from an org
 /// & bucket string pair.
 #[derive(Debug, Error)]
@@ -40,7 +47,7 @@ pub enum NamespaceNameError {
     /// The provided namespace name contains an unacceptable character.
     #[error(
         "namespace name '{}' contains invalid character, character number {} \
-        is a control which is not allowed",
+        is not whitelisted",
         name,
         bad_char_offset
     )]
@@ -91,7 +98,7 @@ impl<'a> NamespaceName<'a> {
         //
         // NOTE: If changing these characters, please update the error message
         // above.
-        if let Some(bad_char_offset) = name.chars().position(|c| c.is_control()) {
+        if let Some(bad_char_offset) = name.chars().position(|c| !is_whitelisted(c)) {
             return Err(NamespaceNameError::BadChars {
                 bad_char_offset,
                 name: name.to_string(),
@@ -197,20 +204,32 @@ mod tests {
 
     #[test]
     fn test_org_bucket_map_db_contains_underscore_and_percent() {
-        let got = NamespaceName::from_org_and_bucket("my%5Forg", "bucket").unwrap();
-        assert_eq!(got.as_str(), "my%5Forg_bucket");
+        let err = NamespaceName::from_org_and_bucket("my%5Forg", "bucket");
+        assert!(matches!(
+            err,
+            Err(OrgBucketMappingError::InvalidNamespaceName { .. })
+        ));
 
-        let got = NamespaceName::from_org_and_bucket("my%5Forg_", "bucket").unwrap();
-        assert_eq!(got.as_str(), "my%5Forg__bucket");
+        let err = NamespaceName::from_org_and_bucket("my%5Forg_", "bucket");
+        assert!(matches!(
+            err,
+            Err(OrgBucketMappingError::InvalidNamespaceName { .. })
+        ));
     }
 
     #[test]
-    fn test_bad_namespace_name_is_encoded() {
-        let got = NamespaceName::from_org_and_bucket("org", "bucket?").unwrap();
-        assert_eq!(got.as_str(), "org_bucket?");
+    fn test_bad_namespace_name_fails_validation() {
+        let err = NamespaceName::from_org_and_bucket("org", "bucket?");
+        assert!(matches!(
+            err,
+            Err(OrgBucketMappingError::InvalidNamespaceName { .. })
+        ));
 
-        let got = NamespaceName::from_org_and_bucket("org!", "bucket").unwrap();
-        assert_eq!(got.as_str(), "org!_bucket");
+        let err = NamespaceName::from_org_and_bucket("org!", "bucket");
+        assert!(matches!(
+            err,
+            Err(OrgBucketMappingError::InvalidNamespaceName { .. })
+        ));
     }
 
     #[test]
@@ -251,30 +270,50 @@ mod tests {
     #[test]
     fn test_bad_chars_null() {
         let got = NamespaceName::new("example\x00").unwrap_err();
-        assert_eq!(got.to_string() , "namespace name 'example\x00' contains invalid character, character number 7 is a control which is not allowed");
+        assert_eq!(got.to_string() , "namespace name 'example\x00' contains invalid character, character number 7 is not whitelisted");
     }
 
     #[test]
     fn test_bad_chars_high_control() {
         let got = NamespaceName::new("\u{007f}example").unwrap_err();
-        assert_eq!(got.to_string() , "namespace name '\u{007f}example' contains invalid character, character number 0 is a control which is not allowed");
+        assert_eq!(got.to_string() , "namespace name '\u{007f}example' contains invalid character, character number 0 is not whitelisted");
     }
 
     #[test]
     fn test_bad_chars_tab() {
         let got = NamespaceName::new("example\tdb").unwrap_err();
-        assert_eq!(got.to_string() , "namespace name 'example\tdb' contains invalid character, character number 7 is a control which is not allowed");
+        assert_eq!(got.to_string() , "namespace name 'example\tdb' contains invalid character, character number 7 is not whitelisted");
     }
 
     #[test]
     fn test_bad_chars_newline() {
         let got = NamespaceName::new("my_example\ndb").unwrap_err();
-        assert_eq!(got.to_string() , "namespace name 'my_example\ndb' contains invalid character, character number 10 is a control which is not allowed");
+        assert_eq!(got.to_string() , "namespace name 'my_example\ndb' contains invalid character, character number 10 is not whitelisted");
+    }
+
+    #[test]
+    fn test_bad_chars_whitespace() {
+        let got = NamespaceName::new("my_example db").unwrap_err();
+        assert_eq!(got.to_string() , "namespace name 'my_example db' contains invalid character, character number 10 is not whitelisted");
+    }
+
+    #[test]
+    fn test_bad_chars_single_quote() {
+        let got = NamespaceName::new("my_example'db").unwrap_err();
+        assert_eq!(got.to_string() , "namespace name 'my_example\'db' contains invalid character, character number 10 is not whitelisted");
     }
 
     #[test]
     fn test_ok_chars() {
-        let db = NamespaceName::new("my-example-db_with_underscores and spaces").unwrap();
-        assert_eq!(&*db, "my-example-db_with_underscores and spaces");
+        let db =
+            NamespaceName::new("my-example-db_with_underscores/and/fwd/slash/AndCaseSensitive")
+                .unwrap();
+        assert_eq!(
+            &*db,
+            "my-example-db_with_underscores/and/fwd/slash/AndCaseSensitive"
+        );
+
+        let db = NamespaceName::new("a_ã_京").unwrap();
+        assert_eq!(&*db, "a_ã_京");
     }
 }
diff --git a/router/src/server/http/write/multi_tenant.rs b/router/src/server/http/write/multi_tenant.rs
index 4cc218ec97..734777adb9 100644
--- a/router/src/server/http/write/multi_tenant.rs
+++ b/router/src/server/http/write/multi_tenant.rs
@@ -78,6 +78,7 @@ fn parse_v2(req: &Request<Body>) -> Result<WriteParams, MultiTenantExtractError>
 #[cfg(test)]
 mod tests {
     use assert_matches::assert_matches;
+    use data_types::NamespaceNameError;
 
     use super::*;
     use crate::server::http::write::Precision;
@@ -189,12 +190,11 @@ mod tests {
     test_parse_v2!(
         encoded_quotation,
         query_string = "?org=cool'confusing&bucket=bucket",
-        want = Ok(WriteParams {
-            namespace,
-            ..
-        }) => {
-            assert_eq!(namespace.as_str(), "cool'confusing_bucket");
-        }
+        want = Err(Error::MultiTenantError(
+            MultiTenantExtractError::InvalidOrgAndBucket(
+                OrgBucketMappingError::InvalidNamespaceName(NamespaceNameError::BadChars { .. })
+            )
+        ))
     );
 
     test_parse_v2!(
diff --git a/router/src/server/http/write/single_tenant/mod.rs b/router/src/server/http/write/single_tenant/mod.rs
index 17853ea93a..b9adfe576b 100644
--- a/router/src/server/http/write/single_tenant/mod.rs
+++ b/router/src/server/http/write/single_tenant/mod.rs
@@ -381,9 +381,9 @@ mod tests {
     test_parse_v1!(
         encoded_quotation,
         query_string = "?db=ban'anas",
-        want = Ok(WriteParams{ namespace, precision: _ }) => {
-            assert_eq!(namespace.as_str(), "ban'anas");
-        }
+        want = Err(Error::SingleTenantError(
+            SingleTenantExtractError::InvalidNamespace(NamespaceNameError::BadChars { .. })
+        ))
     );
 
     test_parse_v1!(
@@ -483,17 +483,24 @@ mod tests {
         ))
     );
 
-    // Do not encode potentially problematic input.
     test_parse_v2!(
-        no_encoding,
+        url_encoding,
         // URL-encoded input that is decoded in the HTTP layer
-        query_string = "?bucket=cool%2Fconfusing%F0%9F%8D%8C&prg=org",
+        query_string = "?bucket=cool%2Fconfusing&prg=org",
         want = Ok(WriteParams {namespace, ..}) => {
             // Yielding a not-encoded string as the namespace.
-            assert_eq!(namespace.as_str(), "cool/confusing🍌");
+            assert_eq!(namespace.as_str(), "cool/confusing");
         }
     );
 
+    test_parse_v2!(
+        encoded_emoji,
+        query_string = "?bucket=confusing%F0%9F%8D%8C&prg=org",
+        want = Err(Error::SingleTenantError(
+            SingleTenantExtractError::InvalidNamespace(NamespaceNameError::BadChars { .. })
+        ))
+    );
+
     test_parse_v2!(
         org_ignored,
         query_string = "?org=wat&bucket=bananas",
@@ -518,14 +525,11 @@ mod tests {
     );
 
     test_parse_v2!(
-        encoded_quotation,
+        single_quotation,
         query_string = "?bucket=buc'ket",
-        want = Ok(WriteParams {
-            namespace,
-            ..
-        }) => {
-            assert_eq!(namespace.as_str(), "buc'ket");
-        }
+        want = Err(Error::SingleTenantError(
+            SingleTenantExtractError::InvalidNamespace(NamespaceNameError::BadChars { .. })
+        ))
     );
 
     test_parse_v2!(
diff --git a/service_grpc_namespace/src/lib.rs b/service_grpc_namespace/src/lib.rs
index f624838367..3fd96297bb 100644
--- a/service_grpc_namespace/src/lib.rs
+++ b/service_grpc_namespace/src/lib.rs
@@ -633,23 +633,26 @@ mod tests {
 
     test_create_namespace_name!(ok, name = "bananas", want = Ok("bananas"));
 
-    test_create_namespace_name!(multi_byte, name = "🍌", want = Ok("🍌"));
+    test_create_namespace_name!(multi_byte, name = "🍌", want = Err(e) => {
+        assert_eq!(e.code(), Code::InvalidArgument);
+        assert_eq!(e.message(), "namespace name '🍌' contains invalid character, character number 0 is not whitelisted");
+    });
 
     test_create_namespace_name!(
         tab,
         name = "it\tis\ttabtasitc",
         want = Err(e) => {
             assert_eq!(e.code(), Code::InvalidArgument);
-            assert_eq!(e.message(), "namespace name 'it\tis\ttabtasitc' contains invalid character, character number 2 is a control which is not allowed");
+            assert_eq!(e.message(), "namespace name 'it\tis\ttabtasitc' contains invalid character, character number 2 is not whitelisted");
         }
     );
 
     test_create_namespace_name!(
         null,
-        name = "bad \0 bananas",
+        name = "bad\0bananas",
         want = Err(e) => {
             assert_eq!(e.code(), Code::InvalidArgument);
-            assert_eq!(e.message(), "namespace name 'bad \0 bananas' contains invalid character, character number 4 is a control which is not allowed");
+            assert_eq!(e.message(), "namespace name 'bad\0bananas' contains invalid character, character number 3 is not whitelisted");
         }
     );
 

From 2769b523214185beec5ee2f6472a941fc35a9e65 Mon Sep 17 00:00:00 2001
From: Dom Dwyer <dom@itsallbroken.com>
Date: Mon, 24 Apr 2023 13:43:39 +0200
Subject: [PATCH 03/25] feat(ingester): blocked write load shedding

This commit bounds the duration of time a write may take to be processed
after being added to the WAL for increased reliability.

In ideal cases, anything that is added to the WAL is buffered /
completed to prevent WAL replay materialising writes that never
completed in the first place (#7111), but in some cases (i.e. catalog
outage, etc) the write becomes blocked, stuck in a retry loop or
otherwise never making progress.

When a write is blocked, the data within it remains in RAM, and the
overhead of the spawned task retrying also consumes CPU resources. If
all writes are blocked for sufficient time, these limitless writes keep
building up until a resource is exhausted (i.e. RAM -> OOM) causing an
outage of the ingester.

Instead, this change allocates a write at most 15 seconds to complete,
before it is cancelled and an error returned to the user (if they're
still connected) to shed load. The error message "buffer apply request
timeout" was chosen to be uniquely identifying - systems have lots of
timeouts!

Normally the inner.apply() call at this point completes in microseconds,
so 15 seconds is more than long enough to avoid shedding legitimate
load.
---
 ingester2/src/dml_sink/trait.rs        |  5 ++
 ingester2/src/server/grpc/rpc_write.rs |  1 +
 ingester2/src/wal/wal_sink.rs          | 95 ++++++++++++++++++++++++--
 3 files changed, 95 insertions(+), 6 deletions(-)

diff --git a/ingester2/src/dml_sink/trait.rs b/ingester2/src/dml_sink/trait.rs
index 9b635de0b3..b46338e766 100644
--- a/ingester2/src/dml_sink/trait.rs
+++ b/ingester2/src/dml_sink/trait.rs
@@ -16,6 +16,11 @@ pub enum DmlError {
     /// An error appending the [`DmlOperation`] to the write-ahead log.
     #[error("wal commit failure: {0}")]
     Wal(String),
+
+    /// The write has hit an internal timeout designed to prevent writes from
+    /// retrying indefinitely.
+    #[error("buffer apply request timeout")]
+    ApplyTimeout,
 }
 
 /// A [`DmlSink`] handles [`DmlOperation`] instances in some abstract way.
diff --git a/ingester2/src/server/grpc/rpc_write.rs b/ingester2/src/server/grpc/rpc_write.rs
index 8ced0359ea..00a1b87d63 100644
--- a/ingester2/src/server/grpc/rpc_write.rs
+++ b/ingester2/src/server/grpc/rpc_write.rs
@@ -63,6 +63,7 @@ impl From<DmlError> for tonic::Status {
         match e {
             DmlError::Buffer(e) => map_write_error(e),
             DmlError::Wal(_) => Self::internal(e.to_string()),
+            DmlError::ApplyTimeout => Self::internal(e.to_string()),
         }
     }
 }
diff --git a/ingester2/src/wal/wal_sink.rs b/ingester2/src/wal/wal_sink.rs
index 8fe67fd1ea..f09dd271f9 100644
--- a/ingester2/src/wal/wal_sink.rs
+++ b/ingester2/src/wal/wal_sink.rs
@@ -2,7 +2,7 @@ use async_trait::async_trait;
 use dml::DmlOperation;
 use generated_types::influxdata::iox::wal::v1::sequenced_wal_op::Op;
 use mutable_batch_pb::encode::encode_write;
-use std::sync::Arc;
+use std::{sync::Arc, time::Duration};
 use tokio::sync::watch::Receiver;
 use wal::{SequencedWalOp, WriteResult};
 
@@ -13,6 +13,17 @@ use crate::{
 
 use super::traits::WalAppender;
 
+/// [`DELEGATE_APPLY_TIMEOUT`] defines how long the inner [`DmlSink`] is given
+/// to complete the write [`DmlSink::apply()`] call.
+///
+/// If this limit weren't enforced, a write that does not make progress would
+/// consume resources forever. Instead, a reasonable duration of time is given
+/// to attempt the write before an error is returned to the caller.
+///
+/// In practice, this limit SHOULD only ever be reached as a symptom of a larger
+/// problem (catalog unavailable, etc) preventing a write from making progress.
+const DELEGATE_APPLY_TIMEOUT: Duration = Duration::from_secs(15);
+
 /// A [`DmlSink`] decorator that ensures any [`DmlOperation`] is committed to
 /// the write-ahead log before passing the operation to the inner [`DmlSink`].
 #[derive(Debug)]
@@ -49,7 +60,11 @@ where
         // durable.
         //
         // Ensure that this future is always driven to completion now that the
-        // WAL entry is being committed, otherwise they'll diverge.
+        // WAL entry is being committed, otherwise they'll diverge. At the same
+        // time, do not allow the spawned task to live forever, consuming
+        // resources without making progress - instead shed load after a
+        // reasonable duration of time (DELEGATE_APPLY_TIMEOUT) has passed,
+        // before returning a write error (if the caller is still listening).
         //
         // If this buffer apply fails, the entry remains in the WAL and will be
         // attempted again during WAL replay after a crash. If this can never
@@ -58,9 +73,14 @@ where
         //  https://github.com/influxdata/influxdb_iox/issues/7111
         //
         let inner = self.inner.clone();
-        CancellationSafe::new(async move { inner.apply(op).await })
-            .await
-            .map_err(Into::into)?;
+        CancellationSafe::new(async move {
+            let res = tokio::time::timeout(DELEGATE_APPLY_TIMEOUT, inner.apply(op))
+                .await
+                .map_err(|_| DmlError::ApplyTimeout)?;
+
+            res.map_err(Into::into)
+        })
+        .await?;
 
         // Wait for the write to be durable before returning to the user
         write_result
@@ -101,7 +121,8 @@ impl WalAppender for Arc<wal::Wal> {
 
 #[cfg(test)]
 mod tests {
-    use std::sync::Arc;
+    use core::{future::Future, marker::Send, pin::Pin};
+    use std::{future, sync::Arc};
 
     use assert_matches::assert_matches;
     use data_types::{NamespaceId, PartitionKey, TableId};
@@ -181,4 +202,66 @@ mod tests {
 
         assert_eq!(want, *payload);
     }
+
+    /// A [`DmlSink`] implementation that hangs forever and never completes.
+    #[derive(Debug, Default, Clone)]
+    struct BlockingDmlSink;
+
+    impl DmlSink for BlockingDmlSink {
+        type Error = DmlError;
+
+        fn apply<'life0, 'async_trait>(
+            &'life0 self,
+            _op: DmlOperation,
+        ) -> Pin<Box<dyn Future<Output = Result<(), Self::Error>> + Send + 'async_trait>>
+        where
+            'life0: 'async_trait,
+            Self: 'async_trait,
+        {
+            Box::pin(future::pending())
+        }
+    }
+
+    #[tokio::test]
+    async fn test_timeout() {
+        let dir = tempfile::tempdir().unwrap();
+
+        // Generate the test op
+        let op = make_write_op(
+            &PartitionKey::from("p1"),
+            NAMESPACE_ID,
+            TABLE_NAME,
+            TABLE_ID,
+            42,
+            r#"bananas,region=Madrid temp=35 4242424242"#,
+        );
+
+        let wal = Wal::new(dir.path())
+            .await
+            .expect("failed to initialise WAL");
+
+        let wal_sink = WalSink::new(BlockingDmlSink::default(), wal);
+
+        // Allow tokio to automatically advance time past the timeout duration,
+        // when all threads are blocked on await points.
+        //
+        // This allows the test to drive the timeout logic without actually
+        // waiting for the timeout duration in the test.
+        tokio::time::pause();
+
+        let start = tokio::time::Instant::now();
+
+        // Apply the op through the decorator, which should time out
+        let err = wal_sink
+            .apply(DmlOperation::Write(op.clone()))
+            .await
+            .expect_err("write should time out");
+
+        assert_matches!(err, DmlError::ApplyTimeout);
+
+        // Ensure that "time" advanced at least the timeout amount of time
+        // before erroring.
+        let duration = tokio::time::Instant::now().duration_since(start);
+        assert!(duration > DELEGATE_APPLY_TIMEOUT);
+    }
 }

From 7b7101b53faa5c92360d2495641bb6990cd0b987 Mon Sep 17 00:00:00 2001
From: "Carol (Nichols || Goulding)" <carol.nichols@gmail.com>
Date: Fri, 24 Feb 2023 13:54:20 -0500
Subject: [PATCH 04/25] fix: In DmlMeta, only use SequenceNumber rather than
 Sequence (shard index plus seq num)

---
 data_types/src/lib.rs                  | 19 -------------------
 dml/src/lib.rs                         | 18 +++++++++---------
 ingester2/benches/write.rs             |  4 ++--
 ingester2/src/buffer_tree/namespace.rs |  3 +--
 ingester2/src/dml_sink/tracing.rs      |  4 ++--
 ingester2/src/init/wal_replay.rs       |  8 ++------
 ingester2/src/server/grpc/rpc_write.rs | 14 +++++---------
 ingester2/src/test_util.rs             | 13 ++++---------
 ingester2/src/wal/wal_sink.rs          |  1 -
 ingester2_test_ctx/src/lib.rs          |  8 +++-----
 10 files changed, 28 insertions(+), 64 deletions(-)

diff --git a/data_types/src/lib.rs b/data_types/src/lib.rs
index 82d3198486..d6006761c3 100644
--- a/data_types/src/lib.rs
+++ b/data_types/src/lib.rs
@@ -2039,25 +2039,6 @@ impl TableSummary {
     }
 }
 
-/// Shard index plus offset
-#[derive(Debug, Copy, Clone, Eq, PartialEq)]
-pub struct Sequence {
-    /// The shard index
-    pub shard_index: ShardIndex,
-    /// The sequence number
-    pub sequence_number: SequenceNumber,
-}
-
-impl Sequence {
-    /// Create a new Sequence
-    pub fn new(shard_index: ShardIndex, sequence_number: SequenceNumber) -> Self {
-        Self {
-            shard_index,
-            sequence_number,
-        }
-    }
-}
-
 /// minimum time that can be represented.
 ///
 /// 1677-09-21 00:12:43.145224194 +0000 UTC
diff --git a/dml/src/lib.rs b/dml/src/lib.rs
index 9cc8a541f3..02849b3254 100644
--- a/dml/src/lib.rs
+++ b/dml/src/lib.rs
@@ -16,8 +16,8 @@
 use std::time::Duration;
 
 use data_types::{
-    DeletePredicate, NamespaceId, NonEmptyString, PartitionKey, Sequence, StatValues, Statistics,
-    TableId,
+    DeletePredicate, NamespaceId, NonEmptyString, PartitionKey, SequenceNumber, StatValues,
+    Statistics, TableId,
 };
 use hashbrown::HashMap;
 use iox_time::{Time, TimeProvider};
@@ -28,7 +28,7 @@ use trace::ctx::SpanContext;
 #[derive(Debug, Default, Clone, PartialEq)]
 pub struct DmlMeta {
     /// The sequence number associated with this write
-    sequence: Option<Sequence>,
+    sequence_number: Option<SequenceNumber>,
 
     /// When this write was ingested into the write buffer
     producer_ts: Option<Time>,
@@ -43,13 +43,13 @@ pub struct DmlMeta {
 impl DmlMeta {
     /// Create a new [`DmlMeta`] for a sequenced operation
     pub fn sequenced(
-        sequence: Sequence,
+        sequence_number: SequenceNumber,
         producer_ts: Time,
         span_ctx: Option<SpanContext>,
         bytes_read: usize,
     ) -> Self {
         Self {
-            sequence: Some(sequence),
+            sequence_number: Some(sequence_number),
             producer_ts: Some(producer_ts),
             span_ctx,
             bytes_read: Some(bytes_read),
@@ -59,7 +59,7 @@ impl DmlMeta {
     /// Create a new [`DmlMeta`] for an unsequenced operation
     pub fn unsequenced(span_ctx: Option<SpanContext>) -> Self {
         Self {
-            sequence: None,
+            sequence_number: None,
             producer_ts: None,
             span_ctx,
             bytes_read: None,
@@ -67,8 +67,8 @@ impl DmlMeta {
     }
 
     /// Gets the sequence number associated with the write if any
-    pub fn sequence(&self) -> Option<&Sequence> {
-        self.sequence.as_ref()
+    pub fn sequence(&self) -> Option<SequenceNumber> {
+        self.sequence_number
     }
 
     /// Gets the producer timestamp associated with the write if any
@@ -438,7 +438,7 @@ pub mod test_util {
             Time::from_timestamp_millis(timestamp.timestamp_millis()).expect("ts in range");
 
         DmlMeta::sequenced(
-            *m.sequence().unwrap(),
+            m.sequence().unwrap(),
             timestamp,
             m.span_context().cloned(),
             m.bytes_read().unwrap(),
diff --git a/ingester2/benches/write.rs b/ingester2/benches/write.rs
index 76e482d698..5f9871b55a 100644
--- a/ingester2/benches/write.rs
+++ b/ingester2/benches/write.rs
@@ -1,7 +1,7 @@
 use std::sync::Arc;
 
 use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput};
-use data_types::{PartitionKey, Sequence, SequenceNumber, ShardIndex};
+use data_types::{PartitionKey, SequenceNumber};
 use dml::{DmlMeta, DmlWrite};
 use futures::{stream::FuturesUnordered, StreamExt};
 use generated_types::influxdata::{
@@ -67,7 +67,7 @@ async fn init(lp: impl AsRef<str>) -> (TestContext<impl IngesterRpcInterface>, D
         batches_by_ids,
         PartitionKey::from(PARTITION_KEY),
         DmlMeta::sequenced(
-            Sequence::new(ShardIndex::new(42), SequenceNumber::new(42)),
+            SequenceNumber::new(42),
             iox_time::SystemProvider::new().now(),
             None,
             50,
diff --git a/ingester2/src/buffer_tree/namespace.rs b/ingester2/src/buffer_tree/namespace.rs
index b886c6fb31..8bcd41742e 100644
--- a/ingester2/src/buffer_tree/namespace.rs
+++ b/ingester2/src/buffer_tree/namespace.rs
@@ -147,8 +147,7 @@ where
         let sequence_number = op
             .meta()
             .sequence()
-            .expect("applying unsequenced op")
-            .sequence_number;
+            .expect("applying unsequenced op");
 
         match op {
             DmlOperation::Write(write) => {
diff --git a/ingester2/src/dml_sink/tracing.rs b/ingester2/src/dml_sink/tracing.rs
index 97e6873a29..d7a2a96f18 100644
--- a/ingester2/src/dml_sink/tracing.rs
+++ b/ingester2/src/dml_sink/tracing.rs
@@ -130,7 +130,7 @@ mod tests {
         // Populate the metadata with a span context.
         let meta = op.meta();
         op.set_meta(DmlMeta::sequenced(
-            *meta.sequence().unwrap(),
+            meta.sequence().unwrap(),
             meta.producer_ts().unwrap(),
             Some(span),
             42,
@@ -166,7 +166,7 @@ mod tests {
         // Populate the metadata with a span context.
         let meta = op.meta();
         op.set_meta(DmlMeta::sequenced(
-            *meta.sequence().unwrap(),
+            meta.sequence().unwrap(),
             meta.producer_ts().unwrap(),
             Some(span),
             42,
diff --git a/ingester2/src/init/wal_replay.rs b/ingester2/src/init/wal_replay.rs
index a8bdad10ab..ddf184ddd9 100644
--- a/ingester2/src/init/wal_replay.rs
+++ b/ingester2/src/init/wal_replay.rs
@@ -1,4 +1,4 @@
-use data_types::{NamespaceId, PartitionKey, Sequence, SequenceNumber, TableId};
+use data_types::{NamespaceId, PartitionKey, SequenceNumber, TableId};
 use dml::{DmlMeta, DmlOperation, DmlWrite};
 use generated_types::influxdata::iox::wal::v1::sequenced_wal_op::Op;
 use metric::U64Counter;
@@ -12,7 +12,6 @@ use crate::{
     dml_sink::{DmlError, DmlSink},
     partition_iter::PartitionIter,
     persist::{drain_buffer::persist_partitions, queue::PersistQueue},
-    TRANSITION_SHARD_INDEX,
 };
 
 /// Errors returned when replaying the write-ahead log.
@@ -235,10 +234,7 @@ where
                 partition_key,
                 // The tracing context should be propagated over the RPC boundary.
                 DmlMeta::sequenced(
-                    Sequence {
-                        shard_index: TRANSITION_SHARD_INDEX, // TODO: remove this from DmlMeta
-                        sequence_number,
-                    },
+                    sequence_number,
                     iox_time::Time::MAX, // TODO: remove this from DmlMeta
                     // TODO: A tracing context should be added for WAL replay.
                     None,
diff --git a/ingester2/src/server/grpc/rpc_write.rs b/ingester2/src/server/grpc/rpc_write.rs
index 00a1b87d63..930817a1ec 100644
--- a/ingester2/src/server/grpc/rpc_write.rs
+++ b/ingester2/src/server/grpc/rpc_write.rs
@@ -1,6 +1,6 @@
 use std::sync::Arc;
 
-use data_types::{NamespaceId, PartitionKey, Sequence, TableId};
+use data_types::{NamespaceId, PartitionKey, TableId};
 use dml::{DmlMeta, DmlOperation, DmlWrite};
 use generated_types::influxdata::iox::ingester::v1::{
     self as proto, write_service_server::WriteService,
@@ -15,7 +15,6 @@ use crate::{
     dml_sink::{DmlError, DmlSink},
     ingest_state::{IngestState, IngestStateError},
     timestamp_oracle::TimestampOracle,
-    TRANSITION_SHARD_INDEX,
 };
 
 /// A list of error states when handling an RPC write request.
@@ -189,10 +188,7 @@ where
                 .collect(),
             partition_key,
             DmlMeta::sequenced(
-                Sequence {
-                    shard_index: TRANSITION_SHARD_INDEX, // TODO: remove this from DmlMeta
-                    sequence_number: self.timestamp.next(),
-                },
+                self.timestamp.next(),
                 iox_time::Time::MAX, // TODO: remove this from DmlMeta
                 // The tracing context should be propagated over the RPC boundary.
                 //
@@ -297,7 +293,7 @@ mod tests {
             assert_eq!(w.namespace_id(), NAMESPACE_ID);
             assert_eq!(w.table_count(), 1);
             assert_eq!(*w.partition_key(), PartitionKey::from(PARTITION_KEY));
-            assert_eq!(w.meta().sequence().unwrap().sequence_number.get(), 1);
+            assert_eq!(w.meta().sequence().unwrap().get(), 1);
         }
     );
 
@@ -405,8 +401,8 @@ mod tests {
         assert_matches!(
             *mock.get_calls(),
             [DmlOperation::Write(ref w1), DmlOperation::Write(ref w2)] => {
-                let w1 = w1.meta().sequence().unwrap().sequence_number.get();
-                let w2 = w2.meta().sequence().unwrap().sequence_number.get();
+                let w1 = w1.meta().sequence().unwrap().get();
+                let w2 = w2.meta().sequence().unwrap().get();
                 assert!(w1 < w2);
             }
         );
diff --git a/ingester2/src/test_util.rs b/ingester2/src/test_util.rs
index c2a15253e5..16a080743f 100644
--- a/ingester2/src/test_util.rs
+++ b/ingester2/src/test_util.rs
@@ -1,8 +1,6 @@
 use std::collections::BTreeMap;
 
-use data_types::{
-    NamespaceId, PartitionKey, Sequence, SequenceNumber, ShardId, ShardIndex, TableId,
-};
+use data_types::{NamespaceId, PartitionKey, SequenceNumber, ShardId, ShardIndex, TableId};
 use dml::{DmlMeta, DmlWrite};
 use iox_catalog::interface::Catalog;
 use mutable_batch_lp::lines_to_batches;
@@ -147,10 +145,7 @@ pub(crate) fn make_write_op(
         tables_by_id,
         partition_key.clone(),
         DmlMeta::sequenced(
-            Sequence {
-                shard_index: ShardIndex::new(i32::MAX),
-                sequence_number: SequenceNumber::new(sequence_number),
-            },
+            SequenceNumber::new(sequence_number),
             iox_time::Time::MIN,
             None,
             42,
@@ -195,8 +190,8 @@ pub(crate) fn assert_dml_writes_eq(a: DmlWrite, b: DmlWrite) {
     assert_eq!(a.partition_key(), b.partition_key(), "partition key");
 
     // Assert sequence numbers were reassigned
-    let seq_a = a.meta().sequence().map(|s| s.sequence_number);
-    let seq_b = b.meta().sequence().map(|s| s.sequence_number);
+    let seq_a = a.meta().sequence();
+    let seq_b = b.meta().sequence();
     assert_eq!(seq_a, seq_b, "sequence numbers differ");
 
     let a = a.into_tables().collect::<BTreeMap<_, _>>();
diff --git a/ingester2/src/wal/wal_sink.rs b/ingester2/src/wal/wal_sink.rs
index f09dd271f9..7cd12c1c26 100644
--- a/ingester2/src/wal/wal_sink.rs
+++ b/ingester2/src/wal/wal_sink.rs
@@ -102,7 +102,6 @@ impl WalAppender for Arc<wal::Wal> {
             .meta()
             .sequence()
             .expect("committing unsequenced dml operation to wal")
-            .sequence_number
             .get() as u64;
 
         let namespace_id = op.namespace_id();
diff --git a/ingester2_test_ctx/src/lib.rs b/ingester2_test_ctx/src/lib.rs
index ac38e98df3..9e89bccbd0 100644
--- a/ingester2_test_ctx/src/lib.rs
+++ b/ingester2_test_ctx/src/lib.rs
@@ -17,8 +17,8 @@ use std::{collections::HashMap, sync::Arc, time::Duration};
 use arrow::record_batch::RecordBatch;
 use arrow_flight::{decode::FlightRecordBatchStream, flight_service_server::FlightService, Ticket};
 use data_types::{
-    Namespace, NamespaceId, NamespaceSchema, ParquetFile, PartitionKey, QueryPoolId, Sequence,
-    SequenceNumber, ShardIndex, TableId, TopicId,
+    Namespace, NamespaceId, NamespaceSchema, ParquetFile, PartitionKey, QueryPoolId,
+    SequenceNumber, TableId, TopicId,
 };
 use dml::{DmlMeta, DmlWrite};
 use futures::{stream::FuturesUnordered, FutureExt, StreamExt, TryStreamExt};
@@ -179,8 +179,6 @@ impl TestContextBuilder {
     }
 }
 
-const SHARD_INDEX: ShardIndex = ShardIndex::new(42);
-
 /// A command interface to the underlying [`ingester2`] instance.
 ///
 /// When the [`TestContext`] is dropped, the underlying [`ingester2`] instance
@@ -309,7 +307,7 @@ where
             batches_by_ids,
             partition_key,
             DmlMeta::sequenced(
-                Sequence::new(SHARD_INDEX, SequenceNumber::new(sequence_number)),
+                SequenceNumber::new(sequence_number),
                 iox_time::SystemProvider::new().now(),
                 None,
                 50,

From 8e7e22d16783c7548d0870bd9f04af56c3ff817f Mon Sep 17 00:00:00 2001
From: Phil Bracikowski <13472206+philjb@users.noreply.github.com>
Date: Mon, 24 Apr 2023 09:57:18 -0700
Subject: [PATCH 05/25] chore(garbage collector): info logging for deleting
 parquet file (#7638)

* for influxdata/idpe#17451
* follow up to #7562
---
 garbage_collector/src/objectstore/deleter.rs | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/garbage_collector/src/objectstore/deleter.rs b/garbage_collector/src/objectstore/deleter.rs
index abc5963e0d..a1255ca426 100644
--- a/garbage_collector/src/objectstore/deleter.rs
+++ b/garbage_collector/src/objectstore/deleter.rs
@@ -1,6 +1,6 @@
 use futures::{StreamExt, TryStreamExt};
 use object_store::{DynObjectStore, ObjectMeta};
-use observability_deps::tracing::{debug, info};
+use observability_deps::tracing::info;
 use snafu::prelude::*;
 use std::sync::Arc;
 use tokio::sync::mpsc;
@@ -23,7 +23,7 @@ pub(crate) async fn perform(
                     info!(?path, "Not deleting due to dry run");
                     Ok(())
                 } else {
-                    debug!("Deleting {path}");
+                    info!("Deleting {path}");
                     object_store
                         .delete(&path)
                         .await

From 2d592a0295f23c97d6bcfa5f3bb15545fd842237 Mon Sep 17 00:00:00 2001
From: "Christopher M. Wolff" <chris.wolff@influxdata.com>
Date: Mon, 24 Apr 2023 10:09:22 -0700
Subject: [PATCH 06/25] feat: teach parser to allow arithmetic functions
 (#7620)

* feat: teach parser to allow arithmetic functions

* refactor: remove unnecessary lowercasing of fn names

---------

Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com>
---
 .../src/expression/conditional.rs             | 31 +++++---
 influxdb_influxql_parser/src/functions.rs     | 74 +++++++++++++++++++
 influxdb_influxql_parser/src/lib.rs           |  1 +
 influxdb_influxql_parser/src/select.rs        | 12 ++-
 .../query_tests2/cases/in/issue_6112.influxql |  6 ++
 .../cases/in/issue_6112.influxql.expected     |  9 +++
 iox_query_influxql/src/plan/planner.rs        | 68 +----------------
 .../src/plan/planner_time_range_expression.rs |  3 +-
 iox_query_influxql/src/plan/rewriter.rs       |  2 +-
 9 files changed, 126 insertions(+), 80 deletions(-)
 create mode 100644 influxdb_influxql_parser/src/functions.rs

diff --git a/influxdb_influxql_parser/src/expression/conditional.rs b/influxdb_influxql_parser/src/expression/conditional.rs
index ba9b3f5ccc..831a62e829 100644
--- a/influxdb_influxql_parser/src/expression/conditional.rs
+++ b/influxdb_influxql_parser/src/expression/conditional.rs
@@ -3,11 +3,12 @@ use crate::expression::arithmetic::{
     arithmetic, call_expression, var_ref, ArithmeticParsers, Expr,
 };
 use crate::expression::Call;
-use crate::internal::Error as InternalError;
-use crate::internal::{expect, verify, ParseResult};
+use crate::functions::is_scalar_math_function;
+use crate::internal::{expect, verify, Error as InternalError, ParseResult};
 use crate::keywords::keyword;
 use crate::literal::{literal_no_regex, literal_regex, Literal};
 use crate::parameter::parameter;
+use crate::select::is_valid_now_call;
 use nom::branch::alt;
 use nom::bytes::complete::tag;
 use nom::character::complete::char;
@@ -289,21 +290,23 @@ fn reduce_expr(
     })
 }
 
-/// Returns true if `expr` is a valid [`Expr::Call`] expression for the `now` function.
-pub(crate) fn is_valid_now_call(expr: &Expr) -> bool {
-    match expr {
-        Expr::Call(Call { name, args }) => name.to_lowercase() == "now" && args.is_empty(),
-        _ => false,
-    }
+/// Returns true if `expr` is a valid [`Expr::Call`] expression for condtional expressions
+/// in the WHERE clause.
+pub(crate) fn is_valid_conditional_call(expr: &Expr) -> bool {
+    is_valid_now_call(expr)
+        || match expr {
+            Expr::Call(Call { name, .. }) => is_scalar_math_function(name),
+            _ => false,
+        }
 }
 
 impl ConditionalExpression {
     /// Parse the `now()` function call
     fn call(i: &str) -> ParseResult<&str, Expr> {
         verify(
-            "invalid expression, the only valid function call is 'now' with no arguments",
+            "invalid expression, the only valid function calls are 'now' with no arguments, or scalar math functions",
             call_expression::<Self>,
-            is_valid_now_call,
+            is_valid_conditional_call,
         )(i)
     }
 }
@@ -384,16 +387,20 @@ mod test {
         let (_, got) = arithmetic_expression("now() + 3").unwrap();
         assert_eq!(got, binary_op!(call!("now"), Add, 3));
 
+        // arithmetic functions calls are permitted
+        let (_, got) = arithmetic_expression("abs(f) + 3").unwrap();
+        assert_eq!(got, binary_op!(call!("abs", var_ref!("f")), Add, 3));
+
         // Fallible cases
 
         assert_expect_error!(
             arithmetic_expression("sum(foo)"),
-            "invalid expression, the only valid function call is 'now' with no arguments"
+            "invalid expression, the only valid function calls are 'now' with no arguments, or scalar math functions"
         );
 
         assert_expect_error!(
             arithmetic_expression("now(1)"),
-            "invalid expression, the only valid function call is 'now' with no arguments"
+            "invalid expression, the only valid function calls are 'now' with no arguments, or scalar math functions"
         );
     }
 
diff --git a/influxdb_influxql_parser/src/functions.rs b/influxdb_influxql_parser/src/functions.rs
new file mode 100644
index 0000000000..b42103e9dc
--- /dev/null
+++ b/influxdb_influxql_parser/src/functions.rs
@@ -0,0 +1,74 @@
+//! # [Functions] supported by InfluxQL
+//!
+//! [Functions]: https://docs.influxdata.com/influxdb/v1.8/query_language/functions/
+
+use std::collections::HashSet;
+
+use once_cell::sync::Lazy;
+
+/// Returns `true` if `name` is a mathematical scalar function
+/// supported by InfluxQL.
+pub fn is_scalar_math_function(name: &str) -> bool {
+    static FUNCTIONS: Lazy<HashSet<&'static str>> = Lazy::new(|| {
+        HashSet::from([
+            "abs", "sin", "cos", "tan", "asin", "acos", "atan", "atan2", "exp", "log", "ln",
+            "log2", "log10", "sqrt", "pow", "floor", "ceil", "round",
+        ])
+    });
+
+    FUNCTIONS.contains(name)
+}
+
+/// Returns `true` if `name` is an aggregate or aggregate function
+/// supported by InfluxQL.
+pub fn is_aggregate_function(name: &str) -> bool {
+    static FUNCTIONS: Lazy<HashSet<&'static str>> = Lazy::new(|| {
+        HashSet::from([
+            // Scalar-like functions
+            "cumulative_sum",
+            "derivative",
+            "difference",
+            "elapsed",
+            "moving_average",
+            "non_negative_derivative",
+            "non_negative_difference",
+            // Selector functions
+            "bottom",
+            "first",
+            "last",
+            "max",
+            "min",
+            "percentile",
+            "sample",
+            "top",
+            // Aggregate functions
+            "count",
+            "integral",
+            "mean",
+            "median",
+            "mode",
+            "spread",
+            "stddev",
+            "sum",
+            // Prediction functions
+            "holt_winters",
+            "holt_winters_with_fit",
+            // Technical analysis functions
+            "chande_momentum_oscillator",
+            "exponential_moving_average",
+            "double_exponential_moving_average",
+            "kaufmans_efficiency_ratio",
+            "kaufmans_adaptive_moving_average",
+            "triple_exponential_moving_average",
+            "triple_exponential_derivative",
+            "relative_strength_index",
+        ])
+    });
+
+    FUNCTIONS.contains(name)
+}
+
+/// Returns `true` if `name` is `"now"`.
+pub fn is_now_function(name: &str) -> bool {
+    name == "now"
+}
diff --git a/influxdb_influxql_parser/src/lib.rs b/influxdb_influxql_parser/src/lib.rs
index 765a31d68e..58adfebb62 100644
--- a/influxdb_influxql_parser/src/lib.rs
+++ b/influxdb_influxql_parser/src/lib.rs
@@ -30,6 +30,7 @@ pub mod delete;
 pub mod drop;
 pub mod explain;
 pub mod expression;
+pub mod functions;
 pub mod identifier;
 mod internal;
 mod keywords;
diff --git a/influxdb_influxql_parser/src/select.rs b/influxdb_influxql_parser/src/select.rs
index 35ea92c874..0a138e1b89 100644
--- a/influxdb_influxql_parser/src/select.rs
+++ b/influxdb_influxql_parser/src/select.rs
@@ -11,8 +11,8 @@ use crate::expression::arithmetic::Expr::Wildcard;
 use crate::expression::arithmetic::{
     arithmetic, call_expression, var_ref, ArithmeticParsers, Expr, WildcardType,
 };
-use crate::expression::conditional::is_valid_now_call;
-use crate::expression::VarRef;
+use crate::expression::{Call, VarRef};
+use crate::functions::is_now_function;
 use crate::identifier::{identifier, Identifier};
 use crate::impl_tuple_clause;
 use crate::internal::{expect, map_fail, verify, ParseResult};
@@ -301,6 +301,14 @@ impl ArithmeticParsers for TimeCallIntervalArgument {
 /// The offset argument accepts either a duration, datetime-like string or `now`.
 struct TimeCallOffsetArgument;
 
+/// Returns true if `expr` is a valid [`Expr::Call`] expression for the `now` function.
+pub(crate) fn is_valid_now_call(expr: &Expr) -> bool {
+    match expr {
+        Expr::Call(Call { name, args }) => is_now_function(&name.to_lowercase()) && args.is_empty(),
+        _ => false,
+    }
+}
+
 impl TimeCallOffsetArgument {
     /// Parse the `now()` function call
     fn now_call(i: &str) -> ParseResult<&str, Expr> {
diff --git a/influxdb_iox/tests/query_tests2/cases/in/issue_6112.influxql b/influxdb_iox/tests/query_tests2/cases/in/issue_6112.influxql
index 90474622eb..2af3ffb2ff 100644
--- a/influxdb_iox/tests/query_tests2/cases/in/issue_6112.influxql
+++ b/influxdb_iox/tests/query_tests2/cases/in/issue_6112.influxql
@@ -233,6 +233,12 @@ SELECT f64 FROM m0 WHERE f64 >= 19.5 AND non_existent = 1;
 SELECT f64 FROM m0 WHERE f64 >= 19.5 AND f64 =~ /foo/;
 SELECT f64 FROM m0 WHERE f64 >= 19.5 OR f64 =~ /foo/;
 
+-- arithmetic scalar function calls work
+SELECT time, floor(f64) FROM m0 WHERE floor(f64) = 19.0;
+
+-- aggregate function calls produce an error
+SELECT *, floor(f64) FROM m0 WHERE sum(f64) > 100.0;
+
 --
 -- Validate column expressions
 --
diff --git a/influxdb_iox/tests/query_tests2/cases/in/issue_6112.influxql.expected b/influxdb_iox/tests/query_tests2/cases/in/issue_6112.influxql.expected
index 196798255f..9581b93e23 100644
--- a/influxdb_iox/tests/query_tests2/cases/in/issue_6112.influxql.expected
+++ b/influxdb_iox/tests/query_tests2/cases/in/issue_6112.influxql.expected
@@ -439,6 +439,15 @@ name: m0
 +---------------------+------+
 | 2022-10-31T02:00:10 | 21.2 |
 +---------------------+------+
+-- InfluxQL: SELECT time, floor(f64) FROM m0 WHERE floor(f64) = 19.0;
+name: m0
++---------------------+-------+
+| time                | floor |
++---------------------+-------+
+| 2022-10-31T02:00:30 | 19.0  |
++---------------------+-------+
+-- InfluxQL: SELECT *, floor(f64) FROM m0 WHERE sum(f64) > 100.0;
+Error while planning query: Error during planning: invalid expression, the only valid function calls are 'now' with no arguments, or scalar math functions at pos 35
 -- InfluxQL: SELECT tag0, f64, f64 * 0.5, f64 + str FROM m0 WHERE f64 > 19;
 name: m0
 +---------------------+-------+------+-------+---------+
diff --git a/iox_query_influxql/src/plan/planner.rs b/iox_query_influxql/src/plan/planner.rs
index b7140c3007..a3f10166bc 100644
--- a/iox_query_influxql/src/plan/planner.rs
+++ b/iox_query_influxql/src/plan/planner.rs
@@ -41,6 +41,9 @@ use influxdb_influxql_parser::expression::{
     Binary, Call, ConditionalBinary, ConditionalExpression, ConditionalOperator, VarRef,
     VarRefDataType,
 };
+use influxdb_influxql_parser::functions::{
+    is_aggregate_function, is_now_function, is_scalar_math_function,
+};
 use influxdb_influxql_parser::select::{
     FillClause, GroupByClause, SLimitClause, SOffsetClause, TimeZoneClause,
 };
@@ -65,7 +68,6 @@ use iox_query::exec::IOxSessionContext;
 use iox_query::logical_optimizer::range_predicate::find_time_range;
 use itertools::Itertools;
 use observability_deps::tracing::debug;
-use once_cell::sync::Lazy;
 use query_functions::selectors::{
     selector_first, selector_last, selector_max, selector_min, SelectorOutput,
 };
@@ -1112,7 +1114,7 @@ impl<'a> InfluxQLToLogicalPlan<'a> {
 
         match ctx.scope {
             ExprScope::Where => {
-                if call.name.eq_ignore_ascii_case("now") {
+                if is_now_function(&call.name) {
                     error::not_implemented("now")
                 } else {
                     let name = &call.name;
@@ -2181,68 +2183,6 @@ pub(crate) fn find_time_column_index(fields: &[Field]) -> Option<usize> {
         .map(|(i, _)| i)
 }
 
-/// Returns `true` if `name` is a mathematical scalar function
-/// supported by InfluxQL.
-pub(crate) fn is_scalar_math_function(name: &str) -> bool {
-    static FUNCTIONS: Lazy<HashSet<&'static str>> = Lazy::new(|| {
-        HashSet::from([
-            "abs", "sin", "cos", "tan", "asin", "acos", "atan", "atan2", "exp", "log", "ln",
-            "log2", "log10", "sqrt", "pow", "floor", "ceil", "round",
-        ])
-    });
-
-    FUNCTIONS.contains(name)
-}
-
-/// Returns `true` if `name` is an aggregate or aggregate function
-/// supported by InfluxQL.
-fn is_aggregate_function(name: &str) -> bool {
-    static FUNCTIONS: Lazy<HashSet<&'static str>> = Lazy::new(|| {
-        HashSet::from([
-            // Scalar-like functions
-            "cumulative_sum",
-            "derivative",
-            "difference",
-            "elapsed",
-            "moving_average",
-            "non_negative_derivative",
-            "non_negative_difference",
-            // Selector functions
-            "bottom",
-            "first",
-            "last",
-            "max",
-            "min",
-            "percentile",
-            "sample",
-            "top",
-            // Aggregate functions
-            "count",
-            "integral",
-            "mean",
-            "median",
-            "mode",
-            "spread",
-            "stddev",
-            "sum",
-            // Prediction functions
-            "holt_winters",
-            "holt_winters_with_fit",
-            // Technical analysis functions
-            "chande_momentum_oscillator",
-            "exponential_moving_average",
-            "double_exponential_moving_average",
-            "kaufmans_efficiency_ratio",
-            "kaufmans_adaptive_moving_average",
-            "triple_exponential_moving_average",
-            "triple_exponential_derivative",
-            "relative_strength_index",
-        ])
-    });
-
-    FUNCTIONS.contains(name)
-}
-
 /// Returns true if the conditional expression is a single node that
 /// refers to the `time` column.
 ///
diff --git a/iox_query_influxql/src/plan/planner_time_range_expression.rs b/iox_query_influxql/src/plan/planner_time_range_expression.rs
index 06bbed1711..2406339bc1 100644
--- a/iox_query_influxql/src/plan/planner_time_range_expression.rs
+++ b/iox_query_influxql/src/plan/planner_time_range_expression.rs
@@ -4,6 +4,7 @@ use crate::plan::util::binary_operator_to_df_operator;
 use datafusion::common::{DataFusionError, Result, ScalarValue};
 use datafusion::logical_expr::{binary_expr, lit, now, BinaryExpr, Expr as DFExpr, Operator};
 use influxdb_influxql_parser::expression::{Binary, BinaryOperator, Call};
+use influxdb_influxql_parser::functions::is_now_function;
 use influxdb_influxql_parser::{expression::Expr, literal::Literal};
 
 type ExprResult = Result<DFExpr>;
@@ -103,7 +104,7 @@ fn reduce_expr(expr: &Expr, tz: Option<chrono_tz::Tz>) -> ExprResult {
     match expr {
         Expr::Binary(v) => reduce_binary_expr(v, tz).map_err(map_expr_err(expr)),
         Expr::Call (Call { name, .. }) => {
-            if !name.eq_ignore_ascii_case("now") {
+            if !is_now_function(name) {
                 return error::query(
                     format!("invalid function call '{name}'"),
                 );
diff --git a/iox_query_influxql/src/plan/rewriter.rs b/iox_query_influxql/src/plan/rewriter.rs
index c8a6d6bcda..b0459f5131 100644
--- a/iox_query_influxql/src/plan/rewriter.rs
+++ b/iox_query_influxql/src/plan/rewriter.rs
@@ -3,12 +3,12 @@
 use crate::plan::expr_type_evaluator::evaluate_type;
 use crate::plan::field::field_name;
 use crate::plan::field_mapper::{field_and_dimensions, FieldTypeMap, TagSet};
-use crate::plan::planner::is_scalar_math_function;
 use crate::plan::{error, util, SchemaProvider};
 use datafusion::common::{DataFusionError, Result};
 use influxdb_influxql_parser::common::{MeasurementName, QualifiedMeasurementName};
 use influxdb_influxql_parser::expression::walk::{walk_expr, walk_expr_mut};
 use influxdb_influxql_parser::expression::{Call, Expr, VarRef, VarRefDataType, WildcardType};
+use influxdb_influxql_parser::functions::is_scalar_math_function;
 use influxdb_influxql_parser::identifier::Identifier;
 use influxdb_influxql_parser::literal::Literal;
 use influxdb_influxql_parser::select::{

From fe5fa3b3e8ce47e9b1ea86194715b2df220874bb Mon Sep 17 00:00:00 2001
From: "Christopher M. Wolff" <chris.wolff@influxdata.com>
Date: Mon, 24 Apr 2023 11:15:08 -0700
Subject: [PATCH 07/25] fix: don't skip failed rules during logical planning
 (#7641)

Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com>
---
 datafusion_util/src/config.rs | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/datafusion_util/src/config.rs b/datafusion_util/src/config.rs
index 1c1a975a4f..7caa94225c 100644
--- a/datafusion_util/src/config.rs
+++ b/datafusion_util/src/config.rs
@@ -22,6 +22,14 @@ pub fn iox_session_config() -> SessionConfig {
     options.execution.parquet.reorder_filters = true;
     options.optimizer.repartition_sorts = true;
 
+    // DataFusion skips failed optimizer rules by default. Relevant issues:
+    // - https://github.com/apache/arrow-datafusion/issues/4615
+    // - https://github.com/apache/arrow-datafusion/issues/4685
+    //
+    // However IOx should not skip them, since that will result in more helpful error messages
+    // when they fail, e.g., 'HandleGapFill`.
+    options.optimizer.skip_failed_rules = false;
+
     SessionConfig::from(options)
         .with_batch_size(BATCH_SIZE)
         .with_create_default_catalog_and_schema(true)

From ebf0b34523ac5503ef651f38eac82ec9ce1420e1 Mon Sep 17 00:00:00 2001
From: Joe-Blount <73478756+Joe-Blount@users.noreply.github.com>
Date: Mon, 24 Apr 2023 15:48:35 -0500
Subject: [PATCH 08/25] feat: add compactor concurrency within single
 partition's plans (#7642)

* feat: trying to get concurrency within partition compaction

* feat: Create a stream then map and try collect

* feat: set compactor concurrency within single partition's plans

* chore: add log about plan concurrency

---------

Co-authored-by: Carol (Nichols || Goulding) <carol.nichols@gmail.com>
---
 compactor2/src/driver.rs             |  41 ++++---
 compactor2/tests/layouts/backfill.rs | 172 +++++++++++++--------------
 compactor2/tests/layouts/knobs.rs    | 124 +++++++++----------
 3 files changed, 172 insertions(+), 165 deletions(-)

diff --git a/compactor2/src/driver.rs b/compactor2/src/driver.rs
index 9963c3a677..2a57d05908 100644
--- a/compactor2/src/driver.rs
+++ b/compactor2/src/driver.rs
@@ -1,7 +1,7 @@
 use std::{num::NonZeroUsize, sync::Arc, time::Duration};
 
 use data_types::{CompactionLevel, ParquetFile, ParquetFileParams, PartitionId};
-use futures::StreamExt;
+use futures::{stream, StreamExt, TryStreamExt};
 use observability_deps::tracing::info;
 use parquet_file::ParquetFilePath;
 use tokio::sync::watch::Sender;
@@ -327,25 +327,32 @@ async fn run_plans(
         split_or_compact,
         input_uuids_inpad,
     );
-    let capacity = plans.iter().map(|p| p.n_output_files()).sum();
-    let mut created_file_params = Vec::with_capacity(capacity);
 
-    for plan_ir in plans
-        .into_iter()
-        .filter(|plan| !matches!(plan, PlanIR::None { .. }))
-    {
-        created_file_params.extend(
-            execute_plan(
-                plan_ir,
-                partition_info,
-                components,
-                Arc::clone(&job_semaphore),
-            )
-            .await?,
+    info!(
+        partition_id = partition_info.partition_id.get(),
+        plan_count = plans.len(),
+        concurrency_limit = job_semaphore.total_permits(),
+        "compacting plans concurrently",
+    );
+
+    let created_file_params: Vec<Vec<_>> = stream::iter(
+        plans
+            .into_iter()
+            .filter(|plan| !matches!(plan, PlanIR::None { .. })),
+    )
+    .map(|plan_ir| {
+        execute_plan(
+            plan_ir,
+            partition_info,
+            components,
+            Arc::clone(&job_semaphore),
         )
-    }
+    })
+    .buffer_unordered(job_semaphore.total_permits())
+    .try_collect()
+    .await?;
 
-    Ok(created_file_params)
+    Ok(created_file_params.into_iter().flatten().collect())
 }
 
 async fn execute_plan(
diff --git a/compactor2/tests/layouts/backfill.rs b/compactor2/tests/layouts/backfill.rs
index 33b799fc3e..86b2f2c526 100644
--- a/compactor2/tests/layouts/backfill.rs
+++ b/compactor2/tests/layouts/backfill.rs
@@ -759,76 +759,76 @@ async fn random_backfill_empty_partition() {
     - "L0                                                                                                                 "
     - "L0.?[76,329] 1.04us 2.96mb|-------------------------------------L0.?--------------------------------------|         "
     - "L0.?[330,356] 1.04us 322.99kb                                                                                 |-L0.?-| "
-    - "**** Simulation run 71, type=split(ReduceOverlap)(split_times=[658]). 1 Input Files, 3.66mb total:"
-    - "L0, all files 3.66mb                                                                                               "
-    - "L0.163[357,670] 1.04us   |-----------------------------------------L0.163-----------------------------------------|"
-    - "**** 2 Output Files (parquet_file_id not yet assigned), 3.66mb total:"
-    - "L0                                                                                                                 "
-    - "L0.?[357,658] 1.04us 3.52mb|----------------------------------------L0.?----------------------------------------|    "
-    - "L0.?[659,670] 1.04us 143.55kb                                                                                      |L0.?|"
-    - "**** Simulation run 72, type=split(ReduceOverlap)(split_times=[329]). 1 Input Files, 3.33mb total:"
-    - "L0, all files 3.33mb                                                                                               "
-    - "L0.165[42,356] 1.04us    |-----------------------------------------L0.165-----------------------------------------|"
-    - "**** 2 Output Files (parquet_file_id not yet assigned), 3.33mb total:"
-    - "L0                                                                                                                 "
-    - "L0.?[42,329] 1.04us 3.04mb|--------------------------------------L0.?--------------------------------------|        "
-    - "L0.?[330,356] 1.04us 292.88kb                                                                                  |L0.?-| "
-    - "**** Simulation run 73, type=split(ReduceOverlap)(split_times=[658]). 1 Input Files, 3.32mb total:"
+    - "**** Simulation run 71, type=split(ReduceOverlap)(split_times=[658]). 1 Input Files, 3.32mb total:"
     - "L0, all files 3.32mb                                                                                               "
     - "L0.166[357,670] 1.04us   |-----------------------------------------L0.166-----------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 3.32mb total:"
     - "L0                                                                                                                 "
     - "L0.?[357,658] 1.04us 3.19mb|----------------------------------------L0.?----------------------------------------|    "
     - "L0.?[659,670] 1.04us 130.17kb                                                                                      |L0.?|"
-    - "**** Simulation run 74, type=split(ReduceOverlap)(split_times=[329]). 1 Input Files, 2.36mb total:"
+    - "**** Simulation run 72, type=split(ReduceOverlap)(split_times=[329]). 1 Input Files, 2.36mb total:"
     - "L0, all files 2.36mb                                                                                               "
     - "L0.168[173,356] 1.04us   |-----------------------------------------L0.168-----------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 2.36mb total:"
     - "L0                                                                                                                 "
     - "L0.?[173,329] 1.04us 2.01mb|-----------------------------------L0.?-----------------------------------|              "
     - "L0.?[330,356] 1.04us 355.83kb                                                                             |---L0.?---| "
-    - "**** Simulation run 75, type=split(ReduceOverlap)(split_times=[658]). 1 Input Files, 4.03mb total:"
+    - "**** Simulation run 73, type=split(ReduceOverlap)(split_times=[658]). 1 Input Files, 4.03mb total:"
     - "L0, all files 4.03mb                                                                                               "
     - "L0.169[357,670] 1.04us   |-----------------------------------------L0.169-----------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 4.03mb total:"
     - "L0                                                                                                                 "
     - "L0.?[357,658] 1.04us 3.87mb|----------------------------------------L0.?----------------------------------------|    "
     - "L0.?[659,670] 1.04us 158.15kb                                                                                      |L0.?|"
-    - "**** Simulation run 76, type=split(ReduceOverlap)(split_times=[329]). 1 Input Files, 5.28mb total:"
+    - "**** Simulation run 74, type=split(ReduceOverlap)(split_times=[329]). 1 Input Files, 5.28mb total:"
     - "L0, all files 5.28mb                                                                                               "
     - "L0.171[50,356] 1.04us    |-----------------------------------------L0.171-----------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 5.28mb total:"
     - "L0                                                                                                                 "
     - "L0.?[50,329] 1.04us 4.82mb|--------------------------------------L0.?--------------------------------------|        "
     - "L0.?[330,356] 1.04us 477.51kb                                                                                  |L0.?-| "
-    - "**** Simulation run 77, type=split(ReduceOverlap)(split_times=[329]). 1 Input Files, 3.27mb total:"
+    - "**** Simulation run 75, type=split(ReduceOverlap)(split_times=[329]). 1 Input Files, 3.27mb total:"
     - "L0, all files 3.27mb                                                                                               "
     - "L0.173[76,356] 1.04us    |-----------------------------------------L0.173-----------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 3.27mb total:"
     - "L0                                                                                                                 "
     - "L0.?[76,329] 1.04us 2.96mb|-------------------------------------L0.?--------------------------------------|         "
     - "L0.?[330,356] 1.04us 322.99kb                                                                                 |-L0.?-| "
-    - "**** Simulation run 78, type=split(ReduceOverlap)(split_times=[658]). 1 Input Files, 3.66mb total:"
+    - "**** Simulation run 76, type=split(ReduceOverlap)(split_times=[658]). 1 Input Files, 3.66mb total:"
     - "L0, all files 3.66mb                                                                                               "
     - "L0.174[357,670] 1.04us   |-----------------------------------------L0.174-----------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 3.66mb total:"
     - "L0                                                                                                                 "
     - "L0.?[357,658] 1.04us 3.52mb|----------------------------------------L0.?----------------------------------------|    "
     - "L0.?[659,670] 1.04us 143.55kb                                                                                      |L0.?|"
-    - "**** Simulation run 79, type=split(ReduceOverlap)(split_times=[329]). 1 Input Files, 3.33mb total:"
+    - "**** Simulation run 77, type=split(ReduceOverlap)(split_times=[329]). 1 Input Files, 3.33mb total:"
     - "L0, all files 3.33mb                                                                                               "
     - "L0.176[42,356] 1.05us    |-----------------------------------------L0.176-----------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 3.33mb total:"
     - "L0                                                                                                                 "
     - "L0.?[42,329] 1.05us 3.04mb|--------------------------------------L0.?--------------------------------------|        "
     - "L0.?[330,356] 1.05us 292.88kb                                                                                  |L0.?-| "
-    - "**** Simulation run 80, type=split(ReduceOverlap)(split_times=[658]). 1 Input Files, 3.32mb total:"
+    - "**** Simulation run 78, type=split(ReduceOverlap)(split_times=[658]). 1 Input Files, 3.32mb total:"
     - "L0, all files 3.32mb                                                                                               "
     - "L0.177[357,670] 1.05us   |-----------------------------------------L0.177-----------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 3.32mb total:"
     - "L0                                                                                                                 "
     - "L0.?[357,658] 1.05us 3.19mb|----------------------------------------L0.?----------------------------------------|    "
     - "L0.?[659,670] 1.05us 130.17kb                                                                                      |L0.?|"
+    - "**** Simulation run 79, type=split(ReduceOverlap)(split_times=[658]). 1 Input Files, 3.66mb total:"
+    - "L0, all files 3.66mb                                                                                               "
+    - "L0.163[357,670] 1.04us   |-----------------------------------------L0.163-----------------------------------------|"
+    - "**** 2 Output Files (parquet_file_id not yet assigned), 3.66mb total:"
+    - "L0                                                                                                                 "
+    - "L0.?[357,658] 1.04us 3.52mb|----------------------------------------L0.?----------------------------------------|    "
+    - "L0.?[659,670] 1.04us 143.55kb                                                                                      |L0.?|"
+    - "**** Simulation run 80, type=split(ReduceOverlap)(split_times=[329]). 1 Input Files, 3.33mb total:"
+    - "L0, all files 3.33mb                                                                                               "
+    - "L0.165[42,356] 1.04us    |-----------------------------------------L0.165-----------------------------------------|"
+    - "**** 2 Output Files (parquet_file_id not yet assigned), 3.33mb total:"
+    - "L0                                                                                                                 "
+    - "L0.?[42,329] 1.04us 3.04mb|--------------------------------------L0.?--------------------------------------|        "
+    - "L0.?[330,356] 1.04us 292.88kb                                                                                  |L0.?-| "
     - "**** Simulation run 81, type=split(ReduceOverlap)(split_times=[329]). 1 Input Files, 2.36mb total:"
     - "L0, all files 2.36mb                                                                                               "
     - "L0.179[173,356] 1.05us   |-----------------------------------------L0.179-----------------------------------------|"
@@ -963,14 +963,14 @@ async fn random_backfill_empty_partition() {
     - "L0.?[649,658] 1.04us 131.79kb                                                                                       |L0.?|"
     - "**** Simulation run 98, type=split(ReduceOverlap)(split_times=[648]). 1 Input Files, 3.52mb total:"
     - "L0, all files 3.52mb                                                                                               "
-    - "L0.233[357,658] 1.04us   |-----------------------------------------L0.233-----------------------------------------|"
+    - "L0.249[357,658] 1.04us   |-----------------------------------------L0.249-----------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 3.52mb total:"
     - "L0                                                                                                                 "
     - "L0.?[357,648] 1.04us 3.4mb|----------------------------------------L0.?-----------------------------------------|   "
     - "L0.?[649,658] 1.04us 119.63kb                                                                                       |L0.?|"
     - "**** Simulation run 99, type=split(ReduceOverlap)(split_times=[648]). 1 Input Files, 3.19mb total:"
     - "L0, all files 3.19mb                                                                                               "
-    - "L0.237[357,658] 1.04us   |-----------------------------------------L0.237-----------------------------------------|"
+    - "L0.233[357,658] 1.04us   |-----------------------------------------L0.233-----------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 3.19mb total:"
     - "L0                                                                                                                 "
     - "L0.?[357,648] 1.04us 3.08mb|----------------------------------------L0.?-----------------------------------------|   "
@@ -984,21 +984,21 @@ async fn random_backfill_empty_partition() {
     - "L0.?[967,986] 1.04us 218.33kb                                                                                    |L0.?|"
     - "**** Simulation run 101, type=split(ReduceOverlap)(split_times=[648]). 1 Input Files, 3.87mb total:"
     - "L0, all files 3.87mb                                                                                               "
-    - "L0.241[357,658] 1.04us   |-----------------------------------------L0.241-----------------------------------------|"
+    - "L0.237[357,658] 1.04us   |-----------------------------------------L0.237-----------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 3.87mb total:"
     - "L0                                                                                                                 "
     - "L0.?[357,648] 1.04us 3.75mb|----------------------------------------L0.?-----------------------------------------|   "
     - "L0.?[649,658] 1.04us 131.79kb                                                                                       |L0.?|"
     - "**** Simulation run 102, type=split(ReduceOverlap)(split_times=[648]). 1 Input Files, 3.52mb total:"
     - "L0, all files 3.52mb                                                                                               "
-    - "L0.247[357,658] 1.04us   |-----------------------------------------L0.247-----------------------------------------|"
+    - "L0.243[357,658] 1.04us   |-----------------------------------------L0.243-----------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 3.52mb total:"
     - "L0                                                                                                                 "
     - "L0.?[357,648] 1.04us 3.4mb|----------------------------------------L0.?-----------------------------------------|   "
     - "L0.?[649,658] 1.04us 119.63kb                                                                                       |L0.?|"
     - "**** Simulation run 103, type=split(ReduceOverlap)(split_times=[648]). 1 Input Files, 3.19mb total:"
     - "L0, all files 3.19mb                                                                                               "
-    - "L0.251[357,658] 1.05us   |-----------------------------------------L0.251-----------------------------------------|"
+    - "L0.247[357,658] 1.05us   |-----------------------------------------L0.247-----------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 3.19mb total:"
     - "L0                                                                                                                 "
     - "L0.?[357,648] 1.05us 3.08mb|----------------------------------------L0.?-----------------------------------------|   "
@@ -1039,7 +1039,7 @@ async fn random_backfill_empty_partition() {
     - "L0.?[671,966] 1.05us 3.14mb|---------------------------------------L0.?---------------------------------------|      "
     - "L0.?[967,986] 1.05us 218.33kb                                                                                    |L0.?|"
     - "Committing partition 1:"
-    - "  Soft Deleting 20 files: L0.145, L0.156, L0.167, L0.178, L0.189, L0.199, L0.205, L0.209, L0.213, L0.219, L0.223, L0.227, L0.233, L0.237, L0.241, L0.247, L0.251, L0.255, L0.261, L0.265"
+    - "  Soft Deleting 20 files: L0.145, L0.156, L0.167, L0.178, L0.189, L0.199, L0.205, L0.209, L0.213, L0.219, L0.223, L0.227, L0.233, L0.237, L0.243, L0.247, L0.249, L0.255, L0.261, L0.265"
     - "  Creating 40 files"
     - "**** Simulation run 109, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[263]). 2 Input Files, 103.14mb total:"
     - "L0                                                                                                                 "
@@ -1132,35 +1132,35 @@ async fn random_backfill_empty_partition() {
     - "L0.?[264,329] 1.04us 789.53kb                                                                  |--------L0.?---------| "
     - "**** Simulation run 121, type=split(ReduceOverlap)(split_times=[263]). 1 Input Files, 3.04mb total:"
     - "L0, all files 3.04mb                                                                                               "
-    - "L0.235[42,329] 1.04us    |-----------------------------------------L0.235-----------------------------------------|"
+    - "L0.251[42,329] 1.04us    |-----------------------------------------L0.251-----------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 3.04mb total:"
     - "L0                                                                                                                 "
     - "L0.?[42,263] 1.04us 2.34mb|-------------------------------L0.?--------------------------------|                     "
     - "L0.?[264,329] 1.04us 715.93kb                                                                     |-------L0.?-------| "
     - "**** Simulation run 122, type=split(ReduceOverlap)(split_times=[263]). 1 Input Files, 2.01mb total:"
     - "L0, all files 2.01mb                                                                                               "
-    - "L0.239[173,329] 1.04us   |-----------------------------------------L0.239-----------------------------------------|"
+    - "L0.235[173,329] 1.04us   |-----------------------------------------L0.235-----------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 2.01mb total:"
     - "L0                                                                                                                 "
     - "L0.?[173,263] 1.04us 1.16mb|----------------------L0.?-----------------------|                                       "
     - "L0.?[264,329] 1.04us 869.81kb                                                    |---------------L0.?----------------| "
     - "**** Simulation run 123, type=split(ReduceOverlap)(split_times=[263]). 1 Input Files, 4.82mb total:"
     - "L0, all files 4.82mb                                                                                               "
-    - "L0.243[50,329] 1.04us    |-----------------------------------------L0.243-----------------------------------------|"
+    - "L0.239[50,329] 1.04us    |-----------------------------------------L0.239-----------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 4.82mb total:"
     - "L0                                                                                                                 "
     - "L0.?[50,263] 1.04us 3.68mb|-------------------------------L0.?-------------------------------|                      "
     - "L0.?[264,329] 1.04us 1.14mb                                                                     |-------L0.?-------| "
     - "**** Simulation run 124, type=split(ReduceOverlap)(split_times=[263]). 1 Input Files, 2.96mb total:"
     - "L0, all files 2.96mb                                                                                               "
-    - "L0.245[76,329] 1.04us    |-----------------------------------------L0.245-----------------------------------------|"
+    - "L0.241[76,329] 1.04us    |-----------------------------------------L0.241-----------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 2.96mb total:"
     - "L0                                                                                                                 "
     - "L0.?[76,263] 1.04us 2.18mb|------------------------------L0.?------------------------------|                        "
     - "L0.?[264,329] 1.04us 789.53kb                                                                  |--------L0.?---------| "
     - "**** Simulation run 125, type=split(ReduceOverlap)(split_times=[263]). 1 Input Files, 3.04mb total:"
     - "L0, all files 3.04mb                                                                                               "
-    - "L0.249[42,329] 1.05us    |-----------------------------------------L0.249-----------------------------------------|"
+    - "L0.245[42,329] 1.05us    |-----------------------------------------L0.245-----------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 3.04mb total:"
     - "L0                                                                                                                 "
     - "L0.?[42,263] 1.05us 2.34mb|-------------------------------L0.?--------------------------------|                     "
@@ -1194,7 +1194,7 @@ async fn random_backfill_empty_partition() {
     - "L0.?[42,263] 1.05us 2.34mb|-------------------------------L0.?--------------------------------|                     "
     - "L0.?[264,329] 1.05us 715.93kb                                                                     |-------L0.?-------| "
     - "Committing partition 1:"
-    - "  Soft Deleting 20 files: L0.197, L0.201, L0.203, L0.207, L0.211, L0.215, L0.217, L0.221, L0.225, L0.229, L0.231, L0.235, L0.239, L0.243, L0.245, L0.249, L0.253, L0.257, L0.259, L0.263"
+    - "  Soft Deleting 20 files: L0.197, L0.201, L0.203, L0.207, L0.211, L0.215, L0.217, L0.221, L0.225, L0.229, L0.231, L0.235, L0.239, L0.241, L0.245, L0.251, L0.253, L0.257, L0.259, L0.263"
     - "  Creating 40 files"
     - "**** Simulation run 130, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[570, 876]). 9 Input Files, 229.77mb total:"
     - "L0                                                                                                                 "
@@ -2050,20 +2050,20 @@ async fn random_backfill_empty_partition() {
     - "L0.522[584,590] 1.04us 84.83kb                                                   |L0.522|                               "
     - "L0.455[591,648] 1.04us 702.84kb                                                    |L0.455|                              "
     - "L0.289[649,658] 1.04us 119.63kb                                                         |L0.289|                         "
-    - "L0.234[659,670] 1.04us 143.55kb                                                          |L0.234|                        "
+    - "L0.250[659,670] 1.04us 143.55kb                                                          |L0.250|                        "
     - "L0.523[671,870] 1.04us 2.34mb                                                           |-----L0.523-----|             "
     - "L0.524[871,876] 1.04us 72.33kb                                                                               |L0.524|   "
     - "L0.388[877,932] 1.04us 675.04kb                                                                               |L0.388|   "
     - "L0.334[42,263] 1.04us 2.34mb|------L0.334-------|                                                                     "
     - "L0.460[264,295] 1.04us 341.44kb                     |L0.460|                                                             "
     - "L0.461[296,329] 1.04us 374.49kb                        |L0.461|                                                          "
-    - "L0.236[330,356] 1.04us 292.88kb                           |L0.236|                                                       "
+    - "L0.252[330,356] 1.04us 292.88kb                           |L0.252|                                                       "
     - "L0.389[357,570] 1.04us 2.26mb                              |------L0.389------|                                        "
     - "L0.525[571,583] 1.04us 131.86kb                                                  |L0.525|                                "
     - "L0.526[584,590] 1.04us 76.92kb                                                   |L0.526|                               "
     - "L0.459[591,648] 1.04us 637.32kb                                                    |L0.459|                              "
     - "L0.291[649,658] 1.04us 108.47kb                                                         |L0.291|                         "
-    - "L0.238[659,670] 1.04us 130.17kb                                                          |L0.238|                        "
+    - "L0.234[659,670] 1.04us 130.17kb                                                          |L0.234|                        "
     - "L0.527[671,870] 1.04us 2.12mb                                                           |-----L0.527-----|             "
     - "L0.528[871,876] 1.04us 65.5kb                                                                               |L0.528|   "
     - "L0.392[877,966] 1.04us 982.47kb                                                                               |L0.392|   "
@@ -2071,20 +2071,20 @@ async fn random_backfill_empty_partition() {
     - "L0.336[173,263] 1.04us 1.16mb            |L0.336|                                                                      "
     - "L0.464[264,295] 1.04us 414.83kb                     |L0.464|                                                             "
     - "L0.465[296,329] 1.04us 454.98kb                        |L0.465|                                                          "
-    - "L0.240[330,356] 1.04us 355.83kb                           |L0.240|                                                       "
+    - "L0.236[330,356] 1.04us 355.83kb                           |L0.236|                                                       "
     - "L0.393[357,570] 1.04us 2.74mb                              |------L0.393------|                                        "
     - "L0.529[571,583] 1.04us 160.2kb                                                  |L0.529|                                "
     - "L0.530[584,590] 1.04us 93.45kb                                                   |L0.530|                               "
     - "L0.463[591,648] 1.04us 774.3kb                                                    |L0.463|                              "
     - "L0.295[649,658] 1.04us 131.79kb                                                         |L0.295|                         "
-    - "L0.242[659,670] 1.04us 158.15kb                                                          |L0.242|                        "
+    - "L0.238[659,670] 1.04us 158.15kb                                                          |L0.238|                        "
     - "L0.531[671,870] 1.04us 2.58mb                                                           |-----L0.531-----|             "
     - "L0.532[871,876] 1.04us 79.64kb                                                                               |L0.532|   "
     - "L0.396[877,950] 1.04us 982.23kb                                                                               |L0.396|   "
     - "L0.338[50,263] 1.04us 3.68mb|------L0.338------|                                                                      "
     - "L0.468[264,295] 1.04us 556.69kb                     |L0.468|                                                             "
     - "L0.469[296,329] 1.04us 610.56kb                        |L0.469|                                                          "
-    - "L0.244[330,356] 1.04us 477.51kb                           |L0.244|                                                       "
+    - "L0.240[330,356] 1.04us 477.51kb                           |L0.240|                                                       "
     - "L0.397[357,570] 1.04us 3.69mb                              |------L0.397------|                                        "
     - "L0.533[571,583] 1.04us 216.68kb                                                  |L0.533|                                "
     - "L0.534[584,590] 1.04us 126.4kb                                                   |L0.534|                               "
@@ -2092,26 +2092,26 @@ async fn random_backfill_empty_partition() {
     - "L0.340[76,263] 1.04us 2.18mb   |----L0.340-----|                                                                      "
     - "L0.472[264,295] 1.04us 376.55kb                     |L0.472|                                                             "
     - "L0.473[296,329] 1.04us 412.99kb                        |L0.473|                                                          "
-    - "L0.246[330,356] 1.04us 322.99kb                           |L0.246|                                                       "
+    - "L0.242[330,356] 1.04us 322.99kb                           |L0.242|                                                       "
     - "L0.399[357,570] 1.04us 2.49mb                              |------L0.399------|                                        "
     - "L0.535[571,583] 1.04us 145.42kb                                                  |L0.535|                                "
     - "L0.536[584,590] 1.04us 84.83kb                                                   |L0.536|                               "
     - "L0.471[591,648] 1.04us 702.84kb                                                    |L0.471|                              "
     - "L0.297[649,658] 1.04us 119.63kb                                                         |L0.297|                         "
-    - "L0.248[659,670] 1.04us 143.55kb                                                          |L0.248|                        "
+    - "L0.244[659,670] 1.04us 143.55kb                                                          |L0.244|                        "
     - "L0.537[671,870] 1.04us 2.34mb                                                           |-----L0.537-----|             "
     - "L0.538[871,876] 1.04us 72.33kb                                                                               |L0.538|   "
     - "L0.402[877,932] 1.04us 675.04kb                                                                               |L0.402|   "
     - "L0.342[42,263] 1.05us 2.34mb|------L0.342-------|                                                                     "
     - "L0.476[264,295] 1.05us 341.44kb                     |L0.476|                                                             "
     - "L0.477[296,329] 1.05us 374.49kb                        |L0.477|                                                          "
-    - "L0.250[330,356] 1.05us 292.88kb                           |L0.250|                                                       "
+    - "L0.246[330,356] 1.05us 292.88kb                           |L0.246|                                                       "
     - "L0.403[357,570] 1.05us 2.26mb                              |------L0.403------|                                        "
     - "L0.539[571,583] 1.05us 131.86kb                                                  |L0.539|                                "
     - "L0.540[584,590] 1.05us 76.92kb                                                   |L0.540|                               "
     - "L0.475[591,648] 1.05us 637.32kb                                                    |L0.475|                              "
     - "L0.299[649,658] 1.05us 108.47kb                                                         |L0.299|                         "
-    - "L0.252[659,670] 1.05us 130.17kb                                                          |L0.252|                        "
+    - "L0.248[659,670] 1.05us 130.17kb                                                          |L0.248|                        "
     - "L0.541[671,870] 1.05us 2.12mb                                                           |-----L0.541-----|             "
     - "L0.542[871,876] 1.05us 65.5kb                                                                               |L0.542|   "
     - "L0.406[877,966] 1.05us 982.47kb                                                                               |L0.406|   "
@@ -3639,76 +3639,76 @@ async fn random_backfill_over_l2s() {
     - "L0                                                                                                                 "
     - "L0.?[592,626] 1.03us 374.74kb|----------------L0.?-----------------|                                                   "
     - "L0.?[627,670] 1.03us 484.96kb                                        |---------------------L0.?----------------------| "
-    - "**** Simulation run 142, type=split(ReduceOverlap)(split_times=[334]). 1 Input Files, 672.54kb total:"
-    - "L0, all files 672.54kb                                                                                             "
-    - "L0.279[295,356] 1.03us   |-----------------------------------------L0.279-----------------------------------------|"
-    - "**** 2 Output Files (parquet_file_id not yet assigned), 672.54kb total:"
-    - "L0                                                                                                                 "
-    - "L0.?[295,334] 1.03us 429.99kb|-------------------------L0.?--------------------------|                                 "
-    - "L0.?[335,356] 1.03us 242.56kb                                                           |------------L0.?------------| "
-    - "**** Simulation run 143, type=split(ReduceOverlap)(split_times=[626]). 1 Input Files, 1.02mb total:"
-    - "L0, all files 1.02mb                                                                                               "
-    - "L0.324[592,670] 1.03us   |-----------------------------------------L0.324-----------------------------------------|"
-    - "**** 2 Output Files (parquet_file_id not yet assigned), 1.02mb total:"
-    - "L0                                                                                                                 "
-    - "L0.?[592,626] 1.03us 455.28kb|----------------L0.?-----------------|                                                   "
-    - "L0.?[627,670] 1.03us 589.19kb                                        |---------------------L0.?----------------------| "
-    - "**** Simulation run 144, type=split(ReduceOverlap)(split_times=[334]). 1 Input Files, 817.09kb total:"
+    - "**** Simulation run 142, type=split(ReduceOverlap)(split_times=[334]). 1 Input Files, 817.09kb total:"
     - "L0, all files 817.09kb                                                                                             "
     - "L0.281[295,356] 1.03us   |-----------------------------------------L0.281-----------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 817.09kb total:"
     - "L0                                                                                                                 "
     - "L0.?[295,334] 1.03us 522.4kb|-------------------------L0.?--------------------------|                                 "
     - "L0.?[335,356] 1.03us 294.69kb                                                           |------------L0.?------------| "
-    - "**** Simulation run 145, type=split(ReduceOverlap)(split_times=[626]). 1 Input Files, 677.02kb total:"
+    - "**** Simulation run 143, type=split(ReduceOverlap)(split_times=[626]). 1 Input Files, 677.02kb total:"
     - "L0, all files 677.02kb                                                                                             "
     - "L0.328[592,629] 1.03us   |-----------------------------------------L0.328-----------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 677.02kb total:"
     - "L0                                                                                                                 "
     - "L0.?[592,626] 1.03us 622.12kb|--------------------------------------L0.?--------------------------------------|        "
     - "L0.?[627,629] 1.03us 54.89kb                                                                                     |L0.?|"
-    - "**** Simulation run 146, type=split(ReduceOverlap)(split_times=[334]). 1 Input Files, 1.07mb total:"
+    - "**** Simulation run 144, type=split(ReduceOverlap)(split_times=[334]). 1 Input Files, 1.07mb total:"
     - "L0, all files 1.07mb                                                                                               "
     - "L0.283[295,356] 1.03us   |-----------------------------------------L0.283-----------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 1.07mb total:"
     - "L0                                                                                                                 "
     - "L0.?[295,334] 1.03us 701.05kb|-------------------------L0.?--------------------------|                                 "
     - "L0.?[335,356] 1.03us 395.46kb                                                           |------------L0.?------------| "
-    - "**** Simulation run 147, type=split(ReduceOverlap)(split_times=[626]). 1 Input Files, 948.08kb total:"
+    - "**** Simulation run 145, type=split(ReduceOverlap)(split_times=[626]). 1 Input Files, 948.08kb total:"
     - "L0, all files 948.08kb                                                                                             "
     - "L0.330[592,670] 1.04us   |-----------------------------------------L0.330-----------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 948.08kb total:"
     - "L0                                                                                                                 "
     - "L0.?[592,626] 1.04us 413.26kb|----------------L0.?-----------------|                                                   "
     - "L0.?[627,670] 1.04us 534.81kb                                        |---------------------L0.?----------------------| "
-    - "**** Simulation run 148, type=split(ReduceOverlap)(split_times=[334]). 1 Input Files, 741.68kb total:"
+    - "**** Simulation run 146, type=split(ReduceOverlap)(split_times=[334]). 1 Input Files, 741.68kb total:"
     - "L0, all files 741.68kb                                                                                             "
     - "L0.285[295,356] 1.04us   |-----------------------------------------L0.285-----------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 741.68kb total:"
     - "L0                                                                                                                 "
     - "L0.?[295,334] 1.04us 474.19kb|-------------------------L0.?--------------------------|                                 "
     - "L0.?[335,356] 1.04us 267.49kb                                                           |------------L0.?------------| "
-    - "**** Simulation run 149, type=split(ReduceOverlap)(split_times=[626]). 1 Input Files, 859.7kb total:"
+    - "**** Simulation run 147, type=split(ReduceOverlap)(split_times=[626]). 1 Input Files, 859.7kb total:"
     - "L0, all files 859.7kb                                                                                              "
     - "L0.334[592,670] 1.04us   |-----------------------------------------L0.334-----------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 859.7kb total:"
     - "L0                                                                                                                 "
     - "L0.?[592,626] 1.04us 374.74kb|----------------L0.?-----------------|                                                   "
     - "L0.?[627,670] 1.04us 484.96kb                                        |---------------------L0.?----------------------| "
-    - "**** Simulation run 150, type=split(ReduceOverlap)(split_times=[334]). 1 Input Files, 672.54kb total:"
+    - "**** Simulation run 148, type=split(ReduceOverlap)(split_times=[334]). 1 Input Files, 672.54kb total:"
     - "L0, all files 672.54kb                                                                                             "
     - "L0.287[295,356] 1.04us   |-----------------------------------------L0.287-----------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 672.54kb total:"
     - "L0                                                                                                                 "
     - "L0.?[295,334] 1.04us 429.99kb|-------------------------L0.?--------------------------|                                 "
     - "L0.?[335,356] 1.04us 242.56kb                                                           |------------L0.?------------| "
-    - "**** Simulation run 151, type=split(ReduceOverlap)(split_times=[626]). 1 Input Files, 1.02mb total:"
+    - "**** Simulation run 149, type=split(ReduceOverlap)(split_times=[626]). 1 Input Files, 1.02mb total:"
     - "L0, all files 1.02mb                                                                                               "
     - "L0.338[592,670] 1.04us   |-----------------------------------------L0.338-----------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 1.02mb total:"
     - "L0                                                                                                                 "
     - "L0.?[592,626] 1.04us 455.28kb|----------------L0.?-----------------|                                                   "
     - "L0.?[627,670] 1.04us 589.19kb                                        |---------------------L0.?----------------------| "
+    - "**** Simulation run 150, type=split(ReduceOverlap)(split_times=[334]). 1 Input Files, 672.54kb total:"
+    - "L0, all files 672.54kb                                                                                             "
+    - "L0.279[295,356] 1.03us   |-----------------------------------------L0.279-----------------------------------------|"
+    - "**** 2 Output Files (parquet_file_id not yet assigned), 672.54kb total:"
+    - "L0                                                                                                                 "
+    - "L0.?[295,334] 1.03us 429.99kb|-------------------------L0.?--------------------------|                                 "
+    - "L0.?[335,356] 1.03us 242.56kb                                                           |------------L0.?------------| "
+    - "**** Simulation run 151, type=split(ReduceOverlap)(split_times=[626]). 1 Input Files, 1.02mb total:"
+    - "L0, all files 1.02mb                                                                                               "
+    - "L0.324[592,670] 1.03us   |-----------------------------------------L0.324-----------------------------------------|"
+    - "**** 2 Output Files (parquet_file_id not yet assigned), 1.02mb total:"
+    - "L0                                                                                                                 "
+    - "L0.?[592,626] 1.03us 455.28kb|----------------L0.?-----------------|                                                   "
+    - "L0.?[627,670] 1.03us 589.19kb                                        |---------------------L0.?----------------------| "
     - "**** Simulation run 152, type=split(ReduceOverlap)(split_times=[334]). 1 Input Files, 817.09kb total:"
     - "L0, all files 817.09kb                                                                                             "
     - "L0.289[295,356] 1.04us   |-----------------------------------------L0.289-----------------------------------------|"
@@ -3910,7 +3910,7 @@ async fn random_backfill_over_l2s() {
     - "L0.?[904,986] 1.03us 918.23kb              |----------------------------------L0.?-----------------------------------| "
     - "**** Simulation run 178, type=split(ReduceOverlap)(split_times=[619]). 1 Input Files, 455.28kb total:"
     - "L0, all files 455.28kb                                                                                             "
-    - "L0.388[592,626] 1.03us   |-----------------------------------------L0.388-----------------------------------------|"
+    - "L0.404[592,626] 1.03us   |-----------------------------------------L0.404-----------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 455.28kb total:"
     - "L0                                                                                                                 "
     - "L0.?[592,619] 1.03us 361.55kb|--------------------------------L0.?---------------------------------|                   "
@@ -3924,14 +3924,14 @@ async fn random_backfill_over_l2s() {
     - "L0.?[904,950] 1.03us 636.2kb                       |------------------------------L0.?------------------------------| "
     - "**** Simulation run 180, type=split(ReduceOverlap)(split_times=[619]). 1 Input Files, 622.12kb total:"
     - "L0, all files 622.12kb                                                                                             "
-    - "L0.392[592,626] 1.03us   |-----------------------------------------L0.392-----------------------------------------|"
+    - "L0.388[592,626] 1.03us   |-----------------------------------------L0.388-----------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 622.12kb total:"
     - "L0                                                                                                                 "
     - "L0.?[592,619] 1.03us 494.04kb|--------------------------------L0.?---------------------------------|                   "
     - "L0.?[620,626] 1.03us 128.08kb                                                                          |----L0.?-----| "
     - "**** Simulation run 181, type=split(ReduceOverlap)(split_times=[619]). 1 Input Files, 413.26kb total:"
     - "L0, all files 413.26kb                                                                                             "
-    - "L0.396[592,626] 1.04us   |-----------------------------------------L0.396-----------------------------------------|"
+    - "L0.392[592,626] 1.04us   |-----------------------------------------L0.392-----------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 413.26kb total:"
     - "L0                                                                                                                 "
     - "L0.?[592,619] 1.04us 328.18kb|--------------------------------L0.?---------------------------------|                   "
@@ -3945,7 +3945,7 @@ async fn random_backfill_over_l2s() {
     - "L0.?[904,932] 1.04us 358.9kb                                |-------------------------L0.?--------------------------| "
     - "**** Simulation run 183, type=split(ReduceOverlap)(split_times=[619]). 1 Input Files, 374.74kb total:"
     - "L0, all files 374.74kb                                                                                             "
-    - "L0.400[592,626] 1.04us   |-----------------------------------------L0.400-----------------------------------------|"
+    - "L0.396[592,626] 1.04us   |-----------------------------------------L0.396-----------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 374.74kb total:"
     - "L0                                                                                                                 "
     - "L0.?[592,619] 1.04us 297.59kb|--------------------------------L0.?---------------------------------|                   "
@@ -3959,7 +3959,7 @@ async fn random_backfill_over_l2s() {
     - "L0.?[904,986] 1.04us 918.23kb              |----------------------------------L0.?-----------------------------------| "
     - "**** Simulation run 185, type=split(ReduceOverlap)(split_times=[619]). 1 Input Files, 455.28kb total:"
     - "L0, all files 455.28kb                                                                                             "
-    - "L0.404[592,626] 1.04us   |-----------------------------------------L0.404-----------------------------------------|"
+    - "L0.400[592,626] 1.04us   |-----------------------------------------L0.400-----------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 455.28kb total:"
     - "L0                                                                                                                 "
     - "L0.?[592,619] 1.04us 361.55kb|--------------------------------L0.?---------------------------------|                   "
@@ -4236,8 +4236,8 @@ async fn random_backfill_over_l2s() {
     - "  Creating 32 files"
     - "**** Simulation run 223, type=split(CompactAndSplitOutput(ManySmallFiles))(split_times=[610]). 200 Input Files, 166.42mb total:"
     - "L0                                                                                                                 "
-    - "L0.386[295,334] 1.03us 429.99kb                        |L0.386|                                                          "
-    - "L0.387[335,356] 1.03us 242.56kb                           |L0.387|                                                       "
+    - "L0.402[295,334] 1.03us 429.99kb                        |L0.402|                                                          "
+    - "L0.403[335,356] 1.03us 242.56kb                           |L0.403|                                                       "
     - "L0.319[358,591] 1.03us 2.48mb                              |-------L0.319-------|                                      "
     - "L0.455[592,619] 1.03us 297.59kb                                                    |L0.455|                              "
     - "L0.456[620,626] 1.03us 77.15kb                                                       |L0.456|                           "
@@ -4247,48 +4247,48 @@ async fn random_backfill_over_l2s() {
     - "L0.458[904,986] 1.03us 918.23kb                                                                                  |L0.458|"
     - "L0.517[173,275] 1.03us 1.31mb            |L0.517-|                                                                     "
     - "L0.518[276,294] 1.03us 250.4kb                      |L0.518|                                                            "
-    - "L0.390[295,334] 1.03us 522.4kb                        |L0.390|                                                          "
-    - "L0.391[335,356] 1.03us 294.69kb                           |L0.391|                                                       "
+    - "L0.386[295,334] 1.03us 522.4kb                        |L0.386|                                                          "
+    - "L0.387[335,356] 1.03us 294.69kb                           |L0.387|                                                       "
     - "L0.216[357,357] 1.03us 0b                              |L0.216|                                                    "
     - "L0.323[358,591] 1.03us 3.01mb                              |-------L0.323-------|                                      "
     - "L0.459[592,619] 1.03us 361.55kb                                                    |L0.459|                              "
     - "L0.460[620,626] 1.03us 93.73kb                                                       |L0.460|                           "
-    - "L0.389[627,670] 1.03us 589.19kb                                                       |L0.389|                           "
+    - "L0.405[627,670] 1.03us 589.19kb                                                       |L0.405|                           "
     - "L0.218[671,672] 1.03us 13.27kb                                                           |L0.218|                       "
     - "L0.325[673,887] 1.03us 2.78mb                                                            |------L0.325------|          "
     - "L0.461[888,903] 1.03us 203.04kb                                                                                |L0.461|  "
     - "L0.462[904,950] 1.03us 636.2kb                                                                                  |L0.462|"
     - "L0.519[50,275] 1.03us 3.89mb|------L0.519-------|                                                                     "
     - "L0.520[276,294] 1.03us 336.03kb                      |L0.520|                                                            "
-    - "L0.394[295,334] 1.03us 701.05kb                        |L0.394|                                                          "
-    - "L0.395[335,356] 1.03us 395.46kb                           |L0.395|                                                       "
+    - "L0.390[295,334] 1.03us 701.05kb                        |L0.390|                                                          "
+    - "L0.391[335,356] 1.03us 395.46kb                           |L0.391|                                                       "
     - "L0.220[357,357] 1.03us 0b                              |L0.220|                                                    "
     - "L0.327[358,591] 1.03us 4.05mb                              |-------L0.327-------|                                      "
     - "L0.463[592,619] 1.03us 494.04kb                                                    |L0.463|                              "
     - "L0.464[620,626] 1.03us 128.08kb                                                       |L0.464|                           "
-    - "L0.393[627,629] 1.03us 54.89kb                                                       |L0.393|                           "
+    - "L0.389[627,629] 1.03us 54.89kb                                                       |L0.389|                           "
     - "L0.521[76,275] 1.04us 2.32mb   |-----L0.521-----|                                                                     "
     - "L0.522[276,294] 1.04us 227.29kb                      |L0.522|                                                            "
-    - "L0.398[295,334] 1.04us 474.19kb                        |L0.398|                                                          "
-    - "L0.399[335,356] 1.04us 267.49kb                           |L0.399|                                                       "
+    - "L0.394[295,334] 1.04us 474.19kb                        |L0.394|                                                          "
+    - "L0.395[335,356] 1.04us 267.49kb                           |L0.395|                                                       "
     - "L0.222[357,357] 1.04us 0b                              |L0.222|                                                    "
     - "L0.329[358,591] 1.04us 2.73mb                              |-------L0.329-------|                                      "
     - "L0.465[592,619] 1.04us 328.18kb                                                    |L0.465|                              "
     - "L0.466[620,626] 1.04us 85.08kb                                                       |L0.466|                           "
-    - "L0.397[627,670] 1.04us 534.81kb                                                       |L0.397|                           "
+    - "L0.393[627,670] 1.04us 534.81kb                                                       |L0.393|                           "
     - "L0.224[671,672] 1.04us 12.05kb                                                           |L0.224|                       "
     - "L0.331[673,887] 1.04us 2.53mb                                                            |------L0.331------|          "
     - "L0.467[888,903] 1.04us 185.64kb                                                                                |L0.467|  "
     - "L0.468[904,932] 1.04us 358.9kb                                                                                  |L0.468|"
     - "L0.523[42,275] 1.04us 2.47mb|-------L0.523-------|                                                                    "
     - "L0.524[276,294] 1.04us 206.1kb                      |L0.524|                                                            "
-    - "L0.402[295,334] 1.04us 429.99kb                        |L0.402|                                                          "
-    - "L0.403[335,356] 1.04us 242.56kb                           |L0.403|                                                       "
+    - "L0.398[295,334] 1.04us 429.99kb                        |L0.398|                                                          "
+    - "L0.399[335,356] 1.04us 242.56kb                           |L0.399|                                                       "
     - "L0.226[357,357] 1.04us 0b                              |L0.226|                                                    "
     - "L0.333[358,591] 1.04us 2.48mb                              |-------L0.333-------|                                      "
     - "L0.469[592,619] 1.04us 297.59kb                                                    |L0.469|                              "
     - "L0.470[620,626] 1.04us 77.15kb                                                       |L0.470|                           "
-    - "L0.401[627,670] 1.04us 484.96kb                                                       |L0.401|                           "
+    - "L0.397[627,670] 1.04us 484.96kb                                                       |L0.397|                           "
     - "L0.228[671,672] 1.04us 10.92kb                                                           |L0.228|                       "
     - "L0.335[673,887] 1.04us 2.29mb                                                            |------L0.335------|          "
     - "L0.471[888,903] 1.04us 165.94kb                                                                                |L0.471|  "
@@ -4301,7 +4301,7 @@ async fn random_backfill_over_l2s() {
     - "L0.337[358,591] 1.04us 3.01mb                              |-------L0.337-------|                                      "
     - "L0.473[592,619] 1.04us 361.55kb                                                    |L0.473|                              "
     - "L0.474[620,626] 1.04us 93.73kb                                                       |L0.474|                           "
-    - "L0.405[627,670] 1.04us 589.19kb                                                       |L0.405|                           "
+    - "L0.401[627,670] 1.04us 589.19kb                                                       |L0.401|                           "
     - "L0.232[671,672] 1.04us 13.27kb                                                           |L0.232|                       "
     - "L0.339[673,887] 1.04us 2.78mb                                                            |------L0.339------|          "
     - "L0.475[888,903] 1.04us 203.04kb                                                                                |L0.475|  "
diff --git a/compactor2/tests/layouts/knobs.rs b/compactor2/tests/layouts/knobs.rs
index 99cde595b6..3dbd177639 100644
--- a/compactor2/tests/layouts/knobs.rs
+++ b/compactor2/tests/layouts/knobs.rs
@@ -858,39 +858,39 @@ async fn all_overlapping_l0_max_input_bytes_per_partition() {
     - "L0.?[171444,200000] 5ns 1.29mb|-----------------------------------------L0.?------------------------------------------| "
     - "**** Simulation run 51, type=split(HighL0OverlapTotalBacklog)(split_times=[171443]). 1 Input Files, 1.29mb total:"
     - "L0, all files 1.29mb                                                                                               "
-    - "L0.52[171443,200000] 6ns |-----------------------------------------L0.52------------------------------------------|"
-    - "**** 2 Output Files (parquet_file_id not yet assigned), 1.29mb total:"
-    - "L0                                                                                                                 "
-    - "L0.?[171443,171443] 6ns 0b|L0.?|                                                                                    "
-    - "L0.?[171444,200000] 6ns 1.29mb|-----------------------------------------L0.?------------------------------------------| "
-    - "**** Simulation run 52, type=split(HighL0OverlapTotalBacklog)(split_times=[171443]). 1 Input Files, 1.29mb total:"
-    - "L0, all files 1.29mb                                                                                               "
-    - "L0.59[171443,200000] 7ns |-----------------------------------------L0.59------------------------------------------|"
-    - "**** 2 Output Files (parquet_file_id not yet assigned), 1.29mb total:"
-    - "L0                                                                                                                 "
-    - "L0.?[171443,171443] 7ns 0b|L0.?|                                                                                    "
-    - "L0.?[171444,200000] 7ns 1.29mb|-----------------------------------------L0.?------------------------------------------| "
-    - "**** Simulation run 53, type=split(HighL0OverlapTotalBacklog)(split_times=[171443]). 1 Input Files, 1.29mb total:"
-    - "L0, all files 1.29mb                                                                                               "
     - "L0.66[171443,200000] 8ns |-----------------------------------------L0.66------------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 1.29mb total:"
     - "L0                                                                                                                 "
     - "L0.?[171443,171443] 8ns 0b|L0.?|                                                                                    "
     - "L0.?[171444,200000] 8ns 1.29mb|-----------------------------------------L0.?------------------------------------------| "
-    - "**** Simulation run 54, type=split(HighL0OverlapTotalBacklog)(split_times=[171443]). 1 Input Files, 1.29mb total:"
+    - "**** Simulation run 52, type=split(HighL0OverlapTotalBacklog)(split_times=[171443]). 1 Input Files, 1.29mb total:"
     - "L0, all files 1.29mb                                                                                               "
     - "L0.73[171443,200000] 9ns |-----------------------------------------L0.73------------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 1.29mb total:"
     - "L0                                                                                                                 "
     - "L0.?[171443,171443] 9ns 0b|L0.?|                                                                                    "
     - "L0.?[171444,200000] 9ns 1.29mb|-----------------------------------------L0.?------------------------------------------| "
-    - "**** Simulation run 55, type=split(HighL0OverlapTotalBacklog)(split_times=[171443]). 1 Input Files, 1.29mb total:"
+    - "**** Simulation run 53, type=split(HighL0OverlapTotalBacklog)(split_times=[171443]). 1 Input Files, 1.29mb total:"
     - "L0, all files 1.29mb                                                                                               "
     - "L0.80[171443,200000] 10ns|-----------------------------------------L0.80------------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 1.29mb total:"
     - "L0                                                                                                                 "
     - "L0.?[171443,171443] 10ns 0b|L0.?|                                                                                    "
     - "L0.?[171444,200000] 10ns 1.29mb|-----------------------------------------L0.?------------------------------------------| "
+    - "**** Simulation run 54, type=split(HighL0OverlapTotalBacklog)(split_times=[171443]). 1 Input Files, 1.29mb total:"
+    - "L0, all files 1.29mb                                                                                               "
+    - "L0.52[171443,200000] 6ns |-----------------------------------------L0.52------------------------------------------|"
+    - "**** 2 Output Files (parquet_file_id not yet assigned), 1.29mb total:"
+    - "L0                                                                                                                 "
+    - "L0.?[171443,171443] 6ns 0b|L0.?|                                                                                    "
+    - "L0.?[171444,200000] 6ns 1.29mb|-----------------------------------------L0.?------------------------------------------| "
+    - "**** Simulation run 55, type=split(HighL0OverlapTotalBacklog)(split_times=[171443]). 1 Input Files, 1.29mb total:"
+    - "L0, all files 1.29mb                                                                                               "
+    - "L0.59[171443,200000] 7ns |-----------------------------------------L0.59------------------------------------------|"
+    - "**** 2 Output Files (parquet_file_id not yet assigned), 1.29mb total:"
+    - "L0                                                                                                                 "
+    - "L0.?[171443,171443] 7ns 0b|L0.?|                                                                                    "
+    - "L0.?[171444,200000] 7ns 1.29mb|-----------------------------------------L0.?------------------------------------------| "
     - "Committing partition 1:"
     - "  Soft Deleting 27 files: L0.42, L0.44, L0.45, L0.49, L0.51, L0.52, L0.56, L0.58, L0.59, L0.63, L0.65, L0.66, L0.70, L0.72, L0.73, L0.77, L0.79, L0.80, L0.99, L0.103, L0.107, L0.111, L0.115, L0.119, L1.121, L1.122, L1.123"
     - "  Creating 55 files"
@@ -1213,7 +1213,7 @@ async fn all_overlapping_l0_max_input_bytes_per_partition() {
     - "L0.?[156351,160867] 6ns 208.25kb                                                                   |--------L0.?--------| "
     - "**** Simulation run 95, type=split(ReduceOverlap)(split_times=[198370]). 1 Input Files, 1.29mb total:"
     - "L0, all files 1.29mb                                                                                               "
-    - "L0.170[171444,200000] 6ns|-----------------------------------------L0.170-----------------------------------------|"
+    - "L0.176[171444,200000] 6ns|-----------------------------------------L0.176-----------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 1.29mb total:"
     - "L0                                                                                                                 "
     - "L0.?[171444,198370] 6ns 1.21mb|---------------------------------------L0.?---------------------------------------|      "
@@ -1227,7 +1227,7 @@ async fn all_overlapping_l0_max_input_bytes_per_partition() {
     - "L0.?[156351,160867] 7ns 208.25kb                                                                   |--------L0.?--------| "
     - "**** Simulation run 97, type=split(ReduceOverlap)(split_times=[198370]). 1 Input Files, 1.29mb total:"
     - "L0, all files 1.29mb                                                                                               "
-    - "L0.172[171444,200000] 7ns|-----------------------------------------L0.172-----------------------------------------|"
+    - "L0.178[171444,200000] 7ns|-----------------------------------------L0.178-----------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 1.29mb total:"
     - "L0                                                                                                                 "
     - "L0.?[171444,198370] 7ns 1.21mb|---------------------------------------L0.?---------------------------------------|      "
@@ -1241,7 +1241,7 @@ async fn all_overlapping_l0_max_input_bytes_per_partition() {
     - "L0.?[156351,160867] 8ns 208.25kb                                                                   |--------L0.?--------| "
     - "**** Simulation run 99, type=split(ReduceOverlap)(split_times=[198370]). 1 Input Files, 1.29mb total:"
     - "L0, all files 1.29mb                                                                                               "
-    - "L0.174[171444,200000] 8ns|-----------------------------------------L0.174-----------------------------------------|"
+    - "L0.170[171444,200000] 8ns|-----------------------------------------L0.170-----------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 1.29mb total:"
     - "L0                                                                                                                 "
     - "L0.?[171444,198370] 8ns 1.21mb|---------------------------------------L0.?---------------------------------------|      "
@@ -1255,7 +1255,7 @@ async fn all_overlapping_l0_max_input_bytes_per_partition() {
     - "L0.?[156351,160867] 9ns 208.25kb                                                                   |--------L0.?--------| "
     - "**** Simulation run 101, type=split(ReduceOverlap)(split_times=[198370]). 1 Input Files, 1.29mb total:"
     - "L0, all files 1.29mb                                                                                               "
-    - "L0.176[171444,200000] 9ns|-----------------------------------------L0.176-----------------------------------------|"
+    - "L0.172[171444,200000] 9ns|-----------------------------------------L0.172-----------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 1.29mb total:"
     - "L0                                                                                                                 "
     - "L0.?[171444,198370] 9ns 1.21mb|---------------------------------------L0.?---------------------------------------|      "
@@ -1269,7 +1269,7 @@ async fn all_overlapping_l0_max_input_bytes_per_partition() {
     - "L0.?[156351,160867] 10ns 208.25kb                                                                   |--------L0.?--------| "
     - "**** Simulation run 103, type=split(ReduceOverlap)(split_times=[198370]). 1 Input Files, 1.29mb total:"
     - "L0, all files 1.29mb                                                                                               "
-    - "L0.178[171444,200000] 10ns|-----------------------------------------L0.178-----------------------------------------|"
+    - "L0.174[171444,200000] 10ns|-----------------------------------------L0.174-----------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 1.29mb total:"
     - "L0                                                                                                                 "
     - "L0.?[171444,198370] 10ns 1.21mb|---------------------------------------L0.?---------------------------------------|      "
@@ -1389,7 +1389,7 @@ async fn all_overlapping_l0_max_input_bytes_per_partition() {
     - "L0.257[142887,156350] 6ns 620.71kb                              |---L0.257---|                                              "
     - "L0.258[156351,160867] 6ns 208.25kb                                            |L0.258|                                      "
     - "L0.186[160868,171442] 6ns 487.56kb                                                |-L0.186--|                               "
-    - "L0.169[171443,171443] 6ns 0b                                                           |L0.169|                       "
+    - "L0.175[171443,171443] 6ns 0b                                                           |L0.175|                       "
     - "L0.259[171444,198370] 6ns 1.21mb                                                            |----------L0.259----------|  "
     - "L0.260[198371,200000] 6ns 75.17kb                                                                                        |L0.260|"
     - "L1                                                                                                                 "
@@ -1404,7 +1404,7 @@ async fn all_overlapping_l0_max_input_bytes_per_partition() {
     - "L1.?[149666,185000] 6ns 10mb                                     |---------------L1.?----------------|                "
     - "L1.?[185001,200000] 6ns 4.25mb                                                                          |----L1.?-----| "
     - "Committing partition 1:"
-    - "  Soft Deleting 14 files: L0.104, L0.144, L0.155, L0.169, L0.186, L1.252, L1.253, L1.254, L1.255, L1.256, L0.257, L0.258, L0.259, L0.260"
+    - "  Soft Deleting 14 files: L0.104, L0.144, L0.155, L0.175, L0.186, L1.252, L1.253, L1.254, L1.255, L1.256, L0.257, L0.258, L0.259, L0.260"
     - "  Creating 3 files"
     - "**** Simulation run 116, type=split(HighL0OverlapTotalBacklog)(split_times=[142886]). 1 Input Files, 10mb total:"
     - "L1, all files 10mb                                                                                                 "
@@ -1743,7 +1743,7 @@ async fn all_overlapping_l0_max_input_bytes_per_partition() {
     - "**** Simulation run 156, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[170977]). 8 Input Files, 19.54mb total:"
     - "L0                                                                                                                 "
     - "L0.190[160868,171442] 7ns 487.56kb                              |----L0.190----|                                            "
-    - "L0.171[171443,171443] 7ns 0b                                              |L0.171|                                    "
+    - "L0.177[171443,171443] 7ns 0b                                              |L0.177|                                    "
     - "L0.309[171444,185000] 7ns 625.13kb                                              |------L0.309------|                        "
     - "L0.310[185001,198370] 7ns 616.55kb                                                                   |------L0.310------|   "
     - "L0.264[198371,200000] 7ns 75.17kb                                                                                       |L0.264|"
@@ -1756,7 +1756,7 @@ async fn all_overlapping_l0_max_input_bytes_per_partition() {
     - "L1.?[140564,170977] 7ns 10mb|--------------------L1.?--------------------|                                            "
     - "L1.?[170978,200000] 7ns 9.54mb                                              |------------------L1.?-------------------| "
     - "Committing partition 1:"
-    - "  Soft Deleting 8 files: L0.171, L0.190, L0.264, L1.302, L1.306, L0.309, L0.310, L1.356"
+    - "  Soft Deleting 8 files: L0.177, L0.190, L0.264, L1.302, L1.306, L0.309, L0.310, L1.356"
     - "  Creating 2 files"
     - "**** Simulation run 157, type=split(ReduceOverlap)(split_times=[170977]). 1 Input Files, 487.56kb total:"
     - "L0, all files 487.56kb                                                                                             "
@@ -1924,7 +1924,7 @@ async fn all_overlapping_l0_max_input_bytes_per_partition() {
     - "L0.266[156351,160867] 8ns 208.25kb                       |L0.266|                                                           "
     - "L0.387[160868,170977] 8ns 466.12kb                              |---L0.387----|                                             "
     - "L0.388[170978,171442] 8ns 21.44kb                                              |L0.388|                                    "
-    - "L0.173[171443,171443] 8ns 0b                                              |L0.173|                                    "
+    - "L0.169[171443,171443] 8ns 0b                                              |L0.169|                                    "
     - "L0.313[171444,185000] 8ns 625.13kb                                              |------L0.313------|                        "
     - "L0.314[185001,198370] 8ns 616.55kb                                                                   |------L0.314------|   "
     - "L0.268[198371,200000] 8ns 75.17kb                                                                                       |L0.268|"
@@ -1937,7 +1937,7 @@ async fn all_overlapping_l0_max_input_bytes_per_partition() {
     - "L1.?[167315,194064] 8ns 10mb                                        |-----------------L1.?-----------------|          "
     - "L1.?[194065,200000] 8ns 2.22mb                                                                                 |-L1.?-| "
     - "Committing partition 1:"
-    - "  Soft Deleting 13 files: L0.159, L0.173, L0.266, L0.268, L0.311, L0.312, L0.313, L0.314, L0.376, L1.385, L1.386, L0.387, L0.388"
+    - "  Soft Deleting 13 files: L0.159, L0.169, L0.266, L0.268, L0.311, L0.312, L0.313, L0.314, L0.376, L1.385, L1.386, L0.387, L0.388"
     - "  Creating 3 files"
     - "**** Simulation run 173, type=split(ReduceOverlap)(split_times=[167314]). 1 Input Files, 466.12kb total:"
     - "L0, all files 466.12kb                                                                                             "
@@ -2118,7 +2118,7 @@ async fn all_overlapping_l0_max_input_bytes_per_partition() {
     - "L0                                                                                                                 "
     - "L0.423[167315,170977] 9ns 168.9kb|-L0.423-|                                                                                "
     - "L0.390[170978,171442] 9ns 21.44kb          |L0.390|                                                                        "
-    - "L0.175[171443,171443] 9ns 0b           |L0.175|                                                                       "
+    - "L0.171[171443,171443] 9ns 0b           |L0.171|                                                                       "
     - "L0.317[171444,185000] 9ns 625.13kb           |--------------L0.317---------------|                                          "
     - "L0.424[185001,194064] 9ns 417.97kb                                                |--------L0.424--------|                  "
     - "L0.425[194065,198370] 9ns 198.58kb                                                                         |-L0.425--|      "
@@ -2131,7 +2131,7 @@ async fn all_overlapping_l0_max_input_bytes_per_partition() {
     - "L1.?[167315,191189] 9ns 10mb|-----------------------------L1.?------------------------------|                         "
     - "L1.?[191190,200000] 9ns 3.69mb                                                                 |---------L1.?---------| "
     - "Committing partition 1:"
-    - "  Soft Deleting 9 files: L0.175, L0.272, L0.317, L0.390, L1.420, L1.421, L0.423, L0.424, L0.425"
+    - "  Soft Deleting 9 files: L0.171, L0.272, L0.317, L0.390, L1.420, L1.421, L0.423, L0.424, L0.425"
     - "  Creating 2 files"
     - "**** Simulation run 189, type=split(ReduceOverlap)(split_times=[191189]). 1 Input Files, 417.97kb total:"
     - "L0, all files 417.97kb                                                                                             "
@@ -2201,7 +2201,7 @@ async fn all_overlapping_l0_max_input_bytes_per_partition() {
     - "L0.458[191190,194064] 10ns 132.59kb                                                                        |L0.458|          "
     - "L0.457[185001,191189] 10ns 285.38kb                                                            |-L0.457--|                   "
     - "L0.321[171444,185000] 10ns 625.13kb                                  |---------L0.321---------|                              "
-    - "L0.177[171443,171443] 10ns 0b                                  |L0.177|                                                "
+    - "L0.173[171443,171443] 10ns 0b                                  |L0.173|                                                "
     - "L0.392[170978,171442] 10ns 21.44kb                                 |L0.392|                                                 "
     - "L0.427[167315,170977] 10ns 168.9kb                          |L0.427|                                                        "
     - "L0.426[160868,167314] 10ns 297.22kb              |--L0.426--|                                                                "
@@ -2217,7 +2217,7 @@ async fn all_overlapping_l0_max_input_bytes_per_partition() {
     - "L1.?[175140,196813] 10ns 10mb                                         |-----------------L1.?------------------|        "
     - "L1.?[196814,200000] 10ns 1.47mb                                                                                   |L1.?| "
     - "Committing partition 1:"
-    - "  Soft Deleting 14 files: L0.177, L0.274, L0.276, L0.321, L0.392, L0.426, L0.427, L0.429, L1.450, L0.454, L1.455, L1.456, L0.457, L0.458"
+    - "  Soft Deleting 14 files: L0.173, L0.274, L0.276, L0.321, L0.392, L0.426, L0.427, L0.429, L1.450, L0.454, L1.455, L1.456, L0.457, L0.458"
     - "  Creating 3 files"
     - "**** Simulation run 193, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[126690, 148648]). 16 Input Files, 22.19mb total:"
     - "L0                                                                                                                 "
@@ -2812,39 +2812,39 @@ async fn all_overlapping_l0_max_input_bytes_per_partition_small_max_desired_file
     - "L0.?[171444,200000] 5ns 1.29mb|-----------------------------------------L0.?------------------------------------------| "
     - "**** Simulation run 51, type=split(HighL0OverlapTotalBacklog)(split_times=[171443]). 1 Input Files, 1.29mb total:"
     - "L0, all files 1.29mb                                                                                               "
-    - "L0.52[171443,200000] 6ns |-----------------------------------------L0.52------------------------------------------|"
-    - "**** 2 Output Files (parquet_file_id not yet assigned), 1.29mb total:"
-    - "L0                                                                                                                 "
-    - "L0.?[171443,171443] 6ns 0b|L0.?|                                                                                    "
-    - "L0.?[171444,200000] 6ns 1.29mb|-----------------------------------------L0.?------------------------------------------| "
-    - "**** Simulation run 52, type=split(HighL0OverlapTotalBacklog)(split_times=[171443]). 1 Input Files, 1.29mb total:"
-    - "L0, all files 1.29mb                                                                                               "
-    - "L0.59[171443,200000] 7ns |-----------------------------------------L0.59------------------------------------------|"
-    - "**** 2 Output Files (parquet_file_id not yet assigned), 1.29mb total:"
-    - "L0                                                                                                                 "
-    - "L0.?[171443,171443] 7ns 0b|L0.?|                                                                                    "
-    - "L0.?[171444,200000] 7ns 1.29mb|-----------------------------------------L0.?------------------------------------------| "
-    - "**** Simulation run 53, type=split(HighL0OverlapTotalBacklog)(split_times=[171443]). 1 Input Files, 1.29mb total:"
-    - "L0, all files 1.29mb                                                                                               "
     - "L0.66[171443,200000] 8ns |-----------------------------------------L0.66------------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 1.29mb total:"
     - "L0                                                                                                                 "
     - "L0.?[171443,171443] 8ns 0b|L0.?|                                                                                    "
     - "L0.?[171444,200000] 8ns 1.29mb|-----------------------------------------L0.?------------------------------------------| "
-    - "**** Simulation run 54, type=split(HighL0OverlapTotalBacklog)(split_times=[171443]). 1 Input Files, 1.29mb total:"
+    - "**** Simulation run 52, type=split(HighL0OverlapTotalBacklog)(split_times=[171443]). 1 Input Files, 1.29mb total:"
     - "L0, all files 1.29mb                                                                                               "
     - "L0.73[171443,200000] 9ns |-----------------------------------------L0.73------------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 1.29mb total:"
     - "L0                                                                                                                 "
     - "L0.?[171443,171443] 9ns 0b|L0.?|                                                                                    "
     - "L0.?[171444,200000] 9ns 1.29mb|-----------------------------------------L0.?------------------------------------------| "
-    - "**** Simulation run 55, type=split(HighL0OverlapTotalBacklog)(split_times=[171443]). 1 Input Files, 1.29mb total:"
+    - "**** Simulation run 53, type=split(HighL0OverlapTotalBacklog)(split_times=[171443]). 1 Input Files, 1.29mb total:"
     - "L0, all files 1.29mb                                                                                               "
     - "L0.80[171443,200000] 10ns|-----------------------------------------L0.80------------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 1.29mb total:"
     - "L0                                                                                                                 "
     - "L0.?[171443,171443] 10ns 0b|L0.?|                                                                                    "
     - "L0.?[171444,200000] 10ns 1.29mb|-----------------------------------------L0.?------------------------------------------| "
+    - "**** Simulation run 54, type=split(HighL0OverlapTotalBacklog)(split_times=[171443]). 1 Input Files, 1.29mb total:"
+    - "L0, all files 1.29mb                                                                                               "
+    - "L0.52[171443,200000] 6ns |-----------------------------------------L0.52------------------------------------------|"
+    - "**** 2 Output Files (parquet_file_id not yet assigned), 1.29mb total:"
+    - "L0                                                                                                                 "
+    - "L0.?[171443,171443] 6ns 0b|L0.?|                                                                                    "
+    - "L0.?[171444,200000] 6ns 1.29mb|-----------------------------------------L0.?------------------------------------------| "
+    - "**** Simulation run 55, type=split(HighL0OverlapTotalBacklog)(split_times=[171443]). 1 Input Files, 1.29mb total:"
+    - "L0, all files 1.29mb                                                                                               "
+    - "L0.59[171443,200000] 7ns |-----------------------------------------L0.59------------------------------------------|"
+    - "**** 2 Output Files (parquet_file_id not yet assigned), 1.29mb total:"
+    - "L0                                                                                                                 "
+    - "L0.?[171443,171443] 7ns 0b|L0.?|                                                                                    "
+    - "L0.?[171444,200000] 7ns 1.29mb|-----------------------------------------L0.?------------------------------------------| "
     - "Committing partition 1:"
     - "  Soft Deleting 27 files: L0.42, L0.44, L0.45, L0.49, L0.51, L0.52, L0.56, L0.58, L0.59, L0.63, L0.65, L0.66, L0.70, L0.72, L0.73, L0.77, L0.79, L0.80, L0.99, L0.103, L0.107, L0.111, L0.115, L0.119, L1.121, L1.122, L1.123"
     - "  Creating 55 files"
@@ -3167,7 +3167,7 @@ async fn all_overlapping_l0_max_input_bytes_per_partition_small_max_desired_file
     - "L0.?[156351,160867] 6ns 208.25kb                                                                   |--------L0.?--------| "
     - "**** Simulation run 95, type=split(ReduceOverlap)(split_times=[198370]). 1 Input Files, 1.29mb total:"
     - "L0, all files 1.29mb                                                                                               "
-    - "L0.170[171444,200000] 6ns|-----------------------------------------L0.170-----------------------------------------|"
+    - "L0.176[171444,200000] 6ns|-----------------------------------------L0.176-----------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 1.29mb total:"
     - "L0                                                                                                                 "
     - "L0.?[171444,198370] 6ns 1.21mb|---------------------------------------L0.?---------------------------------------|      "
@@ -3181,7 +3181,7 @@ async fn all_overlapping_l0_max_input_bytes_per_partition_small_max_desired_file
     - "L0.?[156351,160867] 7ns 208.25kb                                                                   |--------L0.?--------| "
     - "**** Simulation run 97, type=split(ReduceOverlap)(split_times=[198370]). 1 Input Files, 1.29mb total:"
     - "L0, all files 1.29mb                                                                                               "
-    - "L0.172[171444,200000] 7ns|-----------------------------------------L0.172-----------------------------------------|"
+    - "L0.178[171444,200000] 7ns|-----------------------------------------L0.178-----------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 1.29mb total:"
     - "L0                                                                                                                 "
     - "L0.?[171444,198370] 7ns 1.21mb|---------------------------------------L0.?---------------------------------------|      "
@@ -3195,7 +3195,7 @@ async fn all_overlapping_l0_max_input_bytes_per_partition_small_max_desired_file
     - "L0.?[156351,160867] 8ns 208.25kb                                                                   |--------L0.?--------| "
     - "**** Simulation run 99, type=split(ReduceOverlap)(split_times=[198370]). 1 Input Files, 1.29mb total:"
     - "L0, all files 1.29mb                                                                                               "
-    - "L0.174[171444,200000] 8ns|-----------------------------------------L0.174-----------------------------------------|"
+    - "L0.170[171444,200000] 8ns|-----------------------------------------L0.170-----------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 1.29mb total:"
     - "L0                                                                                                                 "
     - "L0.?[171444,198370] 8ns 1.21mb|---------------------------------------L0.?---------------------------------------|      "
@@ -3209,7 +3209,7 @@ async fn all_overlapping_l0_max_input_bytes_per_partition_small_max_desired_file
     - "L0.?[156351,160867] 9ns 208.25kb                                                                   |--------L0.?--------| "
     - "**** Simulation run 101, type=split(ReduceOverlap)(split_times=[198370]). 1 Input Files, 1.29mb total:"
     - "L0, all files 1.29mb                                                                                               "
-    - "L0.176[171444,200000] 9ns|-----------------------------------------L0.176-----------------------------------------|"
+    - "L0.172[171444,200000] 9ns|-----------------------------------------L0.172-----------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 1.29mb total:"
     - "L0                                                                                                                 "
     - "L0.?[171444,198370] 9ns 1.21mb|---------------------------------------L0.?---------------------------------------|      "
@@ -3223,7 +3223,7 @@ async fn all_overlapping_l0_max_input_bytes_per_partition_small_max_desired_file
     - "L0.?[156351,160867] 10ns 208.25kb                                                                   |--------L0.?--------| "
     - "**** Simulation run 103, type=split(ReduceOverlap)(split_times=[198370]). 1 Input Files, 1.29mb total:"
     - "L0, all files 1.29mb                                                                                               "
-    - "L0.178[171444,200000] 10ns|-----------------------------------------L0.178-----------------------------------------|"
+    - "L0.174[171444,200000] 10ns|-----------------------------------------L0.174-----------------------------------------|"
     - "**** 2 Output Files (parquet_file_id not yet assigned), 1.29mb total:"
     - "L0                                                                                                                 "
     - "L0.?[171444,198370] 10ns 1.21mb|---------------------------------------L0.?---------------------------------------|      "
@@ -3343,7 +3343,7 @@ async fn all_overlapping_l0_max_input_bytes_per_partition_small_max_desired_file
     - "L0.257[142887,156350] 6ns 620.71kb                              |---L0.257---|                                              "
     - "L0.258[156351,160867] 6ns 208.25kb                                            |L0.258|                                      "
     - "L0.186[160868,171442] 6ns 487.56kb                                                |-L0.186--|                               "
-    - "L0.169[171443,171443] 6ns 0b                                                           |L0.169|                       "
+    - "L0.175[171443,171443] 6ns 0b                                                           |L0.175|                       "
     - "L0.259[171444,198370] 6ns 1.21mb                                                            |----------L0.259----------|  "
     - "L0.260[198371,200000] 6ns 75.17kb                                                                                        |L0.260|"
     - "L1                                                                                                                 "
@@ -3358,7 +3358,7 @@ async fn all_overlapping_l0_max_input_bytes_per_partition_small_max_desired_file
     - "L1.?[149666,185000] 6ns 10mb                                     |---------------L1.?----------------|                "
     - "L1.?[185001,200000] 6ns 4.25mb                                                                          |----L1.?-----| "
     - "Committing partition 1:"
-    - "  Soft Deleting 14 files: L0.104, L0.144, L0.155, L0.169, L0.186, L1.252, L1.253, L1.254, L1.255, L1.256, L0.257, L0.258, L0.259, L0.260"
+    - "  Soft Deleting 14 files: L0.104, L0.144, L0.155, L0.175, L0.186, L1.252, L1.253, L1.254, L1.255, L1.256, L0.257, L0.258, L0.259, L0.260"
     - "  Creating 3 files"
     - "**** Simulation run 116, type=split(HighL0OverlapTotalBacklog)(split_times=[142886]). 1 Input Files, 10mb total:"
     - "L1, all files 10mb                                                                                                 "
@@ -3697,7 +3697,7 @@ async fn all_overlapping_l0_max_input_bytes_per_partition_small_max_desired_file
     - "**** Simulation run 156, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[170977]). 8 Input Files, 19.54mb total:"
     - "L0                                                                                                                 "
     - "L0.190[160868,171442] 7ns 487.56kb                              |----L0.190----|                                            "
-    - "L0.171[171443,171443] 7ns 0b                                              |L0.171|                                    "
+    - "L0.177[171443,171443] 7ns 0b                                              |L0.177|                                    "
     - "L0.309[171444,185000] 7ns 625.13kb                                              |------L0.309------|                        "
     - "L0.310[185001,198370] 7ns 616.55kb                                                                   |------L0.310------|   "
     - "L0.264[198371,200000] 7ns 75.17kb                                                                                       |L0.264|"
@@ -3710,7 +3710,7 @@ async fn all_overlapping_l0_max_input_bytes_per_partition_small_max_desired_file
     - "L1.?[140564,170977] 7ns 10mb|--------------------L1.?--------------------|                                            "
     - "L1.?[170978,200000] 7ns 9.54mb                                              |------------------L1.?-------------------| "
     - "Committing partition 1:"
-    - "  Soft Deleting 8 files: L0.171, L0.190, L0.264, L1.302, L1.306, L0.309, L0.310, L1.356"
+    - "  Soft Deleting 8 files: L0.177, L0.190, L0.264, L1.302, L1.306, L0.309, L0.310, L1.356"
     - "  Creating 2 files"
     - "**** Simulation run 157, type=split(ReduceOverlap)(split_times=[170977]). 1 Input Files, 487.56kb total:"
     - "L0, all files 487.56kb                                                                                             "
@@ -3878,7 +3878,7 @@ async fn all_overlapping_l0_max_input_bytes_per_partition_small_max_desired_file
     - "L0.266[156351,160867] 8ns 208.25kb                       |L0.266|                                                           "
     - "L0.387[160868,170977] 8ns 466.12kb                              |---L0.387----|                                             "
     - "L0.388[170978,171442] 8ns 21.44kb                                              |L0.388|                                    "
-    - "L0.173[171443,171443] 8ns 0b                                              |L0.173|                                    "
+    - "L0.169[171443,171443] 8ns 0b                                              |L0.169|                                    "
     - "L0.313[171444,185000] 8ns 625.13kb                                              |------L0.313------|                        "
     - "L0.314[185001,198370] 8ns 616.55kb                                                                   |------L0.314------|   "
     - "L0.268[198371,200000] 8ns 75.17kb                                                                                       |L0.268|"
@@ -3891,7 +3891,7 @@ async fn all_overlapping_l0_max_input_bytes_per_partition_small_max_desired_file
     - "L1.?[167315,194064] 8ns 10mb                                        |-----------------L1.?-----------------|          "
     - "L1.?[194065,200000] 8ns 2.22mb                                                                                 |-L1.?-| "
     - "Committing partition 1:"
-    - "  Soft Deleting 13 files: L0.159, L0.173, L0.266, L0.268, L0.311, L0.312, L0.313, L0.314, L0.376, L1.385, L1.386, L0.387, L0.388"
+    - "  Soft Deleting 13 files: L0.159, L0.169, L0.266, L0.268, L0.311, L0.312, L0.313, L0.314, L0.376, L1.385, L1.386, L0.387, L0.388"
     - "  Creating 3 files"
     - "**** Simulation run 173, type=split(ReduceOverlap)(split_times=[167314]). 1 Input Files, 466.12kb total:"
     - "L0, all files 466.12kb                                                                                             "
@@ -4072,7 +4072,7 @@ async fn all_overlapping_l0_max_input_bytes_per_partition_small_max_desired_file
     - "L0                                                                                                                 "
     - "L0.423[167315,170977] 9ns 168.9kb|-L0.423-|                                                                                "
     - "L0.390[170978,171442] 9ns 21.44kb          |L0.390|                                                                        "
-    - "L0.175[171443,171443] 9ns 0b           |L0.175|                                                                       "
+    - "L0.171[171443,171443] 9ns 0b           |L0.171|                                                                       "
     - "L0.317[171444,185000] 9ns 625.13kb           |--------------L0.317---------------|                                          "
     - "L0.424[185001,194064] 9ns 417.97kb                                                |--------L0.424--------|                  "
     - "L0.425[194065,198370] 9ns 198.58kb                                                                         |-L0.425--|      "
@@ -4085,7 +4085,7 @@ async fn all_overlapping_l0_max_input_bytes_per_partition_small_max_desired_file
     - "L1.?[167315,191189] 9ns 10mb|-----------------------------L1.?------------------------------|                         "
     - "L1.?[191190,200000] 9ns 3.69mb                                                                 |---------L1.?---------| "
     - "Committing partition 1:"
-    - "  Soft Deleting 9 files: L0.175, L0.272, L0.317, L0.390, L1.420, L1.421, L0.423, L0.424, L0.425"
+    - "  Soft Deleting 9 files: L0.171, L0.272, L0.317, L0.390, L1.420, L1.421, L0.423, L0.424, L0.425"
     - "  Creating 2 files"
     - "**** Simulation run 189, type=split(ReduceOverlap)(split_times=[191189]). 1 Input Files, 417.97kb total:"
     - "L0, all files 417.97kb                                                                                             "
@@ -4155,7 +4155,7 @@ async fn all_overlapping_l0_max_input_bytes_per_partition_small_max_desired_file
     - "L0.458[191190,194064] 10ns 132.59kb                                                                        |L0.458|          "
     - "L0.457[185001,191189] 10ns 285.38kb                                                            |-L0.457--|                   "
     - "L0.321[171444,185000] 10ns 625.13kb                                  |---------L0.321---------|                              "
-    - "L0.177[171443,171443] 10ns 0b                                  |L0.177|                                                "
+    - "L0.173[171443,171443] 10ns 0b                                  |L0.173|                                                "
     - "L0.392[170978,171442] 10ns 21.44kb                                 |L0.392|                                                 "
     - "L0.427[167315,170977] 10ns 168.9kb                          |L0.427|                                                        "
     - "L0.426[160868,167314] 10ns 297.22kb              |--L0.426--|                                                                "
@@ -4171,7 +4171,7 @@ async fn all_overlapping_l0_max_input_bytes_per_partition_small_max_desired_file
     - "L1.?[175140,196813] 10ns 10mb                                         |-----------------L1.?------------------|        "
     - "L1.?[196814,200000] 10ns 1.47mb                                                                                   |L1.?| "
     - "Committing partition 1:"
-    - "  Soft Deleting 14 files: L0.177, L0.274, L0.276, L0.321, L0.392, L0.426, L0.427, L0.429, L1.450, L0.454, L1.455, L1.456, L0.457, L0.458"
+    - "  Soft Deleting 14 files: L0.173, L0.274, L0.276, L0.321, L0.392, L0.426, L0.427, L0.429, L1.450, L0.454, L1.455, L1.456, L0.457, L0.458"
     - "  Creating 3 files"
     - "**** Simulation run 193, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[126690, 148648]). 16 Input Files, 22.19mb total:"
     - "L0                                                                                                                 "

From 06bc2d62726ac34720a946e27635219889ea64c0 Mon Sep 17 00:00:00 2001
From: Stuart Carnie <stuart.carnie@gmail.com>
Date: Tue, 25 Apr 2023 07:15:44 +1000
Subject: [PATCH 09/25] feat: Add `pre` and `post` visit events for `Literal`
 (#7629)

---
 ...rser__visit__test__delete_statement-2.snap |  4 +++
 ...rser__visit__test__select_statement-6.snap |  6 ++++
 ...__test__show_measurements_statement-4.snap |  2 ++
 ...__test__show_measurements_statement-7.snap |  2 ++
 ...__visit_mut__test__delete_statement-2.snap |  4 +++
 ...__visit_mut__test__select_statement-6.snap |  6 ++++
 ...__test__show_measurements_statement-4.snap |  2 ++
 ...__test__show_measurements_statement-7.snap |  2 ++
 influxdb_influxql_parser/src/visit.rs         | 29 +++++++++++++++++--
 influxdb_influxql_parser/src/visit_mut.rs     | 26 ++++++++++++++++-
 10 files changed, 79 insertions(+), 4 deletions(-)

diff --git a/influxdb_influxql_parser/src/snapshots/influxdb_influxql_parser__visit__test__delete_statement-2.snap b/influxdb_influxql_parser/src/snapshots/influxdb_influxql_parser__visit__test__delete_statement-2.snap
index d727dc5bdf..6958fd4dab 100644
--- a/influxdb_influxql_parser/src/snapshots/influxdb_influxql_parser__visit__test__delete_statement-2.snap
+++ b/influxdb_influxql_parser/src/snapshots/influxdb_influxql_parser__visit__test__delete_statement-2.snap
@@ -9,10 +9,14 @@ expression: "visit_statement!(\"DELETE WHERE 'foo bar' =~ /foo/\")"
 - pre_visit_conditional_binary
 - pre_visit_conditional_expression
 - pre_visit_expr
+- pre_visit_literal
+- post_visit_literal
 - post_visit_expr
 - post_visit_conditional_expression
 - pre_visit_conditional_expression
 - pre_visit_expr
+- pre_visit_literal
+- post_visit_literal
 - post_visit_expr
 - post_visit_conditional_expression
 - post_visit_conditional_binary
diff --git a/influxdb_influxql_parser/src/snapshots/influxdb_influxql_parser__visit__test__select_statement-6.snap b/influxdb_influxql_parser/src/snapshots/influxdb_influxql_parser__visit__test__select_statement-6.snap
index 46499ab53c..0091841665 100644
--- a/influxdb_influxql_parser/src/snapshots/influxdb_influxql_parser__visit__test__select_statement-6.snap
+++ b/influxdb_influxql_parser/src/snapshots/influxdb_influxql_parser__visit__test__select_statement-6.snap
@@ -65,6 +65,8 @@ expression: "visit_statement!(r#\"SELECT value FROM (SELECT usage FROM cpu WHERE
 - post_visit_conditional_expression
 - pre_visit_conditional_expression
 - pre_visit_expr
+- pre_visit_literal
+- post_visit_literal
 - post_visit_expr
 - post_visit_conditional_expression
 - post_visit_conditional_binary
@@ -79,6 +81,8 @@ expression: "visit_statement!(r#\"SELECT value FROM (SELECT usage FROM cpu WHERE
 - post_visit_conditional_expression
 - pre_visit_conditional_expression
 - pre_visit_expr
+- pre_visit_literal
+- post_visit_literal
 - post_visit_expr
 - post_visit_conditional_expression
 - post_visit_conditional_binary
@@ -90,6 +94,8 @@ expression: "visit_statement!(r#\"SELECT value FROM (SELECT usage FROM cpu WHERE
 - pre_visit_select_dimension
 - pre_visit_select_time_dimension
 - pre_visit_expr
+- pre_visit_literal
+- post_visit_literal
 - post_visit_expr
 - post_visit_select_time_dimension
 - post_visit_select_dimension
diff --git a/influxdb_influxql_parser/src/snapshots/influxdb_influxql_parser__visit__test__show_measurements_statement-4.snap b/influxdb_influxql_parser/src/snapshots/influxdb_influxql_parser__visit__test__show_measurements_statement-4.snap
index dbbd9ad010..002605df05 100644
--- a/influxdb_influxql_parser/src/snapshots/influxdb_influxql_parser__visit__test__show_measurements_statement-4.snap
+++ b/influxdb_influxql_parser/src/snapshots/influxdb_influxql_parser__visit__test__show_measurements_statement-4.snap
@@ -15,6 +15,8 @@ expression: "visit_statement!(\"SHOW MEASUREMENTS WHERE host = 'west'\")"
 - post_visit_conditional_expression
 - pre_visit_conditional_expression
 - pre_visit_expr
+- pre_visit_literal
+- post_visit_literal
 - post_visit_expr
 - post_visit_conditional_expression
 - post_visit_conditional_binary
diff --git a/influxdb_influxql_parser/src/snapshots/influxdb_influxql_parser__visit__test__show_measurements_statement-7.snap b/influxdb_influxql_parser/src/snapshots/influxdb_influxql_parser__visit__test__show_measurements_statement-7.snap
index e02e9fbbcb..8f97462aa6 100644
--- a/influxdb_influxql_parser/src/snapshots/influxdb_influxql_parser__visit__test__show_measurements_statement-7.snap
+++ b/influxdb_influxql_parser/src/snapshots/influxdb_influxql_parser__visit__test__show_measurements_statement-7.snap
@@ -23,6 +23,8 @@ expression: "visit_statement!(\"SHOW MEASUREMENTS ON * WITH MEASUREMENT =~ /foo/
 - post_visit_conditional_expression
 - pre_visit_conditional_expression
 - pre_visit_expr
+- pre_visit_literal
+- post_visit_literal
 - post_visit_expr
 - post_visit_conditional_expression
 - post_visit_conditional_binary
diff --git a/influxdb_influxql_parser/src/snapshots/influxdb_influxql_parser__visit_mut__test__delete_statement-2.snap b/influxdb_influxql_parser/src/snapshots/influxdb_influxql_parser__visit_mut__test__delete_statement-2.snap
index 0ab7365019..cd15174ac4 100644
--- a/influxdb_influxql_parser/src/snapshots/influxdb_influxql_parser__visit_mut__test__delete_statement-2.snap
+++ b/influxdb_influxql_parser/src/snapshots/influxdb_influxql_parser__visit_mut__test__delete_statement-2.snap
@@ -9,10 +9,14 @@ expression: "visit_statement!(\"DELETE WHERE 'foo bar' =~ /foo/\")"
 - pre_visit_conditional_binary
 - pre_visit_conditional_expression
 - pre_visit_expr
+- pre_visit_literal
+- post_visit_literal
 - post_visit_expr
 - post_visit_conditional_expression
 - pre_visit_conditional_expression
 - pre_visit_expr
+- pre_visit_literal
+- post_visit_literal
 - post_visit_expr
 - post_visit_conditional_expression
 - post_visit_conditional_binary
diff --git a/influxdb_influxql_parser/src/snapshots/influxdb_influxql_parser__visit_mut__test__select_statement-6.snap b/influxdb_influxql_parser/src/snapshots/influxdb_influxql_parser__visit_mut__test__select_statement-6.snap
index ac3b0e1fda..f14370ca75 100644
--- a/influxdb_influxql_parser/src/snapshots/influxdb_influxql_parser__visit_mut__test__select_statement-6.snap
+++ b/influxdb_influxql_parser/src/snapshots/influxdb_influxql_parser__visit_mut__test__select_statement-6.snap
@@ -65,6 +65,8 @@ expression: "visit_statement!(r#\"SELECT value FROM (SELECT usage FROM cpu WHERE
 - post_visit_conditional_expression
 - pre_visit_conditional_expression
 - pre_visit_expr
+- pre_visit_literal
+- post_visit_literal
 - post_visit_expr
 - post_visit_conditional_expression
 - post_visit_conditional_binary
@@ -79,6 +81,8 @@ expression: "visit_statement!(r#\"SELECT value FROM (SELECT usage FROM cpu WHERE
 - post_visit_conditional_expression
 - pre_visit_conditional_expression
 - pre_visit_expr
+- pre_visit_literal
+- post_visit_literal
 - post_visit_expr
 - post_visit_conditional_expression
 - post_visit_conditional_binary
@@ -90,6 +94,8 @@ expression: "visit_statement!(r#\"SELECT value FROM (SELECT usage FROM cpu WHERE
 - pre_visit_select_dimension
 - pre_visit_select_time_dimension
 - pre_visit_expr
+- pre_visit_literal
+- post_visit_literal
 - post_visit_expr
 - post_visit_select_time_dimension
 - post_visit_select_dimension
diff --git a/influxdb_influxql_parser/src/snapshots/influxdb_influxql_parser__visit_mut__test__show_measurements_statement-4.snap b/influxdb_influxql_parser/src/snapshots/influxdb_influxql_parser__visit_mut__test__show_measurements_statement-4.snap
index a2bc2ff6c9..e5c2e979b1 100644
--- a/influxdb_influxql_parser/src/snapshots/influxdb_influxql_parser__visit_mut__test__show_measurements_statement-4.snap
+++ b/influxdb_influxql_parser/src/snapshots/influxdb_influxql_parser__visit_mut__test__show_measurements_statement-4.snap
@@ -15,6 +15,8 @@ expression: "visit_statement!(\"SHOW MEASUREMENTS WHERE host = 'west'\")"
 - post_visit_conditional_expression
 - pre_visit_conditional_expression
 - pre_visit_expr
+- pre_visit_literal
+- post_visit_literal
 - post_visit_expr
 - post_visit_conditional_expression
 - post_visit_conditional_binary
diff --git a/influxdb_influxql_parser/src/snapshots/influxdb_influxql_parser__visit_mut__test__show_measurements_statement-7.snap b/influxdb_influxql_parser/src/snapshots/influxdb_influxql_parser__visit_mut__test__show_measurements_statement-7.snap
index 9279f2d344..8b801678c4 100644
--- a/influxdb_influxql_parser/src/snapshots/influxdb_influxql_parser__visit_mut__test__show_measurements_statement-7.snap
+++ b/influxdb_influxql_parser/src/snapshots/influxdb_influxql_parser__visit_mut__test__show_measurements_statement-7.snap
@@ -23,6 +23,8 @@ expression: "visit_statement!(\"SHOW MEASUREMENTS ON * WITH MEASUREMENT =~ /foo/
 - post_visit_conditional_expression
 - pre_visit_conditional_expression
 - pre_visit_expr
+- pre_visit_literal
+- post_visit_literal
 - post_visit_expr
 - post_visit_conditional_expression
 - post_visit_conditional_binary
diff --git a/influxdb_influxql_parser/src/visit.rs b/influxdb_influxql_parser/src/visit.rs
index 2faff1d48d..809308c635 100644
--- a/influxdb_influxql_parser/src/visit.rs
+++ b/influxdb_influxql_parser/src/visit.rs
@@ -35,6 +35,7 @@ use crate::explain::ExplainStatement;
 use crate::expression::arithmetic::Expr;
 use crate::expression::conditional::ConditionalExpression;
 use crate::expression::{Binary, Call, ConditionalBinary, VarRef};
+use crate::literal::Literal;
 use crate::select::{
     Dimension, Field, FieldList, FillClause, FromMeasurementClause, GroupByClause,
     MeasurementSelection, SLimitClause, SOffsetClause, SelectStatement, TimeDimension,
@@ -585,6 +586,16 @@ pub trait Visitor: Sized {
     fn post_visit_conditional_binary(self, _n: &ConditionalBinary) -> Result<Self, Self::Error> {
         Ok(self)
     }
+
+    /// Invoked before any children of a literal are visited.
+    fn pre_visit_literal(self, _n: &Literal) -> Result<Recursion<Self>, Self::Error> {
+        Ok(Continue(self))
+    }
+
+    /// Invoked after a literal is visited.
+    fn post_visit_literal(self, _n: &Literal) -> Result<Self, Self::Error> {
+        Ok(self)
+    }
 }
 
 /// Trait for types that can be visited by [`Visitor`]
@@ -1242,18 +1253,28 @@ impl Visitable for Expr {
             Self::Binary(expr) => expr.accept(visitor),
             Self::Nested(expr) => expr.accept(visitor),
             Self::VarRef(expr) => expr.accept(visitor),
+            Self::Literal(expr) => expr.accept(visitor),
 
             // We explicitly list out each enumeration, to ensure
             // we revisit if new items are added to the Expr enumeration.
-            Self::BindParameter(_) | Self::Literal(_) | Self::Wildcard(_) | Self::Distinct(_) => {
-                Ok(visitor)
-            }
+            Self::BindParameter(_) | Self::Wildcard(_) | Self::Distinct(_) => Ok(visitor),
         }?;
 
         visitor.post_visit_expr(self)
     }
 }
 
+impl Visitable for Literal {
+    fn accept<V: Visitor>(&self, visitor: V) -> Result<V, V::Error> {
+        let visitor = match visitor.pre_visit_literal(self)? {
+            Continue(visitor) => visitor,
+            Stop(visitor) => return Ok(visitor),
+        };
+
+        visitor.post_visit_literal(self)
+    }
+}
+
 impl Visitable for OnClause {
     fn accept<V: Visitor>(&self, visitor: V) -> Result<V, V::Error> {
         let visitor = match visitor.pre_visit_on_clause(self)? {
@@ -1331,6 +1352,7 @@ mod test {
     use crate::expression::arithmetic::Expr;
     use crate::expression::conditional::ConditionalExpression;
     use crate::expression::{Binary, Call, ConditionalBinary, VarRef};
+    use crate::literal::Literal;
     use crate::select::{
         Dimension, Field, FieldList, FillClause, FromMeasurementClause, GroupByClause,
         MeasurementSelection, SLimitClause, SOffsetClause, SelectStatement, TimeDimension,
@@ -1427,6 +1449,7 @@ mod test {
         trace_visit!(call, Call);
         trace_visit!(expr_binary, Binary);
         trace_visit!(conditional_binary, ConditionalBinary);
+        trace_visit!(literal, Literal);
     }
 
     macro_rules! visit_statement {
diff --git a/influxdb_influxql_parser/src/visit_mut.rs b/influxdb_influxql_parser/src/visit_mut.rs
index d7830ce5cd..88f8cf9b4f 100644
--- a/influxdb_influxql_parser/src/visit_mut.rs
+++ b/influxdb_influxql_parser/src/visit_mut.rs
@@ -35,6 +35,7 @@ use crate::explain::ExplainStatement;
 use crate::expression::arithmetic::Expr;
 use crate::expression::conditional::ConditionalExpression;
 use crate::expression::{Binary, Call, ConditionalBinary, VarRef};
+use crate::literal::Literal;
 use crate::select::{
     Dimension, Field, FieldList, FillClause, FromMeasurementClause, GroupByClause,
     MeasurementSelection, SLimitClause, SOffsetClause, SelectStatement, TimeDimension,
@@ -616,6 +617,16 @@ pub trait VisitorMut: Sized {
     ) -> Result<(), Self::Error> {
         Ok(())
     }
+
+    /// Invoked before any children of a literal are visited.
+    fn pre_visit_literal(&mut self, _n: &mut Literal) -> Result<Recursion, Self::Error> {
+        Ok(Continue)
+    }
+
+    /// Invoked after a literal is visited.
+    fn post_visit_literal(&mut self, _n: &mut Literal) -> Result<(), Self::Error> {
+        Ok(())
+    }
 }
 
 /// Trait for types that can be visited by [`VisitorMut`]
@@ -1185,16 +1196,27 @@ impl VisitableMut for Expr {
             Self::Binary(expr) => expr.accept(visitor)?,
             Self::Nested(expr) => expr.accept(visitor)?,
             Self::VarRef(expr) => expr.accept(visitor)?,
+            Self::Literal(expr) => expr.accept(visitor)?,
 
             // We explicitly list out each enumeration, to ensure
             // we revisit if new items are added to the Expr enumeration.
-            Self::BindParameter(_) | Self::Literal(_) | Self::Wildcard(_) | Self::Distinct(_) => {}
+            Self::BindParameter(_) | Self::Wildcard(_) | Self::Distinct(_) => {}
         };
 
         visitor.post_visit_expr(self)
     }
 }
 
+impl VisitableMut for Literal {
+    fn accept<V: VisitorMut>(&mut self, visitor: &mut V) -> Result<(), V::Error> {
+        if let Stop = visitor.pre_visit_literal(self)? {
+            return Ok(());
+        };
+
+        visitor.post_visit_literal(self)
+    }
+}
+
 impl VisitableMut for OnClause {
     fn accept<V: VisitorMut>(&mut self, visitor: &mut V) -> Result<(), V::Error> {
         if let Stop = visitor.pre_visit_on_clause(self)? {
@@ -1267,6 +1289,7 @@ mod test {
     use crate::expression::arithmetic::Expr;
     use crate::expression::conditional::ConditionalExpression;
     use crate::expression::{Binary, Call, ConditionalBinary, VarRef};
+    use crate::literal::Literal;
     use crate::parse_statements;
     use crate::select::{
         Dimension, Field, FieldList, FillClause, FromMeasurementClause, GroupByClause,
@@ -1362,6 +1385,7 @@ mod test {
         trace_visit!(call, Call);
         trace_visit!(expr_binary, Binary);
         trace_visit!(conditional_binary, ConditionalBinary);
+        trace_visit!(literal, Literal);
     }
 
     macro_rules! visit_statement {

From b3cdea25ba47012bf830df97eca4eaf8863ed315 Mon Sep 17 00:00:00 2001
From: wiedld <dlw405@gmail.com>
Date: Mon, 24 Apr 2023 17:07:25 -0700
Subject: [PATCH 10/25] chore: update term conventions to allowlist

---
 data_types/src/namespace_name.rs | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/data_types/src/namespace_name.rs b/data_types/src/namespace_name.rs
index 040e8ee9d4..92cb8edd7f 100644
--- a/data_types/src/namespace_name.rs
+++ b/data_types/src/namespace_name.rs
@@ -7,10 +7,10 @@ use thiserror::Error;
 /// A `RangeInclusive` is a closed interval, covering [1, 64]
 const LENGTH_CONSTRAINT: RangeInclusive<usize> = 1..=64;
 
-/// Whitelisted chars for a [`NamespaceName`] name.
+/// Allowlist of chars for a [`NamespaceName`] name.
 ///
 /// '/' | '_' | '-' are utilized by the platforms.
-fn is_whitelisted(c: char) -> bool {
+fn is_allowed(c: char) -> bool {
     c.is_alphanumeric() || matches!(c, '/' | '_' | '-')
 }
 
@@ -98,7 +98,7 @@ impl<'a> NamespaceName<'a> {
         //
         // NOTE: If changing these characters, please update the error message
         // above.
-        if let Some(bad_char_offset) = name.chars().position(|c| !is_whitelisted(c)) {
+        if let Some(bad_char_offset) = name.chars().position(|c| !is_allowed(c)) {
             return Err(NamespaceNameError::BadChars {
                 bad_char_offset,
                 name: name.to_string(),

From 38262f37cb8a2edc952a25fa5f7eb6a5e9c28d14 Mon Sep 17 00:00:00 2001
From: "Carol (Nichols || Goulding)" <carol.nichols@gmail.com>
Date: Sat, 22 Apr 2023 21:10:10 -0400
Subject: [PATCH 11/25] refactor: Take an Arc as an argument to be more
 consistent with similar types

NamespaceData::new doesn't take an Arc of the namespace name loader, but
PartitionData::new does. If they both take Arcs, then making test
constants and helpers they can share is easier. Same deal for
TableData::new.
---
 ingester2/src/buffer_tree/namespace.rs | 8 ++++----
 ingester2/src/buffer_tree/root.rs      | 4 ++--
 ingester2/src/buffer_tree/table.rs     | 8 ++++----
 3 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/ingester2/src/buffer_tree/namespace.rs b/ingester2/src/buffer_tree/namespace.rs
index 8bcd41742e..daf780dfd9 100644
--- a/ingester2/src/buffer_tree/namespace.rs
+++ b/ingester2/src/buffer_tree/namespace.rs
@@ -85,7 +85,7 @@ impl<O> NamespaceData<O> {
     /// Initialize new tables with default partition template of daily
     pub(super) fn new(
         namespace_id: NamespaceId,
-        namespace_name: DeferredLoad<NamespaceName>,
+        namespace_name: Arc<DeferredLoad<NamespaceName>>,
         table_name_resolver: Arc<dyn TableNameProvider>,
         partition_provider: Arc<dyn PartitionProvider>,
         post_write_observer: Arc<O>,
@@ -101,7 +101,7 @@ impl<O> NamespaceData<O> {
 
         Self {
             namespace_id,
-            namespace_name: Arc::new(namespace_name),
+            namespace_name,
             tables: Default::default(),
             table_name_resolver,
             table_count,
@@ -161,7 +161,7 @@ where
                         self.table_count.inc(1);
                         Arc::new(TableData::new(
                             table_id,
-                            self.table_name_resolver.for_table(table_id),
+                            Arc::new(self.table_name_resolver.for_table(table_id)),
                             self.namespace_id,
                             Arc::clone(&self.namespace_name),
                             Arc::clone(&self.partition_provider),
@@ -276,7 +276,7 @@ mod tests {
 
         let ns = NamespaceData::new(
             NAMESPACE_ID,
-            DeferredLoad::new(Duration::from_millis(1), async { NAMESPACE_NAME.into() }),
+            Arc::new(DeferredLoad::new(Duration::from_millis(1), async { NAMESPACE_NAME.into() })),
             Arc::new(MockTableNameProvider::new(TABLE_NAME)),
             partition_provider,
             Arc::new(MockPostWriteObserver::default()),
diff --git a/ingester2/src/buffer_tree/root.rs b/ingester2/src/buffer_tree/root.rs
index 6b0c98961d..d14f868029 100644
--- a/ingester2/src/buffer_tree/root.rs
+++ b/ingester2/src/buffer_tree/root.rs
@@ -180,7 +180,7 @@ where
 
             Arc::new(NamespaceData::new(
                 namespace_id,
-                self.namespace_name_resolver.for_namespace(namespace_id),
+                Arc::new(self.namespace_name_resolver.for_namespace(namespace_id)),
                 Arc::clone(&self.table_name_resolver),
                 Arc::clone(&self.partition_provider),
                 Arc::clone(&self.post_write_observer),
@@ -286,7 +286,7 @@ mod tests {
         // Init the namespace
         let ns = NamespaceData::new(
             NAMESPACE_ID,
-            DeferredLoad::new(Duration::from_millis(1), async { NAMESPACE_NAME.into() }),
+            Arc::new(DeferredLoad::new(Duration::from_millis(1), async { NAMESPACE_NAME.into() })),
             Arc::new(MockTableNameProvider::new(TABLE_NAME)),
             partition_provider,
             Arc::new(MockPostWriteObserver::default()),
diff --git a/ingester2/src/buffer_tree/table.rs b/ingester2/src/buffer_tree/table.rs
index 8741b712b4..5ed810e567 100644
--- a/ingester2/src/buffer_tree/table.rs
+++ b/ingester2/src/buffer_tree/table.rs
@@ -95,7 +95,7 @@ impl<O> TableData<O> {
     /// for the first time.
     pub(super) fn new(
         table_id: TableId,
-        table_name: DeferredLoad<TableName>,
+        table_name: Arc<DeferredLoad<TableName>>,
         namespace_id: NamespaceId,
         namespace_name: Arc<DeferredLoad<NamespaceName>>,
         partition_provider: Arc<dyn PartitionProvider>,
@@ -104,7 +104,7 @@ impl<O> TableData<O> {
     ) -> Self {
         Self {
             table_id,
-            table_name: Arc::new(table_name),
+            table_name,
             namespace_id,
             namespace_name,
             partition_data: Default::default(),
@@ -301,9 +301,9 @@ mod tests {
 
         let table = TableData::new(
             TABLE_ID,
-            DeferredLoad::new(Duration::from_secs(1), async {
+            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
                 TableName::from(TABLE_NAME)
-            }),
+            })),
             NAMESPACE_ID,
             Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
                 NamespaceName::from("platanos")

From 2aaa431d5a93704bca3fc58e3ba21dbf5f31fc02 Mon Sep 17 00:00:00 2001
From: "Carol (Nichols || Goulding)" <carol.nichols@gmail.com>
Date: Mon, 24 Apr 2023 15:51:13 -0400
Subject: [PATCH 12/25] fix: You can't just call everything bananas, Dom

Remove the MockNamespaceNameProvider::default implementation that
hardcodes all namespace names to "bananas"; I just tore some of my hair
out trying to figure out why I was getting bananas WHEN THERE SHOULD
HAVE BEEN NO BANANAS
---
 ingester2/src/buffer_tree/namespace/name_resolver.rs |  6 ------
 ingester2/src/buffer_tree/root.rs                    | 10 +++++-----
 ingester2/src/persist/mod.rs                         |  2 +-
 3 files changed, 6 insertions(+), 12 deletions(-)

diff --git a/ingester2/src/buffer_tree/namespace/name_resolver.rs b/ingester2/src/buffer_tree/namespace/name_resolver.rs
index 71fc8adc12..9d096e4c5e 100644
--- a/ingester2/src/buffer_tree/namespace/name_resolver.rs
+++ b/ingester2/src/buffer_tree/namespace/name_resolver.rs
@@ -90,12 +90,6 @@ pub(crate) mod mock {
         }
     }
 
-    impl Default for MockNamespaceNameProvider {
-        fn default() -> Self {
-            Self::new("bananas")
-        }
-    }
-
     impl NamespaceNameProvider for MockNamespaceNameProvider {
         fn for_namespace(&self, _id: NamespaceId) -> DeferredLoad<NamespaceName> {
             let name = self.name.clone();
diff --git a/ingester2/src/buffer_tree/root.rs b/ingester2/src/buffer_tree/root.rs
index d14f868029..e38c6914d4 100644
--- a/ingester2/src/buffer_tree/root.rs
+++ b/ingester2/src/buffer_tree/root.rs
@@ -356,7 +356,7 @@ mod tests {
 
                     // Init the buffer tree
                     let buf = BufferTree::new(
-                        Arc::new(MockNamespaceNameProvider::default()),
+                        Arc::new(MockNamespaceNameProvider::new(NAMESPACE_NAME)),
                         Arc::new(MockTableNameProvider::new(TABLE_NAME)),
                         partition_provider,
                         Arc::new(MockPostWriteObserver::default()),
@@ -699,7 +699,7 @@ mod tests {
 
         // Init the buffer tree
         let buf = BufferTree::new(
-            Arc::new(MockNamespaceNameProvider::default()),
+            Arc::new(MockNamespaceNameProvider::new(NAMESPACE_NAME)),
             Arc::new(MockTableNameProvider::new(TABLE_NAME)),
             partition_provider,
             Arc::new(MockPostWriteObserver::default()),
@@ -808,7 +808,7 @@ mod tests {
 
         // Init the buffer tree
         let buf = BufferTree::new(
-            Arc::new(MockNamespaceNameProvider::default()),
+            Arc::new(MockNamespaceNameProvider::new(NAMESPACE_NAME)),
             Arc::new(MockTableNameProvider::new(TABLE_NAME)),
             partition_provider,
             Arc::new(MockPostWriteObserver::default()),
@@ -893,7 +893,7 @@ mod tests {
 
         // Init the BufferTree
         let buf = BufferTree::new(
-            Arc::new(MockNamespaceNameProvider::default()),
+            Arc::new(MockNamespaceNameProvider::new(NAMESPACE_NAME)),
             Arc::new(MockTableNameProvider::new(TABLE_NAME)),
             partition_provider,
             Arc::new(MockPostWriteObserver::default()),
@@ -992,7 +992,7 @@ mod tests {
 
         // Init the buffer tree
         let buf = BufferTree::new(
-            Arc::new(MockNamespaceNameProvider::default()),
+            Arc::new(MockNamespaceNameProvider::new(NAMESPACE_NAME)),
             Arc::new(MockTableNameProvider::new(TABLE_NAME)),
             partition_provider,
             Arc::new(MockPostWriteObserver::default()),
diff --git a/ingester2/src/persist/mod.rs b/ingester2/src/persist/mod.rs
index e72fd1d38e..f4f4cfe4e8 100644
--- a/ingester2/src/persist/mod.rs
+++ b/ingester2/src/persist/mod.rs
@@ -75,7 +75,7 @@ mod tests {
 
         // Init the buffer tree
         let buf = BufferTree::new(
-            Arc::new(MockNamespaceNameProvider::default()),
+            Arc::new(MockNamespaceNameProvider::new(NAMESPACE_NAME)),
             Arc::new(MockTableNameProvider::new(TABLE_NAME)),
             Arc::new(CatalogPartitionResolver::new(Arc::clone(&catalog))),
             Arc::new(MockPostWriteObserver::default()),

From 148c103c701599fba491e8dd1a566adc3c3ee398 Mon Sep 17 00:00:00 2001
From: wiedld <wiedld@users.noreply.github.com>
Date: Mon, 24 Apr 2023 17:14:28 -0700
Subject: [PATCH 13/25] refactor: push authz into specific configs (#7607)

* refactor: move authz-addr flag into router-specific config

* refactor: move authz-addr flag into querier-specific config

* refactor: remove global AuthzConfig which is now redundant with the pushdown to individual configs. Keep constant the env vars used universally.

* chore: make errors lowercase, and use the required bool for the authz-addr flag

---------

Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com>
---
 clap_blocks/src/authz.rs                    | 42 -----------------
 clap_blocks/src/lib.rs                      |  2 +-
 clap_blocks/src/querier.rs                  |  9 +++-
 clap_blocks/src/router2.rs                  | 22 +++++++--
 clap_blocks/src/single_tenant.rs            | 11 +++++
 influxdb_iox/src/commands/run/all_in_one.rs | 51 +++++++++------------
 influxdb_iox/src/commands/run/querier.rs    | 17 +------
 influxdb_iox/src/commands/run/router2.rs    | 15 +-----
 ioxd_querier/src/lib.rs                     | 26 +++++++++--
 ioxd_router/src/lib.rs                      | 45 ++++++++++++++----
 10 files changed, 122 insertions(+), 118 deletions(-)
 delete mode 100644 clap_blocks/src/authz.rs
 create mode 100644 clap_blocks/src/single_tenant.rs

diff --git a/clap_blocks/src/authz.rs b/clap_blocks/src/authz.rs
deleted file mode 100644
index eb6f65d428..0000000000
--- a/clap_blocks/src/authz.rs
+++ /dev/null
@@ -1,42 +0,0 @@
-//! CLI config for request authorization.
-
-use ::authz::{Authorizer, IoxAuthorizer};
-use snafu::Snafu;
-use std::{boxed::Box, sync::Arc};
-
-#[derive(Debug, Snafu)]
-#[allow(missing_docs)]
-pub enum Error {
-    #[snafu(display("Invalid authz service address {addr}: {source}"))]
-    BadServiceAddress {
-        addr: String,
-        source: Box<dyn std::error::Error>,
-    },
-}
-
-/// Configuration for optional request authorization.
-#[derive(Clone, Debug, Default, clap::Parser)]
-pub struct AuthzConfig {
-    #[clap(long = "authz-addr", env = "INFLUXDB_IOX_AUTHZ_ADDR")]
-    pub(crate) authz_addr: Option<String>,
-}
-
-impl AuthzConfig {
-    /// Authorizer from the configuration.
-    ///
-    /// An authorizer is optional so will only be created if configured.
-    /// An error will only occur when the authorizer configuration is
-    /// invalid.
-    pub fn authorizer(&self) -> Result<Option<Arc<dyn Authorizer>>, Error> {
-        if let Some(s) = &self.authz_addr {
-            IoxAuthorizer::connect_lazy(s.clone())
-                .map(|c| Some(Arc::new(c) as Arc<dyn Authorizer>))
-                .map_err(|e| Error::BadServiceAddress {
-                    addr: s.clone(),
-                    source: e,
-                })
-        } else {
-            Ok(None)
-        }
-    }
-}
diff --git a/clap_blocks/src/lib.rs b/clap_blocks/src/lib.rs
index 01b5334377..48bba965eb 100644
--- a/clap_blocks/src/lib.rs
+++ b/clap_blocks/src/lib.rs
@@ -12,7 +12,6 @@
     clippy::todo,
     clippy::dbg_macro
 )]
-pub mod authz;
 pub mod catalog_dsn;
 pub mod compactor2;
 pub mod garbage_collector;
@@ -22,4 +21,5 @@ pub mod object_store;
 pub mod querier;
 pub mod router2;
 pub mod run_config;
+pub mod single_tenant;
 pub mod socket_addr;
diff --git a/clap_blocks/src/querier.rs b/clap_blocks/src/querier.rs
index 9c3aec25ca..4d0eacdf3f 100644
--- a/clap_blocks/src/querier.rs
+++ b/clap_blocks/src/querier.rs
@@ -1,11 +1,18 @@
 //! Querier-related configs.
 
-use crate::ingester_address::IngesterAddress;
+use crate::{
+    ingester_address::IngesterAddress,
+    single_tenant::{CONFIG_AUTHZ_ENV_NAME, CONFIG_AUTHZ_FLAG},
+};
 use std::{collections::HashMap, num::NonZeroUsize};
 
 /// CLI config for querier configuration
 #[derive(Debug, Clone, PartialEq, Eq, clap::Parser)]
 pub struct QuerierConfig {
+    /// Addr for connection to authz
+    #[clap(long = CONFIG_AUTHZ_FLAG, env = CONFIG_AUTHZ_ENV_NAME)]
+    pub authz_address: Option<String>,
+
     /// The number of threads to use for queries.
     ///
     /// If not specified, defaults to the number of cores on the system
diff --git a/clap_blocks/src/router2.rs b/clap_blocks/src/router2.rs
index 1905e0c493..149a5219ce 100644
--- a/clap_blocks/src/router2.rs
+++ b/clap_blocks/src/router2.rs
@@ -1,6 +1,11 @@
 //! CLI config for the router using the RPC write path
 
-use crate::ingester_address::IngesterAddress;
+use crate::{
+    ingester_address::IngesterAddress,
+    single_tenant::{
+        CONFIG_AUTHZ_ENV_NAME, CONFIG_AUTHZ_FLAG, CONFIG_CST_ENV_NAME, CONFIG_CST_FLAG,
+    },
+};
 use std::{
     num::{NonZeroUsize, ParseIntError},
     time::Duration,
@@ -10,14 +15,23 @@ use std::{
 #[derive(Debug, Clone, clap::Parser)]
 #[allow(missing_copy_implementations)]
 pub struct Router2Config {
+    /// Addr for connection to authz
+    #[clap(
+        long = CONFIG_AUTHZ_FLAG,
+        env = CONFIG_AUTHZ_ENV_NAME,
+        requires("single_tenant_deployment"),
+    )]
+    pub authz_address: Option<String>,
+
     /// Differential handling based upon deployment to CST vs MT.
     ///
     /// At minimum, differs in supports of v1 endpoint. But also includes
     /// differences in namespace handling, etc.
     #[clap(
-        long = "single-tenancy",
-        env = "INFLUXDB_IOX_SINGLE_TENANCY",
-        default_value = "false"
+        long = CONFIG_CST_FLAG,
+        env = CONFIG_CST_ENV_NAME,
+        default_value = "false",
+        requires_if("true", "authz_address")
     )]
     pub single_tenant_deployment: bool,
 
diff --git a/clap_blocks/src/single_tenant.rs b/clap_blocks/src/single_tenant.rs
new file mode 100644
index 0000000000..fb7fb956ad
--- /dev/null
+++ b/clap_blocks/src/single_tenant.rs
@@ -0,0 +1,11 @@
+//! CLI config for request authorization.
+
+/// Env var providing authz address
+pub const CONFIG_AUTHZ_ENV_NAME: &str = "INFLUXDB_IOX_AUTHZ_ADDR";
+/// CLI flag for authz address
+pub const CONFIG_AUTHZ_FLAG: &str = "authz-addr";
+
+/// Env var for single tenancy deployments
+pub const CONFIG_CST_ENV_NAME: &str = "INFLUXDB_IOX_SINGLE_TENANCY";
+/// CLI flag for single tenancy deployments
+pub const CONFIG_CST_FLAG: &str = "single-tenancy";
diff --git a/influxdb_iox/src/commands/run/all_in_one.rs b/influxdb_iox/src/commands/run/all_in_one.rs
index 0d8ef2e1ee..d996d5146f 100644
--- a/influxdb_iox/src/commands/run/all_in_one.rs
+++ b/influxdb_iox/src/commands/run/all_in_one.rs
@@ -3,9 +3,7 @@
 use crate::process_info::setup_metric_registry;
 
 use super::main;
-use authz::Authorizer;
 use clap_blocks::{
-    authz::AuthzConfig,
     catalog_dsn::CatalogDsnConfig,
     compactor2::Compactor2Config,
     ingester2::Ingester2Config,
@@ -14,6 +12,9 @@ use clap_blocks::{
     querier::QuerierConfig,
     router2::Router2Config,
     run_config::RunConfig,
+    single_tenant::{
+        CONFIG_AUTHZ_ENV_NAME, CONFIG_AUTHZ_FLAG, CONFIG_CST_ENV_NAME, CONFIG_CST_FLAG,
+    },
     socket_addr::SocketAddr,
 };
 use compactor2::object_store::metrics::MetricsStore;
@@ -89,9 +90,6 @@ pub enum Error {
     #[error("Invalid config: {0}")]
     InvalidConfig(#[from] CommonServerStateError),
 
-    #[error("Authz configuration error: {0}")]
-    AuthzConfig(#[from] clap_blocks::authz::Error),
-
     #[error("Authz service error: {0}")]
     AuthzService(#[from] authz::Error),
 }
@@ -173,9 +171,20 @@ pub type Result<T, E = Error> = std::result::Result<T, E>;
 )]
 #[group(skip)]
 pub struct Config {
-    /// Authorizer options.
-    #[clap(flatten)]
-    pub(crate) authz_config: AuthzConfig,
+    #[clap(
+        long = CONFIG_AUTHZ_FLAG,
+        env = CONFIG_AUTHZ_ENV_NAME,
+        requires("single_tenant_deployment"),
+    )]
+    pub(crate) authz_address: Option<String>,
+
+    #[clap(
+        long = CONFIG_CST_FLAG,
+        env = CONFIG_CST_ENV_NAME,
+        default_value = "false",
+        requires_if("true", "authz_address")
+    )]
+    pub(crate) single_tenant_deployment: bool,
 
     /// logging options
     #[clap(flatten)]
@@ -185,17 +194,6 @@ pub struct Config {
     #[clap(flatten)]
     pub(crate) tracing_config: TracingConfig,
 
-    /// Differential handling based upon deployment to CST vs MT.
-    ///
-    /// At minimum, differs in supports of v1 endpoint. But also includes
-    /// differences in namespace handling, etc.
-    #[clap(
-        long = "single-tenancy",
-        env = "INFLUXDB_IOX_SINGLE_TENANCY",
-        default_value = "false"
-    )]
-    pub single_tenant_deployment: bool,
-
     /// Maximum size of HTTP requests.
     #[clap(
         long = "max-http-request-size",
@@ -361,6 +359,7 @@ impl Config {
     /// configuration for each individual IOx service
     fn specialize(self) -> SpecializedConfig {
         let Self {
+            authz_address,
             logging_config,
             tracing_config,
             max_http_request_size,
@@ -381,7 +380,6 @@ impl Config {
             querier_ram_pool_data_bytes,
             querier_max_concurrent_queries,
             exec_mem_pool_bytes,
-            authz_config,
             single_tenant_deployment,
         } = self;
 
@@ -472,6 +470,8 @@ impl Config {
         };
 
         let router_config = Router2Config {
+            authz_address: authz_address.clone(),
+            single_tenant_deployment,
             query_pool_name: QUERY_POOL_NAME.to_string(),
             http_request_limit: 1_000,
             ingester_addresses: ingester_addresses.clone(),
@@ -481,7 +481,6 @@ impl Config {
             topic: QUERY_POOL_NAME.to_string(),
             rpc_write_timeout_seconds: Duration::new(3, 0),
             rpc_write_replicas: None,
-            single_tenant_deployment,
             rpc_write_max_outgoing_bytes: ingester_config.rpc_write_max_incoming_bytes,
         };
 
@@ -514,6 +513,7 @@ impl Config {
         };
 
         let querier_config = QuerierConfig {
+            authz_address,
             num_query_threads: None, // will be ignored
             ingester_addresses,
             ram_pool_metadata_bytes: querier_ram_pool_metadata_bytes,
@@ -536,7 +536,6 @@ impl Config {
             router_config,
             compactor_config,
             querier_config,
-            authz_config,
         }
     }
 }
@@ -564,7 +563,6 @@ struct SpecializedConfig {
     router_config: Router2Config,
     compactor_config: Compactor2Config,
     querier_config: QuerierConfig,
-    authz_config: AuthzConfig,
 }
 
 pub async fn command(config: Config) -> Result<()> {
@@ -578,7 +576,6 @@ pub async fn command(config: Config) -> Result<()> {
         router_config,
         compactor_config,
         querier_config,
-        authz_config,
     } = config.specialize();
 
     let metrics = setup_metric_registry();
@@ -606,10 +603,6 @@ pub async fn command(config: Config) -> Result<()> {
 
     let time_provider: Arc<dyn TimeProvider> = Arc::new(SystemProvider::new());
 
-    let authz = authz_config.authorizer()?;
-    // Verify the connection to the authorizer, if configured.
-    authz.probe().await?;
-
     // create common state from the router and use it below
     let common_state = CommonServerState::from_config(router_run_config.clone())?;
 
@@ -636,7 +629,6 @@ pub async fn command(config: Config) -> Result<()> {
         Arc::clone(&metrics),
         Arc::clone(&catalog),
         Arc::clone(&object_store),
-        authz.as_ref().map(Arc::clone),
         &router_config,
     )
     .await?;
@@ -684,7 +676,6 @@ pub async fn command(config: Config) -> Result<()> {
         exec,
         time_provider,
         querier_config,
-        authz: authz.as_ref().map(Arc::clone),
     })
     .await?;
 
diff --git a/influxdb_iox/src/commands/run/querier.rs b/influxdb_iox/src/commands/run/querier.rs
index 3558c2ae12..ccfed9a637 100644
--- a/influxdb_iox/src/commands/run/querier.rs
+++ b/influxdb_iox/src/commands/run/querier.rs
@@ -3,10 +3,9 @@
 use crate::process_info::setup_metric_registry;
 
 use super::main;
-use authz::Authorizer;
 use clap_blocks::{
-    authz::AuthzConfig, catalog_dsn::CatalogDsnConfig, object_store::make_object_store,
-    querier::QuerierConfig, run_config::RunConfig,
+    catalog_dsn::CatalogDsnConfig, object_store::make_object_store, querier::QuerierConfig,
+    run_config::RunConfig,
 };
 use iox_query::exec::Executor;
 use iox_time::{SystemProvider, TimeProvider};
@@ -41,9 +40,6 @@ pub enum Error {
     #[error("Querier error: {0}")]
     Querier(#[from] ioxd_querier::Error),
 
-    #[error("Authz configuration error: {0}")]
-    AuthzConfig(#[from] clap_blocks::authz::Error),
-
     #[error("Authz service error: {0}")]
     AuthzService(#[from] authz::Error),
 }
@@ -64,10 +60,6 @@ Configuration is loaded from the following sources (highest precedence first):
         - pre-configured default values"
 )]
 pub struct Config {
-    /// Authorizer options.
-    #[clap(flatten)]
-    pub(crate) authz_config: AuthzConfig,
-
     #[clap(flatten)]
     pub(crate) run_config: RunConfig,
 
@@ -100,10 +92,6 @@ pub async fn command(config: Config) -> Result<(), Error> {
 
     let time_provider = Arc::new(SystemProvider::new());
 
-    let authz = config.authz_config.authorizer()?;
-    // Verify the connection to the authorizer, if configured.
-    authz.probe().await?;
-
     let num_query_threads = config.querier_config.num_query_threads();
     let num_threads = num_query_threads.unwrap_or_else(|| {
         NonZeroUsize::new(num_cpus::get()).unwrap_or_else(|| NonZeroUsize::new(1).unwrap())
@@ -126,7 +114,6 @@ pub async fn command(config: Config) -> Result<(), Error> {
         exec,
         time_provider,
         querier_config: config.querier_config,
-        authz: authz.as_ref().map(Arc::clone),
     })
     .await?;
 
diff --git a/influxdb_iox/src/commands/run/router2.rs b/influxdb_iox/src/commands/run/router2.rs
index b06f596db3..2c473eb1a3 100644
--- a/influxdb_iox/src/commands/run/router2.rs
+++ b/influxdb_iox/src/commands/run/router2.rs
@@ -1,10 +1,9 @@
 //! Command line options for running a router2 that uses the RPC write path.
 use super::main;
 use crate::process_info::setup_metric_registry;
-use authz::Authorizer;
 use clap_blocks::{
-    authz::AuthzConfig, catalog_dsn::CatalogDsnConfig, object_store::make_object_store,
-    router2::Router2Config, run_config::RunConfig,
+    catalog_dsn::CatalogDsnConfig, object_store::make_object_store, router2::Router2Config,
+    run_config::RunConfig,
 };
 use iox_time::{SystemProvider, TimeProvider};
 use ioxd_common::{
@@ -36,9 +35,6 @@ pub enum Error {
     #[error("Catalog DSN error: {0}")]
     CatalogDsn(#[from] clap_blocks::catalog_dsn::Error),
 
-    #[error("Authz configuration error: {0}")]
-    AuthzConfig(#[from] clap_blocks::authz::Error),
-
     #[error("Authz service error: {0}")]
     AuthzService(#[from] authz::Error),
 }
@@ -61,9 +57,6 @@ Configuration is loaded from the following sources (highest precedence first):
         - pre-configured default values"
 )]
 pub struct Config {
-    #[clap(flatten)]
-    pub(crate) authz_config: AuthzConfig,
-
     #[clap(flatten)]
     pub(crate) run_config: RunConfig,
 
@@ -98,16 +91,12 @@ pub async fn command(config: Config) -> Result<()> {
         time_provider,
         &metrics,
     ));
-    let authz = config.authz_config.authorizer()?;
-    // Verify the connection to the authorizer, if configured.
-    authz.probe().await?;
 
     let server_type = create_router2_server_type(
         &common_state,
         Arc::clone(&metrics),
         catalog,
         object_store,
-        authz,
         &config.router_config,
     )
     .await?;
diff --git a/ioxd_querier/src/lib.rs b/ioxd_querier/src/lib.rs
index a55729e4cc..805e901287 100644
--- a/ioxd_querier/src/lib.rs
+++ b/ioxd_querier/src/lib.rs
@@ -1,5 +1,5 @@
 use async_trait::async_trait;
-use authz::Authorizer;
+use authz::{Authorizer, IoxAuthorizer};
 use clap_blocks::querier::QuerierConfig;
 use datafusion_util::config::register_iox_object_store;
 use hyper::{Body, Request, Response};
@@ -160,13 +160,18 @@ pub struct QuerierServerTypeArgs<'a> {
     pub exec: Arc<Executor>,
     pub time_provider: Arc<dyn TimeProvider>,
     pub querier_config: QuerierConfig,
-    pub authz: Option<Arc<dyn Authorizer>>,
 }
 
 #[derive(Debug, Error)]
 pub enum Error {
     #[error("querier error: {0}")]
     Querier(#[from] querier::QuerierDatabaseError),
+
+    #[error("authz configuration error for '{addr}': '{source}'")]
+    AuthzConfig {
+        source: Box<dyn std::error::Error>,
+        addr: String,
+    },
 }
 
 /// Instantiate a querier server
@@ -197,6 +202,21 @@ pub async fn create_querier_server_type(
     );
     assert!(existing.is_none());
 
+    let authz = match &args.querier_config.authz_address {
+        Some(addr) => {
+            let authz = IoxAuthorizer::connect_lazy(addr.clone())
+                .map(|c| Arc::new(c) as Arc<dyn Authorizer>)
+                .map_err(|source| Error::AuthzConfig {
+                    source,
+                    addr: addr.clone(),
+                })?;
+            authz.probe().await.expect("Authz connection test failed.");
+
+            Some(authz)
+        }
+        None => None,
+    };
+
     let ingester_connections = if args.querier_config.ingester_addresses.is_empty() {
         None
     } else {
@@ -235,6 +255,6 @@ pub async fn create_querier_server_type(
         querier,
         database,
         args.common_state,
-        args.authz.as_ref().map(Arc::clone),
+        authz,
     )))
 }
diff --git a/ioxd_router/src/lib.rs b/ioxd_router/src/lib.rs
index 77a0caef4f..24ca699d54 100644
--- a/ioxd_router/src/lib.rs
+++ b/ioxd_router/src/lib.rs
@@ -4,7 +4,7 @@ use std::{
 };
 
 use async_trait::async_trait;
-use authz::Authorizer;
+use authz::{Authorizer, IoxAuthorizer};
 use clap_blocks::router2::Router2Config;
 use data_types::{NamespaceName, PartitionTemplate, TemplatePart};
 use hashbrown::HashMap;
@@ -66,6 +66,12 @@ pub enum Error {
 
     #[error("No topic named '{topic_name}' found in the catalog")]
     TopicCatalogLookup { topic_name: String },
+
+    #[error("authz configuration error for '{addr}': '{source}'")]
+    AuthzConfig {
+        source: Box<dyn std::error::Error>,
+        addr: String,
+    },
 }
 
 pub type Result<T, E = Error> = std::result::Result<T, E>;
@@ -193,7 +199,6 @@ pub async fn create_router2_server_type(
     metrics: Arc<metric::Registry>,
     catalog: Arc<dyn Catalog>,
     object_store: Arc<DynObjectStore>,
-    authz: Option<Arc<dyn Authorizer>>,
     router_config: &Router2Config,
 ) -> Result<Arc<dyn ServerType>> {
     let ingester_connections = router_config.ingester_addresses.iter().map(|addr| {
@@ -349,13 +354,35 @@ pub async fn create_router2_server_type(
     let handler_stack = InstrumentationDecorator::new("request", &metrics, handler_stack);
 
     // Initialize the HTTP API delegate
-    let write_request_unifier: Result<Box<dyn WriteRequestUnifier>> =
-        match (router_config.single_tenant_deployment, authz) {
-            (true, Some(auth)) => Ok(Box::new(SingleTenantRequestUnifier::new(auth))),
-            (true, None) => unreachable!("INFLUXDB_IOX_SINGLE_TENANCY is set, but could not create an authz service. Check the INFLUXDB_IOX_AUTHZ_ADDR."),
-            (false, None) => Ok(Box::<MultiTenantRequestUnifier>::default()),
-            (false, Some(_)) => unreachable!("INFLUXDB_IOX_AUTHZ_ADDR is set, but authz only exists for single_tenancy. Check the INFLUXDB_IOX_SINGLE_TENANCY."),
-        };
+    let write_request_unifier: Result<Box<dyn WriteRequestUnifier>> = match (
+        router_config.single_tenant_deployment,
+        &router_config.authz_address,
+    ) {
+        (true, Some(addr)) => {
+            let authz = IoxAuthorizer::connect_lazy(addr.clone())
+                .map(|c| Arc::new(c) as Arc<dyn Authorizer>)
+                .map_err(|source| Error::AuthzConfig {
+                    source,
+                    addr: addr.clone(),
+                })?;
+            authz.probe().await.expect("Authz connection test failed.");
+
+            Ok(Box::new(SingleTenantRequestUnifier::new(authz)))
+        }
+        (true, None) => {
+            // Single tenancy was requested, but no auth was provided - the
+            // router's clap flag parse configuration should not allow this
+            // combination to be accepted and therefore execution should
+            // never reach here.
+            unreachable!("INFLUXDB_IOX_SINGLE_TENANCY is set, but could not create an authz service. Check the INFLUXDB_IOX_AUTHZ_ADDR")
+        }
+        (false, None) => Ok(Box::<MultiTenantRequestUnifier>::default()),
+        (false, Some(_)) => {
+            // As above, this combination should be prevented by the
+            // router's clap flag parse configuration.
+            unreachable!("INFLUXDB_IOX_AUTHZ_ADDR is set, but authz only exists for single_tenancy. Check the INFLUXDB_IOX_SINGLE_TENANCY")
+        }
+    };
     let http = HttpDelegate::new(
         common_state.run_config().max_http_request_size,
         router_config.http_request_limit,

From 740269839d5de5e3245c5e2bc1e5941cbe32305c Mon Sep 17 00:00:00 2001
From: "Carol (Nichols || Goulding)" <carol.nichols@gmail.com>
Date: Mon, 24 Apr 2023 15:56:59 -0400
Subject: [PATCH 14/25] fix: Extract test helper functions

---
 ingester2/src/buffer_tree/namespace.rs        |  74 +--
 ingester2/src/buffer_tree/partition.rs        | 183 +----
 .../buffer_tree/partition/resolver/cache.rs   | 200 +++---
 .../partition/resolver/coalesce.rs            | 111 +---
 .../buffer_tree/partition/resolver/mock.rs    |  25 +-
 .../buffer_tree/partition/resolver/trait.rs   |  55 +-
 ingester2/src/buffer_tree/root.rs             | 628 ++++++++----------
 ingester2/src/buffer_tree/table.rs            |  79 +--
 .../src/buffer_tree/table/name_resolver.rs    |   4 +-
 ingester2/src/init/graceful_shutdown.rs       |  40 +-
 ingester2/src/init/wal_replay.rs              |  92 +--
 ingester2/src/persist/handle.rs               | 171 ++---
 ingester2/src/persist/hot_partitions.rs       |  46 +-
 ingester2/src/persist/mod.rs                  |  36 +-
 ingester2/src/test_util.rs                    | 134 +++-
 ingester2/src/wal/rotate_task.rs              |  55 +-
 16 files changed, 813 insertions(+), 1120 deletions(-)

diff --git a/ingester2/src/buffer_tree/namespace.rs b/ingester2/src/buffer_tree/namespace.rs
index daf780dfd9..7a02f2bb79 100644
--- a/ingester2/src/buffer_tree/namespace.rs
+++ b/ingester2/src/buffer_tree/namespace.rs
@@ -228,56 +228,39 @@ where
 
 #[cfg(test)]
 mod tests {
-    use std::{sync::Arc, time::Duration};
+    use std::sync::Arc;
 
-    use data_types::{PartitionId, PartitionKey, ShardId};
+    use data_types::TRANSITION_SHARD_ID;
     use metric::{Attributes, Metric};
 
     use super::*;
     use crate::{
         buffer_tree::{
-            namespace::NamespaceData,
-            partition::{resolver::mock::MockPartitionProvider, PartitionData, SortKeyState},
+            namespace::NamespaceData, partition::resolver::mock::MockPartitionProvider,
             post_write::mock::MockPostWriteObserver,
-            table::{name_resolver::mock::MockTableNameProvider, TableName},
         },
-        deferred_load::{self, DeferredLoad},
-        test_util::make_write_op,
+        deferred_load,
+        test_util::{
+            make_write_op, PartitionDataBuilder, ARBITRARY_NAMESPACE_ID, ARBITRARY_NAMESPACE_NAME,
+            ARBITRARY_PARTITION_KEY, ARBITRARY_TABLE_ID, ARBITRARY_TABLE_NAME,
+            ARBITRARY_TABLE_NAME_PROVIDER, DEFER_NAMESPACE_NAME_1_MS,
+        },
     };
 
-    const TABLE_NAME: &str = "bananas";
-    const TABLE_ID: TableId = TableId::new(44);
-    const NAMESPACE_NAME: &str = "platanos";
-    const NAMESPACE_ID: NamespaceId = NamespaceId::new(42);
-    const TRANSITION_SHARD_ID: ShardId = ShardId::new(84);
-
     #[tokio::test]
     async fn test_namespace_init_table() {
         let metrics = Arc::new(metric::Registry::default());
 
         // Configure the mock partition provider to return a partition for this
         // table ID.
-        let partition_provider = Arc::new(MockPartitionProvider::default().with_partition(
-            PartitionData::new(
-                PartitionId::new(0),
-                PartitionKey::from("banana-split"),
-                NAMESPACE_ID,
-                Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                    NamespaceName::from(NAMESPACE_NAME)
-                })),
-                TABLE_ID,
-                Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                    TableName::from(TABLE_NAME)
-                })),
-                SortKeyState::Provided(None),
-                TRANSITION_SHARD_ID,
-            ),
-        ));
+        let partition_provider = Arc::new(
+            MockPartitionProvider::default().with_partition(PartitionDataBuilder::new().build()),
+        );
 
         let ns = NamespaceData::new(
-            NAMESPACE_ID,
-            Arc::new(DeferredLoad::new(Duration::from_millis(1), async { NAMESPACE_NAME.into() })),
-            Arc::new(MockTableNameProvider::new(TABLE_NAME)),
+            ARBITRARY_NAMESPACE_ID,
+            Arc::clone(&*DEFER_NAMESPACE_NAME_1_MS),
+            Arc::clone(&*ARBITRARY_TABLE_NAME_PROVIDER),
             partition_provider,
             Arc::new(MockPostWriteObserver::default()),
             &metrics,
@@ -287,27 +270,31 @@ mod tests {
         // Assert the namespace name was stored
         let name = ns.namespace_name().to_string();
         assert!(
-            (name == NAMESPACE_NAME) || (name == deferred_load::UNRESOLVED_DISPLAY_STRING),
+            (name.as_str() == &***ARBITRARY_NAMESPACE_NAME)
+                || (name == deferred_load::UNRESOLVED_DISPLAY_STRING),
             "unexpected namespace name: {name}"
         );
 
         // Assert the namespace does not contain the test data
-        assert!(ns.table(TABLE_ID).is_none());
+        assert!(ns.table(ARBITRARY_TABLE_ID).is_none());
 
         // Write some test data
         ns.apply(DmlOperation::Write(make_write_op(
-            &PartitionKey::from("banana-split"),
-            NAMESPACE_ID,
-            TABLE_NAME,
-            TABLE_ID,
+            &ARBITRARY_PARTITION_KEY,
+            ARBITRARY_NAMESPACE_ID,
+            &ARBITRARY_TABLE_NAME,
+            ARBITRARY_TABLE_ID,
             0,
-            r#"bananas,city=Medford day="sun",temp=55 22"#,
+            &format!(
+                r#"{},city=Medford day="sun",temp=55 22"#,
+                &*ARBITRARY_TABLE_NAME
+            ),
         )))
         .await
         .expect("buffer op should succeed");
 
         // Referencing the table should succeed
-        assert!(ns.table(TABLE_ID).is_some());
+        assert!(ns.table(ARBITRARY_TABLE_ID).is_some());
 
         // And the table counter metric should increase
         let tables = metrics
@@ -320,7 +307,10 @@ mod tests {
 
         // Ensure the deferred namespace name is loaded.
         let name = ns.namespace_name().get().await;
-        assert_eq!(&**name, NAMESPACE_NAME);
-        assert_eq!(ns.namespace_name().to_string(), NAMESPACE_NAME);
+        assert_eq!(&*name, &**ARBITRARY_NAMESPACE_NAME);
+        assert_eq!(
+            ns.namespace_name().to_string().as_str(),
+            &***ARBITRARY_NAMESPACE_NAME
+        );
     }
 }
diff --git a/ingester2/src/buffer_tree/partition.rs b/ingester2/src/buffer_tree/partition.rs
index 04fc4ab4c9..833a4e264a 100644
--- a/ingester2/src/buffer_tree/partition.rs
+++ b/ingester2/src/buffer_tree/partition.rs
@@ -354,20 +354,13 @@ mod tests {
     };
     use datafusion_util::test_collect;
     use iox_catalog::interface::Catalog;
-    use lazy_static::lazy_static;
     use mutable_batch_lp::test_helpers::lp_to_mutable_batch;
 
     use super::*;
-    use crate::{buffer_tree::partition::resolver::SortKeyResolver, test_util::populate_catalog};
-
-    const PARTITION_ID: PartitionId = PartitionId::new(1);
-    const TRANSITION_SHARD_ID: ShardId = ShardId::new(84);
-
-    lazy_static! {
-        static ref PARTITION_KEY: PartitionKey = PartitionKey::from("platanos");
-        static ref TABLE_NAME: TableName = TableName::from("bananas");
-        static ref NAMESPACE_NAME: NamespaceName = NamespaceName::from("namespace-bananas");
-    }
+    use crate::{
+        buffer_tree::partition::resolver::SortKeyResolver,
+        test_util::{populate_catalog, PartitionDataBuilder, ARBITRARY_PARTITION_ID},
+    };
 
     // Write some data and read it back from the buffer.
     //
@@ -375,20 +368,7 @@ mod tests {
     // generation & query all work as intended.
     #[tokio::test]
     async fn test_write_read() {
-        let mut p = PartitionData::new(
-            PARTITION_ID,
-            PARTITION_KEY.clone(),
-            NamespaceId::new(3),
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                NAMESPACE_NAME.clone()
-            })),
-            TableId::new(4),
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                TABLE_NAME.clone()
-            })),
-            SortKeyState::Provided(None),
-            TRANSITION_SHARD_ID,
-        );
+        let mut p = PartitionDataBuilder::new().build();
 
         // And no data should be returned when queried.
         assert!(p.get_query_data().is_none());
@@ -401,7 +381,7 @@ mod tests {
         // The data should be readable.
         {
             let data = p.get_query_data().expect("should return data");
-            assert_eq!(data.partition_id(), PARTITION_ID);
+            assert_eq!(data.partition_id(), ARBITRARY_PARTITION_ID);
 
             let expected = [
                 "+--------+--------+----------+--------------------------------+",
@@ -430,7 +410,7 @@ mod tests {
         // And finally both writes should be readable.
         {
             let data = p.get_query_data().expect("should contain data");
-            assert_eq!(data.partition_id(), PARTITION_ID);
+            assert_eq!(data.partition_id(), ARBITRARY_PARTITION_ID);
 
             let expected = [
                 "+--------+--------+----------+--------------------------------+",
@@ -456,20 +436,7 @@ mod tests {
     // both before, during, and after a persist takes place.
     #[tokio::test]
     async fn test_persist() {
-        let mut p = PartitionData::new(
-            PARTITION_ID,
-            PARTITION_KEY.clone(),
-            NamespaceId::new(3),
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                NAMESPACE_NAME.clone()
-            })),
-            TableId::new(4),
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                TABLE_NAME.clone()
-            })),
-            SortKeyState::Provided(None),
-            TRANSITION_SHARD_ID,
-        );
+        let mut p = PartitionDataBuilder::new().build();
 
         assert!(p.get_query_data().is_none());
 
@@ -485,7 +452,7 @@ mod tests {
         // Begin persisting the partition.
         let persisting_data = p.mark_persisting().expect("must contain existing data");
         // And validate the data being persisted.
-        assert_eq!(persisting_data.partition_id(), PARTITION_ID);
+        assert_eq!(persisting_data.partition_id(), ARBITRARY_PARTITION_ID);
         assert_eq!(persisting_data.record_batches().len(), 1);
         let expected = [
             "+--------+--------+----------+--------------------------------+",
@@ -519,7 +486,7 @@ mod tests {
         // Which must be readable, alongside the ongoing persist data.
         {
             let data = p.get_query_data().expect("must have data");
-            assert_eq!(data.partition_id(), PARTITION_ID);
+            assert_eq!(data.partition_id(), ARBITRARY_PARTITION_ID);
             assert_eq!(data.record_batches().len(), 2);
             let expected = [
                 "+--------+--------+----------+--------------------------------+",
@@ -553,7 +520,7 @@ mod tests {
         // Querying the buffer should now return only the second write.
         {
             let data = p.get_query_data().expect("must have data");
-            assert_eq!(data.partition_id(), PARTITION_ID);
+            assert_eq!(data.partition_id(), ARBITRARY_PARTITION_ID);
             assert_eq!(data.record_batches().len(), 1);
             let expected = [
                 "+--------+--------+---------+--------------------------------+",
@@ -604,26 +571,15 @@ mod tests {
             let input = Arc::new(MemoryExec::try_new(&[batch], schema, projection).unwrap());
 
             // Create and run the deduplicator
-            let exec = Arc::new(iox_query::provider::DeduplicateExec::new(input, sort_keys, false));
+            let exec = Arc::new(iox_query::provider::DeduplicateExec::new(
+                input, sort_keys, false,
+            ));
             let got = test_collect(Arc::clone(&exec) as Arc<dyn ExecutionPlan>).await;
 
             assert_batches_eq!(expect, &*got);
         }
 
-        let mut p = PartitionData::new(
-            PARTITION_ID,
-            PARTITION_KEY.clone(),
-            NamespaceId::new(3),
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                NAMESPACE_NAME.clone()
-            })),
-            TableId::new(4),
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                TABLE_NAME.clone()
-            })),
-            SortKeyState::Provided(None),
-            TRANSITION_SHARD_ID,
-        );
+        let mut p = PartitionDataBuilder::new().build();
 
         // Perform the initial write.
         //
@@ -795,20 +751,7 @@ mod tests {
     // which return the correct SequenceNumberSet instances.
     #[tokio::test]
     async fn test_out_of_order_persist() {
-        let mut p = PartitionData::new(
-            PARTITION_ID,
-            PARTITION_KEY.clone(),
-            NamespaceId::new(3),
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                NAMESPACE_NAME.clone()
-            })),
-            TableId::new(4),
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                TABLE_NAME.clone()
-            })),
-            SortKeyState::Provided(None),
-            TRANSITION_SHARD_ID,
-        );
+        let mut p = PartitionDataBuilder::new().build();
 
         // Perform the initial write.
         //
@@ -982,20 +925,9 @@ mod tests {
         let starting_state =
             SortKeyState::Provided(Some(SortKey::from_columns(["banana", "time"])));
 
-        let mut p = PartitionData::new(
-            PartitionId::new(1),
-            "bananas".into(),
-            NamespaceId::new(42),
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                NAMESPACE_NAME.clone()
-            })),
-            TableId::new(1),
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                TableName::from("platanos")
-            })),
-            starting_state,
-            TRANSITION_SHARD_ID,
-        );
+        let mut p = PartitionDataBuilder::new()
+            .with_sort_key_state(starting_state)
+            .build();
 
         let want = Some(SortKey::from_columns(["banana", "platanos", "time"]));
         p.update_sort_key(want.clone());
@@ -1042,20 +974,9 @@ mod tests {
 
         let starting_state = SortKeyState::Deferred(fetcher);
 
-        let mut p = PartitionData::new(
-            PartitionId::new(1),
-            "bananas".into(),
-            NamespaceId::new(42),
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                NAMESPACE_NAME.clone()
-            })),
-            TableId::new(1),
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                TableName::from("platanos")
-            })),
-            starting_state,
-            shard_id,
-        );
+        let mut p = PartitionDataBuilder::new()
+            .with_sort_key_state(starting_state)
+            .build();
 
         let want = Some(SortKey::from_columns(["banana", "platanos", "time"]));
         p.update_sort_key(want.clone());
@@ -1067,20 +988,7 @@ mod tests {
     // Perform writes with non-monotonic sequence numbers.
     #[tokio::test]
     async fn test_non_monotonic_writes() {
-        let mut p = PartitionData::new(
-            PARTITION_ID,
-            PARTITION_KEY.clone(),
-            NamespaceId::new(3),
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                NAMESPACE_NAME.clone()
-            })),
-            TableId::new(4),
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                TABLE_NAME.clone()
-            })),
-            SortKeyState::Provided(None),
-            TRANSITION_SHARD_ID,
-        );
+        let mut p = PartitionDataBuilder::new().build();
 
         // Perform out of order writes.
         p.buffer_write(
@@ -1115,40 +1023,14 @@ mod tests {
 
     #[tokio::test]
     async fn test_mark_persisting_no_data() {
-        let mut p = PartitionData::new(
-            PARTITION_ID,
-            PARTITION_KEY.clone(),
-            NamespaceId::new(3),
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                NAMESPACE_NAME.clone()
-            })),
-            TableId::new(4),
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                TABLE_NAME.clone()
-            })),
-            SortKeyState::Provided(None),
-            TRANSITION_SHARD_ID,
-        );
+        let mut p = PartitionDataBuilder::new().build();
 
         assert!(p.mark_persisting().is_none());
     }
 
     #[tokio::test]
     async fn test_mark_persisting_twice() {
-        let mut p = PartitionData::new(
-            PARTITION_ID,
-            PARTITION_KEY.clone(),
-            NamespaceId::new(3),
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                NAMESPACE_NAME.clone()
-            })),
-            TableId::new(4),
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                TABLE_NAME.clone()
-            })),
-            SortKeyState::Provided(None),
-            TRANSITION_SHARD_ID,
-        );
+        let mut p = PartitionDataBuilder::new().build();
 
         let mb = lp_to_mutable_batch(r#"bananas,city=London people=2,pigeons="millions" 10"#).1;
         p.buffer_write(mb, SequenceNumber::new(2))
@@ -1162,20 +1044,7 @@ mod tests {
     // QueryAdaptor.
     #[tokio::test]
     async fn test_empty_partition_no_queryadaptor_panic() {
-        let mut p = PartitionData::new(
-            PARTITION_ID,
-            PARTITION_KEY.clone(),
-            NamespaceId::new(3),
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                NAMESPACE_NAME.clone()
-            })),
-            TableId::new(4),
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                TABLE_NAME.clone()
-            })),
-            SortKeyState::Provided(None),
-            TRANSITION_SHARD_ID,
-        );
+        let mut p = PartitionDataBuilder::new().build();
 
         assert!(p.get_query_data().is_none());
     }
diff --git a/ingester2/src/buffer_tree/partition/resolver/cache.rs b/ingester2/src/buffer_tree/partition/resolver/cache.rs
index 360d1c4a7f..fc49b62874 100644
--- a/ingester2/src/buffer_tree/partition/resolver/cache.rs
+++ b/ingester2/src/buffer_tree/partition/resolver/cache.rs
@@ -221,19 +221,19 @@ mod tests {
     // Harmless in tests - saves a bunch of extra vars.
     #![allow(clippy::await_holding_lock)]
 
-    use data_types::ShardId;
+    use data_types::{ShardId, TRANSITION_SHARD_ID};
     use iox_catalog::mem::MemCatalog;
 
     use super::*;
-    use crate::buffer_tree::partition::resolver::mock::MockPartitionProvider;
-
-    const PARTITION_KEY: &str = "bananas";
-    const PARTITION_ID: PartitionId = PartitionId::new(42);
-    const NAMESPACE_ID: NamespaceId = NamespaceId::new(2);
-    const NAMESPACE_NAME: &str = "ns-bananas";
-    const TABLE_ID: TableId = TableId::new(3);
-    const TABLE_NAME: &str = "platanos";
-    const TRANSITION_SHARD_ID: ShardId = ShardId::new(84);
+    use crate::{
+        buffer_tree::partition::resolver::mock::MockPartitionProvider,
+        test_util::{
+            arbitrary_partition, PartitionDataBuilder, ARBITRARY_NAMESPACE_ID,
+            ARBITRARY_NAMESPACE_NAME, ARBITRARY_PARTITION_ID, ARBITRARY_PARTITION_KEY,
+            ARBITRARY_PARTITION_KEY_STR, ARBITRARY_TABLE_ID, ARBITRARY_TABLE_NAME,
+            DEFER_NAMESPACE_NAME_1_SEC, DEFER_TABLE_NAME_1_SEC,
+        },
+    };
 
     fn new_cache<P>(
         inner: MockPartitionProvider,
@@ -253,42 +253,31 @@ mod tests {
 
     #[tokio::test]
     async fn test_miss() {
-        let data = PartitionData::new(
-            PARTITION_ID,
-            PARTITION_KEY.into(),
-            NAMESPACE_ID,
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                NamespaceName::from(NAMESPACE_NAME)
-            })),
-            TABLE_ID,
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                TableName::from(TABLE_NAME)
-            })),
-            SortKeyState::Provided(None),
-            TRANSITION_SHARD_ID,
-        );
+        let data = PartitionDataBuilder::new().build();
         let inner = MockPartitionProvider::default().with_partition(data);
 
         let cache = new_cache(inner, []);
         let got = cache
             .get_partition(
-                PARTITION_KEY.into(),
-                NAMESPACE_ID,
-                Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                    NamespaceName::from(NAMESPACE_NAME)
-                })),
-                TABLE_ID,
-                Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                    TableName::from(TABLE_NAME)
-                })),
+                ARBITRARY_PARTITION_KEY.clone(),
+                ARBITRARY_NAMESPACE_ID,
+                Arc::clone(&*DEFER_NAMESPACE_NAME_1_SEC),
+                ARBITRARY_TABLE_ID,
+                Arc::clone(&*DEFER_TABLE_NAME_1_SEC),
                 TRANSITION_SHARD_ID,
             )
             .await;
 
-        assert_eq!(got.lock().partition_id(), PARTITION_ID);
-        assert_eq!(got.lock().table_id(), TABLE_ID);
-        assert_eq!(&**got.lock().table_name().get().await, TABLE_NAME);
-        assert_eq!(&**got.lock().namespace_name().get().await, NAMESPACE_NAME);
+        assert_eq!(got.lock().partition_id(), ARBITRARY_PARTITION_ID);
+        assert_eq!(got.lock().table_id(), ARBITRARY_TABLE_ID);
+        assert_eq!(
+            &**got.lock().table_name().get().await,
+            &***ARBITRARY_TABLE_NAME
+        );
+        assert_eq!(
+            &**got.lock().namespace_name().get().await,
+            &***ARBITRARY_NAMESPACE_NAME
+        );
         assert!(cache.inner.is_empty());
     }
 
@@ -296,42 +285,40 @@ mod tests {
     async fn test_hit() {
         let inner = MockPartitionProvider::default();
 
-        let stored_partition_key = PartitionKey::from(PARTITION_KEY);
+        let stored_partition_key = PartitionKey::from(ARBITRARY_PARTITION_KEY_STR);
         let partition = Partition {
-            id: PARTITION_ID,
-            shard_id: TRANSITION_SHARD_ID,
-            table_id: TABLE_ID,
             partition_key: stored_partition_key.clone(),
             sort_key: vec!["dos".to_string(), "bananas".to_string()],
-            persisted_sequence_number: Default::default(),
-            new_file_at: Default::default(),
+            ..arbitrary_partition()
         };
 
         let cache = new_cache(inner, [partition]);
 
-        let callers_partition_key = PartitionKey::from(PARTITION_KEY);
+        let callers_partition_key = PartitionKey::from(ARBITRARY_PARTITION_KEY_STR);
         let got = cache
             .get_partition(
                 callers_partition_key.clone(),
-                NAMESPACE_ID,
-                Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                    NamespaceName::from(NAMESPACE_NAME)
-                })),
-                TABLE_ID,
-                Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                    TableName::from(TABLE_NAME)
-                })),
+                ARBITRARY_NAMESPACE_ID,
+                Arc::clone(&*DEFER_NAMESPACE_NAME_1_SEC),
+                ARBITRARY_TABLE_ID,
+                Arc::clone(&*DEFER_TABLE_NAME_1_SEC),
                 TRANSITION_SHARD_ID,
             )
             .await;
 
-        assert_eq!(got.lock().partition_id(), PARTITION_ID);
-        assert_eq!(got.lock().table_id(), TABLE_ID);
-        assert_eq!(&**got.lock().table_name().get().await, TABLE_NAME);
-        assert_eq!(&**got.lock().namespace_name().get().await, NAMESPACE_NAME);
+        assert_eq!(got.lock().partition_id(), ARBITRARY_PARTITION_ID);
+        assert_eq!(got.lock().table_id(), ARBITRARY_TABLE_ID);
+        assert_eq!(
+            &**got.lock().table_name().get().await,
+            &***ARBITRARY_TABLE_NAME
+        );
+        assert_eq!(
+            &**got.lock().namespace_name().get().await,
+            &***ARBITRARY_NAMESPACE_NAME
+        );
         assert_eq!(
             *got.lock().partition_key(),
-            PartitionKey::from(PARTITION_KEY)
+            PartitionKey::from(ARBITRARY_PARTITION_KEY_STR)
         );
 
         // The cache should have been cleaned up as it was consumed.
@@ -350,98 +337,63 @@ mod tests {
     async fn test_miss_partition_key() {
         let other_key = PartitionKey::from("test");
         let other_key_id = PartitionId::new(99);
-        let inner = MockPartitionProvider::default().with_partition(PartitionData::new(
-            other_key_id,
-            other_key.clone(),
-            NAMESPACE_ID,
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                NamespaceName::from(NAMESPACE_NAME)
-            })),
-            TABLE_ID,
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                TableName::from(TABLE_NAME)
-            })),
-            SortKeyState::Provided(None),
-            TRANSITION_SHARD_ID,
-        ));
+        let inner = MockPartitionProvider::default().with_partition(
+            PartitionDataBuilder::new()
+                .with_partition_key(other_key.clone())
+                .with_partition_id(other_key_id)
+                .build(),
+        );
 
-        let partition = Partition {
-            id: PARTITION_ID,
-            shard_id: TRANSITION_SHARD_ID,
-            table_id: TABLE_ID,
-            partition_key: PARTITION_KEY.into(),
-            sort_key: Default::default(),
-            persisted_sequence_number: Default::default(),
-            new_file_at: Default::default(),
-        };
+        let partition = arbitrary_partition();
 
         let cache = new_cache(inner, [partition]);
         let got = cache
             .get_partition(
-                other_key.clone(),
-                NAMESPACE_ID,
-                Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                    NamespaceName::from(NAMESPACE_NAME)
-                })),
-                TABLE_ID,
-                Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                    TableName::from(TABLE_NAME)
-                })),
+                other_key,
+                ARBITRARY_NAMESPACE_ID,
+                Arc::clone(&*DEFER_NAMESPACE_NAME_1_SEC),
+                ARBITRARY_TABLE_ID,
+                Arc::clone(&*DEFER_TABLE_NAME_1_SEC),
                 TRANSITION_SHARD_ID,
             )
             .await;
 
         assert_eq!(got.lock().partition_id(), other_key_id);
-        assert_eq!(got.lock().table_id(), TABLE_ID);
-        assert_eq!(&**got.lock().table_name().get().await, TABLE_NAME);
+        assert_eq!(got.lock().table_id(), ARBITRARY_TABLE_ID);
+        assert_eq!(
+            &**got.lock().table_name().get().await,
+            &***ARBITRARY_TABLE_NAME
+        );
     }
 
     #[tokio::test]
     async fn test_miss_table_id() {
         let other_table = TableId::new(1234);
-        let inner = MockPartitionProvider::default().with_partition(PartitionData::new(
-            PARTITION_ID,
-            PARTITION_KEY.into(),
-            NAMESPACE_ID,
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                NamespaceName::from(NAMESPACE_NAME)
-            })),
-            other_table,
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                TableName::from(TABLE_NAME)
-            })),
-            SortKeyState::Provided(None),
-            TRANSITION_SHARD_ID,
-        ));
+        let inner = MockPartitionProvider::default().with_partition(
+            PartitionDataBuilder::new()
+                .with_table_id(other_table)
+                .build(),
+        );
 
-        let partition = Partition {
-            id: PARTITION_ID,
-            shard_id: TRANSITION_SHARD_ID,
-            table_id: TABLE_ID,
-            partition_key: PARTITION_KEY.into(),
-            sort_key: Default::default(),
-            persisted_sequence_number: Default::default(),
-            new_file_at: Default::default(),
-        };
+        let partition = arbitrary_partition();
 
         let cache = new_cache(inner, [partition]);
         let got = cache
             .get_partition(
-                PARTITION_KEY.into(),
-                NAMESPACE_ID,
-                Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                    NamespaceName::from(NAMESPACE_NAME)
-                })),
+                ARBITRARY_PARTITION_KEY.clone(),
+                ARBITRARY_NAMESPACE_ID,
+                Arc::clone(&*DEFER_NAMESPACE_NAME_1_SEC),
                 other_table,
-                Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                    TableName::from(TABLE_NAME)
-                })),
+                Arc::clone(&*DEFER_TABLE_NAME_1_SEC),
                 TRANSITION_SHARD_ID,
             )
             .await;
 
-        assert_eq!(got.lock().partition_id(), PARTITION_ID);
+        assert_eq!(got.lock().partition_id(), ARBITRARY_PARTITION_ID);
         assert_eq!(got.lock().table_id(), other_table);
-        assert_eq!(&**got.lock().table_name().get().await, TABLE_NAME);
+        assert_eq!(
+            &**got.lock().table_name().get().await,
+            &***ARBITRARY_TABLE_NAME
+        );
     }
 }
diff --git a/ingester2/src/buffer_tree/partition/resolver/coalesce.rs b/ingester2/src/buffer_tree/partition/resolver/coalesce.rs
index 3237bd1b7d..099e271330 100644
--- a/ingester2/src/buffer_tree/partition/resolver/coalesce.rs
+++ b/ingester2/src/buffer_tree/partition/resolver/coalesce.rs
@@ -282,26 +282,16 @@ mod tests {
     use test_helpers::timeout::FutureTimeout;
     use tokio::sync::{Notify, Semaphore};
 
-    use crate::buffer_tree::partition::{resolver::mock::MockPartitionProvider, SortKeyState};
+    use crate::{
+        buffer_tree::partition::{resolver::mock::MockPartitionProvider, SortKeyState},
+        test_util::{
+            PartitionDataBuilder, ARBITRARY_NAMESPACE_ID, ARBITRARY_PARTITION_KEY,
+            ARBITRARY_TABLE_ID, DEFER_NAMESPACE_NAME_1_SEC, DEFER_TABLE_NAME_1_SEC,
+        },
+    };
 
     use super::*;
 
-    const PARTITION_ID: PartitionId = PartitionId::new(4242);
-    const PARTITION_KEY: &str = "bananas";
-    const NAMESPACE_ID: NamespaceId = NamespaceId::new(42);
-    const TABLE_ID: TableId = TableId::new(42);
-
-    lazy_static! {
-        static ref NAMESPACE_NAME: Arc<DeferredLoad<NamespaceName>> =
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                NamespaceName::from("ns-platanos")
-            }));
-        static ref TABLE_NAME: Arc<DeferredLoad<TableName>> =
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                TableName::from("platanos")
-            }));
-    }
-
     /// This test proves that parallel queries for the same partition are
     /// coalesced, returning the same [`PartitionData`] instance and submitting
     /// a single query to the inner resolver.
@@ -309,16 +299,7 @@ mod tests {
     async fn test_coalesce() {
         const MAX_TASKS: usize = 50;
 
-        let data = PartitionData::new(
-            PARTITION_ID,
-            PartitionKey::from(PARTITION_KEY),
-            NAMESPACE_ID,
-            Arc::clone(&*NAMESPACE_NAME),
-            TABLE_ID,
-            Arc::clone(&*TABLE_NAME),
-            SortKeyState::Provided(None),
-            TRANSITION_SHARD_ID,
-        );
+        let data = PartitionDataBuilder::new().build();
 
         // Add a single instance of the partition - if more than one call is
         // made, this will cause a panic.
@@ -327,14 +308,12 @@ mod tests {
 
         let results = (0..MAX_TASKS)
             .map(|_| {
-                let namespace_name = Arc::clone(&*NAMESPACE_NAME);
-                let table_name = Arc::clone(&*TABLE_NAME);
                 layer.get_partition(
-                    PartitionKey::from(PARTITION_KEY),
-                    NAMESPACE_ID,
-                    namespace_name,
-                    TABLE_ID,
-                    table_name,
+                    ARBITRARY_PARTITION_KEY.clone(),
+                    ARBITRARY_NAMESPACE_ID,
+                    Arc::clone(&*DEFER_NAMESPACE_NAME_1_SEC),
+                    ARBITRARY_TABLE_ID,
+                    Arc::clone(&*DEFER_TABLE_NAME_1_SEC),
                     TRANSITION_SHARD_ID,
                 )
             })
@@ -382,7 +361,7 @@ mod tests {
             'life0: 'async_trait,
             Self: 'async_trait,
         {
-            if partition_key == PartitionKey::from(PARTITION_KEY) {
+            if partition_key == *ARBITRARY_PARTITION_KEY {
                 return future::pending().boxed();
             }
             future::ready(Arc::clone(&self.p)).boxed()
@@ -394,16 +373,7 @@ mod tests {
     async fn test_disjoint_parallelised() {
         use futures::Future;
 
-        let data = PartitionData::new(
-            PARTITION_ID,
-            PartitionKey::from(PARTITION_KEY),
-            NAMESPACE_ID,
-            Arc::clone(&*NAMESPACE_NAME),
-            TABLE_ID,
-            Arc::clone(&*TABLE_NAME),
-            SortKeyState::Provided(None),
-            TRANSITION_SHARD_ID,
-        );
+        let data = PartitionDataBuilder::new().build();
 
         // Add a single instance of the partition - if more than one call is
         // made to the mock, it will panic.
@@ -415,19 +385,19 @@ mod tests {
         // The following two partitions are for the same (blocked) partition and
         // neither resolve.
         let pa_1 = layer.get_partition(
-            PartitionKey::from(PARTITION_KEY),
-            NAMESPACE_ID,
-            Arc::clone(&*NAMESPACE_NAME),
-            TABLE_ID,
-            Arc::clone(&*TABLE_NAME),
+            ARBITRARY_PARTITION_KEY.clone(),
+            ARBITRARY_NAMESPACE_ID,
+            Arc::clone(&*DEFER_NAMESPACE_NAME_1_SEC),
+            ARBITRARY_TABLE_ID,
+            Arc::clone(&*DEFER_TABLE_NAME_1_SEC),
             TRANSITION_SHARD_ID,
         );
         let pa_2 = layer.get_partition(
-            PartitionKey::from(PARTITION_KEY),
-            NAMESPACE_ID,
-            Arc::clone(&*NAMESPACE_NAME),
-            TABLE_ID,
-            Arc::clone(&*TABLE_NAME),
+            ARBITRARY_PARTITION_KEY.clone(),
+            ARBITRARY_NAMESPACE_ID,
+            Arc::clone(&*DEFER_NAMESPACE_NAME_1_SEC),
+            ARBITRARY_TABLE_ID,
+            Arc::clone(&*DEFER_TABLE_NAME_1_SEC),
             TRANSITION_SHARD_ID,
         );
 
@@ -444,11 +414,11 @@ mod tests {
         // But a non-blocked partition is resolved without issue.
         let _ = layer
             .get_partition(
-                PartitionKey::from("platanos"),
-                NAMESPACE_ID,
-                Arc::clone(&*NAMESPACE_NAME),
-                TABLE_ID,
-                Arc::clone(&*TABLE_NAME),
+                PartitionKey::from("orange you glad i didn't say bananas"),
+                ARBITRARY_NAMESPACE_ID,
+                Arc::clone(&*DEFER_NAMESPACE_NAME_1_SEC),
+                ARBITRARY_TABLE_ID,
+                Arc::clone(&*DEFER_TABLE_NAME_1_SEC),
                 TRANSITION_SHARD_ID,
             )
             .with_timeout_panic(Duration::from_secs(5))
@@ -510,26 +480,17 @@ mod tests {
         let inner = Arc::new(SemaphoreResolver {
             sem: Arc::clone(&fake_conn_pool),
             wait: Arc::clone(&notify),
-            p: Arc::new(Mutex::new(PartitionData::new(
-                PARTITION_ID,
-                PartitionKey::from(PARTITION_KEY),
-                NAMESPACE_ID,
-                Arc::clone(&*NAMESPACE_NAME),
-                TABLE_ID,
-                Arc::clone(&*TABLE_NAME),
-                SortKeyState::Provided(None),
-                TRANSITION_SHARD_ID,
-            ))),
+            p: Arc::new(Mutex::new(PartitionDataBuilder::new().build())),
         });
 
         let layer = Arc::new(CoalescePartitionResolver::new(inner));
 
         let fut = layer.get_partition(
-            PartitionKey::from(PARTITION_KEY),
-            NAMESPACE_ID,
-            Arc::clone(&*NAMESPACE_NAME),
-            TABLE_ID,
-            Arc::clone(&*TABLE_NAME),
+            ARBITRARY_PARTITION_KEY.clone(),
+            ARBITRARY_NAMESPACE_ID,
+            Arc::clone(&*DEFER_NAMESPACE_NAME_1_SEC),
+            ARBITRARY_TABLE_ID,
+            Arc::clone(&*DEFER_TABLE_NAME_1_SEC),
             TRANSITION_SHARD_ID,
         );
 
diff --git a/ingester2/src/buffer_tree/partition/resolver/mock.rs b/ingester2/src/buffer_tree/partition/resolver/mock.rs
index a50940ce10..84276fe61c 100644
--- a/ingester2/src/buffer_tree/partition/resolver/mock.rs
+++ b/ingester2/src/buffer_tree/partition/resolver/mock.rs
@@ -9,7 +9,7 @@ use parking_lot::Mutex;
 use super::r#trait::PartitionProvider;
 use crate::{
     buffer_tree::{namespace::NamespaceName, partition::PartitionData, table::TableName},
-    deferred_load::DeferredLoad,
+    deferred_load::{self, DeferredLoad},
 };
 
 /// A mock [`PartitionProvider`] for testing that returns pre-initialised
@@ -65,8 +65,27 @@ impl PartitionProvider for MockPartitionProvider {
             });
 
         assert_eq!(p.namespace_id(), namespace_id);
-        assert_eq!(p.namespace_name().to_string(), namespace_name.to_string());
-        assert_eq!(p.table_name().to_string(), table_name.to_string());
+
+        let actual_namespace_name = p.namespace_name().to_string();
+        let expected_namespace_name = namespace_name.get().await.to_string();
+        assert!(
+            (actual_namespace_name.as_str() == expected_namespace_name)
+                || (actual_namespace_name == deferred_load::UNRESOLVED_DISPLAY_STRING),
+            "unexpected namespace name: {actual_namespace_name}. \
+            expected {expected_namespace_name} or {}",
+            deferred_load::UNRESOLVED_DISPLAY_STRING,
+        );
+
+        let actual_table_name = p.table_name().to_string();
+        let expected_table_name = table_name.get().await.to_string();
+        assert!(
+            (actual_table_name.as_str() == expected_table_name)
+                || (actual_table_name == deferred_load::UNRESOLVED_DISPLAY_STRING),
+            "unexpected table name: {actual_table_name}. \
+            expected {expected_table_name} or {}",
+            deferred_load::UNRESOLVED_DISPLAY_STRING,
+        );
+
         Arc::new(Mutex::new(p))
     }
 }
diff --git a/ingester2/src/buffer_tree/partition/resolver/trait.rs b/ingester2/src/buffer_tree/partition/resolver/trait.rs
index a40f1fd4dc..417640634a 100644
--- a/ingester2/src/buffer_tree/partition/resolver/trait.rs
+++ b/ingester2/src/buffer_tree/partition/resolver/trait.rs
@@ -60,54 +60,43 @@ where
 mod tests {
     use std::{sync::Arc, time::Duration};
 
-    use data_types::{PartitionId, ShardId};
+    use data_types::{PartitionId, ShardId, TRANSITION_SHARD_ID};
 
     use super::*;
-    use crate::buffer_tree::partition::{resolver::mock::MockPartitionProvider, SortKeyState};
-
-    const TRANSITION_SHARD_ID: ShardId = ShardId::new(84);
+    use crate::{
+        buffer_tree::partition::{resolver::mock::MockPartitionProvider, SortKeyState},
+        test_util::{
+            PartitionDataBuilder, ARBITRARY_NAMESPACE_ID, ARBITRARY_PARTITION_ID,
+            ARBITRARY_PARTITION_KEY, ARBITRARY_TABLE_ID, DEFER_NAMESPACE_NAME_1_SEC,
+            DEFER_TABLE_NAME_1_SEC,
+        },
+    };
 
     #[tokio::test]
     async fn test_arc_impl() {
-        let key = PartitionKey::from("bananas");
-        let namespace_id = NamespaceId::new(1234);
-        let namespace_name = Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-            NamespaceName::from("ns-platanos")
-        }));
-        let table_id = TableId::new(24);
-        let table_name = Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-            TableName::from("platanos")
-        }));
-        let partition = PartitionId::new(4242);
-        let data = PartitionData::new(
-            partition,
-            "bananas".into(),
-            namespace_id,
-            Arc::clone(&namespace_name),
-            table_id,
-            Arc::clone(&table_name),
-            SortKeyState::Provided(None),
-            TRANSITION_SHARD_ID,
-        );
+        let data = PartitionDataBuilder::new().build();
 
         let mock = Arc::new(MockPartitionProvider::default().with_partition(data));
 
         let got = mock
             .get_partition(
-                key,
-                namespace_id,
-                Arc::clone(&namespace_name),
-                table_id,
-                Arc::clone(&table_name),
+                ARBITRARY_PARTITION_KEY.clone(),
+                ARBITRARY_NAMESPACE_ID,
+                Arc::clone(&*DEFER_NAMESPACE_NAME_1_SEC),
+                ARBITRARY_TABLE_ID,
+                Arc::clone(&*DEFER_TABLE_NAME_1_SEC),
                 TRANSITION_SHARD_ID,
             )
             .await;
-        assert_eq!(got.lock().partition_id(), partition);
-        assert_eq!(got.lock().namespace_id(), namespace_id);
+        assert_eq!(got.lock().partition_id(), ARBITRARY_PARTITION_ID);
+        assert_eq!(got.lock().namespace_id(), ARBITRARY_NAMESPACE_ID);
         assert_eq!(
             got.lock().namespace_name().to_string(),
-            namespace_name.to_string()
+            DEFER_NAMESPACE_NAME_1_SEC.to_string()
+        );
+        assert_eq!(
+            got.lock().table_name().to_string(),
+            DEFER_TABLE_NAME_1_SEC.to_string()
         );
-        assert_eq!(got.lock().table_name().to_string(), table_name.to_string());
     }
 }
diff --git a/ingester2/src/buffer_tree/root.rs b/ingester2/src/buffer_tree/root.rs
index e38c6914d4..c680b5171a 100644
--- a/ingester2/src/buffer_tree/root.rs
+++ b/ingester2/src/buffer_tree/root.rs
@@ -234,7 +234,7 @@ mod tests {
     use std::{sync::Arc, time::Duration};
 
     use assert_matches::assert_matches;
-    use data_types::{PartitionId, PartitionKey};
+    use data_types::{PartitionId, PartitionKey, TRANSITION_SHARD_ID};
     use datafusion::{assert_batches_eq, assert_batches_sorted_eq};
     use futures::{StreamExt, TryStreamExt};
     use metric::{Attributes, Metric};
@@ -242,52 +242,35 @@ mod tests {
     use super::*;
     use crate::{
         buffer_tree::{
-            namespace::{
-                name_resolver::mock::MockNamespaceNameProvider, NamespaceData, NamespaceName,
-            },
-            partition::{resolver::mock::MockPartitionProvider, PartitionData, SortKeyState},
+            namespace::{name_resolver::mock::MockNamespaceNameProvider, NamespaceData},
+            partition::resolver::mock::MockPartitionProvider,
             post_write::mock::MockPostWriteObserver,
-            table::{name_resolver::mock::MockTableNameProvider, TableName},
+            table::TableName,
         },
         deferred_load::{self, DeferredLoad},
         query::partition_response::PartitionResponse,
-        test_util::make_write_op,
+        test_util::{
+            make_write_op, PartitionDataBuilder, ARBITRARY_NAMESPACE_ID, ARBITRARY_NAMESPACE_NAME,
+            ARBITRARY_PARTITION_KEY, ARBITRARY_TABLE_ID, ARBITRARY_TABLE_NAME,
+            ARBITRARY_TABLE_NAME_PROVIDER, DEFER_NAMESPACE_NAME_1_MS,
+        },
     };
 
-    const TABLE_ID: TableId = TableId::new(44);
-    const TABLE_NAME: &str = "bananas";
-    const NAMESPACE_NAME: &str = "platanos";
-    const NAMESPACE_ID: NamespaceId = NamespaceId::new(42);
-    const TRANSITION_SHARD_ID: ShardId = ShardId::new(84);
-
     #[tokio::test]
     async fn test_namespace_init_table() {
         let metrics = Arc::new(metric::Registry::default());
 
         // Configure the mock partition provider to return a partition for this
         // table ID.
-        let partition_provider = Arc::new(MockPartitionProvider::default().with_partition(
-            PartitionData::new(
-                PartitionId::new(0),
-                PartitionKey::from("banana-split"),
-                NAMESPACE_ID,
-                Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                    NamespaceName::from(NAMESPACE_NAME)
-                })),
-                TABLE_ID,
-                Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                    TableName::from(TABLE_NAME)
-                })),
-                SortKeyState::Provided(None),
-                TRANSITION_SHARD_ID,
-            ),
-        ));
+        let partition_provider = Arc::new(
+            MockPartitionProvider::default().with_partition(PartitionDataBuilder::new().build()),
+        );
 
         // Init the namespace
         let ns = NamespaceData::new(
-            NAMESPACE_ID,
-            Arc::new(DeferredLoad::new(Duration::from_millis(1), async { NAMESPACE_NAME.into() })),
-            Arc::new(MockTableNameProvider::new(TABLE_NAME)),
+            ARBITRARY_NAMESPACE_ID,
+            Arc::clone(&*DEFER_NAMESPACE_NAME_1_MS),
+            Arc::clone(&*ARBITRARY_TABLE_NAME_PROVIDER),
             partition_provider,
             Arc::new(MockPostWriteObserver::default()),
             &metrics,
@@ -297,27 +280,31 @@ mod tests {
         // Assert the namespace name was stored
         let name = ns.namespace_name().to_string();
         assert!(
-            (name == NAMESPACE_NAME) || (name == deferred_load::UNRESOLVED_DISPLAY_STRING),
+            (name.as_str() == &***ARBITRARY_NAMESPACE_NAME)
+                || (name == deferred_load::UNRESOLVED_DISPLAY_STRING),
             "unexpected namespace name: {name}"
         );
 
         // Assert the namespace does not contain the test data
-        assert!(ns.table(TABLE_ID).is_none());
+        assert!(ns.table(ARBITRARY_TABLE_ID).is_none());
 
         // Write some test data
         ns.apply(DmlOperation::Write(make_write_op(
-            &PartitionKey::from("banana-split"),
-            NAMESPACE_ID,
-            TABLE_NAME,
-            TABLE_ID,
+            &ARBITRARY_PARTITION_KEY,
+            ARBITRARY_NAMESPACE_ID,
+            &ARBITRARY_TABLE_NAME,
+            ARBITRARY_TABLE_ID,
             0,
-            r#"bananas,city=Madrid day="sun",temp=55 22"#,
+            &format!(
+                r#"{},city=Madrid day="sun",temp=55 22"#,
+                &*ARBITRARY_TABLE_NAME
+            ),
         )))
         .await
         .expect("buffer op should succeed");
 
         // Referencing the table should succeed
-        assert!(ns.table(TABLE_ID).is_some());
+        assert!(ns.table(ARBITRARY_TABLE_ID).is_some());
 
         // And the table counter metric should increase
         let tables = metrics
@@ -330,18 +317,21 @@ mod tests {
 
         // Ensure the deferred namespace name is loaded.
         let name = ns.namespace_name().get().await;
-        assert_eq!(&**name, NAMESPACE_NAME);
-        assert_eq!(ns.namespace_name().to_string(), NAMESPACE_NAME);
+        assert_eq!(&**name, &***ARBITRARY_NAMESPACE_NAME);
+        assert_eq!(
+            ns.namespace_name().to_string().as_str(),
+            &***ARBITRARY_NAMESPACE_NAME
+        );
     }
 
     /// Generate a test that performs a set of writes and assert the data within
-    /// the table with TABLE_ID in the namespace with NAMESPACE_ID.
+    /// the table with ARBITRARY_TABLE_ID in the namespace with ARBITRARY_NAMESPACE_ID.
     macro_rules! test_write_query {
         (
             $name:ident,
             partitions = [$($partition:expr), +], // The set of PartitionData for the mock partition provider
             writes = [$($write:expr), *],         // The set of DmlWrite to apply()
-            want = $want:expr                     // The expected results of querying NAMESPACE_ID and TABLE_ID
+            want = $want:expr                     // The expected results of querying ARBITRARY_NAMESPACE_ID and ARBITRARY_TABLE_ID
         ) => {
             paste::paste! {
                 #[tokio::test]
@@ -356,8 +346,8 @@ mod tests {
 
                     // Init the buffer tree
                     let buf = BufferTree::new(
-                        Arc::new(MockNamespaceNameProvider::new(NAMESPACE_NAME)),
-                        Arc::new(MockTableNameProvider::new(TABLE_NAME)),
+                        Arc::new(MockNamespaceNameProvider::new(&**ARBITRARY_NAMESPACE_NAME)),
+                        Arc::clone(&*ARBITRARY_TABLE_NAME_PROVIDER),
                         partition_provider,
                         Arc::new(MockPostWriteObserver::default()),
                         Arc::new(metric::Registry::default()),
@@ -371,9 +361,9 @@ mod tests {
                             .expect("failed to perform write");
                     )*
 
-                    // Execute the query against NAMESPACE_ID and TABLE_ID
+                    // Execute the query against ARBITRARY_NAMESPACE_ID and ARBITRARY_TABLE_ID
                     let batches = buf
-                        .query_exec(NAMESPACE_ID, TABLE_ID, vec![], None)
+                        .query_exec(ARBITRARY_NAMESPACE_ID, ARBITRARY_TABLE_ID, vec![], None)
                         .await
                         .expect("query should succeed")
                         .into_record_batches()
@@ -381,7 +371,7 @@ mod tests {
                         .await
                         .expect("query failed");
 
-                    // Assert the contents of NAMESPACE_ID and TABLE_ID
+                    // Assert the contents of ARBITRARY_NAMESPACE_ID and ARBITRARY_TABLE_ID
                     assert_batches_sorted_eq!(
                         $want,
                         &batches
@@ -394,27 +384,20 @@ mod tests {
     // A simple "read your writes" test.
     test_write_query!(
         read_writes,
-        partitions = [PartitionData::new(
-            PartitionId::new(0),
-            PartitionKey::from("p1"),
-            NAMESPACE_ID,
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                NamespaceName::from(NAMESPACE_NAME)
-            })),
-            TABLE_ID,
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                TableName::from(TABLE_NAME)
-            })),
-            SortKeyState::Provided(None),
-            TRANSITION_SHARD_ID,
-        )],
+        partitions = [PartitionDataBuilder::new()
+            .with_partition_id(PartitionId::new(0))
+            .with_partition_key(PartitionKey::from("p1"))
+            .build()],
         writes = [make_write_op(
             &PartitionKey::from("p1"),
-            NAMESPACE_ID,
-            TABLE_NAME,
-            TABLE_ID,
+            ARBITRARY_NAMESPACE_ID,
+            &ARBITRARY_TABLE_NAME,
+            ARBITRARY_TABLE_ID,
             0,
-            r#"bananas,region=Asturias temp=35 4242424242"#,
+            &format!(
+                r#"{},region=Asturias temp=35 4242424242"#,
+                &*ARBITRARY_TABLE_NAME
+            ),
         )],
         want = [
             "+----------+------+-------------------------------+",
@@ -430,51 +413,37 @@ mod tests {
     test_write_query!(
         multiple_partitions,
         partitions = [
-            PartitionData::new(
-                PartitionId::new(0),
-                PartitionKey::from("p1"),
-                NAMESPACE_ID,
-                Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                    NamespaceName::from(NAMESPACE_NAME)
-                })),
-                TABLE_ID,
-                Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                    TableName::from(TABLE_NAME)
-                })),
-                SortKeyState::Provided(None),
-                TRANSITION_SHARD_ID,
-            ),
-            PartitionData::new(
-                PartitionId::new(1),
-                PartitionKey::from("p2"),
-                NAMESPACE_ID,
-                Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                    NamespaceName::from(NAMESPACE_NAME)
-                })),
-                TABLE_ID,
-                Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                    TableName::from(TABLE_NAME)
-                })),
-                SortKeyState::Provided(None),
-                TRANSITION_SHARD_ID,
-            )
+            PartitionDataBuilder::new()
+                .with_partition_id(PartitionId::new(0))
+                .with_partition_key(PartitionKey::from("p1"))
+                .build(),
+            PartitionDataBuilder::new()
+                .with_partition_id(PartitionId::new(1))
+                .with_partition_key(PartitionKey::from("p2"))
+                .build()
         ],
         writes = [
             make_write_op(
                 &PartitionKey::from("p1"),
-                NAMESPACE_ID,
-                TABLE_NAME,
-                TABLE_ID,
+                ARBITRARY_NAMESPACE_ID,
+                &ARBITRARY_TABLE_NAME,
+                ARBITRARY_TABLE_ID,
                 0,
-                r#"bananas,region=Madrid temp=35 4242424242"#,
+                &format!(
+                    r#"{},region=Madrid temp=35 4242424242"#,
+                    &*ARBITRARY_TABLE_NAME
+                ),
             ),
             make_write_op(
                 &PartitionKey::from("p2"),
-                NAMESPACE_ID,
-                TABLE_NAME,
-                TABLE_ID,
+                ARBITRARY_NAMESPACE_ID,
+                &ARBITRARY_TABLE_NAME,
+                ARBITRARY_TABLE_ID,
                 0,
-                r#"bananas,region=Asturias temp=25 4242424242"#,
+                &format!(
+                    r#"{},region=Asturias temp=25 4242424242"#,
+                    &*ARBITRARY_TABLE_NAME
+                ),
             )
         ],
         want = [
@@ -492,51 +461,39 @@ mod tests {
     test_write_query!(
         filter_multiple_namespaces,
         partitions = [
-            PartitionData::new(
-                PartitionId::new(0),
-                PartitionKey::from("p1"),
-                NAMESPACE_ID,
-                Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                    NamespaceName::from(NAMESPACE_NAME)
-                })),
-                TABLE_ID,
-                Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                    TableName::from(TABLE_NAME)
-                })),
-                SortKeyState::Provided(None),
-                TRANSITION_SHARD_ID,
-            ),
-            PartitionData::new(
-                PartitionId::new(1),
-                PartitionKey::from("p2"),
-                NamespaceId::new(4321), // A different namespace ID.
-                Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                    NamespaceName::from(NAMESPACE_NAME)
-                })),
-                TableId::new(1234), // A different table ID.
-                Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                    TableName::from(TABLE_NAME)
-                })),
-                SortKeyState::Provided(None),
-                TRANSITION_SHARD_ID,
-            )
+            PartitionDataBuilder::new()
+                .with_partition_id(PartitionId::new(0))
+                .with_partition_key(PartitionKey::from("p1"))
+                .build(),
+            PartitionDataBuilder::new()
+                .with_partition_id(PartitionId::new(1))
+                .with_partition_key(PartitionKey::from("p2"))
+                .with_namespace_id(NamespaceId::new(4321)) // A different namespace ID.
+                .with_table_id(TableId::new(1234)) // A different table ID.
+                .build()
         ],
         writes = [
             make_write_op(
                 &PartitionKey::from("p1"),
-                NAMESPACE_ID,
-                TABLE_NAME,
-                TABLE_ID,
+                ARBITRARY_NAMESPACE_ID,
+                &ARBITRARY_TABLE_NAME,
+                ARBITRARY_TABLE_ID,
                 0,
-                r#"bananas,region=Madrid temp=25 4242424242"#,
+                &format!(
+                    r#"{},region=Madrid temp=25 4242424242"#,
+                    &*ARBITRARY_TABLE_NAME
+                ),
             ),
             make_write_op(
                 &PartitionKey::from("p2"),
                 NamespaceId::new(4321), // A different namespace ID.
-                TABLE_NAME,
+                &ARBITRARY_TABLE_NAME,
                 TableId::new(1234), // A different table ID
                 0,
-                r#"bananas,region=Asturias temp=35 4242424242"#,
+                &format!(
+                    r#"{},region=Asturias temp=35 4242424242"#,
+                    &*ARBITRARY_TABLE_NAME
+                ),
             )
         ],
         want = [
@@ -553,51 +510,38 @@ mod tests {
     test_write_query!(
         filter_multiple_tabls,
         partitions = [
-            PartitionData::new(
-                PartitionId::new(0),
-                PartitionKey::from("p1"),
-                NAMESPACE_ID,
-                Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                    NamespaceName::from(NAMESPACE_NAME)
-                })),
-                TABLE_ID,
-                Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                    TableName::from(TABLE_NAME)
-                })),
-                SortKeyState::Provided(None),
-                TRANSITION_SHARD_ID,
-            ),
-            PartitionData::new(
-                PartitionId::new(1),
-                PartitionKey::from("p2"),
-                NAMESPACE_ID,
-                Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                    NamespaceName::from(NAMESPACE_NAME)
-                })),
-                TableId::new(1234), // A different table ID.
-                Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                    TableName::from(TABLE_NAME)
-                })),
-                SortKeyState::Provided(None),
-                TRANSITION_SHARD_ID,
-            )
+            PartitionDataBuilder::new()
+                .with_partition_id(PartitionId::new(0))
+                .with_partition_key(PartitionKey::from("p1"))
+                .build(),
+            PartitionDataBuilder::new()
+                .with_partition_id(PartitionId::new(1))
+                .with_partition_key(PartitionKey::from("p2"))
+                .with_table_id(TableId::new(1234)) // A different table ID.
+                .build()
         ],
         writes = [
             make_write_op(
                 &PartitionKey::from("p1"),
-                NAMESPACE_ID,
-                TABLE_NAME,
-                TABLE_ID,
+                ARBITRARY_NAMESPACE_ID,
+                &ARBITRARY_TABLE_NAME,
+                ARBITRARY_TABLE_ID,
                 0,
-                r#"bananas,region=Madrid temp=25 4242424242"#,
+                &format!(
+                    r#"{},region=Madrid temp=25 4242424242"#,
+                    &*ARBITRARY_TABLE_NAME
+                ),
             ),
             make_write_op(
                 &PartitionKey::from("p2"),
-                NAMESPACE_ID,
-                TABLE_NAME,
+                ARBITRARY_NAMESPACE_ID,
+                &ARBITRARY_TABLE_NAME,
                 TableId::new(1234), // A different table ID
                 0,
-                r#"bananas,region=Asturias temp=35 4242424242"#,
+                &format!(
+                    r#"{},region=Asturias temp=35 4242424242"#,
+                    &*ARBITRARY_TABLE_NAME
+                ),
             )
         ],
         want = [
@@ -615,36 +559,32 @@ mod tests {
     // writes).
     test_write_query!(
         duplicate_writes,
-        partitions = [PartitionData::new(
-            PartitionId::new(0),
-            PartitionKey::from("p1"),
-            NAMESPACE_ID,
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                NamespaceName::from(NAMESPACE_NAME)
-            })),
-            TABLE_ID,
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                TableName::from(TABLE_NAME)
-            })),
-            SortKeyState::Provided(None),
-            TRANSITION_SHARD_ID,
-        )],
+        partitions = [PartitionDataBuilder::new()
+            .with_partition_id(PartitionId::new(0))
+            .with_partition_key(PartitionKey::from("p1"))
+            .build()],
         writes = [
             make_write_op(
                 &PartitionKey::from("p1"),
-                NAMESPACE_ID,
-                TABLE_NAME,
-                TABLE_ID,
+                ARBITRARY_NAMESPACE_ID,
+                &ARBITRARY_TABLE_NAME,
+                ARBITRARY_TABLE_ID,
                 0,
-                r#"bananas,region=Asturias temp=35 4242424242"#,
+                &format!(
+                    r#"{},region=Asturias temp=35 4242424242"#,
+                    &*ARBITRARY_TABLE_NAME
+                ),
             ),
             make_write_op(
                 &PartitionKey::from("p1"),
-                NAMESPACE_ID,
-                TABLE_NAME,
-                TABLE_ID,
+                ARBITRARY_NAMESPACE_ID,
+                &ARBITRARY_TABLE_NAME,
+                ARBITRARY_TABLE_ID,
                 1,
-                r#"bananas,region=Asturias temp=12 4242424242"#,
+                &format!(
+                    r#"{},region=Asturias temp=12 4242424242"#,
+                    &*ARBITRARY_TABLE_NAME
+                ),
             )
         ],
         want = [
@@ -665,56 +605,43 @@ mod tests {
         // p1.
         let partition_provider = Arc::new(
             MockPartitionProvider::default()
-                .with_partition(PartitionData::new(
-                    PartitionId::new(0),
-                    PartitionKey::from("p1"),
-                    NAMESPACE_ID,
-                    Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                        NamespaceName::from(NAMESPACE_NAME)
-                    })),
-                    TABLE_ID,
-                    Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                        TableName::from(TABLE_NAME)
-                    })),
-                    SortKeyState::Provided(None),
-                    TRANSITION_SHARD_ID,
-                ))
-                .with_partition(PartitionData::new(
-                    PartitionId::new(0),
-                    PartitionKey::from("p2"),
-                    NAMESPACE_ID,
-                    Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                        NamespaceName::from(NAMESPACE_NAME)
-                    })),
-                    TABLE_ID,
-                    Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                        TableName::from(TABLE_NAME)
-                    })),
-                    SortKeyState::Provided(None),
-                    TRANSITION_SHARD_ID,
-                )),
+                .with_partition(
+                    PartitionDataBuilder::new()
+                        .with_partition_id(PartitionId::new(0))
+                        .with_partition_key(PartitionKey::from("p1"))
+                        .build(),
+                )
+                .with_partition(
+                    PartitionDataBuilder::new()
+                        .with_partition_id(PartitionId::new(0))
+                        .with_partition_key(PartitionKey::from("p2"))
+                        .build(),
+                ),
         );
 
         let metrics = Arc::new(metric::Registry::default());
 
         // Init the buffer tree
         let buf = BufferTree::new(
-            Arc::new(MockNamespaceNameProvider::new(NAMESPACE_NAME)),
-            Arc::new(MockTableNameProvider::new(TABLE_NAME)),
+            Arc::new(MockNamespaceNameProvider::new(&**ARBITRARY_NAMESPACE_NAME)),
+            Arc::clone(&*ARBITRARY_TABLE_NAME_PROVIDER),
             partition_provider,
             Arc::new(MockPostWriteObserver::default()),
             Arc::clone(&metrics),
             TRANSITION_SHARD_ID,
         );
 
-        // Write data to partition p1, in table "bananas".
+        // Write data to partition p1, in the arbitrary table
         buf.apply(DmlOperation::Write(make_write_op(
             &PartitionKey::from("p1"),
-            NAMESPACE_ID,
-            TABLE_NAME,
-            TABLE_ID,
+            ARBITRARY_NAMESPACE_ID,
+            &ARBITRARY_TABLE_NAME,
+            ARBITRARY_TABLE_ID,
             0,
-            r#"bananas,region=Asturias temp=35 4242424242"#,
+            &format!(
+                r#"{},region=Asturias temp=35 4242424242"#,
+                &*ARBITRARY_TABLE_NAME
+            ),
         )))
         .await
         .expect("failed to write initial data");
@@ -723,11 +650,14 @@ mod tests {
         // different temp value.
         buf.apply(DmlOperation::Write(make_write_op(
             &PartitionKey::from("p2"),
-            NAMESPACE_ID,
-            TABLE_NAME,
-            TABLE_ID,
+            ARBITRARY_NAMESPACE_ID,
+            &ARBITRARY_TABLE_NAME,
+            ARBITRARY_TABLE_ID,
             1,
-            r#"bananas,region=Asturias temp=12 4242424242"#,
+            &format!(
+                r#"{},region=Asturias temp=12 4242424242"#,
+                &*ARBITRARY_TABLE_NAME
+            ),
         )))
         .await
         .expect("failed to overwrite data");
@@ -757,59 +687,41 @@ mod tests {
     #[tokio::test]
     async fn test_partition_iter() {
         const TABLE2_ID: TableId = TableId::new(1234321);
+        const TABLE2_NAME: &str = "another_table";
 
         // Configure the mock partition provider to return a single partition, named
         // p1.
         let partition_provider = Arc::new(
             MockPartitionProvider::default()
-                .with_partition(PartitionData::new(
-                    PartitionId::new(0),
-                    PartitionKey::from("p1"),
-                    NAMESPACE_ID,
-                    Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                        NamespaceName::from(NAMESPACE_NAME)
-                    })),
-                    TABLE_ID,
-                    Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                        TableName::from(TABLE_NAME)
-                    })),
-                    SortKeyState::Provided(None),
-                    TRANSITION_SHARD_ID,
-                ))
-                .with_partition(PartitionData::new(
-                    PartitionId::new(1),
-                    PartitionKey::from("p2"),
-                    NAMESPACE_ID,
-                    Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                        NamespaceName::from(NAMESPACE_NAME)
-                    })),
-                    TABLE_ID,
-                    Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                        TableName::from(TABLE_NAME)
-                    })),
-                    SortKeyState::Provided(None),
-                    TRANSITION_SHARD_ID,
-                ))
-                .with_partition(PartitionData::new(
-                    PartitionId::new(2),
-                    PartitionKey::from("p3"),
-                    NAMESPACE_ID,
-                    Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                        NamespaceName::from(NAMESPACE_NAME)
-                    })),
-                    TABLE2_ID,
-                    Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                        TableName::from("another_table")
-                    })),
-                    SortKeyState::Provided(None),
-                    TRANSITION_SHARD_ID,
-                )),
+                .with_partition(
+                    PartitionDataBuilder::new()
+                        .with_partition_id(PartitionId::new(0))
+                        .with_partition_key(PartitionKey::from("p1"))
+                        .build(),
+                )
+                .with_partition(
+                    PartitionDataBuilder::new()
+                        .with_partition_id(PartitionId::new(1))
+                        .with_partition_key(PartitionKey::from("p2"))
+                        .build(),
+                )
+                .with_partition(
+                    PartitionDataBuilder::new()
+                        .with_partition_id(PartitionId::new(2))
+                        .with_partition_key(PartitionKey::from("p3"))
+                        .with_table_id(TABLE2_ID)
+                        .with_table_name(Arc::new(DeferredLoad::new(
+                            Duration::from_secs(1),
+                            async move { TableName::from(TABLE2_NAME) },
+                        )))
+                        .build(),
+                ),
         );
 
         // Init the buffer tree
         let buf = BufferTree::new(
-            Arc::new(MockNamespaceNameProvider::new(NAMESPACE_NAME)),
-            Arc::new(MockTableNameProvider::new(TABLE_NAME)),
+            Arc::new(MockNamespaceNameProvider::new(&**ARBITRARY_NAMESPACE_NAME)),
+            Arc::clone(&*ARBITRARY_TABLE_NAME_PROVIDER),
             partition_provider,
             Arc::new(MockPostWriteObserver::default()),
             Arc::clone(&Arc::new(metric::Registry::default())),
@@ -818,28 +730,34 @@ mod tests {
 
         assert_eq!(buf.partitions().count(), 0);
 
-        // Write data to partition p1, in table "bananas".
+        // Write data to partition p1, in the arbitrary table
         buf.apply(DmlOperation::Write(make_write_op(
             &PartitionKey::from("p1"),
-            NAMESPACE_ID,
-            TABLE_NAME,
-            TABLE_ID,
+            ARBITRARY_NAMESPACE_ID,
+            &ARBITRARY_TABLE_NAME,
+            ARBITRARY_TABLE_ID,
             0,
-            r#"bananas,region=Asturias temp=35 4242424242"#,
+            &format!(
+                r#"{},region=Asturias temp=35 4242424242"#,
+                &*ARBITRARY_TABLE_NAME
+            ),
         )))
         .await
         .expect("failed to write initial data");
 
         assert_eq!(buf.partitions().count(), 1);
 
-        // Write data to partition p2, in table "bananas".
+        // Write data to partition p2, in the arbitrary table
         buf.apply(DmlOperation::Write(make_write_op(
             &PartitionKey::from("p2"),
-            NAMESPACE_ID,
-            TABLE_NAME,
-            TABLE_ID,
+            ARBITRARY_NAMESPACE_ID,
+            &ARBITRARY_TABLE_NAME,
+            ARBITRARY_TABLE_ID,
             0,
-            r#"bananas,region=Asturias temp=35 4242424242"#,
+            &format!(
+                r#"{},region=Asturias temp=35 4242424242"#,
+                &*ARBITRARY_TABLE_NAME
+            ),
         )))
         .await
         .expect("failed to write initial data");
@@ -849,11 +767,11 @@ mod tests {
         // Write data to partition p3, in the second table
         buf.apply(DmlOperation::Write(make_write_op(
             &PartitionKey::from("p3"),
-            NAMESPACE_ID,
-            "another_table",
+            ARBITRARY_NAMESPACE_ID,
+            TABLE2_NAME,
             TABLE2_ID,
             0,
-            r#"another_table,region=Asturias temp=35 4242424242"#,
+            &format!(r#"{},region=Asturias temp=35 4242424242"#, TABLE2_NAME),
         )))
         .await
         .expect("failed to write initial data");
@@ -874,27 +792,19 @@ mod tests {
     /// returns no data (as opposed to panicking, etc).
     #[tokio::test]
     async fn test_not_found() {
-        let partition_provider = Arc::new(MockPartitionProvider::default().with_partition(
-            PartitionData::new(
-                PartitionId::new(0),
-                PartitionKey::from("p1"),
-                NAMESPACE_ID,
-                Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                    NamespaceName::from(NAMESPACE_NAME)
-                })),
-                TABLE_ID,
-                Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                    TableName::from(TABLE_NAME)
-                })),
-                SortKeyState::Provided(None),
-                TRANSITION_SHARD_ID,
+        let partition_provider = Arc::new(
+            MockPartitionProvider::default().with_partition(
+                PartitionDataBuilder::new()
+                    .with_partition_id(PartitionId::new(0))
+                    .with_partition_key(PartitionKey::from("p1"))
+                    .build(),
             ),
-        ));
+        );
 
         // Init the BufferTree
         let buf = BufferTree::new(
-            Arc::new(MockNamespaceNameProvider::new(NAMESPACE_NAME)),
-            Arc::new(MockTableNameProvider::new(TABLE_NAME)),
+            Arc::new(MockNamespaceNameProvider::new(&**ARBITRARY_NAMESPACE_NAME)),
+            Arc::clone(&*ARBITRARY_TABLE_NAME_PROVIDER),
             partition_provider,
             Arc::new(MockPostWriteObserver::default()),
             Arc::new(metric::Registry::default()),
@@ -903,37 +813,40 @@ mod tests {
 
         // Query the empty tree
         let err = buf
-            .query_exec(NAMESPACE_ID, TABLE_ID, vec![], None)
+            .query_exec(ARBITRARY_NAMESPACE_ID, ARBITRARY_TABLE_ID, vec![], None)
             .await
             .expect_err("query should fail");
         assert_matches!(err, QueryError::NamespaceNotFound(ns) => {
-            assert_eq!(ns, NAMESPACE_ID);
+            assert_eq!(ns, ARBITRARY_NAMESPACE_ID);
         });
 
-        // Write data to partition p1, in table "bananas".
+        // Write data to partition p1, in the arbitrary table
         buf.apply(DmlOperation::Write(make_write_op(
             &PartitionKey::from("p1"),
-            NAMESPACE_ID,
-            TABLE_NAME,
-            TABLE_ID,
+            ARBITRARY_NAMESPACE_ID,
+            &ARBITRARY_TABLE_NAME,
+            ARBITRARY_TABLE_ID,
             0,
-            r#"bananas,region=Asturias temp=35 4242424242"#,
+            &format!(
+                r#"{},region=Asturias temp=35 4242424242"#,
+                &*ARBITRARY_TABLE_NAME
+            ),
         )))
         .await
         .expect("failed to write data");
 
         // Ensure an unknown table errors
         let err = buf
-            .query_exec(NAMESPACE_ID, TableId::new(1234), vec![], None)
+            .query_exec(ARBITRARY_NAMESPACE_ID, TableId::new(1234), vec![], None)
             .await
             .expect_err("query should fail");
         assert_matches!(err, QueryError::TableNotFound(ns, t) => {
-            assert_eq!(ns, NAMESPACE_ID);
+            assert_eq!(ns, ARBITRARY_NAMESPACE_ID);
             assert_eq!(t, TableId::new(1234));
         });
 
         // Ensure a valid namespace / table does not error
-        buf.query_exec(NAMESPACE_ID, TABLE_ID, vec![], None)
+        buf.query_exec(ARBITRARY_NAMESPACE_ID, ARBITRARY_TABLE_ID, vec![], None)
             .await
             .expect("namespace / table should exist");
     }
@@ -960,54 +873,41 @@ mod tests {
         // p1 and p2.
         let partition_provider = Arc::new(
             MockPartitionProvider::default()
-                .with_partition(PartitionData::new(
-                    PartitionId::new(0),
-                    PartitionKey::from("p1"),
-                    NAMESPACE_ID,
-                    Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                        NamespaceName::from(NAMESPACE_NAME)
-                    })),
-                    TABLE_ID,
-                    Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                        TableName::from(TABLE_NAME)
-                    })),
-                    SortKeyState::Provided(None),
-                    TRANSITION_SHARD_ID,
-                ))
-                .with_partition(PartitionData::new(
-                    PartitionId::new(1),
-                    PartitionKey::from("p2"),
-                    NAMESPACE_ID,
-                    Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                        NamespaceName::from(NAMESPACE_NAME)
-                    })),
-                    TABLE_ID,
-                    Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                        TableName::from(TABLE_NAME)
-                    })),
-                    SortKeyState::Provided(None),
-                    TRANSITION_SHARD_ID,
-                )),
+                .with_partition(
+                    PartitionDataBuilder::new()
+                        .with_partition_id(PartitionId::new(0))
+                        .with_partition_key(PartitionKey::from("p1"))
+                        .build(),
+                )
+                .with_partition(
+                    PartitionDataBuilder::new()
+                        .with_partition_id(PartitionId::new(1))
+                        .with_partition_key(PartitionKey::from("p2"))
+                        .build(),
+                ),
         );
 
         // Init the buffer tree
         let buf = BufferTree::new(
-            Arc::new(MockNamespaceNameProvider::new(NAMESPACE_NAME)),
-            Arc::new(MockTableNameProvider::new(TABLE_NAME)),
+            Arc::new(MockNamespaceNameProvider::new(&**ARBITRARY_NAMESPACE_NAME)),
+            Arc::clone(&*ARBITRARY_TABLE_NAME_PROVIDER),
             partition_provider,
             Arc::new(MockPostWriteObserver::default()),
             Arc::new(metric::Registry::default()),
             TRANSITION_SHARD_ID,
         );
 
-        // Write data to partition p1, in table "bananas".
+        // Write data to partition p1, in the arbitrary table
         buf.apply(DmlOperation::Write(make_write_op(
             &PartitionKey::from("p1"),
-            NAMESPACE_ID,
-            TABLE_NAME,
-            TABLE_ID,
+            ARBITRARY_NAMESPACE_ID,
+            &ARBITRARY_TABLE_NAME,
+            ARBITRARY_TABLE_ID,
             0,
-            r#"bananas,region=Madrid temp=35 4242424242"#,
+            &format!(
+                r#"{},region=Madrid temp=35 4242424242"#,
+                &*ARBITRARY_TABLE_NAME
+            ),
         )))
         .await
         .expect("failed to write initial data");
@@ -1015,7 +915,7 @@ mod tests {
         // Execute a query of the buffer tree, generating the result stream, but
         // DO NOT consume it.
         let stream = buf
-            .query_exec(NAMESPACE_ID, TABLE_ID, vec![], None)
+            .query_exec(ARBITRARY_NAMESPACE_ID, ARBITRARY_TABLE_ID, vec![], None)
             .await
             .expect("query should succeed")
             .into_partition_stream();
@@ -1024,11 +924,14 @@ mod tests {
         // that creates a new partition (p2) in the same table.
         buf.apply(DmlOperation::Write(make_write_op(
             &PartitionKey::from("p2"),
-            NAMESPACE_ID,
-            TABLE_NAME,
-            TABLE_ID,
+            ARBITRARY_NAMESPACE_ID,
+            &ARBITRARY_TABLE_NAME,
+            ARBITRARY_TABLE_ID,
             1,
-            r#"bananas,region=Asturias temp=20 4242424242"#,
+            &format!(
+                r#"{},region=Asturias temp=20 4242424242"#,
+                &*ARBITRARY_TABLE_NAME
+            ),
         )))
         .await
         .expect("failed to perform concurrent write to new partition");
@@ -1037,11 +940,14 @@ mod tests {
         // results snapshot (p1) before the partition is read.
         buf.apply(DmlOperation::Write(make_write_op(
             &PartitionKey::from("p1"),
-            NAMESPACE_ID,
-            TABLE_NAME,
-            TABLE_ID,
+            ARBITRARY_NAMESPACE_ID,
+            &ARBITRARY_TABLE_NAME,
+            ARBITRARY_TABLE_ID,
             2,
-            r#"bananas,region=Murcia temp=30 4242424242"#,
+            &format!(
+                r#"{},region=Murcia temp=30 4242424242"#,
+                &*ARBITRARY_TABLE_NAME
+            ),
         )))
         .await
         .expect("failed to perform concurrent write to existing partition");
diff --git a/ingester2/src/buffer_tree/table.rs b/ingester2/src/buffer_tree/table.rs
index 5ed810e567..450a15355c 100644
--- a/ingester2/src/buffer_tree/table.rs
+++ b/ingester2/src/buffer_tree/table.rs
@@ -260,74 +260,63 @@ where
 
 #[cfg(test)]
 mod tests {
-    use std::{sync::Arc, time::Duration};
+    use std::sync::Arc;
 
-    use data_types::PartitionId;
+    use data_types::TRANSITION_SHARD_ID;
     use mutable_batch_lp::lines_to_batches;
 
     use super::*;
-    use crate::buffer_tree::{
-        partition::{resolver::mock::MockPartitionProvider, PartitionData, SortKeyState},
-        post_write::mock::MockPostWriteObserver,
+    use crate::{
+        buffer_tree::{
+            partition::resolver::mock::MockPartitionProvider,
+            post_write::mock::MockPostWriteObserver,
+        },
+        test_util::{
+            PartitionDataBuilder, ARBITRARY_NAMESPACE_ID, ARBITRARY_PARTITION_KEY,
+            ARBITRARY_TABLE_ID, ARBITRARY_TABLE_NAME, DEFER_NAMESPACE_NAME_1_SEC,
+            DEFER_TABLE_NAME_1_SEC,
+        },
     };
 
-    const TABLE_NAME: &str = "bananas";
-    const TABLE_ID: TableId = TableId::new(44);
-    const NAMESPACE_ID: NamespaceId = NamespaceId::new(42);
-    const PARTITION_KEY: &str = "platanos";
-    const PARTITION_ID: PartitionId = PartitionId::new(0);
-    const TRANSITION_SHARD_ID: ShardId = ShardId::new(84);
-
     #[tokio::test]
     async fn test_partition_init() {
-        // Configure the mock partition provider to return a partition for this
-        // table ID.
-        let partition_provider = Arc::new(MockPartitionProvider::default().with_partition(
-            PartitionData::new(
-                PARTITION_ID,
-                PARTITION_KEY.into(),
-                NAMESPACE_ID,
-                Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                    NamespaceName::from("platanos")
-                })),
-                TABLE_ID,
-                Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                    TableName::from(TABLE_NAME)
-                })),
-                SortKeyState::Provided(None),
-                TRANSITION_SHARD_ID,
-            ),
-        ));
+        // Configure the mock partition provider to return a partition for a table ID.
+        let partition_provider = Arc::new(
+            MockPartitionProvider::default().with_partition(PartitionDataBuilder::new().build()),
+        );
 
         let table = TableData::new(
-            TABLE_ID,
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                TableName::from(TABLE_NAME)
-            })),
-            NAMESPACE_ID,
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                NamespaceName::from("platanos")
-            })),
+            ARBITRARY_TABLE_ID,
+            Arc::clone(&*DEFER_TABLE_NAME_1_SEC),
+            ARBITRARY_NAMESPACE_ID,
+            Arc::clone(&*DEFER_NAMESPACE_NAME_1_SEC),
             partition_provider,
             Arc::new(MockPostWriteObserver::default()),
             TRANSITION_SHARD_ID,
         );
 
-        let batch = lines_to_batches(r#"bananas,bat=man value=24 42"#, 0)
-            .unwrap()
-            .remove(TABLE_NAME)
-            .unwrap();
+        let batch = lines_to_batches(
+            &format!(r#"{},bat=man value=24 42"#, &*ARBITRARY_TABLE_NAME),
+            0,
+        )
+        .unwrap()
+        .remove(&***ARBITRARY_TABLE_NAME)
+        .unwrap();
 
         // Assert the table does not contain the test partition
-        assert!(table.partition_data.get(&PARTITION_KEY.into()).is_none());
+        assert!(table.partition_data.get(&ARBITRARY_PARTITION_KEY).is_none());
 
         // Write some test data
         table
-            .buffer_table_write(SequenceNumber::new(42), batch, PARTITION_KEY.into())
+            .buffer_table_write(
+                SequenceNumber::new(42),
+                batch,
+                ARBITRARY_PARTITION_KEY.clone(),
+            )
             .await
             .expect("buffer op should succeed");
 
         // Referencing the partition should succeed
-        assert!(table.partition_data.get(&PARTITION_KEY.into()).is_some());
+        assert!(table.partition_data.get(&ARBITRARY_PARTITION_KEY).is_some());
     }
 }
diff --git a/ingester2/src/buffer_tree/table/name_resolver.rs b/ingester2/src/buffer_tree/table/name_resolver.rs
index e43b2217d2..576e7ce6cb 100644
--- a/ingester2/src/buffer_tree/table/name_resolver.rs
+++ b/ingester2/src/buffer_tree/table/name_resolver.rs
@@ -49,9 +49,7 @@ impl TableNameResolver {
                     .get_by_id(table_id)
                     .await?
                     .unwrap_or_else(|| {
-                        panic!(
-                            "resolving table name for non-existent table id {table_id}"
-                        )
+                        panic!("resolving table name for non-existent table id {table_id}")
                     })
                     .name
                     .into();
diff --git a/ingester2/src/init/graceful_shutdown.rs b/ingester2/src/init/graceful_shutdown.rs
index 84b44df019..d544b54342 100644
--- a/ingester2/src/init/graceful_shutdown.rs
+++ b/ingester2/src/init/graceful_shutdown.rs
@@ -132,51 +132,29 @@ mod tests {
     use std::{future::ready, sync::Arc, task::Poll};
 
     use assert_matches::assert_matches;
-    use data_types::{NamespaceId, PartitionId, PartitionKey, SequenceNumber, ShardId, TableId};
+    use data_types::SequenceNumber;
     use futures::FutureExt;
-    use lazy_static::lazy_static;
     use mutable_batch_lp::test_helpers::lp_to_mutable_batch;
     use parking_lot::Mutex;
     use test_helpers::timeout::FutureTimeout;
 
     use crate::{
-        buffer_tree::{
-            namespace::NamespaceName, partition::PartitionData, partition::SortKeyState,
-            table::TableName,
-        },
-        deferred_load::DeferredLoad,
+        buffer_tree::partition::PartitionData,
         persist::queue::mock::MockPersistQueue,
+        test_util::{PartitionDataBuilder, ARBITRARY_TABLE_NAME},
     };
 
     use super::*;
 
-    const PARTITION_ID: PartitionId = PartitionId::new(1);
-    const TRANSITION_SHARD_ID: ShardId = ShardId::new(84);
-
-    lazy_static! {
-        static ref PARTITION_KEY: PartitionKey = PartitionKey::from("platanos");
-        static ref TABLE_NAME: TableName = TableName::from("bananas");
-        static ref NAMESPACE_NAME: NamespaceName = NamespaceName::from("namespace-bananas");
-    }
-
     // Initialise a partition containing buffered data.
     fn new_partition() -> Arc<Mutex<PartitionData>> {
-        let mut partition = PartitionData::new(
-            PARTITION_ID,
-            PARTITION_KEY.clone(),
-            NamespaceId::new(3),
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                NAMESPACE_NAME.clone()
-            })),
-            TableId::new(4),
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                TABLE_NAME.clone()
-            })),
-            SortKeyState::Provided(None),
-            TRANSITION_SHARD_ID,
-        );
+        let mut partition = PartitionDataBuilder::new().build();
 
-        let mb = lp_to_mutable_batch(r#"bananas,city=London people=2,pigeons="millions" 10"#).1;
+        let mb = lp_to_mutable_batch(&format!(
+            r#"{},city=London people=2,pigeons="millions" 10"#,
+            &*ARBITRARY_TABLE_NAME
+        ))
+        .1;
         partition
             .buffer_write(mb, SequenceNumber::new(1))
             .expect("failed to write dummy data");
diff --git a/ingester2/src/init/wal_replay.rs b/ingester2/src/init/wal_replay.rs
index ddf184ddd9..8ee8df0d4a 100644
--- a/ingester2/src/init/wal_replay.rs
+++ b/ingester2/src/init/wal_replay.rs
@@ -254,32 +254,28 @@ where
 
 #[cfg(test)]
 mod tests {
-    use std::{sync::Arc, time::Duration};
+    use std::sync::Arc;
 
     use assert_matches::assert_matches;
     use async_trait::async_trait;
-    use data_types::{NamespaceId, PartitionId, PartitionKey, ShardId, TableId};
     use metric::{Attributes, Metric};
     use parking_lot::Mutex;
     use wal::Wal;
 
     use crate::{
-        buffer_tree::partition::{PartitionData, SortKeyState},
-        deferred_load::DeferredLoad,
+        buffer_tree::partition::PartitionData,
         dml_sink::mock_sink::MockDmlSink,
         persist::queue::mock::MockPersistQueue,
-        test_util::{assert_dml_writes_eq, make_write_op},
+        test_util::{
+            assert_dml_writes_eq, make_write_op, PartitionDataBuilder, ARBITRARY_NAMESPACE_ID,
+            ARBITRARY_PARTITION_ID, ARBITRARY_PARTITION_KEY, ARBITRARY_TABLE_ID,
+            ARBITRARY_TABLE_NAME,
+        },
         wal::wal_sink::WalSink,
     };
 
     use super::*;
 
-    const PARTITION_ID: PartitionId = PartitionId::new(42);
-    const TABLE_ID: TableId = TableId::new(44);
-    const TABLE_NAME: &str = "bananas";
-    const NAMESPACE_NAME: &str = "platanos";
-    const NAMESPACE_ID: NamespaceId = NamespaceId::new(42);
-
     #[derive(Debug)]
     struct MockIter {
         sink: MockDmlSink,
@@ -307,28 +303,38 @@ mod tests {
 
         // Generate the test ops that will be appended and read back
         let op1 = make_write_op(
-            &PartitionKey::from("p1"),
-            NAMESPACE_ID,
-            TABLE_NAME,
-            TABLE_ID,
+            &ARBITRARY_PARTITION_KEY,
+            ARBITRARY_NAMESPACE_ID,
+            &ARBITRARY_TABLE_NAME,
+            ARBITRARY_TABLE_ID,
             24,
-            r#"bananas,region=Madrid temp=35 4242424242"#,
+            &format!(
+                r#"{},region=Madrid temp=35 4242424242"#,
+                &*ARBITRARY_TABLE_NAME
+            ),
         );
         let op2 = make_write_op(
-            &PartitionKey::from("p1"),
-            NAMESPACE_ID,
-            TABLE_NAME,
-            TABLE_ID,
+            &ARBITRARY_PARTITION_KEY,
+            ARBITRARY_NAMESPACE_ID,
+            &ARBITRARY_TABLE_NAME,
+            ARBITRARY_TABLE_ID,
             25,
-            r#"bananas,region=Asturias temp=25 4242424242"#,
+            &format!(
+                r#"{},region=Asturias temp=25 4242424242"#,
+                &*ARBITRARY_TABLE_NAME
+            ),
         );
         let op3 = make_write_op(
-            &PartitionKey::from("p1"),
-            NAMESPACE_ID,
-            TABLE_NAME,
-            TABLE_ID,
+            &ARBITRARY_PARTITION_KEY,
+            ARBITRARY_NAMESPACE_ID,
+            &ARBITRARY_TABLE_NAME,
+            ARBITRARY_TABLE_ID,
             42,
-            r#"bananas,region=Asturias temp=15 4242424242"#, // Overwrite op2
+            // Overwrite op2
+            &format!(
+                r#"{},region=Asturias temp=15 4242424242"#,
+                &*ARBITRARY_TABLE_NAME
+            ),
         );
 
         // The write portion of this test.
@@ -380,20 +386,7 @@ mod tests {
         // Replay the results into a mock to capture the DmlWrites and returns
         // some dummy partitions when iterated over.
         let mock_sink = MockDmlSink::default().with_apply_return(vec![Ok(()), Ok(()), Ok(())]);
-        let mut partition = PartitionData::new(
-            PARTITION_ID,
-            PartitionKey::from("bananas"),
-            NAMESPACE_ID,
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                NAMESPACE_NAME.into()
-            })),
-            TABLE_ID,
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                TABLE_NAME.into()
-            })),
-            SortKeyState::Provided(None),
-            ShardId::new(1234),
-        );
+        let mut partition = PartitionDataBuilder::new().build();
         // Put at least one write into the buffer so it is a candidate for persistence
         partition
             .buffer_write(
@@ -415,16 +408,23 @@ mod tests {
 
         // Assert the ops were pushed into the DmlSink exactly as generated.
         let ops = mock_iter.sink.get_calls();
-        assert_matches!(&*ops, &[DmlOperation::Write(ref w1),DmlOperation::Write(ref w2),DmlOperation::Write(ref w3)] => {
-            assert_dml_writes_eq(w1.clone(), op1);
-            assert_dml_writes_eq(w2.clone(), op2);
-            assert_dml_writes_eq(w3.clone(), op3);
-        });
+        assert_matches!(
+            &*ops,
+            &[
+                DmlOperation::Write(ref w1),
+                DmlOperation::Write(ref w2),
+                DmlOperation::Write(ref w3)
+            ] => {
+                assert_dml_writes_eq(w1.clone(), op1);
+                assert_dml_writes_eq(w2.clone(), op2);
+                assert_dml_writes_eq(w3.clone(), op3);
+            }
+        );
 
         // Ensure all partitions were persisted
         let calls = persist.calls();
         assert_matches!(&*calls, [p] => {
-            assert_eq!(p.lock().partition_id(), PARTITION_ID);
+            assert_eq!(p.lock().partition_id(), ARBITRARY_PARTITION_ID);
         });
 
         // Ensure there were no partition persist panics.
diff --git a/ingester2/src/persist/handle.rs b/ingester2/src/persist/handle.rs
index 3cca58dc7b..097f7918c4 100644
--- a/ingester2/src/persist/handle.rs
+++ b/ingester2/src/persist/handle.rs
@@ -209,20 +209,23 @@ impl PersistHandle {
         let queue_duration = metrics
             .register_metric_with_options::<DurationHistogram, _>(
                 "ingester_persist_enqueue_duration",
-                "the distribution of duration a persist job spent enqueued, waiting to be processed in seconds",
-                || DurationHistogramOptions::new([
-                    Duration::from_millis(500),
-                    Duration::from_secs(1),
-                    Duration::from_secs(2),
-                    Duration::from_secs(4),
-                    Duration::from_secs(8),
-                    Duration::from_secs(16),
-                    Duration::from_secs(32),
-                    Duration::from_secs(64),
-                    Duration::from_secs(128),
-                    Duration::from_secs(256),
-                    DURATION_MAX,
-                ])
+                "the distribution of duration a persist job spent enqueued, \
+                waiting to be processed in seconds",
+                || {
+                    DurationHistogramOptions::new([
+                        Duration::from_millis(500),
+                        Duration::from_secs(1),
+                        Duration::from_secs(2),
+                        Duration::from_secs(4),
+                        Duration::from_secs(8),
+                        Duration::from_secs(16),
+                        Duration::from_secs(32),
+                        Duration::from_secs(64),
+                        Duration::from_secs(128),
+                        Duration::from_secs(256),
+                        DURATION_MAX,
+                    ])
+                },
             )
             .recorder(&[]);
 
@@ -472,7 +475,7 @@ mod tests {
     use std::{sync::Arc, task::Poll, time::Duration};
 
     use assert_matches::assert_matches;
-    use data_types::{NamespaceId, PartitionId, PartitionKey, ShardId, TableId};
+    use data_types::TRANSITION_SHARD_ID;
     use dml::DmlOperation;
     use futures::Future;
     use iox_catalog::mem::MemCatalog;
@@ -486,11 +489,9 @@ mod tests {
     use super::*;
     use crate::{
         buffer_tree::{
-            namespace::{name_resolver::mock::MockNamespaceNameProvider, NamespaceName},
+            namespace::name_resolver::mock::MockNamespaceNameProvider,
             partition::resolver::mock::MockPartitionProvider,
-            post_write::mock::MockPostWriteObserver,
-            table::{name_resolver::mock::MockTableNameProvider, TableName},
-            BufferTree,
+            post_write::mock::MockPostWriteObserver, BufferTree,
         },
         deferred_load::DeferredLoad,
         dml_sink::DmlSink,
@@ -499,49 +500,29 @@ mod tests {
             completion_observer::{mock::MockCompletionObserver, NopObserver},
             tests::{assert_metric_counter, assert_metric_gauge},
         },
-        test_util::make_write_op,
+        test_util::{
+            make_write_op, PartitionDataBuilder, ARBITRARY_NAMESPACE_ID, ARBITRARY_NAMESPACE_NAME,
+            ARBITRARY_PARTITION_ID, ARBITRARY_PARTITION_KEY, ARBITRARY_TABLE_ID,
+            ARBITRARY_TABLE_NAME, ARBITRARY_TABLE_NAME_PROVIDER,
+        },
     };
 
-    const PARTITION_ID: PartitionId = PartitionId::new(42);
-    const NAMESPACE_ID: NamespaceId = NamespaceId::new(24);
-    const TABLE_ID: TableId = TableId::new(2442);
-    const TABLE_NAME: &str = "banana-report";
-    const NAMESPACE_NAME: &str = "platanos";
-    const TRANSITION_SHARD_ID: ShardId = ShardId::new(84);
-
     lazy_static! {
         static ref EXEC: Arc<Executor> = Arc::new(Executor::new_testing());
-        static ref PARTITION_KEY: PartitionKey = PartitionKey::from("bananas");
-        static ref NAMESPACE_NAME_LOADER: Arc<DeferredLoad<NamespaceName>> =
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                NamespaceName::from(NAMESPACE_NAME)
-            }));
-        static ref TABLE_NAME_LOADER: Arc<DeferredLoad<TableName>> =
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                TableName::from(TABLE_NAME)
-            }));
     }
 
     /// Construct a partition with the above constants, with the given sort key,
     /// and containing a single write.
-    async fn new_partition(
-        partition_id: PartitionId,
-        sort_key: SortKeyState,
-    ) -> Arc<Mutex<PartitionData>> {
+    async fn new_partition(sort_key: SortKeyState) -> Arc<Mutex<PartitionData>> {
         let buffer_tree = BufferTree::new(
-            Arc::new(MockNamespaceNameProvider::new(NAMESPACE_NAME)),
-            Arc::new(MockTableNameProvider::new(TABLE_NAME)),
+            Arc::new(MockNamespaceNameProvider::new(&**ARBITRARY_NAMESPACE_NAME)),
+            Arc::clone(&*ARBITRARY_TABLE_NAME_PROVIDER),
             Arc::new(
-                MockPartitionProvider::default().with_partition(PartitionData::new(
-                    partition_id,
-                    PARTITION_KEY.clone(),
-                    NAMESPACE_ID,
-                    Arc::clone(&NAMESPACE_NAME_LOADER),
-                    TABLE_ID,
-                    Arc::clone(&TABLE_NAME_LOADER),
-                    sort_key,
-                    TRANSITION_SHARD_ID,
-                )),
+                MockPartitionProvider::default().with_partition(
+                    PartitionDataBuilder::new()
+                        .with_sort_key_state(sort_key)
+                        .build(),
+                ),
             ),
             Arc::new(MockPostWriteObserver::default()),
             Default::default(),
@@ -550,12 +531,12 @@ mod tests {
 
         buffer_tree
             .apply(DmlOperation::Write(make_write_op(
-                &PARTITION_KEY,
-                NAMESPACE_ID,
-                TABLE_NAME,
-                TABLE_ID,
+                &ARBITRARY_PARTITION_KEY,
+                ARBITRARY_NAMESPACE_ID,
+                &ARBITRARY_TABLE_NAME,
+                ARBITRARY_TABLE_ID,
                 0,
-                r#"banana-report,good=yes level=1000 4242424242"#,
+                &format!("{},good=yes level=1000 4242424242", &*ARBITRARY_TABLE_NAME),
             )))
             .await
             .expect("failed to write partition test dataa");
@@ -595,7 +576,7 @@ mod tests {
         handle.worker_queues = JumpHash::new([worker1_tx, worker2_tx]);
 
         // Generate a partition with no known sort key.
-        let p = new_partition(PARTITION_ID, SortKeyState::Provided(None)).await;
+        let p = new_partition(SortKeyState::Provided(None)).await;
         let data = p.lock().mark_persisting().unwrap();
 
         // Enqueue it
@@ -616,7 +597,7 @@ mod tests {
                     .expect("message was not found in either worker")
             }
         };
-        assert_eq!(msg.partition_id(), PARTITION_ID);
+        assert_eq!(msg.partition_id(), ARBITRARY_PARTITION_ID);
 
         // Drop the message, and ensure the notification becomes inactive.
         drop(msg);
@@ -626,7 +607,7 @@ mod tests {
         );
 
         // Enqueue another partition for the same ID.
-        let p = new_partition(PARTITION_ID, SortKeyState::Provided(None)).await;
+        let p = new_partition(SortKeyState::Provided(None)).await;
         let data = p.lock().mark_persisting().unwrap();
 
         // Enqueue it
@@ -636,7 +617,7 @@ mod tests {
         let msg = assigned_worker
             .try_recv()
             .expect("message was not found in either worker");
-        assert_eq!(msg.partition_id(), PARTITION_ID);
+        assert_eq!(msg.partition_id(), ARBITRARY_PARTITION_ID);
     }
 
     /// A test that ensures the correct destination of a partition that has no
@@ -671,12 +652,10 @@ mod tests {
         handle.worker_queues = JumpHash::new([worker1_tx, worker2_tx]);
 
         // Generate a partition with a resolved, but empty sort key.
-        let p = new_partition(
-            PARTITION_ID,
-            SortKeyState::Deferred(Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                None
-            }))),
-        )
+        let p = new_partition(SortKeyState::Deferred(Arc::new(DeferredLoad::new(
+            Duration::from_secs(1),
+            async { None },
+        ))))
         .await;
         let (loader, data) = {
             let mut p = p.lock();
@@ -703,7 +682,7 @@ mod tests {
                     .expect("message was not found in either worker")
             }
         };
-        assert_eq!(msg.partition_id(), PARTITION_ID);
+        assert_eq!(msg.partition_id(), ARBITRARY_PARTITION_ID);
 
         // Drop the message, and ensure the notification becomes inactive.
         drop(msg);
@@ -714,7 +693,7 @@ mod tests {
 
         // Enqueue another partition for the same ID and same (resolved)
         // deferred load instance.
-        let p = new_partition(PARTITION_ID, loader).await;
+        let p = new_partition(loader).await;
         let data = p.lock().mark_persisting().unwrap();
 
         // Enqueue it
@@ -724,7 +703,7 @@ mod tests {
         let msg = assigned_worker
             .try_recv()
             .expect("message was not found in either worker");
-        assert_eq!(msg.partition_id(), PARTITION_ID);
+        assert_eq!(msg.partition_id(), ARBITRARY_PARTITION_ID);
     }
 
     /// A test that ensures the correct destination of a partition that has an
@@ -760,12 +739,10 @@ mod tests {
 
         // Generate a partition with a resolved sort key that does not reflect
         // the data within the partition's buffer.
-        let p = new_partition(
-            PARTITION_ID,
-            SortKeyState::Deferred(Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                Some(SortKey::from_columns(["time", "some-other-column"]))
-            }))),
-        )
+        let p = new_partition(SortKeyState::Deferred(Arc::new(DeferredLoad::new(
+            Duration::from_secs(1),
+            async { Some(SortKey::from_columns(["time", "some-other-column"])) },
+        ))))
         .await;
         let (loader, data) = {
             let mut p = p.lock();
@@ -792,7 +769,7 @@ mod tests {
                     .expect("message was not found in either worker")
             }
         };
-        assert_eq!(msg.partition_id(), PARTITION_ID);
+        assert_eq!(msg.partition_id(), ARBITRARY_PARTITION_ID);
 
         // Drop the message, and ensure the notification becomes inactive.
         drop(msg);
@@ -803,7 +780,7 @@ mod tests {
 
         // Enqueue another partition for the same ID and same (resolved)
         // deferred load instance.
-        let p = new_partition(PARTITION_ID, loader).await;
+        let p = new_partition(loader).await;
         let data = p.lock().mark_persisting().unwrap();
 
         // Enqueue it
@@ -813,7 +790,7 @@ mod tests {
         let msg = assigned_worker
             .try_recv()
             .expect("message was not found in either worker");
-        assert_eq!(msg.partition_id(), PARTITION_ID);
+        assert_eq!(msg.partition_id(), ARBITRARY_PARTITION_ID);
     }
 
     /// A test that a partition that does not require a sort key update is
@@ -848,12 +825,10 @@ mod tests {
 
         // Generate a partition with a resolved sort key that does not reflect
         // the data within the partition's buffer.
-        let p = new_partition(
-            PARTITION_ID,
-            SortKeyState::Deferred(Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                Some(SortKey::from_columns(["time", "good"]))
-            }))),
-        )
+        let p = new_partition(SortKeyState::Deferred(Arc::new(DeferredLoad::new(
+            Duration::from_secs(1),
+            async { Some(SortKey::from_columns(["time", "good"])) },
+        ))))
         .await;
         let (loader, data) = {
             let mut p = p.lock();
@@ -873,7 +848,7 @@ mod tests {
         let msg = global_rx
             .try_recv()
             .expect("task should be in global queue");
-        assert_eq!(msg.partition_id(), PARTITION_ID);
+        assert_eq!(msg.partition_id(), ARBITRARY_PARTITION_ID);
 
         // Drop the message, and ensure the notification becomes inactive.
         drop(msg);
@@ -884,7 +859,7 @@ mod tests {
 
         // Enqueue another partition for the same ID and same (resolved)
         // deferred load instance.
-        let p = new_partition(PARTITION_ID, loader).await;
+        let p = new_partition(loader).await;
         let data = p.lock().mark_persisting().unwrap();
 
         // Enqueue it
@@ -894,7 +869,7 @@ mod tests {
         let msg = global_rx
             .try_recv()
             .expect("task should be in global queue");
-        assert_eq!(msg.partition_id(), PARTITION_ID);
+        assert_eq!(msg.partition_id(), ARBITRARY_PARTITION_ID);
     }
 
     /// A test that a ensures tasks waiting to be enqueued (waiting on the
@@ -930,12 +905,10 @@ mod tests {
         handle.worker_queues = JumpHash::new([worker1_tx, worker2_tx]);
 
         // Generate a partition
-        let p = new_partition(
-            PARTITION_ID,
-            SortKeyState::Deferred(Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                Some(SortKey::from_columns(["time", "good"]))
-            }))),
-        )
+        let p = new_partition(SortKeyState::Deferred(Arc::new(DeferredLoad::new(
+            Duration::from_secs(1),
+            async { Some(SortKey::from_columns(["time", "good"])) },
+        ))))
         .await;
         let data = p.lock().mark_persisting().unwrap();
 
@@ -943,12 +916,10 @@ mod tests {
         let _notify1 = handle.enqueue(p, data).await;
 
         // Generate a second partition
-        let p = new_partition(
-            PARTITION_ID,
-            SortKeyState::Deferred(Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                Some(SortKey::from_columns(["time", "good"]))
-            }))),
-        )
+        let p = new_partition(SortKeyState::Deferred(Arc::new(DeferredLoad::new(
+            Duration::from_secs(1),
+            async { Some(SortKey::from_columns(["time", "good"])) },
+        ))))
         .await;
         let data = p.lock().mark_persisting().unwrap();
 
diff --git a/ingester2/src/persist/hot_partitions.rs b/ingester2/src/persist/hot_partitions.rs
index 6cffb96faa..06cc633c72 100644
--- a/ingester2/src/persist/hot_partitions.rs
+++ b/ingester2/src/persist/hot_partitions.rs
@@ -102,52 +102,28 @@ where
 #[cfg(test)]
 mod tests {
     use std::sync::Arc;
-    use std::time::Duration;
 
     use assert_matches::assert_matches;
-    use data_types::{NamespaceId, PartitionId, PartitionKey, SequenceNumber, ShardId, TableId};
-    use lazy_static::lazy_static;
+    use data_types::SequenceNumber;
     use mutable_batch_lp::test_helpers::lp_to_mutable_batch;
     use parking_lot::Mutex;
 
     use crate::{
-        buffer_tree::{
-            namespace::NamespaceName, partition::PartitionData, partition::SortKeyState,
-            table::TableName,
-        },
-        deferred_load::DeferredLoad,
         persist::queue::mock::MockPersistQueue,
+        test_util::{PartitionDataBuilder, ARBITRARY_TABLE_NAME},
     };
 
     use super::*;
 
-    const PARTITION_ID: PartitionId = PartitionId::new(1);
-    const TRANSITION_SHARD_ID: ShardId = ShardId::new(84);
-
-    lazy_static! {
-        static ref PARTITION_KEY: PartitionKey = PartitionKey::from("pohtaytoes");
-        static ref TABLE_NAME: TableName = TableName::from("potatoes");
-        static ref NAMESPACE_NAME: NamespaceName = NamespaceName::from("namespace-potatoes");
-    }
-
     #[tokio::test]
     async fn test_hot_partition_persist() {
-        let mut p = PartitionData::new(
-            PARTITION_ID,
-            PARTITION_KEY.clone(),
-            NamespaceId::new(3),
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                NAMESPACE_NAME.clone()
-            })),
-            TableId::new(4),
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                TABLE_NAME.clone()
-            })),
-            SortKeyState::Provided(None),
-            TRANSITION_SHARD_ID,
-        );
+        let mut p = PartitionDataBuilder::new().build();
 
-        let mb = lp_to_mutable_batch(r#"potatoes,city=Hereford  people=1,crisps="good" 10"#).1;
+        let mb = lp_to_mutable_batch(&format!(
+            r#"{},city=Hereford  people=1,crisps="good" 10"#,
+            &*ARBITRARY_TABLE_NAME
+        ))
+        .1;
         p.buffer_write(mb, SequenceNumber::new(1))
             .expect("write should succeed");
         let max_cost = p.persist_cost_estimate() + 1; // Require additional data to be buffered before enqueuing
@@ -177,7 +153,11 @@ mod tests {
 
         // Write more data to the partition
         let want_query_data = {
-            let mb = lp_to_mutable_batch(r#"potatoes,city=Worcester people=2,crisps="fine" 5"#).1;
+            let mb = lp_to_mutable_batch(&format!(
+                r#"{},city=Worcester people=2,crisps="fine" 5"#,
+                &*ARBITRARY_TABLE_NAME
+            ))
+            .1;
             let mut guard = p.lock();
             guard
                 .buffer_write(mb, SequenceNumber::new(2))
diff --git a/ingester2/src/persist/mod.rs b/ingester2/src/persist/mod.rs
index f4f4cfe4e8..1785abc21c 100644
--- a/ingester2/src/persist/mod.rs
+++ b/ingester2/src/persist/mod.rs
@@ -15,7 +15,7 @@ mod tests {
     use std::{sync::Arc, time::Duration};
 
     use assert_matches::assert_matches;
-    use data_types::{CompactionLevel, ParquetFile, PartitionKey, SequenceNumber, ShardId};
+    use data_types::{CompactionLevel, ParquetFile, SequenceNumber, TRANSITION_SHARD_ID};
     use dml::DmlOperation;
     use futures::TryStreamExt;
     use iox_catalog::{
@@ -36,28 +36,25 @@ mod tests {
 
     use crate::{
         buffer_tree::{
-            namespace::name_resolver::mock::MockNamespaceNameProvider,
             partition::{resolver::CatalogPartitionResolver, PartitionData, SortKeyState},
             post_write::mock::MockPostWriteObserver,
-            table::name_resolver::mock::MockTableNameProvider,
             BufferTree,
         },
         dml_sink::DmlSink,
         ingest_state::IngestState,
         persist::{completion_observer::mock::MockCompletionObserver, queue::PersistQueue},
-        test_util::{make_write_op, populate_catalog},
+        test_util::{
+            make_write_op, populate_catalog, ARBITRARY_NAMESPACE_NAME,
+            ARBITRARY_NAMESPACE_NAME_PROVIDER, ARBITRARY_PARTITION_KEY, ARBITRARY_TABLE_NAME,
+            ARBITRARY_TABLE_NAME_PROVIDER,
+        },
         TRANSITION_SHARD_INDEX,
     };
 
     use super::handle::PersistHandle;
 
-    const TABLE_NAME: &str = "bananas";
-    const NAMESPACE_NAME: &str = "platanos";
-    const TRANSITION_SHARD_ID: ShardId = ShardId::new(84);
-
     lazy_static! {
         static ref EXEC: Arc<Executor> = Arc::new(Executor::new_testing());
-        static ref PARTITION_KEY: PartitionKey = PartitionKey::from("bananas");
     }
 
     /// Generate a [`PartitionData`] containing one write, and populate the
@@ -68,15 +65,15 @@ mod tests {
         let (_shard_id, namespace_id, table_id) = populate_catalog(
             &*catalog,
             TRANSITION_SHARD_INDEX,
-            NAMESPACE_NAME,
-            TABLE_NAME,
+            &ARBITRARY_NAMESPACE_NAME,
+            &ARBITRARY_TABLE_NAME,
         )
         .await;
 
         // Init the buffer tree
         let buf = BufferTree::new(
-            Arc::new(MockNamespaceNameProvider::new(NAMESPACE_NAME)),
-            Arc::new(MockTableNameProvider::new(TABLE_NAME)),
+            Arc::clone(&*ARBITRARY_NAMESPACE_NAME_PROVIDER),
+            Arc::clone(&*ARBITRARY_TABLE_NAME_PROVIDER),
             Arc::new(CatalogPartitionResolver::new(Arc::clone(&catalog))),
             Arc::new(MockPostWriteObserver::default()),
             Arc::new(metric::Registry::default()),
@@ -84,12 +81,15 @@ mod tests {
         );
 
         let write = make_write_op(
-            &PARTITION_KEY,
+            &ARBITRARY_PARTITION_KEY,
             namespace_id,
-            TABLE_NAME,
+            &ARBITRARY_TABLE_NAME,
             table_id,
             0,
-            r#"bananas,region=Asturias temp=35 4242424242"#,
+            &format!(
+                r#"{},region=Asturias temp=35 4242424242"#,
+                &*ARBITRARY_TABLE_NAME
+            ),
         );
 
         let mut repos = catalog
@@ -103,7 +103,9 @@ mod tests {
 
         // Insert the schema elements into the catalog
         validate_or_insert_schema(
-            write.tables().map(|(_id, data)| (TABLE_NAME, data)),
+            write
+                .tables()
+                .map(|(_id, data)| (&***ARBITRARY_TABLE_NAME, data)),
             &schema,
             &mut *repos,
         )
diff --git a/ingester2/src/test_util.rs b/ingester2/src/test_util.rs
index 16a080743f..7907cbba17 100644
--- a/ingester2/src/test_util.rs
+++ b/ingester2/src/test_util.rs
@@ -1,11 +1,141 @@
-use std::collections::BTreeMap;
+use std::{collections::BTreeMap, sync::Arc, time::Duration};
 
-use data_types::{NamespaceId, PartitionKey, SequenceNumber, ShardId, ShardIndex, TableId};
+use data_types::{
+    NamespaceId, Partition, PartitionId, PartitionKey, SequenceNumber, ShardId, ShardIndex,
+    TableId, TRANSITION_SHARD_ID,
+};
 use dml::{DmlMeta, DmlWrite};
 use iox_catalog::interface::Catalog;
+use lazy_static::lazy_static;
 use mutable_batch_lp::lines_to_batches;
 use schema::Projection;
 
+use crate::{
+    buffer_tree::{
+        namespace::{
+            name_resolver::{mock::MockNamespaceNameProvider, NamespaceNameProvider},
+            NamespaceName,
+        },
+        partition::{PartitionData, SortKeyState},
+        table::{
+            name_resolver::{mock::MockTableNameProvider, TableNameProvider},
+            TableName,
+        },
+    },
+    deferred_load::DeferredLoad,
+};
+
+pub(crate) const ARBITRARY_PARTITION_ID: PartitionId = PartitionId::new(1);
+pub(crate) const ARBITRARY_NAMESPACE_ID: NamespaceId = NamespaceId::new(3);
+pub(crate) const ARBITRARY_TABLE_ID: TableId = TableId::new(4);
+pub(crate) const ARBITRARY_PARTITION_KEY_STR: &str = "platanos";
+
+lazy_static! {
+    pub(crate) static ref ARBITRARY_PARTITION_KEY: PartitionKey =
+        PartitionKey::from(ARBITRARY_PARTITION_KEY_STR);
+    pub(crate) static ref ARBITRARY_NAMESPACE_NAME: NamespaceName =
+        NamespaceName::from("namespace-bananas");
+    pub(crate) static ref DEFER_NAMESPACE_NAME_1_SEC: Arc<DeferredLoad<NamespaceName>> =
+        Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
+            ARBITRARY_NAMESPACE_NAME.clone()
+        }));
+    pub(crate) static ref DEFER_NAMESPACE_NAME_1_MS: Arc<DeferredLoad<NamespaceName>> =
+        Arc::new(DeferredLoad::new(Duration::from_millis(1), async {
+            ARBITRARY_NAMESPACE_NAME.clone()
+        }));
+    pub(crate) static ref ARBITRARY_NAMESPACE_NAME_PROVIDER: Arc<dyn NamespaceNameProvider> =
+        Arc::new(MockNamespaceNameProvider::new(&**ARBITRARY_NAMESPACE_NAME));
+    pub(crate) static ref ARBITRARY_TABLE_NAME: TableName = TableName::from("bananas");
+    pub(crate) static ref DEFER_TABLE_NAME_1_SEC: Arc<DeferredLoad<TableName>> =
+        Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
+            ARBITRARY_TABLE_NAME.clone()
+        }));
+    pub(crate) static ref DEFER_TABLE_NAME_1_MS: Arc<DeferredLoad<TableName>> =
+        Arc::new(DeferredLoad::new(Duration::from_millis(1), async {
+            ARBITRARY_TABLE_NAME.clone()
+        }));
+    pub(crate) static ref ARBITRARY_TABLE_NAME_PROVIDER: Arc<dyn TableNameProvider> =
+        Arc::new(MockTableNameProvider::new(&**ARBITRARY_TABLE_NAME));
+}
+
+/// Build a [`PartitionData`] with mostly arbitrary-yet-valid values for tests.
+#[derive(Debug, Clone, Default)]
+pub(crate) struct PartitionDataBuilder {
+    partition_id: Option<PartitionId>,
+    partition_key: Option<PartitionKey>,
+    namespace_id: Option<NamespaceId>,
+    table_id: Option<TableId>,
+    table_name: Option<Arc<DeferredLoad<TableName>>>,
+    sort_key: Option<SortKeyState>,
+}
+
+impl PartitionDataBuilder {
+    pub(crate) fn new() -> Self {
+        Self::default()
+    }
+
+    pub(crate) fn with_partition_id(mut self, partition_id: PartitionId) -> Self {
+        self.partition_id = Some(partition_id);
+        self
+    }
+
+    pub(crate) fn with_partition_key(mut self, partition_key: PartitionKey) -> Self {
+        self.partition_key = Some(partition_key);
+        self
+    }
+
+    pub(crate) fn with_namespace_id(mut self, namespace_id: NamespaceId) -> Self {
+        self.namespace_id = Some(namespace_id);
+        self
+    }
+
+    pub(crate) fn with_table_id(mut self, table_id: TableId) -> Self {
+        self.table_id = Some(table_id);
+        self
+    }
+
+    pub(crate) fn with_table_name(mut self, table_name: Arc<DeferredLoad<TableName>>) -> Self {
+        self.table_name = Some(table_name);
+        self
+    }
+
+    pub(crate) fn with_sort_key_state(mut self, sort_key_state: SortKeyState) -> Self {
+        self.sort_key = Some(sort_key_state);
+        self
+    }
+
+    /// Generate a valid [`PartitionData`] for use in tests where the exact values (or at least
+    /// some of them) don't particularly matter.
+    pub(crate) fn build(self) -> PartitionData {
+        PartitionData::new(
+            self.partition_id.unwrap_or(ARBITRARY_PARTITION_ID),
+            self.partition_key
+                .unwrap_or_else(|| ARBITRARY_PARTITION_KEY.clone()),
+            self.namespace_id.unwrap_or(ARBITRARY_NAMESPACE_ID),
+            Arc::clone(&*DEFER_NAMESPACE_NAME_1_SEC),
+            self.table_id.unwrap_or(ARBITRARY_TABLE_ID),
+            self.table_name
+                .unwrap_or_else(|| Arc::clone(&*DEFER_TABLE_NAME_1_SEC)),
+            self.sort_key.unwrap_or(SortKeyState::Provided(None)),
+            TRANSITION_SHARD_ID,
+        )
+    }
+}
+
+/// Generate a valid [`Partition`] for use in the tests where the exact values (or at least some of
+/// them) don't particularly matter.
+pub(crate) fn arbitrary_partition() -> Partition {
+    Partition {
+        id: ARBITRARY_PARTITION_ID,
+        shard_id: TRANSITION_SHARD_ID,
+        table_id: ARBITRARY_TABLE_ID,
+        partition_key: ARBITRARY_PARTITION_KEY.clone(),
+        sort_key: Default::default(),
+        persisted_sequence_number: Default::default(),
+        new_file_at: Default::default(),
+    }
+}
+
 /// Generate a [`RecordBatch`] & [`Schema`] with the specified columns and
 /// values:
 ///
diff --git a/ingester2/src/wal/rotate_task.rs b/ingester2/src/wal/rotate_task.rs
index 6ac94a4857..89a116b71c 100644
--- a/ingester2/src/wal/rotate_task.rs
+++ b/ingester2/src/wal/rotate_task.rs
@@ -143,53 +143,25 @@ mod tests {
 
     use assert_matches::assert_matches;
     use async_trait::async_trait;
-    use data_types::{NamespaceId, PartitionId, PartitionKey, SequenceNumber, ShardId, TableId};
-    use lazy_static::lazy_static;
+    use data_types::SequenceNumber;
     use mutable_batch_lp::test_helpers::lp_to_mutable_batch;
     use parking_lot::Mutex;
     use tempfile::tempdir;
     use test_helpers::timeout::FutureTimeout;
     use tokio::sync::oneshot;
 
+    use super::*;
     use crate::{
-        buffer_tree::{
-            namespace::NamespaceName,
-            partition::PartitionData,
-            partition::{persisting::PersistingData, SortKeyState},
-            table::TableName,
-        },
-        deferred_load::DeferredLoad,
+        buffer_tree::{partition::persisting::PersistingData, partition::PartitionData},
         persist::queue::mock::MockPersistQueue,
+        test_util::{PartitionDataBuilder, ARBITRARY_PARTITION_ID},
     };
 
-    use super::*;
-
-    const PARTITION_ID: PartitionId = PartitionId::new(1);
-    const TRANSITION_SHARD_ID: ShardId = ShardId::new(84);
     const TICK_INTERVAL: Duration = Duration::from_millis(10);
 
-    lazy_static! {
-        static ref PARTITION_KEY: PartitionKey = PartitionKey::from("platanos");
-        static ref TABLE_NAME: TableName = TableName::from("bananas");
-        static ref NAMESPACE_NAME: NamespaceName = NamespaceName::from("namespace-bananas");
-    }
-
     #[tokio::test]
     async fn test_persist() {
-        let mut p = PartitionData::new(
-            PARTITION_ID,
-            PARTITION_KEY.clone(),
-            NamespaceId::new(3),
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                NAMESPACE_NAME.clone()
-            })),
-            TableId::new(4),
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                TABLE_NAME.clone()
-            })),
-            SortKeyState::Provided(None),
-            TRANSITION_SHARD_ID,
-        );
+        let mut p = PartitionDataBuilder::new().build();
 
         // Perform a single write to populate the partition.
         let mb = lp_to_mutable_batch(r#"bananas,city=London people=2,pigeons="millions" 10"#).1;
@@ -271,7 +243,7 @@ mod tests {
 
         assert_matches!(persist.calls().as_slice(), [got] => {
             let guard = got.lock();
-            assert_eq!(guard.partition_id(), PARTITION_ID);
+            assert_eq!(guard.partition_id(), ARBITRARY_PARTITION_ID);
         })
     }
 
@@ -305,20 +277,7 @@ mod tests {
 
     #[tokio::test]
     async fn test_persist_ticks_when_blocked() {
-        let mut p = PartitionData::new(
-            PARTITION_ID,
-            PARTITION_KEY.clone(),
-            NamespaceId::new(3),
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                NAMESPACE_NAME.clone()
-            })),
-            TableId::new(4),
-            Arc::new(DeferredLoad::new(Duration::from_secs(1), async {
-                TABLE_NAME.clone()
-            })),
-            SortKeyState::Provided(None),
-            TRANSITION_SHARD_ID,
-        );
+        let mut p = PartitionDataBuilder::new().build();
 
         // Perform a single write to populate the partition.
         let mb = lp_to_mutable_batch(r#"bananas,city=London people=2,pigeons="millions" 10"#).1;

From b74a81e79217f1cbbb0b29e2db486a066f3aba3b Mon Sep 17 00:00:00 2001
From: Nga Tran <nga-tran@live.com>
Date: Tue, 25 Apr 2023 06:16:21 -0400
Subject: [PATCH 15/25] test: add tests on month and year date_bin (#7648)

* test: add tests on month and year date_bin

* fix: add IOX_COMPARE: uuid to get deterministics name for output parquet_file in the explain

---------

Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com>
---
 influxdb_iox/tests/query_tests2/cases.rs      |  12 ++
 .../tests/query_tests2/cases/in/date_bin.sql  | 104 +++++++++++++++
 .../cases/in/date_bin.sql.expected            | 119 ++++++++++++++++++
 3 files changed, 235 insertions(+)
 create mode 100644 influxdb_iox/tests/query_tests2/cases/in/date_bin.sql
 create mode 100644 influxdb_iox/tests/query_tests2/cases/in/date_bin.sql.expected

diff --git a/influxdb_iox/tests/query_tests2/cases.rs b/influxdb_iox/tests/query_tests2/cases.rs
index 1b6a7cdd4b..46345f4cb9 100644
--- a/influxdb_iox/tests/query_tests2/cases.rs
+++ b/influxdb_iox/tests/query_tests2/cases.rs
@@ -49,6 +49,18 @@ async fn basic() {
     .await;
 }
 
+#[tokio::test]
+async fn date_bin() {
+    test_helpers::maybe_start_logging();
+
+    TestCase {
+        input: "cases/in/date_bin.sql",
+        chunk_stage: ChunkStage::All,
+    }
+    .run()
+    .await;
+}
+
 #[tokio::test]
 async fn dedup_and_predicates_parquet() {
     test_helpers::maybe_start_logging();
diff --git a/influxdb_iox/tests/query_tests2/cases/in/date_bin.sql b/influxdb_iox/tests/query_tests2/cases/in/date_bin.sql
new file mode 100644
index 0000000000..6e15ecde38
--- /dev/null
+++ b/influxdb_iox/tests/query_tests2/cases/in/date_bin.sql
@@ -0,0 +1,104 @@
+-- Date_bin tests
+-- IOX_SETUP: OneMeasurementTwoSeries
+
+-- CONSTANT DATA or ARRAY DATA
+
+-- 1 month
+select date_bin(INTERVAL '1 month', column1)
+ from (values
+   (timestamp '2022-01-01 00:00:00'),
+   (timestamp '2022-01-01 01:00:00'),
+   (timestamp '2022-01-02 00:00:00'),
+   (timestamp '2022-02-02 00:00:00'),
+   (timestamp '2022-02-15 00:00:00'),
+   (timestamp '2022-03-31 00:00:00')
+ ) as sq;
+
+-- 1 year
+ select date_bin('1 year', column1)
+ from (values
+   (timestamp '2022-01-01 00:00:00'),
+   (timestamp '2023-01-01 01:00:00'),
+   (timestamp '2022-01-02 00:00:00'),
+   (timestamp '2022-02-02 00:00:00'),
+   (timestamp '2022-02-15 00:00:00'),
+   (timestamp '2022-03-31 00:00:00')
+ ) as sq;
+
+-- origin is last date of the month 1970-12-31T00:15:00Z and not at midnight
+ select date_bin('1 month', column1, '1970-12-31T00:15:00Z')
+ from (values
+   (timestamp '2022-01-01 00:00:00'),
+   (timestamp '2022-01-01 01:00:00'),
+   (timestamp '2022-01-02 00:00:00'),
+   (timestamp '2022-02-02 00:00:00'),
+   (timestamp '2022-02-15 00:00:00'),
+   (timestamp '2022-03-31 00:00:00')
+ ) as sq;
+
+ -- five months interval on constant
+ SELECT DATE_BIN('5 month', '2022-01-01T00:00:00Z');
+
+ -- origin is May 31 (last date of the month) to produce bin on Feb 28
+ SELECT DATE_BIN('3 month', '2022-04-01T00:00:00Z', '2021-05-31T00:04:00Z');
+
+-- origin is on Feb 29 and interval is one month. The bins will be:
+--  # '2000-02-29T00:00:00'
+--  # '2000-01-29T00:00:00'
+--  # '1999-12-29T00:00:00'
+--  # ....
+--  # Reason: Even though 29 (or 28 for non-leap year) is the last date of Feb but it
+--  # is not last date of other month. Months' chrono consider a month before or after that
+--  # will land on the same 29th date.
+select date_bin('1 month', timestamp '2000-01-31T00:00:00', timestamp '2000-02-29T00:00:00');
+
+-- similar for the origin March 29
+select date_bin('1 month', timestamp '2000-01-31T00:00:00', timestamp '2000-03-29T00:00:00');
+
+-- 3 year 1 months = 37 months
+SELECT DATE_BIN('3 years 1 months', '2022-09-01 00:00:00Z');
+
+-- DATA FORM TABLE
+
+-- Input data (by region, time)
+SELECT *
+FROM cpu
+ORDER BY REGION, TIME;
+
+-- Input data (by time)
+SELECT *
+FROM cpu
+ORDER BY TIME;
+
+-- 1 month
+SELECT
+  date_bin('1 month', time) as month,
+  count(cpu.user)
+from cpu
+where time between timestamp '2000-05-05T12:00:00Z' and timestamp '2000-05-05T12:59:00Z'
+group by month;
+
+-- 1 month with origin
+SELECT
+  date_bin('1 month', time, '1970-12-31T00:15:00Z') as month,
+  count(cpu.user)
+from cpu
+where time between timestamp '2000-05-05T12:00:00Z' and timestamp '2000-05-05T12:59:00Z'
+group by month;
+
+-- 3 months with origin on the last date of the month
+select 
+    date_bin('2 month', time, timestamp '2000-02-29T00:00:00') as month,
+    count(cpu.user)
+from cpu
+where time between timestamp '2000-05-05T12:00:00Z' and timestamp '2000-05-05T12:59:00Z'
+group by month;
+
+-- EXPLAIN
+-- IOX_COMPARE: uuid
+EXPLAIN SELECT
+  date_bin('1 month', time, '1970-12-31T00:15:00Z') as month,
+  count(cpu.user)
+from cpu
+where time between timestamp '2000-05-05T12:00:00Z' and timestamp '2000-05-05T12:59:00Z'
+group by month;
\ No newline at end of file
diff --git a/influxdb_iox/tests/query_tests2/cases/in/date_bin.sql.expected b/influxdb_iox/tests/query_tests2/cases/in/date_bin.sql.expected
new file mode 100644
index 0000000000..000b433ed3
--- /dev/null
+++ b/influxdb_iox/tests/query_tests2/cases/in/date_bin.sql.expected
@@ -0,0 +1,119 @@
+-- Test Setup: OneMeasurementTwoSeries
+-- SQL: select date_bin(INTERVAL '1 month', column1) from (values (timestamp '2022-01-01 00:00:00'), (timestamp '2022-01-01 01:00:00'), (timestamp '2022-01-02 00:00:00'), (timestamp '2022-02-02 00:00:00'), (timestamp '2022-02-15 00:00:00'), (timestamp '2022-03-31 00:00:00') ) as sq;
++---------------------------------------------------------------------------+
+| datebin(IntervalMonthDayNano("79228162514264337593543950336"),sq.column1) |
++---------------------------------------------------------------------------+
+| 2022-01-01T00:00:00Z                                                      |
+| 2022-01-01T00:00:00Z                                                      |
+| 2022-01-01T00:00:00Z                                                      |
+| 2022-02-01T00:00:00Z                                                      |
+| 2022-02-01T00:00:00Z                                                      |
+| 2022-03-01T00:00:00Z                                                      |
++---------------------------------------------------------------------------+
+-- SQL: select date_bin('1 year', column1) from (values (timestamp '2022-01-01 00:00:00'), (timestamp '2023-01-01 01:00:00'), (timestamp '2022-01-02 00:00:00'), (timestamp '2022-02-02 00:00:00'), (timestamp '2022-02-15 00:00:00'), (timestamp '2022-03-31 00:00:00') ) as sq;
++------------------------------------+
+| datebin(Utf8("1 year"),sq.column1) |
++------------------------------------+
+| 2022-01-01T00:00:00Z               |
+| 2023-01-01T00:00:00Z               |
+| 2022-01-01T00:00:00Z               |
+| 2022-01-01T00:00:00Z               |
+| 2022-01-01T00:00:00Z               |
+| 2022-01-01T00:00:00Z               |
++------------------------------------+
+-- SQL: select date_bin('1 month', column1, '1970-12-31T00:15:00Z') from (values (timestamp '2022-01-01 00:00:00'), (timestamp '2022-01-01 01:00:00'), (timestamp '2022-01-02 00:00:00'), (timestamp '2022-02-02 00:00:00'), (timestamp '2022-02-15 00:00:00'), (timestamp '2022-03-31 00:00:00') ) as sq;
++------------------------------------------------------------------+
+| datebin(Utf8("1 month"),sq.column1,Utf8("1970-12-31T00:15:00Z")) |
++------------------------------------------------------------------+
+| 2021-12-31T00:15:00Z                                             |
+| 2021-12-31T00:15:00Z                                             |
+| 2021-12-31T00:15:00Z                                             |
+| 2022-01-31T00:15:00Z                                             |
+| 2022-01-31T00:15:00Z                                             |
+| 2022-02-28T00:15:00Z                                             |
++------------------------------------------------------------------+
+-- SQL: SELECT DATE_BIN('5 month', '2022-01-01T00:00:00Z');
++-------------------------------------------------------+
+| datebin(Utf8("5 month"),Utf8("2022-01-01T00:00:00Z")) |
++-------------------------------------------------------+
+| 2021-09-01T00:00:00Z                                  |
++-------------------------------------------------------+
+-- SQL: SELECT DATE_BIN('3 month', '2022-04-01T00:00:00Z', '2021-05-31T00:04:00Z');
++------------------------------------------------------------------------------------+
+| datebin(Utf8("3 month"),Utf8("2022-04-01T00:00:00Z"),Utf8("2021-05-31T00:04:00Z")) |
++------------------------------------------------------------------------------------+
+| 2022-02-28T00:04:00Z                                                               |
++------------------------------------------------------------------------------------+
+-- SQL: select date_bin('1 month', timestamp '2000-01-31T00:00:00', timestamp '2000-02-29T00:00:00');
++----------------------------------------------------------------------------------+
+| datebin(Utf8("1 month"),Utf8("2000-01-31T00:00:00"),Utf8("2000-02-29T00:00:00")) |
++----------------------------------------------------------------------------------+
+| 2000-01-29T00:00:00Z                                                             |
++----------------------------------------------------------------------------------+
+-- SQL: select date_bin('1 month', timestamp '2000-01-31T00:00:00', timestamp '2000-03-29T00:00:00');
++----------------------------------------------------------------------------------+
+| datebin(Utf8("1 month"),Utf8("2000-01-31T00:00:00"),Utf8("2000-03-29T00:00:00")) |
++----------------------------------------------------------------------------------+
+| 2000-01-29T00:00:00Z                                                             |
++----------------------------------------------------------------------------------+
+-- SQL: SELECT DATE_BIN('3 years 1 months', '2022-09-01 00:00:00Z');
++----------------------------------------------------------------+
+| datebin(Utf8("3 years 1 months"),Utf8("2022-09-01 00:00:00Z")) |
++----------------------------------------------------------------+
+| 2022-06-01T00:00:00Z                                           |
++----------------------------------------------------------------+
+-- SQL: SELECT * FROM cpu ORDER BY REGION, TIME;
++------+--------+----------------------+------+
+| idle | region | time                 | user |
++------+--------+----------------------+------+
+| 70.0 | a      | 2000-05-05T12:20:00Z | 23.2 |
+|      | a      | 2000-05-05T12:40:00Z | 21.0 |
+|      | b      | 2000-05-05T12:31:00Z | 25.2 |
+| 60.0 | b      | 2000-05-05T12:39:00Z | 28.9 |
++------+--------+----------------------+------+
+-- SQL: SELECT * FROM cpu ORDER BY TIME;
++------+--------+----------------------+------+
+| idle | region | time                 | user |
++------+--------+----------------------+------+
+| 70.0 | a      | 2000-05-05T12:20:00Z | 23.2 |
+|      | b      | 2000-05-05T12:31:00Z | 25.2 |
+| 60.0 | b      | 2000-05-05T12:39:00Z | 28.9 |
+|      | a      | 2000-05-05T12:40:00Z | 21.0 |
++------+--------+----------------------+------+
+-- SQL: SELECT date_bin('1 month', time) as month, count(cpu.user) from cpu where time between timestamp '2000-05-05T12:00:00Z' and timestamp '2000-05-05T12:59:00Z' group by month;
++----------------------+-----------------+
+| month                | COUNT(cpu.user) |
++----------------------+-----------------+
+| 2000-05-01T00:00:00Z | 4               |
++----------------------+-----------------+
+-- SQL: SELECT date_bin('1 month', time, '1970-12-31T00:15:00Z') as month, count(cpu.user) from cpu where time between timestamp '2000-05-05T12:00:00Z' and timestamp '2000-05-05T12:59:00Z' group by month;
++----------------------+-----------------+
+| month                | COUNT(cpu.user) |
++----------------------+-----------------+
+| 2000-04-30T00:15:00Z | 4               |
++----------------------+-----------------+
+-- SQL: select date_bin('2 month', time, timestamp '2000-02-29T00:00:00') as month, count(cpu.user) from cpu where time between timestamp '2000-05-05T12:00:00Z' and timestamp '2000-05-05T12:59:00Z' group by month;
++----------------------+-----------------+
+| month                | COUNT(cpu.user) |
++----------------------+-----------------+
+| 2000-04-29T00:00:00Z | 4               |
++----------------------+-----------------+
+-- SQL: EXPLAIN SELECT date_bin('1 month', time, '1970-12-31T00:15:00Z') as month, count(cpu.user) from cpu where time between timestamp '2000-05-05T12:00:00Z' and timestamp '2000-05-05T12:59:00Z' group by month;
+-- Results After Normalizing UUIDs
+----------
+| plan_type    | plan    |
+----------
+| logical_plan    | Projection: datebin(Utf8("1 month"),cpu.time,Utf8("1970-12-31T00:15:00Z")) AS month, COUNT(cpu.user)    |
+|    |   Aggregate: groupBy=[[datebin(IntervalMonthDayNano("79228162514264337593543950336"), cpu.time, TimestampNanosecond(31450500000000000, None)) AS datebin(Utf8("1 month"),cpu.time,Utf8("1970-12-31T00:15:00Z"))]], aggr=[[COUNT(cpu.user)]]    |
+|    |     TableScan: cpu projection=[time, user], full_filters=[cpu.time >= TimestampNanosecond(957528000000000000, None), cpu.time <= TimestampNanosecond(957531540000000000, None)]    |
+| physical_plan    | ProjectionExec: expr=[datebin(Utf8("1 month"),cpu.time,Utf8("1970-12-31T00:15:00Z"))@0 as month, COUNT(cpu.user)@1 as COUNT(cpu.user)]    |
+|    |   AggregateExec: mode=FinalPartitioned, gby=[datebin(Utf8("1 month"),cpu.time,Utf8("1970-12-31T00:15:00Z"))@0 as datebin(Utf8("1 month"),cpu.time,Utf8("1970-12-31T00:15:00Z"))], aggr=[COUNT(cpu.user)]    |
+|    |     CoalesceBatchesExec: target_batch_size=8192    |
+|    |       RepartitionExec: partitioning=Hash([Column { name: "datebin(Utf8(\"1 month\"),cpu.time,Utf8(\"1970-12-31T00:15:00Z\"))", index: 0 }], 4), input_partitions=4    |
+|    |         AggregateExec: mode=Partial, gby=[datebin(79228162514264337593543950336, time@0, 31450500000000000) as datebin(Utf8("1 month"),cpu.time,Utf8("1970-12-31T00:15:00Z"))], aggr=[COUNT(cpu.user)]    |
+|    |           RepartitionExec: partitioning=RoundRobinBatch(4), input_partitions=1    |
+|    |             CoalesceBatchesExec: target_batch_size=8192    |
+|    |               FilterExec: time@0 >= 957528000000000000 AND time@0 <= 957531540000000000    |
+|    |                 ParquetExec: limit=None, partitions={1 group: [[1/1/1/00000000-0000-0000-0000-000000000000.parquet]]}, predicate=time@2 >= 957528000000000000 AND time@2 <= 957531540000000000, pruning_predicate=time_max@0 >= 957528000000000000 AND time_min@1 <= 957531540000000000, output_ordering=[time@0 ASC], projection=[time, user]    |
+|    |    |
+----------
\ No newline at end of file

From 159a254ada7b80b096a34d73d09aeb0349e9bbaf Mon Sep 17 00:00:00 2001
From: Andrew Lamb <alamb@influxdata.com>
Date: Tue, 25 Apr 2023 10:46:03 -0400
Subject: [PATCH 16/25] fix: Lower message about unsupported predicates on the
 ingester to debug level (#7655)

---
 ingester2/src/server/grpc/query.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ingester2/src/server/grpc/query.rs b/ingester2/src/server/grpc/query.rs
index dc4d5f33c9..4ea7f2551f 100644
--- a/ingester2/src/server/grpc/query.rs
+++ b/ingester2/src/server/grpc/query.rs
@@ -174,7 +174,7 @@ where
 
         // Predicate pushdown is part of the API, but not implemented.
         if let Some(p) = request.predicate {
-            warn!(predicate=?p, "ignoring query predicate (unsupported)");
+            debug!(predicate=?p, "ignoring query predicate (unsupported)");
         }
 
         let response = match self

From 06fbab264980da93097d45ad8738b8611e96de96 Mon Sep 17 00:00:00 2001
From: Phil Bracikowski <13472206+philjb@users.noreply.github.com>
Date: Tue, 25 Apr 2023 08:57:23 -0700
Subject: [PATCH 17/25] fix(garbage collector): limit catalog update for files
 to delete (#7623)

* fix(garbage collector): limit catalog update for files to delete

Impose a 1000 LIMIT on flag_for_delete_by_retention so the garbage
collector's load on the catalog is limited. 1000 is used as the fixed
limit in another catalog DML.

* follow up to requests in #7562

* chore: add test for limit on update

---------

Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com>
---
 iox_catalog/src/interface.rs | 42 +++++++++++++++++++++++++++++++++++-
 iox_catalog/src/lib.rs       |  1 -
 iox_catalog/src/mem.rs       |  8 ++++++-
 iox_catalog/src/postgres.rs  | 29 ++++++++++++++-----------
 iox_catalog/src/sqlite.rs    | 29 ++++++++++++++-----------
 5 files changed, 80 insertions(+), 29 deletions(-)

diff --git a/iox_catalog/src/interface.rs b/iox_catalog/src/interface.rs
index fd583f8d31..ee56af5afe 100644
--- a/iox_catalog/src/interface.rs
+++ b/iox_catalog/src/interface.rs
@@ -16,6 +16,10 @@ use std::{
 };
 use uuid::Uuid;
 
+/// Maximum number of files deleted by [`ParquetFileRepo::delete_old_ids_only`] and
+/// [`ParquetFileRepo::flag_for_delete_by_retention`] at a time.
+pub const MAX_PARQUET_FILES_SELECTED_ONCE: i64 = 1_000;
+
 /// An error wrapper detailing the reason for a compare-and-swap failure.
 #[derive(Debug)]
 pub enum CasFailure<T> {
@@ -1835,6 +1839,7 @@ pub(crate) mod test_helpers {
             .expect("delete namespace should succeed");
     }
 
+    /// tests many interactions with the catalog and parquet files. See the individual conditions herein
     async fn test_parquet_file(catalog: Arc<dyn Catalog>) {
         let mut repos = catalog.repositories().await;
         let topic = repos.topics().create_or_get("foo").await.unwrap();
@@ -2074,7 +2079,7 @@ pub(crate) mod test_helpers {
             min_time: Timestamp::new(50),
             max_time: Timestamp::new(60),
             max_sequence_number: SequenceNumber::new(11),
-            ..f1_params
+            ..f1_params.clone()
         };
         let f2 = repos
             .parquet_files()
@@ -2219,6 +2224,41 @@ pub(crate) mod test_helpers {
             .await
             .unwrap();
         assert!(ids.is_empty());
+
+        // test that flag_for_delete_by_retention respects UPDATE LIMIT
+        // create limit + the meaning of life parquet files that are all older than the retention (>1hr)
+        const LIMIT: usize = 1000;
+        const MOL: usize = 42;
+        for _ in 0..LIMIT + MOL {
+            let params = ParquetFileParams {
+                object_store_id: Uuid::new_v4(),
+                max_time: Timestamp::new(
+                    // a bit over an hour ago
+                    (catalog.time_provider().now() - Duration::from_secs(60 * 65))
+                        .timestamp_nanos(),
+                ),
+                ..f1_params.clone()
+            };
+            repos.parquet_files().create(params.clone()).await.unwrap();
+        }
+        let ids = repos
+            .parquet_files()
+            .flag_for_delete_by_retention()
+            .await
+            .unwrap();
+        assert_eq!(ids.len(), LIMIT);
+        let ids = repos
+            .parquet_files()
+            .flag_for_delete_by_retention()
+            .await
+            .unwrap();
+        assert_eq!(ids.len(), MOL); // second call took remainder
+        let ids = repos
+            .parquet_files()
+            .flag_for_delete_by_retention()
+            .await
+            .unwrap();
+        assert_eq!(ids.len(), 0); // none left
     }
 
     async fn test_parquet_file_delete_broken(catalog: Arc<dyn Catalog>) {
diff --git a/iox_catalog/src/lib.rs b/iox_catalog/src/lib.rs
index 6564cc6f73..a1caf178d8 100644
--- a/iox_catalog/src/lib.rs
+++ b/iox_catalog/src/lib.rs
@@ -37,7 +37,6 @@ pub const DEFAULT_MAX_COLUMNS_PER_TABLE: i32 = 200;
 /// Default retention period for data in the catalog.
 pub const DEFAULT_RETENTION_PERIOD: Option<i64> = None;
 
-/// A string value representing an infinite retention policy.
 pub mod interface;
 pub mod mem;
 pub mod metrics;
diff --git a/iox_catalog/src/mem.rs b/iox_catalog/src/mem.rs
index b9412a4903..00f97e4ffa 100644
--- a/iox_catalog/src/mem.rs
+++ b/iox_catalog/src/mem.rs
@@ -6,6 +6,7 @@ use crate::{
         sealed::TransactionFinalize, CasFailure, Catalog, ColumnRepo, ColumnTypeMismatchSnafu,
         Error, NamespaceRepo, ParquetFileRepo, PartitionRepo, QueryPoolRepo, RepoCollection,
         Result, ShardRepo, SoftDeletedRows, TableRepo, TopicMetadataRepo, Transaction,
+        MAX_PARQUET_FILES_SELECTED_ONCE,
     },
     metrics::MetricDecorator,
     DEFAULT_MAX_COLUMNS_PER_TABLE, DEFAULT_MAX_TABLES, SHARED_TOPIC_ID, SHARED_TOPIC_NAME,
@@ -1034,6 +1035,7 @@ impl ParquetFileRepo for MemTxn {
                         })
                     })
             })
+            .take(MAX_PARQUET_FILES_SELECTED_ONCE as usize)
             .collect())
     }
 
@@ -1090,7 +1092,11 @@ impl ParquetFileRepo for MemTxn {
 
         stage.parquet_files = keep;
 
-        let delete = delete.into_iter().map(|f| f.id).collect();
+        let delete = delete
+            .into_iter()
+            .take(MAX_PARQUET_FILES_SELECTED_ONCE as usize)
+            .map(|f| f.id)
+            .collect();
         Ok(delete)
     }
 
diff --git a/iox_catalog/src/postgres.rs b/iox_catalog/src/postgres.rs
index 82bf8d062c..43f81f9ccd 100644
--- a/iox_catalog/src/postgres.rs
+++ b/iox_catalog/src/postgres.rs
@@ -5,7 +5,7 @@ use crate::{
         self, sealed::TransactionFinalize, CasFailure, Catalog, ColumnRepo,
         ColumnTypeMismatchSnafu, Error, NamespaceRepo, ParquetFileRepo, PartitionRepo,
         QueryPoolRepo, RepoCollection, Result, ShardRepo, SoftDeletedRows, TableRepo,
-        TopicMetadataRepo, Transaction,
+        TopicMetadataRepo, Transaction, MAX_PARQUET_FILES_SELECTED_ONCE,
     },
     metrics::MetricDecorator,
     DEFAULT_MAX_COLUMNS_PER_TABLE, DEFAULT_MAX_TABLES, SHARED_TOPIC_ID, SHARED_TOPIC_NAME,
@@ -31,9 +31,6 @@ use std::{collections::HashMap, fmt::Display, str::FromStr, sync::Arc, time::Dur
 
 static MIGRATOR: Migrator = sqlx::migrate!();
 
-/// Maximum number of files deleted by [`ParquetFileRepo::delete_old_ids_only].
-const MAX_PARQUET_FILES_DELETED_ONCE: i64 = 1_000;
-
 /// Postgres connection options.
 #[derive(Debug, Clone)]
 pub struct PostgresConnectionOptions {
@@ -1537,17 +1534,23 @@ RETURNING *;
         // TODO - include check of table retention period once implemented
         let flagged = sqlx::query(
             r#"
-                UPDATE parquet_file
-                SET to_delete = $1
-                FROM namespace
-                WHERE namespace.retention_period_ns IS NOT NULL
-                AND parquet_file.to_delete IS NULL
-                AND parquet_file.max_time < $1 - namespace.retention_period_ns
-                AND namespace.id = parquet_file.namespace_id
-                RETURNING parquet_file.id;
+WITH parquet_file_ids as (
+    SELECT parquet_file.id
+    FROM namespace, parquet_file
+    WHERE namespace.retention_period_ns IS NOT NULL
+    AND parquet_file.to_delete IS NULL
+    AND parquet_file.max_time < $1 - namespace.retention_period_ns
+    AND namespace.id = parquet_file.namespace_id
+    LIMIT $2
+)
+UPDATE parquet_file
+SET to_delete = $1
+WHERE id IN (SELECT id FROM parquet_file_ids)
+RETURNING id;
             "#,
         )
         .bind(flagged_at) // $1
+        .bind(MAX_PARQUET_FILES_SELECTED_ONCE) // $2
         .fetch_all(&mut self.inner)
         .await
         .map_err(|e| Error::SqlxError { source: e })?;
@@ -1634,7 +1637,7 @@ RETURNING id;
              "#,
         )
         .bind(older_than) // $1
-        .bind(MAX_PARQUET_FILES_DELETED_ONCE) // $2
+        .bind(MAX_PARQUET_FILES_SELECTED_ONCE) // $2
         .fetch_all(&mut self.inner)
         .await
         .map_err(|e| Error::SqlxError { source: e })?;
diff --git a/iox_catalog/src/sqlite.rs b/iox_catalog/src/sqlite.rs
index d6d2e13025..5abcea3686 100644
--- a/iox_catalog/src/sqlite.rs
+++ b/iox_catalog/src/sqlite.rs
@@ -5,7 +5,7 @@ use crate::{
         self, sealed::TransactionFinalize, CasFailure, Catalog, ColumnRepo,
         ColumnTypeMismatchSnafu, Error, NamespaceRepo, ParquetFileRepo, PartitionRepo,
         QueryPoolRepo, RepoCollection, Result, ShardRepo, SoftDeletedRows, TableRepo,
-        TopicMetadataRepo, Transaction,
+        TopicMetadataRepo, Transaction, MAX_PARQUET_FILES_SELECTED_ONCE,
     },
     metrics::MetricDecorator,
     DEFAULT_MAX_COLUMNS_PER_TABLE, DEFAULT_MAX_TABLES, SHARED_TOPIC_ID, SHARED_TOPIC_NAME,
@@ -36,9 +36,6 @@ use std::sync::Arc;
 
 static MIGRATOR: Migrator = sqlx::migrate!("sqlite/migrations");
 
-/// Maximum number of files deleted by [`ParquetFileRepo::delete_old_ids_only].
-const MAX_PARQUET_FILES_DELETED_ONCE: i64 = 1_000;
-
 /// SQLite connection options.
 #[derive(Debug, Clone)]
 pub struct SqliteConnectionOptions {
@@ -1405,17 +1402,23 @@ RETURNING *;
         // TODO - include check of table retention period once implemented
         let flagged = sqlx::query(
             r#"
-                UPDATE parquet_file
-                SET to_delete = $1
-                FROM namespace
-                WHERE namespace.retention_period_ns IS NOT NULL
-                AND parquet_file.to_delete IS NULL
-                AND parquet_file.max_time < $1 - namespace.retention_period_ns
-                AND namespace.id = parquet_file.namespace_id
-                RETURNING parquet_file.id;
+WITH parquet_file_ids as (
+    SELECT parquet_file.id
+    FROM namespace, parquet_file
+    WHERE namespace.retention_period_ns IS NOT NULL
+    AND parquet_file.to_delete IS NULL
+    AND parquet_file.max_time < $1 - namespace.retention_period_ns
+    AND namespace.id = parquet_file.namespace_id
+    LIMIT $2
+)
+UPDATE parquet_file
+SET to_delete = $1
+WHERE id IN (SELECT id FROM parquet_file_ids)
+RETURNING id;
             "#,
         )
         .bind(flagged_at) // $1
+        .bind(MAX_PARQUET_FILES_SELECTED_ONCE) // $2
         .fetch_all(self.inner.get_mut())
         .await
         .map_err(|e| Error::SqlxError { source: e })?;
@@ -1511,7 +1514,7 @@ RETURNING id;
              "#,
         )
         .bind(older_than) // $1
-        .bind(MAX_PARQUET_FILES_DELETED_ONCE) // $2
+        .bind(MAX_PARQUET_FILES_SELECTED_ONCE) // $2
         .fetch_all(self.inner.get_mut())
         .await
         .map_err(|e| Error::SqlxError { source: e })?;

From bc324c4ac2a942c30646ba3550131a7ab8e78c6e Mon Sep 17 00:00:00 2001
From: Andrew Lamb <alamb@influxdata.com>
Date: Tue, 25 Apr 2023 14:29:23 -0400
Subject: [PATCH 18/25] chore: Update DataFusion and arrow/parquet/arrow-flight
 to `38.0.0` (#7662)

* chore: Update DataFusion and arrow/parquet/arrow-flight to `38.0.0`

* chore: Run cargo hakari tasks

---------

Co-authored-by: CircleCI[bot] <circleci@influxdata.com>
---
 Cargo.lock                | 82 +++++++++++++++++++--------------------
 Cargo.toml                | 10 ++---
 workspace-hack/Cargo.toml | 18 ++++-----
 3 files changed, 55 insertions(+), 55 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 4be019bcf4..de0e25d4af 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -149,9 +149,9 @@ checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6"
 
 [[package]]
 name = "arrow"
-version = "37.0.0"
+version = "38.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1aea9fcb25bbb70f7f922f95b99ca29c1013dab47f6df61a6f24861842dd7f2e"
+checksum = "c107a57b5913d852da9d5a40e280e4695f2258b5b87733c13b770c63a7117287"
 dependencies = [
  "ahash 0.8.3",
  "arrow-arith",
@@ -171,9 +171,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-arith"
-version = "37.0.0"
+version = "38.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d967b42f7b12c91fd78acd396b20c2973b184c8866846674abbb00c963e93ab"
+checksum = "ace6aa3d5617c5d03041a05e01c6819428a8ddf49dd0b055df9b40fef9d96094"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -186,9 +186,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-array"
-version = "37.0.0"
+version = "38.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3190f208ee7aa0f3596fa0098d42911dec5e123ca88c002a08b24877ad14c71e"
+checksum = "104a04520692cc674e6afd7682f213ca41f9b13ff1873f63a5a2857a590b87b3"
 dependencies = [
  "ahash 0.8.3",
  "arrow-buffer",
@@ -203,9 +203,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-buffer"
-version = "37.0.0"
+version = "38.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d33c733c5b6c44a0fc526f29c09546e04eb56772a7a21e48e602f368be381f6"
+checksum = "72c875bcb9530ec403998fb0b2dc6d180a7c64563ca4bc22b90eafb84b113143"
 dependencies = [
  "half 2.2.1",
  "num",
@@ -213,9 +213,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-cast"
-version = "37.0.0"
+version = "38.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "abd349520b6a1ed4924ae2afc9d23330a3044319e4ec3d5b124c09e4d440ae87"
+checksum = "d6d6e18281636c8fc0b93be59834da6bf9a72bb70fd0c98ddfdaf124da466c28"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -230,9 +230,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-csv"
-version = "37.0.0"
+version = "38.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c80af3c3e290a2a7e1cc518f1471dff331878cb4af9a5b088bf030b89debf649"
+checksum = "3197dab0963a236ff8e7c82e2272535745955ac1321eb740c29f2f88b353f54e"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -249,9 +249,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-data"
-version = "37.0.0"
+version = "38.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1c8361947aaa96d331da9df3f7a08bdd8ab805a449994c97f5c4d24c4b7e2cf"
+checksum = "eb68113d6ecdbe8bba48b2c4042c151bf9e1c61244e45072a50250a6fc59bafe"
 dependencies = [
  "arrow-buffer",
  "arrow-schema",
@@ -261,9 +261,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-flight"
-version = "37.0.0"
+version = "38.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bd1fc687f3e4ffe91ccb7f2ffb06143ff97029448d427a9641006242bcbd0c24"
+checksum = "52045fe4f34dc1529dfb6bb19542cd76d093e4d2f00ac58822755ec59fc14160"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -281,9 +281,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-ipc"
-version = "37.0.0"
+version = "38.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9a46ee000b9fbd1e8db6e8b26acb8c760838512b39d8c9f9d73892cb55351d50"
+checksum = "eab4bbf2dd3078facb5ce0a9641316a64f42bfd8cf357e6775c8a5e6708e3a8d"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -295,9 +295,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-json"
-version = "37.0.0"
+version = "38.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4bf2366607be867ced681ad7f272371a5cf1fc2941328eef7b4fee14565166fb"
+checksum = "48c5b650d23746a494665d914a7fa3d21d939153cff9d53bdebe39bffa88f263"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -315,9 +315,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-ord"
-version = "37.0.0"
+version = "38.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "304069901c867200e21ec868ae7521165875470ef2f1f6d58f979a443d63997e"
+checksum = "68c6fce28e5011e30acc7466b5efcb8ed0197c396240bd2b10e167f275a3c208"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -330,9 +330,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-row"
-version = "37.0.0"
+version = "38.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0d57fe8ceef3392fdd493269d8a2d589de17bafce151aacbffbddac7a57f441a"
+checksum = "f20a421f19799d8b93eb8edde5217e910fa1e2d6ceb3c529f000e57b6db144c0"
 dependencies = [
  "ahash 0.8.3",
  "arrow-array",
@@ -345,15 +345,15 @@ dependencies = [
 
 [[package]]
 name = "arrow-schema"
-version = "37.0.0"
+version = "38.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a16b88a93ac8350f0200b1cd336a1f887315925b8dd7aa145a37b8bdbd8497a4"
+checksum = "bc85923d8d6662cc66ac6602c7d1876872e671002d60993dfdf492a6badeae92"
 
 [[package]]
 name = "arrow-select"
-version = "37.0.0"
+version = "38.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "98e8a4d6ca37d5212439b24caad4d80743fcbb706706200dd174bb98e68fe9d8"
+checksum = "f6ab6613ce65b61d85a3410241744e84e48fbab0fe06e1251b4429d21b3470fd"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -364,9 +364,9 @@ dependencies = [
 
 [[package]]
 name = "arrow-string"
-version = "37.0.0"
+version = "38.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cbb594efa397eb6a546f42b1f8df3d242ea84dbfda5232e06035dc2b2e2c8459"
+checksum = "f3008641239e884aefba66d8b8532da6af40d14296349fcc85935de4ba67b89e"
 dependencies = [
  "arrow-array",
  "arrow-buffer",
@@ -1441,7 +1441,7 @@ dependencies = [
 [[package]]
 name = "datafusion"
 version = "23.0.0"
-source = "git+https://github.com/apache/arrow-datafusion.git?rev=181e5ccf2816ccaa05d8aaef0b375d4b7bbceece#181e5ccf2816ccaa05d8aaef0b375d4b7bbceece"
+source = "git+https://github.com/apache/arrow-datafusion.git?rev=451e81eafe5e404de50f6ede1bebf25ed90f5eb0#451e81eafe5e404de50f6ede1bebf25ed90f5eb0"
 dependencies = [
  "ahash 0.8.3",
  "arrow",
@@ -1490,7 +1490,7 @@ dependencies = [
 [[package]]
 name = "datafusion-common"
 version = "23.0.0"
-source = "git+https://github.com/apache/arrow-datafusion.git?rev=181e5ccf2816ccaa05d8aaef0b375d4b7bbceece#181e5ccf2816ccaa05d8aaef0b375d4b7bbceece"
+source = "git+https://github.com/apache/arrow-datafusion.git?rev=451e81eafe5e404de50f6ede1bebf25ed90f5eb0#451e81eafe5e404de50f6ede1bebf25ed90f5eb0"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -1504,7 +1504,7 @@ dependencies = [
 [[package]]
 name = "datafusion-execution"
 version = "23.0.0"
-source = "git+https://github.com/apache/arrow-datafusion.git?rev=181e5ccf2816ccaa05d8aaef0b375d4b7bbceece#181e5ccf2816ccaa05d8aaef0b375d4b7bbceece"
+source = "git+https://github.com/apache/arrow-datafusion.git?rev=451e81eafe5e404de50f6ede1bebf25ed90f5eb0#451e81eafe5e404de50f6ede1bebf25ed90f5eb0"
 dependencies = [
  "dashmap",
  "datafusion-common",
@@ -1521,7 +1521,7 @@ dependencies = [
 [[package]]
 name = "datafusion-expr"
 version = "23.0.0"
-source = "git+https://github.com/apache/arrow-datafusion.git?rev=181e5ccf2816ccaa05d8aaef0b375d4b7bbceece#181e5ccf2816ccaa05d8aaef0b375d4b7bbceece"
+source = "git+https://github.com/apache/arrow-datafusion.git?rev=451e81eafe5e404de50f6ede1bebf25ed90f5eb0#451e81eafe5e404de50f6ede1bebf25ed90f5eb0"
 dependencies = [
  "ahash 0.8.3",
  "arrow",
@@ -1532,7 +1532,7 @@ dependencies = [
 [[package]]
 name = "datafusion-optimizer"
 version = "23.0.0"
-source = "git+https://github.com/apache/arrow-datafusion.git?rev=181e5ccf2816ccaa05d8aaef0b375d4b7bbceece#181e5ccf2816ccaa05d8aaef0b375d4b7bbceece"
+source = "git+https://github.com/apache/arrow-datafusion.git?rev=451e81eafe5e404de50f6ede1bebf25ed90f5eb0#451e81eafe5e404de50f6ede1bebf25ed90f5eb0"
 dependencies = [
  "arrow",
  "async-trait",
@@ -1549,7 +1549,7 @@ dependencies = [
 [[package]]
 name = "datafusion-physical-expr"
 version = "23.0.0"
-source = "git+https://github.com/apache/arrow-datafusion.git?rev=181e5ccf2816ccaa05d8aaef0b375d4b7bbceece#181e5ccf2816ccaa05d8aaef0b375d4b7bbceece"
+source = "git+https://github.com/apache/arrow-datafusion.git?rev=451e81eafe5e404de50f6ede1bebf25ed90f5eb0#451e81eafe5e404de50f6ede1bebf25ed90f5eb0"
 dependencies = [
  "ahash 0.8.3",
  "arrow",
@@ -1581,7 +1581,7 @@ dependencies = [
 [[package]]
 name = "datafusion-proto"
 version = "23.0.0"
-source = "git+https://github.com/apache/arrow-datafusion.git?rev=181e5ccf2816ccaa05d8aaef0b375d4b7bbceece#181e5ccf2816ccaa05d8aaef0b375d4b7bbceece"
+source = "git+https://github.com/apache/arrow-datafusion.git?rev=451e81eafe5e404de50f6ede1bebf25ed90f5eb0#451e81eafe5e404de50f6ede1bebf25ed90f5eb0"
 dependencies = [
  "arrow",
  "chrono",
@@ -1595,7 +1595,7 @@ dependencies = [
 [[package]]
 name = "datafusion-row"
 version = "23.0.0"
-source = "git+https://github.com/apache/arrow-datafusion.git?rev=181e5ccf2816ccaa05d8aaef0b375d4b7bbceece#181e5ccf2816ccaa05d8aaef0b375d4b7bbceece"
+source = "git+https://github.com/apache/arrow-datafusion.git?rev=451e81eafe5e404de50f6ede1bebf25ed90f5eb0#451e81eafe5e404de50f6ede1bebf25ed90f5eb0"
 dependencies = [
  "arrow",
  "datafusion-common",
@@ -1606,7 +1606,7 @@ dependencies = [
 [[package]]
 name = "datafusion-sql"
 version = "23.0.0"
-source = "git+https://github.com/apache/arrow-datafusion.git?rev=181e5ccf2816ccaa05d8aaef0b375d4b7bbceece#181e5ccf2816ccaa05d8aaef0b375d4b7bbceece"
+source = "git+https://github.com/apache/arrow-datafusion.git?rev=451e81eafe5e404de50f6ede1bebf25ed90f5eb0#451e81eafe5e404de50f6ede1bebf25ed90f5eb0"
 dependencies = [
  "arrow",
  "arrow-schema",
@@ -3930,9 +3930,9 @@ dependencies = [
 
 [[package]]
 name = "parquet"
-version = "37.0.0"
+version = "38.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b5022d98333271f4ca3e87bab760498e61726bf5a6ca919123c80517e20ded29"
+checksum = "4cbd51311f8d9ff3d2697b1522b18a588782e097d313a1a278b0faf2ccf2d3f6"
 dependencies = [
  "ahash 0.8.3",
  "arrow-array",
diff --git a/Cargo.toml b/Cargo.toml
index 539df59e59..0b1c54db2a 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -113,13 +113,13 @@ edition = "2021"
 license = "MIT OR Apache-2.0"
 
 [workspace.dependencies]
-arrow = { version = "37.0.0" }
-arrow-flight = { version = "37.0.0" }
+arrow = { version = "38.0.0" }
+arrow-flight = { version = "38.0.0" }
 chrono-english = { git = "https://github.com/stevedonovan/chrono-english.git", rev = "def5941ebee24b55e1174eb18ab33d91603f907a" }
-datafusion = { git = "https://github.com/apache/arrow-datafusion.git", rev="181e5ccf2816ccaa05d8aaef0b375d4b7bbceece", default-features = false }
-datafusion-proto = { git = "https://github.com/apache/arrow-datafusion.git", rev="181e5ccf2816ccaa05d8aaef0b375d4b7bbceece" }
+datafusion = { git = "https://github.com/apache/arrow-datafusion.git", rev="451e81eafe5e404de50f6ede1bebf25ed90f5eb0", default-features = false }
+datafusion-proto = { git = "https://github.com/apache/arrow-datafusion.git", rev="451e81eafe5e404de50f6ede1bebf25ed90f5eb0" }
 hashbrown = { version = "0.13.2" }
-parquet = { version = "37.0.0" }
+parquet = { version = "38.0.0" }
 tonic = { version = "0.9.2", features = ["tls", "tls-webpki-roots"] }
 tonic-build = { version = "0.9.2" }
 tonic-health = { version = "0.9.2" }
diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml
index 9998eeca98..f092badafe 100644
--- a/workspace-hack/Cargo.toml
+++ b/workspace-hack/Cargo.toml
@@ -17,11 +17,11 @@ license.workspace = true
 ### BEGIN HAKARI SECTION
 [dependencies]
 ahash = { version = "0.8", default-features = false, features = ["runtime-rng"] }
-arrow = { version = "37", features = ["dyn_cmp_dict", "prettyprint"] }
-arrow-array = { version = "37", default-features = false, features = ["chrono-tz"] }
-arrow-flight = { version = "37", features = ["flight-sql-experimental"] }
-arrow-ord = { version = "37", default-features = false, features = ["dyn_cmp_dict"] }
-arrow-string = { version = "37", default-features = false, features = ["dyn_cmp_dict"] }
+arrow = { version = "38", features = ["dyn_cmp_dict", "prettyprint"] }
+arrow-array = { version = "38", default-features = false, features = ["chrono-tz"] }
+arrow-flight = { version = "38", features = ["flight-sql-experimental"] }
+arrow-ord = { version = "38", default-features = false, features = ["dyn_cmp_dict"] }
+arrow-string = { version = "38", default-features = false, features = ["dyn_cmp_dict"] }
 base64-594e8ee84c453af0 = { package = "base64", version = "0.13" }
 base64-647d43efb71741da = { package = "base64", version = "0.21" }
 bitflags = { version = "1" }
@@ -30,9 +30,9 @@ bytes = { version = "1" }
 chrono = { version = "0.4", default-features = false, features = ["alloc", "clock", "serde"] }
 crossbeam-utils = { version = "0.8" }
 crypto-common = { version = "0.1", default-features = false, features = ["std"] }
-datafusion = { git = "https://github.com/apache/arrow-datafusion.git", rev = "181e5ccf2816ccaa05d8aaef0b375d4b7bbceece" }
-datafusion-optimizer = { git = "https://github.com/apache/arrow-datafusion.git", rev = "181e5ccf2816ccaa05d8aaef0b375d4b7bbceece", default-features = false, features = ["crypto_expressions", "regex_expressions", "unicode_expressions"] }
-datafusion-physical-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "181e5ccf2816ccaa05d8aaef0b375d4b7bbceece", default-features = false, features = ["crypto_expressions", "regex_expressions", "unicode_expressions"] }
+datafusion = { git = "https://github.com/apache/arrow-datafusion.git", rev = "451e81eafe5e404de50f6ede1bebf25ed90f5eb0" }
+datafusion-optimizer = { git = "https://github.com/apache/arrow-datafusion.git", rev = "451e81eafe5e404de50f6ede1bebf25ed90f5eb0", default-features = false, features = ["crypto_expressions", "regex_expressions", "unicode_expressions"] }
+datafusion-physical-expr = { git = "https://github.com/apache/arrow-datafusion.git", rev = "451e81eafe5e404de50f6ede1bebf25ed90f5eb0", default-features = false, features = ["crypto_expressions", "regex_expressions", "unicode_expressions"] }
 digest = { version = "0.10", features = ["mac", "std"] }
 either = { version = "1" }
 fixedbitset = { version = "0.4" }
@@ -61,7 +61,7 @@ num-traits = { version = "0.2", features = ["i128", "libm"] }
 object_store = { version = "0.5", default-features = false, features = ["aws", "azure", "gcp"] }
 once_cell = { version = "1", features = ["parking_lot"] }
 parking_lot = { version = "0.12", features = ["arc_lock"] }
-parquet = { version = "37", features = ["async", "experimental"] }
+parquet = { version = "38", features = ["async", "experimental"] }
 petgraph = { version = "0.6" }
 phf_shared = { version = "0.11" }
 predicates = { version = "3" }

From 2341306c267c4a89a3b408d5b507dc733571f106 Mon Sep 17 00:00:00 2001
From: Andrew Lamb <alamb@influxdata.com>
Date: Tue, 25 Apr 2023 16:16:47 -0400
Subject: [PATCH 19/25] refactor: Use upstream copy of FlightDataEncoder
 (#7656)

---
 service_grpc_flight/src/lib.rs | 102 ++-------------------------------
 1 file changed, 5 insertions(+), 97 deletions(-)

diff --git a/service_grpc_flight/src/lib.rs b/service_grpc_flight/src/lib.rs
index ab39c73716..d7163a1793 100644
--- a/service_grpc_flight/src/lib.rs
+++ b/service_grpc_flight/src/lib.rs
@@ -3,20 +3,15 @@
 
 mod request;
 
-use arrow::{
-    datatypes::SchemaRef, error::ArrowError, ipc::writer::IpcWriteOptions,
-    record_batch::RecordBatch,
-};
+use arrow::error::ArrowError;
 use arrow_flight::{
     encode::{FlightDataEncoder, FlightDataEncoderBuilder},
     flight_descriptor::DescriptorType,
     flight_service_server::{FlightService as Flight, FlightServiceServer as FlightServer},
     Action, ActionType, Criteria, Empty, FlightData, FlightDescriptor, FlightEndpoint, FlightInfo,
-    HandshakeRequest, HandshakeResponse, PutResult, SchemaAsIpc, SchemaResult, Ticket,
+    HandshakeRequest, HandshakeResponse, PutResult, SchemaResult, Ticket,
 };
-use arrow_util::flight::prepare_schema_for_flight;
 use authz::Authorizer;
-use bytes::Bytes;
 use data_types::NamespaceNameError;
 use datafusion::{error::DataFusionError, physical_plan::ExecutionPlan};
 use flightsql::FlightSQLCommand;
@@ -799,7 +794,7 @@ fn flightsql_permissions(namespace_name: &str, cmd: &FlightSQLCommand) -> Vec<au
 /// Wrapper over a FlightDataEncodeStream that adds IOx specfic
 /// metadata and records completion
 struct GetStream {
-    inner: IOxFlightDataEncoder,
+    inner: FlightDataEncoder,
     #[allow(dead_code)]
     permit: InstrumentedAsyncOwnedSemaphorePermit,
     query_completed_token: QueryCompletedToken,
@@ -830,7 +825,8 @@ impl GetStream {
             });
 
         // setup inner stream
-        let inner = IOxFlightDataEncoderBuilder::new(schema)
+        let inner = FlightDataEncoderBuilder::new()
+            .with_schema(schema)
             .with_metadata(app_metadata.encode_to_vec().into())
             .build(query_results);
 
@@ -843,94 +839,6 @@ impl GetStream {
     }
 }
 
-/// workaround for <https://github.com/apache/arrow-rs/issues/3591>
-///
-/// data encoder stream that always sends a Schema message even if the
-/// underlying stream is empty
-struct IOxFlightDataEncoder {
-    inner: FlightDataEncoder,
-    // The schema of the inner stream. Set to None when a schema
-    // message has been sent.
-    schema: Option<SchemaRef>,
-    done: bool,
-}
-
-impl IOxFlightDataEncoder {
-    fn new(inner: FlightDataEncoder, schema: SchemaRef) -> Self {
-        Self {
-            inner,
-            schema: Some(schema),
-            done: false,
-        }
-    }
-}
-
-#[derive(Debug)]
-struct IOxFlightDataEncoderBuilder {
-    inner: FlightDataEncoderBuilder,
-    schema: SchemaRef,
-}
-
-impl IOxFlightDataEncoderBuilder {
-    fn new(schema: SchemaRef) -> Self {
-        Self {
-            inner: FlightDataEncoderBuilder::new().with_schema(Arc::clone(&schema)),
-            schema: prepare_schema_for_flight(schema),
-        }
-    }
-
-    pub fn with_metadata(mut self, app_metadata: Bytes) -> Self {
-        self.inner = self.inner.with_metadata(app_metadata);
-        self
-    }
-
-    pub fn build<S>(self, input: S) -> IOxFlightDataEncoder
-    where
-        S: Stream<Item = arrow_flight::error::Result<RecordBatch>> + Send + 'static,
-    {
-        let Self { inner, schema } = self;
-
-        IOxFlightDataEncoder::new(inner.build(input), schema)
-    }
-}
-
-impl Stream for IOxFlightDataEncoder {
-    type Item = arrow_flight::error::Result<FlightData>;
-
-    fn poll_next(
-        mut self: Pin<&mut Self>,
-        cx: &mut std::task::Context<'_>,
-    ) -> Poll<Option<Self::Item>> {
-        loop {
-            if self.done {
-                return Poll::Ready(None);
-            }
-
-            let res = ready!(self.inner.poll_next_unpin(cx));
-            match res {
-                None => {
-                    self.done = true;
-                    // return a schema message if we haven't sent any data
-                    if let Some(schema) = self.schema.take() {
-                        let options = IpcWriteOptions::default();
-                        let data: FlightData = SchemaAsIpc::new(schema.as_ref(), &options).into();
-                        return Poll::Ready(Some(Ok(data)));
-                    }
-                }
-                Some(Ok(data)) => {
-                    // If any data is returned from the underlying stream no need to resend schema
-                    self.schema = None;
-                    return Poll::Ready(Some(Ok(data)));
-                }
-                Some(Err(e)) => {
-                    self.done = true;
-                    return Poll::Ready(Some(Err(e)));
-                }
-            }
-        }
-    }
-}
-
 impl Stream for GetStream {
     type Item = Result<FlightData, tonic::Status>;
 

From eca86a0f06a16d9b1e4eb1f59915de4c0c80af8a Mon Sep 17 00:00:00 2001
From: Chunchun Ye <14298407+appletreeisyellow@users.noreply.github.com>
Date: Tue, 25 Apr 2023 16:06:27 -0500
Subject: [PATCH 20/25] feat(flightsql): Support `CommandGetXdbcTypeInfo`
 metadata endpoint with an empty RecordBatch (#7576)

* feat: WIP support CommandGetXdbcTypeInfo metadata endpoint with tests

* chore: update test case and add jdbc test

* chore: uncomment jdbc getCoumns test

* chore: lint

---------

Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com>
---
 flightsql/src/cmd.rs                          | 17 ++++++-
 flightsql/src/planner.rs                      | 48 ++++++++++++++++++-
 .../tests/end_to_end_cases/flightsql.rs       | 42 ++++++++++++++++
 influxdb_iox/tests/jdbc_client/Main.java      | 13 +++--
 influxdb_iox_client/src/client/flightsql.rs   | 23 ++++++++-
 service_grpc_flight/src/lib.rs                |  1 +
 6 files changed, 135 insertions(+), 9 deletions(-)

diff --git a/flightsql/src/cmd.rs b/flightsql/src/cmd.rs
index feb1e082f1..7e8df8d999 100644
--- a/flightsql/src/cmd.rs
+++ b/flightsql/src/cmd.rs
@@ -6,7 +6,7 @@ use arrow_flight::sql::{
     ActionClosePreparedStatementRequest, ActionCreatePreparedStatementRequest, Any,
     CommandGetCatalogs, CommandGetCrossReference, CommandGetDbSchemas, CommandGetExportedKeys,
     CommandGetImportedKeys, CommandGetPrimaryKeys, CommandGetSqlInfo, CommandGetTableTypes,
-    CommandGetTables, CommandPreparedStatementQuery, CommandStatementQuery,
+    CommandGetTables, CommandGetXdbcTypeInfo, CommandPreparedStatementQuery, CommandStatementQuery,
 };
 use bytes::Bytes;
 use prost::Message;
@@ -94,7 +94,10 @@ pub enum FlightSQLCommand {
     CommandGetPrimaryKeys(CommandGetPrimaryKeys),
     /// Get a list of the available tables
     CommandGetTables(CommandGetTables),
-    /// Get a list of the available table tyypes
+    /// Get information about data types supported.
+    /// See [`CommandGetXdbcTypeInfo`] for details.
+    CommandGetXdbcTypeInfo(CommandGetXdbcTypeInfo),
+    /// Get a list of the available table types
     CommandGetTableTypes(CommandGetTableTypes),
     /// Create a prepared statement
     ActionCreatePreparedStatementRequest(ActionCreatePreparedStatementRequest),
@@ -224,6 +227,13 @@ impl Display for FlightSQLCommand {
             Self::CommandGetTableTypes(CommandGetTableTypes {}) => {
                 write!(f, "CommandGetTableTypes")
             }
+            Self::CommandGetXdbcTypeInfo(CommandGetXdbcTypeInfo { data_type }) => {
+                write!(
+                    f,
+                    "CommandGetXdbcTypeInfo(data_type={})",
+                    data_type.as_ref().copied().unwrap_or(0),
+                )
+            }
             Self::ActionCreatePreparedStatementRequest(ActionCreatePreparedStatementRequest {
                 query,
             }) => {
@@ -269,6 +279,8 @@ impl FlightSQLCommand {
             Ok(Self::CommandGetTables(decode_cmd))
         } else if let Some(decoded_cmd) = Any::unpack::<CommandGetTableTypes>(&msg)? {
             Ok(Self::CommandGetTableTypes(decoded_cmd))
+        } else if let Some(decoded_cmd) = Any::unpack::<CommandGetXdbcTypeInfo>(&msg)? {
+            Ok(Self::CommandGetXdbcTypeInfo(decoded_cmd))
         } else if let Some(decoded_cmd) = Any::unpack::<ActionCreatePreparedStatementRequest>(&msg)?
         {
             Ok(Self::ActionCreatePreparedStatementRequest(decoded_cmd))
@@ -308,6 +320,7 @@ impl FlightSQLCommand {
             FlightSQLCommand::CommandGetPrimaryKeys(cmd) => Any::pack(&cmd),
             FlightSQLCommand::CommandGetTables(cmd) => Any::pack(&cmd),
             FlightSQLCommand::CommandGetTableTypes(cmd) => Any::pack(&cmd),
+            FlightSQLCommand::CommandGetXdbcTypeInfo(cmd) => Any::pack(&cmd),
             FlightSQLCommand::ActionCreatePreparedStatementRequest(cmd) => Any::pack(&cmd),
             FlightSQLCommand::ActionClosePreparedStatementRequest(handle) => {
                 let prepared_statement_handle = handle.encode();
diff --git a/flightsql/src/planner.rs b/flightsql/src/planner.rs
index c43988f024..3af41511e9 100644
--- a/flightsql/src/planner.rs
+++ b/flightsql/src/planner.rs
@@ -13,7 +13,7 @@ use arrow_flight::{
         ActionCreatePreparedStatementRequest, ActionCreatePreparedStatementResult, Any,
         CommandGetCatalogs, CommandGetCrossReference, CommandGetDbSchemas, CommandGetExportedKeys,
         CommandGetImportedKeys, CommandGetPrimaryKeys, CommandGetSqlInfo, CommandGetTableTypes,
-        CommandGetTables, CommandStatementQuery,
+        CommandGetTables, CommandGetXdbcTypeInfo, CommandStatementQuery,
     },
     IpcMessage, SchemaAsIpc,
 };
@@ -86,6 +86,9 @@ impl FlightSQLPlanner {
             FlightSQLCommand::CommandGetTableTypes(CommandGetTableTypes { .. }) => {
                 encode_schema(&GET_TABLE_TYPE_SCHEMA)
             }
+            FlightSQLCommand::CommandGetXdbcTypeInfo(CommandGetXdbcTypeInfo { .. }) => {
+                encode_schema(&GET_XDBC_TYPE_INFO_SCHEMA)
+            }
             FlightSQLCommand::ActionCreatePreparedStatementRequest(_)
             | FlightSQLCommand::ActionClosePreparedStatementRequest(_) => ProtocolSnafu {
                 cmd: format!("{cmd:?}"),
@@ -239,6 +242,11 @@ impl FlightSQLPlanner {
                 let plan = plan_get_table_types(ctx).await?;
                 Ok(ctx.create_physical_plan(&plan).await?)
             }
+            FlightSQLCommand::CommandGetXdbcTypeInfo(CommandGetXdbcTypeInfo { data_type }) => {
+                debug!(?data_type, "Planning GetXdbcTypeInfo query");
+                let plan = plan_get_xdbc_type_info(ctx, data_type).await?;
+                Ok(ctx.create_physical_plan(&plan).await?)
+            }
             FlightSQLCommand::ActionClosePreparedStatementRequest(_)
             | FlightSQLCommand::ActionCreatePreparedStatementRequest(_) => ProtocolSnafu {
                 cmd: format!("{cmd:?}"),
@@ -417,6 +425,15 @@ async fn plan_get_table_types(ctx: &IOxSessionContext) -> Result<LogicalPlan> {
     Ok(ctx.batch_to_logical_plan(TABLE_TYPES_RECORD_BATCH.clone())?)
 }
 
+/// Return a `LogicalPlan` for GetXdbcTypeInfo
+async fn plan_get_xdbc_type_info(
+    ctx: &IOxSessionContext,
+    _data_type: Option<i32>,
+) -> Result<LogicalPlan> {
+    let batch = RecordBatch::new_empty(Arc::clone(&GET_XDBC_TYPE_INFO_SCHEMA));
+    Ok(ctx.batch_to_logical_plan(batch)?)
+}
+
 /// The schema for GetTableTypes
 static GET_TABLE_TYPE_SCHEMA: Lazy<SchemaRef> = Lazy::new(|| {
     Arc::new(Schema::new(vec![Field::new(
@@ -505,3 +522,32 @@ static GET_PRIMARY_KEYS_SCHEMA: Lazy<SchemaRef> = Lazy::new(|| {
         Field::new("key_sequence", DataType::Int32, false),
     ]))
 });
+
+// From https://github.com/apache/arrow/blob/9588da967c756b2923e213ccc067378ba6c90a86/format/FlightSql.proto#L1064-L1113
+static GET_XDBC_TYPE_INFO_SCHEMA: Lazy<SchemaRef> = Lazy::new(|| {
+    Arc::new(Schema::new(vec![
+        Field::new("type_name", DataType::Utf8, false),
+        Field::new("data_type", DataType::Int32, false),
+        Field::new("column_size", DataType::Int32, true),
+        Field::new("literal_prefix", DataType::Utf8, true),
+        Field::new("literal_suffix", DataType::Utf8, true),
+        Field::new(
+            "create_params",
+            DataType::List(Arc::new(Field::new("item", DataType::Utf8, false))),
+            true,
+        ),
+        Field::new("nullable", DataType::Int32, false), // Nullable enum: https://github.com/apache/arrow/blob/9588da967c756b2923e213ccc067378ba6c90a86/format/FlightSql.proto#L1014-L1029
+        Field::new("case_sensitive", DataType::Boolean, false),
+        Field::new("searchable", DataType::Int32, false), // Searchable enum: https://github.com/apache/arrow/blob/9588da967c756b2923e213ccc067378ba6c90a86/format/FlightSql.proto#L1031-L1056
+        Field::new("unsigned_attribute", DataType::Boolean, true),
+        Field::new("fixed_prec_scale", DataType::Boolean, false),
+        Field::new("auto_increment", DataType::Boolean, true),
+        Field::new("local_type_name", DataType::Utf8, true),
+        Field::new("minimum_scale", DataType::Int32, true),
+        Field::new("maximum_scale", DataType::Int32, true),
+        Field::new("sql_data_type", DataType::Int32, false),
+        Field::new("datetime_subcode", DataType::Int32, true),
+        Field::new("num_prec_radix", DataType::Int32, true),
+        Field::new("interval_precision", DataType::Int32, true),
+    ]))
+});
diff --git a/influxdb_iox/tests/end_to_end_cases/flightsql.rs b/influxdb_iox/tests/end_to_end_cases/flightsql.rs
index 6fafe0437b..99f2715d49 100644
--- a/influxdb_iox/tests/end_to_end_cases/flightsql.rs
+++ b/influxdb_iox/tests/end_to_end_cases/flightsql.rs
@@ -1134,6 +1134,48 @@ async fn flightsql_get_primary_keys() {
     .await
 }
 
+#[tokio::test]
+async fn flightsql_get_xdbc_type_info() {
+    test_helpers::maybe_start_logging();
+    let database_url = maybe_skip_integration!();
+
+    let table_name = "the_table";
+
+    // Set up the cluster  ====================================
+    let mut cluster = MiniCluster::create_shared2(database_url).await;
+
+    StepTest::new(
+        &mut cluster,
+        vec![
+            Step::WriteLineProtocol(format!(
+                "{table_name},tag1=A,tag2=B val=42i 123456\n\
+                 {table_name},tag1=A,tag2=C val=43i 123457"
+            )),
+            Step::Custom(Box::new(move |state: &mut StepTestState| {
+                async move {
+                    let mut client = flightsql_client(state.cluster());
+                    let data_type: Option<i32> = None;
+
+                    let stream = client.get_xdbc_type_info(data_type).await.unwrap();
+                    let batches = collect_stream(stream).await;
+
+                    insta::assert_yaml_snapshot!(
+                        batches_to_sorted_lines(&batches),
+                        @r###"
+                    ---
+                    - ++
+                    - ++
+                    "###
+                    );
+                }
+                .boxed()
+            })),
+        ],
+    )
+    .run()
+    .await
+}
+
 #[tokio::test]
 /// Runs  the `jdbc_client` program against IOx to verify JDBC via FlightSQL is working
 ///
diff --git a/influxdb_iox/tests/jdbc_client/Main.java b/influxdb_iox/tests/jdbc_client/Main.java
index feff02da77..b147264ab1 100644
--- a/influxdb_iox/tests/jdbc_client/Main.java
+++ b/influxdb_iox/tests/jdbc_client/Main.java
@@ -169,12 +169,15 @@ public class Main {
         System.out.println("**************");
         print_result_set(md.getTableTypes());
 
+        System.out.println("**************");
+        System.out.println("getColumns:");
+        System.out.println("**************");
+        print_result_set(md.getColumns(null, null, null, null));
 
-        // TODO uncomment when GetTables is implemented
-        //System.out.println("**************");
-        //System.out.println("getColumns:");
-        //System.out.println("**************");
-        //print_result_set(md.getColumns(null, null, null, null));
+        System.out.println("**************");
+        System.out.println("Type Info:");
+        System.out.println("**************");
+        print_result_set(md.getTypeInfo());
 
         System.out.println("**************");
         System.out.println("getFunctions:");
diff --git a/influxdb_iox_client/src/client/flightsql.rs b/influxdb_iox_client/src/client/flightsql.rs
index 1cf8c10e85..6a68823142 100644
--- a/influxdb_iox_client/src/client/flightsql.rs
+++ b/influxdb_iox_client/src/client/flightsql.rs
@@ -31,7 +31,8 @@ use arrow_flight::{
         ActionCreatePreparedStatementRequest, ActionCreatePreparedStatementResult, Any,
         CommandGetCatalogs, CommandGetCrossReference, CommandGetDbSchemas, CommandGetExportedKeys,
         CommandGetImportedKeys, CommandGetPrimaryKeys, CommandGetSqlInfo, CommandGetTableTypes,
-        CommandGetTables, CommandPreparedStatementQuery, CommandStatementQuery, ProstMessageExt,
+        CommandGetTables, CommandGetXdbcTypeInfo, CommandPreparedStatementQuery,
+        CommandStatementQuery, ProstMessageExt,
     },
     Action, FlightClient, FlightDescriptor, FlightInfo, IpcMessage, Ticket,
 };
@@ -361,6 +362,26 @@ impl FlightSqlClient {
         self.do_get_with_cmd(msg.as_any()).await
     }
 
+    /// List information about data type supported on this server
+    /// using a [`CommandGetXdbcTypeInfo`] message.
+    ///
+    /// # Parameters
+    ///
+    /// Definition from <https://github.com/apache/arrow/blob/9588da967c756b2923e213ccc067378ba6c90a86/format/FlightSql.proto#L1058-L1123>
+    ///
+    /// data_type: Specifies the data type to search for the info.
+    ///
+    /// This implementation does not support alternate endpoints
+    pub async fn get_xdbc_type_info(
+        &mut self,
+        data_type: Option<impl Into<i32> + Send>,
+    ) -> Result<FlightRecordBatchStream> {
+        let msg = CommandGetXdbcTypeInfo {
+            data_type: data_type.map(|dt| dt.into()),
+        };
+        self.do_get_with_cmd(msg.as_any()).await
+    }
+
     /// Implements the canonical interaction for most FlightSQL messages:
     ///
     /// 1. Call `GetFlightInfo` with the provided message, and get a
diff --git a/service_grpc_flight/src/lib.rs b/service_grpc_flight/src/lib.rs
index d7163a1793..bc095955bc 100644
--- a/service_grpc_flight/src/lib.rs
+++ b/service_grpc_flight/src/lib.rs
@@ -785,6 +785,7 @@ fn flightsql_permissions(namespace_name: &str, cmd: &FlightSQLCommand) -> Vec<au
         FlightSQLCommand::CommandGetPrimaryKeys(_) => authz::Action::ReadSchema,
         FlightSQLCommand::CommandGetTables(_) => authz::Action::ReadSchema,
         FlightSQLCommand::CommandGetTableTypes(_) => authz::Action::ReadSchema,
+        FlightSQLCommand::CommandGetXdbcTypeInfo(_) => authz::Action::ReadSchema,
         FlightSQLCommand::ActionCreatePreparedStatementRequest(_) => authz::Action::Read,
         FlightSQLCommand::ActionClosePreparedStatementRequest(_) => authz::Action::Read,
     };

From d36c9dced557708bd411b4b605673191494db30a Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 26 Apr 2023 02:02:42 +0000
Subject: [PATCH 21/25] chore(deps): Bump tracing from 0.1.37 to 0.1.38

Bumps [tracing](https://github.com/tokio-rs/tracing) from 0.1.37 to 0.1.38.
- [Release notes](https://github.com/tokio-rs/tracing/releases)
- [Commits](https://github.com/tokio-rs/tracing/compare/tracing-0.1.37...tracing-0.1.38)

---
updated-dependencies:
- dependency-name: tracing
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 Cargo.lock | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index de0e25d4af..4cdbd9b8cb 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6087,11 +6087,10 @@ dependencies = [
 
 [[package]]
 name = "tracing"
-version = "0.1.37"
+version = "0.1.38"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8"
+checksum = "cf9cf6a813d3f40c88b0b6b6f29a5c95c6cdbf97c1f9cc53fb820200f5ad814d"
 dependencies = [
- "cfg-if",
  "log",
  "pin-project-lite",
  "tracing-attributes",
@@ -6100,13 +6099,13 @@ dependencies = [
 
 [[package]]
 name = "tracing-attributes"
-version = "0.1.23"
+version = "0.1.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a"
+checksum = "0f57e3ca2a01450b1a921183a9c9cbfda207fd822cef4ccb00a65402cbba7a74"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 1.0.109",
+ "syn 2.0.15",
 ]
 
 [[package]]

From d72179fbfc45d8ced30a0082ee147fa56263d7f1 Mon Sep 17 00:00:00 2001
From: "CircleCI[bot]" <circleci@influxdata.com>
Date: Wed, 26 Apr 2023 02:03:57 +0000
Subject: [PATCH 22/25] chore: Run cargo hakari tasks

---
 workspace-hack/Cargo.toml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml
index f092badafe..4d76dfdd33 100644
--- a/workspace-hack/Cargo.toml
+++ b/workspace-hack/Cargo.toml
@@ -146,8 +146,8 @@ sha2 = { version = "0.10" }
 smallvec = { version = "1", default-features = false, features = ["union"] }
 sqlx-core = { version = "0.6", default-features = false, features = ["any", "migrate", "postgres", "runtime-tokio-rustls", "sqlite", "uuid"] }
 sqlx-macros = { version = "0.6", default-features = false, features = ["json", "migrate", "postgres", "runtime-tokio-rustls", "sqlite", "uuid"] }
-syn-dff4ba8e3ae991db = { package = "syn", version = "1", features = ["extra-traits", "full", "visit", "visit-mut"] }
-syn-f595c2ba2a3f28df = { package = "syn", version = "2", features = ["full", "visit-mut"] }
+syn-dff4ba8e3ae991db = { package = "syn", version = "1", features = ["extra-traits", "full", "visit-mut"] }
+syn-f595c2ba2a3f28df = { package = "syn", version = "2", features = ["extra-traits", "full", "visit-mut"] }
 tokio = { version = "1", features = ["full", "test-util", "tracing"] }
 tokio-stream = { version = "0.1", features = ["fs", "net"] }
 url = { version = "2" }

From 0b9240cbbe86fe687c1ed0a8c43f9bda61de212a Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 26 Apr 2023 09:24:39 +0000
Subject: [PATCH 23/25] chore(deps): Bump tokio-util from 0.7.7 to 0.7.8
 (#7665)

Bumps [tokio-util](https://github.com/tokio-rs/tokio) from 0.7.7 to 0.7.8.
- [Release notes](https://github.com/tokio-rs/tokio/releases)
- [Commits](https://github.com/tokio-rs/tokio/compare/tokio-util-0.7.7...tokio-util-0.7.8)

---
updated-dependencies:
- dependency-name: tokio-util
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com>
---
 Cargo.lock                        | 4 ++--
 cache_system/Cargo.toml           | 2 +-
 compactor2/Cargo.toml             | 2 +-
 executor/Cargo.toml               | 2 +-
 garbage_collector/Cargo.toml      | 2 +-
 influxdb_iox/Cargo.toml           | 2 +-
 ingester2/Cargo.toml              | 2 +-
 ingester2_test_ctx/Cargo.toml     | 2 +-
 ioxd_common/Cargo.toml            | 2 +-
 ioxd_compactor2/Cargo.toml        | 2 +-
 ioxd_garbage_collector/Cargo.toml | 2 +-
 ioxd_ingester2/Cargo.toml         | 2 +-
 ioxd_querier/Cargo.toml           | 2 +-
 ioxd_router/Cargo.toml            | 2 +-
 ioxd_test/Cargo.toml              | 2 +-
 querier/Cargo.toml                | 2 +-
 tracker/Cargo.toml                | 2 +-
 17 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 4cdbd9b8cb..ffb48e1585 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -5837,9 +5837,9 @@ dependencies = [
 
 [[package]]
 name = "tokio-util"
-version = "0.7.7"
+version = "0.7.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5427d89453009325de0d8f342c9490009f76e999cb7672d77e46267448f7e6b2"
+checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d"
 dependencies = [
  "bytes",
  "futures-core",
diff --git a/cache_system/Cargo.toml b/cache_system/Cargo.toml
index 8c95f4a635..f2f748b98a 100644
--- a/cache_system/Cargo.toml
+++ b/cache_system/Cargo.toml
@@ -16,7 +16,7 @@ parking_lot = { version = "0.12", features = ["arc_lock"] }
 pdatastructs = { version = "0.7", default-features = false, features = ["fixedbitset"] }
 rand = "0.8.3"
 tokio = { version = "1.27", features = ["macros", "parking_lot", "rt-multi-thread", "sync", "time"] }
-tokio-util = { version = "0.7.7" }
+tokio-util = { version = "0.7.8" }
 trace = { path = "../trace"}
 workspace-hack = { version = "0.1", path = "../workspace-hack" }
 
diff --git a/compactor2/Cargo.toml b/compactor2/Cargo.toml
index f57589a1cc..4694bed32c 100644
--- a/compactor2/Cargo.toml
+++ b/compactor2/Cargo.toml
@@ -25,7 +25,7 @@ rand = "0.8.3"
 schema = { path = "../schema" }
 sharder = { path = "../sharder" }
 tokio = { version = "1", features = ["macros", "rt", "sync"] }
-tokio-util = { version = "0.7.7" }
+tokio-util = { version = "0.7.8" }
 tracker = { path = "../tracker" }
 uuid = { version = "1", features = ["v4"] }
 workspace-hack = { version = "0.1", path = "../workspace-hack" }
diff --git a/executor/Cargo.toml b/executor/Cargo.toml
index 109027b65c..0281207c74 100644
--- a/executor/Cargo.toml
+++ b/executor/Cargo.toml
@@ -12,7 +12,7 @@ once_cell = { version = "1.17", features = ["parking_lot"] }
 parking_lot = "0.12"
 pin-project = "1.0"
 tokio = { version = "1.27" }
-tokio-util = { version = "0.7.7" }
+tokio-util = { version = "0.7.8" }
 workspace-hack = { version = "0.1", path = "../workspace-hack" }
 
 # use libc on unix like platforms to set worker priority in DedicatedExecutor
diff --git a/garbage_collector/Cargo.toml b/garbage_collector/Cargo.toml
index cdebc0c8ef..59e7bcca0d 100644
--- a/garbage_collector/Cargo.toml
+++ b/garbage_collector/Cargo.toml
@@ -18,7 +18,7 @@ observability_deps = { path = "../observability_deps" }
 snafu = "0.7"
 tokio = { version = "1", features = ["macros", "rt", "sync"] }
 tokio-stream = "0.1"
-tokio-util = { version = "0.7.7" }
+tokio-util = { version = "0.7.8" }
 uuid = { version = "1", features = ["v4"] }
 
 [dev-dependencies]
diff --git a/influxdb_iox/Cargo.toml b/influxdb_iox/Cargo.toml
index 3204528d7e..360615e2d0 100644
--- a/influxdb_iox/Cargo.toml
+++ b/influxdb_iox/Cargo.toml
@@ -70,7 +70,7 @@ thiserror = "1.0.40"
 tikv-jemalloc-ctl = { version = "0.5.0", optional = true }
 tokio = { version = "1.27", features = ["macros", "net", "parking_lot", "rt-multi-thread", "signal", "sync", "time", "io-std"] }
 tokio-stream = { version = "0.1", features = ["net"] }
-tokio-util = { version = "0.7.7", features = ["compat"] }
+tokio-util = { version = "0.7.8", features = ["compat"] }
 tonic = { workspace = true }
 uuid = { version = "1", features = ["v4"] }
 # jemalloc-sys with unprefixed_malloc_on_supported_platforms feature and heappy are mutually exclusive
diff --git a/ingester2/Cargo.toml b/ingester2/Cargo.toml
index 6c636b1539..19525282ec 100644
--- a/ingester2/Cargo.toml
+++ b/ingester2/Cargo.toml
@@ -43,7 +43,7 @@ sharder = { version = "0.1.0", path = "../sharder" }
 test_helpers = { path = "../test_helpers", features = ["future_timeout"], optional = true }
 thiserror = "1.0.40"
 tokio = { version = "1.27", features = ["macros", "parking_lot", "rt-multi-thread", "sync", "time"] }
-tokio-util = "0.7.7"
+tokio-util = "0.7.8"
 tonic = { workspace = true }
 trace = { version = "0.1.0", path = "../trace" }
 uuid = "1.3.1"
diff --git a/ingester2_test_ctx/Cargo.toml b/ingester2_test_ctx/Cargo.toml
index 18dda522fd..66fcbc1e16 100644
--- a/ingester2_test_ctx/Cargo.toml
+++ b/ingester2_test_ctx/Cargo.toml
@@ -31,7 +31,7 @@ prost = { version = "0.11.9", default-features = false, features = ["std"] }
 tempfile = { version = "3.5.0" }
 test_helpers = { path = "../test_helpers", features = ["future_timeout"] }
 tokio = { version = "1.27", features = ["macros", "parking_lot", "rt-multi-thread", "sync", "time"] }
-tokio-util = "0.7.7"
+tokio-util = "0.7.8"
 tonic = { workspace = true }
 wal = { version = "0.1.0", path = "../wal" }
 workspace-hack = { version = "0.1", path = "../workspace-hack" }
diff --git a/ioxd_common/Cargo.toml b/ioxd_common/Cargo.toml
index bc69b3c4bf..0123ef3d89 100644
--- a/ioxd_common/Cargo.toml
+++ b/ioxd_common/Cargo.toml
@@ -43,7 +43,7 @@ serde_urlencoded = "0.7.0"
 snafu = "0.7"
 tokio = { version = "1.27", features = ["macros", "net", "parking_lot", "rt-multi-thread", "signal", "sync", "time"] }
 tokio-stream = { version = "0.1", features = ["net"] }
-tokio-util = { version = "0.7.7" }
+tokio-util = { version = "0.7.8" }
 tonic  = { workspace = true }
 tonic-health  = { workspace = true }
 tonic-reflection = { workspace = true }
diff --git a/ioxd_compactor2/Cargo.toml b/ioxd_compactor2/Cargo.toml
index b7658ed3ae..db3cd589e9 100644
--- a/ioxd_compactor2/Cargo.toml
+++ b/ioxd_compactor2/Cargo.toml
@@ -18,6 +18,6 @@ iox_query = { path = "../iox_query" }
 ioxd_common = { path = "../ioxd_common" }
 metric = { path = "../metric" }
 parquet_file = { path = "../parquet_file" }
-tokio-util = "0.7.7"
+tokio-util = "0.7.8"
 trace = { path = "../trace" }
 workspace-hack = { version = "0.1", path = "../workspace-hack" }
diff --git a/ioxd_garbage_collector/Cargo.toml b/ioxd_garbage_collector/Cargo.toml
index 89e0862297..bc108d8bbc 100644
--- a/ioxd_garbage_collector/Cargo.toml
+++ b/ioxd_garbage_collector/Cargo.toml
@@ -17,4 +17,4 @@ snafu = "0.7"
 tokio = { version = "1", features = ["sync"] }
 trace = { path = "../trace" }
 workspace-hack = { version = "0.1", path = "../workspace-hack" }
-tokio-util = "0.7.7"
+tokio-util = "0.7.8"
diff --git a/ioxd_ingester2/Cargo.toml b/ioxd_ingester2/Cargo.toml
index 4959793c2e..c9507d39ba 100644
--- a/ioxd_ingester2/Cargo.toml
+++ b/ioxd_ingester2/Cargo.toml
@@ -20,6 +20,6 @@ metric = { path = "../metric" }
 parquet_file = { version = "0.1.0", path = "../parquet_file" }
 thiserror = "1.0.40"
 tokio = { version = "1.27", features = ["macros", "net", "parking_lot", "rt-multi-thread", "signal", "sync", "time"] }
-tokio-util = { version = "0.7.7" }
+tokio-util = { version = "0.7.8" }
 trace = { path = "../trace" }
 workspace-hack = { version = "0.1", path = "../workspace-hack" }
diff --git a/ioxd_querier/Cargo.toml b/ioxd_querier/Cargo.toml
index 885fae88a9..252e231a16 100644
--- a/ioxd_querier/Cargo.toml
+++ b/ioxd_querier/Cargo.toml
@@ -31,7 +31,7 @@ thiserror = "1.0.40"
 tokio = { version = "1.27", features = ["macros", "net", "parking_lot", "rt-multi-thread", "signal", "sync", "time"] }
 tonic = { workspace = true }
 workspace-hack = { version = "0.1", path = "../workspace-hack" }
-tokio-util = "0.7.7"
+tokio-util = "0.7.8"
 
 [dev-dependencies]
 # Workspace dependencies, in alphabetical order
diff --git a/ioxd_router/Cargo.toml b/ioxd_router/Cargo.toml
index 60bd848a43..6ed964332d 100644
--- a/ioxd_router/Cargo.toml
+++ b/ioxd_router/Cargo.toml
@@ -21,6 +21,6 @@ router = { path = "../router" }
 sharder = { path = "../sharder" }
 thiserror = "1.0.40"
 tokio = { version = "1.27", features = ["macros", "net", "parking_lot", "rt-multi-thread", "signal", "sync", "time"] }
-tokio-util = { version = "0.7.7" }
+tokio-util = { version = "0.7.8" }
 trace = { path = "../trace" }
 workspace-hack = { version = "0.1", path = "../workspace-hack" }
diff --git a/ioxd_test/Cargo.toml b/ioxd_test/Cargo.toml
index a3c96397b8..58e6ad2588 100644
--- a/ioxd_test/Cargo.toml
+++ b/ioxd_test/Cargo.toml
@@ -16,5 +16,5 @@ async-trait = "0.1"
 clap = { version = "4", features = ["derive", "env"] }
 hyper = "0.14"
 snafu = "0.7"
-tokio-util = "0.7.7"
+tokio-util = "0.7.8"
 workspace-hack = { version = "0.1", path = "../workspace-hack" }
diff --git a/querier/Cargo.toml b/querier/Cargo.toml
index 3f97b29b13..79fc7ddbfb 100644
--- a/querier/Cargo.toml
+++ b/querier/Cargo.toml
@@ -39,7 +39,7 @@ schema = { path = "../schema" }
 snafu = "0.7"
 thiserror = "1.0"
 tokio = { version = "1.27", features = ["macros", "parking_lot", "rt-multi-thread", "sync", "time"] }
-tokio-util = { version = "0.7.7" }
+tokio-util = { version = "0.7.8" }
 tonic = { workspace = true }
 trace = { path = "../trace" }
 trace_exporters = { path = "../trace_exporters" }
diff --git a/tracker/Cargo.toml b/tracker/Cargo.toml
index a561cd7051..10d408320c 100644
--- a/tracker/Cargo.toml
+++ b/tracker/Cargo.toml
@@ -16,7 +16,7 @@ parking_lot = "0.12"
 pin-project = "1.0"
 iox_time = { path = "../iox_time" }
 tokio = { version = "1.27", features = ["macros", "parking_lot", "sync", "time"] }
-tokio-util = { version = "0.7.7" }
+tokio-util = { version = "0.7.8" }
 trace = { path = "../trace"}
 workspace-hack = { version = "0.1", path = "../workspace-hack" }
 

From 09d6b4ae50ada08aa398d2652e993366b58a0990 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 26 Apr 2023 09:31:04 +0000
Subject: [PATCH 24/25] chore(deps): Bump tokio-stream from 0.1.12 to 0.1.13
 (#7666)

Bumps [tokio-stream](https://github.com/tokio-rs/tokio) from 0.1.12 to 0.1.13.
- [Release notes](https://github.com/tokio-rs/tokio/releases)
- [Commits](https://github.com/tokio-rs/tokio/compare/tokio-stream-0.1.12...tokio-stream-0.1.13)

---
updated-dependencies:
- dependency-name: tokio-stream
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Dom <dom@itsallbroken.com>
Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com>
---
 Cargo.lock                     | 4 ++--
 influxdb_iox_client/Cargo.toml | 2 +-
 router/Cargo.toml              | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index ffb48e1585..022f8c3048 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -5826,9 +5826,9 @@ dependencies = [
 
 [[package]]
 name = "tokio-stream"
-version = "0.1.12"
+version = "0.1.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313"
+checksum = "76cd2598a37719e3cd4c28af93f978506a97a2920ef4d96e4b12e38b8cbc8940"
 dependencies = [
  "futures-core",
  "pin-project-lite",
diff --git a/influxdb_iox_client/Cargo.toml b/influxdb_iox_client/Cargo.toml
index 514c01074f..fed3fa1229 100644
--- a/influxdb_iox_client/Cargo.toml
+++ b/influxdb_iox_client/Cargo.toml
@@ -26,7 +26,7 @@ reqwest = { version = "0.11", default-features = false, features = ["stream", "r
 schema = { path = "../schema" }
 serde_json = "1.0.96"
 tokio = { version = "1.27", features = ["macros", "parking_lot", "rt-multi-thread"] }
-tokio-stream = "0.1.12"
+tokio-stream = "0.1.13"
 thiserror = "1.0.40"
 tonic = { workspace = true }
 
diff --git a/router/Cargo.toml b/router/Cargo.toml
index 51b5715007..95776cb938 100644
--- a/router/Cargo.toml
+++ b/router/Cargo.toml
@@ -54,7 +54,7 @@ pretty_assertions = "1.3.0"
 rand = "0.8.3"
 test_helpers = { version = "0.1.0", path = "../test_helpers", features = ["future_timeout"] }
 tokio = { version = "1", features = ["test-util"] }
-tokio-stream = { version = "0.1.12", default_features = false, features = [] }
+tokio-stream = { version = "0.1.13", default_features = false, features = [] }
 
 [lib]
 # Allow --save-baseline to work

From 3f69f5eea663e9fd86f1ad63db99f2e7b742eeb5 Mon Sep 17 00:00:00 2001
From: Andrew Lamb <alamb@influxdata.com>
Date: Wed, 26 Apr 2023 06:19:03 -0400
Subject: [PATCH 25/25] fix: Fix `The data type inlist should be same` error
 (#7645)

Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com>
---
 influxdb_iox/tests/query_tests2/cases.rs      | 12 ++++++++++++
 .../tests/query_tests2/cases/in/bugs.sql      |  7 +++++++
 .../query_tests2/cases/in/bugs.sql.expected   | 15 +++++++++++++++
 influxdb_iox/tests/query_tests2/setups.rs     | 19 +++++++++++++++++++
 4 files changed, 53 insertions(+)
 create mode 100644 influxdb_iox/tests/query_tests2/cases/in/bugs.sql
 create mode 100644 influxdb_iox/tests/query_tests2/cases/in/bugs.sql.expected

diff --git a/influxdb_iox/tests/query_tests2/cases.rs b/influxdb_iox/tests/query_tests2/cases.rs
index 46345f4cb9..7113ae1eca 100644
--- a/influxdb_iox/tests/query_tests2/cases.rs
+++ b/influxdb_iox/tests/query_tests2/cases.rs
@@ -325,6 +325,18 @@ async fn different_tag_sets() {
     .await;
 }
 
+#[tokio::test]
+async fn bugs() {
+    test_helpers::maybe_start_logging();
+
+    TestCase {
+        input: "cases/in/bugs.sql",
+        chunk_stage: ChunkStage::Ingester,
+    }
+    .run()
+    .await;
+}
+
 mod influxql {
     use super::*;
 
diff --git a/influxdb_iox/tests/query_tests2/cases/in/bugs.sql b/influxdb_iox/tests/query_tests2/cases/in/bugs.sql
new file mode 100644
index 0000000000..1bc31198bd
--- /dev/null
+++ b/influxdb_iox/tests/query_tests2/cases/in/bugs.sql
@@ -0,0 +1,7 @@
+-- IOX_SETUP: Bugs
+
+-- https://github.com/influxdata/influxdb_iox/issues/7644
+-- internal error in select list
+SELECT id FROM checks WHERE id in ('2','3','4','5');
+
+SELECT id FROM checks WHERE id in ('1', '2','3','4','5');
diff --git a/influxdb_iox/tests/query_tests2/cases/in/bugs.sql.expected b/influxdb_iox/tests/query_tests2/cases/in/bugs.sql.expected
new file mode 100644
index 0000000000..3d3bcc151f
--- /dev/null
+++ b/influxdb_iox/tests/query_tests2/cases/in/bugs.sql.expected
@@ -0,0 +1,15 @@
+-- Test Setup: Bugs
+-- SQL: SELECT id FROM checks WHERE id in ('2','3','4','5');
++----+
+| id |
++----+
++----+
+-- SQL: SELECT id FROM checks WHERE id in ('1', '2','3','4','5');
++----+
+| id |
++----+
+| 1  |
+| 1  |
+| 1  |
+| 1  |
++----+
\ No newline at end of file
diff --git a/influxdb_iox/tests/query_tests2/setups.rs b/influxdb_iox/tests/query_tests2/setups.rs
index 822d7ae02c..74d5b2df59 100644
--- a/influxdb_iox/tests/query_tests2/setups.rs
+++ b/influxdb_iox/tests/query_tests2/setups.rs
@@ -21,6 +21,25 @@ static RETENTION_SETUP: Lazy<RetentionSetup> = Lazy::new(RetentionSetup::new);
 /// All possible setups for the [`TestCase`][crate::TestCase]s to use, indexed by name
 pub static SETUPS: Lazy<HashMap<SetupName, SetupSteps>> = Lazy::new(|| {
     HashMap::from([
+        (
+            "Bugs",
+            vec![
+                Step::RecordNumParquetFiles,
+                Step::WriteLineProtocol(
+                    [
+                        r#"checks,id=1,method=POST,name=Writes,url="https://example.com",user_id=1 elapsed=66909i,status=204i 1678578528255989730"#,
+                        r#"checks,id=1,method=POST,name=Writes,url="https://example.com",user_id=1 elapsed=112928i,status=204i 1678578532192972168"#,
+                        r#"checks,id=1,method=POST,name=Writes,url="https://example.com",user_id=1 elapsed=147683i,status=204i 1678578588416052133"#,
+                        r#"checks,id=1,method=POST,name=Writes,url="https://example.com",user_id=1 elapsed=78000i,status=204i 1678578592147893402"#,
+                    ]
+                    .join("\n"),
+                ),
+                Step::Persist,
+                Step::WaitForPersisted2 {
+                    expected_increase: 1,
+                },
+            ],
+        ),
         (
             "TwoMeasurements",
             vec![