Merge branch 'main' into crepererum/dml_shard_new_types

Cargo.lock

@@ -453,9 +453,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.0.71"
+version = "1.0.72"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "79c2681d6594606957bbb8631c4b90a7fcaaa72cdb714743a437b156d6a7eedd"
+checksum = "22a9137b95ea06864e018375b72adfb7db6e6f68cfc8df5a04d00288050485ee"
 dependencies = [
  "jobserver",
 ]
@@ -641,9 +641,9 @@ dependencies = [
 
 [[package]]
 name = "crc32fast"
-version = "1.2.1"
+version = "1.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a"
+checksum = "3825b1e8580894917dc4468cb634a1b4e9745fddc854edad72d9c04644c0319f"
 dependencies = [
  "cfg-if",
 ]
@@ -1111,9 +1111,9 @@ dependencies = [
 
 [[package]]
 name = "futures-channel"
-version = "0.3.17"
+version = "0.3.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5da6ba8c3bb3c165d3c7319fc1cc8304facf1fb8db99c5de877183c08a273888"
+checksum = "7fc8cd39e3dbf865f7340dce6a2d401d24fd37c6fe6c4f0ee0de8bfca2252d27"
 dependencies = [
  "futures-core",
  "futures-sink",
@@ -1121,9 +1121,9 @@ dependencies = [
 
 [[package]]
 name = "futures-core"
-version = "0.3.17"
+version = "0.3.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "88d1c26957f23603395cd326b0ffe64124b818f4449552f960d815cfba83a53d"
+checksum = "629316e42fe7c2a0b9a65b47d159ceaa5453ab14e8f0a3c5eedbb8cd55b4a445"
 
 [[package]]
 name = "futures-executor"
@@ -1138,9 +1138,9 @@ dependencies = [
 
 [[package]]
 name = "futures-io"
-version = "0.3.17"
+version = "0.3.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "522de2a0fe3e380f1bc577ba0474108faf3f6b18321dbf60b3b9c39a75073377"
+checksum = "e481354db6b5c353246ccf6a728b0c5511d752c08da7260546fc0933869daa11"
 
 [[package]]
 name = "futures-macro"
@@ -1157,15 +1157,15 @@ dependencies = [
 
 [[package]]
 name = "futures-sink"
-version = "0.3.17"
+version = "0.3.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "36ea153c13024fe480590b3e3d4cad89a0cfacecc24577b68f86c6ced9c2bc11"
+checksum = "996c6442437b62d21a32cd9906f9c41e7dc1e19a9579843fad948696769305af"
 
 [[package]]
 name = "futures-task"
-version = "0.3.17"
+version = "0.3.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1d3d00f4eddb73e498a54394f228cd55853bdf059259e8e7bc6e69d408892e99"
+checksum = "dabf1872aaab32c886832f2276d2f5399887e2bd613698a02359e4ea83f8de12"
 
 [[package]]
 name = "futures-test"
@@ -3053,9 +3053,9 @@ dependencies = [
 
 [[package]]
 name = "predicates"
-version = "2.0.3"
+version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5c6ce811d0b2e103743eec01db1c50612221f173084ce2f7941053e94b6bb474"
+checksum = "95e5a7689e456ab905c22c2b48225bb921aba7c8dfa58440d68ba13f6222a715"
 dependencies = [
  "difflib",
  "float-cmp",
@@ -3372,9 +3372,9 @@ dependencies = [
 
 [[package]]
 name = "rdkafka"
-version = "0.27.0"
+version = "0.28.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a22ce72c78b471baba6c75bda6e03511ef2ee1bae3729902d2bb38951db1048"
+checksum = "1de127f294f2dba488ed46760b129d5ecbeabbd337ccbf3739cb29d50db2161c"
 dependencies = [
  "futures",
  "libc",
@@ -3389,9 +3389,9 @@ dependencies = [
 
 [[package]]
 name = "rdkafka-sys"
-version = "4.1.0+1.7.0"
+version = "4.2.0+1.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e212ccf56a2d4e0b9f874a1cad295495769e0cdbabe60e02b4f654d369a2d6a1"
+checksum = "9e542c6863b04ce0fa0c5719bc6b7b348cf8dd21af1bb03c9db5f9805b2a6473"
 dependencies = [
  "libc",
  "libz-sys",
@@ -3838,9 +3838,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.71"
+version = "1.0.72"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "063bf466a64011ac24040a49009724ee60a57da1b437617ceb32e53ad61bfb19"
+checksum = "d0ffa0837f2dfa6fb90868c2b5468cad482e175f7dad97e7421951e663f2b527"
 dependencies = [
  "indexmap",
  "itoa",

generated_types/protos/influxdata/platform/storage/service.proto

@@ -39,4 +39,7 @@ service Storage {
     rpc MeasurementTagValues(MeasurementTagValuesRequest) returns (stream StringValuesResponse);
 
     rpc MeasurementFields(MeasurementFieldsRequest) returns (stream MeasurementFieldsResponse);
+
+    // Offsets gets the partition offsets of the node
+    rpc Offsets (google.protobuf.Empty) returns (OffsetsResponse);
 }

generated_types/src/database_rules/lifecycle.rs

@@ -58,6 +58,21 @@ impl TryFrom<management::LifecycleRules> for LifecycleRules {
     type Error = FieldViolation;
 
     fn try_from(proto: management::LifecycleRules) -> Result<Self, Self::Error> {
+        let persist_age_threshold_seconds = NonZeroU32::new(proto.persist_age_threshold_seconds)
+            .unwrap_or_else(|| NonZeroU32::new(DEFAULT_PERSIST_AGE_THRESHOLD_SECONDS).unwrap());
+
+        let late_arrive_window_seconds = NonZeroU32::new(proto.late_arrive_window_seconds)
+            .unwrap_or_else(|| NonZeroU32::new(DEFAULT_LATE_ARRIVE_WINDOW_SECONDS).unwrap());
+
+        if persist_age_threshold_seconds < late_arrive_window_seconds {
+            return Err(FieldViolation {
+                field: "persist_age_threshold_seconds".to_string(),
+                description:
+                    "persist_age_threshold_seconds must not be less than late_arrive_window_seconds"
+                        .to_string(),
+            });
+        }
+
         Ok(Self {
             buffer_size_soft: (proto.buffer_size_soft as usize).try_into().ok(),
             buffer_size_hard: (proto.buffer_size_hard as usize).try_into().ok(),
@@ -79,14 +94,12 @@ impl TryFrom<management::LifecycleRules> for LifecycleRules {
                 Some(d) => d.try_into().scope("catalog_transaction_prune_age")?,
                 None => DEFAULT_CATALOG_TRANSACTION_PRUNE_AGE,
             },
-            late_arrive_window_seconds: NonZeroU32::new(proto.late_arrive_window_seconds)
+            late_arrive_window_seconds,
-                .unwrap_or_else(|| NonZeroU32::new(DEFAULT_LATE_ARRIVE_WINDOW_SECONDS).unwrap()),
             persist_row_threshold: NonZeroUsize::new(proto.persist_row_threshold as usize)
                 .unwrap_or_else(|| {
                     NonZeroUsize::new(DEFAULT_PERSIST_ROW_THRESHOLD as usize).unwrap()
                 }),
-            persist_age_threshold_seconds: NonZeroU32::new(proto.persist_age_threshold_seconds)
+            persist_age_threshold_seconds,
-                .unwrap_or_else(|| NonZeroU32::new(DEFAULT_PERSIST_AGE_THRESHOLD_SECONDS).unwrap()),
             mub_row_threshold: NonZeroUsize::new(proto.mub_row_threshold as usize)
                 .unwrap_or_else(|| NonZeroUsize::new(DEFAULT_MUB_ROW_THRESHOLD).unwrap()),
             parquet_cache_limit: NonZeroU64::new(proto.parquet_cache_limit),
@@ -116,7 +129,7 @@ mod tests {
 
     #[test]
     fn lifecycle_rules() {
-        let protobuf = management::LifecycleRules {
+        let mut protobuf = management::LifecycleRules {
             buffer_size_soft: 353,
             buffer_size_hard: 232,
             persist: true,
@@ -132,7 +145,7 @@ mod tests {
             }),
             late_arrive_window_seconds: 23,
             persist_row_threshold: 57,
-            persist_age_threshold_seconds: 23,
+            persist_age_threshold_seconds: 60,
             mub_row_threshold: 3454,
             parquet_cache_limit: 10,
         };
@@ -181,6 +194,12 @@ mod tests {
             protobuf.parquet_cache_limit
         );
         assert_eq!(back.parquet_cache_limit, protobuf.parquet_cache_limit);
+
+        protobuf.late_arrive_window_seconds = 20;
+        protobuf.persist_age_threshold_seconds = 4;
+
+        let e = LifecycleRules::try_from(protobuf).unwrap_err().to_string();
+        assert_eq!(e, "Violation for field \"persist_age_threshold_seconds\": persist_age_threshold_seconds must not be less than late_arrive_window_seconds");
     }
 
     #[test]

influxdb2_client/Cargo.toml

@@ -9,7 +9,7 @@ bytes = "1.0"
 futures = { version = "0.3", default-features = false }
 reqwest = { version = "0.11", features = ["stream", "json"] }
 serde = { version = "1.0", features = ["derive"] }
-serde_json = "1.0.71"
+serde_json = "1.0.72"
 snafu = "0.6.6"
 url = "2.1.1"
 workspace-hack = { path = "../workspace-hack"}

influxdb_iox/Cargo.toml

@@ -70,7 +70,7 @@ pprof = { version = "^0.5", default-features = false, features = ["flamegraph",
 prost = "0.8"
 rustyline = { version = "9.0", default-features = false }
 serde = { version = "1.0", features = ["derive"] }
-serde_json = "1.0.71"
+serde_json = "1.0.72"
 serde_urlencoded = "0.7.0"
 snafu = "0.6.9"
 structopt = "0.3.25"
@@ -105,7 +105,7 @@ write_buffer = { path = "../write_buffer" }
 # Crates.io dependencies, in alphabetical order
 assert_cmd = "2.0.2"
 hex = "0.4.2"
-predicates = "2.0.3"
+predicates = "2.1.0"
 rand = "0.8.3"
 reqwest = "0.11"
 tempfile = "3.1.0"

influxdb_iox/src/influxdb_ioxd/server_type/database/rpc/storage/service.rs

@@ -11,12 +11,12 @@ use tonic::Status;
 
 use data_types::{error::ErrorLogger, names::org_and_bucket_to_database, DatabaseName};
 use generated_types::{
-    google::protobuf::Empty, storage_server::Storage, CapabilitiesResponse, Capability,
+    google::protobuf::Empty, offsets_response::PartitionOffsetResponse, storage_server::Storage,
-    Int64ValuesResponse, MeasurementFieldsRequest, MeasurementFieldsResponse,
+    CapabilitiesResponse, Capability, Int64ValuesResponse, MeasurementFieldsRequest,
-    MeasurementNamesRequest, MeasurementTagKeysRequest, MeasurementTagValuesRequest, Predicate,
+    MeasurementFieldsResponse, MeasurementNamesRequest, MeasurementTagKeysRequest,
-    ReadFilterRequest, ReadGroupRequest, ReadResponse, ReadSeriesCardinalityRequest,
+    MeasurementTagValuesRequest, OffsetsResponse, Predicate, ReadFilterRequest, ReadGroupRequest,
-    ReadWindowAggregateRequest, StringValuesResponse, TagKeyMetaNames, TagKeysRequest,
+    ReadResponse, ReadSeriesCardinalityRequest, ReadWindowAggregateRequest, StringValuesResponse,
-    TagValuesRequest, TimestampRange,
+    TagKeyMetaNames, TagKeysRequest, TagValuesRequest, TimestampRange,
 };
 use observability_deps::tracing::{error, info, trace};
 use predicate::predicate::PredicateBuilder;
@@ -674,6 +674,18 @@ where
 
         Ok(tonic::Response::new(ReceiverStream::new(rx)))
     }
+
+    async fn offsets(
+        &self,
+        _req: tonic::Request<Empty>,
+    ) -> Result<tonic::Response<OffsetsResponse>, Status> {
+        // We present ourselves to the rest of IDPE as a single storage node with 1 partition.
+        // (Returning offset 1 just in case offset 0 is interpreted by query nodes as being special)
+        let the_partition = PartitionOffsetResponse { id: 0, offset: 1 };
+        Ok(tonic::Response::new(OffsetsResponse {
+            partitions: vec![the_partition],
+        }))
+    }
 }
 
 trait SetRange {

influxdb_iox/tests/end_to_end_cases/storage_api.rs

@@ -7,12 +7,14 @@ use generated_types::{
     google::protobuf::Empty,
     measurement_fields_response::FieldType,
     node::{Comparison, Type as NodeType, Value},
+    offsets_response::PartitionOffsetResponse,
     read_group_request::Group,
     read_response::{frame::Data, *},
     storage_client::StorageClient,
     Aggregate, MeasurementFieldsRequest, MeasurementNamesRequest, MeasurementTagKeysRequest,
-    MeasurementTagValuesRequest, Node, Predicate, ReadFilterRequest, ReadGroupRequest,
+    MeasurementTagValuesRequest, Node, OffsetsResponse, Predicate, ReadFilterRequest,
-    ReadWindowAggregateRequest, Tag, TagKeysRequest, TagValuesRequest, TimestampRange,
+    ReadGroupRequest, ReadWindowAggregateRequest, Tag, TagKeysRequest, TagValuesRequest,
+    TimestampRange,
 };
 use influxdb_iox_client::connection::Connection;
 use influxdb_storage_client::tag_key_bytes_to_strings;
@@ -30,7 +32,6 @@ pub async fn test() {
     scenario.create_database(&mut management_client).await;
     scenario.load_data(&influxdb2).await;
 
-    capabilities_endpoint(&mut storage_client).await;
     read_filter_endpoint(&mut storage_client, &scenario).await;
     tag_keys_endpoint(&mut storage_client, &scenario).await;
     tag_values_endpoint(&mut storage_client, &scenario).await;
@@ -41,9 +42,16 @@ pub async fn test() {
 }
 
 /// Validate that capabilities storage endpoint is hooked up
-async fn capabilities_endpoint(storage_client: &mut StorageClient<Connection>) {
+#[tokio::test]
-    let capabilities_response = storage_client.capabilities(Empty {}).await.unwrap();
+async fn capabilities_endpoint() {
-    let capabilities_response = capabilities_response.into_inner();
+    let server_fixture = ServerFixture::create_shared(ServerType::Database).await;
+    let mut storage_client = StorageClient::new(server_fixture.grpc_channel());
+
+    let capabilities_response = storage_client
+        .capabilities(Empty {})
+        .await
+        .unwrap()
+        .into_inner();
     assert_eq!(
         capabilities_response.caps.len(),
         2,
@@ -52,6 +60,19 @@ async fn capabilities_endpoint(storage_client: &mut StorageClient<Connection>) {
     );
 }
 
+/// Validate that storage offsets endpoint is hooked up (required by internal Influx cloud)
+#[tokio::test]
+async fn offsets_endpoint() {
+    let server_fixture = ServerFixture::create_shared(ServerType::Database).await;
+    let mut storage_client = StorageClient::new(server_fixture.grpc_channel());
+
+    let offsets_response = storage_client.offsets(Empty {}).await.unwrap().into_inner();
+    let expected = OffsetsResponse {
+        partitions: vec![PartitionOffsetResponse { id: 0, offset: 1 }],
+    };
+    assert_eq!(offsets_response, expected);
+}
+
 async fn read_filter_endpoint(storage_client: &mut StorageClient<Connection>, scenario: &Scenario) {
     let read_source = scenario.read_source();
     let range = scenario.timestamp_range();

influxdb_iox_client/Cargo.toml

@@ -27,7 +27,7 @@ mutable_batch_pb = { path = "../mutable_batch_pb", optional = true }
 prost = "0.8"
 rand = "0.8.3"
 serde = "1.0.128"
-serde_json = { version = "1.0.71", optional = true }
+serde_json = { version = "1.0.72", optional = true }
 thiserror = "1.0.30"
 tonic = { version = "0.5.0" }
 uuid = { version = "0.8", features = ["v4"] }

iox_data_generator/Cargo.toml

@@ -19,7 +19,7 @@ influxdb_iox_client = { path = "../influxdb_iox_client" }
 itertools = "0.10.0"
 rand = { version = "0.8.3", features = ["small_rng"] }
 serde = { version = "1.0", features = ["derive"] }
-serde_json = "1.0.71"
+serde_json = "1.0.72"
 snafu = "0.6.8"
 tokio = { version = "1.13", features = ["macros", "rt-multi-thread"] }
 toml = "0.5.6"

predicate/Cargo.toml

@@ -13,7 +13,7 @@ schema = { path = "../schema" }
 observability_deps = { path = "../observability_deps" }
 ordered-float = "2"
 regex = "1"
-serde_json = "1.0.71"
+serde_json = "1.0.72"
 snafu = "0.6.9"
 sqlparser = "0.12.0"
 workspace-hack = { path = "../workspace-hack"}

query/src/lib.rs

@@ -165,7 +165,6 @@ pub trait QueryChunk: QueryChunkMeta + Debug + Send + Sync {
         &self,
         predicate: &Predicate,
         selection: Selection<'_>,
-        delete_predicates: &[Arc<Predicate>],
     ) -> Result<SendableRecordBatchStream, Self::Error>;
 
     /// Returns true if data of this chunk is sorted

query/src/provider/physical.rs

@@ -118,22 +118,14 @@ impl<C: QueryChunk + 'static> ExecutionPlan for IOxReadFilterNode<C> {
         let selection_cols = restrict_selection(selection_cols, &chunk_table_schema);
         let selection = Selection::Some(&selection_cols);
 
-        let del_preds = chunk.delete_predicates();
+        let stream = chunk.read_filter(&self.predicate, selection).map_err(|e| {
-        let del_preds: Vec<Arc<Predicate>> = del_preds
+            DataFusionError::Execution(format!(
-            .iter()
+                "Error creating scan for table {} chunk {}: {}",
-            .map(|pred| Arc::new(pred.as_ref().clone().into()))
+                self.table_name,
-            .collect();
+                chunk.id(),
-
+                e
-        let stream = chunk
+            ))
-            .read_filter(&self.predicate, selection, &del_preds)
+        })?;
-            .map_err(|e| {
-                DataFusionError::Execution(format!(
-                    "Error creating scan for table {} chunk {}: {}",
-                    self.table_name,
-                    chunk.id(),
-                    e
-                ))
-            })?;
 
         // all CPU time is now done, pass in baseline metrics to adapter
         timer.done();

query/src/test.rs

@@ -834,7 +834,6 @@ impl QueryChunk for TestChunk {
         &self,
         predicate: &Predicate,
         _selection: Selection<'_>,
-        _delete_predicates: &[Arc<Predicate>],
     ) -> Result<SendableRecordBatchStream, Self::Error> {
         self.check_error()?;
 
@@ -938,9 +937,8 @@ pub async fn raw_data(chunks: &[Arc<TestChunk>]) -> Vec<RecordBatch> {
     for c in chunks {
         let pred = Predicate::default();
         let selection = Selection::All;
-        let delete_predicates: Vec<Arc<Predicate>> = vec![];
         let mut stream = c
-            .read_filter(&pred, selection, &delete_predicates)
+            .read_filter(&pred, selection)
             .expect("Error in read_filter");
         while let Some(b) = stream.next().await {
             let b = b.expect("Error in stream");

scripts/parquet-meta-iox

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -eu -o pipefail
+
+if ! command -v parquet &> /dev/null; then
+  echo "parquet CLI tool required"
+  echo "  brew install parquet-cli"
+  echo
+  echo "  (yes there is also linuxbrew; and probably you can find it somewhere else)"
+fi
+
+parquet meta "$1" \
+ | grep IOX:metadata: \
+ | awk '{print $2}' \
+ | base64 -d \
+ | ./scripts/prototxt \
+     decode influxdata.iox.preserved_catalog.v1.IoxMetadata

server/Cargo.toml

@@ -11,7 +11,7 @@ async-trait = "0.1"
 bytes = "1.0"
 chrono = "0.4"
 cache_loader_async = { version = "0.1.2", features = ["ttl-cache"] }
-crc32fast = "1.2.0"
+crc32fast = "1.2.2"
 data_types = { path = "../data_types" }
 datafusion = { path = "../datafusion" }
 datafusion_util = { path = "../datafusion_util" }

server/src/db.rs

@@ -1921,7 +1921,7 @@ mod tests {
 
     async fn collect_read_filter(chunk: &DbChunk) -> Vec<RecordBatch> {
         chunk
-            .read_filter(&Default::default(), Selection::All, &[])
+            .read_filter(&Default::default(), Selection::All)
             .unwrap()
             .collect::<Vec<_>>()
             .await

server/src/db/chunk.rs

@@ -224,7 +224,7 @@ impl DbChunk {
     /// predicate depends on the schema of the chunk. Callers should validate
     /// predicates against chunks they are to be executed against using
     /// `read_buffer::Chunk::validate_predicate`
-    pub fn to_rub_negated_predicates(
+    fn to_rub_negated_predicates(
         delete_predicates: &[Arc<Predicate>],
     ) -> Result<Vec<read_buffer::Predicate>> {
         let mut rub_preds: Vec<read_buffer::Predicate> = vec![];
@@ -346,7 +346,6 @@ impl QueryChunk for DbChunk {
         &self,
         predicate: &Predicate,
         selection: Selection<'_>,
-        delete_predicates: &[Arc<Predicate>],
     ) -> Result<SendableRecordBatchStream, Self::Error> {
         // Predicate is not required to be applied for correctness. We only pushed it down
         // when possible for performance gain
@@ -354,11 +353,16 @@ impl QueryChunk for DbChunk {
         debug!(?predicate, "Input Predicate to read_filter");
         self.access_recorder.record_access();
 
-        debug!(?delete_predicates, "Input Delete Predicates to read_filter");
+        let delete_predicates: Vec<_> = self
+            .meta
+            .delete_predicates
+            .iter()
+            .map(|pred| Arc::new(pred.as_ref().clone().into()))
+            .collect();
 
         // merge the negated delete predicates into the select predicate
         let mut pred_with_deleted_exprs = predicate.clone();
-        pred_with_deleted_exprs.merge_delete_predicates(delete_predicates);
+        pred_with_deleted_exprs.merge_delete_predicates(&delete_predicates);
         debug!(
             ?pred_with_deleted_exprs,
             "Input Predicate plus deleted ranges and deleted predicates"
@@ -381,7 +385,7 @@ impl QueryChunk for DbChunk {
                 debug!(?rb_predicate, "Predicate pushed down to RUB");
 
                 // combine all delete expressions to RUB's negated ones
-                let negated_delete_exprs = Self::to_rub_negated_predicates(delete_predicates)?
+                let negated_delete_exprs = Self::to_rub_negated_predicates(&delete_predicates)?
                     .into_iter()
                     // Any delete predicates unsupported by the Read Buffer will be elided.
                     .filter_map(|p| chunk.validate_predicate(p).ok())
@@ -581,7 +585,7 @@ mod tests {
 
         let t1 = time.inc(Duration::from_secs(1));
         snapshot
-            .read_filter(&Default::default(), Selection::All, &[])
+            .read_filter(&Default::default(), Selection::All)
             .unwrap();
         let m3 = chunk.access_recorder().get_metrics();
 

server/src/db/lifecycle/write.rs

@@ -24,8 +24,7 @@ use persistence_windows::{
     checkpoint::{DatabaseCheckpoint, PartitionCheckpoint, PersistCheckpointBuilder},
     persistence_windows::FlushHandle,
 };
-use predicate::predicate::Predicate;
+use query::QueryChunk;
-use query::{QueryChunk, QueryChunkMeta};
 use schema::selection::Selection;
 use snafu::ResultExt;
 use std::{future::Future, sync::Arc};
@@ -95,13 +94,8 @@ pub(super) fn write_chunk_to_object_store(
             collect_checkpoints(flush_handle.checkpoint(), &db.catalog);
 
         // Get RecordBatchStream of data from the read buffer chunk
-        let del_preds: Vec<Arc<Predicate>> = db_chunk
-            .delete_predicates()
-            .iter()
-            .map(|pred| Arc::new(pred.as_ref().clone().into()))
-            .collect();
         let stream = db_chunk
-            .read_filter(&Default::default(), Selection::All, &del_preds)
+            .read_filter(&Default::default(), Selection::All)
             .expect("read filter should be infallible");
 
         // check that the upcoming state change will very likely succeed

write_buffer/Cargo.toml

@@ -19,7 +19,7 @@ observability_deps = { path = "../observability_deps" }
 parking_lot = "0.11.2"
 pin-project = "1.0"
 prost = "0.8"
-rdkafka = "0.27.0"
+rdkafka = "0.28.0"
 time = { path = "../time" }
 tokio = { version = "1.13", features = ["macros", "fs"] }
 tokio-util = "0.6.9"