fix: Disable parquet file filtering in the querier based on max seq num in RPC write mode (#6443)

Connects to #6421.

Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com>
pull/24376/head
Carol (Nichols || Goulding) 2022-12-19 13:01:21 -05:00 committed by GitHub
parent 9b22ede3f0
commit 200f4fe9bd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 13 additions and 1 deletions

View File

@ -337,6 +337,7 @@ impl QuerierTable {
Arc::clone(&self.table_name),
Arc::clone(&self.namespace_name),
Arc::clone(self.chunk_adapter.catalog_cache()),
self.rpc_write(),
);
// create parquet files

View File

@ -33,6 +33,9 @@ pub struct Reconciler {
table_name: Arc<str>,
namespace_name: Arc<str>,
catalog_cache: Arc<CatalogCache>,
/// Whether the querier is running in RPC write mode. This can be removed when the switch to
/// the RPC write design is complete.
rpc_write: bool,
}
impl Reconciler {
@ -40,11 +43,13 @@ impl Reconciler {
table_name: Arc<str>,
namespace_name: Arc<str>,
catalog_cache: Arc<CatalogCache>,
rpc_write: bool,
) -> Self {
Self {
table_name,
namespace_name,
catalog_cache,
rpc_write,
}
}
@ -113,7 +118,13 @@ impl Reconciler {
.push(tombstone);
}
let parquet_files = filter_parquet_files(ingester_partitions, parquet_files)?;
// Do not filter based on max sequence number in RPC write mode because sequence numbers
// are no longer relevant
let parquet_files = if self.rpc_write {
parquet_files
} else {
filter_parquet_files(ingester_partitions, parquet_files)?
};
debug!(
parquet_ids=?parquet_files.iter().map(|f| f.meta().parquet_file_id()).collect::<Vec<_>>(),