feat(ingester2): New objecstore paths will have no shard id (#6735)

Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com>
pull/24376/head
Marko Mikulicic 2023-01-27 18:16:54 +01:00 committed by GitHub
parent f6e7724d19
commit db7e6335ca
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 35 additions and 8 deletions

View File

@ -24,6 +24,9 @@ use data_types::{NamespaceId, ParquetFile, ParquetFileParams, PartitionId, Shard
use object_store::path::Path; use object_store::path::Path;
use uuid::Uuid; use uuid::Uuid;
// Ingester2 creates partitions in this shard.
const TRANSITION_SHARD_ID: ShardId = ShardId::new(1234);
/// Location of a Parquet file within a namespace's object store. /// Location of a Parquet file within a namespace's object store.
/// The exact format is an implementation detail and is subject to change. /// The exact format is an implementation detail and is subject to change.
#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd)] #[derive(Debug, Clone, Copy, Eq, PartialEq, Hash, Ord, PartialOrd)]
@ -62,7 +65,14 @@ impl ParquetFilePath {
partition_id, partition_id,
object_store_id, object_store_id,
} = self; } = self;
if shard_id == &TRANSITION_SHARD_ID {
Path::from_iter([
namespace_id.to_string().as_str(),
table_id.to_string().as_str(),
partition_id.to_string().as_str(),
&format!("{}.parquet", object_store_id),
])
} else {
Path::from_iter([ Path::from_iter([
namespace_id.to_string().as_str(), namespace_id.to_string().as_str(),
table_id.to_string().as_str(), table_id.to_string().as_str(),
@ -71,6 +81,7 @@ impl ParquetFilePath {
&format!("{}.parquet", object_store_id), &format!("{}.parquet", object_store_id),
]) ])
} }
}
/// Get object store ID. /// Get object store ID.
pub fn objest_store_id(&self) -> Uuid { pub fn objest_store_id(&self) -> Uuid {
@ -147,4 +158,20 @@ mod tests {
"1/2/3/4/00000000-0000-0000-0000-000000000000.parquet".to_string(), "1/2/3/4/00000000-0000-0000-0000-000000000000.parquet".to_string(),
); );
} }
#[test]
fn parquet_file_without_shard_id() {
let pfp = ParquetFilePath::new(
NamespaceId::new(1),
TableId::new(2),
TRANSITION_SHARD_ID,
PartitionId::new(4),
Uuid::nil(),
);
let path = pfp.object_store_path();
assert_eq!(
path.to_string(),
"1/2/4/00000000-0000-0000-0000-000000000000.parquet".to_string(),
);
}
} }