Influxdata
/
influxdb
mirror of https://github.com/influxdata/influxdb.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

feat: Add namespace_id to the parquet_files table; object store paths need it

pull/24376/head
Carol (Nichols || Goulding) 2022-03-24 11:51:45 -04:00
parent a373c90415
commit f3f792fd08
No known key found for this signature in database
GPG Key ID: E907EE5A736F87D4
9 changed files with 32 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
compactor/src/compact.rs
View File

@ -1499,6 +1499,7 @@ mod tests {
ParquetFile {
id: ParquetFileId::new(0),
sequencer_id: SequencerId::new(0),
namespace_id: NamespaceId::new(0),
table_id: TableId::new(0),
partition_id: PartitionId::new(0),
object_store_id: Uuid::new_v4(),
@ -1691,6 +1692,7 @@ mod tests {
let p1 = ParquetFileParams {
sequencer_id: sequencer.id,
namespace_id: namespace.id,
table_id: table.id,
partition_id: partition.id,
object_store_id: Uuid::new_v4(),
@ -1993,6 +1995,7 @@ mod tests {
// Prepare metadata in form of ParquetFileParams to get added with tombstone
let parquet = ParquetFileParams {
sequencer_id: sequencer.id,
namespace_id: namespace.id,
table_id: table.id,
partition_id: partition.id,
object_store_id: Uuid::new_v4(),

4
data_types2/src/lib.rs
View File

@ -710,6 +710,8 @@ pub struct ParquetFile {
pub id: ParquetFileId,
/// the sequencer that sequenced writes that went into this file
pub sequencer_id: SequencerId,
/// the namespace
pub namespace_id: NamespaceId,
/// the table
pub table_id: TableId,
/// the partition
@ -743,6 +745,8 @@ pub struct ParquetFile {
pub struct ParquetFileParams {
/// the sequencer that sequenced writes that went into this file
pub sequencer_id: SequencerId,
/// the namespace
pub namespace_id: NamespaceId,
/// the table
pub table_id: TableId,
/// the partition

1
ingester/src/data.rs
View File

@ -1897,6 +1897,7 @@ mod tests {
.unwrap();
let parquet_file_params = ParquetFileParams {
sequencer_id: sequencer.id,
namespace_id: namespace.id,
table_id: table.id,
partition_id: partition.id,
object_store_id: Uuid::new_v4(),

10
iox_catalog/migrations/20220324152729_add_namespace_id_to_parquet_file.sql Normal file
View File

@ -0,0 +1,10 @@
ALTER TABLE
IF EXISTS parquet_file
ADD COLUMN namespace_id INT NOT NULL;
ALTER TABLE
IF EXISTS parquet_file
ADD FOREIGN KEY (namespace_id)
REFERENCES namespace (id) MATCH SIMPLE
ON UPDATE NO ACTION
ON DELETE NO ACTION
NOT VALID;

6
iox_catalog/src/interface.rs
View File

@ -902,6 +902,7 @@ pub(crate) mod test_helpers {
.await
.unwrap();
let parquet_file_params = ParquetFileParams {
namespace_id: namespace.id,
sequencer_id: seq.id,
table_id: t.id,
partition_id: partition.id,
@ -1503,6 +1504,7 @@ pub(crate) mod test_helpers {
let parquet_file_params = ParquetFileParams {
sequencer_id: sequencer.id,
namespace_id: namespace.id,
table_id: partition.table_id,
partition_id: partition.id,
object_store_id: Uuid::new_v4(),
@ -1712,6 +1714,7 @@ pub(crate) mod test_helpers {
let parquet_file_params = ParquetFileParams {
sequencer_id: sequencer.id,
namespace_id: namespace.id,
table_id: partition.table_id,
partition_id: partition.id,
object_store_id: Uuid::new_v4(),
@ -2038,6 +2041,7 @@ pub(crate) mod test_helpers {
let parquet_file_params = ParquetFileParams {
sequencer_id: sequencer.id,
namespace_id: namespace.id,
table_id: partition.table_id,
partition_id: partition.id,
object_store_id: Uuid::new_v4(),
@ -2166,6 +2170,7 @@ pub(crate) mod test_helpers {
// Create a file with times entirely within the window
let parquet_file_params = ParquetFileParams {
sequencer_id: sequencer.id,
namespace_id: namespace.id,
table_id: partition.table_id,
partition_id: partition.id,
object_store_id: Uuid::new_v4(),
@ -2366,6 +2371,7 @@ pub(crate) mod test_helpers {
// Create a file with times entirely within the window
let parquet_file_params = ParquetFileParams {
sequencer_id: sequencer.id,
namespace_id: namespace.id,
table_id: partition.table_id,
partition_id: partition.id,
object_store_id: Uuid::new_v4(),

2
iox_catalog/src/mem.rs
View File

@ -902,6 +902,7 @@ impl ParquetFileRepo for MemTxn {
let ParquetFileParams {
sequencer_id,
namespace_id,
table_id,
partition_id,
object_store_id,
@ -926,6 +927,7 @@ impl ParquetFileRepo for MemTxn {
let parquet_file = ParquetFile {
id: ParquetFileId::new(stage.parquet_files.len() as i64 + 1),
sequencer_id,
namespace_id,
table_id,
partition_id,
object_store_id,

6
iox_catalog/src/postgres.rs
View File

@ -1377,6 +1377,7 @@ impl ParquetFileRepo for PostgresTxn {
async fn create(&mut self, parquet_file_params: ParquetFileParams) -> Result<ParquetFile> {
let ParquetFileParams {
sequencer_id,
namespace_id,
table_id,
partition_id,
object_store_id,
@ -1395,8 +1396,8 @@ impl ParquetFileRepo for PostgresTxn {
INSERT INTO parquet_file (
sequencer_id, table_id, partition_id, object_store_id, min_sequence_number,
max_sequence_number, min_time, max_time, file_size_bytes, parquet_metadata,
row_count, compaction_level, created_at )
VALUES ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13 )
row_count, compaction_level, created_at, namespace_id )
VALUES ( $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14 )
RETURNING *;
"#,
)
@ -1413,6 +1414,7 @@ RETURNING *;
.bind(row_count) // $11
.bind(INITIAL_COMPACTION_LEVEL) // $12
.bind(created_at) // $13
.bind(namespace_id) // $14
.fetch_one(&mut self.inner)
.await
.map_err(|e| {

1
iox_tests/src/util.rs
View File

@ -438,6 +438,7 @@ impl TestPartition {
let parquet_file_params = ParquetFileParams {
sequencer_id: self.sequencer.sequencer.id,
namespace_id: self.namespace.namespace.id,
table_id: self.table.table.id,
partition_id: self.partition.id,
object_store_id,

1
parquet_file/src/metadata.rs
View File

@ -662,6 +662,7 @@ impl IoxMetadata {
) -> ParquetFileParams {
ParquetFileParams {
sequencer_id: self.sequencer_id,
namespace_id: self.namespace_id,
table_id: self.table_id,
partition_id: self.partition_id,
object_store_id: self.object_store_id,