From 8399d2a1597deb0d2e27d18c277ff13135867c57 Mon Sep 17 00:00:00 2001 From: Andrew Lamb Date: Thu, 5 Nov 2020 08:52:22 -0500 Subject: [PATCH] refactor: rename delorean_table to packers (#409) --- Cargo.lock | 38 +++++++++++----------- Cargo.toml | 4 +-- benches/line_protocol_to_parquet.rs | 2 +- benches/packers.rs | 2 +- delorean_ingest/Cargo.toml | 2 +- delorean_ingest/src/lib.rs | 17 ++++------ delorean_mem_qe/Cargo.toml | 2 +- delorean_mem_qe/src/encoding.rs | 6 ++-- delorean_parquet/Cargo.toml | 2 +- delorean_parquet/src/stats.rs | 2 +- delorean_parquet/src/writer.rs | 8 ++--- delorean_parquet/tests/read_write.rs | 2 +- delorean_segment_store/Cargo.toml | 2 +- {delorean_table => packers}/Cargo.toml | 2 +- {delorean_table => packers}/src/lib.rs | 2 +- {delorean_table => packers}/src/packers.rs | 0 {delorean_table => packers}/src/sorter.rs | 0 {delorean_table => packers}/src/stats.rs | 2 +- src/commands/convert.rs | 2 +- src/commands/input.rs | 2 +- src/commands/stats.rs | 2 +- 21 files changed, 49 insertions(+), 52 deletions(-) rename {delorean_table => packers}/Cargo.toml (95%) rename {delorean_table => packers}/src/lib.rs (98%) rename {delorean_table => packers}/src/packers.rs (100%) rename {delorean_table => packers}/src/sorter.rs (100%) rename {delorean_table => packers}/src/stats.rs (99%) diff --git a/Cargo.lock b/Cargo.lock index 29050e08d9..ec2031d99f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -715,7 +715,6 @@ dependencies = [ "delorean_parquet", "delorean_segment_store", "delorean_storage", - "delorean_table", "delorean_test_helpers", "delorean_tsm", "delorean_wal", @@ -730,6 +729,7 @@ dependencies = [ "hyper", "influxdb2_client", "libflate", + "packers", "predicates", "prost", "prost-types", @@ -793,11 +793,11 @@ version = "0.1.0" dependencies = [ "data_types", "delorean_line_parser", - "delorean_table", "delorean_test_helpers", "delorean_tsm", "env_logger", "libflate", + "packers", "snafu", "tracing", ] @@ -823,9 +823,9 @@ dependencies = [ "croaring", "crossbeam", "delorean_arrow", - "delorean_table", "env_logger", "human_format", + "packers", "snafu", "tracing", ] @@ -853,8 +853,8 @@ version = "0.1.0" dependencies = [ "data_types", "delorean_arrow", - "delorean_table", "delorean_test_helpers", + "packers", "snafu", "tracing", ] @@ -866,7 +866,7 @@ dependencies = [ "criterion", "croaring", "delorean_arrow", - "delorean_table", + "packers", "rand", ] @@ -887,20 +887,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "delorean_table" -version = "0.1.0" -dependencies = [ - "data_types", - "delorean_arrow", - "delorean_test_helpers", - "delorean_tsm", - "human_format", - "rand", - "snafu", - "tracing", -] - [[package]] name = "delorean_test_helpers" version = "0.1.0" @@ -2083,6 +2069,20 @@ dependencies = [ "libm", ] +[[package]] +name = "packers" +version = "0.1.0" +dependencies = [ + "data_types", + "delorean_arrow", + "delorean_test_helpers", + "delorean_tsm", + "human_format", + "rand", + "snafu", + "tracing", +] + [[package]] name = "parquet" version = "3.0.0-SNAPSHOT" diff --git a/Cargo.toml b/Cargo.toml index 0d07b7319c..9b34ca5417 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ members = [ "delorean_mem_qe", "delorean_parquet", "delorean_segment_store", - "delorean_table", + "packers", "delorean_test_helpers", "delorean_tsm", "delorean_storage", @@ -39,7 +39,7 @@ delorean_line_parser = { path = "delorean_line_parser" } delorean_mem_qe = { path = "delorean_mem_qe" } delorean_parquet = { path = "delorean_parquet" } delorean_segment_store = { path = "delorean_segment_store" } -delorean_table = { path = "delorean_table" } +packers = { path = "packers" } delorean_wal = { path = "delorean_wal" } delorean_wal_writer = { path = "delorean_wal_writer" } delorean_write_buffer = { path = "delorean_write_buffer" } diff --git a/benches/line_protocol_to_parquet.rs b/benches/line_protocol_to_parquet.rs index ec38d2c12d..ee79fec143 100644 --- a/benches/line_protocol_to_parquet.rs +++ b/benches/line_protocol_to_parquet.rs @@ -6,7 +6,7 @@ use delorean_parquet::{ writer::{CompressionLevel, DeloreanParquetTableWriter}, TryClone, }; -use delorean_table::{DeloreanTableWriter, DeloreanTableWriterSource, Error as TableError}; +use packers::{DeloreanTableWriter, DeloreanTableWriterSource, Error as TableError}; use std::time::Duration; use std::io::{Seek, SeekFrom, Write}; diff --git a/benches/packers.rs b/benches/packers.rs index 6ff3e88eb9..9dccd2364c 100644 --- a/benches/packers.rs +++ b/benches/packers.rs @@ -4,7 +4,7 @@ use rand::Rng; use std::convert::TryFrom; use std::mem; -use delorean_table::Packer; +use packers::Packer; const BATCH_SIZES: [usize; 6] = [10, 100, 1000, 10_000, 100_000, 1_000_000]; diff --git a/delorean_ingest/Cargo.toml b/delorean_ingest/Cargo.toml index e46d630284..0bb040d860 100644 --- a/delorean_ingest/Cargo.toml +++ b/delorean_ingest/Cargo.toml @@ -11,7 +11,7 @@ snafu = "0.6.2" env_logger = "0.7.1" tracing = "0.1" delorean_line_parser = { path = "../delorean_line_parser" } -delorean_table = { path = "../delorean_table" } +packers = { path = "../packers" } data_types = { path = "../data_types" } delorean_tsm = { path = "../delorean_tsm" } diff --git a/delorean_ingest/src/lib.rs b/delorean_ingest/src/lib.rs index e71f3de837..bdee7c3ea8 100644 --- a/delorean_ingest/src/lib.rs +++ b/delorean_ingest/src/lib.rs @@ -11,15 +11,14 @@ use data_types::table_schema::{DataType, Schema, SchemaBuilder}; use delorean_line_parser::{FieldValue, ParsedLine}; -use delorean_table::{ - packers::{Packer, Packers}, - ByteArray, DeloreanTableWriter, DeloreanTableWriterSource, Error as TableError, -}; use delorean_tsm::{ mapper::{ColumnData, MeasurementTable, TSMMeasurementMapper}, reader::{BlockDecoder, TSMBlockReader, TSMIndexReader}, BlockType, TSMError, }; +use packers::{ + ByteArray, DeloreanTableWriter, DeloreanTableWriterSource, Error as TableError, Packer, Packers, +}; use snafu::{ensure, OptionExt, ResultExt, Snafu}; use std::{ collections::{BTreeMap, BTreeSet}, @@ -47,7 +46,7 @@ impl Default for ConversionSettings { } } -/// Converts `ParsedLines` into the delorean_table internal columnar +/// Converts `ParsedLines` into the packers internal columnar /// data format and then passes that converted data to a /// `DeloreanTableWriter` pub struct LineProtocolConverter<'a> { @@ -601,7 +600,7 @@ fn pack_lines<'a>(schema: &Schema, lines: &[ParsedLine<'a>]) -> Vec { // // let sort = [0, 7, 6, 12]; // // let sort = [8, 4, 9, 0, 1, 7, 10, 6, 5, 2, 3, 12]; // let sort = [3, 2, 5, 6, 10, 7, 1, 0, 9, 4, 8, 12]; -// delorean_table::sorter::sort(&mut chunked_packers, &sort).unwrap(); +// packers::sorter::sort(&mut chunked_packers, &sort).unwrap(); // println!( // "Writing {:?} packers with size: {:?}", @@ -656,7 +655,7 @@ fn pack_lines<'a>(schema: &Schema, lines: &[ParsedLine<'a>]) -> Vec { // w.write(&record_batch).unwrap(); // } -/// Converts one or more TSM files into the delorean_table internal columnar +/// Converts one or more TSM files into the packers internal columnar /// data format and then passes that converted data to a `DeloreanTableWriter`. pub struct TSMFileConverter { table_writer_source: Box, @@ -1088,14 +1087,12 @@ impl std::fmt::Debug for TSMFileConverter { mod delorean_ingest_tests { use super::*; use data_types::table_schema::ColumnDefinition; - use delorean_table::{ - DeloreanTableWriter, DeloreanTableWriterSource, Error as TableError, Packers, - }; use delorean_test_helpers::approximately_equal; use delorean_tsm::{ reader::{BlockData, MockBlockDecoder}, Block, }; + use packers::{DeloreanTableWriter, DeloreanTableWriterSource, Error as TableError, Packers}; use libflate::gzip; use std::fs::File; diff --git a/delorean_mem_qe/Cargo.toml b/delorean_mem_qe/Cargo.toml index 2abbf76247..ae23363ad4 100644 --- a/delorean_mem_qe/Cargo.toml +++ b/delorean_mem_qe/Cargo.toml @@ -9,7 +9,7 @@ edition = "2018" [dependencies] delorean_arrow = { path = "../delorean_arrow" } -delorean_table = { path = "../delorean_table" } +packers = { path = "../packers" } snafu = "0.6.8" croaring = "0.4.5" crossbeam = "0.7.3" diff --git a/delorean_mem_qe/src/encoding.rs b/delorean_mem_qe/src/encoding.rs index f39b7e2750..80c8485842 100644 --- a/delorean_mem_qe/src/encoding.rs +++ b/delorean_mem_qe/src/encoding.rs @@ -1031,13 +1031,13 @@ impl std::convert::From> for DictionaryRLE { } // TODO(edd): improve perf here.... -impl std::convert::From<&delorean_table::Packer> for DictionaryRLE { - fn from(p: &delorean_table::Packer) -> Self { +impl std::convert::From<&packers::Packer> for DictionaryRLE { + fn from(p: &packers::Packer) -> Self { let mut drle = Self::new(); for v in p.values() { let s = v .clone() - .unwrap_or_else(|| delorean_table::ByteArray::from("NULL")); + .unwrap_or_else(|| packers::ByteArray::from("NULL")); drle.push(s.as_utf8().unwrap()); } drle diff --git a/delorean_parquet/Cargo.toml b/delorean_parquet/Cargo.toml index df71e6c16e..e03d2cab45 100644 --- a/delorean_parquet/Cargo.toml +++ b/delorean_parquet/Cargo.toml @@ -11,7 +11,7 @@ snafu = "0.6.2" tracing = "0.1" delorean_arrow = { path = "../delorean_arrow" } -delorean_table = { path = "../delorean_table" } +packers = { path = "../packers" } data_types = { path = "../data_types" } diff --git a/delorean_parquet/src/stats.rs b/delorean_parquet/src/stats.rs index 3a43395a42..d2363cf1d0 100644 --- a/delorean_parquet/src/stats.rs +++ b/delorean_parquet/src/stats.rs @@ -3,7 +3,7 @@ use delorean_arrow::parquet::{ basic::{Compression, Encoding}, file::reader::{FileReader, SerializedFileReader}, }; -use delorean_table::{ +use packers::{ stats::{ColumnStatsBuilder, FileStats, FileStatsBuilder}, Name, }; diff --git a/delorean_parquet/src/writer.rs b/delorean_parquet/src/writer.rs index 4cb1bd3c0c..e11bea1344 100644 --- a/delorean_parquet/src/writer.rs +++ b/delorean_parquet/src/writer.rs @@ -20,7 +20,7 @@ use std::{ use tracing::debug; use crate::metadata::parquet_schema_as_string; -use delorean_table::{DeloreanTableWriter, Error as TableError, Packers}; +use packers::{DeloreanTableWriter, Error as TableError, Packers}; #[derive(Debug, Snafu)] pub enum Error { @@ -75,7 +75,7 @@ impl FromStr for CompressionLevel { } /// A `DeloreanParquetTableWriter` is used for writing batches of rows -/// represented using the structures in `delorean_table` to parquet files. +/// parquet files. pub struct DeloreanParquetTableWriter where W: ParquetWriter, @@ -95,8 +95,8 @@ where /// # use std::fs; /// # use data_types::table_schema; /// # use data_types::table_schema::DataType; - /// # use delorean_table::DeloreanTableWriter; - /// # use delorean_table::packers::{Packer, Packers}; + /// # use packers::DeloreanTableWriter; + /// # use packers::{Packer, Packers}; /// # use delorean_parquet::writer::{DeloreanParquetTableWriter, CompressionLevel}; /// # use delorean_arrow::parquet::data_type::ByteArray; /// diff --git a/delorean_parquet/tests/read_write.rs b/delorean_parquet/tests/read_write.rs index 84717af5af..61dc6fbcec 100644 --- a/delorean_parquet/tests/read_write.rs +++ b/delorean_parquet/tests/read_write.rs @@ -1,5 +1,5 @@ use delorean_parquet::writer::{CompressionLevel, DeloreanParquetTableWriter}; -use delorean_table::{packers::Packer, DeloreanTableWriter, Packers}; +use packers::{DeloreanTableWriter, Packer, Packers}; use delorean_arrow::parquet::data_type::ByteArray; use std::fs; diff --git a/delorean_segment_store/Cargo.toml b/delorean_segment_store/Cargo.toml index 6746791e43..d824235447 100644 --- a/delorean_segment_store/Cargo.toml +++ b/delorean_segment_store/Cargo.toml @@ -9,7 +9,7 @@ edition = "2018" [dependencies] delorean_arrow = { path = "../delorean_arrow" } -delorean_table = { path = "../delorean_table" } +packers = { path = "../packers" } croaring = "0.4.5" [dev-dependencies] diff --git a/delorean_table/Cargo.toml b/packers/Cargo.toml similarity index 95% rename from delorean_table/Cargo.toml rename to packers/Cargo.toml index 846d04fed1..945a760b81 100644 --- a/delorean_table/Cargo.toml +++ b/packers/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "delorean_table" +name = "packers" version = "0.1.0" authors = ["Andrew Lamb "] edition = "2018" diff --git a/delorean_table/src/lib.rs b/packers/src/lib.rs similarity index 98% rename from delorean_table/src/lib.rs rename to packers/src/lib.rs index 418636f4c5..3c4c7c6894 100644 --- a/delorean_table/src/lib.rs +++ b/packers/src/lib.rs @@ -12,9 +12,9 @@ pub mod stats; use snafu::Snafu; +pub use crate::packers::{Packer, Packers}; use data_types::table_schema::Schema; pub use delorean_arrow::parquet::data_type::ByteArray; -pub use packers::{Packer, Packers}; use std::borrow::Cow; diff --git a/delorean_table/src/packers.rs b/packers/src/packers.rs similarity index 100% rename from delorean_table/src/packers.rs rename to packers/src/packers.rs diff --git a/delorean_table/src/sorter.rs b/packers/src/sorter.rs similarity index 100% rename from delorean_table/src/sorter.rs rename to packers/src/sorter.rs diff --git a/delorean_table/src/stats.rs b/packers/src/stats.rs similarity index 99% rename from delorean_table/src/stats.rs rename to packers/src/stats.rs index 95490690ed..e2f4b1f693 100644 --- a/delorean_table/src/stats.rs +++ b/packers/src/stats.rs @@ -69,7 +69,7 @@ impl fmt::Display for ColumnStats { /// # Example: /// ``` /// use data_types::table_schema::DataType; -/// use delorean_table::stats::ColumnStatsBuilder; +/// use packers::stats::ColumnStatsBuilder; /// /// let stats = ColumnStatsBuilder::new("My Column", 3, DataType::Float) /// .compression("GZIP") diff --git a/src/commands/convert.rs b/src/commands/convert.rs index 95bb7b74fb..426d39af1f 100644 --- a/src/commands/convert.rs +++ b/src/commands/convert.rs @@ -5,7 +5,7 @@ use delorean_ingest::{ use delorean_line_parser::parse_lines; use delorean_parquet::writer::Error as ParquetWriterError; use delorean_parquet::writer::{CompressionLevel, DeloreanParquetTableWriter}; -use delorean_table::{DeloreanTableWriter, DeloreanTableWriterSource, Error as TableError}; +use packers::{DeloreanTableWriter, DeloreanTableWriterSource, Error as TableError}; use snafu::{ResultExt, Snafu}; use std::{ convert::TryInto, diff --git a/src/commands/input.rs b/src/commands/input.rs index 47d2a7222c..a41d09768c 100644 --- a/src/commands/input.rs +++ b/src/commands/input.rs @@ -1,8 +1,8 @@ use delorean_arrow::parquet::file::serialized_reader::{FileSource, SliceableCursor}; use delorean_parquet::ChunkReader; -use delorean_table::Name; /// Module to handle input files (and maybe urls?) use libflate::gzip; +use packers::Name; use snafu::{ResultExt, Snafu}; use std::{ borrow::Cow, diff --git a/src/commands/stats.rs b/src/commands/stats.rs index 6837c66349..0cee8ed822 100644 --- a/src/commands/stats.rs +++ b/src/commands/stats.rs @@ -1,7 +1,7 @@ //! This module contains code to report compression statistics for storage files use delorean_parquet::{error::Error as DeloreanParquetError, stats as parquet_stats}; -use delorean_table::{ +use packers::{ stats::{FileSetStatsBuilder, FileStats}, Name, };