diff --git a/Cargo.lock b/Cargo.lock index 289485cd2e..1699eda220 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -716,8 +716,6 @@ dependencies = [ "delorean_storage", "delorean_test_helpers", "delorean_tsm", - "delorean_wal", - "delorean_wal_writer", "delorean_write_buffer", "dirs 3.0.1", "dotenv", @@ -744,6 +742,7 @@ dependencies = [ "tonic", "tracing", "tracing-futures", + "wal", ] [[package]] @@ -894,35 +893,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "delorean_wal" -version = "0.1.0" -dependencies = [ - "byteorder", - "crc32fast", - "delorean_test_helpers", - "futures", - "itertools 0.9.0", - "once_cell", - "regex", - "snafu", - "snap", - "tokio", -] - -[[package]] -name = "delorean_wal_writer" -version = "0.1.0" -dependencies = [ - "delorean_wal", - "futures", - "serde", - "serde_json", - "snafu", - "tokio", - "tracing", -] - [[package]] name = "delorean_write_buffer" version = "0.1.0" @@ -936,14 +906,13 @@ dependencies = [ "delorean_line_parser", "delorean_storage", "delorean_test_helpers", - "delorean_wal", - "delorean_wal_writer", "flatbuffers", "snafu", "sqlparser", "string-interner", "tokio", "tracing", + "wal", ] [[package]] @@ -3632,6 +3601,25 @@ dependencies = [ "libc", ] +[[package]] +name = "wal" +version = "0.1.0" +dependencies = [ + "byteorder", + "crc32fast", + "delorean_test_helpers", + "futures", + "itertools 0.9.0", + "once_cell", + "regex", + "serde", + "serde_json", + "snafu", + "snap", + "tokio", + "tracing", +] + [[package]] name = "walkdir" version = "2.3.1" diff --git a/Cargo.toml b/Cargo.toml index 144edabe2d..47ad4ba442 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,8 +21,7 @@ members = [ "delorean_test_helpers", "delorean_tsm", "delorean_storage", - "delorean_wal", - "delorean_wal_writer", + "wal", "delorean_write_buffer", "influxdb2_client", ] @@ -40,12 +39,11 @@ delorean_mem_qe = { path = "delorean_mem_qe" } delorean_parquet = { path = "delorean_parquet" } delorean_segment_store = { path = "delorean_segment_store" } packers = { path = "packers" } -delorean_wal = { path = "delorean_wal" } -delorean_wal_writer = { path = "delorean_wal_writer" } delorean_write_buffer = { path = "delorean_write_buffer" } object_store = { path = "object_store" } delorean_storage = { path = "delorean_storage" } delorean_tsm = { path = "delorean_tsm" } +wal = { path = "wal" } bytes = "0.5.4" hyper = "0.13" diff --git a/delorean_wal_writer/Cargo.toml b/delorean_wal_writer/Cargo.toml deleted file mode 100644 index 06ca489424..0000000000 --- a/delorean_wal_writer/Cargo.toml +++ /dev/null @@ -1,17 +0,0 @@ -[package] -name = "delorean_wal_writer" -version = "0.1.0" -authors = ["Andrew Lamb "] -edition = "2018" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -delorean_wal = { path = "../delorean_wal" } - -tokio = { version = "0.2", features = ["full"] } -serde_json = "1.0.44" -serde = { version = "1.0", features = ["derive"] } -futures = "0.3.1" -tracing = "0.1" -snafu = "0.6.2" diff --git a/delorean_write_buffer/Cargo.toml b/delorean_write_buffer/Cargo.toml index f52ddfa2c8..1ca984aced 100644 --- a/delorean_write_buffer/Cargo.toml +++ b/delorean_write_buffer/Cargo.toml @@ -11,8 +11,7 @@ data_types = { path = "../data_types" } delorean_generated_types = { path = "../delorean_generated_types" } delorean_line_parser = { path = "../delorean_line_parser" } delorean_storage = { path = "../delorean_storage" } -delorean_wal = { path = "../delorean_wal" } -delorean_wal_writer = { path = "../delorean_wal_writer" } +wal = { path = "../wal" } delorean_test_helpers = { path = "../delorean_test_helpers" } async-trait = "0.1" diff --git a/delorean_write_buffer/benches/benchmark.rs b/delorean_write_buffer/benches/benchmark.rs index 4a344406fd..7bbcf8f7cf 100644 --- a/delorean_write_buffer/benches/benchmark.rs +++ b/delorean_write_buffer/benches/benchmark.rs @@ -1,8 +1,8 @@ use criterion::{criterion_group, criterion_main, Criterion, Throughput}; use delorean_line_parser as line_parser; use delorean_storage::Database; -use delorean_wal::{Entry, WalBuilder}; use delorean_write_buffer::{restore_partitions_from_wal, Db}; +use wal::{Entry, WalBuilder}; type Error = Box; type Result = std::result::Result; diff --git a/delorean_write_buffer/src/database.rs b/delorean_write_buffer/src/database.rs index 95938c6c85..0eb74ea1a5 100644 --- a/delorean_write_buffer/src/database.rs +++ b/delorean_write_buffer/src/database.rs @@ -8,8 +8,10 @@ use delorean_storage::{ predicate::Predicate, Database, }; -use delorean_wal::WalBuilder; -use delorean_wal_writer::{start_wal_sync_task, Error as WalWriterError, WalDetails}; +use wal::{ + writer::{start_wal_sync_task, Error as WalWriterError, WalDetails}, + WalBuilder, +}; use crate::column::Column; use crate::partition::Partition; @@ -65,7 +67,7 @@ pub enum Error { #[snafu(display("Error opening WAL for database {}: {}", database, source))] LoadingWal { database: String, - source: delorean_wal::Error, + source: wal::Error, }, #[snafu(display("Error recovering WAL for database {}: {}", database, source))] diff --git a/delorean_write_buffer/src/partition.rs b/delorean_write_buffer/src/partition.rs index 0d9ed5e9df..54a359bd91 100644 --- a/delorean_write_buffer/src/partition.rs +++ b/delorean_write_buffer/src/partition.rs @@ -4,8 +4,8 @@ use delorean_arrow::{ datafusion::scalar::ScalarValue, }; use delorean_generated_types::wal as wb; -use delorean_wal::{Entry as WalEntry, Result as WalResult}; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use wal::{Entry as WalEntry, Result as WalResult}; use data_types::TIME_COLUMN_NAME; use delorean_storage::{ @@ -21,7 +21,7 @@ use snafu::{OptionExt, ResultExt, Snafu}; #[derive(Debug, Snafu)] pub enum Error { #[snafu(display("Could not read WAL entry: {}", source))] - WalEntryRead { source: delorean_wal::Error }, + WalEntryRead { source: wal::Error }, #[snafu(display("Partition {} not found", partition))] PartitionNotFound { partition: String }, diff --git a/delorean_wal/Cargo.toml b/wal/Cargo.toml similarity index 78% rename from delorean_wal/Cargo.toml rename to wal/Cargo.toml index 673a290fd7..42fbe8dbc4 100644 --- a/delorean_wal/Cargo.toml +++ b/wal/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "delorean_wal" +name = "wal" version = "0.1.0" authors = ["Paul Dix "] edition = "2018" @@ -12,8 +12,12 @@ snap = "1.0.0" regex = "1.3.7" itertools = "0.9.0" once_cell = "1.4.0" +futures = "0.3.4" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0.44" +tracing = "0.1" +tokio = { version = "0.2", features = ["full"] } + [dev-dependencies] delorean_test_helpers = { path = "../delorean_test_helpers" } -futures = "0.3.4" -tokio = { version = "0.2", features = ["full"] } diff --git a/delorean_wal/src/lib.rs b/wal/src/lib.rs similarity index 99% rename from delorean_wal/src/lib.rs rename to wal/src/lib.rs index 414e60498b..34113a4a92 100644 --- a/delorean_wal/src/lib.rs +++ b/wal/src/lib.rs @@ -2,12 +2,11 @@ #![warn( missing_copy_implementations, missing_debug_implementations, - missing_docs, clippy::explicit_iter_loop, clippy::use_self )] -//! # delorean_wal +//! # wal //! //! This crate provides a WAL tailored for delorean `Partition`s to optionally use. //! @@ -30,6 +29,9 @@ use std::{ path::{Path, PathBuf}, }; +/// WAL Writer and related utilties +pub mod writer; + /// Opaque public `Error` type #[derive(Debug, Snafu)] pub struct Error(InternalError); @@ -209,7 +211,7 @@ impl WalBuilder { /// /// ``` /// # fn example(root_path: &std::path::Path) -> Result<(), Box> { -/// use delorean_wal::{WalBuilder, WritePayload}; +/// use wal::{WalBuilder, WritePayload}; /// /// // This wal should be either protected with a mutex or moved into a single /// // worker thread that receives writes from channels. diff --git a/delorean_wal_writer/src/lib.rs b/wal/src/writer.rs similarity index 97% rename from delorean_wal_writer/src/lib.rs rename to wal/src/writer.rs index 676a608458..9163ca13e7 100644 --- a/delorean_wal_writer/src/lib.rs +++ b/wal/src/writer.rs @@ -5,7 +5,7 @@ clippy::explicit_iter_loop, clippy::use_self )] -use delorean_wal::{Error as WalError, SequenceNumber, WalBuilder, WritePayload}; +use crate::{Error as WalError, SequenceNumber, WalBuilder, WritePayload}; use futures::{channel::mpsc, SinkExt, StreamExt}; use snafu::{ResultExt, Snafu}; @@ -16,6 +16,7 @@ use tracing::{error, info}; use std::path::PathBuf; #[derive(Debug, Snafu)] +/// Error type pub enum Error { #[snafu(display("Wal Writer error using WAL: {}", source))] UnderlyingWalError { source: WalError }, @@ -154,6 +155,6 @@ pub async fn start_wal_sync_task(wal_builder: WalBuilder) -> Result mod tests { #[test] fn it_works_but_has_no_tests() { - // :thinkin_face: + // :thinking_face: } } diff --git a/delorean_wal/tests/delete-up-to.rs b/wal/tests/delete-up-to.rs similarity index 98% rename from delorean_wal/tests/delete-up-to.rs rename to wal/tests/delete-up-to.rs index 248a6732d0..277846517a 100644 --- a/delorean_wal/tests/delete-up-to.rs +++ b/wal/tests/delete-up-to.rs @@ -1,4 +1,4 @@ -use delorean_wal::{WalBuilder, WritePayload}; +use wal::{WalBuilder, WritePayload}; #[macro_use] mod helpers; diff --git a/delorean_wal/tests/file_rollover.rs b/wal/tests/file_rollover.rs similarity index 98% rename from delorean_wal/tests/file_rollover.rs rename to wal/tests/file_rollover.rs index b4c330b3f7..ff08a79ac6 100644 --- a/delorean_wal/tests/file_rollover.rs +++ b/wal/tests/file_rollover.rs @@ -1,5 +1,5 @@ -use delorean_wal::{WalBuilder, WritePayload}; use std::fs; +use wal::{WalBuilder, WritePayload}; #[macro_use] mod helpers; diff --git a/delorean_wal/tests/helpers/mod.rs b/wal/tests/helpers/mod.rs similarity index 98% rename from delorean_wal/tests/helpers/mod.rs rename to wal/tests/helpers/mod.rs index 84496f1d7e..b7313a8b7f 100644 --- a/delorean_wal/tests/helpers/mod.rs +++ b/wal/tests/helpers/mod.rs @@ -1,8 +1,8 @@ #![allow(unused_macros)] #![allow(dead_code)] -use delorean_wal::{Entry, WalBuilder}; use std::{fs, path::PathBuf}; +use wal::{Entry, WalBuilder}; type TestError = Box; pub type Result = std::result::Result; diff --git a/delorean_wal/tests/no_concurrency.rs b/wal/tests/no_concurrency.rs similarity index 94% rename from delorean_wal/tests/no_concurrency.rs rename to wal/tests/no_concurrency.rs index c88a6ae154..49e54ea1c6 100644 --- a/delorean_wal/tests/no_concurrency.rs +++ b/wal/tests/no_concurrency.rs @@ -1,4 +1,4 @@ -use delorean_wal::{WalBuilder, WritePayload}; +use wal::{WalBuilder, WritePayload}; type TestError = Box; type Result = std::result::Result; diff --git a/delorean_wal/tests/total_size.rs b/wal/tests/total_size.rs similarity index 95% rename from delorean_wal/tests/total_size.rs rename to wal/tests/total_size.rs index 3251aa440e..e9b68b456f 100644 --- a/delorean_wal/tests/total_size.rs +++ b/wal/tests/total_size.rs @@ -1,5 +1,5 @@ -use delorean_wal::{WalBuilder, WritePayload}; use std::fs; +use wal::{WalBuilder, WritePayload}; #[macro_use] mod helpers; @@ -52,7 +52,7 @@ fn total_size() -> Result { assert_eq!(wal.total_size(), total_file_size_before_delete); // Pretend the process restarts - let wal: delorean_wal::Wal = builder.wal()?; + let wal = builder.wal()?; // Total size should be that of all the files, so without the file deleted out-of-band assert_eq!(wal.total_size(), helpers::total_size_on_disk(&dir.as_ref()));