refactor: rename delorean_wal --> wal, conslidate wal_writer (#411)

Cargo.lock

@@ -716,8 +716,6 @@ dependencies = [
  "delorean_storage",
  "delorean_test_helpers",
  "delorean_tsm",
- "delorean_wal",
- "delorean_wal_writer",
  "delorean_write_buffer",
  "dirs 3.0.1",
  "dotenv",
@@ -744,6 +742,7 @@ dependencies = [
  "tonic",
  "tracing",
  "tracing-futures",
+ "wal",
 ]
 
 [[package]]
@@ -894,35 +893,6 @@ dependencies = [
  "tracing",
 ]
 
-[[package]]
-name = "delorean_wal"
-version = "0.1.0"
-dependencies = [
- "byteorder",
- "crc32fast",
- "delorean_test_helpers",
- "futures",
- "itertools 0.9.0",
- "once_cell",
- "regex",
- "snafu",
- "snap",
- "tokio",
-]
-
-[[package]]
-name = "delorean_wal_writer"
-version = "0.1.0"
-dependencies = [
- "delorean_wal",
- "futures",
- "serde",
- "serde_json",
- "snafu",
- "tokio",
- "tracing",
-]
-
 [[package]]
 name = "delorean_write_buffer"
 version = "0.1.0"
@@ -936,14 +906,13 @@ dependencies = [
  "delorean_line_parser",
  "delorean_storage",
  "delorean_test_helpers",
- "delorean_wal",
- "delorean_wal_writer",
  "flatbuffers",
  "snafu",
  "sqlparser",
  "string-interner",
  "tokio",
  "tracing",
+ "wal",
 ]
 
 [[package]]
@@ -3632,6 +3601,25 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "wal"
+version = "0.1.0"
+dependencies = [
+ "byteorder",
+ "crc32fast",
+ "delorean_test_helpers",
+ "futures",
+ "itertools 0.9.0",
+ "once_cell",
+ "regex",
+ "serde",
+ "serde_json",
+ "snafu",
+ "snap",
+ "tokio",
+ "tracing",
+]
+
 [[package]]
 name = "walkdir"
 version = "2.3.1"

Cargo.toml

@@ -21,8 +21,7 @@ members = [
     "delorean_test_helpers",
     "delorean_tsm",
     "delorean_storage",
-    "delorean_wal",
-    "delorean_wal_writer",
+    "wal",
     "delorean_write_buffer",
     "influxdb2_client",
 ]
@@ -40,12 +39,11 @@ delorean_mem_qe = { path = "delorean_mem_qe" }
 delorean_parquet = { path = "delorean_parquet" }
 delorean_segment_store = { path = "delorean_segment_store" }
 packers = { path = "packers" }
-delorean_wal = { path = "delorean_wal" }
-delorean_wal_writer = { path = "delorean_wal_writer" }
 delorean_write_buffer = { path = "delorean_write_buffer" }
 object_store = { path = "object_store" }
 delorean_storage = { path = "delorean_storage" }
 delorean_tsm = { path = "delorean_tsm" }
+wal = { path = "wal" }
 
 bytes = "0.5.4"
 hyper = "0.13"

delorean_wal_writer/Cargo.toml

@@ -1,17 +0,0 @@
-[package]
-name = "delorean_wal_writer"
-version = "0.1.0"
-authors = ["Andrew Lamb <andrew@nerdnetworks.org>"]
-edition = "2018"
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[dependencies]
-delorean_wal = { path = "../delorean_wal" }
-
-tokio = { version = "0.2", features = ["full"] }
-serde_json = "1.0.44"
-serde = { version = "1.0", features = ["derive"] }
-futures = "0.3.1"
-tracing = "0.1"
-snafu = "0.6.2"

delorean_write_buffer/Cargo.toml

@@ -11,8 +11,7 @@ data_types = { path = "../data_types" }
 delorean_generated_types = { path = "../delorean_generated_types" }
 delorean_line_parser = { path = "../delorean_line_parser" }
 delorean_storage = { path = "../delorean_storage" }
-delorean_wal = { path = "../delorean_wal" }
-delorean_wal_writer = { path = "../delorean_wal_writer" }
+wal = { path = "../wal" }
 delorean_test_helpers = { path = "../delorean_test_helpers" }
 
 async-trait = "0.1"

delorean_write_buffer/benches/benchmark.rs

@@ -1,8 +1,8 @@
 use criterion::{criterion_group, criterion_main, Criterion, Throughput};
 use delorean_line_parser as line_parser;
 use delorean_storage::Database;
-use delorean_wal::{Entry, WalBuilder};
 use delorean_write_buffer::{restore_partitions_from_wal, Db};
+use wal::{Entry, WalBuilder};
 
 type Error = Box<dyn std::error::Error + Send + Sync + 'static>;
 type Result<T, E = Error> = std::result::Result<T, E>;

delorean_write_buffer/src/database.rs

@@ -8,8 +8,10 @@ use delorean_storage::{
     predicate::Predicate,
     Database,
 };
-use delorean_wal::WalBuilder;
-use delorean_wal_writer::{start_wal_sync_task, Error as WalWriterError, WalDetails};
+use wal::{
+    writer::{start_wal_sync_task, Error as WalWriterError, WalDetails},
+    WalBuilder,
+};
 
 use crate::column::Column;
 use crate::partition::Partition;
@@ -65,7 +67,7 @@ pub enum Error {
     #[snafu(display("Error opening WAL for database {}: {}", database, source))]
     LoadingWal {
         database: String,
-        source: delorean_wal::Error,
+        source: wal::Error,
     },
 
     #[snafu(display("Error recovering WAL for database {}: {}", database, source))]

delorean_write_buffer/src/partition.rs

@@ -4,8 +4,8 @@ use delorean_arrow::{
     datafusion::scalar::ScalarValue,
 };
 use delorean_generated_types::wal as wb;
-use delorean_wal::{Entry as WalEntry, Result as WalResult};
 use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
+use wal::{Entry as WalEntry, Result as WalResult};
 
 use data_types::TIME_COLUMN_NAME;
 use delorean_storage::{
@@ -21,7 +21,7 @@ use snafu::{OptionExt, ResultExt, Snafu};
 #[derive(Debug, Snafu)]
 pub enum Error {
     #[snafu(display("Could not read WAL entry: {}", source))]
-    WalEntryRead { source: delorean_wal::Error },
+    WalEntryRead { source: wal::Error },
 
     #[snafu(display("Partition {} not found", partition))]
     PartitionNotFound { partition: String },

wal/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "delorean_wal"
+name = "wal"
 version = "0.1.0"
 authors = ["Paul Dix <paul@pauldix.net>"]
 edition = "2018"
@@ -12,8 +12,12 @@ snap = "1.0.0"
 regex = "1.3.7"
 itertools = "0.9.0"
 once_cell = "1.4.0"
+futures = "0.3.4"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0.44"
+tracing = "0.1"
+tokio = { version = "0.2", features = ["full"] }
+
 
 [dev-dependencies]
 delorean_test_helpers = { path = "../delorean_test_helpers" }
-futures = "0.3.4"
-tokio = { version = "0.2", features = ["full"] }

wal/src/lib.rs

@@ -2,12 +2,11 @@
 #![warn(
     missing_copy_implementations,
     missing_debug_implementations,
-    missing_docs,
     clippy::explicit_iter_loop,
     clippy::use_self
 )]
 
-//! # delorean_wal
+//! # wal
 //!
 //! This crate provides a WAL tailored for delorean `Partition`s to optionally use.
 //!
@@ -30,6 +29,9 @@ use std::{
     path::{Path, PathBuf},
 };
 
+/// WAL Writer and related utilties
+pub mod writer;
+
 /// Opaque public `Error` type
 #[derive(Debug, Snafu)]
 pub struct Error(InternalError);
@@ -209,7 +211,7 @@ impl WalBuilder {
 ///
 /// ```
 /// # fn example(root_path: &std::path::Path) -> Result<(), Box<dyn std::error::Error>> {
-/// use delorean_wal::{WalBuilder, WritePayload};
+/// use wal::{WalBuilder, WritePayload};
 ///
 /// // This wal should be either protected with a mutex or moved into a single
 /// // worker thread that receives writes from channels.

wal/src/writer.rs

@@ -5,7 +5,7 @@
     clippy::explicit_iter_loop,
     clippy::use_self
 )]
-use delorean_wal::{Error as WalError, SequenceNumber, WalBuilder, WritePayload};
+use crate::{Error as WalError, SequenceNumber, WalBuilder, WritePayload};
 
 use futures::{channel::mpsc, SinkExt, StreamExt};
 use snafu::{ResultExt, Snafu};
@@ -16,6 +16,7 @@ use tracing::{error, info};
 use std::path::PathBuf;
 
 #[derive(Debug, Snafu)]
+/// Error type
 pub enum Error {
     #[snafu(display("Wal Writer error using WAL: {}", source))]
     UnderlyingWalError { source: WalError },
@@ -154,6 +155,6 @@ pub async fn start_wal_sync_task(wal_builder: WalBuilder) -> Result<WalDetails>
 mod tests {
     #[test]
     fn it_works_but_has_no_tests() {
-        // :thinkin_face:
+        // :thinking_face:
     }
 }

wal/tests/delete-up-to.rs

@@ -1,4 +1,4 @@
-use delorean_wal::{WalBuilder, WritePayload};
+use wal::{WalBuilder, WritePayload};
 
 #[macro_use]
 mod helpers;

wal/tests/file_rollover.rs

@@ -1,5 +1,5 @@
-use delorean_wal::{WalBuilder, WritePayload};
 use std::fs;
+use wal::{WalBuilder, WritePayload};
 
 #[macro_use]
 mod helpers;

wal/tests/helpers/mod.rs

@@ -1,8 +1,8 @@
 #![allow(unused_macros)]
 #![allow(dead_code)]
 
-use delorean_wal::{Entry, WalBuilder};
 use std::{fs, path::PathBuf};
+use wal::{Entry, WalBuilder};
 
 type TestError = Box<dyn std::error::Error + Send + Sync + 'static>;
 pub type Result<T = (), E = TestError> = std::result::Result<T, E>;

wal/tests/no_concurrency.rs

@@ -1,4 +1,4 @@
-use delorean_wal::{WalBuilder, WritePayload};
+use wal::{WalBuilder, WritePayload};
 
 type TestError = Box<dyn std::error::Error + Send + Sync + 'static>;
 type Result<T = (), E = TestError> = std::result::Result<T, E>;

wal/tests/total_size.rs

@@ -1,5 +1,5 @@
-use delorean_wal::{WalBuilder, WritePayload};
 use std::fs;
+use wal::{WalBuilder, WritePayload};
 
 #[macro_use]
 mod helpers;
@@ -52,7 +52,7 @@ fn total_size() -> Result {
     assert_eq!(wal.total_size(), total_file_size_before_delete);
 
     // Pretend the process restarts
-    let wal: delorean_wal::Wal = builder.wal()?;
+    let wal = builder.wal()?;
 
     // Total size should be that of all the files, so without the file deleted out-of-band
     assert_eq!(wal.total_size(), helpers::total_size_on_disk(&dir.as_ref()));