chore: Update arrow to latest version (#335)

* chore: Update arrow to latest version

* fix: Updates needed by new version of datafusion

Cargo.lock

@@ -78,7 +78,7 @@ checksum = "cff77d8686867eceff3105329d4698d96c2391c176d5d03adc90c7389162b5b8"
 [[package]]
 name = "arrow"
 version = "2.0.0-SNAPSHOT"
-source = "git+https://github.com/apache/arrow.git?rev=a1157b770dc5d51de22f32ad7a12131a1670aaca#a1157b770dc5d51de22f32ad7a12131a1670aaca"
+source = "git+https://github.com/apache/arrow.git?rev=238a9497269f39ab4d5bf20c28c2431a1b4e6673#238a9497269f39ab4d5bf20c28c2431a1b4e6673"
 dependencies = [
  "chrono",
  "csv",
@@ -646,9 +646,9 @@ dependencies = [
 [[package]]
 name = "datafusion"
 version = "2.0.0-SNAPSHOT"
-source = "git+https://github.com/apache/arrow.git?rev=a1157b770dc5d51de22f32ad7a12131a1670aaca#a1157b770dc5d51de22f32ad7a12131a1670aaca"
+source = "git+https://github.com/apache/arrow.git?rev=238a9497269f39ab4d5bf20c28c2431a1b4e6673#238a9497269f39ab4d5bf20c28c2431a1b4e6673"
 dependencies = [
- "arrow 2.0.0-SNAPSHOT (git+https://github.com/apache/arrow.git?rev=a1157b770dc5d51de22f32ad7a12131a1670aaca)",
+ "arrow 2.0.0-SNAPSHOT (git+https://github.com/apache/arrow.git?rev=238a9497269f39ab4d5bf20c28c2431a1b4e6673)",
  "async-trait",
  "chrono",
  "clap",
@@ -718,7 +718,7 @@ dependencies = [
 name = "delorean_arrow"
 version = "0.1.0"
 dependencies = [
- "arrow 2.0.0-SNAPSHOT (git+https://github.com/apache/arrow.git?rev=a1157b770dc5d51de22f32ad7a12131a1670aaca)",
+ "arrow 2.0.0-SNAPSHOT (git+https://github.com/apache/arrow.git?rev=238a9497269f39ab4d5bf20c28c2431a1b4e6673)",
  "datafusion",
  "parquet",
 ]
@@ -2108,9 +2108,9 @@ dependencies = [
 [[package]]
 name = "parquet"
 version = "2.0.0-SNAPSHOT"
-source = "git+https://github.com/apache/arrow.git?rev=a1157b770dc5d51de22f32ad7a12131a1670aaca#a1157b770dc5d51de22f32ad7a12131a1670aaca"
+source = "git+https://github.com/apache/arrow.git?rev=238a9497269f39ab4d5bf20c28c2431a1b4e6673#238a9497269f39ab4d5bf20c28c2431a1b4e6673"
 dependencies = [
- "arrow 2.0.0-SNAPSHOT (git+https://github.com/apache/arrow.git?rev=a1157b770dc5d51de22f32ad7a12131a1670aaca)",
+ "arrow 2.0.0-SNAPSHOT (git+https://github.com/apache/arrow.git?rev=238a9497269f39ab4d5bf20c28c2431a1b4e6673)",
  "brotli",
  "byteorder",
  "chrono",

delorean_arrow/Cargo.toml

@@ -11,10 +11,10 @@ description = "Apache Arrow / Parquet / DataFusion dependencies for delorean, to
 [dependencies]
 # We are using development version of arrow/parquet/datafusion and the dependencies are at the same rev
 
-# Check out the current rev @ https://github.com/apache/arrow/commit/a1157b770dc5d51de22f32ad7a12131a1670aaca
+# The version can be found here: https://github.com/apache/arrow/commit/238a9497269f39ab4d5bf20c28c2431a1b4e6673
 #
-arrow = { git = "https://github.com/apache/arrow.git", rev = "a1157b770dc5d51de22f32ad7a12131a1670aaca"}
+arrow = { git = "https://github.com/apache/arrow.git", rev = "238a9497269f39ab4d5bf20c28c2431a1b4e6673"}
-datafusion = { git = "https://github.com/apache/arrow.git", rev = "a1157b770dc5d51de22f32ad7a12131a1670aaca" }
+datafusion = { git = "https://github.com/apache/arrow.git", rev = "238a9497269f39ab4d5bf20c28c2431a1b4e6673" }
 # Turn off the "arrow" feature; it currently has a bug that causes the crate to rebuild every time
 # and we're not currently using it anyway
-parquet = { git = "https://github.com/apache/arrow.git", rev = "a1157b770dc5d51de22f32ad7a12131a1670aaca", default-features = false, features = ["snap", "brotli", "flate2", "lz4", "zstd"] }
+parquet = { git = "https://github.com/apache/arrow.git", rev = "238a9497269f39ab4d5bf20c28c2431a1b4e6673", default-features = false, features = ["snap", "brotli", "flate2", "lz4", "zstd"] }

delorean_mem_qe/src/bin/main.rs

@@ -110,7 +110,7 @@ fn build_store(
     let mut total_rows_read = 0;
     let start = std::time::Instant::now();
     loop {
-        let rb = reader.next_batch();
+        let rb = reader.next().transpose();
         match rb {
             Err(e) => println!("WARNING: error reading batch: {:?}, SKIPPING", e),
             Ok(Some(rb)) => {

delorean_storage/src/exec/schema_pivot.rs

@@ -219,7 +219,8 @@ impl ExecutionPlan for SchemaPivotExec {
             let input_batch = input_reader
                 .lock()
                 .expect("locked input mutex")
-                .next_batch()?;
+                .next()
+                .transpose()?;
 
             keep_searching = match input_batch {
                 Some(input_batch) => {
@@ -429,7 +430,7 @@ mod tests {
         let mut batches = Vec::new();
         let mut reader = reader.lock().expect("locking record batch reader");
         // process the record batches one by one
-        while let Some(record_batch) = reader.next_batch().expect("reading next batch") {
+        while let Some(record_batch) = reader.next().transpose().expect("reading next batch") {
             batches.push(record_batch)
         }
         batches