diff --git a/Cargo.lock b/Cargo.lock index 632fc1538c..3d2aefbcd7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -828,7 +828,7 @@ dependencies = [ [[package]] name = "datafusion" version = "5.1.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=299ab7d1c37c707fcd500d3428abbdbe4dc5399b#299ab7d1c37c707fcd500d3428abbdbe4dc5399b" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=1c858ce7baab1929cfdba97051ef4e5e4d0a866b#1c858ce7baab1929cfdba97051ef4e5e4d0a866b" dependencies = [ "ahash", "arrow", diff --git a/datafusion/Cargo.toml b/datafusion/Cargo.toml index f91e868b22..7708d1e1bb 100644 --- a/datafusion/Cargo.toml +++ b/datafusion/Cargo.toml @@ -9,4 +9,4 @@ description = "Re-exports datafusion at a specific version" # Rename to workaround doctest bug # Turn off optional datafusion features (e.g. various crypo functions) -upstream = { git = "https://github.com/apache/arrow-datafusion.git", rev="299ab7d1c37c707fcd500d3428abbdbe4dc5399b", default-features = false, package = "datafusion" } +upstream = { git = "https://github.com/apache/arrow-datafusion.git", rev="1c858ce7baab1929cfdba97051ef4e5e4d0a866b", default-features = false, package = "datafusion" } diff --git a/predicate/src/predicate.rs b/predicate/src/predicate.rs index 16f6905a46..bb67c324c5 100644 --- a/predicate/src/predicate.rs +++ b/predicate/src/predicate.rs @@ -11,9 +11,8 @@ use std::{ use data_types::timestamp::TimestampRange; use datafusion::{ error::DataFusionError, - logical_plan::{col, lit, Column, Expr, Operator}, + logical_plan::{col, lit, lit_timestamp_nano, Column, Expr, Operator}, optimizer::utils, - scalar::ScalarValue, }; use datafusion_util::{make_range_expr, AndExprBuilder}; use internal_types::schema::TIME_COLUMN_NAME; @@ -212,15 +211,11 @@ impl Predicate { // Time range if let Some(range) = pred.range { - // cast int to timestamp - // NGA todo: add in DF a function timestamp_lit(i64_val) which does lit(ScalarValue::TimestampNanosecond(Some(i64_val)) - // and use it here - let ts_start = ScalarValue::TimestampNanosecond(Some(range.start)); - let ts_end = ScalarValue::TimestampNanosecond(Some(range.end)); // time_expr = NOT(start <= time_range < end) + // Equivalent to: (time < start OR time >= end) let time_expr = col(TIME_COLUMN_NAME) - .lt(lit(ts_start)) - .or(col(TIME_COLUMN_NAME).gt_eq(lit(ts_end))); + .lt(lit_timestamp_nano(range.start)) + .or(col(TIME_COLUMN_NAME).gt_eq(lit_timestamp_nano(range.end))); match expr { None => expr = Some(time_expr),