diff --git a/Cargo.lock b/Cargo.lock index dd2e2ac98f..6115ea9430 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -197,7 +197,7 @@ dependencies = [ "arrow-flight", "chrono", "comfy-table", - "datafusion 0.1.0", + "datafusion", "hashbrown", "num-traits", "rand", @@ -767,7 +767,7 @@ dependencies = [ "backoff", "bytes", "data_types", - "datafusion 0.1.0", + "datafusion", "futures", "generated_types", "iox_catalog", @@ -1125,15 +1125,6 @@ dependencies = [ "workspace-hack", ] -[[package]] -name = "datafusion" -version = "0.1.0" -dependencies = [ - "datafusion 13.0.0", - "datafusion-proto", - "workspace-hack", -] - [[package]] name = "datafusion" version = "13.0.0" @@ -1248,7 +1239,7 @@ source = "git+https://github.com/apache/arrow-datafusion.git?rev=48f73c6af3b0cc7 dependencies = [ "arrow", "async-trait", - "datafusion 13.0.0", + "datafusion", "datafusion-common", "datafusion-expr", "prost 0.11.0", @@ -1282,7 +1273,7 @@ name = "datafusion_util" version = "0.1.0" dependencies = [ "async-trait", - "datafusion 0.1.0", + "datafusion", "futures", "observability_deps", "pin-project", @@ -1676,7 +1667,8 @@ dependencies = [ "base64", "bytes", "data_types", - "datafusion 0.1.0", + "datafusion", + "datafusion-proto", "observability_deps", "pbjson", "pbjson-build", @@ -2143,7 +2135,7 @@ dependencies = [ "compactor", "console-subscriber", "data_types", - "datafusion 0.1.0", + "datafusion", "dotenvy", "flate2", "futures", @@ -2291,7 +2283,7 @@ dependencies = [ "bytes", "chrono", "data_types", - "datafusion 0.1.0", + "datafusion", "datafusion_util", "dml", "flatbuffers", @@ -2435,7 +2427,7 @@ dependencies = [ "async-trait", "chrono", "data_types", - "datafusion 0.1.0", + "datafusion", "datafusion_util", "executor", "futures", @@ -2463,7 +2455,7 @@ dependencies = [ "arrow", "bytes", "data_types", - "datafusion 0.1.0", + "datafusion", "datafusion_util", "futures", "iox_catalog", @@ -3426,7 +3418,7 @@ dependencies = [ "base64", "bytes", "data_types", - "datafusion 0.1.0", + "datafusion", "datafusion_util", "futures", "generated_types", @@ -3452,7 +3444,7 @@ dependencies = [ name = "parquet_to_line_protocol" version = "0.1.0" dependencies = [ - "datafusion 0.1.0", + "datafusion", "futures", "influxdb_line_protocol", "mutable_batch", @@ -3654,7 +3646,7 @@ dependencies = [ "arrow", "chrono", "data_types", - "datafusion 0.1.0", + "datafusion", "datafusion_util", "itertools", "observability_deps", @@ -3915,7 +3907,7 @@ dependencies = [ "cache_system", "client_util", "data_types", - "datafusion 0.1.0", + "datafusion", "datafusion_util", "futures", "generated_types", @@ -3956,7 +3948,7 @@ version = "0.1.0" dependencies = [ "arrow", "chrono", - "datafusion 0.1.0", + "datafusion", "datafusion_util", "itertools", "observability_deps", @@ -3978,7 +3970,7 @@ dependencies = [ "async-trait", "backoff", "data_types", - "datafusion 0.1.0", + "datafusion", "datafusion_util", "dml", "futures", @@ -4485,7 +4477,7 @@ name = "service_common" version = "0.1.0" dependencies = [ "async-trait", - "datafusion 0.1.0", + "datafusion", "iox_query", "metric", "parking_lot 0.12.1", @@ -4520,7 +4512,7 @@ dependencies = [ "arrow_util", "bytes", "data_types", - "datafusion 0.1.0", + "datafusion", "futures", "generated_types", "iox_query", @@ -4547,7 +4539,7 @@ dependencies = [ "arrow", "async-trait", "data_types", - "datafusion 0.1.0", + "datafusion", "datafusion_util", "futures", "generated_types", @@ -6008,11 +6000,11 @@ dependencies = [ "chrono", "crossbeam-utils", "crypto-common", + "datafusion", "digest", "either", "fixedbitset", "flatbuffers", - "flate2", "futures-channel", "futures-core", "futures-sink", diff --git a/Cargo.toml b/Cargo.toml index 1a188495e7..7d28b0dafa 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,6 @@ members = [ "client_util", "compactor", "data_types", - "datafusion", "datafusion_util", "dml", "executor", @@ -108,6 +107,10 @@ authors = ["IOx Project Developers"] edition = "2021" license = "MIT OR Apache-2.0" +[workspace.dependencies] +datafusion = { git = "https://github.com/apache/arrow-datafusion.git", rev="48f73c6af3b0cc747c38b4a9c7a610f4630e8736", default-features = false } +datafusion-proto = { git = "https://github.com/apache/arrow-datafusion.git", rev="48f73c6af3b0cc747c38b4a9c7a610f4630e8736" } + # This profile optimizes for runtime performance and small binary size at the expense of longer # build times. It's most suitable for final release builds. [profile.release] diff --git a/arrow_util/Cargo.toml b/arrow_util/Cargo.toml index f35803ca9c..ba874e848a 100644 --- a/arrow_util/Cargo.toml +++ b/arrow_util/Cargo.toml @@ -13,7 +13,7 @@ arrow = { version = "25.0.0", features = ["prettyprint", "dyn_cmp_dict"] } # used by arrow anyway (needed for printing workaround) chrono = { version = "0.4", default-features = false } comfy-table = { version = "6.1", default-features = false } -datafusion = { path = "../datafusion" } +datafusion = { workspace = true } hashbrown = "0.12" num-traits = "0.2" snafu = "0.7" diff --git a/compactor/Cargo.toml b/compactor/Cargo.toml index fd43d6ea4f..75ca783e61 100644 --- a/compactor/Cargo.toml +++ b/compactor/Cargo.toml @@ -11,7 +11,7 @@ async-trait = "0.1.58" backoff = { path = "../backoff" } bytes = "1.2" data_types = { path = "../data_types" } -datafusion = { path = "../datafusion" } +datafusion = { workspace = true } futures = "0.3" generated_types = { path = "../generated_types" } iox_catalog = { path = "../iox_catalog" } diff --git a/datafusion/Cargo.toml b/datafusion/Cargo.toml deleted file mode 100644 index fa2f7b2988..0000000000 --- a/datafusion/Cargo.toml +++ /dev/null @@ -1,15 +0,0 @@ -[package] -name = "datafusion" -description = "Re-exports datafusion at a specific version" -version.workspace = true -authors.workspace = true -edition.workspace = true -license.workspace = true - -[dependencies] - -# Rename to workaround doctest bug -# Turn off optional datafusion features (e.g. don't get support for crypto functions or avro) -upstream = { git = "https://github.com/apache/arrow-datafusion.git", rev="48f73c6af3b0cc747c38b4a9c7a610f4630e8736", default-features = false, package = "datafusion" } -datafusion-proto = { git = "https://github.com/apache/arrow-datafusion.git", rev="48f73c6af3b0cc747c38b4a9c7a610f4630e8736" } -workspace-hack = { path = "../workspace-hack"} diff --git a/datafusion/src/lib.rs b/datafusion/src/lib.rs deleted file mode 100644 index bb694f7556..0000000000 --- a/datafusion/src/lib.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub use upstream::*; - -// Publically re-export datafusion-proto crate as well -pub use datafusion_proto; diff --git a/datafusion_util/Cargo.toml b/datafusion_util/Cargo.toml index 0182df186f..1ffb366553 100644 --- a/datafusion_util/Cargo.toml +++ b/datafusion_util/Cargo.toml @@ -8,7 +8,7 @@ license.workspace = true [dependencies] async-trait = "0.1" -datafusion = { path = "../datafusion" } +datafusion = { workspace = true } futures = "0.3" observability_deps = { path = "../observability_deps" } pin-project = "1.0" diff --git a/generated_types/Cargo.toml b/generated_types/Cargo.toml index ce3fbec307..d91f8f4e4d 100644 --- a/generated_types/Cargo.toml +++ b/generated_types/Cargo.toml @@ -9,7 +9,8 @@ license.workspace = true base64 = "0.13" bytes = "1.2" data_types = { path = "../data_types", optional = true } -datafusion = { path = "../datafusion", optional = true } +datafusion = { workspace = true, optional = true } +datafusion-proto = { workspace = true, optional = true } observability_deps = { path = "../observability_deps" } pbjson = "0.5" pbjson-types = "0.5" @@ -28,9 +29,9 @@ pbjson-build = "0.5" [dev-dependencies] data_types = { path = "../data_types" } -datafusion = { path = "../datafusion" } +datafusion = { workspace = true } predicate = { path = "../predicate" } [features] default = ["data_types_conversions"] -data_types_conversions = ["data_types", "datafusion", "predicate"] +data_types_conversions = ["data_types", "datafusion", "datafusion-proto", "predicate"] diff --git a/generated_types/src/ingester.rs b/generated_types/src/ingester.rs index 109417b27a..080c0c24d8 100644 --- a/generated_types/src/ingester.rs +++ b/generated_types/src/ingester.rs @@ -1,6 +1,7 @@ use crate::{google::FieldViolation, influxdata::iox::ingester::v1 as proto}; use data_types::TimestampRange; -use datafusion::{common::DataFusionError, datafusion_proto::bytes::Serializeable, prelude::Expr}; +use datafusion::{common::DataFusionError, prelude::Expr}; +use datafusion_proto::bytes::Serializeable; use predicate::{Predicate, ValueExpr}; use prost::Message; use snafu::{ResultExt, Snafu}; diff --git a/influxdb_iox/Cargo.toml b/influxdb_iox/Cargo.toml index 584abeb0cb..950b8e212d 100644 --- a/influxdb_iox/Cargo.toml +++ b/influxdb_iox/Cargo.toml @@ -11,7 +11,7 @@ license.workspace = true clap_blocks = { path = "../clap_blocks" } compactor = { path = "../compactor" } data_types = { path = "../data_types" } -datafusion = { path = "../datafusion" } +datafusion = { workspace = true } generated_types = { path = "../generated_types" } import = { path = "../import" } influxdb_iox_client = { path = "../influxdb_iox_client", features = ["flight", "format"] } diff --git a/ingester/Cargo.toml b/ingester/Cargo.toml index 41761d61e8..29496d30f3 100644 --- a/ingester/Cargo.toml +++ b/ingester/Cargo.toml @@ -12,7 +12,7 @@ arrow_util = { path = "../arrow_util" } async-trait = "0.1.58" backoff = { path = "../backoff" } bytes = "1.2" -datafusion = { path = "../datafusion" } +datafusion = { workspace = true } datafusion_util = { path = "../datafusion_util" } data_types = { path = "../data_types" } flatbuffers = "2.1.2" diff --git a/iox_query/Cargo.toml b/iox_query/Cargo.toml index 9bf45bb708..81a85b751b 100644 --- a/iox_query/Cargo.toml +++ b/iox_query/Cargo.toml @@ -20,7 +20,7 @@ arrow_util = { path = "../arrow_util" } async-trait = "0.1" chrono = { version = "0.4", default-features = false } data_types = { path = "../data_types" } -datafusion = { path = "../datafusion" } +datafusion = { workspace = true } datafusion_util = { path = "../datafusion_util" } executor = { path = "../executor"} futures = "0.3" diff --git a/iox_tests/Cargo.toml b/iox_tests/Cargo.toml index 5a89220909..750f443cca 100644 --- a/iox_tests/Cargo.toml +++ b/iox_tests/Cargo.toml @@ -10,7 +10,7 @@ license.workspace = true arrow = "25.0.0" bytes = "1.2" data_types = { path = "../data_types" } -datafusion = { path = "../datafusion" } +datafusion = { workspace = true } datafusion_util = { path = "../datafusion_util" } iox_catalog = { path = "../iox_catalog" } iox_time = { path = "../iox_time" } diff --git a/parquet_file/Cargo.toml b/parquet_file/Cargo.toml index bba47cf6c3..8b4a72c3db 100644 --- a/parquet_file/Cargo.toml +++ b/parquet_file/Cargo.toml @@ -10,7 +10,7 @@ arrow = { version = "25.0.0", features = ["prettyprint"] } base64 = "0.13" bytes = "1.2" data_types = { path = "../data_types" } -datafusion = { path = "../datafusion" } +datafusion = { workspace = true } datafusion_util = { path = "../datafusion_util" } futures = "0.3" generated_types = { path = "../generated_types" } diff --git a/parquet_to_line_protocol/Cargo.toml b/parquet_to_line_protocol/Cargo.toml index e77fe9c4a8..05650f311d 100644 --- a/parquet_to_line_protocol/Cargo.toml +++ b/parquet_to_line_protocol/Cargo.toml @@ -6,7 +6,7 @@ edition.workspace = true license.workspace = true [dependencies] -datafusion = { path = "../datafusion" } +datafusion = { workspace = true } influxdb_line_protocol = { path = "../influxdb_line_protocol" } futures = {version = "0.3"} num_cpus = "1.13.1" diff --git a/predicate/Cargo.toml b/predicate/Cargo.toml index a5431c73c1..cd3bbd636e 100644 --- a/predicate/Cargo.toml +++ b/predicate/Cargo.toml @@ -9,7 +9,7 @@ license.workspace = true arrow = { version = "25.0.0", features = ["prettyprint"] } chrono = { version = "0.4", default-features = false } data_types = { path = "../data_types" } -datafusion = { path = "../datafusion" } +datafusion = { workspace = true } datafusion_util = { path = "../datafusion_util" } itertools = "0.10" observability_deps = { path = "../observability_deps" } diff --git a/querier/Cargo.toml b/querier/Cargo.toml index a6d6b2068a..fb3f651c0a 100644 --- a/querier/Cargo.toml +++ b/querier/Cargo.toml @@ -13,7 +13,7 @@ bytes = "1.2" cache_system = { path = "../cache_system" } client_util = { path = "../client_util" } data_types = { path = "../data_types" } -datafusion = { path = "../datafusion" } +datafusion = { workspace = true } datafusion_util = { path = "../datafusion_util" } futures = "0.3" generated_types = { path = "../generated_types" } diff --git a/query_functions/Cargo.toml b/query_functions/Cargo.toml index 45b680681c..29b7c3f857 100644 --- a/query_functions/Cargo.toml +++ b/query_functions/Cargo.toml @@ -8,7 +8,7 @@ license.workspace = true [dependencies] arrow = { version = "25.0.0", features = ["prettyprint"] } chrono = { version = "0.4", default-features = false } -datafusion = { path = "../datafusion" } +datafusion = { workspace = true } itertools = "0.10.5" observability_deps = { path = "../observability_deps" } once_cell = "1" diff --git a/query_tests/Cargo.toml b/query_tests/Cargo.toml index 00cf97a752..6006d0e98a 100644 --- a/query_tests/Cargo.toml +++ b/query_tests/Cargo.toml @@ -12,7 +12,7 @@ arrow_util = { path = "../arrow_util" } async-trait = "0.1" backoff = { path = "../backoff" } data_types = { path = "../data_types" } -datafusion = { path = "../datafusion" } +datafusion = { workspace = true } datafusion_util = { path = "../datafusion_util" } dml = { path = "../dml" } futures = "0.3" diff --git a/service_common/Cargo.toml b/service_common/Cargo.toml index 433c5d4e50..7b6154de69 100644 --- a/service_common/Cargo.toml +++ b/service_common/Cargo.toml @@ -5,17 +5,14 @@ authors.workspace = true edition.workspace = true license.workspace = true -[dependencies] -# Workspace dependencies, in alphabetical order -datafusion = { path = "../datafusion" } -predicate = { path = "../predicate" } +[dependencies] # In alphabetical order +async-trait = "0.1.58" +datafusion = { workspace = true } iox_query = { path = "../iox_query" } metric = { path = "../metric" } parking_lot = "0.12" +predicate = { path = "../predicate" } +tonic = "0.8" trace = { path = "../trace" } tracker = { path = "../tracker" } -tonic = "0.8" workspace-hack = { path = "../workspace-hack"} - -# Crates.io dependencies, in alphabetical order -async-trait = "0.1.58" diff --git a/service_grpc_flight/Cargo.toml b/service_grpc_flight/Cargo.toml index 7906ceabee..b52a8df1b7 100644 --- a/service_grpc_flight/Cargo.toml +++ b/service_grpc_flight/Cargo.toml @@ -9,7 +9,7 @@ license.workspace = true # Workspace dependencies, in alphabetical order arrow_util = { path = "../arrow_util" } data_types = { path = "../data_types" } -datafusion = { path = "../datafusion" } +datafusion = { workspace = true } generated_types = { path = "../generated_types" } observability_deps = { path = "../observability_deps" } iox_query = { path = "../iox_query" } diff --git a/service_grpc_influxrpc/Cargo.toml b/service_grpc_influxrpc/Cargo.toml index 86de5c0c70..c248a28763 100644 --- a/service_grpc_influxrpc/Cargo.toml +++ b/service_grpc_influxrpc/Cargo.toml @@ -8,7 +8,7 @@ license.workspace = true [dependencies] # Workspace dependencies, in alphabetical order data_types = { path = "../data_types" } -datafusion = { path = "../datafusion" } +datafusion = { workspace = true } datafusion_util = { path = "../datafusion_util" } generated_types = { path = "../generated_types" } observability_deps = { path = "../observability_deps" } diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index 23b1922b6c..5abf9011ec 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -25,11 +25,11 @@ bytes = { version = "1", features = ["std"] } chrono = { version = "0.4", default-features = false, features = ["alloc", "clock", "iana-time-zone", "serde", "std", "winapi"] } crossbeam-utils = { version = "0.8", features = ["std"] } crypto-common = { version = "0.1", default-features = false, features = ["std"] } +datafusion = { git = "https://github.com/apache/arrow-datafusion.git", rev = "48f73c6af3b0cc747c38b4a9c7a610f4630e8736", features = ["crypto_expressions", "regex_expressions", "unicode_expressions"] } digest = { version = "0.10", features = ["alloc", "block-buffer", "core-api", "mac", "std", "subtle"] } either = { version = "1", features = ["use_std"] } fixedbitset = { version = "0.4", features = ["std"] } flatbuffers = { version = "2", features = ["thiserror"] } -flate2 = { version = "1", features = ["miniz_oxide", "rust_backend"] } futures-channel = { version = "0.3", features = ["alloc", "futures-sink", "sink", "std"] } futures-core = { version = "0.3", features = ["alloc", "std"] } futures-sink = { version = "0.3", features = ["alloc", "std"] }