diff --git a/Cargo.lock b/Cargo.lock index 2d11451927..beabbdff66 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -118,11 +118,11 @@ checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" [[package]] name = "arrow" -version = "22.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5936b4185aa57cb9790d8742aab22859045ce5cc6a3023796240cd101c19335" +version = "23.0.0" +source = "git+https://github.com/alamb/arrow-rs.git?branch=alamb/23.0.0_patched#d6b733d19fc8ae05de716e77d05391ddd1e999a5" dependencies = [ "ahash 0.8.0", + "arrow-buffer", "bitflags", "chrono", "comfy-table", @@ -141,11 +141,19 @@ dependencies = [ "serde_json", ] +[[package]] +name = "arrow-buffer" +version = "23.0.0" +source = "git+https://github.com/alamb/arrow-rs.git?branch=alamb/23.0.0_patched#d6b733d19fc8ae05de716e77d05391ddd1e999a5" +dependencies = [ + "half 2.1.0", + "num", +] + [[package]] name = "arrow-flight" -version = "22.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "660ed54d9d068c21281154fbf1d91905df679a81ba4a83f1742f48c40c91804f" +version = "23.0.0" +source = "git+https://github.com/alamb/arrow-rs.git?branch=alamb/23.0.0_patched#d6b733d19fc8ae05de716e77d05391ddd1e999a5" dependencies = [ "arrow", "base64", @@ -1044,7 +1052,7 @@ dependencies = [ [[package]] name = "datafusion" version = "12.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=a6fbf24ddb81e34258a940a48e371b1efe5f6e49#a6fbf24ddb81e34258a940a48e371b1efe5f6e49" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=c7f3a70a79ee84070f7e8f770981fc982d8df0a4#c7f3a70a79ee84070f7e8f770981fc982d8df0a4" dependencies = [ "ahash 0.8.0", "arrow", @@ -1083,7 +1091,7 @@ dependencies = [ [[package]] name = "datafusion-common" version = "12.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=a6fbf24ddb81e34258a940a48e371b1efe5f6e49#a6fbf24ddb81e34258a940a48e371b1efe5f6e49" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=c7f3a70a79ee84070f7e8f770981fc982d8df0a4#c7f3a70a79ee84070f7e8f770981fc982d8df0a4" dependencies = [ "arrow", "object_store", @@ -1095,7 +1103,7 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "12.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=a6fbf24ddb81e34258a940a48e371b1efe5f6e49#a6fbf24ddb81e34258a940a48e371b1efe5f6e49" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=c7f3a70a79ee84070f7e8f770981fc982d8df0a4#c7f3a70a79ee84070f7e8f770981fc982d8df0a4" dependencies = [ "ahash 0.8.0", "arrow", @@ -1106,7 +1114,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "12.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=a6fbf24ddb81e34258a940a48e371b1efe5f6e49#a6fbf24ddb81e34258a940a48e371b1efe5f6e49" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=c7f3a70a79ee84070f7e8f770981fc982d8df0a4#c7f3a70a79ee84070f7e8f770981fc982d8df0a4" dependencies = [ "arrow", "async-trait", @@ -1121,7 +1129,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "12.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=a6fbf24ddb81e34258a940a48e371b1efe5f6e49#a6fbf24ddb81e34258a940a48e371b1efe5f6e49" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=c7f3a70a79ee84070f7e8f770981fc982d8df0a4#c7f3a70a79ee84070f7e8f770981fc982d8df0a4" dependencies = [ "ahash 0.8.0", "arrow", @@ -1145,7 +1153,7 @@ dependencies = [ [[package]] name = "datafusion-proto" version = "12.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=a6fbf24ddb81e34258a940a48e371b1efe5f6e49#a6fbf24ddb81e34258a940a48e371b1efe5f6e49" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=c7f3a70a79ee84070f7e8f770981fc982d8df0a4#c7f3a70a79ee84070f7e8f770981fc982d8df0a4" dependencies = [ "arrow", "datafusion 12.0.0", @@ -1158,7 +1166,7 @@ dependencies = [ [[package]] name = "datafusion-row" version = "12.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=a6fbf24ddb81e34258a940a48e371b1efe5f6e49#a6fbf24ddb81e34258a940a48e371b1efe5f6e49" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=c7f3a70a79ee84070f7e8f770981fc982d8df0a4#c7f3a70a79ee84070f7e8f770981fc982d8df0a4" dependencies = [ "arrow", "datafusion-common", @@ -1169,7 +1177,7 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "12.0.0" -source = "git+https://github.com/apache/arrow-datafusion.git?rev=a6fbf24ddb81e34258a940a48e371b1efe5f6e49#a6fbf24ddb81e34258a940a48e371b1efe5f6e49" +source = "git+https://github.com/apache/arrow-datafusion.git?rev=c7f3a70a79ee84070f7e8f770981fc982d8df0a4#c7f3a70a79ee84070f7e8f770981fc982d8df0a4" dependencies = [ "ahash 0.8.0", "arrow", @@ -1707,6 +1715,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad6a9459c9c30b177b925162351f97e7d967c7ea8bab3b8352805327daf45554" dependencies = [ "crunchy", + "num-traits", ] [[package]] @@ -2137,7 +2146,7 @@ version = "0.1.0" dependencies = [ "flate2", "hex", - "integer-encoding 3.0.4", + "integer-encoding", "observability_deps", "rand", "snafu", @@ -2217,12 +2226,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "integer-encoding" -version = "1.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48dc51180a9b377fd75814d0cc02199c20f8e99433d6762f650d39cdbbd3b56f" - [[package]] name = "integer-encoding" version = "3.0.4" @@ -3279,9 +3282,8 @@ dependencies = [ [[package]] name = "parquet" -version = "22.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "474c423be6f10921adab3b94b42ec7fe87c1b87e1360dee150976caee444224f" +version = "23.0.0" +source = "git+https://github.com/alamb/arrow-rs.git?branch=alamb/23.0.0_patched#d6b733d19fc8ae05de716e77d05391ddd1e999a5" dependencies = [ "ahash 0.8.0", "arrow", @@ -3295,7 +3297,6 @@ dependencies = [ "lz4", "num", "num-bigint", - "parquet-format", "rand", "seq-macro", "snap", @@ -3304,15 +3305,6 @@ dependencies = [ "zstd", ] -[[package]] -name = "parquet-format" -version = "4.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f0c06cdcd5460967c485f9c40a821746f5955ad81990533c7fae95dbd9bc0b5" -dependencies = [ - "thrift", -] - [[package]] name = "parquet_file" version = "0.1.0" @@ -3330,7 +3322,6 @@ dependencies = [ "observability_deps", "parking_lot 0.12.1", "parquet", - "parquet-format", "pbjson-types", "predicate", "prost 0.11.0", @@ -4218,7 +4209,7 @@ dependencies = [ "chrono", "crc32c", "futures", - "integer-encoding 3.0.4", + "integer-encoding", "parking_lot 0.12.1", "pin-project-lite", "rand", @@ -5068,12 +5059,12 @@ dependencies = [ [[package]] name = "thrift" -version = "0.13.0" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c6d965454947cc7266d22716ebfd07b18d84ebaf35eec558586bbb2a8cb6b5b" +checksum = "09678c4cdbb4eed72e18b7c2af1329c69825ed16fcbac62d083fc3e2b0590ff0" dependencies = [ "byteorder", - "integer-encoding 1.1.7", + "integer-encoding", "log", "ordered-float 1.1.1", "threadpool", @@ -5927,6 +5918,7 @@ dependencies = [ "sqlx-core", "sqlx-macros", "syn", + "thrift", "tokio", "tokio-stream", "tokio-util", diff --git a/Cargo.toml b/Cargo.toml index 37f7b6b2c6..e69b911035 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -121,3 +121,19 @@ inherits = "release" codegen-units = 16 lto = false incremental = true + + +# Work around https://github.com/apache/arrow-rs/issues/2779 until fixis released +# +# Use https://github.com/alamb/arrow-rs/tree/alamb/23.0.0_patched +# Created by: +# +# git checkout 23.0.0 +# git checkout -b alamb/23.0.0_patched +# # get fix for #2779 +# git cherry-pick a7cf274765945af4111fddaeec26d672715de9d0 + +[patch.crates-io] +arrow = { git = "https://github.com/alamb/arrow-rs.git", branch="alamb/23.0.0_patched" } +parquet = { git = "https://github.com/alamb/arrow-rs.git", branch="alamb/23.0.0_patched" } +arrow-flight = { git = "https://github.com/alamb/arrow-rs.git", branch="alamb/23.0.0_patched" } diff --git a/arrow_util/Cargo.toml b/arrow_util/Cargo.toml index 02cb9cc238..ec530515f5 100644 --- a/arrow_util/Cargo.toml +++ b/arrow_util/Cargo.toml @@ -8,7 +8,7 @@ description = "Apache Arrow utilities" [dependencies] ahash = { version = "0.8.0", default-features = false, features = ["runtime-rng"] } # need dyn_cmp_dict feature for comparing dictionary arrays -arrow = { version = "22.0.0", features = ["prettyprint", "dyn_cmp_dict"] } +arrow = { version = "23.0.0", features = ["prettyprint", "dyn_cmp_dict"] } # used by arrow anyway (needed for printing workaround) chrono = { version = "0.4", default-features = false } comfy-table = { version = "6.1", default-features = false } @@ -19,5 +19,5 @@ snafu = "0.7" workspace-hack = { path = "../workspace-hack"} [dev-dependencies] -arrow-flight = "22.0.0" +arrow-flight = "23.0.0" rand = "0.8.3" diff --git a/arrow_util/src/util.rs b/arrow_util/src/util.rs index ab496bfd49..c3e403dd43 100644 --- a/arrow_util/src/util.rs +++ b/arrow_util/src/util.rs @@ -5,6 +5,7 @@ use std::sync::Arc; use arrow::{ array::{new_null_array, ArrayRef, StringArray}, + compute::concat_batches, datatypes::SchemaRef, error::ArrowError, record_batch::RecordBatch, @@ -73,5 +74,5 @@ pub fn merge_record_batches( return Ok(None); } - Ok(Some(RecordBatch::concat(output_schema, &batches)?)) + Ok(Some(concat_batches(output_schema, &batches)?)) } diff --git a/compactor/Cargo.toml b/compactor/Cargo.toml index 4fa507cd17..8a366ab903 100644 --- a/compactor/Cargo.toml +++ b/compactor/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Luke Bond "] edition = "2021" [dependencies] -arrow = { version = "22.0.0", features = ["prettyprint"] } +arrow = { version = "23.0.0", features = ["prettyprint"] } async-trait = "0.1.57" backoff = { path = "../backoff" } bytes = "1.2" diff --git a/datafusion/Cargo.toml b/datafusion/Cargo.toml index 727530bf4f..4e20f711d2 100644 --- a/datafusion/Cargo.toml +++ b/datafusion/Cargo.toml @@ -9,8 +9,6 @@ description = "Re-exports datafusion at a specific version" # Rename to workaround doctest bug # Turn off optional datafusion features (e.g. don't get support for crypto functions or avro) -upstream = { git = "https://github.com/apache/arrow-datafusion.git", rev="a6fbf24ddb81e34258a940a48e371b1efe5f6e49", default-features = false, package = "datafusion" } -datafusion-proto = { git = "https://github.com/apache/arrow-datafusion.git", rev="a6fbf24ddb81e34258a940a48e371b1efe5f6e49" } - - +upstream = { git = "https://github.com/apache/arrow-datafusion.git", rev="c7f3a70a79ee84070f7e8f770981fc982d8df0a4", default-features = false, package = "datafusion" } +datafusion-proto = { git = "https://github.com/apache/arrow-datafusion.git", rev="c7f3a70a79ee84070f7e8f770981fc982d8df0a4" } workspace-hack = { path = "../workspace-hack"} diff --git a/influxdb_iox/Cargo.toml b/influxdb_iox/Cargo.toml index 1f0bf62f1e..23f892b2db 100644 --- a/influxdb_iox/Cargo.toml +++ b/influxdb_iox/Cargo.toml @@ -40,7 +40,7 @@ trogging = { path = "../trogging", default-features = false, features = ["clap"] # Crates.io dependencies, in alphabetical order nu-ansi-term = "0.46.0" -arrow = { version = "22.0.0", features = ["prettyprint"] } +arrow = { version = "23.0.0", features = ["prettyprint"] } backtrace = "0.3" bytes = "1.2" clap = { version = "3", features = ["derive", "env"] } diff --git a/influxdb_iox_client/Cargo.toml b/influxdb_iox_client/Cargo.toml index eb4fdad7c0..b4bd715810 100644 --- a/influxdb_iox_client/Cargo.toml +++ b/influxdb_iox_client/Cargo.toml @@ -20,8 +20,8 @@ mutable_batch_lp = { path = "../mutable_batch_lp", optional = true } mutable_batch_pb = { path = "../mutable_batch_pb", optional = true } # Crates.io dependencies, in alphabetical order -arrow = { version = "22.0.0", optional = true } -arrow-flight = { version = "22.0.0", optional = true } +arrow = { version = "23.0.0", optional = true } +arrow-flight = { version = "23.0.0", optional = true } bytes = "1.2" futures-util = { version = "0.3", optional = true } prost = "0.11" diff --git a/ingester/Cargo.toml b/ingester/Cargo.toml index 58cef2e0a6..beb94c37e9 100644 --- a/ingester/Cargo.toml +++ b/ingester/Cargo.toml @@ -5,8 +5,8 @@ authors = ["Nga Tran "] edition = "2021" [dependencies] -arrow = { version = "22.0.0", features = ["prettyprint"] } -arrow-flight = "22.0.0" +arrow = { version = "23.0.0", features = ["prettyprint"] } +arrow-flight = "23.0.0" arrow_util = { path = "../arrow_util" } async-trait = "0.1.57" backoff = { path = "../backoff" } diff --git a/ingester/src/data/partition.rs b/ingester/src/data/partition.rs index 0055321e97..e232be2cdf 100644 --- a/ingester/src/data/partition.rs +++ b/ingester/src/data/partition.rs @@ -264,10 +264,11 @@ impl PartitionData { // Merge all result record batches into one record batch // and make a snapshot for it let snapshot = if !record_batches.is_empty() { - let record_batch = RecordBatch::concat(&record_batches[0].schema(), &record_batches) - .unwrap_or_else(|e| { - panic!("unable to concat record batches: {:?}", e); - }); + let record_batch = + arrow::compute::concat_batches(&record_batches[0].schema(), &record_batches) + .unwrap_or_else(|e| { + panic!("unable to concat record batches: {:?}", e); + }); let snapshot = SnapshotBatch { min_sequence_number, max_sequence_number, diff --git a/iox_query/Cargo.toml b/iox_query/Cargo.toml index eefa53d5b7..dd52271c26 100644 --- a/iox_query/Cargo.toml +++ b/iox_query/Cargo.toml @@ -14,7 +14,7 @@ description = "IOx Query Interface and Executor" # 2. Allow for query logic testing without bringing in all the storage systems. [dependencies] # In alphabetical order -arrow = { version = "22.0.0", features = ["prettyprint"] } +arrow = { version = "23.0.0", features = ["prettyprint"] } arrow_util = { path = "../arrow_util" } async-trait = "0.1" chrono = { version = "0.4", default-features = false } diff --git a/iox_query/src/exec/seriesset/converter.rs b/iox_query/src/exec/seriesset/converter.rs index 96b8a879ed..6c85358e4f 100644 --- a/iox_query/src/exec/seriesset/converter.rs +++ b/iox_query/src/exec/seriesset/converter.rs @@ -100,7 +100,8 @@ impl SeriesSetConverter { let batches = collect(it).await.context(ReadingSnafu)?; let batch = if !batches.is_empty() { - RecordBatch::concat(&batches[0].schema(), &batches).context(ConcatenatingSnafu)? + arrow::compute::concat_batches(&batches[0].schema(), &batches) + .context(ConcatenatingSnafu)? } else { return Ok(vec![]); }; diff --git a/iox_tests/Cargo.toml b/iox_tests/Cargo.toml index e4a60bae04..8760728d4e 100644 --- a/iox_tests/Cargo.toml +++ b/iox_tests/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" description = "IOx test utils and tests" [dependencies] -arrow = "22.0.0" +arrow = "23.0.0" bytes = "1.2" data_types = { path = "../data_types" } datafusion = { path = "../datafusion" } diff --git a/iox_tests/src/util.rs b/iox_tests/src/util.rs index fd81a009af..946782a632 100644 --- a/iox_tests/src/util.rs +++ b/iox_tests/src/util.rs @@ -961,5 +961,5 @@ fn dedup_batch(record_batch: RecordBatch, sort_key: &SortKey) -> RecordBatch { batches.push(batch); } - RecordBatch::concat(&schema, &batches).unwrap() + arrow::compute::concat_batches(&schema, &batches).unwrap() } diff --git a/ioxd_querier/Cargo.toml b/ioxd_querier/Cargo.toml index 53514f5219..fb92e7e988 100644 --- a/ioxd_querier/Cargo.toml +++ b/ioxd_querier/Cargo.toml @@ -22,7 +22,7 @@ iox_time = { path = "../iox_time" } trace = { path = "../trace" } # Crates.io dependencies, in alphabetical order -arrow-flight = "22.0.0" +arrow-flight = "23.0.0" async-trait = "0.1" hyper = "0.14" thiserror = "1.0.36" diff --git a/mutable_batch/Cargo.toml b/mutable_batch/Cargo.toml index 547865cfb2..b6d571dbdb 100644 --- a/mutable_batch/Cargo.toml +++ b/mutable_batch/Cargo.toml @@ -5,7 +5,7 @@ edition = "2021" description = "A mutable arrow RecordBatch" [dependencies] -arrow = { version = "22.0.0", features = ["prettyprint"] } +arrow = { version = "23.0.0", features = ["prettyprint"] } arrow_util = { path = "../arrow_util" } chrono = { version = "0.4", default-features = false } data_types = { path = "../data_types" } diff --git a/packers/Cargo.toml b/packers/Cargo.toml index e5993df046..cdc31dceaa 100644 --- a/packers/Cargo.toml +++ b/packers/Cargo.toml @@ -5,11 +5,11 @@ authors = ["Andrew Lamb "] edition = "2021" [dependencies] # In alphabetical order -arrow = { version = "22.0.0", features = ["prettyprint"] } +arrow = { version = "23.0.0", features = ["prettyprint"] } influxdb_tsm = { path = "../influxdb_tsm" } schema = { path = "../schema" } snafu = "0.7" -parquet = "22.0.0" +parquet = "23.0.0" workspace-hack = { path = "../workspace-hack"} [dev-dependencies] # In alphabetical order diff --git a/parquet_file/Cargo.toml b/parquet_file/Cargo.toml index 0ed9d496a2..0b6d39548e 100644 --- a/parquet_file/Cargo.toml +++ b/parquet_file/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Nga Tran "] edition = "2021" [dependencies] # In alphabetical order -arrow = { version = "22.0.0", features = ["prettyprint"] } +arrow = { version = "23.0.0", features = ["prettyprint"] } base64 = "0.13" bytes = "1.2" data_types = { path = "../data_types" } @@ -17,14 +17,13 @@ iox_time = { path = "../iox_time" } object_store = "0.5.0" observability_deps = { path = "../observability_deps" } parking_lot = "0.12" -parquet = {version = "22.0.0", features = ["experimental"]} -parquet-format = "4.0" +parquet = {version = "23.0.0", features = ["experimental"]} pbjson-types = "0.5" predicate = { path = "../predicate" } prost = "0.11" schema = { path = "../schema" } snafu = "0.7" -thrift = "0.13" +thrift = "0.16" tokio = { version = "1.21", features = ["macros", "parking_lot", "rt", "rt-multi-thread", "sync"] } uuid = { version = "1", features = ["v4"] } zstd = "0.11" diff --git a/parquet_file/src/metadata.rs b/parquet_file/src/metadata.rs index e72acd333c..3ce398f509 100644 --- a/parquet_file/src/metadata.rs +++ b/parquet_file/src/metadata.rs @@ -604,7 +604,7 @@ impl IoxParquetMetaData { .map(|rg| rg.to_thrift()) .collect(); - let thrift_file_metadata = parquet_format::FileMetaData { + let thrift_file_metadata = parquet::format::FileMetaData { version: file_metadata.version(), schema: thrift_schema, @@ -635,7 +635,7 @@ impl IoxParquetMetaData { // step 2: load thrift data from byte stream let thrift_file_metadata = { let mut protocol = TCompactInputProtocol::new(&data[..]); - parquet_format::FileMetaData::read_from_in_protocol(&mut protocol) + parquet::format::FileMetaData::read_from_in_protocol(&mut protocol) .context(ThriftReadFailureSnafu {})? }; @@ -674,10 +674,10 @@ impl IoxParquetMetaData { } } -impl TryFrom for IoxParquetMetaData { +impl TryFrom for IoxParquetMetaData { type Error = Error; - fn try_from(v: parquet_format::FileMetaData) -> Result { + fn try_from(v: parquet::format::FileMetaData) -> Result { let mut buffer = Vec::new(); { let mut protocol = TCompactOutputProtocol::new(&mut buffer); diff --git a/parquet_file/src/serialize.rs b/parquet_file/src/serialize.rs index ad1b667241..96131a9254 100644 --- a/parquet_file/src/serialize.rs +++ b/parquet_file/src/serialize.rs @@ -81,13 +81,13 @@ pub enum CodecError { /// data was serialised. /// /// [`proto::IoxMetadata`]: generated_types::influxdata::iox::ingester::v1 -/// [`FileMetaData`]: parquet_format::FileMetaData +/// [`FileMetaData`]: parquet::format::FileMetaData /// [`IoxParquetMetaData`]: crate::metadata::IoxParquetMetaData pub async fn to_parquet( batches: S, meta: &IoxMetadata, sink: W, -) -> Result +) -> Result where S: Stream> + Send, W: Write + Send, @@ -148,7 +148,7 @@ where pub async fn to_parquet_bytes( batches: S, meta: &IoxMetadata, -) -> Result<(Vec, parquet_format::FileMetaData), CodecError> +) -> Result<(Vec, parquet::format::FileMetaData), CodecError> where S: Stream> + Send, { diff --git a/predicate/Cargo.toml b/predicate/Cargo.toml index 5532a624d4..1d2b1d0147 100644 --- a/predicate/Cargo.toml +++ b/predicate/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0" edition = "2021" [dependencies] -arrow = { version = "22.0.0", features = ["prettyprint"] } +arrow = { version = "23.0.0", features = ["prettyprint"] } chrono = { version = "0.4", default-features = false } data_types = { path = "../data_types" } datafusion = { path = "../datafusion" } diff --git a/querier/Cargo.toml b/querier/Cargo.toml index 941c6f3e8e..9d55643c4b 100644 --- a/querier/Cargo.toml +++ b/querier/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0" edition = "2021" [dependencies] -arrow = "22.0.0" +arrow = "23.0.0" async-trait = "0.1.57" backoff = { path = "../backoff" } bytes = "1.2" diff --git a/query_functions/Cargo.toml b/query_functions/Cargo.toml index 22545dee63..2c421acc7e 100644 --- a/query_functions/Cargo.toml +++ b/query_functions/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -arrow = { version = "22.0.0", features = ["prettyprint"] } +arrow = { version = "23.0.0", features = ["prettyprint"] } chrono = { version = "0.4", default-features = false } datafusion = { path = "../datafusion" } itertools = "0.10.5" diff --git a/query_tests/Cargo.toml b/query_tests/Cargo.toml index ce71f59f05..bdbde84d64 100644 --- a/query_tests/Cargo.toml +++ b/query_tests/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" description = "Tests of the query engine against different database configurations" [dependencies] -arrow = { version = "22.0.0", features = ["prettyprint"] } +arrow = { version = "23.0.0", features = ["prettyprint"] } arrow_util = { path = "../arrow_util" } async-trait = "0.1" backoff = { path = "../backoff" } @@ -34,7 +34,7 @@ workspace-hack = { path = "../workspace-hack"} parquet_file = { version = "0.1.0", path = "../parquet_file" } [dev-dependencies] -arrow = { version = "22.0.0", features = ["prettyprint"] } +arrow = { version = "23.0.0", features = ["prettyprint"] } arrow_util = { path = "../arrow_util" } snafu = "0.7" tempfile = "3.1.0" diff --git a/query_tests/src/runner.rs b/query_tests/src/runner.rs index 6d2c46bb42..0a9e633877 100644 --- a/query_tests/src/runner.rs +++ b/query_tests/src/runner.rs @@ -285,7 +285,8 @@ impl Runner { // compare against sorted results, if requested if query.sorted_compare() && !results.is_empty() { let schema = results[0].schema(); - let batch = RecordBatch::concat(&schema, &results).expect("concatenating batches"); + let batch = arrow::compute::concat_batches(&schema, &results) + .expect("concatenating batches"); results = vec![sort_record_batch(batch)]; } diff --git a/read_buffer/Cargo.toml b/read_buffer/Cargo.toml index d810235dc0..562c0ed8d5 100644 --- a/read_buffer/Cargo.toml +++ b/read_buffer/Cargo.toml @@ -11,7 +11,7 @@ edition = "2021" # 2. Keep change/compile/link time down during development when working on just this crate [dependencies] # In alphabetical order -arrow = { version = "22.0.0", features = ["prettyprint"] } +arrow = { version = "23.0.0", features = ["prettyprint"] } arrow_util = { path = "../arrow_util" } croaring = "0.6" data_types = { path = "../data_types" } diff --git a/read_buffer/src/chunk.rs b/read_buffer/src/chunk.rs index ef8e5a2aca..9a4ffce38e 100644 --- a/read_buffer/src/chunk.rs +++ b/read_buffer/src/chunk.rs @@ -407,8 +407,7 @@ impl ChunkBuilder { // contiguous block of memory. To reduce memory overhead we can avoid // the concatenation and teach the `RowGroup` to build itself from a // vec of record batches. - let concat_batch = - arrow::record_batch::RecordBatch::concat(&self.schema, &self.record_batches)?; + let concat_batch = arrow::compute::concat_batches(&self.schema, &self.record_batches)?; self.row_groups .push(record_batch_to_row_group(concat_batch)); diff --git a/schema/Cargo.toml b/schema/Cargo.toml index a9c8e9bc77..a54ae90e4b 100644 --- a/schema/Cargo.toml +++ b/schema/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" description = "IOx Schema definition" [dependencies] -arrow = { version = "22.0.0", features = ["prettyprint"] } +arrow = { version = "23.0.0", features = ["prettyprint"] } hashbrown = "0.12" indexmap = { version = "1.9", features = ["std"] } itertools = "0.10.5" diff --git a/service_grpc_flight/Cargo.toml b/service_grpc_flight/Cargo.toml index 5773067d17..172e89b560 100644 --- a/service_grpc_flight/Cargo.toml +++ b/service_grpc_flight/Cargo.toml @@ -19,8 +19,8 @@ trace_http = { path = "../trace_http"} tracker = { path = "../tracker" } # Crates.io dependencies, in alphabetical order -arrow = { version = "22.0.0", features = ["prettyprint"] } -arrow-flight = "22.0.0" +arrow = { version = "23.0.0", features = ["prettyprint"] } +arrow-flight = "23.0.0" bytes = "1.2" futures = "0.3" pin-project = "1.0" diff --git a/service_grpc_influxrpc/Cargo.toml b/service_grpc_influxrpc/Cargo.toml index 43e8ba3693..ea4169e05c 100644 --- a/service_grpc_influxrpc/Cargo.toml +++ b/service_grpc_influxrpc/Cargo.toml @@ -19,7 +19,7 @@ trace_http = { path = "../trace_http"} tracker = { path = "../tracker" } # Crates.io dependencies, in alphabetical order -arrow = { version = "22.0.0", features = ["prettyprint"] } +arrow = { version = "23.0.0", features = ["prettyprint"] } async-trait = "0.1" futures = "0.3" pin-project = "1.0" diff --git a/test_helpers_end_to_end/Cargo.toml b/test_helpers_end_to_end/Cargo.toml index 36258d40f1..c54aeb940e 100644 --- a/test_helpers_end_to_end/Cargo.toml +++ b/test_helpers_end_to_end/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0" edition = "2021" [dependencies] # In alphabetical order -arrow = { version = "22.0.0", features = ["prettyprint"] } +arrow = { version = "23.0.0", features = ["prettyprint"] } arrow_util = { path = "../arrow_util" } assert_cmd = "2.0.2" bytes = "1.2" diff --git a/trace_exporters/Cargo.toml b/trace_exporters/Cargo.toml index f3c1064782..e4fe065f5a 100644 --- a/trace_exporters/Cargo.toml +++ b/trace_exporters/Cargo.toml @@ -13,7 +13,7 @@ clap = { version = "3", features = ["derive", "env"] } futures = "0.3" observability_deps = { path = "../observability_deps" } snafu = "0.7" -thrift = { version = "0.13.0" } +thrift = { version = "0.16.0" } tokio = { version = "1.21", features = ["macros", "parking_lot", "rt", "sync"] } trace = { path = "../trace" } workspace-hack = { path = "../workspace-hack"} diff --git a/workspace-hack/Cargo.toml b/workspace-hack/Cargo.toml index 75cf4e9aba..6e466e461d 100644 --- a/workspace-hack/Cargo.toml +++ b/workspace-hack/Cargo.toml @@ -14,7 +14,7 @@ publish = false ### BEGIN HAKARI SECTION [dependencies] ahash-ca01ad9e24f5d932 = { package = "ahash", version = "0.7", features = ["std"] } -arrow = { version = "22", features = ["comfy-table", "csv", "csv_crate", "dyn_cmp_dict", "flatbuffers", "ipc", "json", "prettyprint", "serde", "serde_json"] } +arrow = { git = "https://github.com/alamb/arrow-rs.git", branch = "alamb/23.0.0_patched", features = ["comfy-table", "csv", "csv_crate", "dyn_cmp_dict", "flatbuffers", "ipc", "json", "prettyprint", "serde", "serde_json"] } base64 = { version = "0.13", features = ["std"] } bitflags = { version = "1" } byteorder = { version = "1", features = ["std"] } @@ -45,7 +45,7 @@ num-traits = { version = "0.2", features = ["i128", "libm", "std"] } object_store = { version = "0.5", default-features = false, features = ["aws", "azure", "base64", "cloud", "gcp", "quick-xml", "rand", "reqwest", "ring", "rustls-pemfile", "serde", "serde_json"] } once_cell = { version = "1", features = ["alloc", "parking_lot", "parking_lot_core", "race", "std"] } parking_lot = { version = "0.12", features = ["arc_lock"] } -parquet = { version = "22", features = ["arrow", "async", "base64", "brotli", "experimental", "flate2", "futures", "lz4", "snap", "tokio", "zstd"] } +parquet = { git = "https://github.com/alamb/arrow-rs.git", branch = "alamb/23.0.0_patched", features = ["arrow", "async", "base64", "brotli", "experimental", "flate2", "futures", "lz4", "snap", "tokio", "zstd"] } predicates = { version = "2", features = ["diff", "difflib", "float-cmp", "normalize-line-endings", "regex"] } prost = { version = "0.11", features = ["prost-derive", "std"] } prost-types = { version = "0.11", features = ["std"] } @@ -61,6 +61,7 @@ sha2 = { version = "0.10", features = ["std"] } smallvec = { version = "1", default-features = false, features = ["union"] } sqlx = { version = "0.6", features = ["_rt-tokio", "json", "macros", "migrate", "postgres", "runtime-tokio-rustls", "sqlx-macros", "tls", "uuid"] } sqlx-core = { version = "0.6", default-features = false, features = ["_rt-tokio", "_tls-rustls", "any", "base64", "crc", "dirs", "hkdf", "hmac", "json", "md-5", "migrate", "postgres", "rand", "runtime-tokio-rustls", "rustls", "rustls-pemfile", "serde", "serde_json", "sha1", "sha2", "tokio-stream", "uuid", "webpki-roots", "whoami"] } +thrift = { version = "0.16", features = ["log", "server", "threadpool"] } tokio = { version = "1", features = ["bytes", "fs", "io-std", "io-util", "libc", "macros", "memchr", "mio", "net", "num_cpus", "once_cell", "parking_lot", "rt", "rt-multi-thread", "signal", "signal-hook-registry", "socket2", "sync", "time", "tokio-macros", "tracing"] } tokio-stream = { version = "0.1", features = ["fs", "net", "time"] } tokio-util = { version = "0.7", features = ["codec", "tracing"] }