Influxdata
/
influxdb
mirror of https://github.com/influxdata/influxdb.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

chore: Update datafusion and `arrow`/`parquet`/`arrow-flight` to 23.0.0, `thrift` to 0.16.0 (#5694) 

* chore: Update datafusion and `arrow`/`parquet`/`arrow-flight`  to 23.0.0

* chore: Update thrift / remove parquet_format

* fix: Update APIs

* chore: Update lock + Run cargo hakari tasks

* fix: use patched version of arrow-rs to work around https://github.com/apache/arrow-rs/issues/2779

* chore: Run cargo hakari tasks

Co-authored-by: CircleCI[bot] <circleci@influxdata.com>
Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com>
pull/24376/head
Andrew Lamb 2022-09-27 08:50:54 -04:00 committed by GitHub
parent 8139d01d08
commit 66dbb9541f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
33 changed files with 101 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
Cargo.lock generated
View File

@ -118,11 +118,11 @@ checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6"
[[package]]
name = "arrow"
version = "22.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c5936b4185aa57cb9790d8742aab22859045ce5cc6a3023796240cd101c19335"
version = "23.0.0"
source = "git+https://github.com/alamb/arrow-rs.git?branch=alamb/23.0.0_patched#d6b733d19fc8ae05de716e77d05391ddd1e999a5"
dependencies = [
"ahash 0.8.0",
"arrow-buffer",
"bitflags",
"chrono",
"comfy-table",
@ -141,11 +141,19 @@ dependencies = [
"serde_json",
]
[[package]]
name = "arrow-buffer"
version = "23.0.0"
source = "git+https://github.com/alamb/arrow-rs.git?branch=alamb/23.0.0_patched#d6b733d19fc8ae05de716e77d05391ddd1e999a5"
dependencies = [
"half 2.1.0",
"num",
]
[[package]]
name = "arrow-flight"
version = "22.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "660ed54d9d068c21281154fbf1d91905df679a81ba4a83f1742f48c40c91804f"
version = "23.0.0"
source = "git+https://github.com/alamb/arrow-rs.git?branch=alamb/23.0.0_patched#d6b733d19fc8ae05de716e77d05391ddd1e999a5"
dependencies = [
"arrow",
"base64",
@ -1044,7 +1052,7 @@ dependencies = [
[[package]]
name = "datafusion"
version = "12.0.0"
source = "git+https://github.com/apache/arrow-datafusion.git?rev=a6fbf24ddb81e34258a940a48e371b1efe5f6e49#a6fbf24ddb81e34258a940a48e371b1efe5f6e49"
source = "git+https://github.com/apache/arrow-datafusion.git?rev=c7f3a70a79ee84070f7e8f770981fc982d8df0a4#c7f3a70a79ee84070f7e8f770981fc982d8df0a4"
dependencies = [
"ahash 0.8.0",
"arrow",
@ -1083,7 +1091,7 @@ dependencies = [
[[package]]
name = "datafusion-common"
version = "12.0.0"
source = "git+https://github.com/apache/arrow-datafusion.git?rev=a6fbf24ddb81e34258a940a48e371b1efe5f6e49#a6fbf24ddb81e34258a940a48e371b1efe5f6e49"
source = "git+https://github.com/apache/arrow-datafusion.git?rev=c7f3a70a79ee84070f7e8f770981fc982d8df0a4#c7f3a70a79ee84070f7e8f770981fc982d8df0a4"
dependencies = [
"arrow",
"object_store",
@ -1095,7 +1103,7 @@ dependencies = [
[[package]]
name = "datafusion-expr"
version = "12.0.0"
source = "git+https://github.com/apache/arrow-datafusion.git?rev=a6fbf24ddb81e34258a940a48e371b1efe5f6e49#a6fbf24ddb81e34258a940a48e371b1efe5f6e49"
source = "git+https://github.com/apache/arrow-datafusion.git?rev=c7f3a70a79ee84070f7e8f770981fc982d8df0a4#c7f3a70a79ee84070f7e8f770981fc982d8df0a4"
dependencies = [
"ahash 0.8.0",
"arrow",
@ -1106,7 +1114,7 @@ dependencies = [
[[package]]
name = "datafusion-optimizer"
version = "12.0.0"
source = "git+https://github.com/apache/arrow-datafusion.git?rev=a6fbf24ddb81e34258a940a48e371b1efe5f6e49#a6fbf24ddb81e34258a940a48e371b1efe5f6e49"
source = "git+https://github.com/apache/arrow-datafusion.git?rev=c7f3a70a79ee84070f7e8f770981fc982d8df0a4#c7f3a70a79ee84070f7e8f770981fc982d8df0a4"
dependencies = [
"arrow",
"async-trait",
@ -1121,7 +1129,7 @@ dependencies = [
[[package]]
name = "datafusion-physical-expr"
version = "12.0.0"
source = "git+https://github.com/apache/arrow-datafusion.git?rev=a6fbf24ddb81e34258a940a48e371b1efe5f6e49#a6fbf24ddb81e34258a940a48e371b1efe5f6e49"
source = "git+https://github.com/apache/arrow-datafusion.git?rev=c7f3a70a79ee84070f7e8f770981fc982d8df0a4#c7f3a70a79ee84070f7e8f770981fc982d8df0a4"
dependencies = [
"ahash 0.8.0",
"arrow",
@ -1145,7 +1153,7 @@ dependencies = [
[[package]]
name = "datafusion-proto"
version = "12.0.0"
source = "git+https://github.com/apache/arrow-datafusion.git?rev=a6fbf24ddb81e34258a940a48e371b1efe5f6e49#a6fbf24ddb81e34258a940a48e371b1efe5f6e49"
source = "git+https://github.com/apache/arrow-datafusion.git?rev=c7f3a70a79ee84070f7e8f770981fc982d8df0a4#c7f3a70a79ee84070f7e8f770981fc982d8df0a4"
dependencies = [
"arrow",
"datafusion 12.0.0",
@ -1158,7 +1166,7 @@ dependencies = [
[[package]]
name = "datafusion-row"
version = "12.0.0"
source = "git+https://github.com/apache/arrow-datafusion.git?rev=a6fbf24ddb81e34258a940a48e371b1efe5f6e49#a6fbf24ddb81e34258a940a48e371b1efe5f6e49"
source = "git+https://github.com/apache/arrow-datafusion.git?rev=c7f3a70a79ee84070f7e8f770981fc982d8df0a4#c7f3a70a79ee84070f7e8f770981fc982d8df0a4"
dependencies = [
"arrow",
"datafusion-common",
@ -1169,7 +1177,7 @@ dependencies = [
[[package]]
name = "datafusion-sql"
version = "12.0.0"
source = "git+https://github.com/apache/arrow-datafusion.git?rev=a6fbf24ddb81e34258a940a48e371b1efe5f6e49#a6fbf24ddb81e34258a940a48e371b1efe5f6e49"
source = "git+https://github.com/apache/arrow-datafusion.git?rev=c7f3a70a79ee84070f7e8f770981fc982d8df0a4#c7f3a70a79ee84070f7e8f770981fc982d8df0a4"
dependencies = [
"ahash 0.8.0",
"arrow",
@ -1707,6 +1715,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ad6a9459c9c30b177b925162351f97e7d967c7ea8bab3b8352805327daf45554"
dependencies = [
"crunchy",
"num-traits",
]
[[package]]
@ -2137,7 +2146,7 @@ version = "0.1.0"
dependencies = [
"flate2",
"hex",
"integer-encoding 3.0.4",
"integer-encoding",
"observability_deps",
"rand",
"snafu",
@ -2217,12 +2226,6 @@ dependencies = [
"cfg-if",
]
[[package]]
name = "integer-encoding"
version = "1.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48dc51180a9b377fd75814d0cc02199c20f8e99433d6762f650d39cdbbd3b56f"
[[package]]
name = "integer-encoding"
version = "3.0.4"
@ -3279,9 +3282,8 @@ dependencies = [
[[package]]
name = "parquet"
version = "22.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "474c423be6f10921adab3b94b42ec7fe87c1b87e1360dee150976caee444224f"
version = "23.0.0"
source = "git+https://github.com/alamb/arrow-rs.git?branch=alamb/23.0.0_patched#d6b733d19fc8ae05de716e77d05391ddd1e999a5"
dependencies = [
"ahash 0.8.0",
"arrow",
@ -3295,7 +3297,6 @@ dependencies = [
"lz4",
"num",
"num-bigint",
"parquet-format",
"rand",
"seq-macro",
"snap",
@ -3304,15 +3305,6 @@ dependencies = [
"zstd",
]
[[package]]
name = "parquet-format"
version = "4.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f0c06cdcd5460967c485f9c40a821746f5955ad81990533c7fae95dbd9bc0b5"
dependencies = [
"thrift",
]
[[package]]
name = "parquet_file"
version = "0.1.0"
@ -3330,7 +3322,6 @@ dependencies = [
"observability_deps",
"parking_lot 0.12.1",
"parquet",
"parquet-format",
"pbjson-types",
"predicate",
"prost 0.11.0",
@ -4218,7 +4209,7 @@ dependencies = [
"chrono",
"crc32c",
"futures",
"integer-encoding 3.0.4",
"integer-encoding",
"parking_lot 0.12.1",
"pin-project-lite",
"rand",
@ -5068,12 +5059,12 @@ dependencies = [
[[package]]
name = "thrift"
version = "0.13.0"
version = "0.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c6d965454947cc7266d22716ebfd07b18d84ebaf35eec558586bbb2a8cb6b5b"
checksum = "09678c4cdbb4eed72e18b7c2af1329c69825ed16fcbac62d083fc3e2b0590ff0"
dependencies = [
"byteorder",
"integer-encoding 1.1.7",
"integer-encoding",
"log",
"ordered-float 1.1.1",
"threadpool",
@ -5927,6 +5918,7 @@ dependencies = [
"sqlx-core",
"sqlx-macros",
"syn",
"thrift",
"tokio",
"tokio-stream",
"tokio-util",

16
Cargo.toml
View File

@ -121,3 +121,19 @@ inherits = "release"
codegen-units = 16
lto = false
incremental = true
# Work around https://github.com/apache/arrow-rs/issues/2779 until fixis released
#
# Use https://github.com/alamb/arrow-rs/tree/alamb/23.0.0_patched
# Created by:
#
# git checkout 23.0.0
# git checkout -b alamb/23.0.0_patched
# # get fix for #2779
# git cherry-pick a7cf274765945af4111fddaeec26d672715de9d0
[patch.crates-io]
arrow = { git = "https://github.com/alamb/arrow-rs.git", branch="alamb/23.0.0_patched" }
parquet = { git = "https://github.com/alamb/arrow-rs.git", branch="alamb/23.0.0_patched" }
arrow-flight = { git = "https://github.com/alamb/arrow-rs.git", branch="alamb/23.0.0_patched" }

4
arrow_util/Cargo.toml
View File

@ -8,7 +8,7 @@ description = "Apache Arrow utilities"
[dependencies]
ahash = { version = "0.8.0", default-features = false, features = ["runtime-rng"] }
# need dyn_cmp_dict feature for comparing dictionary arrays
arrow = { version = "22.0.0", features = ["prettyprint", "dyn_cmp_dict"] }
arrow = { version = "23.0.0", features = ["prettyprint", "dyn_cmp_dict"] }
# used by arrow anyway (needed for printing workaround)
chrono = { version = "0.4", default-features = false }
comfy-table = { version = "6.1", default-features = false }
@ -19,5 +19,5 @@ snafu = "0.7"
workspace-hack = { path = "../workspace-hack"}
[dev-dependencies]
arrow-flight = "22.0.0"
arrow-flight = "23.0.0"
rand = "0.8.3"

3
arrow_util/src/util.rs
View File

@ -5,6 +5,7 @@ use std::sync::Arc;
use arrow::{
array::{new_null_array, ArrayRef, StringArray},
compute::concat_batches,
datatypes::SchemaRef,
error::ArrowError,
record_batch::RecordBatch,
@ -73,5 +74,5 @@ pub fn merge_record_batches(
return Ok(None);
}
Ok(Some(RecordBatch::concat(output_schema, &batches)?))
Ok(Some(concat_batches(output_schema, &batches)?))
}

2
compactor/Cargo.toml
View File

@ -5,7 +5,7 @@ authors = ["Luke Bond <luke.n.bond@gmail.com>"]
edition = "2021"
[dependencies]
arrow = { version = "22.0.0", features = ["prettyprint"] }
arrow = { version = "23.0.0", features = ["prettyprint"] }
async-trait = "0.1.57"
backoff = { path = "../backoff" }
bytes = "1.2"

6
datafusion/Cargo.toml
View File

@ -9,8 +9,6 @@ description = "Re-exports datafusion at a specific version"
# Rename to workaround doctest bug
# Turn off optional datafusion features (e.g. don't get support for crypto functions or avro)
upstream = { git = "https://github.com/apache/arrow-datafusion.git", rev="a6fbf24ddb81e34258a940a48e371b1efe5f6e49", default-features = false, package = "datafusion" }
datafusion-proto = { git = "https://github.com/apache/arrow-datafusion.git", rev="a6fbf24ddb81e34258a940a48e371b1efe5f6e49" }
upstream = { git = "https://github.com/apache/arrow-datafusion.git", rev="c7f3a70a79ee84070f7e8f770981fc982d8df0a4", default-features = false, package = "datafusion" }
datafusion-proto = { git = "https://github.com/apache/arrow-datafusion.git", rev="c7f3a70a79ee84070f7e8f770981fc982d8df0a4" }
workspace-hack = { path = "../workspace-hack"}

2
influxdb_iox/Cargo.toml
View File

@ -40,7 +40,7 @@ trogging = { path = "../trogging", default-features = false, features = ["clap"]
# Crates.io dependencies, in alphabetical order
nu-ansi-term = "0.46.0"
arrow = { version = "22.0.0", features = ["prettyprint"] }
arrow = { version = "23.0.0", features = ["prettyprint"] }
backtrace = "0.3"
bytes = "1.2"
clap = { version = "3", features = ["derive", "env"] }

4
influxdb_iox_client/Cargo.toml
View File

@ -20,8 +20,8 @@ mutable_batch_lp = { path = "../mutable_batch_lp", optional = true }
mutable_batch_pb = { path = "../mutable_batch_pb", optional = true }
# Crates.io dependencies, in alphabetical order
arrow = { version = "22.0.0", optional = true }
arrow-flight = { version = "22.0.0", optional = true }
arrow = { version = "23.0.0", optional = true }
arrow-flight = { version = "23.0.0", optional = true }
bytes = "1.2"
futures-util = { version = "0.3", optional = true }
prost = "0.11"

4
ingester/Cargo.toml
View File

@ -5,8 +5,8 @@ authors = ["Nga Tran <nga-tran@live.com>"]
edition = "2021"
[dependencies]
arrow = { version = "22.0.0", features = ["prettyprint"] }
arrow-flight = "22.0.0"
arrow = { version = "23.0.0", features = ["prettyprint"] }
arrow-flight = "23.0.0"
arrow_util = { path = "../arrow_util" }
async-trait = "0.1.57"
backoff = { path = "../backoff" }

9
ingester/src/data/partition.rs
View File

@ -243,10 +243,11 @@ impl PartitionData {
// Merge all result record batches into one record batch
// and make a snapshot for it
let snapshot = if !record_batches.is_empty() {
let record_batch = RecordBatch::concat(&record_batches[0].schema(), &record_batches)
.unwrap_or_else(|e| {
panic!("unable to concat record batches: {:?}", e);
});
let record_batch =
arrow::compute::concat_batches(&record_batches[0].schema(), &record_batches)
.unwrap_or_else(|e| {
panic!("unable to concat record batches: {:?}", e);
});
let snapshot = SnapshotBatch {
min_sequence_number,
max_sequence_number,

2
iox_query/Cargo.toml
View File

@ -14,7 +14,7 @@ description = "IOx Query Interface and Executor"
# 2. Allow for query logic testing without bringing in all the storage systems.
[dependencies] # In alphabetical order
arrow = { version = "22.0.0", features = ["prettyprint"] }
arrow = { version = "23.0.0", features = ["prettyprint"] }
arrow_util = { path = "../arrow_util" }
async-trait = "0.1"
chrono = { version = "0.4", default-features = false }

3
iox_query/src/exec/seriesset/converter.rs
View File

@ -100,7 +100,8 @@ impl SeriesSetConverter {
let batches = collect(it).await.context(ReadingSnafu)?;
let batch = if !batches.is_empty() {
RecordBatch::concat(&batches[0].schema(), &batches).context(ConcatenatingSnafu)?
arrow::compute::concat_batches(&batches[0].schema(), &batches)
.context(ConcatenatingSnafu)?
} else {
return Ok(vec![]);
};

2
iox_tests/Cargo.toml
View File

@ -6,7 +6,7 @@ edition = "2021"
description = "IOx test utils and tests"
[dependencies]
arrow = "22.0.0"
arrow = "23.0.0"
bytes = "1.2"
data_types = { path = "../data_types" }
datafusion = { path = "../datafusion" }

2
iox_tests/src/util.rs
View File

@ -961,5 +961,5 @@ fn dedup_batch(record_batch: RecordBatch, sort_key: &SortKey) -> RecordBatch {
batches.push(batch);
}
RecordBatch::concat(&schema, &batches).unwrap()
arrow::compute::concat_batches(&schema, &batches).unwrap()
}

2
ioxd_querier/Cargo.toml
View File

@ -22,7 +22,7 @@ iox_time = { path = "../iox_time" }
trace = { path = "../trace" }
# Crates.io dependencies, in alphabetical order
arrow-flight = "22.0.0"
arrow-flight = "23.0.0"
async-trait = "0.1"
hyper = "0.14"
thiserror = "1.0.36"

2
mutable_batch/Cargo.toml
View File

@ -5,7 +5,7 @@ edition = "2021"
description = "A mutable arrow RecordBatch"
[dependencies]
arrow = { version = "22.0.0", features = ["prettyprint"] }
arrow = { version = "23.0.0", features = ["prettyprint"] }
arrow_util = { path = "../arrow_util" }
chrono = { version = "0.4", default-features = false }
data_types = { path = "../data_types" }

4
packers/Cargo.toml
View File

@ -5,11 +5,11 @@ authors = ["Andrew Lamb <andrew@nerdnetworks.org>"]
edition = "2021"
[dependencies] # In alphabetical order
arrow = { version = "22.0.0", features = ["prettyprint"] }
arrow = { version = "23.0.0", features = ["prettyprint"] }
influxdb_tsm = { path = "../influxdb_tsm" }
schema = { path = "../schema" }
snafu = "0.7"
parquet = "22.0.0"
parquet = "23.0.0"
workspace-hack = { path = "../workspace-hack"}
[dev-dependencies] # In alphabetical order

7
parquet_file/Cargo.toml
View File

@ -5,7 +5,7 @@ authors = ["Nga Tran <nga-tran@live.com>"]
edition = "2021"
[dependencies] # In alphabetical order
arrow = { version = "22.0.0", features = ["prettyprint"] }
arrow = { version = "23.0.0", features = ["prettyprint"] }
base64 = "0.13"
bytes = "1.2"
data_types = { path = "../data_types" }
@ -17,14 +17,13 @@ iox_time = { path = "../iox_time" }
object_store = "0.5.0"
observability_deps = { path = "../observability_deps" }
parking_lot = "0.12"
parquet = {version = "22.0.0", features = ["experimental"]}
parquet-format = "4.0"
parquet = {version = "23.0.0", features = ["experimental"]}
pbjson-types = "0.5"
predicate = { path = "../predicate" }
prost = "0.11"
schema = { path = "../schema" }
snafu = "0.7"
thrift = "0.13"
thrift = "0.16"
tokio = { version = "1.21", features = ["macros", "parking_lot", "rt", "rt-multi-thread", "sync"] }
uuid = { version = "1", features = ["v4"] }
zstd = "0.11"

8
parquet_file/src/metadata.rs
View File

@ -604,7 +604,7 @@ impl IoxParquetMetaData {
.map(|rg| rg.to_thrift())
.collect();
let thrift_file_metadata = parquet_format::FileMetaData {
let thrift_file_metadata = parquet::format::FileMetaData {
version: file_metadata.version(),
schema: thrift_schema,
@ -635,7 +635,7 @@ impl IoxParquetMetaData {
// step 2: load thrift data from byte stream
let thrift_file_metadata = {
let mut protocol = TCompactInputProtocol::new(&data[..]);
parquet_format::FileMetaData::read_from_in_protocol(&mut protocol)
parquet::format::FileMetaData::read_from_in_protocol(&mut protocol)
.context(ThriftReadFailureSnafu {})?
};
@ -674,10 +674,10 @@ impl IoxParquetMetaData {
}
}
impl TryFrom<parquet_format::FileMetaData> for IoxParquetMetaData {
impl TryFrom<parquet::format::FileMetaData> for IoxParquetMetaData {
type Error = Error;
fn try_from(v: parquet_format::FileMetaData) -> Result<Self, Self::Error> {
fn try_from(v: parquet::format::FileMetaData) -> Result<Self, Self::Error> {
let mut buffer = Vec::new();
{
let mut protocol = TCompactOutputProtocol::new(&mut buffer);

6
parquet_file/src/serialize.rs
View File

@ -81,13 +81,13 @@ pub enum CodecError {
/// data was serialised.
///
/// [`proto::IoxMetadata`]: generated_types::influxdata::iox::ingester::v1
/// [`FileMetaData`]: parquet_format::FileMetaData
/// [`FileMetaData`]: parquet::format::FileMetaData
/// [`IoxParquetMetaData`]: crate::metadata::IoxParquetMetaData
pub async fn to_parquet<S, W>(
batches: S,
meta: &IoxMetadata,
sink: W,
) -> Result<parquet_format::FileMetaData, CodecError>
) -> Result<parquet::format::FileMetaData, CodecError>
where
S: Stream<Item = Result<RecordBatch, ArrowError>> + Send,
W: Write + Send,
@ -148,7 +148,7 @@ where
pub async fn to_parquet_bytes<S>(
batches: S,
meta: &IoxMetadata,
) -> Result<(Vec<u8>, parquet_format::FileMetaData), CodecError>
) -> Result<(Vec<u8>, parquet::format::FileMetaData), CodecError>
where
S: Stream<Item = Result<RecordBatch, ArrowError>> + Send,
{

2
predicate/Cargo.toml
View File

@ -4,7 +4,7 @@ version = "0.1.0"
edition = "2021"
[dependencies]
arrow = { version = "22.0.0", features = ["prettyprint"] }
arrow = { version = "23.0.0", features = ["prettyprint"] }
chrono = { version = "0.4", default-features = false }
data_types = { path = "../data_types" }
datafusion = { path = "../datafusion" }

2
querier/Cargo.toml
View File

@ -4,7 +4,7 @@ version = "0.1.0"
edition = "2021"
[dependencies]
arrow = "22.0.0"
arrow = "23.0.0"
async-trait = "0.1.57"
backoff = { path = "../backoff" }
bytes = "1.2"

2
query_functions/Cargo.toml
View File

@ -6,7 +6,7 @@ edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
arrow = { version = "22.0.0", features = ["prettyprint"] }
arrow = { version = "23.0.0", features = ["prettyprint"] }
chrono = { version = "0.4", default-features = false }
datafusion = { path = "../datafusion" }
itertools = "0.10.5"

4
query_tests/Cargo.toml
View File

@ -6,7 +6,7 @@ edition = "2021"
description = "Tests of the query engine against different database configurations"
[dependencies]
arrow = { version = "22.0.0", features = ["prettyprint"] }
arrow = { version = "23.0.0", features = ["prettyprint"] }
arrow_util = { path = "../arrow_util" }
async-trait = "0.1"
backoff = { path = "../backoff" }
@ -34,7 +34,7 @@ workspace-hack = { path = "../workspace-hack"}
parquet_file = { version = "0.1.0", path = "../parquet_file" }
[dev-dependencies]
arrow = { version = "22.0.0", features = ["prettyprint"] }
arrow = { version = "23.0.0", features = ["prettyprint"] }
arrow_util = { path = "../arrow_util" }
snafu = "0.7"
tempfile = "3.1.0"

3
query_tests/src/runner.rs
View File

@ -285,7 +285,8 @@ impl<W: Write> Runner<W> {
// compare against sorted results, if requested
if query.sorted_compare() && !results.is_empty() {
let schema = results[0].schema();
let batch = RecordBatch::concat(&schema, &results).expect("concatenating batches");
let batch = arrow::compute::concat_batches(&schema, &results)
.expect("concatenating batches");
results = vec![sort_record_batch(batch)];
}

2
read_buffer/Cargo.toml
View File

@ -11,7 +11,7 @@ edition = "2021"
# 2. Keep change/compile/link time down during development when working on just this crate
[dependencies] # In alphabetical order
arrow = { version = "22.0.0", features = ["prettyprint"] }
arrow = { version = "23.0.0", features = ["prettyprint"] }
arrow_util = { path = "../arrow_util" }
croaring = "0.6"
data_types = { path = "../data_types" }

3
read_buffer/src/chunk.rs
View File

@ -407,8 +407,7 @@ impl ChunkBuilder {
// contiguous block of memory. To reduce memory overhead we can avoid
// the concatenation and teach the `RowGroup` to build itself from a
// vec of record batches.
let concat_batch =
arrow::record_batch::RecordBatch::concat(&self.schema, &self.record_batches)?;
let concat_batch = arrow::compute::concat_batches(&self.schema, &self.record_batches)?;
self.row_groups
.push(record_batch_to_row_group(concat_batch));

2
schema/Cargo.toml
View File

@ -6,7 +6,7 @@ edition = "2021"
description = "IOx Schema definition"
[dependencies]
arrow = { version = "22.0.0", features = ["prettyprint"] }
arrow = { version = "23.0.0", features = ["prettyprint"] }
hashbrown = "0.12"
indexmap = { version = "1.9", features = ["std"] }
itertools = "0.10.5"

4
service_grpc_flight/Cargo.toml
View File

@ -19,8 +19,8 @@ trace_http = { path = "../trace_http"}
tracker = { path = "../tracker" }
# Crates.io dependencies, in alphabetical order
arrow = { version = "22.0.0", features = ["prettyprint"] }
arrow-flight = "22.0.0"
arrow = { version = "23.0.0", features = ["prettyprint"] }
arrow-flight = "23.0.0"
bytes = "1.2"
futures = "0.3"
pin-project = "1.0"

2
service_grpc_influxrpc/Cargo.toml
View File

@ -19,7 +19,7 @@ trace_http = { path = "../trace_http"}
tracker = { path = "../tracker" }
# Crates.io dependencies, in alphabetical order
arrow = { version = "22.0.0", features = ["prettyprint"] }
arrow = { version = "23.0.0", features = ["prettyprint"] }
async-trait = "0.1"
futures = "0.3"
pin-project = "1.0"

2
test_helpers_end_to_end/Cargo.toml
View File

@ -4,7 +4,7 @@ version = "0.1.0"
edition = "2021"
[dependencies] # In alphabetical order
arrow = { version = "22.0.0", features = ["prettyprint"] }
arrow = { version = "23.0.0", features = ["prettyprint"] }
arrow_util = { path = "../arrow_util" }
assert_cmd = "2.0.2"
bytes = "1.2"

2
trace_exporters/Cargo.toml
View File

@ -13,7 +13,7 @@ clap = { version = "3", features = ["derive", "env"] }
futures = "0.3"
observability_deps = { path = "../observability_deps" }
snafu = "0.7"
thrift = { version = "0.13.0" }
thrift = { version = "0.16.0" }
tokio = { version = "1.21", features = ["macros", "parking_lot", "rt", "sync"] }
trace = { path = "../trace" }
workspace-hack = { path = "../workspace-hack"}

5
workspace-hack/Cargo.toml
View File

@ -14,7 +14,7 @@ publish = false
### BEGIN HAKARI SECTION
[dependencies]
ahash-ca01ad9e24f5d932 = { package = "ahash", version = "0.7", features = ["std"] }
arrow = { version = "22", features = ["comfy-table", "csv", "csv_crate", "dyn_cmp_dict", "flatbuffers", "ipc", "json", "prettyprint", "serde", "serde_json"] }
arrow = { git = "https://github.com/alamb/arrow-rs.git", branch = "alamb/23.0.0_patched", features = ["comfy-table", "csv", "csv_crate", "dyn_cmp_dict", "flatbuffers", "ipc", "json", "prettyprint", "serde", "serde_json"] }
base64 = { version = "0.13", features = ["std"] }
bitflags = { version = "1" }
byteorder = { version = "1", features = ["std"] }
@ -45,7 +45,7 @@ num-traits = { version = "0.2", features = ["i128", "libm", "std"] }
object_store = { version = "0.5", default-features = false, features = ["aws", "azure", "base64", "cloud", "gcp", "quick-xml", "rand", "reqwest", "ring", "rustls-pemfile", "serde", "serde_json"] }
once_cell = { version = "1", features = ["alloc", "parking_lot", "parking_lot_core", "race", "std"] }
parking_lot = { version = "0.12", features = ["arc_lock"] }
parquet = { version = "22", features = ["arrow", "async", "base64", "brotli", "experimental", "flate2", "futures", "lz4", "snap", "tokio", "zstd"] }
parquet = { git = "https://github.com/alamb/arrow-rs.git", branch = "alamb/23.0.0_patched", features = ["arrow", "async", "base64", "brotli", "experimental", "flate2", "futures", "lz4", "snap", "tokio", "zstd"] }
predicates = { version = "2", features = ["diff", "difflib", "float-cmp", "normalize-line-endings", "regex"] }
prost = { version = "0.11", features = ["prost-derive", "std"] }
prost-types = { version = "0.11", features = ["std"] }
@ -61,6 +61,7 @@ sha2 = { version = "0.10", features = ["std"] }
smallvec = { version = "1", default-features = false, features = ["union"] }
sqlx = { version = "0.6", features = ["_rt-tokio", "json", "macros", "migrate", "postgres", "runtime-tokio-rustls", "sqlx-macros", "tls", "uuid"] }
sqlx-core = { version = "0.6", default-features = false, features = ["_rt-tokio", "_tls-rustls", "any", "base64", "crc", "dirs", "hkdf", "hmac", "json", "md-5", "migrate", "postgres", "rand", "runtime-tokio-rustls", "rustls", "rustls-pemfile", "serde", "serde_json", "sha1", "sha2", "tokio-stream", "uuid", "webpki-roots", "whoami"] }
thrift = { version = "0.16", features = ["log", "server", "threadpool"] }
tokio = { version = "1", features = ["bytes", "fs", "io-std", "io-util", "libc", "macros", "memchr", "mio", "net", "num_cpus", "once_cell", "parking_lot", "rt", "rt-multi-thread", "signal", "signal-hook-registry", "socket2", "sync", "time", "tokio-macros", "tracing"] }
tokio-stream = { version = "0.1", features = ["fs", "net", "time"] }
tokio-util = { version = "0.7", features = ["codec", "tracing"] }