Influxdata
/
influxdb
mirror of https://github.com/influxdata/influxdb.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

chore: sync latest core (#25138) 

* chore: sync latest core

* chore: clippy
pull/25126/merge
Trevor Hilton 2024-07-10 12:25:09 -04:00 committed by GitHub
parent cd6734a7c4
commit 8fd50cefe1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 455 additions and 483 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

810
Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

112
Cargo.toml
View File

@ -32,13 +32,13 @@ license = "MIT OR Apache-2.0"
[workspace.dependencies]
anyhow = "1.0"
arrow = { version = "51.0.0", features = ["prettyprint", "chrono-tz"] }
arrow-array = "51.0.0"
arrow-buffer = "51.0.0"
arrow-csv = "51.0.0"
arrow-flight = { version = "51.0.0", features = ["flight-sql-experimental"] }
arrow-json = "51.0.0"
arrow-schema = "51.0.0"
arrow = { version = "52.1.0", features = ["prettyprint", "chrono-tz"] }
arrow-array = "52.1.0"
arrow-buffer = "52.1.0"
arrow-csv = "52.1.0"
arrow-flight = { version = "52.1.0", features = ["flight-sql-experimental"] }
arrow-json = "52.1.0"
arrow-schema = "52.1.0"
assert_cmd = "2.0.14"
async-trait = "0.1"
backtrace = "0.3"
@ -49,8 +49,8 @@ chrono = "0.4"
clap = { version = "4", features = ["derive", "env", "string"] }
crc32fast = "1.2.0"
crossbeam-channel = "0.5.11"
datafusion = { git = "https://github.com/influxdata/arrow-datafusion.git", rev = "e495f1f6bfa8e65f4058829912283ee98ab46318" }
datafusion-proto = { git = "https://github.com/influxdata/arrow-datafusion.git", rev = "e495f1f6bfa8e65f4058829912283ee98ab46318" }
datafusion = { git = "https://github.com/apache/datafusion.git", rev = "a64df83502821f18067fb4ff65dd217815b305c9" }
datafusion-proto = { git = "https://github.com/apache/datafusion.git", rev = "a64df83502821f18067fb4ff65dd217815b305c9" }
csv = "1.3.0"
dotenvy = "0.15.7"
flate2 = "1.0.27"
@ -66,10 +66,10 @@ indexmap = { version = "2.2.6" }
libc = { version = "0.2" }
mockito = { version = "1.4.0", default-features = false }
num_cpus = "1.16.0"
object_store = "0.9.1"
object_store = "0.10.1"
once_cell = { version = "1.18", features = ["parking_lot"] }
parking_lot = "0.12.1"
parquet = { version = "51.0.0", features = ["object_store"] }
parquet = { version = "52.1.0", features = ["object_store"] }
pbjson = "0.6.0"
pbjson-build = "0.6.2"
pbjson-types = "0.6.0"
@ -87,7 +87,7 @@ serde_urlencoded = "0.7.0"
serde_with = "3.8.1"
sha2 = "0.10.8"
snap = "1.0.0"
sqlparser = "0.47.0"
sqlparser = "0.48.0"
sysinfo = "0.30.8"
thiserror = "1.0"
tokio = { version = "1.35", features = ["full"] }
@ -106,36 +106,36 @@ uuid = { version = "1", features = ["v4"] }
# Currently influxdb is pointed at a revision from the experimental branch
# in influxdb3_core, hiltontj/17-june-2024-iox-sync-exp, instead of main.
# See https://github.com/influxdata/influxdb3_core/pull/23
arrow_util = { git = "https://github.com/influxdata/influxdb3_core", rev = "ca040f2a6e5b6470e469825a8d35060d0199297c"}
authz = { git = "https://github.com/influxdata/influxdb3_core", rev = "ca040f2a6e5b6470e469825a8d35060d0199297c", features = ["http"] }
clap_blocks = { git = "https://github.com/influxdata/influxdb3_core", rev = "ca040f2a6e5b6470e469825a8d35060d0199297c" }
data_types = { git = "https://github.com/influxdata/influxdb3_core", rev = "ca040f2a6e5b6470e469825a8d35060d0199297c" }
datafusion_util = { git = "https://github.com/influxdata/influxdb3_core", rev = "ca040f2a6e5b6470e469825a8d35060d0199297c" }
influxdb-line-protocol = { git = "https://github.com/influxdata/influxdb3_core", rev = "ca040f2a6e5b6470e469825a8d35060d0199297c" }
influxdb_influxql_parser = { git = "https://github.com/influxdata/influxdb3_core", rev = "ca040f2a6e5b6470e469825a8d35060d0199297c" }
influxdb_iox_client = { git = "https://github.com/influxdata/influxdb3_core", rev = "ca040f2a6e5b6470e469825a8d35060d0199297c" }
iox_catalog = { git = "https://github.com/influxdata/influxdb3_core", rev = "ca040f2a6e5b6470e469825a8d35060d0199297c" }
iox_http = { git = "https://github.com/influxdata/influxdb3_core", rev = "ca040f2a6e5b6470e469825a8d35060d0199297c" }
iox_query = { git = "https://github.com/influxdata/influxdb3_core", rev = "ca040f2a6e5b6470e469825a8d35060d0199297c" }
iox_query_params = { git = "https://github.com/influxdata/influxdb3_core", rev = "ca040f2a6e5b6470e469825a8d35060d0199297c" }
iox_query_influxql = { git = "https://github.com/influxdata/influxdb3_core", rev = "ca040f2a6e5b6470e469825a8d35060d0199297c" }
iox_system_tables = { git = "https://github.com/influxdata/influxdb3_core", rev = "ca040f2a6e5b6470e469825a8d35060d0199297c" }
iox_time = { git = "https://github.com/influxdata/influxdb3_core", rev = "ca040f2a6e5b6470e469825a8d35060d0199297c" }
metric = { git = "https://github.com/influxdata/influxdb3_core", rev = "ca040f2a6e5b6470e469825a8d35060d0199297c" }
metric_exporters = { git = "https://github.com/influxdata/influxdb3_core", rev = "ca040f2a6e5b6470e469825a8d35060d0199297c" }
observability_deps = { git = "https://github.com/influxdata/influxdb3_core", rev = "ca040f2a6e5b6470e469825a8d35060d0199297c" }
panic_logging = { git = "https://github.com/influxdata/influxdb3_core", rev = "ca040f2a6e5b6470e469825a8d35060d0199297c" }
parquet_file = { git = "https://github.com/influxdata/influxdb3_core", rev = "ca040f2a6e5b6470e469825a8d35060d0199297c" }
schema = { git = "https://github.com/influxdata/influxdb3_core", rev = "ca040f2a6e5b6470e469825a8d35060d0199297c" }
service_common = { git = "https://github.com/influxdata/influxdb3_core", rev = "ca040f2a6e5b6470e469825a8d35060d0199297c" }
service_grpc_flight = { git = "https://github.com/influxdata/influxdb3_core", rev = "ca040f2a6e5b6470e469825a8d35060d0199297c" }
test_helpers = { git = "https://github.com/influxdata/influxdb3_core", rev = "ca040f2a6e5b6470e469825a8d35060d0199297c" }
tokio_metrics_bridge = { git = "https://github.com/influxdata/influxdb3_core", rev = "ca040f2a6e5b6470e469825a8d35060d0199297c" }
trace = { git = "https://github.com/influxdata/influxdb3_core", rev = "ca040f2a6e5b6470e469825a8d35060d0199297c" }
trace_exporters = { git = "https://github.com/influxdata/influxdb3_core", rev = "ca040f2a6e5b6470e469825a8d35060d0199297c" }
trace_http = { git = "https://github.com/influxdata/influxdb3_core", rev = "ca040f2a6e5b6470e469825a8d35060d0199297c" }
tracker = { git = "https://github.com/influxdata/influxdb3_core", rev = "ca040f2a6e5b6470e469825a8d35060d0199297c" }
trogging = { git = "https://github.com/influxdata/influxdb3_core", rev = "ca040f2a6e5b6470e469825a8d35060d0199297c", default-features = true, features = ["clap"] }
arrow_util = { git = "https://github.com/influxdata/influxdb3_core", rev = "d81f63ddc10e3cf1c28b05e6c1cef03b71da7f8a"}
authz = { git = "https://github.com/influxdata/influxdb3_core", rev = "d81f63ddc10e3cf1c28b05e6c1cef03b71da7f8a", features = ["http"] }
clap_blocks = { git = "https://github.com/influxdata/influxdb3_core", rev = "d81f63ddc10e3cf1c28b05e6c1cef03b71da7f8a" }
data_types = { git = "https://github.com/influxdata/influxdb3_core", rev = "d81f63ddc10e3cf1c28b05e6c1cef03b71da7f8a" }
datafusion_util = { git = "https://github.com/influxdata/influxdb3_core", rev = "d81f63ddc10e3cf1c28b05e6c1cef03b71da7f8a" }
influxdb-line-protocol = { git = "https://github.com/influxdata/influxdb3_core", rev = "d81f63ddc10e3cf1c28b05e6c1cef03b71da7f8a" }
influxdb_influxql_parser = { git = "https://github.com/influxdata/influxdb3_core", rev = "d81f63ddc10e3cf1c28b05e6c1cef03b71da7f8a" }
influxdb_iox_client = { git = "https://github.com/influxdata/influxdb3_core", rev = "d81f63ddc10e3cf1c28b05e6c1cef03b71da7f8a" }
iox_catalog = { git = "https://github.com/influxdata/influxdb3_core", rev = "d81f63ddc10e3cf1c28b05e6c1cef03b71da7f8a" }
iox_http = { git = "https://github.com/influxdata/influxdb3_core", rev = "d81f63ddc10e3cf1c28b05e6c1cef03b71da7f8a" }
iox_query = { git = "https://github.com/influxdata/influxdb3_core", rev = "d81f63ddc10e3cf1c28b05e6c1cef03b71da7f8a" }
iox_query_params = { git = "https://github.com/influxdata/influxdb3_core", rev = "d81f63ddc10e3cf1c28b05e6c1cef03b71da7f8a" }
iox_query_influxql = { git = "https://github.com/influxdata/influxdb3_core", rev = "d81f63ddc10e3cf1c28b05e6c1cef03b71da7f8a" }
iox_system_tables = { git = "https://github.com/influxdata/influxdb3_core", rev = "d81f63ddc10e3cf1c28b05e6c1cef03b71da7f8a" }
iox_time = { git = "https://github.com/influxdata/influxdb3_core", rev = "d81f63ddc10e3cf1c28b05e6c1cef03b71da7f8a" }
metric = { git = "https://github.com/influxdata/influxdb3_core", rev = "d81f63ddc10e3cf1c28b05e6c1cef03b71da7f8a" }
metric_exporters = { git = "https://github.com/influxdata/influxdb3_core", rev = "d81f63ddc10e3cf1c28b05e6c1cef03b71da7f8a" }
observability_deps = { git = "https://github.com/influxdata/influxdb3_core", rev = "d81f63ddc10e3cf1c28b05e6c1cef03b71da7f8a" }
panic_logging = { git = "https://github.com/influxdata/influxdb3_core", rev = "d81f63ddc10e3cf1c28b05e6c1cef03b71da7f8a" }
parquet_file = { git = "https://github.com/influxdata/influxdb3_core", rev = "d81f63ddc10e3cf1c28b05e6c1cef03b71da7f8a" }
schema = { git = "https://github.com/influxdata/influxdb3_core", rev = "d81f63ddc10e3cf1c28b05e6c1cef03b71da7f8a" }
service_common = { git = "https://github.com/influxdata/influxdb3_core", rev = "d81f63ddc10e3cf1c28b05e6c1cef03b71da7f8a" }
service_grpc_flight = { git = "https://github.com/influxdata/influxdb3_core", rev = "d81f63ddc10e3cf1c28b05e6c1cef03b71da7f8a" }
test_helpers = { git = "https://github.com/influxdata/influxdb3_core", rev = "d81f63ddc10e3cf1c28b05e6c1cef03b71da7f8a" }
tokio_metrics_bridge = { git = "https://github.com/influxdata/influxdb3_core", rev = "d81f63ddc10e3cf1c28b05e6c1cef03b71da7f8a" }
trace = { git = "https://github.com/influxdata/influxdb3_core", rev = "d81f63ddc10e3cf1c28b05e6c1cef03b71da7f8a" }
trace_exporters = { git = "https://github.com/influxdata/influxdb3_core", rev = "d81f63ddc10e3cf1c28b05e6c1cef03b71da7f8a" }
trace_http = { git = "https://github.com/influxdata/influxdb3_core", rev = "d81f63ddc10e3cf1c28b05e6c1cef03b71da7f8a" }
tracker = { git = "https://github.com/influxdata/influxdb3_core", rev = "d81f63ddc10e3cf1c28b05e6c1cef03b71da7f8a" }
trogging = { git = "https://github.com/influxdata/influxdb3_core", rev = "d81f63ddc10e3cf1c28b05e6c1cef03b71da7f8a", default-features = true, features = ["clap"] }
[workspace.lints.rust]
rust_2018_idioms = "deny"
@ -176,13 +176,21 @@ opt-level = 3
[profile.dev.package.similar]
opt-level = 3
# patch arrow-flight crate to allow for prepared statement parameters
# see related arrow-rs PR https://github.com/apache/arrow-rs/pull/5433
[patch.crates-io]
arrow-array = { git = "https://github.com/influxdata/arrow-rs.git", rev = "ea454d74707357731535d4bf20e9508e838f5f5d" }
arrow-schema = { git = "https://github.com/influxdata/arrow-rs.git", rev = "ea454d74707357731535d4bf20e9508e838f5f5d" }
arrow-data = { git = "https://github.com/influxdata/arrow-rs.git", rev = "ea454d74707357731535d4bf20e9508e838f5f5d" }
arrow-buffer = { git = "https://github.com/influxdata/arrow-rs.git", rev = "ea454d74707357731535d4bf20e9508e838f5f5d" }
arrow-ipc = { git = "https://github.com/influxdata/arrow-rs.git", rev = "ea454d74707357731535d4bf20e9508e838f5f5d" }
arrow-flight = { git = "https://github.com/influxdata/arrow-rs.git", rev = "ea454d74707357731535d4bf20e9508e838f5f5d" }
parquet = { git = "https://github.com/influxdata/arrow-rs.git", rev = "ea454d74707357731535d4bf20e9508e838f5f5d" }
# Patching Arrow
#
# Assuming you have a local checkout of Arrow in a directory alongside your local checkout of influxdb3_core,
# and you have changes to Arrow in your local checkout that you want to test out with influxdb3_core,
# uncomment this `[patch.crates-io]` section to tell Cargo to use your local arrow versions for all
# transitive dependencies. The entries for the `arrow-*` crates are needed because `datafusion` has
# a direct dependency on them.
#
# WARNING: Do not merge in a PR uncommenting this change! This is for local testing only!
#
# [patch.crates-io]
# arrow = { path = "../arrow-rs/arrow" }
# parquet = { path = "../arrow-rs/parquet" }
# arrow-array = { path = "../arrow-rs/arrow-array" }
# arrow-schema = { path = "../arrow-rs/arrow-schema" }
# arrow-data = { path = "../arrow-rs/arrow-data" }
# arrow-buffer = { path = "../arrow-rs/arrow-buffer" }
# arrow-ipc = { path = "../arrow-rs/arrow-ipc" }

2
influxdb3/tests/server/system_tables.rs
View File

@ -31,7 +31,7 @@ async fn queries_table() {
assert_batches_sorted_eq!(
[
"+----------+",
"| COUNT(*) |",
"| count(*) |",
"+----------+",
"| 0 |",
"+----------+",

2
influxdb3_write/src/cache.rs
View File

@ -72,7 +72,7 @@ impl ParquetCache {
let path = parquet_path.to_string();
task::block_in_place(move || -> Result<_, Error> {
Handle::current()
.block_on(self.object_store.put(&parquet_path, parquet.bytes))
.block_on(self.object_store.put(&parquet_path, parquet.bytes.into()))
.map_err(Into::into)
})?;

4
influxdb3_write/src/catalog/serialize.rs
View File

@ -173,9 +173,7 @@ impl<'a> From<&'a ArrowDataType> for DataType<'a> {
ArrowDataType::Float16 => Self::F16,
ArrowDataType::Float32 => Self::F32,
ArrowDataType::Float64 => Self::F64,
// Arrow's TimeUnit does not impl Copy, so we cheaply clone it:
// See <https://github.com/apache/arrow-rs/issues/5839>
ArrowDataType::Timestamp(unit, tz) => Self::Time(unit.clone().into(), tz.as_deref()),
ArrowDataType::Timestamp(unit, tz) => Self::Time((*unit).into(), tz.as_deref()),
ArrowDataType::Date32 => todo!(),
ArrowDataType::Date64 => todo!(),
ArrowDataType::Time32(_) => todo!(),

8
influxdb3_write/src/persister.rs
View File

@ -232,7 +232,7 @@ impl Persister for PersisterImpl {
let catalog_path = CatalogFilePath::new(segment_id);
let json = serde_json::to_vec_pretty(&catalog)?;
self.object_store
.put(catalog_path.as_ref(), Bytes::from(json))
.put(catalog_path.as_ref(), json.into())
.await?;
Ok(())
}
@ -241,7 +241,7 @@ impl Persister for PersisterImpl {
let segment_file_path = SegmentInfoFilePath::new(persisted_segment.segment_id);
let json = serde_json::to_vec_pretty(persisted_segment)?;
self.object_store
.put(segment_file_path.as_ref(), Bytes::from(json))
.put(segment_file_path.as_ref(), json.into())
.await?;
Ok(())
}
@ -253,7 +253,9 @@ impl Persister for PersisterImpl {
) -> Result<(u64, FileMetaData)> {
let parquet = self.serialize_to_parquet(record_batch).await?;
let bytes_written = parquet.bytes.len() as u64;
self.object_store.put(path.as_ref(), parquet.bytes).await?;
self.object_store
.put(path.as_ref(), parquet.bytes.into())
.await?;
Ok((bytes_written, parquet.meta_data))
}