Influxdata
/
influxdb
mirror of https://github.com/influxdata/influxdb.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

chore: merge main to branch

pull/24376/head
Nga Tran 2021-05-19 15:11:15 -04:00
parent 1f13842550 41093b54d6
commit 11561111d5
26 changed files with 1739 additions and 373 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

256
Cargo.lock generated
View File

@ -31,6 +31,17 @@ version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
[[package]]
name = "ahash"
version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "796540673305a66d127804eef19ad696f1f204b8c1025aaca4958c17eab32877"
dependencies = [
"getrandom 0.2.2",
"once_cell",
"version_check",
]
[[package]] [[package]]
name = "ahash" name = "ahash"
version = "0.7.2" version = "0.7.2"
@ -96,6 +107,15 @@ version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544"
[[package]]
name = "arrayvec"
version = "0.4.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd9fd44efafa8690358b7408d253adf110036b88f55672a933f01d616ad9b1b9"
dependencies = [
"nodrop",
]
[[package]] [[package]]
name = "arrayvec" name = "arrayvec"
version = "0.5.2" version = "0.5.2"
@ -159,7 +179,7 @@ dependencies = [
name = "arrow_util" name = "arrow_util"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"ahash", "ahash 0.7.2",
"arrow 0.1.0", "arrow 0.1.0",
"futures", "futures",
"hashbrown 0.11.2", "hashbrown 0.11.2",
@ -193,6 +213,15 @@ dependencies = [
"wait-timeout", "wait-timeout",
] ]
[[package]]
name = "async-mutex"
version = "1.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "479db852db25d9dbf6204e6cb6253698f175c15726470f78af0d918e99d6156e"
dependencies = [
"event-listener",
]
[[package]] [[package]]
name = "async-stream" name = "async-stream"
version = "0.3.1" version = "0.3.1"
@ -372,7 +401,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "afa748e348ad3be8263be728124b24a24f268266f6f5d58af9d75f6a40b5c587" checksum = "afa748e348ad3be8263be728124b24a24f268266f6f5d58af9d75f6a40b5c587"
dependencies = [ dependencies = [
"arrayref", "arrayref",
"arrayvec", "arrayvec 0.5.2",
"constant_time_eq", "constant_time_eq",
] ]
@ -424,6 +453,12 @@ version = "3.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "63396b8a4b9de3f4fdfb320ab6080762242f66a8ef174c49d8e19b674db4cdbe" checksum = "63396b8a4b9de3f4fdfb320ab6080762242f66a8ef174c49d8e19b674db4cdbe"
[[package]]
name = "bytemuck"
version = "1.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bed57e2090563b83ba8f83366628ce535a7584c9afa4c9fc0612a03925c6df58"
[[package]] [[package]]
name = "byteorder" name = "byteorder"
version = "1.4.3" version = "1.4.3"
@ -439,6 +474,40 @@ dependencies = [
"serde", "serde",
] ]
[[package]]
name = "cached"
version = "0.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e2afe73808fbaac302e39c9754bfc3c4b4d0f99c9c240b9f4e4efc841ad1b74"
dependencies = [
"async-mutex",
"async-trait",
"cached_proc_macro",
"cached_proc_macro_types",
"futures",
"hashbrown 0.9.1",
"once_cell",
]
[[package]]
name = "cached_proc_macro"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf857ae42d910aede5c5186e62684b0d7a597ce2fe3bd14448ab8f7ef439848c"
dependencies = [
"async-mutex",
"cached_proc_macro_types",
"darling",
"quote",
"syn",
]
[[package]]
name = "cached_proc_macro_types"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a4f925191b4367301851c6d99b09890311d74b0d43f274c0b34c86d308a3663"
[[package]] [[package]]
name = "cast" name = "cast"
version = "0.2.5" version = "0.2.5"
@ -512,7 +581,7 @@ dependencies = [
"ansi_term 0.11.0", "ansi_term 0.11.0",
"atty", "atty",
"bitflags", "bitflags",
"strsim", "strsim 0.8.0",
"textwrap", "textwrap",
"unicode-width", "unicode-width",
"vec_map", "vec_map",
@ -767,6 +836,41 @@ dependencies = [
"sct", "sct",
] ]
[[package]]
name = "darling"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0d706e75d87e35569db781a9b5e2416cff1236a47ed380831f959382ccd5f858"
dependencies = [
"darling_core",
"darling_macro",
]
[[package]]
name = "darling_core"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f0c960ae2da4de88a91b2d920c2a7233b400bc33cb28453a2987822d8392519b"
dependencies = [
"fnv",
"ident_case",
"proc-macro2",
"quote",
"strsim 0.9.3",
"syn",
]
[[package]]
name = "darling_macro"
version = "0.10.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d9b5a2f4ac4969822c62224815d069952656cadc7084fdca9751e6d959189b72"
dependencies = [
"darling_core",
"quote",
"syn",
]
[[package]] [[package]]
name = "dashmap" name = "dashmap"
version = "4.0.2" version = "4.0.2"
@ -804,7 +908,7 @@ name = "datafusion"
version = "4.0.0-SNAPSHOT" version = "4.0.0-SNAPSHOT"
source = "git+https://github.com/apache/arrow-datafusion.git?rev=9cf32cf2cda8472b87130142c4eee1126d4d9cbe#9cf32cf2cda8472b87130142c4eee1126d4d9cbe" source = "git+https://github.com/apache/arrow-datafusion.git?rev=9cf32cf2cda8472b87130142c4eee1126d4d9cbe#9cf32cf2cda8472b87130142c4eee1126d4d9cbe"
dependencies = [ dependencies = [
"ahash", "ahash 0.7.2",
"arrow 4.0.0-SNAPSHOT", "arrow 4.0.0-SNAPSHOT",
"async-trait", "async-trait",
"chrono", "chrono",
@ -830,6 +934,15 @@ dependencies = [
"futures", "futures",
] ]
[[package]]
name = "debugid"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f91cf5a8c2f2097e2a32627123508635d47ce10563d999ec1a95addf08b502ba"
dependencies = [
"uuid",
]
[[package]] [[package]]
name = "difference" name = "difference"
version = "2.0.0" version = "2.0.0"
@ -975,6 +1088,12 @@ dependencies = [
"termcolor", "termcolor",
] ]
[[package]]
name = "event-listener"
version = "2.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7531096570974c3a9dcf9e4b8e1cede1ec26cf5046219fb3b9d897503b9be59"
[[package]] [[package]]
name = "extend" name = "extend"
version = "0.1.2" version = "0.1.2"
@ -1319,7 +1438,7 @@ version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
dependencies = [ dependencies = [
"ahash", "ahash 0.7.2",
] ]
[[package]] [[package]]
@ -1456,6 +1575,12 @@ dependencies = [
"tokio-native-tls", "tokio-native-tls",
] ]
[[package]]
name = "ident_case"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
[[package]] [[package]]
name = "idna" name = "idna"
version = "0.2.3" version = "0.2.3"
@ -1477,6 +1602,24 @@ dependencies = [
"hashbrown 0.9.1", "hashbrown 0.9.1",
] ]
[[package]]
name = "inferno"
version = "0.10.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "37fb405dbcc505ed20838c4f8dad7b901094de90add237755df54bd5dcda2fdd"
dependencies = [
"ahash 0.6.3",
"atty",
"indexmap",
"itoa",
"lazy_static",
"log",
"num-format",
"quick-xml",
"rgb",
"str_stack",
]
[[package]] [[package]]
name = "influxdb2_client" name = "influxdb2_client"
version = "0.1.0" version = "0.1.0"
@ -1541,6 +1684,7 @@ dependencies = [
"panic_logging", "panic_logging",
"parking_lot", "parking_lot",
"parquet 0.1.0", "parquet 0.1.0",
"pprof",
"predicates", "predicates",
"prettytable-rs", "prettytable-rs",
"prost", "prost",
@ -1738,7 +1882,7 @@ version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6607c62aa161d23d17a9072cc5da0be67cdfc89d3afb1e8d9c842bebc2525ffe" checksum = "6607c62aa161d23d17a9072cc5da0be67cdfc89d3afb1e8d9c842bebc2525ffe"
dependencies = [ dependencies = [
"arrayvec", "arrayvec 0.5.2",
"bitflags", "bitflags",
"cfg-if", "cfg-if",
"ryu", "ryu",
@ -1859,6 +2003,16 @@ version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc" checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc"
[[package]]
name = "memmap"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b"
dependencies = [
"libc",
"winapi",
]
[[package]] [[package]]
name = "memoffset" name = "memoffset"
version = "0.6.3" version = "0.6.3"
@ -2024,6 +2178,12 @@ dependencies = [
"libc", "libc",
] ]
[[package]]
name = "nodrop"
version = "0.1.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb"
[[package]] [[package]]
name = "nom" name = "nom"
version = "5.1.2" version = "5.1.2"
@ -2095,6 +2255,16 @@ dependencies = [
"num-traits", "num-traits",
] ]
[[package]]
name = "num-format"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bafe4179722c2894288ee77a9f044f02811c86af699344c498b0840c698a2465"
dependencies = [
"arrayvec 0.4.12",
"itoa",
]
[[package]] [[package]]
name = "num-integer" name = "num-integer"
version = "0.1.44" version = "0.1.44"
@ -2625,6 +2795,27 @@ dependencies = [
"plotters-backend", "plotters-backend",
] ]
[[package]]
name = "pprof"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "234cb1ca0d59aa771d9bc7e268739d7aef6ca7e9e8d3b78d92f266cd663fd0c1"
dependencies = [
"backtrace",
"inferno",
"lazy_static",
"libc",
"log",
"nix",
"parking_lot",
"prost",
"prost-build",
"prost-derive",
"symbolic-demangle",
"tempfile",
"thiserror",
]
[[package]] [[package]]
name = "ppv-lite86" name = "ppv-lite86"
version = "0.2.10" version = "0.2.10"
@ -2824,6 +3015,15 @@ version = "1.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
[[package]]
name = "quick-xml"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26aab6b48e2590e4a64d1ed808749ba06257882b461d01ca71baeb747074a6dd"
dependencies = [
"memchr",
]
[[package]] [[package]]
name = "quote" name = "quote"
version = "1.0.9" version = "1.0.9"
@ -3095,6 +3295,15 @@ dependencies = [
"winreg", "winreg",
] ]
[[package]]
name = "rgb"
version = "0.8.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fddb3b23626145d1776addfc307e1a1851f60ef6ca64f376bcb889697144cf0"
dependencies = [
"bytemuck",
]
[[package]] [[package]]
name = "ring" name = "ring"
version = "0.16.20" version = "0.16.20"
@ -3461,6 +3670,7 @@ dependencies = [
"arrow_util", "arrow_util",
"async-trait", "async-trait",
"bytes", "bytes",
"cached",
"chrono", "chrono",
"crc32fast", "crc32fast",
"criterion", "criterion",
@ -3702,12 +3912,24 @@ version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0"
[[package]]
name = "str_stack"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9091b6114800a5f2141aee1d1b9d6ca3592ac062dc5decb3764ec5895a47b4eb"
[[package]] [[package]]
name = "strsim" name = "strsim"
version = "0.8.0" version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a" checksum = "8ea5119cdb4c55b55d432abb513a0429384878c15dde60cc77b1c99de1a95a6a"
[[package]]
name = "strsim"
version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6446ced80d6c486436db5c078dde11a9f73d42b57fb273121e160b84f63d894c"
[[package]] [[package]]
name = "structopt" name = "structopt"
version = "0.3.21" version = "0.3.21"
@ -3738,6 +3960,28 @@ version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e81da0851ada1f3e9d4312c704aa4f8806f0f9d69faaf8df2f3464b4a9437c2" checksum = "1e81da0851ada1f3e9d4312c704aa4f8806f0f9d69faaf8df2f3464b4a9437c2"
[[package]]
name = "symbolic-common"
version = "8.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "be7dfa630954f18297ceae1ff2890cb7f5008a0b2d2106b0468dafc45b0b6b12"
dependencies = [
"debugid",
"memmap",
"stable_deref_trait",
"uuid",
]
[[package]]
name = "symbolic-demangle"
version = "8.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b4ba42bd1221803e965054767b1899f2db9a12c89969965c6cb3a02af7014eb"
dependencies = [
"rustc-demangle",
"symbolic-common",
]
[[package]] [[package]]
name = "syn" name = "syn"
version = "1.0.67" version = "1.0.67"

1
Cargo.toml
View File

@ -92,6 +92,7 @@ parking_lot = "0.11.1"
itertools = "0.9.0" itertools = "0.9.0"
# used by arrow/datafusion anyway # used by arrow/datafusion anyway
prettytable-rs = "0.8" prettytable-rs = "0.8"
pprof = { version = "^0.4", default-features = false, features = ["flamegraph", "protobuf"] }
prost = "0.7" prost = "0.7"
# Forked to upgrade hyper and tokio # Forked to upgrade hyper and tokio
routerify = { git = "https://github.com/influxdata/routerify", rev = "274e250" } routerify = { git = "https://github.com/influxdata/routerify", rev = "274e250" }

4
data_types/src/error.rs
View File

@ -4,8 +4,8 @@ use std::fmt::Debug;
use observability_deps::tracing::error; use observability_deps::tracing::error;
/// Add ability for Results to log error messages via `error!` logs. /// Add ability for Results to log error messages via `error!` logs.
/// This is useful when using async tasks that may not have a natural /// This is useful when using async tasks that may not have any code
/// return error /// checking their return values.
pub trait ErrorLogger { pub trait ErrorLogger {
/// Log the contents of self with a string of context. The context /// Log the contents of self with a string of context. The context
/// should appear in a message such as /// should appear in a message such as

1
docs/README.md
View File

@ -17,6 +17,7 @@ We hold monthly Tech Talks that explain the project's technical underpinnings. Y
* Rust style and Idiom guide: [style_guide.md](style_guide.md) * Rust style and Idiom guide: [style_guide.md](style_guide.md)
* Tracing and logging Guide: [tracing.md](tracing.md) * Tracing and logging Guide: [tracing.md](tracing.md)
* Profiling Guide: [profiling.md](profiling.md)
* How InfluxDB IOx manages the lifecycle of time series data: [data_management.md](data_management.md) * How InfluxDB IOx manages the lifecycle of time series data: [data_management.md](data_management.md)
* Thoughts on parquet encoding and compression for timeseries data: [encoding_thoughts.md](encoding_thoughts.md) * Thoughts on parquet encoding and compression for timeseries data: [encoding_thoughts.md](encoding_thoughts.md)
* Thoughts on using multiple cores: [multi_core_tasks.md](multi_core_tasks.md) * Thoughts on using multiple cores: [multi_core_tasks.md](multi_core_tasks.md)

BIN
docs/images/flame_graph.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 1.1 MiB

46
docs/profiling.md Normal file
View File

@ -0,0 +1,46 @@
# IOx — Profiling
IOx includes an embedded `pprof` exporter compatible with the [go pprof](https://golang.org/pkg/net/http/pprof/) tool.
To use it, aim your favorite tool at your IOx host at the HTTP `/debug/pprof/profile` endpoint.
# Use the Go `pprof` tool:
Example
```shell
go tool pprof 'http://localhost:8080/debug/pprof/profile?seconds=5'
```
And you get output like:
```
Fetching profile over HTTP from http://localhost:8080/debug/pprof/profile?seconds=5
Saved profile in /Users/mkm/pprof/pprof.cpu.006.pb.gz
Type: cpu
Entering interactive mode (type "help" for commands, "o" for options)
(pprof) top
Showing nodes accounting for 93, 100% of 93 total
Showing top 10 nodes out of 185
flat flat% sum% cum cum%
93 100% 100% 93 100% backtrace::backtrace::libunwind::trace
0 0% 100% 1 1.08% <&str as nom::traits::InputTakeAtPosition>::split_at_position1_complete
0 0% 100% 1 1.08% <(FnA,FnB) as nom::sequence::Tuple<Input,(A,B),Error>>::parse
0 0% 100% 1 1.08% <(FnA,FnB,FnC) as nom::sequence::Tuple<Input,(A,B,C),Error>>::parse
0 0% 100% 5 5.38% <F as futures_core::future::TryFuture>::try_poll
0 0% 100% 1 1.08% <T as alloc::slice::hack::ConvertVec>::to_vec
0 0% 100% 1 1.08% <alloc::alloc::Global as core::alloc::Allocator>::allocate
0 0% 100% 1 1.08% <alloc::borrow::Cow<B> as core::clone::Clone>::clone
0 0% 100% 3 3.23% <alloc::vec::Vec<T,A> as alloc::vec::spec_extend::SpecExtend<T,I>>::spec_extend
0 0% 100% 1 1.08% <alloc::vec::Vec<T,A> as core::iter::traits::collect::Extend<T>>::extend
```
# Use the built in flame graph renderer
IOx also knows how to render a flamegraph SVG directly if opened directly in the browser:
For example, if you aim your browser at an IOx server with a URL such as http://localhost:8080/debug/pprof/profile?seconds=5
You will see a beautiful flame graph such as
![Flame Graph](images/flame_graph.png)

24
mutable_buffer/src/chunk/snapshot.rs
View File

@ -8,7 +8,7 @@ use internal_types::selection::Selection;
use snafu::{OptionExt, ResultExt, Snafu}; use snafu::{OptionExt, ResultExt, Snafu};
use super::Chunk; use super::Chunk;
use data_types::partition_metadata::Statistics; use data_types::{error::ErrorLogger, partition_metadata::Statistics};
#[derive(Debug, Snafu)] #[derive(Debug, Snafu)]
pub enum Error { pub enum Error {
@ -57,8 +57,15 @@ impl ChunkSnapshot {
let mut records: HashMap<String, TableSnapshot> = Default::default(); let mut records: HashMap<String, TableSnapshot> = Default::default();
let table = &chunk.table; let table = &chunk.table;
let schema = table.schema(&chunk.dictionary, Selection::All).unwrap(); let schema = table
let batch = table.to_arrow(&chunk.dictionary, Selection::All).unwrap(); .schema(&chunk.dictionary, Selection::All)
.log_if_error("ChunkSnapshot getting table schema")
.unwrap();
let batch = table
.to_arrow(&chunk.dictionary, Selection::All)
.log_if_error("ChunkSnapshot converting table to arrow")
.unwrap();
let name = chunk.table_name.as_ref(); let name = chunk.table_name.as_ref();
let timestamp_range = let timestamp_range =
@ -87,10 +94,13 @@ impl ChunkSnapshot {
}, },
); );
Self { let chunk_id = chunk
chunk_id: chunk.id.expect("cannot snapshot chunk without an ID"), .id
records, .ok_or("cannot snapshot chunk without an ID")
} .log_if_error("ChunkSnapshot determining chunk id")
.unwrap();
Self { chunk_id, records }
} }
/// return the ID of the chunk this is a snapshot of /// return the ID of the chunk this is a snapshot of

86
parquet_file/src/catalog.rs
View File

@ -251,24 +251,33 @@ where
S: CatalogState, S: CatalogState,
{ {
/// Create new catalog w/o any data. /// Create new catalog w/o any data.
pub fn new_empty( ///
/// An empty transaction will be used to mark the catalog start so that concurrent open but still-empty catalogs can
/// easily be detected.
pub async fn new_empty(
object_store: Arc<ObjectStore>, object_store: Arc<ObjectStore>,
server_id: ServerId, server_id: ServerId,
db_name: impl Into<String>, db_name: impl Into<String>,
state_data: S::EmptyInput, state_data: S::EmptyInput,
) -> Self { ) -> Result<Self> {
let inner = PreservedCatalogInner { let inner = PreservedCatalogInner {
previous_tkey: None, previous_tkey: None,
state: Arc::new(S::new_empty(state_data)), state: Arc::new(S::new_empty(state_data)),
}; };
Self { let catalog = Self {
inner: RwLock::new(inner), inner: RwLock::new(inner),
transaction_semaphore: Semaphore::new(1), transaction_semaphore: Semaphore::new(1),
object_store, object_store,
server_id, server_id,
db_name: db_name.into(), db_name: db_name.into(),
} };
// add empty transaction
let transaction = catalog.open_transaction().await;
transaction.commit().await?;
Ok(catalog)
} }
/// Load existing catalog from store, if it exists. /// Load existing catalog from store, if it exists.
@ -383,14 +392,13 @@ where
} }
/// Get latest revision counter. /// Get latest revision counter.
/// pub fn revision_counter(&self) -> u64 {
/// This can be `None` for a newly created catalog.
pub fn revision_counter(&self) -> Option<u64> {
self.inner self.inner
.read() .read()
.previous_tkey .previous_tkey
.clone() .clone()
.map(|tkey| tkey.revision_counter) .map(|tkey| tkey.revision_counter)
.expect("catalog should have at least an empty transaction")
} }
} }
@ -960,6 +968,42 @@ pub mod tests {
use super::test_helpers::TestCatalogState; use super::test_helpers::TestCatalogState;
use super::*; use super::*;
#[tokio::test]
async fn test_create_empty() {
let object_store = make_object_store();
let server_id = make_server_id();
let db_name = "db1";
assert!(PreservedCatalog::<TestCatalogState>::load(
Arc::clone(&object_store),
server_id,
db_name.to_string(),
()
)
.await
.unwrap()
.is_none());
PreservedCatalog::<TestCatalogState>::new_empty(
Arc::clone(&object_store),
server_id,
db_name.to_string(),
(),
)
.await
.unwrap();
assert!(PreservedCatalog::<TestCatalogState>::load(
Arc::clone(&object_store),
server_id,
db_name.to_string(),
()
)
.await
.unwrap()
.is_some());
}
#[tokio::test] #[tokio::test]
async fn test_inmem_commit_semantics() { async fn test_inmem_commit_semantics() {
let object_store = make_object_store(); let object_store = make_object_store();
@ -1064,12 +1108,7 @@ pub mod tests {
(), (),
) )
.await; .await;
assert!(matches!( assert_eq!(res.unwrap_err().to_string(), "Missing transaction: 0",);
res,
Err(Error::MissingTransaction {
revision_counter: 0
})
));
} }
#[tokio::test] #[tokio::test]
@ -1327,14 +1366,16 @@ pub mod tests {
make_server_id(), make_server_id(),
"db1".to_string(), "db1".to_string(),
(), (),
); )
.await
.unwrap();
let mut t = catalog.open_transaction().await; let mut t = catalog.open_transaction().await;
// open transaction // open transaction
t.transaction.as_mut().unwrap().proto.uuid = Uuid::nil().to_string(); t.transaction.as_mut().unwrap().proto.uuid = Uuid::nil().to_string();
assert_eq!( assert_eq!(
format!("{:?}", t), format!("{:?}", t),
"TransactionHandle(open, 0.00000000-0000-0000-0000-000000000000)" "TransactionHandle(open, 1.00000000-0000-0000-0000-000000000000)"
); );
// "closed" transaction // "closed" transaction
@ -1521,7 +1562,9 @@ pub mod tests {
server_id, server_id,
db_name.to_string(), db_name.to_string(),
(), (),
); )
.await
.unwrap();
// get some test metadata // get some test metadata
let metadata1 = make_metadata(object_store, "foo").await; let metadata1 = make_metadata(object_store, "foo").await;
@ -1531,8 +1574,9 @@ pub mod tests {
let mut trace = TestTrace::new(); let mut trace = TestTrace::new();
// empty catalog has no data // empty catalog has no data
assert!(catalog.revision_counter().is_none()); assert_eq!(catalog.revision_counter(), 0);
assert_catalog_parquet_files(&catalog, &[]); assert_catalog_parquet_files(&catalog, &[]);
trace.record(&catalog);
// fill catalog with examples // fill catalog with examples
{ {
@ -1548,7 +1592,7 @@ pub mod tests {
t.commit().await.unwrap(); t.commit().await.unwrap();
} }
assert_eq!(catalog.revision_counter().unwrap(), 0); assert_eq!(catalog.revision_counter(), 1);
assert_catalog_parquet_files( assert_catalog_parquet_files(
&catalog, &catalog,
&[ &[
@ -1578,7 +1622,7 @@ pub mod tests {
t.commit().await.unwrap(); t.commit().await.unwrap();
} }
assert_eq!(catalog.revision_counter().unwrap(), 1); assert_eq!(catalog.revision_counter(), 2);
assert_catalog_parquet_files( assert_catalog_parquet_files(
&catalog, &catalog,
&[ &[
@ -1599,7 +1643,7 @@ pub mod tests {
// NO commit here! // NO commit here!
} }
assert_eq!(catalog.revision_counter().unwrap(), 1); assert_eq!(catalog.revision_counter(), 2);
assert_catalog_parquet_files( assert_catalog_parquet_files(
&catalog, &catalog,
&[ &[
@ -1629,7 +1673,7 @@ pub mod tests {
.unwrap() .unwrap()
.unwrap(); .unwrap();
assert_eq!( assert_eq!(
catalog.revision_counter().unwrap(), catalog.revision_counter(),
trace.tkeys.last().unwrap().revision_counter trace.tkeys.last().unwrap().revision_counter
); );
assert_catalog_parquet_files( assert_catalog_parquet_files(

18
query/src/exec/context.rs
View File

@ -8,7 +8,7 @@ use datafusion::{
execution::context::{ExecutionContextState, QueryPlanner}, execution::context::{ExecutionContextState, QueryPlanner},
logical_plan::{LogicalPlan, UserDefinedLogicalNode}, logical_plan::{LogicalPlan, UserDefinedLogicalNode},
physical_plan::{ physical_plan::{
collect, collect, displayable,
merge::MergeExec, merge::MergeExec,
planner::{DefaultPhysicalPlanner, ExtensionPlanner}, planner::{DefaultPhysicalPlanner, ExtensionPlanner},
ExecutionPlan, PhysicalPlanner, SendableRecordBatchStream, ExecutionPlan, PhysicalPlanner, SendableRecordBatchStream,
@ -151,21 +151,15 @@ impl IOxExecutionContext {
/// Prepare (optimize + plan) a pre-created logical plan for execution /// Prepare (optimize + plan) a pre-created logical plan for execution
pub fn prepare_plan(&self, plan: &LogicalPlan) -> Result<Arc<dyn ExecutionPlan>> { pub fn prepare_plan(&self, plan: &LogicalPlan) -> Result<Arc<dyn ExecutionPlan>> {
debug!( debug!(text=%plan.display_indent_schema(), "initial plan");
"Creating plan: Initial plan\n----\n{}\n{}\n----",
plan.display_indent_schema(),
plan.display_graphviz(),
);
let plan = self.inner.optimize(&plan)?; let plan = self.inner.optimize(&plan)?;
debug!(text=%plan.display_indent_schema(), graphviz=%plan.display_graphviz(), "optimized plan");
debug!( let physical_plan = self.inner.create_physical_plan(&plan)?;
"Creating plan: Optimized plan\n----\n{}\n{}\n----",
plan.display_indent_schema(),
plan.display_graphviz(),
);
self.inner.create_physical_plan(&plan) debug!(text=%displayable(physical_plan.as_ref()).indent(), "optimized physical plan");
Ok(physical_plan)
} }
/// Executes the logical plan using DataFusion on a separate /// Executes the logical plan using DataFusion on a separate

121
query/src/predicate.rs
View File

@ -3,7 +3,7 @@
//! mode as well as for arbitrary other predicates that are expressed //! mode as well as for arbitrary other predicates that are expressed
//! by DataFusion's `Expr` type. //! by DataFusion's `Expr` type.
use std::collections::{BTreeSet, HashSet}; use std::{collections::{BTreeSet, HashSet}, fmt};
use data_types::timestamp::TimestampRange; use data_types::timestamp::TimestampRange;
use datafusion::{ use datafusion::{
@ -27,9 +27,9 @@ pub const EMPTY_PREDICATE: Predicate = Predicate {
/// Represents a parsed predicate for evaluation by the /// Represents a parsed predicate for evaluation by the
/// TSDatabase InfluxDB IOx query engine. /// TSDatabase InfluxDB IOx query engine.
/// ///
/// Note that the data model of TSDatabase (e.g. ParsedLine's) /// Note that the InfluxDB data model (e.g. ParsedLine's)
/// distinguishes between some types of columns (tags and fields), and /// distinguishes between some types of columns (tags and fields), and
/// likewise the semantics of this structure has some types of /// likewise the semantics of this structure can express some types of
/// restrictions that only apply to certain types of columns. /// restrictions that only apply to certain types of columns.
#[derive(Clone, Debug, Default, PartialEq)] #[derive(Clone, Debug, Default, PartialEq)]
pub struct Predicate { pub struct Predicate {
@ -37,23 +37,23 @@ pub struct Predicate {
/// to only tables whose names are in `table_names` /// to only tables whose names are in `table_names`
pub table_names: Option<BTreeSet<String>>, pub table_names: Option<BTreeSet<String>>,
// Optional field restriction. If present, restricts the results to only /// Optional field restriction. If present, restricts the results to only
// tables which have *at least one* of the fields in field_columns. /// tables which have *at least one* of the fields in field_columns.
pub field_columns: Option<BTreeSet<String>>, pub field_columns: Option<BTreeSet<String>>,
/// Optional partition key filter
pub partition_key: Option<String>,
/// Optional timestamp range: only rows within this range are included in
/// results. Other rows are excluded
pub range: Option<TimestampRange>,
/// Optional arbitrary predicates, represented as list of /// Optional arbitrary predicates, represented as list of
/// DataFusion expressions applied a logical conjunction (aka they /// DataFusion expressions applied a logical conjunction (aka they
/// are 'AND'ed together). Only rows that evaluate to TRUE for all /// are 'AND'ed together). Only rows that evaluate to TRUE for all
/// these expressions should be returned. Other rows are excluded /// these expressions should be returned. Other rows are excluded
/// from the results. /// from the results.
pub exprs: Vec<Expr>, pub exprs: Vec<Expr>,
/// Optional timestamp range: only rows within this range are included in
/// results. Other rows are excluded
pub range: Option<TimestampRange>,
/// Optional partition key filter
pub partition_key: Option<String>,
} }
impl Predicate { impl Predicate {
@ -108,8 +108,66 @@ impl Predicate {
} }
} }
impl fmt::Display for Predicate {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fn iter_to_str<S>(s: impl IntoIterator<Item = S>) -> String
where
S: ToString,
{
s.into_iter()
.map(|v| v.to_string())
.collect::<Vec<_>>()
.join(", ")
}
write!(f, "Predicate")?;
if let Some(table_names) = &self.table_names {
write!(f, " table_names: {{{}}}", iter_to_str(table_names))?;
}
if let Some(field_columns) = &self.field_columns {
write!(f, " field_columns: {{{}}}", iter_to_str(field_columns))?;
}
if let Some(partition_key) = &self.partition_key {
write!(f, " partition_key: '{}'", partition_key)?;
}
if let Some(range) = &self.range {
// TODO: could be nice to show this as actual timestamps (not just numbers)?
write!(f, " range: [{} - {}]", range.start, range.end)?;
}
if !self.exprs.is_empty() {
// Expr doesn't implement `Display` yet, so just the debug version
// See https://github.com/apache/arrow-datafusion/issues/347
let display_exprs = self.exprs.iter().map(|e| format!("{:?}", e));
write!(f, " exprs: [{}]", iter_to_str(display_exprs))?;
}
Ok(())
}
}
#[derive(Debug, Default)] #[derive(Debug, Default)]
/// Structure for building `Predicate`s /// Structure for building [`Predicate`]s
///
/// Example:
/// ```
/// use query::predicate::PredicateBuilder;
/// use datafusion::logical_plan::{col, lit};
///
/// let p = PredicateBuilder::new()
/// .timestamp_range(1, 100)
/// .add_expr(col("foo").eq(lit(42)))
/// .build();
///
/// assert_eq!(
/// p.to_string(),
/// "Predicate range: [1 - 100] exprs: [#foo Eq Int32(42)]"
/// );
/// ```
pub struct PredicateBuilder { pub struct PredicateBuilder {
inner: Predicate, inner: Predicate,
} }
@ -315,9 +373,8 @@ impl PredicateBuilder {
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use datafusion::logical_plan::{col, lit};
use super::*; use super::*;
use datafusion::logical_plan::{col, lit};
#[test] #[test]
fn test_default_predicate_is_empty() { fn test_default_predicate_is_empty() {
@ -392,4 +449,38 @@ mod tests {
assert_eq!(predicate.exprs[5], col("city").eq(lit("Boston"))); assert_eq!(predicate.exprs[5], col("city").eq(lit("Boston")));
assert_eq!(predicate.exprs[6], col("city").not_eq(lit("Braintree"))); assert_eq!(predicate.exprs[6], col("city").not_eq(lit("Braintree")));
} }
#[test]
fn predicate_display_ts() {
// TODO make this a doc example?
let p = PredicateBuilder::new().timestamp_range(1, 100).build();
assert_eq!(p.to_string(), "Predicate range: [1 - 100]");
}
#[test]
fn predicate_display_ts_and_expr() {
let p = PredicateBuilder::new()
.timestamp_range(1, 100)
.add_expr(col("foo").eq(lit(42)).and(col("bar").lt(lit(11))))
.build();
assert_eq!(
p.to_string(),
"Predicate range: [1 - 100] exprs: [#foo Eq Int32(42) And #bar Lt Int32(11)]"
);
}
#[test]
fn predicate_display_full() {
let p = PredicateBuilder::new()
.timestamp_range(1, 100)
.add_expr(col("foo").eq(lit(42)))
.table("my_table")
.field_columns(vec!["f1", "f2"])
.partition_key("the_key")
.build();
assert_eq!(p.to_string(), "Predicate table_names: {my_table} field_columns: {f1, f2} partition_key: 'the_key' range: [1 - 100] exprs: [#foo Eq Int32(42)]");
}
} }

19
query/src/provider/physical.rs
View File

@ -1,11 +1,11 @@
//! Implementation of a DataFusion PhysicalPlan node across partition chunks //! Implementation of a DataFusion PhysicalPlan node across partition chunks
use std::sync::Arc; use std::{fmt, sync::Arc};
use arrow::datatypes::SchemaRef; use arrow::datatypes::SchemaRef;
use datafusion::{ use datafusion::{
error::DataFusionError, error::DataFusionError,
physical_plan::{ExecutionPlan, Partitioning, SendableRecordBatchStream}, physical_plan::{DisplayFormatType, ExecutionPlan, Partitioning, SendableRecordBatchStream},
}; };
use internal_types::{schema::Schema, selection::Selection}; use internal_types::{schema::Schema, selection::Selection};
@ -116,6 +116,21 @@ impl<C: PartitionChunk + 'static> ExecutionPlan for IOxReadFilterNode<C> {
Ok(Box::pin(adapter)) Ok(Box::pin(adapter))
} }
fn fmt_as(&self, t: DisplayFormatType, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match t {
DisplayFormatType::Default => {
// Note Predicate doesn't implement Display so punt on showing that now
write!(
f,
"IOxReadFilterNode: table_name={}, chunks={} predicate={}",
self.table_name,
self.chunk_and_infos.len(),
self.predicate,
)
}
}
}
} }
/// Removes any columns that are not present in schema, returning a possibly /// Removes any columns that are not present in schema, returning a possibly

8
read_buffer/benches/plain.rs
View File

@ -116,7 +116,7 @@ fn benchmark_plain_sum(
|b, input| { |b, input| {
b.iter(|| { b.iter(|| {
// do work // do work
let _ = encoding.sum(&input); let _ = encoding.sum::<i64>(&input);
}); });
}, },
); );
@ -142,7 +142,7 @@ fn benchmark_plain_sum(
|b, input| { |b, input| {
b.iter(|| { b.iter(|| {
// do work // do work
let _ = encoding.sum::<i32>(&input); let _ = encoding.sum::<i64>(&input);
}); });
}, },
); );
@ -161,7 +161,7 @@ fn benchmark_plain_sum(
|b, input| { |b, input| {
b.iter(|| { b.iter(|| {
// do work // do work
let _ = encoding.sum(&input); let _ = encoding.sum::<i64>(&input);
}); });
}, },
); );
@ -206,7 +206,7 @@ fn benchmark_plain_sum(
|b, input| { |b, input| {
b.iter(|| { b.iter(|| {
// do work // do work
let _ = encoding.sum(&input); let _ = encoding.sum::<i64>(&input);
}); });
}, },
); );

10
read_buffer/benches/sum_fixed.rs
View File

@ -116,7 +116,7 @@ fn benchmark_none_sum(
|b, input| { |b, input| {
b.iter(|| { b.iter(|| {
// do work // do work
let _ = encoding.sum(&input); let _ = encoding.sum::<i64>(&input);
}); });
}, },
); );
@ -142,7 +142,7 @@ fn benchmark_none_sum(
|b, input| { |b, input| {
b.iter(|| { b.iter(|| {
// do work // do work
let _ = encoding.sum::<i32>(&input); let _ = encoding.sum::<i64>(&input);
}); });
}, },
); );
@ -161,7 +161,7 @@ fn benchmark_none_sum(
|b, input| { |b, input| {
b.iter(|| { b.iter(|| {
// do work // do work
let _ = encoding.sum(&input); let _ = encoding.sum::<i64>(&input);
}); });
}, },
); );
@ -187,7 +187,7 @@ fn benchmark_none_sum(
|b, input| { |b, input| {
b.iter(|| { b.iter(|| {
// do work // do work
let _ = encoding.sum::<i16>(&input); let _ = encoding.sum::<i64>(&input);
}); });
}, },
); );
@ -206,7 +206,7 @@ fn benchmark_none_sum(
|b, input| { |b, input| {
b.iter(|| { b.iter(|| {
// do work // do work
let _ = encoding.sum(&input); let _ = encoding.sum::<i64>(&input);
}); });
}, },
); );

13
read_buffer/src/column/encoding/scalar/fixed.rs
View File

@ -21,7 +21,7 @@ use arrow::array::Array;
use crate::column::{cmp, RowIDs}; use crate::column::{cmp, RowIDs};
#[derive(Debug, Default)] #[derive(Debug, Default, PartialEq, PartialOrd)]
/// A Fixed encoding is one in which every value has a fixed width, and is /// A Fixed encoding is one in which every value has a fixed width, and is
/// stored contiguous in a backing vector. Fixed encodings do not support NULL /// stored contiguous in a backing vector. Fixed encodings do not support NULL
/// values, so are suitable for columns known to not have NULL values that we /// values, so are suitable for columns known to not have NULL values that we
@ -566,13 +566,6 @@ macro_rules! fixed_from_arrow_impls {
} }
fixed_from_arrow_impls! { fixed_from_arrow_impls! {
(&arrow::array::Int64Array, i64),
(&arrow::array::Int64Array, i32),
(&arrow::array::Int64Array, i16),
(&arrow::array::Int64Array, i8),
(&arrow::array::Int64Array, u32),
(&arrow::array::Int64Array, u16),
(&arrow::array::Int64Array, u8),
(arrow::array::Int64Array, i64), (arrow::array::Int64Array, i64),
(arrow::array::Int64Array, i32), (arrow::array::Int64Array, i32),
(arrow::array::Int64Array, i16), (arrow::array::Int64Array, i16),
@ -581,10 +574,6 @@ fixed_from_arrow_impls! {
(arrow::array::Int64Array, u16), (arrow::array::Int64Array, u16),
(arrow::array::Int64Array, u8), (arrow::array::Int64Array, u8),
(&arrow::array::UInt64Array, u64),
(&arrow::array::UInt64Array, u32),
(&arrow::array::UInt64Array, u16),
(&arrow::array::UInt64Array, u8),
(arrow::array::UInt64Array, u64), (arrow::array::UInt64Array, u64),
(arrow::array::UInt64Array, u32), (arrow::array::UInt64Array, u32),
(arrow::array::UInt64Array, u16), (arrow::array::UInt64Array, u16),

270
read_buffer/src/column/encoding/scalar/fixed_null.rs
View File

@ -14,6 +14,7 @@
//! consumer of these encodings. //! consumer of these encodings.
use std::cmp::Ordering; use std::cmp::Ordering;
use std::fmt::Debug; use std::fmt::Debug;
use std::iter::FromIterator;
use std::mem::size_of; use std::mem::size_of;
use arrow::{ use arrow::{
@ -23,7 +24,7 @@ use arrow::{
use crate::column::{cmp, RowIDs}; use crate::column::{cmp, RowIDs};
#[derive(Debug)] #[derive(Debug, PartialEq)]
pub struct FixedNull<T> pub struct FixedNull<T>
where where
T: ArrowNumericType, T: ArrowNumericType,
@ -116,26 +117,30 @@ where
// //
// //
/// Return the logical (decoded) value at the provided row ID. A NULL value /// Return the logical (decoded) value at the provided row ID according to
/// the logical type of the column, which is specified by `U`. A NULL value
/// is represented by None. /// is represented by None.
pub fn value(&self, row_id: u32) -> Option<T::Native> { pub fn value<U>(&self, row_id: u32) -> Option<U>
where
U: From<T::Native>,
{
if self.arr.is_null(row_id as usize) { if self.arr.is_null(row_id as usize) {
return None; return None;
} }
Some(self.arr.value(row_id as usize)) Some(U::from(self.arr.value(row_id as usize)))
} }
/// Returns the logical (decoded) values for the provided row IDs. /// Returns the logical (decoded) values for the provided row IDs according
/// to the logical type of the column, which is specified by `U`.
/// ///
/// NULL values are represented by None. /// NULL values are represented by None.
/// ///
/// TODO(edd): Perf - we should return a vector of values and a vector of /// TODO(edd): Perf - we should return a vector of values and a vector of
/// integers representing the null validity bitmap. /// integers representing the null validity bitmap.
pub fn values( pub fn values<U>(&self, row_ids: &[u32], mut dst: Vec<Option<U>>) -> Vec<Option<U>>
&self, where
row_ids: &[u32], U: From<T::Native>,
mut dst: Vec<Option<T::Native>>, {
) -> Vec<Option<T::Native>> {
dst.clear(); dst.clear();
dst.reserve(row_ids.len()); dst.reserve(row_ids.len());
@ -143,20 +148,24 @@ where
if self.arr.is_null(row_id as usize) { if self.arr.is_null(row_id as usize) {
dst.push(None) dst.push(None)
} else { } else {
dst.push(Some(self.arr.value(row_id as usize))) dst.push(Some(U::from(self.arr.value(row_id as usize))))
} }
} }
assert_eq!(dst.len(), row_ids.len()); assert_eq!(dst.len(), row_ids.len());
dst dst
} }
/// Returns the logical (decoded) values for all the rows in the column. /// Returns the logical (decoded) values for all the rows in the column
/// according to the logical type of the column, which is specified by `U`.
/// ///
/// NULL values are represented by None. /// NULL values are represented by None.
/// ///
/// TODO(edd): Perf - we should return a vector of values and a vector of /// TODO(edd): Perf - we should return a vector of values and a vector of
/// integers representing the null validity bitmap. /// integers representing the null validity bitmap.
pub fn all_values(&self, mut dst: Vec<Option<T::Native>>) -> Vec<Option<T::Native>> { pub fn all_values<U>(&self, mut dst: Vec<Option<U>>) -> Vec<Option<U>>
where
U: From<T::Native>,
{
dst.clear(); dst.clear();
dst.reserve(self.arr.len()); dst.reserve(self.arr.len());
@ -164,7 +173,7 @@ where
if self.arr.is_null(i) { if self.arr.is_null(i) {
dst.push(None) dst.push(None)
} else { } else {
dst.push(Some(self.arr.value(i))) dst.push(Some(U::from(self.arr.value(i))))
} }
} }
assert_eq!(dst.len(), self.num_rows() as usize); assert_eq!(dst.len(), self.num_rows() as usize);
@ -202,23 +211,23 @@ where
/// this implementation (about 85% in the `sum` case). We will revisit /// this implementation (about 85% in the `sum` case). We will revisit
/// them in the future as they do would the implementation of these /// them in the future as they do would the implementation of these
/// aggregation functions. /// aggregation functions.
pub fn sum(&self, row_ids: &[u32]) -> Option<T::Native> pub fn sum<U>(&self, row_ids: &[u32]) -> Option<U>
where where
T::Native: std::ops::Add<Output = T::Native>, U: Default + From<T::Native> + std::ops::Add<Output = U>,
{ {
let mut result = T::Native::default(); let mut result = U::default();
if self.arr.null_count() == 0 { if self.arr.null_count() == 0 {
for chunks in row_ids.chunks_exact(4) { for chunks in row_ids.chunks_exact(4) {
result = result + self.arr.value(chunks[3] as usize); result = result + U::from(self.arr.value(chunks[3] as usize));
result = result + self.arr.value(chunks[2] as usize); result = result + U::from(self.arr.value(chunks[2] as usize));
result = result + self.arr.value(chunks[1] as usize); result = result + U::from(self.arr.value(chunks[1] as usize));
result = result + self.arr.value(chunks[0] as usize); result = result + U::from(self.arr.value(chunks[0] as usize));
} }
let rem = row_ids.len() % 4; let rem = row_ids.len() % 4;
for &i in &row_ids[row_ids.len() - rem..row_ids.len()] { for &i in &row_ids[row_ids.len() - rem..row_ids.len()] {
result = result + self.arr.value(i as usize); result = result + U::from(self.arr.value(i as usize));
} }
return Some(result); return Some(result);
@ -230,7 +239,7 @@ where
continue; continue;
} }
is_none = false; is_none = false;
result = result + self.arr.value(i as usize); result = result + U::from(self.arr.value(i as usize));
} }
if is_none { if is_none {
@ -241,20 +250,29 @@ where
/// Returns the first logical (decoded) value from the provided /// Returns the first logical (decoded) value from the provided
/// row IDs. /// row IDs.
pub fn first(&self, row_ids: &[u32]) -> Option<T::Native> { pub fn first<U>(&self, row_ids: &[u32]) -> Option<U>
where
U: From<T::Native>,
{
self.value(row_ids[0]) self.value(row_ids[0])
} }
/// Returns the last logical (decoded) value from the provided /// Returns the last logical (decoded) value from the provided
/// row IDs. /// row IDs.
pub fn last(&self, row_ids: &[u32]) -> Option<T::Native> { pub fn last<U>(&self, row_ids: &[u32]) -> Option<U>
where
U: From<T::Native>,
{
self.value(row_ids[row_ids.len() - 1]) self.value(row_ids[row_ids.len() - 1])
} }
/// Returns the minimum logical (decoded) non-null value from the provided /// Returns the minimum logical (decoded) non-null value from the provided
/// row IDs. /// row IDs.
pub fn min(&self, row_ids: &[u32]) -> Option<T::Native> { pub fn min<U>(&self, row_ids: &[u32]) -> Option<U>
let mut min: Option<T::Native> = self.value(row_ids[0]); where
U: From<T::Native> + PartialOrd,
{
let mut min: Option<U> = self.value(row_ids[0]);
for &v in row_ids.iter().skip(1) { for &v in row_ids.iter().skip(1) {
if self.arr.is_null(v as usize) { if self.arr.is_null(v as usize) {
continue; continue;
@ -269,8 +287,11 @@ where
/// Returns the maximum logical (decoded) non-null value from the provided /// Returns the maximum logical (decoded) non-null value from the provided
/// row IDs. /// row IDs.
pub fn max(&self, row_ids: &[u32]) -> Option<T::Native> { pub fn max<U>(&self, row_ids: &[u32]) -> Option<U>
let mut max: Option<T::Native> = self.value(row_ids[0]); where
U: From<T::Native> + PartialOrd,
{
let mut max: Option<U> = self.value(row_ids[0]);
for &v in row_ids.iter().skip(1) { for &v in row_ids.iter().skip(1) {
if self.arr.is_null(v as usize) { if self.arr.is_null(v as usize) {
continue; continue;
@ -414,11 +435,11 @@ where
/// `x {>, >=, <, <=} value1 AND x {>, >=, <, <=} value2`. /// `x {>, >=, <, <=} value1 AND x {>, >=, <, <=} value2`.
pub fn row_ids_filter_range( pub fn row_ids_filter_range(
&self, &self,
left: (T::Native, cmp::Operator), left: (T::Native, &cmp::Operator),
right: (T::Native, cmp::Operator), right: (T::Native, &cmp::Operator),
dst: RowIDs, dst: RowIDs,
) -> RowIDs { ) -> RowIDs {
match (&left.1, &right.1) { match (left.1, right.1) {
(cmp::Operator::GT, cmp::Operator::LT) (cmp::Operator::GT, cmp::Operator::LT)
| (cmp::Operator::GT, cmp::Operator::LTE) | (cmp::Operator::GT, cmp::Operator::LTE)
| (cmp::Operator::GTE, cmp::Operator::LT) | (cmp::Operator::GTE, cmp::Operator::LT)
@ -509,81 +530,93 @@ where
// This macro implements the From trait for slices of various logical types. // This macro implements the From trait for slices of various logical types.
// //
// Here is an example implementation: // Here are example implementations:
// //
// impl From<&[i64]> for FixedNull<arrow::datatypes::Int64Type> { // impl From<Vec<i64>> for FixedNull<Int64Type> {
// fn from(v: &[i64]) -> Self { // fn from(v: Vec<i64>) -> Self {
// Self{ // Self{
// arr: PrimitiveArray::from(v.to_vec()), // arr: PrimitiveArray::from(v),
// } // }
// } // }
// } // }
// //
// impl From<&[Option<i64>]> for // impl From<&[i64]> for FixedNull<Int64Type> {
// FixedNull<arrow::datatypes::Int64Type> { fn from(v: &[i64]) // fn from(v: &[i64]) -> Self {
// -> Self { Self{ // Self::from(v.to_vec())
// arr: PrimitiveArray::from(v.to_vec()), // }
// }
//
// impl From<Vec<Option<i64>>> for FixedNull<Int64Type> {
// fn from(v: Vec<Option<i64>>) -> Self {
// Self{
// arr: PrimitiveArray::from(v),
// } // }
// } // }
// } // }
// //
// impl From<&[Option<i64>]> for FixedNull<Int64Type> {
macro_rules! fixed_from_slice_impls { // fn from(v: &[i64]) -> Self {
// Self::from(v.to_vec())
// }
// }
//
macro_rules! fixed_null_from_native_types {
($(($type_from:ty, $type_to:ty),)*) => { ($(($type_from:ty, $type_to:ty),)*) => {
$( $(
impl From<Vec<$type_from>> for FixedNull<$type_to> {
fn from(v: Vec<$type_from>) -> Self {
Self{
arr: PrimitiveArray::from(v),
}
}
}
impl From<&[$type_from]> for FixedNull<$type_to> { impl From<&[$type_from]> for FixedNull<$type_to> {
fn from(v: &[$type_from]) -> Self { fn from(v: &[$type_from]) -> Self {
Self::from(v.to_vec())
}
}
impl From<Vec<Option<$type_from>>> for FixedNull<$type_to> {
fn from(v: Vec<Option<$type_from>>) -> Self {
Self{ Self{
arr: PrimitiveArray::from(v.to_vec()), arr: PrimitiveArray::from(v),
} }
} }
} }
impl From<&[Option<$type_from>]> for FixedNull<$type_to> { impl From<&[Option<$type_from>]> for FixedNull<$type_to> {
fn from(v: &[Option<$type_from>]) -> Self { fn from(v: &[Option<$type_from>]) -> Self {
Self{ Self::from(v.to_vec())
arr: PrimitiveArray::from(v.to_vec()),
}
} }
} }
)* )*
}; };
} }
// Supported logical and physical datatypes for the FixedNull encoding. fixed_null_from_native_types! {
//
// Need to look at possibility of initialising smaller datatypes...
fixed_from_slice_impls! {
(i64, arrow::datatypes::Int64Type), (i64, arrow::datatypes::Int64Type),
// (i64, arrow::datatypes::Int32Type), (i32, arrow::datatypes::Int32Type),
// (i64, arrow::datatypes::Int16Type), (i16, arrow::datatypes::Int16Type),
// (i64, arrow::datatypes::Int8Type), (i8, arrow::datatypes::Int8Type),
// (i64, arrow::datatypes::UInt32Type), (u64, arrow::datatypes::UInt64Type),
// (i64, arrow::datatypes::UInt16Type), (u32, arrow::datatypes::UInt32Type),
// (i64, arrow::datatypes::UInt8Type), (u16, arrow::datatypes::UInt16Type),
(i32, arrow::datatypes::Int32Type), (u8, arrow::datatypes::UInt8Type),
// (i32, arrow::datatypes::Int16Type), (f64, arrow::datatypes::Float64Type),
// (i32, arrow::datatypes::Int8Type),
// (i32, arrow::datatypes::UInt16Type),
// (i32, arrow::datatypes::UInt8Type),
(i16, arrow::datatypes::Int16Type),
// (i16, arrow::datatypes::Int8Type),
// (i16, arrow::datatypes::UInt8Type),
(i8, arrow::datatypes::Int8Type),
(u64, arrow::datatypes::UInt64Type),
// (u64, arrow::datatypes::UInt32Type),
// (u64, arrow::datatypes::UInt16Type),
// (u64, arrow::datatypes::UInt8Type),
(u32, arrow::datatypes::UInt32Type),
// (u32, arrow::datatypes::UInt16Type),
// (u32, arrow::datatypes::UInt8Type),
(u16, arrow::datatypes::UInt16Type),
// (u16, arrow::datatypes::UInt8Type),
(u8, arrow::datatypes::UInt8Type),
(f64, arrow::datatypes::Float64Type),
} }
macro_rules! fixed_from_arrow_impls { // This macro implements the From trait for Arrow arrays
//
// Implementation:
//
// impl From<Int64Array> for FixedNull<Int64Type> {
// fn from(arr: Int64Array) -> Self {
// Self{arr}
// }
// }
//
macro_rules! fixed_null_from_arrow_types {
($(($type_from:ty, $type_to:ty),)*) => { ($(($type_from:ty, $type_to:ty),)*) => {
$( $(
impl From<$type_from> for FixedNull<$type_to> { impl From<$type_from> for FixedNull<$type_to> {
@ -595,27 +628,76 @@ macro_rules! fixed_from_arrow_impls {
}; };
} }
// Supported logical and physical datatypes for the Plain encoding. fixed_null_from_arrow_types! {
//
// Need to look at possibility of initialising smaller datatypes...
fixed_from_arrow_impls! {
(arrow::array::Int64Array, arrow::datatypes::Int64Type), (arrow::array::Int64Array, arrow::datatypes::Int64Type),
(arrow::array::UInt64Array, arrow::datatypes::UInt64Type), (arrow::array::UInt64Array, arrow::datatypes::UInt64Type),
(arrow::array::Float64Array, arrow::datatypes::Float64Type), (arrow::array::Float64Array, arrow::datatypes::Float64Type),
}
// TODO(edd): add more datatypes // This macro implements the From trait for Arrow arrays where some down-casting
// to a smaller physical type happens. It is the caller's responsibility to
// ensure that this down-casting is safe.
//
// Example implementation:
//
// impl From<Int64Array> for FixedNull<Int32Type> {
// fn from(arr: Int64Array) -> Self {
// let arr: PrimitiveArray<Int32Type> =
// PrimitiveArray::from_iter(arr.iter().map(|v| v.map(|v| v as i32)));
// Self { arr }
// }
// }
//
macro_rules! fixed_null_from_arrow_types_down_cast {
($(($type_from:ty, $type_to:ty, $rust_type:ty),)*) => {
$(
impl From<$type_from> for FixedNull<$type_to> {
fn from(arr: $type_from) -> Self {
let arr: PrimitiveArray<$type_to> =
PrimitiveArray::from_iter(arr.iter().map(|v| v.map(|v| v as $rust_type)));
Self { arr }
}
}
)*
};
}
fixed_null_from_arrow_types_down_cast! {
(arrow::array::Int64Array, arrow::datatypes::Int32Type, i32),
(arrow::array::Int64Array, arrow::datatypes::UInt32Type, u32),
(arrow::array::Int64Array, arrow::datatypes::Int16Type, i16),
(arrow::array::Int64Array, arrow::datatypes::UInt16Type, u16),
(arrow::array::Int64Array, arrow::datatypes::Int8Type, i8),
(arrow::array::Int64Array, arrow::datatypes::UInt8Type, u8),
(arrow::array::UInt64Array, arrow::datatypes::UInt32Type, u32),
(arrow::array::UInt64Array, arrow::datatypes::UInt16Type, u16),
(arrow::array::UInt64Array, arrow::datatypes::UInt8Type, u8),
} }
#[cfg(test)] #[cfg(test)]
mod test { mod test {
use super::cmp::Operator; use super::cmp::Operator;
use super::*; use super::*;
use arrow::array::*;
use arrow::datatypes::*; use arrow::datatypes::*;
fn some_vec<T: Copy>(v: Vec<T>) -> Vec<Option<T>> { fn some_vec<T: Copy>(v: Vec<T>) -> Vec<Option<T>> {
v.iter().map(|x| Some(*x)).collect() v.iter().map(|x| Some(*x)).collect()
} }
#[test]
fn from_arrow_downcast() {
let arr = Int64Array::from(vec![100, u8::MAX as i64]);
let exp_values = arr.iter().collect::<Vec<Option<i64>>>();
let enc: FixedNull<UInt8Type> = FixedNull::from(arr);
assert_eq!(enc.all_values(vec![]), exp_values);
let arr = Int64Array::from(vec![100, i32::MAX as i64]);
let exp_values = arr.iter().collect::<Vec<Option<i64>>>();
let enc: FixedNull<UInt32Type> = FixedNull::from(arr);
assert_eq!(enc.all_values(vec![]), exp_values);
}
#[test] #[test]
fn size() { fn size() {
let v = FixedNull::<UInt64Type>::from(vec![None, None, Some(100), Some(2222)].as_slice()); let v = FixedNull::<UInt64Type>::from(vec![None, None, Some(100), Some(2222)].as_slice());
@ -879,36 +961,36 @@ mod test {
); );
let row_ids = v.row_ids_filter_range( let row_ids = v.row_ids_filter_range(
(100, Operator::GTE), (100, &Operator::GTE),
(240, Operator::LT), (240, &Operator::LT),
RowIDs::new_vector(), RowIDs::new_vector(),
); );
assert_eq!(row_ids.to_vec(), vec![0, 1, 5, 6, 13, 17]); assert_eq!(row_ids.to_vec(), vec![0, 1, 5, 6, 13, 17]);
let row_ids = v.row_ids_filter_range( let row_ids = v.row_ids_filter_range(
(100, Operator::GT), (100, &Operator::GT),
(240, Operator::LT), (240, &Operator::LT),
RowIDs::new_vector(), RowIDs::new_vector(),
); );
assert_eq!(row_ids.to_vec(), vec![1, 6, 13]); assert_eq!(row_ids.to_vec(), vec![1, 6, 13]);
let row_ids = v.row_ids_filter_range( let row_ids = v.row_ids_filter_range(
(10, Operator::LT), (10, &Operator::LT),
(-100, Operator::GT), (-100, &Operator::GT),
RowIDs::new_vector(), RowIDs::new_vector(),
); );
assert_eq!(row_ids.to_vec(), vec![11, 14, 15]); assert_eq!(row_ids.to_vec(), vec![11, 14, 15]);
let row_ids = v.row_ids_filter_range( let row_ids = v.row_ids_filter_range(
(21, Operator::GTE), (21, &Operator::GTE),
(21, Operator::LTE), (21, &Operator::LTE),
RowIDs::new_vector(), RowIDs::new_vector(),
); );
assert_eq!(row_ids.to_vec(), vec![16]); assert_eq!(row_ids.to_vec(), vec![16]);
let row_ids = v.row_ids_filter_range( let row_ids = v.row_ids_filter_range(
(10000, Operator::LTE), (10000, &Operator::LTE),
(3999, Operator::GT), (3999, &Operator::GT),
RowIDs::new_vector(), RowIDs::new_vector(),
); );
assert_eq!(row_ids.to_vec(), Vec::<u32>::new()); assert_eq!(row_ids.to_vec(), Vec::<u32>::new());
@ -926,8 +1008,8 @@ mod test {
.as_slice(), .as_slice(),
); );
let row_ids = v.row_ids_filter_range( let row_ids = v.row_ids_filter_range(
(200, Operator::GTE), (200, &Operator::GTE),
(300, Operator::LTE), (300, &Operator::LTE),
RowIDs::new_vector(), RowIDs::new_vector(),
); );
assert_eq!(row_ids.to_vec(), vec![1, 2, 4]); assert_eq!(row_ids.to_vec(), vec![1, 2, 4]);

762
read_buffer/src/column/integer.rs
View File

@ -1,12 +1,19 @@
use std::mem::size_of; use std::mem::size_of;
use arrow::{self, array::Array}; use arrow::{
self, array::Array, datatypes::Int16Type as ArrowInt16Type,
datatypes::Int32Type as ArrowInt32Type, datatypes::Int64Type as ArrowInt64Type,
datatypes::Int8Type as ArrowInt8Type, datatypes::UInt16Type as ArrowUInt16Type,
datatypes::UInt32Type as ArrowUInt32Type, datatypes::UInt64Type as ArrowUInt64Type,
datatypes::UInt8Type as ArrowUInt8Type,
};
use super::encoding::{scalar::Fixed, scalar::FixedNull}; use super::encoding::{scalar::Fixed, scalar::FixedNull};
use super::{cmp, Statistics}; use super::{cmp, Statistics};
use crate::column::{EncodedValues, RowIDs, Scalar, Value, Values}; use crate::column::{EncodedValues, RowIDs, Scalar, Value, Values};
pub enum IntegerEncoding { pub enum IntegerEncoding {
// non-null encodings. These are backed by `Vec<T>`
I64I64(Fixed<i64>), I64I64(Fixed<i64>),
I64I32(Fixed<i32>), I64I32(Fixed<i32>),
I64U32(Fixed<u32>), I64U32(Fixed<u32>),
@ -14,15 +21,97 @@ pub enum IntegerEncoding {
I64U16(Fixed<u16>), I64U16(Fixed<u16>),
I64I8(Fixed<i8>), I64I8(Fixed<i8>),
I64U8(Fixed<u8>), I64U8(Fixed<u8>),
U64U64(Fixed<u64>), U64U64(Fixed<u64>),
U64U32(Fixed<u32>), U64U32(Fixed<u32>),
U64U16(Fixed<u16>), U64U16(Fixed<u16>),
U64U8(Fixed<u8>), U64U8(Fixed<u8>),
// Nullable encodings - TODO, add variants for smaller physical types. // Nullable encodings. These are backed by an Arrow array.
I64I64N(FixedNull<arrow::datatypes::Int64Type>), I64I64N(FixedNull<ArrowInt64Type>),
U64U64N(FixedNull<arrow::datatypes::UInt64Type>), I64I32N(FixedNull<ArrowInt32Type>),
I64U32N(FixedNull<ArrowUInt32Type>),
I64I16N(FixedNull<ArrowInt16Type>),
I64U16N(FixedNull<ArrowUInt16Type>),
I64I8N(FixedNull<ArrowInt8Type>),
I64U8N(FixedNull<ArrowUInt8Type>),
U64U64N(FixedNull<ArrowUInt64Type>),
U64U32N(FixedNull<ArrowUInt32Type>),
U64U16N(FixedNull<ArrowUInt16Type>),
U64U8N(FixedNull<ArrowUInt8Type>),
}
impl PartialEq for IntegerEncoding {
fn eq(&self, other: &Self) -> bool {
match (self, other) {
(Self::I64I64(a), Self::I64I64(b)) => a == b,
(Self::I64I32(a), Self::I64I32(b)) => a == b,
(Self::I64U32(a), Self::I64U32(b)) => a == b,
(Self::I64I16(a), Self::I64I16(b)) => a == b,
(Self::I64U16(a), Self::I64U16(b)) => a == b,
(Self::I64I8(a), Self::I64I8(b)) => a == b,
(Self::I64U8(a), Self::I64U8(b)) => a == b,
(Self::U64U64(a), Self::U64U64(b)) => a == b,
(Self::U64U32(a), Self::U64U32(b)) => a == b,
(Self::U64U16(a), Self::U64U16(b)) => a == b,
(Self::U64U8(a), Self::U64U8(b)) => a == b,
(Self::I64I64N(a), Self::I64I64N(b)) => {
let a = a.all_values::<i64>(vec![]);
let b = b.all_values(vec![]);
a == b
}
(Self::I64I32N(a), Self::I64I32N(b)) => {
let a = a.all_values::<i64>(vec![]);
let b = b.all_values(vec![]);
a == b
}
(Self::I64U32N(a), Self::I64U32N(b)) => {
let a = a.all_values::<i64>(vec![]);
let b = b.all_values(vec![]);
a == b
}
(Self::I64I16N(a), Self::I64I16N(b)) => {
let a = a.all_values::<i64>(vec![]);
let b = b.all_values(vec![]);
a == b
}
(Self::I64U16N(a), Self::I64U16N(b)) => {
let a = a.all_values::<i64>(vec![]);
let b = b.all_values(vec![]);
a == b
}
(Self::I64I8N(a), Self::I64I8N(b)) => {
let a = a.all_values::<i64>(vec![]);
let b = b.all_values(vec![]);
a == b
}
(Self::I64U8N(a), Self::I64U8N(b)) => {
let a = a.all_values::<i64>(vec![]);
let b = b.all_values(vec![]);
a == b
}
(Self::U64U64N(a), Self::U64U64N(b)) => {
let a = a.all_values::<u64>(vec![]);
let b = b.all_values(vec![]);
a == b
}
(Self::U64U32N(a), Self::U64U32N(b)) => {
let a = a.all_values::<u64>(vec![]);
let b = b.all_values(vec![]);
a == b
}
(Self::U64U16N(a), Self::U64U16N(b)) => {
let a = a.all_values::<u64>(vec![]);
let b = b.all_values(vec![]);
a == b
}
(Self::U64U8N(a), Self::U64U8N(b)) => {
let a = a.all_values::<u64>(vec![]);
let b = b.all_values(vec![]);
a == b
}
(_, _) => false,
}
}
} }
impl IntegerEncoding { impl IntegerEncoding {
@ -41,7 +130,16 @@ impl IntegerEncoding {
Self::U64U16(enc) => enc.size(), Self::U64U16(enc) => enc.size(),
Self::U64U8(enc) => enc.size(), Self::U64U8(enc) => enc.size(),
Self::I64I64N(enc) => enc.size(), Self::I64I64N(enc) => enc.size(),
Self::I64I32N(enc) => enc.size(),
Self::I64U32N(enc) => enc.size(),
Self::I64I16N(enc) => enc.size(),
Self::I64U16N(enc) => enc.size(),
Self::I64I8N(enc) => enc.size(),
Self::I64U8N(enc) => enc.size(),
Self::U64U64N(enc) => enc.size(), Self::U64U64N(enc) => enc.size(),
Self::U64U32N(enc) => enc.size(),
Self::U64U16N(enc) => enc.size(),
Self::U64U8N(enc) => enc.size(),
} }
} }
@ -68,7 +166,16 @@ impl IntegerEncoding {
} }
Self::I64I64N(enc) => enc.size_raw(include_nulls), Self::I64I64N(enc) => enc.size_raw(include_nulls),
Self::I64I32N(enc) => enc.size_raw(include_nulls),
Self::I64U32N(enc) => enc.size_raw(include_nulls),
Self::I64I16N(enc) => enc.size_raw(include_nulls),
Self::I64U16N(enc) => enc.size_raw(include_nulls),
Self::I64I8N(enc) => enc.size_raw(include_nulls),
Self::I64U8N(enc) => enc.size_raw(include_nulls),
Self::U64U64N(enc) => enc.size_raw(include_nulls), Self::U64U64N(enc) => enc.size_raw(include_nulls),
Self::U64U32N(enc) => enc.size_raw(include_nulls),
Self::U64U16N(enc) => enc.size_raw(include_nulls),
Self::U64U8N(enc) => enc.size_raw(include_nulls),
} }
} }
@ -87,7 +194,16 @@ impl IntegerEncoding {
Self::U64U16(enc) => enc.num_rows(), Self::U64U16(enc) => enc.num_rows(),
Self::U64U8(enc) => enc.num_rows(), Self::U64U8(enc) => enc.num_rows(),
Self::I64I64N(enc) => enc.num_rows(), Self::I64I64N(enc) => enc.num_rows(),
Self::I64I32N(enc) => enc.num_rows(),
Self::I64U32N(enc) => enc.num_rows(),
Self::I64I16N(enc) => enc.num_rows(),
Self::I64U16N(enc) => enc.num_rows(),
Self::I64I8N(enc) => enc.num_rows(),
Self::I64U8N(enc) => enc.num_rows(),
Self::U64U64N(enc) => enc.num_rows(), Self::U64U64N(enc) => enc.num_rows(),
Self::U64U32N(enc) => enc.num_rows(),
Self::U64U16N(enc) => enc.num_rows(),
Self::U64U8N(enc) => enc.num_rows(),
} }
} }
@ -108,7 +224,16 @@ impl IntegerEncoding {
pub fn contains_null(&self) -> bool { pub fn contains_null(&self) -> bool {
match self { match self {
Self::I64I64N(enc) => enc.contains_null(), Self::I64I64N(enc) => enc.contains_null(),
Self::I64I32N(enc) => enc.contains_null(),
Self::I64U32N(enc) => enc.contains_null(),
Self::I64I16N(enc) => enc.contains_null(),
Self::I64U16N(enc) => enc.contains_null(),
Self::I64I8N(enc) => enc.contains_null(),
Self::I64U8N(enc) => enc.contains_null(),
Self::U64U64N(enc) => enc.contains_null(), Self::U64U64N(enc) => enc.contains_null(),
Self::U64U32N(enc) => enc.contains_null(),
Self::U64U16N(enc) => enc.contains_null(),
Self::U64U8N(enc) => enc.contains_null(),
_ => false, _ => false,
} }
} }
@ -128,7 +253,16 @@ impl IntegerEncoding {
Self::U64U16(_) => 0, Self::U64U16(_) => 0,
Self::U64U8(_) => 0, Self::U64U8(_) => 0,
Self::I64I64N(enc) => enc.null_count(), Self::I64I64N(enc) => enc.null_count(),
Self::I64I32N(enc) => enc.null_count(),
Self::I64U32N(enc) => enc.null_count(),
Self::I64I16N(enc) => enc.null_count(),
Self::I64U16N(enc) => enc.null_count(),
Self::I64I8N(enc) => enc.null_count(),
Self::I64U8N(enc) => enc.null_count(),
Self::U64U64N(enc) => enc.null_count(), Self::U64U64N(enc) => enc.null_count(),
Self::U64U32N(enc) => enc.null_count(),
Self::U64U16N(enc) => enc.null_count(),
Self::U64U8N(enc) => enc.null_count(),
} }
} }
@ -136,7 +270,16 @@ impl IntegerEncoding {
pub fn has_any_non_null_value(&self) -> bool { pub fn has_any_non_null_value(&self) -> bool {
match self { match self {
Self::I64I64N(enc) => enc.has_any_non_null_value(), Self::I64I64N(enc) => enc.has_any_non_null_value(),
Self::I64I32N(enc) => enc.has_any_non_null_value(),
Self::I64U32N(enc) => enc.has_any_non_null_value(),
Self::I64I16N(enc) => enc.has_any_non_null_value(),
Self::I64U16N(enc) => enc.has_any_non_null_value(),
Self::I64I8N(enc) => enc.has_any_non_null_value(),
Self::I64U8N(enc) => enc.has_any_non_null_value(),
Self::U64U64N(enc) => enc.has_any_non_null_value(), Self::U64U64N(enc) => enc.has_any_non_null_value(),
Self::U64U32N(enc) => enc.has_any_non_null_value(),
Self::U64U16N(enc) => enc.has_any_non_null_value(),
Self::U64U8N(enc) => enc.has_any_non_null_value(),
_ => true, _ => true,
} }
} }
@ -146,7 +289,16 @@ impl IntegerEncoding {
pub fn has_non_null_value(&self, row_ids: &[u32]) -> bool { pub fn has_non_null_value(&self, row_ids: &[u32]) -> bool {
match self { match self {
Self::I64I64N(enc) => enc.has_non_null_value(row_ids), Self::I64I64N(enc) => enc.has_non_null_value(row_ids),
Self::I64I32N(enc) => enc.has_non_null_value(row_ids),
Self::I64U32N(enc) => enc.has_non_null_value(row_ids),
Self::I64I16N(enc) => enc.has_non_null_value(row_ids),
Self::I64U16N(enc) => enc.has_non_null_value(row_ids),
Self::I64I8N(enc) => enc.has_non_null_value(row_ids),
Self::I64U8N(enc) => enc.has_non_null_value(row_ids),
Self::U64U64N(enc) => enc.has_non_null_value(row_ids), Self::U64U64N(enc) => enc.has_non_null_value(row_ids),
Self::U64U32N(enc) => enc.has_non_null_value(row_ids),
Self::U64U16N(enc) => enc.has_non_null_value(row_ids),
Self::U64U8N(enc) => enc.has_non_null_value(row_ids),
_ => !row_ids.is_empty(), // all rows will be non-null _ => !row_ids.is_empty(), // all rows will be non-null
} }
} }
@ -158,25 +310,64 @@ impl IntegerEncoding {
// `c.value` should return as the logical type // `c.value` should return as the logical type
// signed 64-bit variants - logical type is i64 for all these // signed 64-bit variants - logical type is i64 for all these
Self::I64I64(c) => Value::Scalar(Scalar::I64(c.value(row_id))), Self::I64I64(enc) => Value::Scalar(Scalar::I64(enc.value(row_id))),
Self::I64I32(c) => Value::Scalar(Scalar::I64(c.value(row_id))), Self::I64I32(enc) => Value::Scalar(Scalar::I64(enc.value(row_id))),
Self::I64U32(c) => Value::Scalar(Scalar::I64(c.value(row_id))), Self::I64U32(enc) => Value::Scalar(Scalar::I64(enc.value(row_id))),
Self::I64I16(c) => Value::Scalar(Scalar::I64(c.value(row_id))), Self::I64I16(enc) => Value::Scalar(Scalar::I64(enc.value(row_id))),
Self::I64U16(c) => Value::Scalar(Scalar::I64(c.value(row_id))), Self::I64U16(enc) => Value::Scalar(Scalar::I64(enc.value(row_id))),
Self::I64I8(c) => Value::Scalar(Scalar::I64(c.value(row_id))), Self::I64I8(enc) => Value::Scalar(Scalar::I64(enc.value(row_id))),
Self::I64U8(c) => Value::Scalar(Scalar::I64(c.value(row_id))), Self::I64U8(enc) => Value::Scalar(Scalar::I64(enc.value(row_id))),
// unsigned 64-bit variants - logical type is u64 for all these // unsigned 64-bit variants - logical type is u64 for all these
Self::U64U64(c) => Value::Scalar(Scalar::U64(c.value(row_id))), Self::U64U64(enc) => Value::Scalar(Scalar::U64(enc.value(row_id))),
Self::U64U32(c) => Value::Scalar(Scalar::U64(c.value(row_id))), Self::U64U32(enc) => Value::Scalar(Scalar::U64(enc.value(row_id))),
Self::U64U16(c) => Value::Scalar(Scalar::U64(c.value(row_id))), Self::U64U16(enc) => Value::Scalar(Scalar::U64(enc.value(row_id))),
Self::U64U8(c) => Value::Scalar(Scalar::U64(c.value(row_id))), Self::U64U8(enc) => Value::Scalar(Scalar::U64(enc.value(row_id))),
Self::I64I64N(c) => match c.value(row_id) { // signed 64-bit variants
Self::I64I64N(enc) => match enc.value(row_id) {
Some(v) => Value::Scalar(Scalar::I64(v)), Some(v) => Value::Scalar(Scalar::I64(v)),
None => Value::Null, None => Value::Null,
}, },
Self::U64U64N(c) => match c.value(row_id) { Self::I64I32N(enc) => match enc.value(row_id) {
Some(v) => Value::Scalar(Scalar::I64(v)),
None => Value::Null,
},
Self::I64U32N(enc) => match enc.value(row_id) {
Some(v) => Value::Scalar(Scalar::I64(v)),
None => Value::Null,
},
Self::I64I16N(enc) => match enc.value(row_id) {
Some(v) => Value::Scalar(Scalar::I64(v)),
None => Value::Null,
},
Self::I64U16N(enc) => match enc.value(row_id) {
Some(v) => Value::Scalar(Scalar::I64(v)),
None => Value::Null,
},
Self::I64I8N(enc) => match enc.value(row_id) {
Some(v) => Value::Scalar(Scalar::I64(v)),
None => Value::Null,
},
Self::I64U8N(enc) => match enc.value(row_id) {
Some(v) => Value::Scalar(Scalar::I64(v)),
None => Value::Null,
},
// unsigned 64-bit variants
Self::U64U64N(enc) => match enc.value(row_id) {
Some(v) => Value::Scalar(Scalar::U64(v)),
None => Value::Null,
},
Self::U64U32N(enc) => match enc.value(row_id) {
Some(v) => Value::Scalar(Scalar::U64(v)),
None => Value::Null,
},
Self::U64U16N(enc) => match enc.value(row_id) {
Some(v) => Value::Scalar(Scalar::U64(v)),
None => Value::Null,
},
Self::U64U8N(enc) => match enc.value(row_id) {
Some(v) => Value::Scalar(Scalar::U64(v)), Some(v) => Value::Scalar(Scalar::U64(v)),
None => Value::Null, None => Value::Null,
}, },
@ -190,22 +381,34 @@ impl IntegerEncoding {
pub fn values(&self, row_ids: &[u32]) -> Values<'_> { pub fn values(&self, row_ids: &[u32]) -> Values<'_> {
match &self { match &self {
// signed 64-bit variants - logical type is i64 for all these // signed 64-bit variants - logical type is i64 for all these
Self::I64I64(c) => Values::I64(c.values::<i64>(row_ids, vec![])), Self::I64I64(enc) => Values::I64(enc.values::<i64>(row_ids, vec![])),
Self::I64I32(c) => Values::I64(c.values::<i64>(row_ids, vec![])), Self::I64I32(enc) => Values::I64(enc.values::<i64>(row_ids, vec![])),
Self::I64U32(c) => Values::I64(c.values::<i64>(row_ids, vec![])), Self::I64U32(enc) => Values::I64(enc.values::<i64>(row_ids, vec![])),
Self::I64I16(c) => Values::I64(c.values::<i64>(row_ids, vec![])), Self::I64I16(enc) => Values::I64(enc.values::<i64>(row_ids, vec![])),
Self::I64U16(c) => Values::I64(c.values::<i64>(row_ids, vec![])), Self::I64U16(enc) => Values::I64(enc.values::<i64>(row_ids, vec![])),
Self::I64I8(c) => Values::I64(c.values::<i64>(row_ids, vec![])), Self::I64I8(enc) => Values::I64(enc.values::<i64>(row_ids, vec![])),
Self::I64U8(c) => Values::I64(c.values::<i64>(row_ids, vec![])), Self::I64U8(enc) => Values::I64(enc.values::<i64>(row_ids, vec![])),
// unsigned 64-bit variants - logical type is u64 for all these // unsigned 64-bit variants - logical type is u64 for all these
Self::U64U64(c) => Values::U64(c.values::<u64>(row_ids, vec![])), Self::U64U64(enc) => Values::U64(enc.values::<u64>(row_ids, vec![])),
Self::U64U32(c) => Values::U64(c.values::<u64>(row_ids, vec![])), Self::U64U32(enc) => Values::U64(enc.values::<u64>(row_ids, vec![])),
Self::U64U16(c) => Values::U64(c.values::<u64>(row_ids, vec![])), Self::U64U16(enc) => Values::U64(enc.values::<u64>(row_ids, vec![])),
Self::U64U8(c) => Values::U64(c.values::<u64>(row_ids, vec![])), Self::U64U8(enc) => Values::U64(enc.values::<u64>(row_ids, vec![])),
Self::I64I64N(c) => Values::I64N(c.values(row_ids, vec![])), // signed 64-bit nullable variants - logical type is i64 for all these.
Self::U64U64N(c) => Values::U64N(c.values(row_ids, vec![])), Self::I64I64N(enc) => Values::I64N(enc.values(row_ids, vec![])),
Self::I64I32N(enc) => Values::I64N(enc.values(row_ids, vec![])),
Self::I64U32N(enc) => Values::I64N(enc.values(row_ids, vec![])),
Self::I64I16N(enc) => Values::I64N(enc.values(row_ids, vec![])),
Self::I64U16N(enc) => Values::I64N(enc.values(row_ids, vec![])),
Self::I64I8N(enc) => Values::I64N(enc.values(row_ids, vec![])),
Self::I64U8N(enc) => Values::I64N(enc.values(row_ids, vec![])),
// unsigned 64-bit nullable variants - logical type is u64 for all these.
Self::U64U64N(enc) => Values::U64N(enc.values(row_ids, vec![])),
Self::U64U32N(enc) => Values::U64N(enc.values(row_ids, vec![])),
Self::U64U16N(enc) => Values::U64N(enc.values(row_ids, vec![])),
Self::U64U8N(enc) => Values::U64N(enc.values(row_ids, vec![])),
} }
} }
@ -230,8 +433,20 @@ impl IntegerEncoding {
Self::U64U16(c) => Values::U64(c.all_values::<u64>(vec![])), Self::U64U16(c) => Values::U64(c.all_values::<u64>(vec![])),
Self::U64U8(c) => Values::U64(c.all_values::<u64>(vec![])), Self::U64U8(c) => Values::U64(c.all_values::<u64>(vec![])),
Self::I64I64N(c) => Values::I64N(c.all_values(vec![])), // signed 64-bit nullable variants - logical type is i64 for all these.
Self::U64U64N(c) => Values::U64N(c.all_values(vec![])), Self::I64I64N(enc) => Values::I64N(enc.all_values(vec![])),
Self::I64I32N(enc) => Values::I64N(enc.all_values(vec![])),
Self::I64U32N(enc) => Values::I64N(enc.all_values(vec![])),
Self::I64I16N(enc) => Values::I64N(enc.all_values(vec![])),
Self::I64U16N(enc) => Values::I64N(enc.all_values(vec![])),
Self::I64I8N(enc) => Values::I64N(enc.all_values(vec![])),
Self::I64U8N(enc) => Values::I64N(enc.all_values(vec![])),
// unsigned 64-bit nullable variants - logical type is u64 for all these.
Self::U64U64N(enc) => Values::U64N(enc.all_values(vec![])),
Self::U64U32N(enc) => Values::U64N(enc.all_values(vec![])),
Self::U64U16N(enc) => Values::U64N(enc.all_values(vec![])),
Self::U64U8N(enc) => Values::U64N(enc.all_values(vec![])),
} }
} }
@ -297,8 +512,20 @@ impl IntegerEncoding {
Self::U64U16(c) => c.row_ids_filter(value.as_u16(), op, dst), Self::U64U16(c) => c.row_ids_filter(value.as_u16(), op, dst),
Self::U64U8(c) => c.row_ids_filter(value.as_u8(), op, dst), Self::U64U8(c) => c.row_ids_filter(value.as_u8(), op, dst),
Self::I64I64N(c) => c.row_ids_filter(value.as_i64(), op, dst), // signed 64-bit nullable variants - logical type is i64 for all these.
Self::U64U64N(c) => c.row_ids_filter(value.as_u64(), op, dst), Self::I64I64N(enc) => enc.row_ids_filter(value.as_i64(), op, dst),
Self::I64I32N(enc) => enc.row_ids_filter(value.as_i32(), op, dst),
Self::I64U32N(enc) => enc.row_ids_filter(value.as_u32(), op, dst),
Self::I64I16N(enc) => enc.row_ids_filter(value.as_i16(), op, dst),
Self::I64U16N(enc) => enc.row_ids_filter(value.as_u16(), op, dst),
Self::I64I8N(enc) => enc.row_ids_filter(value.as_i8(), op, dst),
Self::I64U8N(enc) => enc.row_ids_filter(value.as_u8(), op, dst),
// unsigned 64-bit nullable variants - logical type is u64 for all these.
Self::U64U64N(enc) => enc.row_ids_filter(value.as_u64(), op, dst),
Self::U64U32N(enc) => enc.row_ids_filter(value.as_u32(), op, dst),
Self::U64U16N(enc) => enc.row_ids_filter(value.as_u16(), op, dst),
Self::U64U8N(enc) => enc.row_ids_filter(value.as_u8(), op, dst),
} }
} }
@ -349,8 +576,41 @@ impl IntegerEncoding {
c.row_ids_filter_range((low.1.as_u8(), low.0), (high.1.as_u8(), high.0), dst) c.row_ids_filter_range((low.1.as_u8(), low.0), (high.1.as_u8(), high.0), dst)
} }
Self::I64I64N(_) => todo!(), Self::I64I64N(enc) => {
Self::U64U64N(_) => todo!(), enc.row_ids_filter_range((low.1.as_i64(), low.0), (high.1.as_i64(), high.0), dst)
}
Self::I64I32N(enc) => {
enc.row_ids_filter_range((low.1.as_i32(), low.0), (high.1.as_i32(), high.0), dst)
}
Self::I64U32N(enc) => {
enc.row_ids_filter_range((low.1.as_u32(), low.0), (high.1.as_u32(), high.0), dst)
}
Self::I64I16N(enc) => {
enc.row_ids_filter_range((low.1.as_i16(), low.0), (high.1.as_i16(), high.0), dst)
}
Self::I64U16N(enc) => {
enc.row_ids_filter_range((low.1.as_u16(), low.0), (high.1.as_u16(), high.0), dst)
}
Self::I64I8N(enc) => {
enc.row_ids_filter_range((low.1.as_i8(), low.0), (high.1.as_i8(), high.0), dst)
}
Self::I64U8N(enc) => {
enc.row_ids_filter_range((low.1.as_u8(), low.0), (high.1.as_u8(), high.0), dst)
}
// unsigned 64-bit nullable variants - logical type is u64 for all these.
Self::U64U64N(enc) => {
enc.row_ids_filter_range((low.1.as_u64(), low.0), (high.1.as_u64(), high.0), dst)
}
Self::U64U32N(enc) => {
enc.row_ids_filter_range((low.1.as_u32(), low.0), (high.1.as_u32(), high.0), dst)
}
Self::U64U16N(enc) => {
enc.row_ids_filter_range((low.1.as_u16(), low.0), (high.1.as_u16(), high.0), dst)
}
Self::U64U8N(enc) => {
enc.row_ids_filter_range((low.1.as_u8(), low.0), (high.1.as_u8(), high.0), dst)
}
} }
} }
@ -367,11 +627,49 @@ impl IntegerEncoding {
Self::U64U32(c) => Value::Scalar(Scalar::U64(c.min(row_ids))), Self::U64U32(c) => Value::Scalar(Scalar::U64(c.min(row_ids))),
Self::U64U16(c) => Value::Scalar(Scalar::U64(c.min(row_ids))), Self::U64U16(c) => Value::Scalar(Scalar::U64(c.min(row_ids))),
Self::U64U8(c) => Value::Scalar(Scalar::U64(c.min(row_ids))), Self::U64U8(c) => Value::Scalar(Scalar::U64(c.min(row_ids))),
Self::I64I64N(c) => match c.min(row_ids) {
Self::I64I64N(enc) => match enc.min(row_ids) {
Some(v) => Value::Scalar(Scalar::I64(v)), Some(v) => Value::Scalar(Scalar::I64(v)),
None => Value::Null, None => Value::Null,
}, },
Self::U64U64N(c) => match c.min(row_ids) { Self::I64I32N(enc) => match enc.min(row_ids) {
Some(v) => Value::Scalar(Scalar::I64(v)),
None => Value::Null,
},
Self::I64U32N(enc) => match enc.min(row_ids) {
Some(v) => Value::Scalar(Scalar::I64(v)),
None => Value::Null,
},
Self::I64I16N(enc) => match enc.min(row_ids) {
Some(v) => Value::Scalar(Scalar::I64(v)),
None => Value::Null,
},
Self::I64U16N(enc) => match enc.min(row_ids) {
Some(v) => Value::Scalar(Scalar::I64(v)),
None => Value::Null,
},
Self::I64I8N(enc) => match enc.min(row_ids) {
Some(v) => Value::Scalar(Scalar::I64(v)),
None => Value::Null,
},
Self::I64U8N(enc) => match enc.min(row_ids) {
Some(v) => Value::Scalar(Scalar::I64(v)),
None => Value::Null,
},
Self::U64U64N(enc) => match enc.min(row_ids) {
Some(v) => Value::Scalar(Scalar::U64(v)),
None => Value::Null,
},
Self::U64U32N(enc) => match enc.min(row_ids) {
Some(v) => Value::Scalar(Scalar::U64(v)),
None => Value::Null,
},
Self::U64U16N(enc) => match enc.min(row_ids) {
Some(v) => Value::Scalar(Scalar::U64(v)),
None => Value::Null,
},
Self::U64U8N(enc) => match enc.min(row_ids) {
Some(v) => Value::Scalar(Scalar::U64(v)), Some(v) => Value::Scalar(Scalar::U64(v)),
None => Value::Null, None => Value::Null,
}, },
@ -391,11 +689,48 @@ impl IntegerEncoding {
Self::U64U32(c) => Value::Scalar(Scalar::U64(c.max(row_ids))), Self::U64U32(c) => Value::Scalar(Scalar::U64(c.max(row_ids))),
Self::U64U16(c) => Value::Scalar(Scalar::U64(c.max(row_ids))), Self::U64U16(c) => Value::Scalar(Scalar::U64(c.max(row_ids))),
Self::U64U8(c) => Value::Scalar(Scalar::U64(c.max(row_ids))), Self::U64U8(c) => Value::Scalar(Scalar::U64(c.max(row_ids))),
Self::I64I64N(c) => match c.max(row_ids) { Self::I64I64N(enc) => match enc.max(row_ids) {
Some(v) => Value::Scalar(Scalar::I64(v)), Some(v) => Value::Scalar(Scalar::I64(v)),
None => Value::Null, None => Value::Null,
}, },
Self::U64U64N(c) => match c.max(row_ids) { Self::I64I32N(enc) => match enc.max(row_ids) {
Some(v) => Value::Scalar(Scalar::I64(v)),
None => Value::Null,
},
Self::I64U32N(enc) => match enc.max(row_ids) {
Some(v) => Value::Scalar(Scalar::I64(v)),
None => Value::Null,
},
Self::I64I16N(enc) => match enc.max(row_ids) {
Some(v) => Value::Scalar(Scalar::I64(v)),
None => Value::Null,
},
Self::I64U16N(enc) => match enc.max(row_ids) {
Some(v) => Value::Scalar(Scalar::I64(v)),
None => Value::Null,
},
Self::I64I8N(enc) => match enc.max(row_ids) {
Some(v) => Value::Scalar(Scalar::I64(v)),
None => Value::Null,
},
Self::I64U8N(enc) => match enc.max(row_ids) {
Some(v) => Value::Scalar(Scalar::I64(v)),
None => Value::Null,
},
Self::U64U64N(enc) => match enc.max(row_ids) {
Some(v) => Value::Scalar(Scalar::U64(v)),
None => Value::Null,
},
Self::U64U32N(enc) => match enc.max(row_ids) {
Some(v) => Value::Scalar(Scalar::U64(v)),
None => Value::Null,
},
Self::U64U16N(enc) => match enc.max(row_ids) {
Some(v) => Value::Scalar(Scalar::U64(v)),
None => Value::Null,
},
Self::U64U8N(enc) => match enc.max(row_ids) {
Some(v) => Value::Scalar(Scalar::U64(v)), Some(v) => Value::Scalar(Scalar::U64(v)),
None => Value::Null, None => Value::Null,
}, },
@ -415,11 +750,48 @@ impl IntegerEncoding {
Self::U64U32(c) => Scalar::U64(c.sum(row_ids)), Self::U64U32(c) => Scalar::U64(c.sum(row_ids)),
Self::U64U16(c) => Scalar::U64(c.sum(row_ids)), Self::U64U16(c) => Scalar::U64(c.sum(row_ids)),
Self::U64U8(c) => Scalar::U64(c.sum(row_ids)), Self::U64U8(c) => Scalar::U64(c.sum(row_ids)),
Self::I64I64N(c) => match c.sum(row_ids) { Self::I64I64N(enc) => match enc.sum(row_ids) {
Some(v) => Scalar::I64(v), Some(v) => Scalar::I64(v),
None => Scalar::Null, None => Scalar::Null,
}, },
Self::U64U64N(c) => match c.sum(row_ids) { Self::I64I32N(enc) => match enc.sum(row_ids) {
Some(v) => Scalar::I64(v),
None => Scalar::Null,
},
Self::I64U32N(enc) => match enc.sum(row_ids) {
Some(v) => Scalar::I64(v),
None => Scalar::Null,
},
Self::I64I16N(enc) => match enc.sum(row_ids) {
Some(v) => Scalar::I64(v),
None => Scalar::Null,
},
Self::I64U16N(enc) => match enc.sum(row_ids) {
Some(v) => Scalar::I64(v),
None => Scalar::Null,
},
Self::I64I8N(enc) => match enc.sum(row_ids) {
Some(v) => Scalar::I64(v),
None => Scalar::Null,
},
Self::I64U8N(enc) => match enc.sum(row_ids) {
Some(v) => Scalar::I64(v),
None => Scalar::Null,
},
Self::U64U64N(enc) => match enc.sum(row_ids) {
Some(v) => Scalar::U64(v),
None => Scalar::Null,
},
Self::U64U32N(enc) => match enc.sum(row_ids) {
Some(v) => Scalar::U64(v),
None => Scalar::Null,
},
Self::U64U16N(enc) => match enc.sum(row_ids) {
Some(v) => Scalar::U64(v),
None => Scalar::Null,
},
Self::U64U8N(enc) => match enc.sum(row_ids) {
Some(v) => Scalar::U64(v), Some(v) => Scalar::U64(v),
None => Scalar::Null, None => Scalar::Null,
}, },
@ -439,8 +811,17 @@ impl IntegerEncoding {
Self::U64U32(c) => c.count(row_ids), Self::U64U32(c) => c.count(row_ids),
Self::U64U16(c) => c.count(row_ids), Self::U64U16(c) => c.count(row_ids),
Self::U64U8(c) => c.count(row_ids), Self::U64U8(c) => c.count(row_ids),
Self::I64I64N(c) => c.count(row_ids), Self::I64I64N(enc) => enc.count(row_ids),
Self::U64U64N(c) => c.count(row_ids), Self::I64I32N(enc) => enc.count(row_ids),
Self::I64U32N(enc) => enc.count(row_ids),
Self::I64I16N(enc) => enc.count(row_ids),
Self::I64U16N(enc) => enc.count(row_ids),
Self::I64I8N(enc) => enc.count(row_ids),
Self::I64U8N(enc) => enc.count(row_ids),
Self::U64U64N(enc) => enc.count(row_ids),
Self::U64U32N(enc) => enc.count(row_ids),
Self::U64U16N(enc) => enc.count(row_ids),
Self::U64U8N(enc) => enc.count(row_ids),
} }
} }
@ -459,7 +840,16 @@ impl IntegerEncoding {
Self::U64U16(_) => "BT_U16", Self::U64U16(_) => "BT_U16",
Self::U64U8(_) => "BT_U8", Self::U64U8(_) => "BT_U8",
Self::I64I64N(_) => "None", Self::I64I64N(_) => "None",
Self::I64I32N(_) => "BT_I32N",
Self::I64U32N(_) => "BT_U32N",
Self::I64I16N(_) => "BT_U16N",
Self::I64U16N(_) => "BT_U16N",
Self::I64I8N(_) => "BT_I8N",
Self::I64U8N(_) => "BT_U8N",
Self::U64U64N(_) => "None", Self::U64U64N(_) => "None",
Self::U64U32N(_) => "BT_U32N",
Self::U64U16N(_) => "BT_U16N",
Self::U64U8N(_) => "BT_U8N",
} }
} }
@ -478,7 +868,16 @@ impl IntegerEncoding {
Self::U64U16(_) => "u64", Self::U64U16(_) => "u64",
Self::U64U8(_) => "u64", Self::U64U8(_) => "u64",
Self::I64I64N(_) => "i64", Self::I64I64N(_) => "i64",
Self::I64I32N(_) => "i64",
Self::I64U32N(_) => "i64",
Self::I64I16N(_) => "i64",
Self::I64U16N(_) => "i64",
Self::I64I8N(_) => "i64",
Self::I64U8N(_) => "i64",
Self::U64U64N(_) => "u64", Self::U64U64N(_) => "u64",
Self::U64U32N(_) => "u64",
Self::U64U16N(_) => "u64",
Self::U64U8N(_) => "u64",
} }
} }
} }
@ -499,7 +898,46 @@ impl std::fmt::Display for IntegerEncoding {
Self::U64U16(enc) => write!(f, "[{}]: {}", name, enc), Self::U64U16(enc) => write!(f, "[{}]: {}", name, enc),
Self::U64U8(enc) => write!(f, "[{}]: {}", name, enc), Self::U64U8(enc) => write!(f, "[{}]: {}", name, enc),
Self::I64I64N(enc) => write!(f, "[{}]: {}", name, enc), Self::I64I64N(enc) => write!(f, "[{}]: {}", name, enc),
Self::I64I32N(enc) => write!(f, "[{}]: {}", name, enc),
Self::I64U32N(enc) => write!(f, "[{}]: {}", name, enc),
Self::I64I16N(enc) => write!(f, "[{}]: {}", name, enc),
Self::I64U16N(enc) => write!(f, "[{}]: {}", name, enc),
Self::I64I8N(enc) => write!(f, "[{}]: {}", name, enc),
Self::I64U8N(enc) => write!(f, "[{}]: {}", name, enc),
Self::U64U64N(enc) => write!(f, "[{}]: {}", name, enc), Self::U64U64N(enc) => write!(f, "[{}]: {}", name, enc),
Self::U64U32N(enc) => write!(f, "[{}]: {}", name, enc),
Self::U64U16N(enc) => write!(f, "[{}]: {}", name, enc),
Self::U64U8N(enc) => write!(f, "[{}]: {}", name, enc),
}
}
}
impl std::fmt::Debug for IntegerEncoding {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let name = self.name();
match self {
Self::I64I64(enc) => enc.fmt(f),
Self::I64I32(enc) => write!(f, "[{}]: {:?}", name, enc),
Self::I64U32(enc) => write!(f, "[{}]: {:?}", name, enc),
Self::I64I16(enc) => write!(f, "[{}]: {:?}", name, enc),
Self::I64U16(enc) => write!(f, "[{}]: {:?}", name, enc),
Self::I64I8(enc) => write!(f, "[{}]: {:?}", name, enc),
Self::I64U8(enc) => write!(f, "[{}]: {:?}", name, enc),
Self::U64U64(enc) => write!(f, "[{}]: {:?}", name, enc),
Self::U64U32(enc) => write!(f, "[{}]: {:?}", name, enc),
Self::U64U16(enc) => write!(f, "[{}]: {:?}", name, enc),
Self::U64U8(enc) => write!(f, "[{}]: {:?}", name, enc),
Self::I64I64N(enc) => write!(f, "[{}]: {:?}", name, enc),
Self::I64I32N(enc) => write!(f, "[{}]: {:?}", name, enc),
Self::I64U32N(enc) => write!(f, "[{}]: {:?}", name, enc),
Self::I64I16N(enc) => write!(f, "[{}]: {:?}", name, enc),
Self::I64U16N(enc) => write!(f, "[{}]: {:?}", name, enc),
Self::I64I8N(enc) => write!(f, "[{}]: {:?}", name, enc),
Self::I64U8N(enc) => write!(f, "[{}]: {:?}", name, enc),
Self::U64U64N(enc) => write!(f, "[{}]: {:?}", name, enc),
Self::U64U32N(enc) => write!(f, "[{}]: {:?}", name, enc),
Self::U64U16N(enc) => write!(f, "[{}]: {:?}", name, enc),
Self::U64U8N(enc) => write!(f, "[{}]: {:?}", name, enc),
} }
} }
} }
@ -553,17 +991,48 @@ impl From<&[i64]> for IntegerEncoding {
/// Converts an Arrow array into an IntegerEncoding. /// Converts an Arrow array into an IntegerEncoding.
/// ///
/// TODO(edd): convert underlying type of Arrow data to smallest physical /// The most compact physical Arrow array type is used to store the column
/// representation. /// within a `FixedNull` encoding.
impl From<arrow::array::Int64Array> for IntegerEncoding { impl From<arrow::array::Int64Array> for IntegerEncoding {
fn from(arr: arrow::array::Int64Array) -> Self { fn from(arr: arrow::array::Int64Array) -> Self {
if arr.null_count() == 0 { if arr.null_count() == 0 {
return Self::from(arr.values()); return Self::from(arr.values());
} }
// TODO(edd): currently fixed null only supports 64-bit logical/physical // determine min and max values.
// types. Need to add support for storing as smaller physical types. let min = arrow::compute::kernels::aggregate::min(&arr);
Self::I64I64N(FixedNull::<arrow::datatypes::Int64Type>::from(arr)) let max = arrow::compute::kernels::aggregate::max(&arr);
// This match is carefully ordered. It prioritises smaller physical
// datatypes that can safely represent the provided logical data
match (min, max) {
// encode as u8 values
(min, max) if min >= Some(0) && max <= Some(u8::MAX as i64) => {
Self::I64U8N(FixedNull::<ArrowUInt8Type>::from(arr))
}
// encode as i8 values
(min, max) if min >= Some(i8::MIN as i64) && max <= Some(i8::MAX as i64) => {
Self::I64I8N(FixedNull::<ArrowInt8Type>::from(arr))
}
// encode as u16 values
(min, max) if min >= Some(0) && max <= Some(u16::MAX as i64) => {
Self::I64U16N(FixedNull::<ArrowUInt16Type>::from(arr))
}
// encode as i16 values
(min, max) if min >= Some(i16::MIN as i64) && max <= Some(i16::MAX as i64) => {
Self::I64I16N(FixedNull::<ArrowInt16Type>::from(arr))
}
// encode as u32 values
(min, max) if min >= Some(0) && max <= Some(u32::MAX as i64) => {
Self::I64U32N(FixedNull::<ArrowUInt32Type>::from(arr))
}
// encode as i32 values
(min, max) if min >= Some(i32::MIN as i64) && max <= Some(i32::MAX as i64) => {
Self::I64I32N(FixedNull::<ArrowInt32Type>::from(arr))
}
// otherwise, encode with the same physical type (i64)
(_, _) => Self::I64I64N(FixedNull::<ArrowInt64Type>::from(arr)),
}
} }
} }
@ -600,23 +1069,42 @@ impl From<&[u64]> for IntegerEncoding {
/// Converts an Arrow array into an IntegerEncoding. /// Converts an Arrow array into an IntegerEncoding.
/// ///
/// TODO(edd): convert underlying type of Arrow data to smallest physical /// The most compact physical Arrow array type is used to store the column
/// representation. /// within a `FixedNull` encoding.
impl From<arrow::array::UInt64Array> for IntegerEncoding { impl From<arrow::array::UInt64Array> for IntegerEncoding {
fn from(arr: arrow::array::UInt64Array) -> Self { fn from(arr: arrow::array::UInt64Array) -> Self {
if arr.null_count() == 0 { if arr.null_count() == 0 {
return Self::from(arr.values()); return Self::from(arr.values());
} }
// TODO(edd): currently fixed null only supports 64-bit logical/physical // determine max value.
// types. Need to add support for storing as smaller physical types. let max = arrow::compute::kernels::aggregate::max(&arr);
Self::U64U64N(FixedNull::<arrow::datatypes::UInt64Type>::from(arr))
// This match is carefully ordered. It prioritises smaller physical
// datatypes that can safely represent the provided logical data
match max {
// encode as u8 values
max if max <= Some(u8::MAX as u64) => {
Self::U64U8N(FixedNull::<ArrowUInt8Type>::from(arr))
}
// encode as u16 values
max if max <= Some(u16::MAX as u64) => {
Self::U64U16N(FixedNull::<ArrowUInt16Type>::from(arr))
}
// encode as u32 values
max if max <= Some(u32::MAX as u64) => {
Self::U64U32N(FixedNull::<ArrowUInt32Type>::from(arr))
}
// otherwise, encode with the same physical type (u64)
_ => Self::U64U64N(FixedNull::<ArrowUInt64Type>::from(arr)),
}
} }
} }
#[cfg(test)] #[cfg(test)]
mod test { mod test {
use arrow::datatypes::Int64Type; use arrow::array::{Int64Array, UInt64Array};
use std::iter;
use super::*; use super::*;
@ -628,7 +1116,7 @@ mod test {
vec![399_i64, 2, 2452, 3], vec![399_i64, 2, 2452, 3],
vec![-399_i64, 2, 2452, 3], vec![-399_i64, 2, 2452, 3],
vec![u32::MAX as i64, 2, 245, 3], vec![u32::MAX as i64, 2, 245, 3],
vec![i32::MAX as i64, 2, 245, 3], vec![i32::MIN as i64, 2, 245, 3],
vec![0_i64, 2, 245, u32::MAX as i64 + 1], vec![0_i64, 2, 245, u32::MAX as i64 + 1],
]; ];
@ -642,9 +1130,156 @@ mod test {
IntegerEncoding::I64I64(Fixed::<i64>::from(cases[6].as_slice())), IntegerEncoding::I64I64(Fixed::<i64>::from(cases[6].as_slice())),
]; ];
for (_case, _exp) in cases.iter().zip(exp.iter()) { for (case, exp) in cases.into_iter().zip(exp.into_iter()) {
// TODO - add debug assert_eq!(IntegerEncoding::from(case.as_slice()), exp);
//assert_eq!(IntegerEncoding::from(&case), exp); }
}
#[test]
fn from_slice_u64() {
let cases = vec![
vec![0_u64, 2, 245, 3],
vec![399_u64, 2, 2452, 3],
vec![u32::MAX as u64, 2, 245, 3],
vec![0_u64, 2, 245, u32::MAX as u64 + 1],
];
let exp = vec![
IntegerEncoding::U64U8(Fixed::<u8>::from(cases[0].as_slice())),
IntegerEncoding::U64U16(Fixed::<u16>::from(cases[1].as_slice())),
IntegerEncoding::U64U32(Fixed::<u32>::from(cases[2].as_slice())),
IntegerEncoding::U64U64(Fixed::<u64>::from(cases[3].as_slice())),
];
for (case, exp) in cases.into_iter().zip(exp.into_iter()) {
assert_eq!(IntegerEncoding::from(case.as_slice()), exp);
}
}
#[test]
fn from_arrow_i64_array() {
let cases = vec![
vec![0_i64, 2, 245, 3],
vec![0_i64, -120, 127, 3],
vec![399_i64, 2, 2452, 3],
vec![-399_i64, 2, 2452, 3],
vec![u32::MAX as i64, 2, 245, 3],
vec![i32::MIN as i64, 2, 245, 3],
vec![0_i64, 2, 245, u32::MAX as i64 + 1],
];
let exp = vec![
IntegerEncoding::I64U8(Fixed::<u8>::from(cases[0].as_slice())),
IntegerEncoding::I64I8(Fixed::<i8>::from(cases[1].as_slice())),
IntegerEncoding::I64U16(Fixed::<u16>::from(cases[2].as_slice())),
IntegerEncoding::I64I16(Fixed::<i16>::from(cases[3].as_slice())),
IntegerEncoding::I64U32(Fixed::<u32>::from(cases[4].as_slice())),
IntegerEncoding::I64I32(Fixed::<i32>::from(cases[5].as_slice())),
IntegerEncoding::I64I64(Fixed::<i64>::from(cases[6].as_slice())),
];
// for Arrow arrays with no nulls we can store the column using a
// non-nullable fixed encoding
for (case, exp) in cases.iter().cloned().zip(exp.into_iter()) {
let arr = Int64Array::from(case);
assert_eq!(IntegerEncoding::from(arr), exp);
}
// Tack a NULL onto each of the input cases.
let cases = cases
.iter()
.map(|case| {
case.iter()
.map(|x| Some(*x))
.chain(iter::repeat(None).take(1))
.collect::<Vec<_>>()
})
.collect::<Vec<_>>();
// when a NULL value is present then we need to use a nullable encoding.
let exp = vec![
IntegerEncoding::I64U8N(FixedNull::<ArrowUInt8Type>::from(Int64Array::from(
cases[0].clone(),
))),
IntegerEncoding::I64I8N(FixedNull::<ArrowInt8Type>::from(Int64Array::from(
cases[1].clone(),
))),
IntegerEncoding::I64U16N(FixedNull::<ArrowUInt16Type>::from(Int64Array::from(
cases[2].clone(),
))),
IntegerEncoding::I64I16N(FixedNull::<ArrowInt16Type>::from(Int64Array::from(
cases[3].clone(),
))),
IntegerEncoding::I64U32N(FixedNull::<ArrowUInt32Type>::from(Int64Array::from(
cases[4].clone(),
))),
IntegerEncoding::I64I32N(FixedNull::<ArrowInt32Type>::from(Int64Array::from(
cases[5].clone(),
))),
IntegerEncoding::I64I64N(FixedNull::<ArrowInt64Type>::from(Int64Array::from(
cases[6].clone(),
))),
];
for (case, exp) in cases.into_iter().zip(exp.into_iter()) {
let arr = Int64Array::from(case.clone());
assert_eq!(IntegerEncoding::from(arr), exp);
}
}
#[test]
fn from_arrow_u64_array() {
let cases = vec![
vec![0_u64, 2, 245, 3],
vec![399_u64, 2, 2452, 3],
vec![u32::MAX as u64, 2, 245, 3],
vec![0_u64, 2, 245, u32::MAX as u64 + 1],
];
let exp = vec![
IntegerEncoding::U64U8(Fixed::<u8>::from(cases[0].as_slice())),
IntegerEncoding::U64U16(Fixed::<u16>::from(cases[1].as_slice())),
IntegerEncoding::U64U32(Fixed::<u32>::from(cases[2].as_slice())),
IntegerEncoding::U64U64(Fixed::<u64>::from(cases[3].as_slice())),
];
// for Arrow arrays with no nulls we can store the column using a
// non-nullable fixed encoding
for (case, exp) in cases.iter().cloned().zip(exp.into_iter()) {
let arr = UInt64Array::from(case);
assert_eq!(IntegerEncoding::from(arr), exp);
}
// Tack a NULL onto each of the input cases.
let cases = cases
.iter()
.map(|case| {
case.iter()
.map(|x| Some(*x))
.chain(iter::repeat(None).take(1))
.collect::<Vec<_>>()
})
.collect::<Vec<_>>();
// when a NULL value is present then we need to use a nullable encoding.
let exp = vec![
IntegerEncoding::U64U8N(FixedNull::<ArrowUInt8Type>::from(UInt64Array::from(
cases[0].clone(),
))),
IntegerEncoding::U64U16N(FixedNull::<ArrowUInt16Type>::from(UInt64Array::from(
cases[1].clone(),
))),
IntegerEncoding::U64U32N(FixedNull::<ArrowUInt32Type>::from(UInt64Array::from(
cases[2].clone(),
))),
IntegerEncoding::U64U64N(FixedNull::<ArrowUInt64Type>::from(UInt64Array::from(
cases[3].clone(),
))),
];
for (case, exp) in cases.into_iter().zip(exp.into_iter()) {
let arr = UInt64Array::from(case.clone());
assert_eq!(IntegerEncoding::from(arr), exp);
} }
} }
@ -660,12 +1295,7 @@ mod test {
assert_eq!(enc.size_raw(true), 56); assert_eq!(enc.size_raw(true), 56);
assert_eq!(enc.size_raw(false), 56); assert_eq!(enc.size_raw(false), 56);
let enc = IntegerEncoding::I64I64N(FixedNull::<Int64Type>::from(&[2, 22, 12, 31][..])); let enc = IntegerEncoding::I64I64N(FixedNull::<ArrowInt64Type>::from(
// (4 * 8) + 24
assert_eq!(enc.size_raw(true), 56);
assert_eq!(enc.size_raw(false), 56);
let enc = IntegerEncoding::I64I64N(FixedNull::<Int64Type>::from(
&[Some(2), Some(22), Some(12), None, None, Some(31)][..], &[Some(2), Some(22), Some(12), None, None, Some(31)][..],
)); ));
// (6 * 8) + 24 // (6 * 8) + 24

1
server/Cargo.toml
View File

@ -11,6 +11,7 @@ arrow_flight = { path = "../arrow_flight" }
async-trait = "0.1" async-trait = "0.1"
bytes = { version = "1.0" } bytes = { version = "1.0" }
chrono = "0.4" chrono = "0.4"
cached = "0.23.0"
crc32fast = "1.2.0" crc32fast = "1.2.0"
data_types = { path = "../data_types" } data_types = { path = "../data_types" }
datafusion = { path = "../datafusion" } datafusion = { path = "../datafusion" }

45
server/src/config.rs
View File

@ -6,10 +6,14 @@ use std::{
use data_types::{database_rules::DatabaseRules, server_id::ServerId, DatabaseName}; use data_types::{database_rules::DatabaseRules, server_id::ServerId, DatabaseName};
use metrics::MetricRegistry; use metrics::MetricRegistry;
use object_store::{path::ObjectStorePath, ObjectStore}; use object_store::{path::ObjectStorePath, ObjectStore};
use parquet_file::catalog::PreservedCatalog;
use query::exec::Executor; use query::exec::Executor;
/// This module contains code for managing the configuration of the server. /// This module contains code for managing the configuration of the server.
use crate::{db::Db, Error, JobRegistry, Result}; use crate::{
db::{catalog::Catalog, Db},
Error, JobRegistry, Result,
};
use observability_deps::tracing::{self, error, info, warn, Instrument}; use observability_deps::tracing::{self, error, info, warn, Instrument};
use tokio::task::JoinHandle; use tokio::task::JoinHandle;
use tokio_util::sync::CancellationToken; use tokio_util::sync::CancellationToken;
@ -125,6 +129,7 @@ impl Config {
server_id: ServerId, server_id: ServerId,
object_store: Arc<ObjectStore>, object_store: Arc<ObjectStore>,
exec: Arc<Executor>, exec: Arc<Executor>,
preserved_catalog: PreservedCatalog<Catalog>,
) { ) {
let mut state = self.state.write().expect("mutex poisoned"); let mut state = self.state.write().expect("mutex poisoned");
let name = state let name = state
@ -148,7 +153,7 @@ impl Config {
exec, exec,
write_buffer, write_buffer,
Arc::clone(&self.jobs), Arc::clone(&self.jobs),
Arc::clone(&self.metric_registry), preserved_catalog,
)); ));
let shutdown = self.shutdown.child_token(); let shutdown = self.shutdown.child_token();
@ -203,6 +208,10 @@ impl Config {
info!("database background workers shutdown"); info!("database background workers shutdown");
} }
pub fn metrics_registry(&self) -> Arc<MetricRegistry> {
Arc::clone(&self.metric_registry)
}
} }
pub fn object_store_path_for_database_config<P: ObjectStorePath>( pub fn object_store_path_for_database_config<P: ObjectStorePath>(
@ -274,9 +283,15 @@ impl<'a> CreateDatabaseHandle<'a> {
server_id: ServerId, server_id: ServerId,
object_store: Arc<ObjectStore>, object_store: Arc<ObjectStore>,
exec: Arc<Executor>, exec: Arc<Executor>,
preserved_catalog: PreservedCatalog<Catalog>,
) { ) {
self.config self.config.commit(
.commit(self.rules.take().unwrap(), server_id, object_store, exec) self.rules.take().unwrap(),
server_id,
object_store,
exec,
preserved_catalog,
)
} }
pub(crate) fn rules(&self) -> &DatabaseRules { pub(crate) fn rules(&self) -> &DatabaseRules {
@ -298,6 +313,8 @@ mod test {
use object_store::{memory::InMemory, ObjectStore, ObjectStoreApi}; use object_store::{memory::InMemory, ObjectStore, ObjectStoreApi};
use crate::db::load_preserved_catalog;
use super::*; use super::*;
#[tokio::test] #[tokio::test]
@ -317,7 +334,15 @@ mod test {
let server_id = ServerId::try_from(1).unwrap(); let server_id = ServerId::try_from(1).unwrap();
let store = Arc::new(ObjectStore::new_in_memory(InMemory::new())); let store = Arc::new(ObjectStore::new_in_memory(InMemory::new()));
let exec = Arc::new(Executor::new(1)); let exec = Arc::new(Executor::new(1));
db_reservation.commit(server_id, store, exec); let preserved_catalog = load_preserved_catalog(
&name,
Arc::clone(&store),
server_id,
config.metrics_registry(),
)
.await
.unwrap();
db_reservation.commit(server_id, store, exec, preserved_catalog);
assert!(config.db(&name).is_some()); assert!(config.db(&name).is_some());
assert_eq!(config.db_names_sorted(), vec![name.clone()]); assert_eq!(config.db_names_sorted(), vec![name.clone()]);
@ -345,7 +370,15 @@ mod test {
let server_id = ServerId::try_from(1).unwrap(); let server_id = ServerId::try_from(1).unwrap();
let store = Arc::new(ObjectStore::new_in_memory(InMemory::new())); let store = Arc::new(ObjectStore::new_in_memory(InMemory::new()));
let exec = Arc::new(Executor::new(1)); let exec = Arc::new(Executor::new(1));
db_reservation.commit(server_id, store, exec); let preserved_catalog = load_preserved_catalog(
&name,
Arc::clone(&store),
server_id,
config.metrics_registry(),
)
.await
.unwrap();
db_reservation.commit(server_id, store, exec, preserved_catalog);
let token = config let token = config
.state .state

151
server/src/db.rs
View File

@ -310,6 +310,33 @@ pub struct Db {
metric_labels: Vec<KeyValue>, metric_labels: Vec<KeyValue>,
} }
/// Load preserved catalog state from store.
pub async fn load_preserved_catalog(
db_name: &str,
object_store: Arc<ObjectStore>,
server_id: ServerId,
metrics_registry: Arc<MetricRegistry>,
) -> std::result::Result<PreservedCatalog<Catalog>, parquet_file::catalog::Error> {
let metric_labels = vec![
KeyValue::new("db_name", db_name.to_string()),
KeyValue::new("svr_id", format!("{}", server_id)),
];
let metrics_domain =
metrics_registry.register_domain_with_labels("catalog", metric_labels.clone());
PreservedCatalog::new_empty(
Arc::clone(&object_store),
server_id,
db_name.to_string(),
CatalogEmptyInput {
domain: metrics_domain,
metrics_registry: Arc::clone(&metrics_registry),
metric_labels: metric_labels.clone(),
},
)
.await
}
impl Db { impl Db {
pub fn new( pub fn new(
rules: DatabaseRules, rules: DatabaseRules,
@ -318,33 +345,19 @@ impl Db {
exec: Arc<Executor>, exec: Arc<Executor>,
write_buffer: Option<Buffer>, write_buffer: Option<Buffer>,
jobs: Arc<JobRegistry>, jobs: Arc<JobRegistry>,
metrics_registry: Arc<MetricRegistry>, preserved_catalog: PreservedCatalog<Catalog>,
) -> Self { ) -> Self {
let db_name = rules.name.clone(); let db_name = rules.name.clone();
let metric_labels = vec![
KeyValue::new("db_name", db_name.to_string()),
KeyValue::new("svr_id", format!("{}", server_id)),
];
let metrics_domain =
metrics_registry.register_domain_with_labels("catalog", metric_labels.clone());
let rules = RwLock::new(rules); let rules = RwLock::new(rules);
let server_id = server_id; let server_id = server_id;
let store = Arc::clone(&object_store); let store = Arc::clone(&object_store);
let write_buffer = write_buffer.map(Mutex::new); let write_buffer = write_buffer.map(Mutex::new);
let catalog = PreservedCatalog::new_empty( let system_tables =
Arc::clone(&object_store), SystemSchemaProvider::new(&db_name, preserved_catalog.state(), Arc::clone(&jobs));
server_id,
db_name.to_string(),
CatalogEmptyInput {
domain: metrics_domain,
metrics_registry: Arc::clone(&metrics_registry),
metric_labels: metric_labels.clone(),
},
);
let system_tables = SystemSchemaProvider::new(&db_name, catalog.state(), Arc::clone(&jobs));
let system_tables = Arc::new(system_tables); let system_tables = Arc::new(system_tables);
let metrics_registry = Arc::clone(&preserved_catalog.state().metrics_registry);
let metric_labels = preserved_catalog.state().metric_labels.clone();
let process_clock = process_clock::ProcessClock::new(); let process_clock = process_clock::ProcessClock::new();
@ -353,7 +366,7 @@ impl Db {
server_id, server_id,
store, store,
exec, exec,
catalog, catalog: preserved_catalog,
write_buffer, write_buffer,
jobs, jobs,
metrics_registry, metrics_registry,
@ -1205,10 +1218,10 @@ mod tests {
type Error = Box<dyn std::error::Error + Send + Sync + 'static>; type Error = Box<dyn std::error::Error + Send + Sync + 'static>;
type Result<T, E = Error> = std::result::Result<T, E>; type Result<T, E = Error> = std::result::Result<T, E>;
#[test] #[tokio::test]
fn write_no_mutable_buffer() { async fn write_no_mutable_buffer() {
// Validate that writes are rejected if there is no mutable buffer // Validate that writes are rejected if there is no mutable buffer
let db = make_db().db; let db = make_db().await.db;
db.rules.write().lifecycle_rules.immutable = true; db.rules.write().lifecycle_rules.immutable = true;
let entry = lp_to_entry("cpu bar=1 10"); let entry = lp_to_entry("cpu bar=1 10");
let res = db.store_entry(entry); let res = db.store_entry(entry);
@ -1220,7 +1233,7 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn read_write() { async fn read_write() {
let db = Arc::new(make_db().db); let db = Arc::new(make_db().await.db);
write_lp(&db, "cpu bar=1 10"); write_lp(&db, "cpu bar=1 10");
let batches = run_query(db, "select * from cpu").await; let batches = run_query(db, "select * from cpu").await;
@ -1252,7 +1265,7 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn metrics_during_rollover() { async fn metrics_during_rollover() {
let test_db = make_db(); let test_db = make_db().await;
let db = Arc::new(test_db.db); let db = Arc::new(test_db.db);
write_lp(db.as_ref(), "cpu bar=1 10"); write_lp(db.as_ref(), "cpu bar=1 10");
@ -1372,7 +1385,7 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn write_with_rollover() { async fn write_with_rollover() {
let db = Arc::new(make_db().db); let db = Arc::new(make_db().await.db);
write_lp(db.as_ref(), "cpu bar=1 10"); write_lp(db.as_ref(), "cpu bar=1 10");
assert_eq!(vec!["1970-01-01T00"], db.partition_keys().unwrap()); assert_eq!(vec!["1970-01-01T00"], db.partition_keys().unwrap());
@ -1420,7 +1433,7 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn write_with_missing_tags_are_null() { async fn write_with_missing_tags_are_null() {
let db = Arc::new(make_db().db); let db = Arc::new(make_db().await.db);
// Note the `region` tag is introduced in the second line, so // Note the `region` tag is introduced in the second line, so
// the values in prior rows for the region column are // the values in prior rows for the region column are
// null. Likewise the `core` tag is introduced in the third // null. Likewise the `core` tag is introduced in the third
@ -1457,7 +1470,7 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn read_from_read_buffer() { async fn read_from_read_buffer() {
// Test that data can be loaded into the ReadBuffer // Test that data can be loaded into the ReadBuffer
let test_db = make_db(); let test_db = make_db().await;
let db = Arc::new(test_db.db); let db = Arc::new(test_db.db);
write_lp(db.as_ref(), "cpu bar=1 10"); write_lp(db.as_ref(), "cpu bar=1 10");
@ -1543,7 +1556,7 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn load_to_read_buffer_sorted() { async fn load_to_read_buffer_sorted() {
let test_db = make_db(); let test_db = make_db().await;
let db = Arc::new(test_db.db); let db = Arc::new(test_db.db);
write_lp(db.as_ref(), "cpu,tag1=cupcakes bar=1 10"); write_lp(db.as_ref(), "cpu,tag1=cupcakes bar=1 10");
@ -1655,7 +1668,8 @@ mod tests {
.server_id(server_id) .server_id(server_id)
.object_store(Arc::clone(&object_store)) .object_store(Arc::clone(&object_store))
.db_name(db_name) .db_name(db_name)
.build(); .build()
.await;
let db = Arc::new(test_db.db); let db = Arc::new(test_db.db);
@ -1754,7 +1768,8 @@ mod tests {
.server_id(server_id) .server_id(server_id)
.object_store(Arc::clone(&object_store)) .object_store(Arc::clone(&object_store))
.db_name(db_name) .db_name(db_name)
.build(); .build()
.await;
let db = Arc::new(test_db.db); let db = Arc::new(test_db.db);
@ -1866,9 +1881,9 @@ mod tests {
assert_batches_eq!(expected, &record_batches); assert_batches_eq!(expected, &record_batches);
} }
#[test] #[tokio::test]
fn write_updates_last_write_at() { async fn write_updates_last_write_at() {
let db = Arc::new(make_db().db); let db = Arc::new(make_db().await.db);
let before_create = Utc::now(); let before_create = Utc::now();
let partition_key = "1970-01-01T00"; let partition_key = "1970-01-01T00";
@ -1896,7 +1911,7 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn test_chunk_timestamps() { async fn test_chunk_timestamps() {
let start = Utc::now(); let start = Utc::now();
let db = Arc::new(make_db().db); let db = Arc::new(make_db().await.db);
// Given data loaded into two chunks // Given data loaded into two chunks
write_lp(&db, "cpu bar=1 10"); write_lp(&db, "cpu bar=1 10");
@ -1931,9 +1946,9 @@ mod tests {
assert!(chunk.time_closed().unwrap() < after_rollover); assert!(chunk.time_closed().unwrap() < after_rollover);
} }
#[test] #[tokio::test]
fn test_chunk_closing() { async fn test_chunk_closing() {
let db = Arc::new(make_db().db); let db = Arc::new(make_db().await.db);
db.rules.write().lifecycle_rules.mutable_size_threshold = db.rules.write().lifecycle_rules.mutable_size_threshold =
Some(NonZeroUsize::new(2).unwrap()); Some(NonZeroUsize::new(2).unwrap());
@ -1952,9 +1967,9 @@ mod tests {
assert!(matches!(chunks[1].read().state(), ChunkState::Closed(_))); assert!(matches!(chunks[1].read().state(), ChunkState::Closed(_)));
} }
#[test] #[tokio::test]
fn chunks_sorted_by_times() { async fn chunks_sorted_by_times() {
let db = Arc::new(make_db().db); let db = Arc::new(make_db().await.db);
write_lp(&db, "cpu val=1 1"); write_lp(&db, "cpu val=1 1");
write_lp(&db, "mem val=2 400000000000001"); write_lp(&db, "mem val=2 400000000000001");
write_lp(&db, "cpu val=1 2"); write_lp(&db, "cpu val=1 2");
@ -1987,7 +2002,7 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn chunk_id_listing() { async fn chunk_id_listing() {
// Test that chunk id listing is hooked up // Test that chunk id listing is hooked up
let db = Arc::new(make_db().db); let db = Arc::new(make_db().await.db);
let partition_key = "1970-01-01T00"; let partition_key = "1970-01-01T00";
write_lp(&db, "cpu bar=1 10"); write_lp(&db, "cpu bar=1 10");
@ -2056,7 +2071,7 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn partition_chunk_summaries() { async fn partition_chunk_summaries() {
// Test that chunk id listing is hooked up // Test that chunk id listing is hooked up
let db = Arc::new(make_db().db); let db = Arc::new(make_db().await.db);
write_lp(&db, "cpu bar=1 1"); write_lp(&db, "cpu bar=1 1");
db.rollover_partition("1970-01-01T00", "cpu").await.unwrap(); db.rollover_partition("1970-01-01T00", "cpu").await.unwrap();
@ -2104,7 +2119,7 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn partition_chunk_summaries_timestamp() { async fn partition_chunk_summaries_timestamp() {
let db = Arc::new(make_db().db); let db = Arc::new(make_db().await.db);
let start = Utc::now(); let start = Utc::now();
write_lp(&db, "cpu bar=1 1"); write_lp(&db, "cpu bar=1 1");
let after_first_write = Utc::now(); let after_first_write = Utc::now();
@ -2155,7 +2170,7 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn chunk_summaries() { async fn chunk_summaries() {
// Test that chunk id listing is hooked up // Test that chunk id listing is hooked up
let db = Arc::new(make_db().db); let db = Arc::new(make_db().await.db);
// get three chunks: one open, one closed in mb and one close in rb // get three chunks: one open, one closed in mb and one close in rb
write_lp(&db, "cpu bar=1 1"); write_lp(&db, "cpu bar=1 1");
@ -2250,7 +2265,7 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn partition_summaries() { async fn partition_summaries() {
// Test that chunk id listing is hooked up // Test that chunk id listing is hooked up
let db = Arc::new(make_db().db); let db = Arc::new(make_db().await.db);
write_lp(&db, "cpu bar=1 1"); write_lp(&db, "cpu bar=1 1");
let chunk_id = db let chunk_id = db
@ -2460,7 +2475,7 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn write_chunk_to_object_store_in_background() { async fn write_chunk_to_object_store_in_background() {
// Test that data can be written to object store using a background task // Test that data can be written to object store using a background task
let db = Arc::new(make_db().db); let db = Arc::new(make_db().await.db);
// create MB partition // create MB partition
write_lp(db.as_ref(), "cpu bar=1 10"); write_lp(db.as_ref(), "cpu bar=1 10");
@ -2501,9 +2516,9 @@ mod tests {
assert_eq!(read_parquet_file_chunk_ids(&db, partition_key), vec![0]); assert_eq!(read_parquet_file_chunk_ids(&db, partition_key), vec![0]);
} }
#[test] #[tokio::test]
fn write_hard_limit() { async fn write_hard_limit() {
let db = Arc::new(make_db().db); let db = Arc::new(make_db().await.db);
db.rules.write().lifecycle_rules.buffer_size_hard = Some(NonZeroUsize::new(10).unwrap()); db.rules.write().lifecycle_rules.buffer_size_hard = Some(NonZeroUsize::new(10).unwrap());
// inserting first line does not trigger hard buffer limit // inserting first line does not trigger hard buffer limit
@ -2516,9 +2531,9 @@ mod tests {
)); ));
} }
#[test] #[tokio::test]
fn write_goes_to_write_buffer_if_configured() { async fn write_goes_to_write_buffer_if_configured() {
let db = Arc::new(TestDb::builder().write_buffer(true).build().db); let db = Arc::new(TestDb::builder().write_buffer(true).build().await.db);
assert_eq!(db.write_buffer.as_ref().unwrap().lock().size(), 0); assert_eq!(db.write_buffer.as_ref().unwrap().lock().size(), 0);
write_lp(db.as_ref(), "cpu bar=1 10"); write_lp(db.as_ref(), "cpu bar=1 10");
@ -2536,7 +2551,8 @@ mod tests {
.server_id(server_id) .server_id(server_id)
.object_store(Arc::clone(&object_store)) .object_store(Arc::clone(&object_store))
.db_name(db_name) .db_name(db_name)
.build(); .build()
.await;
let db = Arc::new(test_db.db); let db = Arc::new(test_db.db);
@ -2704,9 +2720,22 @@ mod tests {
.object_store(Arc::clone(&object_store)) .object_store(Arc::clone(&object_store))
.server_id(server_id) .server_id(server_id)
.db_name(db_name) .db_name(db_name)
.build(); .build()
.await;
let db = Arc::new(test_db.db); let db = Arc::new(test_db.db);
// at this point, an empty preserved catalog exists
let maybe_preserved_catalog =
PreservedCatalog::<parquet_file::catalog::test_helpers::TestCatalogState>::load(
Arc::clone(&object_store),
server_id,
db_name.to_string(),
(),
)
.await
.unwrap();
assert!(maybe_preserved_catalog.is_some());
// Write some line protocols in Mutable buffer of the DB // Write some line protocols in Mutable buffer of the DB
write_lp(db.as_ref(), "cpu bar=1 10"); write_lp(db.as_ref(), "cpu bar=1 10");
@ -2722,18 +2751,6 @@ mod tests {
.await .await
.unwrap(); .unwrap();
// at this point, no preserved catalog exists
let maybe_preserved_catalog =
PreservedCatalog::<parquet_file::catalog::test_helpers::TestCatalogState>::load(
Arc::clone(&object_store),
server_id,
db_name.to_string(),
(),
)
.await
.unwrap();
assert!(maybe_preserved_catalog.is_none());
// Write the RB chunk to Object Store but keep it in RB // Write the RB chunk to Object Store but keep it in RB
db.write_chunk_to_object_store(partition_key, "cpu", mb_chunk.id()) db.write_chunk_to_object_store(partition_key, "cpu", mb_chunk.id())
.await .await

18
server/src/db/process_clock.rs
View File

@ -75,11 +75,11 @@ mod tests {
use entry::test_helpers::lp_to_entry; use entry::test_helpers::lp_to_entry;
use std::{sync::Arc, thread, time::Duration}; use std::{sync::Arc, thread, time::Duration};
#[test] #[tokio::test]
fn process_clock_defaults_to_current_time_in_ns() { async fn process_clock_defaults_to_current_time_in_ns() {
let before = system_clock_now(); let before = system_clock_now();
let db = Arc::new(TestDb::builder().build().db); let db = Arc::new(TestDb::builder().build().await.db);
let db_process_clock = db.process_clock.inner.load(Ordering::SeqCst); let db_process_clock = db.process_clock.inner.load(Ordering::SeqCst);
let after = system_clock_now(); let after = system_clock_now();
@ -98,12 +98,12 @@ mod tests {
); );
} }
#[test] #[tokio::test]
fn process_clock_incremented_and_set_on_sequenced_entry() { async fn process_clock_incremented_and_set_on_sequenced_entry() {
let before = system_clock_now(); let before = system_clock_now();
let before = ClockValue::try_from(before).unwrap(); let before = ClockValue::try_from(before).unwrap();
let db = Arc::new(TestDb::builder().write_buffer(true).build().db); let db = Arc::new(TestDb::builder().write_buffer(true).build().await.db);
let entry = lp_to_entry("cpu bar=1 10"); let entry = lp_to_entry("cpu bar=1 10");
db.store_entry(entry).unwrap(); db.store_entry(entry).unwrap();
@ -147,10 +147,10 @@ mod tests {
); );
} }
#[test] #[tokio::test]
fn next_process_clock_always_increments() { async fn next_process_clock_always_increments() {
// Process clock defaults to the current time // Process clock defaults to the current time
let db = Arc::new(TestDb::builder().write_buffer(true).build().db); let db = Arc::new(TestDb::builder().write_buffer(true).build().await.db);
// Set the process clock value to a time in the future, so that when compared to the // Set the process clock value to a time in the future, so that when compared to the
// current time, the process clock value will be greater // current time, the process clock value will be greater

70
server/src/lib.rs
View File

@ -72,6 +72,8 @@ use std::sync::Arc;
use async_trait::async_trait; use async_trait::async_trait;
use bytes::BytesMut; use bytes::BytesMut;
use cached::proc_macro::cached;
use db::load_preserved_catalog;
use futures::stream::TryStreamExt; use futures::stream::TryStreamExt;
use observability_deps::tracing::{error, info, warn}; use observability_deps::tracing::{error, info, warn};
use parking_lot::Mutex; use parking_lot::Mutex;
@ -175,6 +177,8 @@ pub enum Error {
}, },
#[snafu(display("remote error: {}", source))] #[snafu(display("remote error: {}", source))]
RemoteError { source: ConnectionManagerError }, RemoteError { source: ConnectionManagerError },
#[snafu(display("cannot load catalog: {}", source))]
CatalogLoadError { source: DatabaseError },
} }
pub type Result<T, E = Error> = std::result::Result<T, E>; pub type Result<T, E = Error> = std::result::Result<T, E>;
@ -418,12 +422,26 @@ impl<M: ConnectionManager> Server<M> {
pub async fn create_database(&self, rules: DatabaseRules, server_id: ServerId) -> Result<()> { pub async fn create_database(&self, rules: DatabaseRules, server_id: ServerId) -> Result<()> {
// Return an error if this server hasn't yet been setup with an id // Return an error if this server hasn't yet been setup with an id
self.require_id()?; self.require_id()?;
let db_reservation = self.config.create_db(rules)?;
let preserved_catalog = load_preserved_catalog(
rules.db_name(),
Arc::clone(&self.store),
server_id,
self.config.metrics_registry(),
)
.await
.map_err(|e| Box::new(e) as _)
.context(CatalogLoadError)?;
let db_reservation = self.config.create_db(rules)?;
self.persist_database_rules(db_reservation.rules().clone()) self.persist_database_rules(db_reservation.rules().clone())
.await?; .await?;
db_reservation.commit(
db_reservation.commit(server_id, Arc::clone(&self.store), Arc::clone(&self.exec)); server_id,
Arc::clone(&self.store),
Arc::clone(&self.exec),
preserved_catalog,
);
Ok(()) Ok(())
} }
@ -507,10 +525,24 @@ impl<M: ConnectionManager> Server<M> {
Err(e) => { Err(e) => {
error!("error parsing database config {:?} from store: {}", path, e) error!("error parsing database config {:?} from store: {}", path, e)
} }
Ok(rules) => match config.create_db(rules) { Ok(rules) => {
Err(e) => error!("error adding database to config: {}", e), match load_preserved_catalog(
Ok(handle) => handle.commit(server_id, store, exec), rules.db_name(),
}, Arc::clone(&store),
server_id,
config.metrics_registry(),
)
.await
{
Err(e) => error!("cannot load database: {}", e),
Ok(preserved_catalog) => match config.create_db(rules) {
Err(e) => error!("error adding database to config: {}", e),
Ok(handle) => {
handle.commit(server_id, store, exec, preserved_catalog)
}
},
}
}
}; };
}) })
}) })
@ -905,17 +937,25 @@ impl ConnectionManager for ConnectionManagerImpl {
&self, &self,
connect: &str, connect: &str,
) -> Result<Arc<Self::RemoteServer>, ConnectionManagerError> { ) -> Result<Arc<Self::RemoteServer>, ConnectionManagerError> {
// TODO(mkm): cache the connections cached_remote_server(connect.to_string()).await
let connection = Builder::default()
.build(connect)
.await
.map_err(|e| Box::new(e) as _)
.context(RemoteServerConnectError)?;
let client = write::Client::new(connection);
Ok(Arc::new(RemoteServerImpl { client }))
} }
} }
// cannot be an associated function
// argument need to have static lifetime because they become caching keys
#[cached(result = true)]
async fn cached_remote_server(
connect: String,
) -> Result<Arc<RemoteServerImpl>, ConnectionManagerError> {
let connection = Builder::default()
.build(&connect)
.await
.map_err(|e| Box::new(e) as _)
.context(RemoteServerConnectError)?;
let client = write::Client::new(connection);
Ok(Arc::new(RemoteServerImpl { client }))
}
/// An implementation for communicating with other IOx servers. This should /// An implementation for communicating with other IOx servers. This should
/// be moved into and implemented in an influxdb_iox_client create at a later /// be moved into and implemented in an influxdb_iox_client create at a later
/// date. /// date.

6
server/src/query_tests/influxrpc/read_window_aggregate.rs
View File

@ -162,7 +162,7 @@ impl DbSetup for MeasurementForWindowAggregateMonths {
// partition keys are: ["2020-03-02T00", "2020-03-01T00", "2020-04-01T00", // partition keys are: ["2020-03-02T00", "2020-03-01T00", "2020-04-01T00",
// "2020-04-02T00"] // "2020-04-02T00"]
let db = make_db().db; let db = make_db().await.db;
let data = lp_lines.join("\n"); let data = lp_lines.join("\n");
write_lp(&db, &data); write_lp(&db, &data);
let scenario1 = DbScenario { let scenario1 = DbScenario {
@ -170,7 +170,7 @@ impl DbSetup for MeasurementForWindowAggregateMonths {
db, db,
}; };
let db = make_db().db; let db = make_db().await.db;
let data = lp_lines.join("\n"); let data = lp_lines.join("\n");
write_lp(&db, &data); write_lp(&db, &data);
db.rollover_partition("2020-03-01T00", "h2o").await.unwrap(); db.rollover_partition("2020-03-01T00", "h2o").await.unwrap();
@ -182,7 +182,7 @@ impl DbSetup for MeasurementForWindowAggregateMonths {
db, db,
}; };
let db = make_db().db; let db = make_db().await.db;
let data = lp_lines.join("\n"); let data = lp_lines.join("\n");
write_lp(&db, &data); write_lp(&db, &data);
// roll over and load chunks into both RUB and OS // roll over and load chunks into both RUB and OS

38
server/src/query_tests/scenarios.rs
View File

@ -36,7 +36,7 @@ impl DbSetup for NoData {
// Scenario 1: No data in the DB yet // Scenario 1: No data in the DB yet
// //
let db = make_db().db; let db = make_db().await.db;
let scenario1 = DbScenario { let scenario1 = DbScenario {
scenario_name: "New, Empty Database".into(), scenario_name: "New, Empty Database".into(),
db, db,
@ -45,7 +45,7 @@ impl DbSetup for NoData {
// Scenario 2: listing partitions (which may create an entry in a map) // Scenario 2: listing partitions (which may create an entry in a map)
// in an empty database // in an empty database
// //
let db = make_db().db; let db = make_db().await.db;
assert_eq!(count_mutable_buffer_chunks(&db), 0); assert_eq!(count_mutable_buffer_chunks(&db), 0);
assert_eq!(count_read_buffer_chunks(&db), 0); assert_eq!(count_read_buffer_chunks(&db), 0);
assert_eq!(count_object_store_chunks(&db), 0); assert_eq!(count_object_store_chunks(&db), 0);
@ -56,7 +56,7 @@ impl DbSetup for NoData {
// Scenario 3: the database has had data loaded into RB and then deleted // Scenario 3: the database has had data loaded into RB and then deleted
// //
let db = make_db().db; let db = make_db().await.db;
let data = "cpu,region=west user=23.2 100"; let data = "cpu,region=west user=23.2 100";
write_lp(&db, data); write_lp(&db, data);
// move data out of open chunk // move data out of open chunk
@ -94,7 +94,7 @@ impl DbSetup for NoData {
// Scenario 4: the database has had data loaded into RB & Object Store and then deleted // Scenario 4: the database has had data loaded into RB & Object Store and then deleted
// //
let db = make_db().db; let db = make_db().await.db;
let data = "cpu,region=west user=23.2 100"; let data = "cpu,region=west user=23.2 100";
write_lp(&db, data); write_lp(&db, data);
// move data out of open chunk // move data out of open chunk
@ -253,7 +253,7 @@ pub struct TwoMeasurementsManyFieldsOneChunk {}
#[async_trait] #[async_trait]
impl DbSetup for TwoMeasurementsManyFieldsOneChunk { impl DbSetup for TwoMeasurementsManyFieldsOneChunk {
async fn make(&self) -> Vec<DbScenario> { async fn make(&self) -> Vec<DbScenario> {
let db = make_db().db; let db = make_db().await.db;
let lp_lines = vec![ let lp_lines = vec![
"h2o,state=MA,city=Boston temp=70.4 50", "h2o,state=MA,city=Boston temp=70.4 50",
@ -277,7 +277,7 @@ pub struct TwoMeasurementsManyFieldsTwoChunks {}
#[async_trait] #[async_trait]
impl DbSetup for TwoMeasurementsManyFieldsTwoChunks { impl DbSetup for TwoMeasurementsManyFieldsTwoChunks {
async fn make(&self) -> Vec<DbScenario> { async fn make(&self) -> Vec<DbScenario> {
let db = make_db().db; let db = make_db().await.db;
let partition_key = "1970-01-01T00"; let partition_key = "1970-01-01T00";
@ -314,7 +314,7 @@ impl DbSetup for TwoMeasurementsManyFieldsLifecycle {
async fn make(&self) -> Vec<DbScenario> { async fn make(&self) -> Vec<DbScenario> {
let partition_key = "1970-01-01T00"; let partition_key = "1970-01-01T00";
let db = std::sync::Arc::new(make_db().db); let db = std::sync::Arc::new(make_db().await.db);
write_lp( write_lp(
&db, &db,
@ -397,7 +397,7 @@ impl DbSetup for EndToEndTest {
let lp_data = lp_lines.join("\n"); let lp_data = lp_lines.join("\n");
let db = make_db().db; let db = make_db().await.db;
write_lp(&db, &lp_data); write_lp(&db, &lp_data);
let scenario1 = DbScenario { let scenario1 = DbScenario {
@ -413,7 +413,7 @@ impl DbSetup for EndToEndTest {
/// ///
pub(crate) async fn make_one_chunk_scenarios(partition_key: &str, data: &str) -> Vec<DbScenario> { pub(crate) async fn make_one_chunk_scenarios(partition_key: &str, data: &str) -> Vec<DbScenario> {
// Scenario 1: One open chunk in MUB // Scenario 1: One open chunk in MUB
let db = make_db().db; let db = make_db().await.db;
write_lp(&db, data); write_lp(&db, data);
let scenario1 = DbScenario { let scenario1 = DbScenario {
scenario_name: "Data in open chunk of mutable buffer".into(), scenario_name: "Data in open chunk of mutable buffer".into(),
@ -421,7 +421,7 @@ pub(crate) async fn make_one_chunk_scenarios(partition_key: &str, data: &str) ->
}; };
// Scenario 2: One closed chunk in MUB // Scenario 2: One closed chunk in MUB
let db = make_db().db; let db = make_db().await.db;
let table_names = write_lp(&db, data); let table_names = write_lp(&db, data);
for table_name in &table_names { for table_name in &table_names {
db.rollover_partition(partition_key, &table_name) db.rollover_partition(partition_key, &table_name)
@ -434,7 +434,7 @@ pub(crate) async fn make_one_chunk_scenarios(partition_key: &str, data: &str) ->
}; };
// Scenario 3: One closed chunk in RUB // Scenario 3: One closed chunk in RUB
let db = make_db().db; let db = make_db().await.db;
let table_names = write_lp(&db, data); let table_names = write_lp(&db, data);
for table_name in &table_names { for table_name in &table_names {
db.rollover_partition(partition_key, &table_name) db.rollover_partition(partition_key, &table_name)
@ -450,7 +450,7 @@ pub(crate) async fn make_one_chunk_scenarios(partition_key: &str, data: &str) ->
}; };
// Scenario 4: One closed chunk in both RUb and OS // Scenario 4: One closed chunk in both RUb and OS
let db = make_db().db; let db = make_db().await.db;
let table_names = write_lp(&db, data); let table_names = write_lp(&db, data);
for table_name in &table_names { for table_name in &table_names {
db.rollover_partition(partition_key, &table_name) db.rollover_partition(partition_key, &table_name)
@ -469,7 +469,7 @@ pub(crate) async fn make_one_chunk_scenarios(partition_key: &str, data: &str) ->
}; };
// Scenario 5: One closed chunk in OS only // Scenario 5: One closed chunk in OS only
let db = make_db().db; let db = make_db().await.db;
let table_names = write_lp(&db, data); let table_names = write_lp(&db, data);
for table_name in &table_names { for table_name in &table_names {
db.rollover_partition(partition_key, &table_name) db.rollover_partition(partition_key, &table_name)
@ -504,7 +504,7 @@ pub async fn make_two_chunk_scenarios(
data1: &str, data1: &str,
data2: &str, data2: &str,
) -> Vec<DbScenario> { ) -> Vec<DbScenario> {
let db = make_db().db; let db = make_db().await.db;
write_lp(&db, data1); write_lp(&db, data1);
write_lp(&db, data2); write_lp(&db, data2);
let scenario1 = DbScenario { let scenario1 = DbScenario {
@ -513,7 +513,7 @@ pub async fn make_two_chunk_scenarios(
}; };
// spread across 2 mutable buffer chunks // spread across 2 mutable buffer chunks
let db = make_db().db; let db = make_db().await.db;
let table_names = write_lp(&db, data1); let table_names = write_lp(&db, data1);
for table_name in &table_names { for table_name in &table_names {
db.rollover_partition(partition_key, &table_name) db.rollover_partition(partition_key, &table_name)
@ -527,7 +527,7 @@ pub async fn make_two_chunk_scenarios(
}; };
// spread across 1 mutable buffer, 1 read buffer chunks // spread across 1 mutable buffer, 1 read buffer chunks
let db = make_db().db; let db = make_db().await.db;
let table_names = write_lp(&db, data1); let table_names = write_lp(&db, data1);
for table_name in &table_names { for table_name in &table_names {
db.rollover_partition(partition_key, &table_name) db.rollover_partition(partition_key, &table_name)
@ -544,7 +544,7 @@ pub async fn make_two_chunk_scenarios(
}; };
// in 2 read buffer chunks // in 2 read buffer chunks
let db = make_db().db; let db = make_db().await.db;
let table_names = write_lp(&db, data1); let table_names = write_lp(&db, data1);
for table_name in &table_names { for table_name in &table_names {
db.rollover_partition(partition_key, &table_name) db.rollover_partition(partition_key, &table_name)
@ -571,7 +571,7 @@ pub async fn make_two_chunk_scenarios(
}; };
// in 2 read buffer chunks that also loaded into object store // in 2 read buffer chunks that also loaded into object store
let db = make_db().db; let db = make_db().await.db;
let table_names = write_lp(&db, data1); let table_names = write_lp(&db, data1);
for table_name in &table_names { for table_name in &table_names {
db.rollover_partition(partition_key, &table_name) db.rollover_partition(partition_key, &table_name)
@ -606,7 +606,7 @@ pub async fn make_two_chunk_scenarios(
}; };
// Scenario 6: Two closed chunk in OS only // Scenario 6: Two closed chunk in OS only
let db = make_db().db; let db = make_db().await.db;
let table_names = write_lp(&db, data1); let table_names = write_lp(&db, data1);
for table_name in &table_names { for table_name in &table_names {
db.rollover_partition(partition_key, &table_name) db.rollover_partition(partition_key, &table_name)

24
server/src/query_tests/utils.rs
View File

@ -7,7 +7,11 @@ use data_types::{
use object_store::{memory::InMemory, ObjectStore}; use object_store::{memory::InMemory, ObjectStore};
use query::{exec::Executor, Database}; use query::{exec::Executor, Database};
use crate::{buffer::Buffer, db::Db, JobRegistry}; use crate::{
buffer::Buffer,
db::{load_preserved_catalog, Db},
JobRegistry,
};
use std::{borrow::Cow, convert::TryFrom, sync::Arc}; use std::{borrow::Cow, convert::TryFrom, sync::Arc};
// A wrapper around a Db and a metrics registry allowing for isolated testing // A wrapper around a Db and a metrics registry allowing for isolated testing
@ -37,7 +41,7 @@ impl TestDbBuilder {
Self::default() Self::default()
} }
pub fn build(self) -> TestDb { pub async fn build(self) -> TestDb {
let server_id = self let server_id = self
.server_id .server_id
.unwrap_or_else(|| ServerId::try_from(1).unwrap()); .unwrap_or_else(|| ServerId::try_from(1).unwrap());
@ -64,9 +68,17 @@ impl TestDbBuilder {
} else { } else {
None None
}; };
let preserved_catalog = load_preserved_catalog(
db_name.as_str(),
Arc::clone(&object_store),
server_id,
Arc::clone(&metrics_registry),
)
.await
.unwrap();
TestDb { TestDb {
metric_registry: metrics::TestMetricRegistry::new(Arc::clone(&metrics_registry)), metric_registry: metrics::TestMetricRegistry::new(metrics_registry),
db: Db::new( db: Db::new(
DatabaseRules::new(db_name), DatabaseRules::new(db_name),
server_id, server_id,
@ -74,7 +86,7 @@ impl TestDbBuilder {
exec, exec,
write_buffer, write_buffer,
Arc::new(JobRegistry::new()), Arc::new(JobRegistry::new()),
metrics_registry, preserved_catalog,
), ),
} }
} }
@ -101,8 +113,8 @@ impl TestDbBuilder {
} }
/// Used for testing: create a Database with a local store /// Used for testing: create a Database with a local store
pub fn make_db() -> TestDb { pub async fn make_db() -> TestDb {
TestDb::builder().build() TestDb::builder().build().await
} }
fn chunk_summary_iter(db: &Db) -> impl Iterator<Item = ChunkSummary> + '_ { fn chunk_summary_iter(db: &Db) -> impl Iterator<Item = ChunkSummary> + '_ {

1
server/src/snapshot.rs
View File

@ -279,6 +279,7 @@ cpu,host=B,region=east user=10.0,system=74.1 1
let db = TestDb::builder() let db = TestDb::builder()
.object_store(Arc::new(ObjectStore::new_in_memory(InMemory::new()))) .object_store(Arc::new(ObjectStore::new_in_memory(InMemory::new())))
.build() .build()
.await
.db; .db;
write_lp(&db, &lp); write_lp(&db, &lp);

119
src/influxdb_ioxd/http.rs
View File

@ -26,21 +26,24 @@ use server::{ConnectionManager, Server as AppServer};
use bytes::{Bytes, BytesMut}; use bytes::{Bytes, BytesMut};
use futures::{self, StreamExt}; use futures::{self, StreamExt};
use http::header::{CONTENT_ENCODING, CONTENT_TYPE}; use http::header::{CONTENT_ENCODING, CONTENT_TYPE};
use hyper::{Body, Method, Request, Response, StatusCode}; use hyper::{http::HeaderValue, Body, Method, Request, Response, StatusCode};
use observability_deps::{ use observability_deps::{
opentelemetry::KeyValue, opentelemetry::KeyValue,
tracing::{self, debug, error}, tracing::{self, debug, error, info},
}; };
use routerify::{prelude::*, Middleware, RequestInfo, Router, RouterError, RouterService}; use routerify::{prelude::*, Middleware, RequestInfo, Router, RouterError, RouterService};
use serde::Deserialize; use serde::Deserialize;
use snafu::{OptionExt, ResultExt, Snafu}; use snafu::{OptionExt, ResultExt, Snafu};
use hyper::server::conn::AddrIncoming; use hyper::server::conn::AddrIncoming;
use pprof::protos::Message;
use std::num::NonZeroI32;
use std::{ use std::{
fmt::Debug, fmt::Debug,
str::{self, FromStr}, str::{self, FromStr},
sync::Arc, sync::Arc,
}; };
use tokio::time::Duration;
use tokio_util::sync::CancellationToken; use tokio_util::sync::CancellationToken;
/// Constants used in API error codes. /// Constants used in API error codes.
@ -217,6 +220,15 @@ pub enum ApplicationError {
partition: String, partition: String,
table_name: String, table_name: String,
}, },
#[snafu(display("PProf error: {}", source))]
PProf { source: pprof::Error },
#[snafu(display("Protobuf error: {}", source))]
Prost { source: prost::EncodeError },
#[snafu(display("Empty flamegraph"))]
EmptyFlamegraph,
} }
impl ApplicationError { impl ApplicationError {
@ -251,6 +263,9 @@ impl ApplicationError {
Self::ParsingFormat { .. } => self.bad_request(), Self::ParsingFormat { .. } => self.bad_request(),
Self::Planning { .. } => self.bad_request(), Self::Planning { .. } => self.bad_request(),
Self::NoSnapshot { .. } => self.not_modified(), Self::NoSnapshot { .. } => self.not_modified(),
Self::PProf { .. } => self.internal_error(),
Self::Prost { .. } => self.internal_error(),
Self::EmptyFlamegraph => self.no_content(),
} }
} }
@ -282,6 +297,13 @@ impl ApplicationError {
.unwrap() .unwrap()
} }
fn no_content(&self) -> Response<Body> {
Response::builder()
.status(StatusCode::NO_CONTENT)
.body(self.body())
.unwrap()
}
fn body(&self) -> Body { fn body(&self) -> Body {
let json = let json =
serde_json::json!({"error": self.to_string(), "error_code": self.api_error_code()}) serde_json::json!({"error": self.to_string(), "error_code": self.api_error_code()})
@ -340,6 +362,8 @@ where
.get("/iox/api/v1/databases/:name/query", query::<M>) .get("/iox/api/v1/databases/:name/query", query::<M>)
.get("/api/v1/partitions", list_partitions::<M>) .get("/api/v1/partitions", list_partitions::<M>)
.post("/api/v1/snapshot", snapshot_partition::<M>) .post("/api/v1/snapshot", snapshot_partition::<M>)
.get("/debug/pprof", pprof_home::<M>)
.get("/debug/pprof/profile", pprof_profile::<M>)
// Specify the error handler to handle any errors caused by // Specify the error handler to handle any errors caused by
// a route or any middleware. // a route or any middleware.
.err_handler_with_info(error_handler) .err_handler_with_info(error_handler)
@ -785,6 +809,97 @@ async fn snapshot_partition<M: ConnectionManager + Send + Sync + Debug + 'static
} }
} }
#[tracing::instrument(level = "debug")]
async fn pprof_home<M: ConnectionManager + Send + Sync + Debug + 'static>(
req: Request<Body>,
) -> Result<Response<Body>, ApplicationError> {
let default_host = HeaderValue::from_static("localhost");
let host = req
.headers()
.get("host")
.unwrap_or(&default_host)
.to_str()
.unwrap_or_default();
let cmd = format!(
"/debug/pprof/profile?seconds={}",
PProfArgs::default_seconds()
);
Ok(Response::new(Body::from(format!(
r#"<a href="{}">http://{}{}</a>"#,
cmd, host, cmd
))))
}
async fn dump_rsprof(seconds: u64, frequency: i32) -> pprof::Result<pprof::Report> {
let guard = pprof::ProfilerGuard::new(frequency)?;
info!(
"start profiling {} seconds with frequency {} /s",
seconds, frequency
);
tokio::time::sleep(Duration::from_secs(seconds)).await;
info!(
"done profiling {} seconds with frequency {} /s",
seconds, frequency
);
guard.report().build()
}
#[derive(Debug, Deserialize)]
struct PProfArgs {
#[serde(default = "PProfArgs::default_seconds")]
seconds: u64,
#[serde(default = "PProfArgs::default_frequency")]
frequency: NonZeroI32,
}
impl PProfArgs {
fn default_seconds() -> u64 {
30
}
// 99Hz to avoid coinciding with special periods
fn default_frequency() -> NonZeroI32 {
NonZeroI32::new(99).unwrap()
}
}
#[tracing::instrument(level = "debug")]
async fn pprof_profile<M: ConnectionManager + Send + Sync + Debug + 'static>(
req: Request<Body>,
) -> Result<Response<Body>, ApplicationError> {
let query_string = req.uri().query().unwrap_or_default();
let query: PProfArgs =
serde_urlencoded::from_str(query_string).context(InvalidQueryString { query_string })?;
let report = dump_rsprof(query.seconds, query.frequency.get())
.await
.context(PProf)?;
let mut body: Vec<u8> = Vec::new();
// render flamegraph when opening in the browser
// otherwise render as protobuf; works great with: go tool pprof http://..../debug/pprof/profile
if req
.headers()
.get_all("Accept")
.iter()
.flat_map(|i| i.to_str().unwrap_or_default().split(','))
.any(|i| i == "text/html" || i == "image/svg+xml")
{
report.flamegraph(&mut body).context(PProf)?;
if body.is_empty() {
return EmptyFlamegraph.fail();
}
} else {
let profile = report.pprof().context(PProf)?;
profile.encode(&mut body).context(Prost)?;
}
Ok(Response::new(Body::from(body)))
}
pub async fn serve<M>( pub async fn serve<M>(
addr: AddrIncoming, addr: AddrIncoming,
server: Arc<AppServer<M>>, server: Arc<AppServer<M>>,