diff --git a/.cargo/config b/.cargo/config new file mode 100644 index 0000000000..9a552b1589 --- /dev/null +++ b/.cargo/config @@ -0,0 +1,4 @@ +[target.x86_64-unknown-linux-gnu] +rustflags = [ + "-C", "link-arg=-fuse-ld=lld", +] \ No newline at end of file diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..05de81c3b6 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,3 @@ +generated_types/protos/google/ linguist-generated=true +generated_types/protos/grpc/ linguist-generated=true +generated_types/src/wal_generated.rs linguist-generated=true diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index b882ff9cc0..5ec1f00efb 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -19,6 +19,11 @@ on: [pull_request] name: ci +env: + # Disable full debug symbol generation to speed up CI build + # "1" means line tables only, which is useful for panic tracebacks. + RUSTFLAGS: "-C debuginfo=1" + jobs: build: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index daf59984e6..4ee1c6732b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -186,3 +186,20 @@ cargo clippy --all-targets --workspace -- -D warnings [`rustfmt`]: https://github.com/rust-lang/rustfmt [`clippy`]: https://github.com/rust-lang/rust-clippy + +## Upgrading the `flatbuffers` crate + +IOx uses Flatbuffers for its write-ahead log. The structure is defined in +[`generated_types/protos/wal.fbs`]. We have then used the `flatc` Flatbuffers compiler to generate +the corresponding Rust code in [`generated_types/src/wal_generated.rs`], which is checked in to the +repository. + +The checked-in code is compatible with the `flatbuffers` crate version in the `Cargo.lock` file. If +upgrading the version of the `flatbuffers` crate that IOx depends on, the generated code will need +to be updated as well. + +Instructions for updating the generated code are in [`docs/regenerating_flatbuffers.md`]. + +[`generated_types/protos/wal.fbs`]: generated_types/protos/wal.fbs +[`generated_types/src/wal_generated.rs`]: generated_types/src/wal_generated.rs +[`docs/regenerating_flatbuffers.md`]: docs/regenerating_flatbuffers.md diff --git a/Cargo.lock b/Cargo.lock index a2903ca1ab..521c5b1d71 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,5 +1,15 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +[[package]] +name = "Inflector" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" +dependencies = [ + "lazy_static", + "regex", +] + [[package]] name = "RustyXML" version = "0.3.0" @@ -29,9 +39,9 @@ checksum = "739f4a8db6605981345c5654f3a85b056ce52f37a39d34da03f25bf2151ea16e" [[package]] name = "ahash" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dd69f6bbb6851e9bba54499698ed6b50640e35544e6b552e7604ad6258778b9" +checksum = "7f200cbb1e856866d9eade941cf3aa0c5d7dd36f74311c4273b494f4ef036957" dependencies = [ "getrandom 0.2.2", "once_cell", @@ -101,12 +111,12 @@ checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" [[package]] name = "arrow" version = "4.0.0-SNAPSHOT" -source = "git+https://github.com/apache/arrow.git?rev=4f6adc700d1cebc50a0594b5aa671f64491cc20e#4f6adc700d1cebc50a0594b5aa671f64491cc20e" +source = "git+https://github.com/apache/arrow.git?rev=6208a79739d0228ecc566fa8436ee61068452212#6208a79739d0228ecc566fa8436ee61068452212" dependencies = [ "cfg_aliases", "chrono", "csv", - "flatbuffers 0.8.3", + "flatbuffers", "hex", "indexmap", "lazy_static", @@ -124,7 +134,7 @@ dependencies = [ [[package]] name = "arrow-flight" version = "4.0.0-SNAPSHOT" -source = "git+https://github.com/apache/arrow.git?rev=4f6adc700d1cebc50a0594b5aa671f64491cc20e#4f6adc700d1cebc50a0594b5aa671f64491cc20e" +source = "git+https://github.com/apache/arrow.git?rev=6208a79739d0228ecc566fa8436ee61068452212#6208a79739d0228ecc566fa8436ee61068452212" dependencies = [ "arrow", "bytes", @@ -195,9 +205,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.42" +version = "0.1.48" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d3a45e77e34375a7923b1e8febb049bb011f064714a8e17a1a616fef01da13d" +checksum = "36ea56748e10732c49404c153638a15ec3d6211ec5ff35d9bb20e13b93576adf" dependencies = [ "proc-macro2", "quote", @@ -277,7 +287,7 @@ dependencies = [ "serde_json", "smallvec", "thiserror", - "time 0.2.25", + "time 0.2.26", "url", "uuid", ] @@ -324,7 +334,7 @@ dependencies = [ "cexpr", "clang-sys", "clap", - "env_logger 0.8.3", + "env_logger", "lazy_static", "lazycell", "log", @@ -404,9 +414,9 @@ checksum = "63396b8a4b9de3f4fdfb320ab6080762242f66a8ef174c49d8e19b674db4cdbe" [[package]] name = "byteorder" -version = "1.4.2" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae44d1a3d5a19df61dd0c8beb138458ac2a53a7ac09eba97d55592540004306b" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "bytes" @@ -423,6 +433,13 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "catalog" +version = "0.1.0" +dependencies = [ + "snafu", +] + [[package]] name = "cc" version = "1.0.67" @@ -501,9 +518,9 @@ dependencies = [ [[package]] name = "cloud-storage" -version = "0.8.0" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bce9e4bdb24400b2e2403aecceb676c355c0309e3476a79c80c34858fddd611f" +checksum = "a27c92803d7c48c97d828f468d0bb3069f21a2531ccc8361486a7e05ba9518ec" dependencies = [ "base64 0.13.0", "bytes", @@ -635,26 +652,16 @@ dependencies = [ [[package]] name = "crossbeam" -version = "0.7.3" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69323bff1fb41c635347b8ead484a5ca6c3f11914d784170b158d8449ab07f8e" +checksum = "fd01a6eb3daaafa260f6fc94c3a6c36390abc2080e38e3e34ced87393fb77d80" dependencies = [ - "cfg-if 0.1.10", - "crossbeam-channel 0.4.4", - "crossbeam-deque 0.7.3", - "crossbeam-epoch 0.8.2", + "cfg-if 1.0.0", + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-epoch", "crossbeam-queue", - "crossbeam-utils 0.7.2", -] - -[[package]] -name = "crossbeam-channel" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b153fe7cbef478c567df0f972e02e6d736db11affe43dfc9c56a9374d1adfb87" -dependencies = [ - "crossbeam-utils 0.7.2", - "maybe-uninit", + "crossbeam-utils", ] [[package]] @@ -664,18 +671,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775" dependencies = [ "cfg-if 1.0.0", - "crossbeam-utils 0.8.3", -] - -[[package]] -name = "crossbeam-deque" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f02af974daeee82218205558e51ec8768b48cf524bd01d550abe5573a608285" -dependencies = [ - "crossbeam-epoch 0.8.2", - "crossbeam-utils 0.7.2", - "maybe-uninit", + "crossbeam-utils", ] [[package]] @@ -685,23 +681,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9" dependencies = [ "cfg-if 1.0.0", - "crossbeam-epoch 0.9.3", - "crossbeam-utils 0.8.3", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" -dependencies = [ - "autocfg", - "cfg-if 0.1.10", - "crossbeam-utils 0.7.2", - "lazy_static", - "maybe-uninit", - "memoffset 0.5.6", - "scopeguard", + "crossbeam-epoch", + "crossbeam-utils", ] [[package]] @@ -711,32 +692,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2584f639eb95fea8c798496315b297cf81b9b58b6d30ab066a75455333cf4b12" dependencies = [ "cfg-if 1.0.0", - "crossbeam-utils 0.8.3", + "crossbeam-utils", "lazy_static", - "memoffset 0.6.1", + "memoffset", "scopeguard", ] [[package]] name = "crossbeam-queue" -version = "0.2.3" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570" +checksum = "0f6cb3c7f5b8e51bc3ebb73a2327ad4abdbd119dc13223f14f961d2f38486756" dependencies = [ - "cfg-if 0.1.10", - "crossbeam-utils 0.7.2", - "maybe-uninit", -] - -[[package]] -name = "crossbeam-utils" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" -dependencies = [ - "autocfg", - "cfg-if 0.1.10", - "lazy_static", + "cfg-if 1.0.0", + "crossbeam-utils", ] [[package]] @@ -762,9 +731,9 @@ dependencies = [ [[package]] name = "csv" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9d58633299b24b515ac72a3f869f8b91306a3cec616a602843a383acd6f9e97" +checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1" dependencies = [ "bstr", "csv-core", @@ -795,26 +764,26 @@ dependencies = [ name = "data_types" version = "0.1.0" dependencies = [ - "arrow_deps", "chrono", - "crc32fast", - "criterion", - "flatbuffers 0.6.1", "generated_types", "influxdb_line_protocol", "percent-encoding", + "prost", + "regex", "serde", + "serde_regex", "snafu", "test_helpers", + "tonic", "tracing", ] [[package]] name = "datafusion" version = "4.0.0-SNAPSHOT" -source = "git+https://github.com/apache/arrow.git?rev=4f6adc700d1cebc50a0594b5aa671f64491cc20e#4f6adc700d1cebc50a0594b5aa671f64491cc20e" +source = "git+https://github.com/apache/arrow.git?rev=6208a79739d0228ecc566fa8436ee61068452212#6208a79739d0228ecc566fa8436ee61068452212" dependencies = [ - "ahash 0.7.1", + "ahash 0.7.2", "arrow", "async-trait", "chrono", @@ -827,10 +796,9 @@ dependencies = [ "parquet", "paste", "pin-project-lite", - "sqlparser 0.8.0", + "sqlparser", "tokio", "tokio-stream", - "unicode-segmentation", ] [[package]] @@ -945,19 +913,6 @@ dependencies = [ "cfg-if 1.0.0", ] -[[package]] -name = "env_logger" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44533bbbb3bb3c1fa17d9f2e4e38bbbaf8396ba82193c4cb1b6445d711445d36" -dependencies = [ - "atty", - "humantime 1.3.0", - "log", - "regex", - "termcolor", -] - [[package]] name = "env_logger" version = "0.8.3" @@ -965,7 +920,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17392a012ea30ef05a610aa97dfb49496e71c9f676b27879922ea5bdf60d9d3f" dependencies = [ "atty", - "humantime 2.1.0", + "humantime", "log", "regex", "termcolor", @@ -1011,15 +966,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d" -[[package]] -name = "flatbuffers" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a788f068dd10687940565bf4b5480ee943176cbd114b12e811074bcf7c04e4b9" -dependencies = [ - "smallvec", -] - [[package]] name = "flatbuffers" version = "0.8.3" @@ -1195,7 +1141,7 @@ dependencies = [ name = "generated_types" version = "0.1.0" dependencies = [ - "flatbuffers 0.6.1", + "flatbuffers", "futures", "google_types", "prost", @@ -1203,6 +1149,7 @@ dependencies = [ "prost-types", "tonic", "tonic-build", + "tracing", ] [[package]] @@ -1255,9 +1202,6 @@ version = "0.1.0" dependencies = [ "prost", "prost-build", - "prost-types", - "tonic", - "tracing", ] [[package]] @@ -1314,9 +1258,9 @@ dependencies = [ [[package]] name = "hex" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "644f9158b2f133fd50f5fb3242878846d9eb792e445c893805ff0e3824006e35" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hmac" @@ -1367,15 +1311,6 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86cce260d758a9aa3d7c4b99d55c815a540f8a37514ba6046ab6be402a157cb0" -[[package]] -name = "humantime" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df004cfca50ef23c36850aaaa59ad52cc70d0e90243c3c7737a4dd32dc7a3c4f" -dependencies = [ - "quick-error", -] - [[package]] name = "humantime" version = "2.1.0" @@ -1449,9 +1384,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.6.1" +version = "1.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb1fa934250de4de8aef298d81c729a7d33d8c239daa3a7575e6b92bfc7313b" +checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3" dependencies = [ "autocfg", "hashbrown", @@ -1486,7 +1421,7 @@ dependencies = [ "data_types", "dirs 3.0.1", "dotenv", - "env_logger 0.7.1", + "env_logger", "flate2", "futures", "generated_types", @@ -1498,18 +1433,22 @@ dependencies = [ "influxdb_line_protocol", "influxdb_tsm", "ingest", + "internal_types", "logfmt", "mem_qe", "mutable_buffer", "object_store", + "once_cell", "opentelemetry", "opentelemetry-jaeger", "packers", "panic_logging", + "parking_lot", "predicates", + "prettytable-rs", "prost", "query", - "rand 0.7.3", + "rand 0.8.3", "read_buffer", "reqwest", "routerify", @@ -1524,6 +1463,7 @@ dependencies = [ "thiserror", "tokio", "tokio-stream", + "tokio-util", "tonic", "tonic-health", "tracing", @@ -1569,7 +1509,7 @@ dependencies = [ "flate2", "hex", "integer-encoding", - "rand 0.7.3", + "rand 0.8.3", "snafu", "snap", "test_helpers", @@ -1581,10 +1521,10 @@ name = "ingest" version = "0.1.0" dependencies = [ "arrow_deps", - "data_types", "flate2", "influxdb_line_protocol", "influxdb_tsm", + "internal_types", "packers", "parking_lot", "snafu", @@ -1607,6 +1547,23 @@ version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48dc51180a9b377fd75814d0cc02199c20f8e99433d6762f650d39cdbbd3b56f" +[[package]] +name = "internal_types" +version = "0.1.0" +dependencies = [ + "arrow_deps", + "chrono", + "crc32fast", + "criterion", + "data_types", + "flatbuffers", + "generated_types", + "influxdb_line_protocol", + "ouroboros", + "snafu", + "tracing", +] + [[package]] name = "ipnet" version = "2.3.0" @@ -1696,9 +1653,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.86" +version = "0.2.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7282d924be3275cec7f6756ff4121987bc6481325397dde6ba3e7802b1a8b1c" +checksum = "538c092e5586f4cdd7dd8078c4a79220e3e168880218124dcbce860f0ea938c6" [[package]] name = "libloading" @@ -1786,12 +1743,6 @@ version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" -[[package]] -name = "maybe-uninit" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" - [[package]] name = "md5" version = "0.7.0" @@ -1807,7 +1758,7 @@ dependencies = [ "criterion", "croaring", "crossbeam", - "env_logger 0.7.1", + "env_logger", "human_format", "packers", "snafu", @@ -1820,15 +1771,6 @@ version = "2.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" -[[package]] -name = "memoffset" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" -dependencies = [ - "autocfg", -] - [[package]] name = "memoffset" version = "0.6.1" @@ -1856,9 +1798,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.7.9" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5dede4e2065b3842b8b0af444119f3aa331cc7cc2dd20388bfb0f5d5a38823a" +checksum = "2182a122f3b7f3f5329cb1972cee089ba2459a0a80a56935e6e674f096f8d839" dependencies = [ "libc", "log", @@ -1897,9 +1839,9 @@ dependencies = [ [[package]] name = "multimap" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1255076139a83bb467426e7f8d0134968a8118844faa755985e077cf31850333" +checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" [[package]] name = "mutable_buffer" @@ -1910,9 +1852,10 @@ dependencies = [ "chrono", "criterion", "data_types", - "flatbuffers 0.6.1", + "flatbuffers", "generated_types", "influxdb_line_protocol", + "internal_types", "snafu", "string-interner", "test_helpers", @@ -1970,7 +1913,7 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b7a8e9be5e039e2ff869df49155f1c06bd01ade2117ec783e56ab0932b67a8f" dependencies = [ - "num-bigint 0.3.1", + "num-bigint 0.3.2", "num-complex", "num-integer", "num-iter", @@ -1991,9 +1934,9 @@ dependencies = [ [[package]] name = "num-bigint" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e9a41747ae4633fce5adffb4d2e81ffc5e89593cb19917f8fb2cc5ff76507bf" +checksum = "7d0a3d5e207573f948a9e5376662aa743a2ea13f7c50a554d7af443a73fbfeba" dependencies = [ "autocfg", "num-integer", @@ -2037,7 +1980,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12ac428b1cb17fce6f731001d307d351ec70a6d202fc2e60f7d4c5e42d8f4f07" dependencies = [ "autocfg", - "num-bigint 0.3.1", + "num-bigint 0.3.2", "num-integer", "num-traits", ] @@ -2114,9 +2057,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.7.0" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10acf907b94fc1b1a152d08ef97e7759650268cf986bf127f387e602b02c7e5a" +checksum = "af8b08b04175473088b46763e51ee54da5f9a164bc162f615b91bc179dbf15a3" dependencies = [ "parking_lot", ] @@ -2135,15 +2078,15 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "openssl" -version = "0.10.32" +version = "0.10.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "038d43985d1ddca7a9900630d8cd031b56e4794eecc2e9ea39dd17aa04399a70" +checksum = "a61075b62a23fef5a29815de7536d940aa35ce96d18ce0cc5076272db678a577" dependencies = [ "bitflags", "cfg-if 1.0.0", "foreign-types", - "lazy_static", "libc", + "once_cell", "openssl-sys", ] @@ -2155,9 +2098,9 @@ checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" [[package]] name = "openssl-sys" -version = "0.9.60" +version = "0.9.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "921fc71883267538946025deffb622905ecad223c28efbfdef9bb59a0175f3e6" +checksum = "313752393519e876837e09e1fa183ddef0be7735868dced3196f4472d536277f" dependencies = [ "autocfg", "cc", @@ -2216,6 +2159,29 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ouroboros" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f6d5c203fe8d786d9d7bec8203cbbff3eb2cf8410c0d70cfd05b3d5f5d545da" +dependencies = [ + "ouroboros_macro", + "stable_deref_trait", +] + +[[package]] +name = "ouroboros_macro" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "129943a960e6a08c7e70ca5a09f113c273fe7f10ae8420992c78293e3dffdf65" +dependencies = [ + "Inflector", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "packed_simd_2" version = "0.3.4" @@ -2231,10 +2197,10 @@ name = "packers" version = "0.1.0" dependencies = [ "arrow_deps", - "data_types", "human_format", "influxdb_tsm", - "rand 0.7.3", + "internal_types", + "rand 0.8.3", "snafu", "test_helpers", "tracing", @@ -2275,7 +2241,7 @@ dependencies = [ [[package]] name = "parquet" version = "4.0.0-SNAPSHOT" -source = "git+https://github.com/apache/arrow.git?rev=4f6adc700d1cebc50a0594b5aa671f64491cc20e#4f6adc700d1cebc50a0594b5aa671f64491cc20e" +source = "git+https://github.com/apache/arrow.git?rev=6208a79739d0228ecc566fa8436ee61068452212#6208a79739d0228ecc566fa8436ee61068452212" dependencies = [ "arrow", "base64 0.12.3", @@ -2284,7 +2250,7 @@ dependencies = [ "chrono", "flate2", "lz4", - "num-bigint 0.3.1", + "num-bigint 0.3.2", "parquet-format", "snap", "thrift", @@ -2302,9 +2268,9 @@ dependencies = [ [[package]] name = "paste" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5d65c4d95931acda4498f675e332fcbdc9a06705cd07086c510e9b6009cd1c1" +checksum = "acbf547ad0c65e31259204bd90935776d1c693cec2f4ff7abb7a1bbbd40dfe58" [[package]] name = "peeking_take_while" @@ -2387,9 +2353,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.4" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439697af366c49a6d0a010c56a0d97685bc140ce0d377b13a2ea2aa42d64a827" +checksum = "dc0e1f259c92177c30a4c9d177246edd0a3568b25756a977d0632cf8fa37e905" [[package]] name = "pin-utils" @@ -2587,9 +2553,10 @@ dependencies = [ "data_types", "futures", "influxdb_line_protocol", + "internal_types", "parking_lot", "snafu", - "sqlparser 0.6.1", + "sqlparser", "test_helpers", "tokio", "tokio-stream", @@ -2676,12 +2643,12 @@ dependencies = [ [[package]] name = "rand_distr" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9e9532ada3929fb8b2e9dbe28d1e06c9b2cc65813f074fcb6bd5fbefeff9d56" +checksum = "da9e8f32ad24fb80d07d2323a9a2ce8b30d68a62b8cb4df88119ff49a698f038" dependencies = [ "num-traits", - "rand 0.7.3", + "rand 0.8.3", ] [[package]] @@ -2709,7 +2676,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674" dependencies = [ "autocfg", - "crossbeam-deque 0.8.0", + "crossbeam-deque", "either", "rayon-core", ] @@ -2720,9 +2687,9 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a" dependencies = [ - "crossbeam-channel 0.5.0", - "crossbeam-deque 0.8.0", - "crossbeam-utils 0.8.3", + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils", "lazy_static", "num_cpus", ] @@ -2734,13 +2701,13 @@ dependencies = [ "arrow_deps", "criterion", "croaring", - "data_types", "either", "hashbrown", + "internal_types", "itertools 0.9.0", "packers", "permutation", - "rand 0.7.3", + "rand 0.8.3", "rand_distr", "snafu", ] @@ -2783,14 +2750,13 @@ dependencies = [ [[package]] name = "regex" -version = "1.4.3" +version = "1.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9251239e129e16308e70d853559389de218ac275b515068abc96829d05b948a" +checksum = "957056ecddbeba1b26965114e191d2e8589ce74db242b6ea25fc4062427a5c19" dependencies = [ "aho-corasick", "memchr", "regex-syntax", - "thread_local", ] [[package]] @@ -2805,9 +2771,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.22" +version = "0.6.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5eb417147ba9860a96cfe72a0b93bf88fee1744b5636ec99ab20c1aa9376581" +checksum = "24d5f089152e60f62d28b835fbff2cd2e8dc0baf1ac13343bef92ab7eed84548" [[package]] name = "remove_dir_all" @@ -2820,9 +2786,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0460542b551950620a3648c6aa23318ac6b3cd779114bd873209e6e8b5eb1c34" +checksum = "bf12057f289428dbf5c591c74bf10392e4a8003f993405a902f20117019022d4" dependencies = [ "base64 0.13.0", "bytes", @@ -2962,7 +2928,7 @@ dependencies = [ "rustc_version", "serde", "sha2", - "time 0.2.25", + "time 0.2.26", "tokio", ] @@ -2975,7 +2941,7 @@ dependencies = [ "base64 0.13.0", "blake2b_simd", "constant_time_eq", - "crossbeam-utils 0.8.3", + "crossbeam-utils", ] [[package]] @@ -3067,9 +3033,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.1.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dfd318104249865096c8da1dfabf09ddbb6d0330ea176812a62ec75e40c4166" +checksum = "d493c5f39e02dfb062cd8f33301f90f9b13b650e8c1b1d0fd75c19dd64bff69d" dependencies = [ "bitflags", "core-foundation", @@ -3105,9 +3071,9 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" [[package]] name = "serde" -version = "1.0.123" +version = "1.0.124" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92d5161132722baa40d802cc70b15262b98258453e85e5d1d365c757c73869ae" +checksum = "bd761ff957cb2a45fbb9ab3da6512de9de55872866160b23c25f1a841e99d29f" dependencies = [ "serde_derive", ] @@ -3136,9 +3102,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.123" +version = "1.0.124" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9391c295d64fc0abb2c556bad848f33cb8296276b1ad2677d1ae1ace4f258f31" +checksum = "1800f7693e94e186f5e25a28291ae1570da908aff7d97a095dec1e56ff99069b" dependencies = [ "proc-macro2", "quote", @@ -3147,9 +3113,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.63" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43535db9747a4ba938c0ce0a98cc631a46ebf943c9e1d604e091df6007620bf6" +checksum = "799e97dc9fdae36a5c8b8f2cae9ce2ee9fdce2058c57a93e6099d919fd982f79" dependencies = [ "indexmap", "itoa", @@ -3157,6 +3123,16 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_regex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8136f1a4ea815d7eac4101cfd0b16dc0cb5e1fe1b8609dfd728058656b7badf" +dependencies = [ + "regex", + "serde", +] + [[package]] name = "serde_urlencoded" version = "0.6.1" @@ -3191,10 +3167,12 @@ dependencies = [ "chrono", "crc32fast", "data_types", - "flatbuffers 0.6.1", + "flatbuffers", "futures", "generated_types", + "hashbrown", "influxdb_line_protocol", + "internal_types", "mutable_buffer", "object_store", "parking_lot", @@ -3207,6 +3185,7 @@ dependencies = [ "snap", "test_helpers", "tokio", + "tokio-util", "tracing", "uuid", ] @@ -3326,15 +3305,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" -[[package]] -name = "sqlparser" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53fa7478852b3ea28f0d21a42b2d7dade24ba4aa72e22bf66982e4b587a7f608" -dependencies = [ - "log", -] - [[package]] name = "sqlparser" version = "0.8.0" @@ -3344,6 +3314,12 @@ dependencies = [ "log", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "standback" version = "0.2.15" @@ -3457,9 +3433,9 @@ checksum = "1e81da0851ada1f3e9d4312c704aa4f8806f0f9d69faaf8df2f3464b4a9437c2" [[package]] name = "syn" -version = "1.0.60" +version = "1.0.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c700597eca8a5a762beb35753ef6b94df201c81cca676604f547495a0d7f0081" +checksum = "3fd9d1e9976102a03c542daa2eff1b43f9d72306342f3f8b3ed5fb8908195d6f" dependencies = [ "proc-macro2", "quote", @@ -3595,9 +3571,9 @@ dependencies = [ [[package]] name = "time" -version = "0.2.25" +version = "0.2.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1195b046942c221454c2539395f85413b33383a067449d78aab2b7b052a142f7" +checksum = "08a8cbfbf47955132d0202d1662f49b2423ae35862aee471f3ba4b133358f372" dependencies = [ "const_fn", "libc", @@ -3633,9 +3609,9 @@ dependencies = [ [[package]] name = "tinytemplate" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2ada8616fad06a2d0c455adc530de4ef57605a8120cc65da9653e0e9623ca74" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" dependencies = [ "serde", "serde_json", @@ -3658,9 +3634,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8190d04c665ea9e6b6a0dc45523ade572c088d2e6566244c1122671dbf4ae3a" +checksum = "8d56477f6ed99e10225f38f9f75f872f29b8b8bd8c0b946f63345bb144e9eeda" dependencies = [ "autocfg", "bytes", @@ -3710,9 +3686,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1981ad97df782ab506a1f43bf82c967326960d278acf3bf8279809648c3ff3ea" +checksum = "c535f53c0cfa1acace62995a8994fc9cc1f12d202420da96ff306ee24d576469" dependencies = [ "futures-core", "pin-project-lite", @@ -3721,9 +3697,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebb7cb2f00c5ae8df755b252306272cd1790d39728363936e01827e11f0b017b" +checksum = "ec31e5cc6b46e653cf57762f36f71d5e6386391d88a72fd6db4508f8f676fb29" dependencies = [ "bytes", "futures-core", @@ -3791,9 +3767,9 @@ dependencies = [ [[package]] name = "tower" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "713c629c07a3a97f741c140e474e7304294fabec66a43a33f0832e98315ab07f" +checksum = "f715efe02c0862926eb463e49368d38ddb119383475686178e32e26d15d06a66" dependencies = [ "futures-core", "futures-util", @@ -3836,9 +3812,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.13" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8a9bd1db7706f2373a190b0d067146caa39350c486f3d455b0e33b431f94c07" +checksum = "c42e6fa53307c8a17e4ccd4dc81cf5ec38db9209f59b222210375b54ee40d1e2" dependencies = [ "proc-macro2", "quote", @@ -3900,9 +3876,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ab8966ac3ca27126141f7999361cc97dd6fb4b71da04c02044fa9045d98bb96" +checksum = "705096c6f83bf68ea5d357a6aa01829ddbdac531b357b45abeca842938085baa" dependencies = [ "ansi_term 0.12.1", "chrono", @@ -3935,9 +3911,9 @@ checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" [[package]] name = "typenum" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "373c8a200f9e67a0c95e62a4f52fbf80c23b4381c05a17845531982fa99e6b33" +checksum = "879f6906492a7cd215bfa4cf595b600146ccfac0c79bcbd1f3000162af5e8b06" [[package]] name = "unicode-bidi" @@ -4018,9 +3994,9 @@ checksum = "f1bddf1187be692e79c5ffeab891132dfb0f236ed36a43c7ed39f1165ee20191" [[package]] name = "version_check" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" +checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" [[package]] name = "wait-timeout" @@ -4253,18 +4229,18 @@ checksum = "81a974bcdd357f0dca4d41677db03436324d45a4c9ed2d0b873a5a360ce41c36" [[package]] name = "zstd" -version = "0.6.0+zstd.1.4.8" +version = "0.6.1+zstd.1.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d4e44664feba7f2f1a9f300c1f6157f2d1bfc3c15c6f3cf4beabf3f5abe9c237" +checksum = "5de55e77f798f205d8561b8fe2ef57abfb6e0ff2abe7fd3c089e119cdb5631a3" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "3.0.0+zstd.1.4.8" +version = "3.0.1+zstd.1.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9447afcd795693ad59918c7bbffe42fdd6e467d708f3537e3dc14dc598c573f" +checksum = "1387cabcd938127b30ce78c4bf00b30387dddf704e3f0881dbc4ff62b5566f8c" dependencies = [ "libc", "zstd-sys", @@ -4272,12 +4248,10 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "1.4.19+zstd.1.4.8" +version = "1.4.20+zstd.1.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec24a9273d24437afb8e71b16f3d9a5d569193cccdb7896213b59f552f387674" +checksum = "ebd5b733d7cf2d9447e2c3e76a5589b4f5e5ae065c22a2bc0b023cbc331b6c8e" dependencies = [ "cc", - "glob", - "itertools 0.9.0", "libc", ] diff --git a/Cargo.toml b/Cargo.toml index cb6c61d6ba..fea17d944c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,10 +4,12 @@ version = "0.1.0" authors = ["Paul Dix "] edition = "2018" default-run = "influxdb_iox" +readme = "README.md" [workspace] # In alphabetical order members = [ "arrow_deps", + "catalog", "data_types", "generated_types", "google_types", @@ -16,6 +18,7 @@ members = [ "influxdb_tsm", "influxdb2_client", "ingest", + "internal_types", "logfmt", "mem_qe", "mutable_buffer", @@ -40,9 +43,10 @@ debug = true arrow_deps = { path = "arrow_deps" } data_types = { path = "data_types" } generated_types = { path = "generated_types" } -influxdb_iox_client = { path = "influxdb_iox_client" } +influxdb_iox_client = { path = "influxdb_iox_client", features = ["format"] } influxdb_line_protocol = { path = "influxdb_line_protocol" } influxdb_tsm = { path = "influxdb_tsm" } +internal_types = { path = "internal_types" } ingest = { path = "ingest" } logfmt = { path = "logfmt" } mem_qe = { path = "mem_qe" } @@ -63,13 +67,15 @@ clap = "2.33.1" csv = "1.1" dirs = "3.0.1" dotenv = "0.15.0" -env_logger = "0.7.1" +env_logger = "0.8.3" flate2 = "1.0" futures = "0.3.1" http = "0.2.0" hyper = "0.14" opentelemetry = { version = "0.12", default-features = false, features = ["trace", "tokio-support"] } opentelemetry-jaeger = { version = "0.11", features = ["tokio"] } +# used by arrow/datafusion anyway +prettytable-rs = "0.8" prost = "0.7" # Forked to upgrade hyper and tokio routerify = { git = "https://github.com/influxdata/routerify", rev = "274e250" } @@ -79,8 +85,9 @@ serde_urlencoded = "0.7.0" snafu = "0.6.9" structopt = "0.3.21" thiserror = "1.0.23" -tokio = { version = "1.0", features = ["macros", "rt-multi-thread", "parking_lot"] } +tokio = { version = "1.0", features = ["macros", "rt-multi-thread", "parking_lot", "signal"] } tokio-stream = { version = "0.1.2", features = ["net"] } +tokio-util = { version = "0.6.3" } tonic = "0.4.0" tonic-health = "0.3.0" tracing = { version = "0.1", features = ["release_max_level_debug"] } @@ -93,6 +100,8 @@ tracing-subscriber = { version = "0.2.15", features = ["parking_lot"] } influxdb2_client = { path = "influxdb2_client" } influxdb_iox_client = { path = "influxdb_iox_client", features = ["flight"] } test_helpers = { path = "test_helpers" } +once_cell = { version = "1.4.0", features = ["parking_lot"] } +parking_lot = "0.11.1" # Crates.io dependencies, in alphabetical order assert_cmd = "1.0.0" @@ -100,14 +109,10 @@ criterion = "0.3" flate2 = "1.0" hex = "0.4.2" predicates = "1.0.4" -rand = "0.7.2" +rand = "0.8.3" reqwest = "0.11" tempfile = "3.1.0" -[[bin]] -name = "cpu_feature_check" -path = "src/cpu_feature_check/main.rs" - [[bench]] name = "encoders" harness = false diff --git a/Dockerfile b/Dockerfile index 51079c774e..bcbf98dd29 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,7 +20,7 @@ RUN \ FROM debian:buster-slim RUN apt-get update \ - && apt-get install -y libssl1.1 libgcc1 libc6 --no-install-recommends \ + && apt-get install -y libssl1.1 libgcc1 libc6 ca-certificates --no-install-recommends \ && rm -rf /var/lib/{apt,dpkg,cache,log} RUN groupadd -g 1500 rust \ @@ -36,3 +36,5 @@ COPY --from=build /root/influxdb_iox /usr/bin/influxdb_iox EXPOSE 8080 8082 ENTRYPOINT ["/usr/bin/influxdb_iox"] + +CMD ["run"] diff --git a/README.md b/README.md index 3030b7d14c..0daa900ed7 100644 --- a/README.md +++ b/README.md @@ -28,8 +28,7 @@ We're also hosting monthly tech talks and community office hours on the project ## Quick Start -To compile and run InfluxDB IOx from source, you'll need a Rust compiler and a `flatc` FlatBuffers -compiler. +To compile and run InfluxDB IOx from source, you'll need a Rust compiler and `clang`. ### Build a Docker Image @@ -80,36 +79,6 @@ rustc --version and you should see a nightly version of Rust! -### Installing `flatc` - -InfluxDB IOx uses the [FlatBuffer] serialization format for its write-ahead log. The [`flatc` -compiler] reads the schema in `generated_types/wal.fbs` and generates the corresponding Rust code. - -Install `flatc` >= 1.12.0 with one of these methods as appropriate to your operating system: - -* Using a [Windows binary release] -* Using the [`flatbuffers` package for conda] -* Using the [`flatbuffers` package for Arch Linux] -* Using the [`flatbuffers` package for Homebrew] - -Once you have installed the packages, you should be able to run: - -```shell -flatc --version -``` - -and see the version displayed. - -You won't have to run `flatc` directly; once it's available, Rust's Cargo build tool manages the -compilation process by calling `flatc` for you. - -[FlatBuffer]: https://google.github.io/flatbuffers/ -[`flatc` compiler]: https://google.github.io/flatbuffers/flatbuffers_guide_using_schema_compiler.html -[Windows binary release]: https://github.com/google/flatbuffers/releases -[`flatbuffers` package for conda]: https://anaconda.org/conda-forge/flatbuffers -[`flatbuffers` package for Arch Linux]: https://www.archlinux.org/packages/community/x86_64/flatbuffers/ -[`flatbuffers` package for Homebrew]: https://github.com/Homebrew/homebrew-core/blob/HEAD/Formula/flatbuffers.rb - ### Installing `clang` An installation of `clang` is required to build the [`croaring`] dependency - if @@ -133,17 +102,16 @@ takes its configuration as environment variables. You can see a list of the current configuration values by running `influxdb_iox --help`, as well as the specific subcommand config options such as `influxdb_iox -server --help`. +run --help`. Should you desire specifying config via a file, you can do so using a `.env` formatted file in the working directory. You can use the provided [example](docs/env.example) as a template if you want: -```bash +```shell cp docs/env.example .env ``` - ### Compiling and Starting the Server InfluxDB IOx is built using Cargo, Rust's package manager and build tool. @@ -163,7 +131,7 @@ which will create a binary in `target/debug` that you can run with: You can compile and run with one command by using: ```shell -cargo run +cargo run -- server ``` When compiling for performance testing, build in release mode by using: @@ -175,13 +143,13 @@ cargo build --release which will create the corresponding binary in `target/release`: ```shell -./target/release/influxdb_iox +./target/release/influxdb_iox run ``` Similarly, you can do this in one step with: ```shell -cargo run --release +cargo run --release -- server ``` The server will, by default, start an HTTP API server on port `8080` and a gRPC server on port @@ -190,34 +158,60 @@ The server will, by default, start an HTTP API server on port `8080` and a gRPC ### Writing and Reading Data Each IOx instance requires a writer ID. -This can be set three ways: +This can be set one of 4 ways: - set an environment variable `INFLUXDB_IOX_ID=42` - set a flag `--writer-id 42` -- send an HTTP PUT request: -``` -curl --request PUT \ - --url http://localhost:8080/iox/api/v1/id \ - --header 'Content-Type: application/json' \ - --data '{ - "id": 42 - }' +- use the API (not convered here) +- use the CLI +```shell +influxdb_iox writer set 42 ``` -To write data, you need a destination database. -This is set via HTTP PUT, identifying the database by org `company` and bucket `sensors`: -``` -curl --request PUT \ - --url http://localhost:8080/iox/api/v1/databases/company_sensors \ - --header 'Content-Type: application/json' \ - --data '{ -}' +To write data, you need to create a database. You can do so via the API or using the CLI. For example, to create a database called `company_sensors` with a 100MB mutable buffer, use this command: + +```shell +influxdb_iox database create company_sensors -m 100 ``` -Data can be stored in InfluxDB IOx by sending it in [line protocol] format to the `/api/v2/write` -endpoint. Data is stored by organization and bucket names. Here's an example using [`curl`] with -the organization name `company` and the bucket name `sensors` that will send the data in the -`tests/fixtures/lineproto/metrics.lp` file in this repository, assuming that you're running the -server on the default port: +Data can be stored in InfluxDB IOx by sending it in [line protocol] +format to the `/api/v2/write` endpoint or using the CLI. For example, +here is a command that will send the data in the +`tests/fixtures/lineproto/metrics.lp` file in this repository, +assuming that you're running the server on the default port into +the `company_sensors` database, you can use: + +```shell +influxdb_iox database write company_sensors tests/fixtures/lineproto/metrics.lp +``` + +To query data stored in the `company_sensors` database: + +```shell +influxdb_iox database query company_sensors "SELECT * FROM cpu LIMIT 10" +``` + +### Using the CLI + +To ease deloyment, IOx is packaged as a combined binary which has +commands to start the IOx server as well as a CLI interface for +interacting with and configuring such servers. + +The CLI itself is documented via extensive built in help which you can +access by runing `influxdb_iox --help` + + +### InfluxDB 2.0 compatibility + +InfluxDB IOx allows seamless interoperability with InfluxDB 2.0. + +InfluxDB 2.0 stores data in organization and buckets, but InfluxDB IOx +stores data in named databases. IOx maps `organization` and `bucket` +to a database named with the two parts separated by an underscore +(`_`): `organization_bucket`. + +Here's an example using [`curl`] command to send the same data into +the `company_sensors` database using the InfluxDB 2.0 `/api/v2/write` +API: ```shell curl -v "http://127.0.0.1:8080/api/v2/write?org=company&bucket=sensors" --data-binary @tests/fixtures/lineproto/metrics.lp @@ -226,29 +220,46 @@ curl -v "http://127.0.0.1:8080/api/v2/write?org=company&bucket=sensors" --data-b [line protocol]: https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/ [`curl`]: https://curl.se/ -To query stored data, use the `/api/v2/read` endpoint with a SQL query. This example will return -all data in the `company` organization's `sensors` bucket for the `processes` measurement: - -```shell -curl -v -G -d 'org=company' -d 'bucket=sensors' --data-urlencode 'sql_query=select * from processes' "http://127.0.0.1:8080/api/v2/read" -``` ### Health Checks The HTTP API exposes a healthcheck endpoint at `/health` -```shell +```console $ curl http://127.0.0.1:8080/health OK ``` The gRPC API implements the [gRPC Health Checking Protocol](https://github.com/grpc/grpc/blob/master/doc/health-checking.md). This can be tested with [grpc-health-probe](https://github.com/grpc-ecosystem/grpc-health-probe) -```shell +```console $ grpc_health_probe -addr 127.0.0.1:8082 -service influxdata.platform.storage.Storage status: SERVING ``` +### Manually calling gRPC API + +If you want to manually invoke one of the gRPC APIs, you can use any gRPC CLI client; +a good one is [grpcurl](https://github.com/fullstorydev/grpcurl). + +Tonic (the gRPC server library we're using) currently doesn't have support for gRPC reflection, +hence you must pass all `.proto` files to your client. You can find a conventient `grpcurl` wrapper +that does that in the `scripts` directory: + +```console +$ ./scripts/grpcurl -plaintext 127.0.0.1:8082 list +grpc.health.v1.Health +influxdata.iox.management.v1.ManagementService +influxdata.platform.storage.IOxTesting +influxdata.platform.storage.Storage +$ ./scripts/grpcurl -plaintext 127.0.0.1:8082 influxdata.iox.management.v1.ManagementService.ListDatabases +{ + "names": [ + "foobar_weather" + ] +} +``` + ## Contributing We welcome community contributions from anyone! diff --git a/arrow_deps/Cargo.toml b/arrow_deps/Cargo.toml index 492657c5cb..ed63129754 100644 --- a/arrow_deps/Cargo.toml +++ b/arrow_deps/Cargo.toml @@ -8,14 +8,14 @@ description = "Apache Arrow / Parquet / DataFusion dependencies for InfluxDB IOx [dependencies] # In alphabetical order # We are using development version of arrow/parquet/datafusion and the dependencies are at the same rev -# The version can be found here: https://github.com/apache/arrow/commit/4f6adc700d1cebc50a0594b5aa671f64491cc20e +# The version can be found here: https://github.com/apache/arrow/commit/6208a79739d0228ecc566fa8436ee61068452212 # -arrow = { git = "https://github.com/apache/arrow.git", rev = "4f6adc700d1cebc50a0594b5aa671f64491cc20e" , features = ["simd"] } -arrow-flight = { git = "https://github.com/apache/arrow.git", rev = "4f6adc700d1cebc50a0594b5aa671f64491cc20e" } +arrow = { git = "https://github.com/apache/arrow.git", rev = "6208a79739d0228ecc566fa8436ee61068452212" , features = ["simd"] } +arrow-flight = { git = "https://github.com/apache/arrow.git", rev = "6208a79739d0228ecc566fa8436ee61068452212" } # Turn off optional datafusion features (function packages) -datafusion = { git = "https://github.com/apache/arrow.git", rev = "4f6adc700d1cebc50a0594b5aa671f64491cc20e", default-features = false } +datafusion = { git = "https://github.com/apache/arrow.git", rev = "6208a79739d0228ecc566fa8436ee61068452212", default-features = false } # Turn off the "arrow" feature; it currently has a bug that causes the crate to rebuild every time # and we're not currently using it anyway -parquet = { git = "https://github.com/apache/arrow.git", rev = "4f6adc700d1cebc50a0594b5aa671f64491cc20e", default-features = false, features = ["snap", "brotli", "flate2", "lz4", "zstd"] } +parquet = { git = "https://github.com/apache/arrow.git", rev = "6208a79739d0228ecc566fa8436ee61068452212", default-features = false, features = ["snap", "brotli", "flate2", "lz4", "zstd"] } diff --git a/arrow_deps/src/test_util.rs b/arrow_deps/src/test_util.rs index 11c17507f4..87ffb5fd4a 100644 --- a/arrow_deps/src/test_util.rs +++ b/arrow_deps/src/test_util.rs @@ -44,7 +44,7 @@ pub fn sort_record_batch(batch: RecordBatch) -> RecordBatch { }) .collect(); - let sort_output = lexsort(&sort_input).expect("Sorting to complete"); + let sort_output = lexsort(&sort_input, None).expect("Sorting to complete"); RecordBatch::try_new(batch.schema(), sort_output).unwrap() } diff --git a/benches/encoders.rs b/benches/encoders.rs index 34f2ea1ede..602fdb8ad0 100644 --- a/benches/encoders.rs +++ b/benches/encoders.rs @@ -204,7 +204,7 @@ fn integer_encode_random(c: &mut Criterion) { &LARGER_BATCH_SIZES, |batch_size| { (1..batch_size) - .map(|_| rand::thread_rng().gen_range(0, 100)) + .map(|_| rand::thread_rng().gen_range(0..100)) .collect() }, influxdb_tsm::encoders::integer::encode, @@ -323,7 +323,7 @@ fn integer_decode_random(c: &mut Criterion) { &LARGER_BATCH_SIZES, |batch_size| { let decoded: Vec = (1..batch_size) - .map(|_| rand::thread_rng().gen_range(0, 100)) + .map(|_| rand::thread_rng().gen_range(0..100)) .collect(); let mut encoded = vec![]; influxdb_tsm::encoders::integer::encode(&decoded, &mut encoded).unwrap(); diff --git a/benches/line_protocol_to_parquet.rs b/benches/line_protocol_to_parquet.rs index bc416d6980..93160bf438 100644 --- a/benches/line_protocol_to_parquet.rs +++ b/benches/line_protocol_to_parquet.rs @@ -1,5 +1,4 @@ use criterion::{criterion_group, criterion_main, Criterion, Throughput}; -use data_types::schema::Schema; use influxdb_line_protocol::parse_lines; use ingest::{ parquet::{ @@ -8,6 +7,7 @@ use ingest::{ }, ConversionSettings, LineProtocolConverter, }; +use internal_types::schema::Schema; use packers::{Error as TableError, IOxTableWriter, IOxTableWriterSource}; use std::time::Duration; diff --git a/benches/packers.rs b/benches/packers.rs index b0baadcc82..a34a66958c 100644 --- a/benches/packers.rs +++ b/benches/packers.rs @@ -52,7 +52,7 @@ fn i64_vec_with_nulls(size: usize, null_percent: usize) -> Vec> { let mut a = Vec::with_capacity(size); // insert 10% null values for _ in 0..size { - if rng.gen_range(0, null_percent) == 0 { + if rng.gen_range(0..null_percent) == 0 { a.push(None); } else { a.push(Some(1_u64)); diff --git a/buf.yaml b/buf.yaml index fabedb23ea..e728e5f408 100644 --- a/buf.yaml +++ b/buf.yaml @@ -2,12 +2,13 @@ version: v1beta1 build: roots: - generated_types/protos/ - excludes: - - generated_types/protos/com - - generated_types/protos/influxdata/platform - - generated_types/protos/grpc lint: + ignore: + - google + - grpc + - com/github/influxdata/idpe/storage/read + - influxdata/platform use: - DEFAULT - STYLE_DEFAULT diff --git a/catalog/Cargo.toml b/catalog/Cargo.toml new file mode 100644 index 0000000000..3b5fd318b5 --- /dev/null +++ b/catalog/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "catalog" +version = "0.1.0" +authors = ["Andrew Lamb "] +edition = "2018" +description = "InfluxDB IOx Metadata catalog implementation" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +snafu = "0.6" diff --git a/catalog/src/chunk.rs b/catalog/src/chunk.rs new file mode 100644 index 0000000000..ace990fb16 --- /dev/null +++ b/catalog/src/chunk.rs @@ -0,0 +1,69 @@ +use std::sync::Arc; + +/// The state +#[derive(Debug, PartialEq, Clone, Copy)] +pub enum ChunkState { + /// Chunk can accept new writes + Open, + + /// Chunk can still accept new writes, but will likely be closed soon + Closing, + + /// Chunk is closed for new writes and has become read only + Closed, + + /// Chunk is closed for new writes, and is actively moving to the read + /// buffer + Moving, + + /// Chunk has been completely loaded in the read buffer + Moved, +} + +/// The catalog representation of a Chunk in IOx. Note that a chunk +/// may exist in several physical locations at any given time (e.g. in +/// mutable buffer and in read buffer) +#[derive(Debug, PartialEq)] +pub struct Chunk { + /// What partition does the chunk belong to? + partition_key: Arc, + + /// The ID of the chunk + id: u32, + + /// The state of this chunk + state: ChunkState, + /* TODO: Additional fields + * such as object_store_path, etc */ +} + +impl Chunk { + /// Create a new chunk in the Open state + pub(crate) fn new(partition_key: impl Into, id: u32) -> Self { + let partition_key = Arc::new(partition_key.into()); + + Self { + partition_key, + id, + state: ChunkState::Open, + } + } + + pub fn id(&self) -> u32 { + self.id + } + + pub fn key(&self) -> &str { + self.partition_key.as_ref() + } + + pub fn state(&self) -> ChunkState { + self.state + } + + pub fn set_state(&mut self, state: ChunkState) { + // TODO add state transition validation here? + + self.state = state; + } +} diff --git a/catalog/src/lib.rs b/catalog/src/lib.rs new file mode 100644 index 0000000000..d77d52a6dc --- /dev/null +++ b/catalog/src/lib.rs @@ -0,0 +1,361 @@ +//! This module contains the implementation of the InfluxDB IOx Metadata catalog +#![deny(rust_2018_idioms)] +#![warn( + missing_debug_implementations, + clippy::explicit_iter_loop, + clippy::use_self, + clippy::clone_on_ref_ptr +)] +use std::collections::{btree_map::Entry, BTreeMap}; + +use snafu::{OptionExt, Snafu}; + +#[derive(Debug, Snafu)] +pub enum Error { + #[snafu(display("unknown partition: {}", partition_key))] + UnknownPartition { partition_key: String }, + + #[snafu(display("unknown chunk: {}:{}", partition_key, chunk_id))] + UnknownChunk { + partition_key: String, + chunk_id: u32, + }, + + #[snafu(display("partition already exists: {}", partition_key))] + PartitionAlreadyExists { partition_key: String }, + + #[snafu(display("chunk already exists: {}:{}", partition_key, chunk_id))] + ChunkAlreadyExists { + partition_key: String, + chunk_id: u32, + }, +} +pub type Result = std::result::Result; + +pub mod chunk; +pub mod partition; + +use chunk::Chunk; +use partition::Partition; + +/// InfluxDB IOx Metadata Catalog +/// +/// The Catalog stores information such as which chunks exist, what +/// state they are in, and what objects on object store are used, etc. +/// +/// The catalog is also responsible for (eventually) persisting this +/// information as well as ensuring that references between different +/// objects remain valid (e.g. that the `partition_key` field of all +/// Chunk's refer to valid partitions). +#[derive(Default, Debug)] +pub struct Catalog { + /// key is partition_key + partitions: BTreeMap, +} + +impl Catalog { + pub fn new() -> Self { + Self { + ..Default::default() + } + } + + /// Return an immutable chunk reference given the specified partition and + /// chunk id + pub fn chunk(&self, partition_key: impl AsRef, chunk_id: u32) -> Result<&Chunk> { + let partition_key = partition_key.as_ref(); + self.valid_partition(partition_key)?.chunk(chunk_id) + } + + /// Return an mutable chunk reference given the specified partition and + /// chunk id + pub fn chunk_mut( + &mut self, + partition_key: impl AsRef, + chunk_id: u32, + ) -> Result<&mut Chunk> { + let partition_key = partition_key.as_ref(); + self.valid_partition_mut(partition_key)?.chunk_mut(chunk_id) + } + + /// Creates a new `Chunk` with id `id` within a specified Partition. + pub fn create_chunk(&mut self, partition_key: impl AsRef, chunk_id: u32) -> Result<()> { + let partition_key = partition_key.as_ref(); + self.valid_partition_mut(partition_key)? + .create_chunk(chunk_id) + } + + /// Removes the specified `Chunk` from the catalog + pub fn drop_chunk(&mut self, partition_key: impl AsRef, chunk_id: u32) -> Result<()> { + let partition_key = partition_key.as_ref(); + self.valid_partition_mut(partition_key)? + .drop_chunk(chunk_id) + } + + /// List all `Chunk`s in this database + pub fn chunks(&self) -> impl Iterator { + self.partitions.values().flat_map(|p| p.chunks()) + } + + /// List all `Chunk`s in a particular partition + pub fn partition_chunks( + &self, + partition_key: impl AsRef, + ) -> Result> { + let partition_key = partition_key.as_ref(); + let iter = self.valid_partition(partition_key)?.chunks(); + Ok(iter) + } + + // List all partitions in this database + pub fn partitions(&self) -> impl Iterator { + self.partitions.values() + } + + // Get a specific partition by name, returning `None` if there is no such + // partition + pub fn partition(&self, partition_key: impl AsRef) -> Option<&Partition> { + let partition_key = partition_key.as_ref(); + self.partitions.get(partition_key) + } + + // Create a new partition in the catalog, returning an error if it already + // exists + pub fn create_partition(&mut self, partition_key: impl Into) -> Result<()> { + let partition_key = partition_key.into(); + + let entry = self.partitions.entry(partition_key); + match entry { + Entry::Vacant(entry) => { + let partition = Partition::new(entry.key()); + entry.insert(partition); + Ok(()) + } + Entry::Occupied(entry) => PartitionAlreadyExists { + partition_key: entry.key(), + } + .fail(), + } + } + + /// Internal helper to return the specified partition or an error + /// if there is no such partition + fn valid_partition(&self, partition_key: &str) -> Result<&Partition> { + self.partitions + .get(partition_key) + .context(UnknownPartition { partition_key }) + } + + /// Internal helper to return the specified partition as a mutable + /// reference or an error if there is no such partition + fn valid_partition_mut(&mut self, partition_key: &str) -> Result<&mut Partition> { + self.partitions + .get_mut(partition_key) + .context(UnknownPartition { partition_key }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn partition_create() { + let mut catalog = Catalog::new(); + catalog.create_partition("p1").unwrap(); + + let err = catalog.create_partition("p1").unwrap_err(); + assert_eq!(err.to_string(), "partition already exists: p1"); + } + + #[test] + fn partition_get() { + let mut catalog = Catalog::new(); + catalog.create_partition("p1").unwrap(); + catalog.create_partition("p2").unwrap(); + + let p1 = catalog.partition("p1").unwrap(); + assert_eq!(p1.key(), "p1"); + + let p2 = catalog.partition("p2").unwrap(); + assert_eq!(p2.key(), "p2"); + + let p3 = catalog.partition("p3"); + assert!(p3.is_none()); + } + + #[test] + fn partition_list() { + let mut catalog = Catalog::new(); + + assert_eq!(catalog.partitions().count(), 0); + + catalog.create_partition("p1").unwrap(); + catalog.create_partition("p2").unwrap(); + catalog.create_partition("p3").unwrap(); + + let mut partition_keys: Vec = + catalog.partitions().map(|p| p.key().into()).collect(); + partition_keys.sort_unstable(); + + assert_eq!(partition_keys, vec!["p1", "p2", "p3"]); + } + + #[test] + fn chunk_create_no_partition() { + let mut catalog = Catalog::new(); + let err = catalog + .create_chunk("non existent partition", 0) + .unwrap_err(); + assert_eq!(err.to_string(), "unknown partition: non existent partition"); + } + + #[test] + fn chunk_create() { + let mut catalog = Catalog::new(); + catalog.create_partition("p1").unwrap(); + catalog.create_chunk("p1", 0).unwrap(); + catalog.create_chunk("p1", 1).unwrap(); + + let c1_0 = catalog.chunk("p1", 0).unwrap(); + assert_eq!(c1_0.key(), "p1"); + assert_eq!(c1_0.id(), 0); + + let c1_0 = catalog.chunk_mut("p1", 0).unwrap(); + assert_eq!(c1_0.key(), "p1"); + assert_eq!(c1_0.id(), 0); + + let c1_1 = catalog.chunk("p1", 1).unwrap(); + assert_eq!(c1_1.key(), "p1"); + assert_eq!(c1_1.id(), 1); + + let err = catalog.chunk("p3", 0).unwrap_err(); + assert_eq!(err.to_string(), "unknown partition: p3"); + + let err = catalog.chunk("p1", 100).unwrap_err(); + assert_eq!(err.to_string(), "unknown chunk: p1:100"); + } + + #[test] + fn chunk_create_dupe() { + let mut catalog = Catalog::new(); + catalog.create_partition("p1").unwrap(); + catalog.create_chunk("p1", 0).unwrap(); + + let res = catalog.create_chunk("p1", 0).unwrap_err(); + assert_eq!(res.to_string(), "chunk already exists: p1:0"); + } + + #[test] + fn chunk_list() { + let mut catalog = Catalog::new(); + assert_eq!(catalog.chunks().count(), 0); + + catalog.create_partition("p1").unwrap(); + catalog.create_chunk("p1", 0).unwrap(); + catalog.create_chunk("p1", 1).unwrap(); + + catalog.create_partition("p2").unwrap(); + catalog.create_chunk("p2", 100).unwrap(); + + assert_eq!( + chunk_strings(&catalog), + vec!["Chunk p1:0", "Chunk p1:1", "Chunk p2:100"] + ); + + assert_eq!( + partition_chunk_strings(&catalog, "p1"), + vec!["Chunk p1:0", "Chunk p1:1"] + ); + assert_eq!( + partition_chunk_strings(&catalog, "p2"), + vec!["Chunk p2:100"] + ); + } + + #[test] + fn chunk_list_err() { + let catalog = Catalog::new(); + + match catalog.partition_chunks("p3") { + Err(err) => assert_eq!(err.to_string(), "unknown partition: p3"), + Ok(_) => panic!("unexpected success"), + }; + } + + fn chunk_strings(catalog: &Catalog) -> Vec { + let mut chunks: Vec = catalog + .chunks() + .map(|c| format!("Chunk {}:{}", c.key(), c.id())) + .collect(); + chunks.sort_unstable(); + + chunks + } + + fn partition_chunk_strings(catalog: &Catalog, partition_key: &str) -> Vec { + let mut chunks: Vec = catalog + .partition_chunks(partition_key) + .unwrap() + .map(|c| format!("Chunk {}:{}", c.key(), c.id())) + .collect(); + chunks.sort_unstable(); + + chunks + } + + #[test] + fn chunk_drop() { + let mut catalog = Catalog::new(); + + catalog.create_partition("p1").unwrap(); + catalog.create_chunk("p1", 0).unwrap(); + catalog.create_chunk("p1", 1).unwrap(); + + catalog.create_partition("p2").unwrap(); + catalog.create_chunk("p2", 0).unwrap(); + + assert_eq!(catalog.chunks().count(), 3); + + catalog.drop_chunk("p1", 1).unwrap(); + catalog.chunk("p1", 1).unwrap_err(); // chunk is gone + assert_eq!(catalog.chunks().count(), 2); + + catalog.drop_chunk("p2", 0).unwrap(); + catalog.chunk("p2", 0).unwrap_err(); // chunk is gone + assert_eq!(catalog.chunks().count(), 1); + } + + #[test] + fn chunk_drop_non_existent_partition() { + let mut catalog = Catalog::new(); + let err = catalog.drop_chunk("p3", 0).unwrap_err(); + assert_eq!(err.to_string(), "unknown partition: p3"); + } + + #[test] + fn chunk_drop_non_existent_chunk() { + let mut catalog = Catalog::new(); + catalog.create_partition("p3").unwrap(); + + let err = catalog.drop_chunk("p3", 0).unwrap_err(); + assert_eq!(err.to_string(), "unknown chunk: p3:0"); + } + + #[test] + fn chunk_recreate_dropped() { + let mut catalog = Catalog::new(); + + catalog.create_partition("p1").unwrap(); + catalog.create_chunk("p1", 0).unwrap(); + catalog.create_chunk("p1", 1).unwrap(); + assert_eq!(catalog.chunks().count(), 2); + + catalog.drop_chunk("p1", 0).unwrap(); + assert_eq!(catalog.chunks().count(), 1); + + // should be ok to recreate + catalog.create_chunk("p1", 0).unwrap(); + assert_eq!(catalog.chunks().count(), 2); + } +} diff --git a/catalog/src/partition.rs b/catalog/src/partition.rs new file mode 100644 index 0000000000..2b0a4c6690 --- /dev/null +++ b/catalog/src/partition.rs @@ -0,0 +1,108 @@ +//! The catalog representation of a Partition + +use crate::chunk::Chunk; +use std::collections::{btree_map::Entry, BTreeMap}; + +use super::{ChunkAlreadyExists, Result, UnknownChunk}; +use snafu::OptionExt; + +/// IOx Catalog Partition +/// +/// A partition contains multiple Chunks. +#[derive(Debug, Default)] +pub struct Partition { + /// The partition key + key: String, + + /// The chunks that make up this partition, indexed by id + chunks: BTreeMap, +} + +impl Partition { + /// Return the partition_key of this Partition + pub fn key(&self) -> &str { + &self.key + } +} + +impl Partition { + /// Create a new partition catalog object. + /// + /// This function is not pub because `Partition`s should be + /// created using the interfaces on [`Catalog`] and not + /// instantiated directly. + pub(crate) fn new(key: impl Into) -> Self { + let key = key.into(); + + Self { + key, + ..Default::default() + } + } + + /// Create a new Chunk + /// + /// This function is not pub because `Chunks`s should be created + /// using the interfaces on [`Catalog`] and not instantiated + /// directly. + pub(crate) fn create_chunk(&mut self, chunk_id: u32) -> Result<()> { + let entry = self.chunks.entry(chunk_id); + match entry { + Entry::Vacant(entry) => { + entry.insert(Chunk::new(&self.key, chunk_id)); + Ok(()) + } + Entry::Occupied(_) => ChunkAlreadyExists { + partition_key: self.key(), + chunk_id, + } + .fail(), + } + } + + /// Drop the specified + /// + /// This function is not pub because `Chunks`s should be dropped + /// using the interfaces on [`Catalog`] and not instantiated + /// directly. + pub(crate) fn drop_chunk(&mut self, chunk_id: u32) -> Result<()> { + match self.chunks.remove(&chunk_id) { + Some(_) => Ok(()), + None => UnknownChunk { + partition_key: self.key(), + chunk_id, + } + .fail(), + } + } + + /// Return an immutable chunk reference by chunk id + /// + /// This function is not pub because `Chunks`s should be + /// accessed using the interfaces on [`Catalog`] + pub(crate) fn chunk(&self, chunk_id: u32) -> Result<&Chunk> { + self.chunks.get(&chunk_id).context(UnknownChunk { + partition_key: self.key(), + chunk_id, + }) + } + + /// Return a mutable chunk reference by chunk id + /// + /// This function is not pub because `Chunks`s should be + /// accessed using the interfaces on [`Catalog`] + pub(crate) fn chunk_mut(&mut self, chunk_id: u32) -> Result<&mut Chunk> { + self.chunks.get_mut(&chunk_id).context(UnknownChunk { + partition_key: &self.key, + chunk_id, + }) + } + + /// Return a iterator over chunks + /// + /// This function is not pub because `Chunks`s should be + /// accessed using the interfaces on [`Catalog`] + pub(crate) fn chunks(&self) -> impl Iterator { + self.chunks.values() + } +} diff --git a/data_types/Cargo.toml b/data_types/Cargo.toml index 80cfdc44bc..8c99f52135 100644 --- a/data_types/Cargo.toml +++ b/data_types/Cargo.toml @@ -2,24 +2,22 @@ name = "data_types" version = "0.1.0" authors = ["pauldix "] +description = "InfluxDB IOx data_types, shared between IOx instances and IOx clients" edition = "2018" +readme = "README.md" [dependencies] # In alphabetical order -arrow_deps = { path = "../arrow_deps" } chrono = { version = "0.4", features = ["serde"] } -crc32fast = "1.2.0" -flatbuffers = "0.6" generated_types = { path = "../generated_types" } influxdb_line_protocol = { path = "../influxdb_line_protocol" } percent-encoding = "2.1.0" +prost = "0.7" +regex = "1.4" serde = "1.0" +serde_regex = "1.1" snafu = "0.6" +tonic = { version = "0.4.0" } tracing = "0.1" [dev-dependencies] # In alphabetical order -criterion = "0.3" test_helpers = { path = "../test_helpers" } - -[[bench]] -name = "benchmark" -harness = false diff --git a/data_types/README.md b/data_types/README.md new file mode 100644 index 0000000000..2911ec8088 --- /dev/null +++ b/data_types/README.md @@ -0,0 +1,5 @@ +# Data Types + +This crate contains types that are designed for external consumption (in `influxdb_iox_client` and other "client" facing uses). + +*Client facing* in this case means exposed via management API or CLI and where changing the structs may require additional coordination / organization with clients. diff --git a/data_types/src/chunk.rs b/data_types/src/chunk.rs new file mode 100644 index 0000000000..b68d618b5c --- /dev/null +++ b/data_types/src/chunk.rs @@ -0,0 +1,176 @@ +//! Module contains a representation of chunk metadata +use std::{convert::TryFrom, sync::Arc}; + +use crate::field_validation::FromField; +use generated_types::{google::FieldViolation, influxdata::iox::management::v1 as management}; +use serde::{Deserialize, Serialize}; + +/// Which storage system is a chunk located in? +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Serialize, Deserialize)] +pub enum ChunkStorage { + /// The chunk is still open for new writes, in the Mutable Buffer + OpenMutableBuffer, + + /// The chunk is no longer open for writes, in the Mutable Buffer + ClosedMutableBuffer, + + /// The chunk is in the Read Buffer (where it can not be mutated) + ReadBuffer, + + /// The chunk is stored in Object Storage (where it can not be mutated) + ObjectStore, +} + +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Serialize, Deserialize)] +/// Represents metadata about a chunk in a database. +/// A chunk can contain one or more tables. +pub struct ChunkSummary { + /// The partitition key of this chunk + pub partition_key: Arc, + + /// The id of this chunk + pub id: u32, + + /// How is this chunk stored? + pub storage: ChunkStorage, + + /// The total estimated size of this chunk, in bytes + pub estimated_bytes: usize, +} + +/// Conversion code to management API chunk structure +impl From for management::Chunk { + fn from(summary: ChunkSummary) -> Self { + let ChunkSummary { + partition_key, + id, + storage, + estimated_bytes, + } = summary; + + let storage: management::ChunkStorage = storage.into(); + let storage = storage.into(); // convert to i32 + + let estimated_bytes = estimated_bytes as u64; + + let partition_key = match Arc::try_unwrap(partition_key) { + // no one else has a reference so take the string + Ok(partition_key) => partition_key, + // some other refernece exists to this string, so clone it + Err(partition_key) => partition_key.as_ref().clone(), + }; + + Self { + partition_key, + id, + storage, + estimated_bytes, + } + } +} + +impl From for management::ChunkStorage { + fn from(storage: ChunkStorage) -> Self { + match storage { + ChunkStorage::OpenMutableBuffer => Self::OpenMutableBuffer, + ChunkStorage::ClosedMutableBuffer => Self::ClosedMutableBuffer, + ChunkStorage::ReadBuffer => Self::ReadBuffer, + ChunkStorage::ObjectStore => Self::ObjectStore, + } + } +} + +/// Conversion code from management API chunk structure +impl TryFrom for ChunkSummary { + type Error = FieldViolation; + + fn try_from(proto: management::Chunk) -> Result { + // Use prost enum conversion + let storage = proto.storage().scope("storage")?; + + let management::Chunk { + partition_key, + id, + estimated_bytes, + .. + } = proto; + + let estimated_bytes = estimated_bytes as usize; + let partition_key = Arc::new(partition_key); + + Ok(Self { + partition_key, + id, + storage, + estimated_bytes, + }) + } +} + +impl TryFrom for ChunkStorage { + type Error = FieldViolation; + + fn try_from(proto: management::ChunkStorage) -> Result { + match proto { + management::ChunkStorage::OpenMutableBuffer => Ok(Self::OpenMutableBuffer), + management::ChunkStorage::ClosedMutableBuffer => Ok(Self::ClosedMutableBuffer), + management::ChunkStorage::ReadBuffer => Ok(Self::ReadBuffer), + management::ChunkStorage::ObjectStore => Ok(Self::ObjectStore), + management::ChunkStorage::Unspecified => Err(FieldViolation::required("")), + } + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn valid_proto_to_summary() { + let proto = management::Chunk { + partition_key: "foo".to_string(), + id: 42, + estimated_bytes: 1234, + storage: management::ChunkStorage::ObjectStore.into(), + }; + + let summary = ChunkSummary::try_from(proto).expect("conversion successful"); + let expected = ChunkSummary { + partition_key: Arc::new("foo".to_string()), + id: 42, + estimated_bytes: 1234, + storage: ChunkStorage::ObjectStore, + }; + + assert_eq!( + summary, expected, + "Actual:\n\n{:?}\n\nExpected:\n\n{:?}\n\n", + summary, expected + ); + } + + #[test] + fn valid_summary_to_proto() { + let summary = ChunkSummary { + partition_key: Arc::new("foo".to_string()), + id: 42, + estimated_bytes: 1234, + storage: ChunkStorage::ObjectStore, + }; + + let proto = management::Chunk::try_from(summary).expect("conversion successful"); + + let expected = management::Chunk { + partition_key: "foo".to_string(), + id: 42, + estimated_bytes: 1234, + storage: management::ChunkStorage::ObjectStore.into(), + }; + + assert_eq!( + proto, expected, + "Actual:\n\n{:?}\n\nExpected:\n\n{:?}\n\n", + proto, expected + ); + } +} diff --git a/data_types/src/database_rules.rs b/data_types/src/database_rules.rs index 70909620e4..00e12f28bf 100644 --- a/data_types/src/database_rules.rs +++ b/data_types/src/database_rules.rs @@ -1,6 +1,7 @@ use std::convert::{TryFrom, TryInto}; use chrono::{DateTime, TimeZone, Utc}; +use regex::Regex; use serde::{Deserialize, Serialize}; use snafu::Snafu; @@ -33,73 +34,11 @@ pub struct DatabaseRules { /// database call, so an empty default is fine. #[serde(default)] pub name: String, // TODO: Use DatabaseName here + /// Template that generates a partition key for each row inserted into the /// db #[serde(default)] pub partition_template: PartitionTemplate, - /// The set of host groups that data should be replicated to. Which host a - /// write goes to within a host group is determined by consistent hashing of - /// the partition key. We'd use this to create a host group per - /// availability zone, so you might have 5 availability zones with 2 - /// hosts in each. Replication will ensure that N of those zones get a - /// write. For each zone, only a single host needs to get the write. - /// Replication is for ensuring a write exists across multiple hosts - /// before returning success. Its purpose is to ensure write durability, - /// rather than write availability for query (this is covered by - /// subscriptions). - #[serde(default)] - pub replication: Vec, - /// The minimum number of host groups to replicate a write to before success - /// is returned. This can be overridden on a per request basis. - /// Replication will continue to write to the other host groups in the - /// background. - #[serde(default)] - pub replication_count: u8, - /// How long the replication queue can get before either rejecting writes or - /// dropping missed writes. The queue is kept in memory on a - /// per-database basis. A queue size of zero means it will only try to - /// replicate synchronously and drop any failures. - #[serde(default)] - pub replication_queue_max_size: usize, - /// `subscriptions` are used for query servers to get data via either push - /// or pull as it arrives. They are separate from replication as they - /// have a different purpose. They're for query servers or other clients - /// that want to subscribe to some subset of data being written in. This - /// could either be specific partitions, ranges of partitions, tables, or - /// rows matching some predicate. This is step #3 from the diagram. - #[serde(default)] - pub subscriptions: Vec, - - /// If set to `true`, this server should answer queries from one or more of - /// of its local write buffer and any read-only partitions that it knows - /// about. In this case, results will be merged with any others from the - /// remote goups or read-only partitions. - #[serde(default)] - pub query_local: bool, - /// Set `primary_query_group` to a host group if remote servers should be - /// issued queries for this database. All hosts in the group should be - /// queried with this server acting as the coordinator that merges - /// results together. If a specific host in the group is unavailable, - /// another host in the same position from a secondary group should be - /// queried. For example, imagine we've partitioned the data in this DB into - /// 4 partitions and we are replicating the data across 3 availability - /// zones. We have 4 hosts in each of those AZs, thus they each have 1 - /// partition. We'd set the primary group to be the 4 hosts in the same - /// AZ as this one, and the secondary groups as the hosts in the other 2 - /// AZs. - #[serde(default)] - pub primary_query_group: Option, - #[serde(default)] - pub secondary_query_groups: Vec, - - /// Use `read_only_partitions` when a server should answer queries for - /// partitions that come from object storage. This can be used to start - /// up a new query server to handle queries by pointing it at a - /// collection of partitions and then telling it to also pull - /// data from the replication servers (writes that haven't been snapshotted - /// into a partition). - #[serde(default)] - pub read_only_partitions: Vec, /// When set this will buffer WAL writes in memory based on the /// configuration. @@ -113,6 +52,16 @@ pub struct DatabaseRules { /// in object storage. #[serde(default = "MutableBufferConfig::default_option")] pub mutable_buffer_config: Option, + + /// An optional config to split writes into different "shards". A shard + /// is a logical concept, but the usage is meant to split data into + /// mutually exclusive areas. The rough order of organization is: + /// database -> shard -> partition -> chunk. For example, you could shard + /// based on table name and assign to 1 of 10 shards. Within each + /// shard you would have partitions, which would likely be based off time. + /// This makes it possible to horizontally scale out writes. + #[serde(default)] + pub shard_config: Option, } impl DatabaseRules { @@ -149,28 +98,9 @@ impl Partitioner for DatabaseRules { impl From for management::DatabaseRules { fn from(rules: DatabaseRules) -> Self { - let subscriptions: Vec = - rules.subscriptions.into_iter().map(Into::into).collect(); - - let replication_config = management::ReplicationConfig { - replications: rules.replication, - replication_count: rules.replication_count as _, - replication_queue_max_size: rules.replication_queue_max_size as _, - }; - - let query_config = management::QueryConfig { - query_local: rules.query_local, - primary: rules.primary_query_group.unwrap_or_default(), - secondaries: rules.secondary_query_groups, - read_only_partitions: rules.read_only_partitions, - }; - Self { name: rules.name, partition_template: Some(rules.partition_template.into()), - replication_config: Some(replication_config), - subscription_config: Some(management::SubscriptionConfig { subscriptions }), - query_config: Some(query_config), wal_buffer_config: rules.wal_buffer_config.map(Into::into), mutable_buffer_config: rules.mutable_buffer_config.map(Into::into), } @@ -183,15 +113,6 @@ impl TryFrom for DatabaseRules { fn try_from(proto: management::DatabaseRules) -> Result { DatabaseName::new(&proto.name).field("name")?; - let subscriptions = proto - .subscription_config - .map(|s| { - s.subscriptions - .vec_field("subscription_config.subscriptions") - }) - .transpose()? - .unwrap_or_default(); - let wal_buffer_config = proto.wal_buffer_config.optional("wal_buffer_config")?; let mutable_buffer_config = proto @@ -203,22 +124,12 @@ impl TryFrom for DatabaseRules { .optional("partition_template")? .unwrap_or_default(); - let query = proto.query_config.unwrap_or_default(); - let replication = proto.replication_config.unwrap_or_default(); - Ok(Self { name: proto.name, partition_template, - replication: replication.replications, - replication_count: replication.replication_count as _, - replication_queue_max_size: replication.replication_queue_max_size as _, - subscriptions, - query_local: query.query_local, - primary_query_group: query.primary.optional(), - secondary_query_groups: query.secondaries, - read_only_partitions: query.read_only_partitions, wal_buffer_config, mutable_buffer_config, + shard_config: None, }) } } @@ -718,10 +629,19 @@ impl TryFrom for PartitionTemplate { /// part of a partition key. #[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone)] pub enum TemplatePart { + /// The name of a table Table, + /// The value in a named column Column(String), + /// Applies a `strftime` format to the "time" column. + /// + /// For example, a time format of "%Y-%m-%d %H:%M:%S" will produce + /// partition key parts such as "2021-03-14 12:25:21" and + /// "2021-04-14 12:24:21" TimeFormat(String), + /// Applies a regex to the value in a string column RegexCapture(RegexCapture), + /// Applies a `strftime` pattern to some column other than "time" StrftimeColumn(StrftimeColumn), } @@ -733,8 +653,15 @@ pub struct RegexCapture { regex: String, } -/// `StrftimeColumn` can be used to create a time based partition key off some +/// [`StrftimeColumn`] is used to create a time based partition key off some /// column other than the builtin `time` column. +/// +/// The value of the named column is formatted using a `strftime` +/// style string. +/// +/// For example, a time format of "%Y-%m-%d %H:%M:%S" will produce +/// partition key parts such as "2021-03-14 12:25:21" and +/// "2021-04-14 12:24:21" #[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone)] pub struct StrftimeColumn { column: String, @@ -802,66 +729,192 @@ impl TryFrom for TemplatePart { } } -/// `PartitionId` is the object storage identifier for a specific partition. It -/// should be a path that can be used against an object store to locate all the -/// files and subdirectories for a partition. It takes the form of `////`. -pub type PartitionId = String; -pub type WriterId = u32; - -/// `Subscription` represents a group of hosts that want to receive data as it -/// arrives. The subscription has a matcher that is used to determine what data -/// will match it, and an optional queue for storing matched writes. Subscribers -/// that recieve some subeset of an individual replicated write will get a new -/// replicated write, but with the same originating writer ID and sequence -/// number for the consuming subscriber's tracking purposes. -/// -/// For pull based subscriptions, the requester will send a matcher, which the -/// receiver will execute against its in-memory WAL. +/// ShardConfig defines rules for assigning a line/row to an individual +/// host or a group of hosts. A shard +/// is a logical concept, but the usage is meant to split data into +/// mutually exclusive areas. The rough order of organization is: +/// database -> shard -> partition -> chunk. For example, you could shard +/// based on table name and assign to 1 of 10 shards. Within each +/// shard you would have partitions, which would likely be based off time. +/// This makes it possible to horizontally scale out writes. #[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone)] -pub struct Subscription { - pub name: String, - pub host_group_id: HostGroupId, - pub matcher: Matcher, +pub struct ShardConfig { + /// An optional matcher. If there is a match, the route will be evaluated to + /// the given targets, otherwise the hash ring will be evaluated. This is + /// useful for overriding the hashring function on some hot spot. For + /// example, if you use the table name as the input to the hash function + /// and your ring has 4 slots. If two tables that are very hot get + /// assigned to the same slot you can override that by putting in a + /// specific matcher to pull that table over to a different node. + pub specific_targets: Option, + /// An optional default hasher which will route to one in a collection of + /// nodes. + pub hash_ring: Option, + /// If set to true the router will ignore any errors sent by the remote + /// targets in this route. That is, the write request will succeed + /// regardless of this route's success. + pub ignore_errors: bool, } -impl From for management::subscription_config::Subscription { - fn from(s: Subscription) -> Self { +/// Maps a matcher with specific target group. If the line/row matches +/// it should be sent to the group. +#[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone, Default)] +pub struct MatcherToTargets { + pub matcher: Matcher, + pub target: NodeGroup, +} + +/// A collection of IOx nodes +pub type NodeGroup = Vec; + +/// HashRing is a rule for creating a hash key for a row and mapping that to +/// an individual node on a ring. +#[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone)] +pub struct HashRing { + /// If true the table name will be included in the hash key + pub table_name: bool, + /// include the values of these columns in the hash key + pub columns: Vec, + /// ring of node groups. Each group holds a shard + pub node_groups: Vec, +} + +/// A matcher is used to match routing rules or subscriptions on a row-by-row +/// (or line) basis. +#[derive(Debug, Serialize, Deserialize, Clone, Default)] +pub struct Matcher { + /// if provided, match if the table name matches against the regex + #[serde(with = "serde_regex")] + pub table_name_regex: Option, + // paul: what should we use for predicate matching here against a single row/line? + pub predicate: Option, +} + +impl PartialEq for Matcher { + fn eq(&self, other: &Self) -> bool { + // this is kind of janky, but it's only used during tests and should get the job + // done + format!("{:?}{:?}", self.table_name_regex, self.predicate) + == format!("{:?}{:?}", other.table_name_regex, other.predicate) + } +} +impl Eq for Matcher {} + +impl From for management::ShardConfig { + fn from(shard_config: ShardConfig) -> Self { Self { - name: s.name, - host_group_id: s.host_group_id, - matcher: Some(s.matcher.into()), + specific_targets: shard_config.specific_targets.map(|i| i.into()), + hash_ring: shard_config.hash_ring.map(|i| i.into()), + ignore_errors: shard_config.ignore_errors, } } } -impl TryFrom for Subscription { +impl TryFrom for ShardConfig { type Error = FieldViolation; - fn try_from(proto: management::subscription_config::Subscription) -> Result { + fn try_from(proto: management::ShardConfig) -> Result { Ok(Self { - name: proto.name.required("name")?, - host_group_id: proto.host_group_id.required("host_group_id")?, - matcher: proto.matcher.optional("matcher")?.unwrap_or_default(), + specific_targets: proto + .specific_targets + .map(|i| i.try_into()) + .map_or(Ok(None), |r| r.map(Some))?, + hash_ring: proto + .hash_ring + .map(|i| i.try_into()) + .map_or(Ok(None), |r| r.map(Some))?, + ignore_errors: proto.ignore_errors, }) } } -/// `Matcher` specifies the rule against the table name and/or a predicate -/// against the row to determine if it matches the write rule. -#[derive(Debug, Default, Serialize, Deserialize, Eq, PartialEq, Clone)] -pub struct Matcher { - pub tables: MatchTables, - // TODO: make this work with query::Predicate - #[serde(skip_serializing_if = "Option::is_none")] - pub predicate: Option, +/// Returns none if v matches its default value. +fn none_if_default(v: T) -> Option { + if v == Default::default() { + None + } else { + Some(v) + } +} + +impl From for management::MatcherToTargets { + fn from(matcher_to_targets: MatcherToTargets) -> Self { + Self { + matcher: none_if_default(matcher_to_targets.matcher.into()), + target: none_if_default(from_node_group_for_management_node_group( + matcher_to_targets.target, + )), + } + } +} + +impl TryFrom for MatcherToTargets { + type Error = FieldViolation; + + fn try_from(proto: management::MatcherToTargets) -> Result { + Ok(Self { + matcher: proto.matcher.unwrap_or_default().try_into()?, + target: try_from_management_node_group_for_node_group( + proto.target.unwrap_or_default(), + )?, + }) + } +} + +impl From for management::HashRing { + fn from(hash_ring: HashRing) -> Self { + Self { + table_name: hash_ring.table_name, + columns: hash_ring.columns, + node_groups: hash_ring + .node_groups + .into_iter() + .map(from_node_group_for_management_node_group) + .collect(), + } + } +} + +impl TryFrom for HashRing { + type Error = FieldViolation; + + fn try_from(proto: management::HashRing) -> Result { + Ok(Self { + table_name: proto.table_name, + columns: proto.columns, + node_groups: proto + .node_groups + .into_iter() + .map(try_from_management_node_group_for_node_group) + .collect::, _>>()?, + }) + } +} + +// cannot (and/or don't know how to) add impl From inside prost generated code +fn from_node_group_for_management_node_group(node_group: NodeGroup) -> management::NodeGroup { + management::NodeGroup { + nodes: node_group + .into_iter() + .map(|id| management::node_group::Node { id }) + .collect(), + } +} + +fn try_from_management_node_group_for_node_group( + proto: management::NodeGroup, +) -> Result { + Ok(proto.nodes.into_iter().map(|i| i.id).collect()) } impl From for management::Matcher { - fn from(m: Matcher) -> Self { + fn from(matcher: Matcher) -> Self { Self { - predicate: m.predicate.unwrap_or_default(), - table_matcher: Some(m.tables.into()), + table_name_regex: matcher + .table_name_regex + .map(|r| r.to_string()) + .unwrap_or_default(), + predicate: matcher.predicate.unwrap_or_default(), } } } @@ -870,61 +923,31 @@ impl TryFrom for Matcher { type Error = FieldViolation; fn try_from(proto: management::Matcher) -> Result { + let table_name_regex = match &proto.table_name_regex as &str { + "" => None, + re => Some(Regex::new(re).map_err(|e| FieldViolation { + field: "table_name_regex".to_string(), + description: e.to_string(), + })?), + }; + let predicate = match proto.predicate { + p if p.is_empty() => None, + p => Some(p), + }; + Ok(Self { - tables: proto.table_matcher.required("table_matcher")?, - predicate: proto.predicate.optional(), + table_name_regex, + predicate, }) } } -/// `MatchTables` looks at the table name of a row to determine if it should -/// match the rule. -#[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone)] -#[serde(rename_all = "camelCase")] -pub enum MatchTables { - #[serde(rename = "*")] - All, - Table(String), - Regex(String), -} - -impl Default for MatchTables { - fn default() -> Self { - Self::All - } -} - -impl From for management::matcher::TableMatcher { - fn from(m: MatchTables) -> Self { - match m { - MatchTables::All => Self::All(Empty {}), - MatchTables::Table(table) => Self::Table(table), - MatchTables::Regex(regex) => Self::Regex(regex), - } - } -} - -impl TryFrom for MatchTables { - type Error = FieldViolation; - - fn try_from(proto: management::matcher::TableMatcher) -> Result { - use management::matcher::TableMatcher; - Ok(match proto { - TableMatcher::All(_) => Self::All, - TableMatcher::Table(table) => Self::Table(table.required("table_matcher.table")?), - TableMatcher::Regex(regex) => Self::Regex(regex.required("table_matcher.regex")?), - }) - } -} - -pub type HostGroupId = String; - -#[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Clone)] -pub struct HostGroup { - pub id: HostGroupId, - /// `hosts` is a vector of connection strings for remote hosts. - pub hosts: Vec, -} +/// `PartitionId` is the object storage identifier for a specific partition. It +/// should be a path that can be used against an object store to locate all the +/// files and subdirectories for a partition. It takes the form of `////`. +pub type PartitionId = String; +pub type WriterId = u32; #[cfg(test)] mod tests { @@ -1107,16 +1130,9 @@ mod tests { assert_eq!(protobuf.name, back.name); assert_eq!(rules.partition_template.parts.len(), 0); - assert_eq!(rules.subscriptions.len(), 0); - assert!(rules.primary_query_group.is_none()); - assert_eq!(rules.read_only_partitions.len(), 0); - assert_eq!(rules.secondary_query_groups.len(), 0); // These will be defaulted as optionality not preserved on non-protobuf // DatabaseRules - assert_eq!(back.replication_config, Some(Default::default())); - assert_eq!(back.subscription_config, Some(Default::default())); - assert_eq!(back.query_config, Some(Default::default())); assert_eq!(back.partition_template, Some(Default::default())); // These should be none as preserved on non-protobuf DatabaseRules @@ -1124,65 +1140,6 @@ mod tests { assert!(back.mutable_buffer_config.is_none()); } - #[test] - fn test_database_rules_query() { - let readonly = vec!["readonly1".to_string(), "readonly2".to_string()]; - let secondaries = vec!["secondary1".to_string(), "secondary2".to_string()]; - - let protobuf = management::DatabaseRules { - name: "database".to_string(), - query_config: Some(management::QueryConfig { - query_local: true, - primary: "primary".to_string(), - secondaries: secondaries.clone(), - read_only_partitions: readonly.clone(), - }), - ..Default::default() - }; - - let rules: DatabaseRules = protobuf.clone().try_into().unwrap(); - let back: management::DatabaseRules = rules.clone().into(); - - assert_eq!(rules.name, protobuf.name); - assert_eq!(protobuf.name, back.name); - - assert_eq!(rules.read_only_partitions, readonly); - assert_eq!(rules.primary_query_group, Some("primary".to_string())); - assert_eq!(rules.secondary_query_groups, secondaries); - assert_eq!(rules.subscriptions.len(), 0); - assert_eq!(rules.partition_template.parts.len(), 0); - - // Should be the same as was specified - assert_eq!(back.query_config, protobuf.query_config); - assert!(back.wal_buffer_config.is_none()); - assert!(back.mutable_buffer_config.is_none()); - - // These will be defaulted as optionality not preserved on non-protobuf - // DatabaseRules - assert_eq!(back.replication_config, Some(Default::default())); - assert_eq!(back.subscription_config, Some(Default::default())); - assert_eq!(back.partition_template, Some(Default::default())); - } - - #[test] - fn test_query_config_default() { - let protobuf = management::DatabaseRules { - name: "database".to_string(), - query_config: Some(Default::default()), - ..Default::default() - }; - - let rules: DatabaseRules = protobuf.clone().try_into().unwrap(); - let back: management::DatabaseRules = rules.clone().into(); - - assert!(rules.primary_query_group.is_none()); - assert_eq!(rules.secondary_query_groups.len(), 0); - assert_eq!(rules.read_only_partitions.len(), 0); - assert_eq!(rules.query_local, false); - - assert_eq!(protobuf.query_config, back.query_config); - } - #[test] fn test_partition_template_default() { let protobuf = management::DatabaseRules { @@ -1317,87 +1274,6 @@ mod tests { assert_eq!(&err.description, "Duration must be positive"); } - #[test] - fn test_matcher_default() { - let protobuf: management::Matcher = Default::default(); - - let res: Result = protobuf.try_into(); - let err = res.expect_err("expected failure"); - - assert_eq!(&err.field, "table_matcher"); - assert_eq!(&err.description, "Field is required"); - } - - #[test] - fn test_matcher() { - let protobuf = management::Matcher { - predicate: Default::default(), - table_matcher: Some(management::matcher::TableMatcher::Regex( - "regex".to_string(), - )), - }; - let matcher: Matcher = protobuf.try_into().unwrap(); - - assert_eq!(matcher.tables, MatchTables::Regex("regex".to_string())); - assert!(matcher.predicate.is_none()); - } - - #[test] - fn test_subscription_default() { - let pb_matcher = Some(management::Matcher { - predicate: "predicate1".to_string(), - table_matcher: Some(management::matcher::TableMatcher::Table( - "table".to_string(), - )), - }); - - let matcher = Matcher { - tables: MatchTables::Table("table".to_string()), - predicate: Some("predicate1".to_string()), - }; - - let subscription_config = management::SubscriptionConfig { - subscriptions: vec![ - management::subscription_config::Subscription { - name: "subscription1".to_string(), - host_group_id: "host group".to_string(), - matcher: pb_matcher.clone(), - }, - management::subscription_config::Subscription { - name: "subscription2".to_string(), - host_group_id: "host group".to_string(), - matcher: pb_matcher, - }, - ], - }; - - let protobuf = management::DatabaseRules { - name: "database".to_string(), - subscription_config: Some(subscription_config), - ..Default::default() - }; - - let rules: DatabaseRules = protobuf.clone().try_into().unwrap(); - let back: management::DatabaseRules = rules.clone().into(); - - assert_eq!(protobuf.subscription_config, back.subscription_config); - assert_eq!( - rules.subscriptions, - vec![ - Subscription { - name: "subscription1".to_string(), - host_group_id: "host group".to_string(), - matcher: matcher.clone() - }, - Subscription { - name: "subscription2".to_string(), - host_group_id: "host group".to_string(), - matcher - } - ] - ) - } - #[test] fn mutable_buffer_config_default() { let protobuf: management::MutableBufferConfig = Default::default(); @@ -1528,4 +1404,128 @@ mod tests { assert_eq!(err3.field, "column.column_name"); assert_eq!(err3.description, "Field is required"); } + + #[test] + fn test_matcher_default() { + let protobuf = management::Matcher { + ..Default::default() + }; + + let matcher: Matcher = protobuf.clone().try_into().unwrap(); + let back: management::Matcher = matcher.clone().into(); + + assert!(matcher.table_name_regex.is_none()); + assert_eq!(protobuf.table_name_regex, back.table_name_regex); + + assert_eq!(matcher.predicate, None); + assert_eq!(protobuf.predicate, back.predicate); + } + + #[test] + fn test_matcher_regexp() { + let protobuf = management::Matcher { + table_name_regex: "^foo$".into(), + ..Default::default() + }; + + let matcher: Matcher = protobuf.clone().try_into().unwrap(); + let back: management::Matcher = matcher.clone().into(); + + assert_eq!(matcher.table_name_regex.unwrap().to_string(), "^foo$"); + assert_eq!(protobuf.table_name_regex, back.table_name_regex); + } + + #[test] + fn test_matcher_bad_regexp() { + let protobuf = management::Matcher { + table_name_regex: "*".into(), + ..Default::default() + }; + + let matcher: Result = protobuf.try_into(); + assert!(matcher.is_err()); + assert_eq!(matcher.err().unwrap().field, "table_name_regex"); + } + + #[test] + fn test_hash_ring_default() { + let protobuf = management::HashRing { + ..Default::default() + }; + + let hash_ring: HashRing = protobuf.clone().try_into().unwrap(); + let back: management::HashRing = hash_ring.clone().into(); + + assert_eq!(hash_ring.table_name, false); + assert_eq!(protobuf.table_name, back.table_name); + assert!(hash_ring.columns.is_empty()); + assert_eq!(protobuf.columns, back.columns); + assert!(hash_ring.node_groups.is_empty()); + assert_eq!(protobuf.node_groups, back.node_groups); + } + + #[test] + fn test_hash_ring_nodes() { + let protobuf = management::HashRing { + node_groups: vec![ + management::NodeGroup { + nodes: vec![ + management::node_group::Node { id: 10 }, + management::node_group::Node { id: 11 }, + management::node_group::Node { id: 12 }, + ], + }, + management::NodeGroup { + nodes: vec![management::node_group::Node { id: 20 }], + }, + ], + ..Default::default() + }; + + let hash_ring: HashRing = protobuf.try_into().unwrap(); + + assert_eq!(hash_ring.node_groups.len(), 2); + assert_eq!(hash_ring.node_groups[0].len(), 3); + assert_eq!(hash_ring.node_groups[1].len(), 1); + } + + #[test] + fn test_matcher_to_targets_default() { + let protobuf = management::MatcherToTargets { + ..Default::default() + }; + + let matcher_to_targets: MatcherToTargets = protobuf.clone().try_into().unwrap(); + let back: management::MatcherToTargets = matcher_to_targets.clone().into(); + + assert_eq!( + matcher_to_targets.matcher, + Matcher { + ..Default::default() + } + ); + assert_eq!(protobuf.matcher, back.matcher); + + assert_eq!(matcher_to_targets.target, Vec::::new()); + assert_eq!(protobuf.target, back.target); + } + + #[test] + fn test_shard_config_default() { + let protobuf = management::ShardConfig { + ..Default::default() + }; + + let shard_config: ShardConfig = protobuf.clone().try_into().unwrap(); + let back: management::ShardConfig = shard_config.clone().into(); + + assert!(shard_config.specific_targets.is_none()); + assert_eq!(protobuf.specific_targets, back.specific_targets); + + assert!(shard_config.hash_ring.is_none()); + assert_eq!(protobuf.hash_ring, back.hash_ring); + + assert_eq!(shard_config.ignore_errors, false); + assert_eq!(protobuf.ignore_errors, back.ignore_errors); + } } diff --git a/data_types/src/http.rs b/data_types/src/http.rs index 2f595fb82a..904c2dc129 100644 --- a/data_types/src/http.rs +++ b/data_types/src/http.rs @@ -18,9 +18,3 @@ pub struct WalMetadataQuery { pub struct WalMetadataResponse { pub segments: Vec, } - -#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)] -/// Body of the response to the /databases endpoint. -pub struct ListDatabasesResponse { - pub names: Vec, -} diff --git a/data_types/src/job.rs b/data_types/src/job.rs new file mode 100644 index 0000000000..0a308bdda9 --- /dev/null +++ b/data_types/src/job.rs @@ -0,0 +1,160 @@ +use generated_types::google::{protobuf::Any, FieldViolation, FieldViolationExt}; +use generated_types::{ + google::longrunning, influxdata::iox::management::v1 as management, protobuf_type_url_eq, +}; +use serde::{Deserialize, Serialize}; +use std::convert::TryFrom; + +/// Metadata associated with a set of background tasks +/// Used in combination with TrackerRegistry +/// +/// TODO: Serde is temporary until prost adds JSON support +#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)] +pub enum Job { + Dummy { + nanos: Vec, + }, + + /// Persist a WAL segment to object store + PersistSegment { + writer_id: u32, + segment_id: u64, + }, + + /// Move a chunk from mutable buffer to read buffer + CloseChunk { + db_name: String, + partition_key: String, + chunk_id: u32, + }, +} + +impl From for management::operation_metadata::Job { + fn from(job: Job) -> Self { + match job { + Job::Dummy { nanos } => Self::Dummy(management::Dummy { nanos }), + Job::PersistSegment { + writer_id, + segment_id, + } => Self::PersistSegment(management::PersistSegment { + writer_id, + segment_id, + }), + Job::CloseChunk { + db_name, + partition_key, + chunk_id, + } => Self::CloseChunk(management::CloseChunk { + db_name, + partition_key, + chunk_id, + }), + } + } +} + +impl From for Job { + fn from(value: management::operation_metadata::Job) -> Self { + use management::operation_metadata::Job; + match value { + Job::Dummy(management::Dummy { nanos }) => Self::Dummy { nanos }, + Job::PersistSegment(management::PersistSegment { + writer_id, + segment_id, + }) => Self::PersistSegment { + writer_id, + segment_id, + }, + Job::CloseChunk(management::CloseChunk { + db_name, + partition_key, + chunk_id, + }) => Self::CloseChunk { + db_name, + partition_key, + chunk_id, + }, + } + } +} + +/// The status of a running operation +#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)] +pub enum OperationStatus { + /// A task associated with the operation is running + Running, + /// All tasks associated with the operation have finished + /// + /// Note: This does not indicate success or failure only that + /// no tasks associated with the operation are running + Complete, + /// The operation was cancelled and no associated tasks are running + Cancelled, + /// An operation error was returned + /// + /// Note: The tracker system currently will never return this + Errored, +} + +/// A group of asynchronous tasks being performed by an IOx server +/// +/// TODO: Temporary until prost adds JSON support +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Operation { + /// ID of the running operation + pub id: usize, + /// Number of subtasks for this operation + pub task_count: u64, + /// Number of pending tasks for this operation + pub pending_count: u64, + /// Wall time spent executing this operation + pub wall_time: std::time::Duration, + /// CPU time spent executing this operation + pub cpu_time: std::time::Duration, + /// Additional job metadata + pub job: Option, + /// The status of the running operation + pub status: OperationStatus, +} + +impl TryFrom for Operation { + type Error = FieldViolation; + + fn try_from(operation: longrunning::Operation) -> Result { + let metadata: Any = operation + .metadata + .ok_or_else(|| FieldViolation::required("metadata"))?; + + if !protobuf_type_url_eq(&metadata.type_url, management::OPERATION_METADATA) { + return Err(FieldViolation { + field: "metadata.type_url".to_string(), + description: "Unexpected field type".to_string(), + }); + } + + let meta: management::OperationMetadata = + prost::Message::decode(metadata.value).field("metadata.value")?; + + let status = match &operation.result { + None => OperationStatus::Running, + Some(longrunning::operation::Result::Response(_)) => OperationStatus::Complete, + Some(longrunning::operation::Result::Error(status)) => { + if status.code == tonic::Code::Cancelled as i32 { + OperationStatus::Cancelled + } else { + OperationStatus::Errored + } + } + }; + + Ok(Self { + id: operation.name.parse().field("name")?, + task_count: meta.task_count, + pending_count: meta.pending_count, + wall_time: std::time::Duration::from_nanos(meta.wall_nanos), + cpu_time: std::time::Duration::from_nanos(meta.cpu_nanos), + job: meta.job.map(Into::into), + status, + }) + } +} diff --git a/data_types/src/lib.rs b/data_types/src/lib.rs index 9b294abaab..1d0333bf34 100644 --- a/data_types/src/lib.rs +++ b/data_types/src/lib.rs @@ -10,28 +10,17 @@ clippy::clone_on_ref_ptr )] -pub use schema::TIME_COLUMN_NAME; +pub use database_name::*; -/// The name of the column containing table names returned by a call to -/// `table_names`. -pub const TABLE_NAMES_COLUMN_NAME: &str = "table"; - -/// The name of the column containing column names returned by a call to -/// `column_names`. -pub const COLUMN_NAMES_COLUMN_NAME: &str = "column"; - -pub mod data; +pub mod chunk; pub mod database_rules; pub mod error; pub mod http; +pub mod job; pub mod names; pub mod partition_metadata; -pub mod schema; -pub mod selection; pub mod timestamp; pub mod wal; mod database_name; -pub use database_name::*; - pub(crate) mod field_validation; diff --git a/docker/Dockerfile.ci b/docker/Dockerfile.ci index 6404c75bbf..dcf337d32d 100644 --- a/docker/Dockerfile.ci +++ b/docker/Dockerfile.ci @@ -9,25 +9,9 @@ # docker build -f docker/Dockerfile.ci . ## -# Build any binaries that can be copied into the CI image -# Note we build flatbuffers from source (pinned to a particualar version) -FROM rust:slim-buster AS flatc -ARG flatbuffers_version="v1.12.0" -RUN apt-get update \ - && mkdir -p /usr/share/man/man1 \ - && apt-get install -y \ - git make clang cmake llvm \ - --no-install-recommends \ - && git clone -b ${flatbuffers_version} -- https://github.com/google/flatbuffers.git /usr/local/src/flatbuffers \ - && cmake -S /usr/local/src/flatbuffers -B /usr/local/src/flatbuffers \ - -G "Unix Makefiles" \ - -DCMAKE_BUILD_TYPE=Release \ - && make -C /usr/local/src/flatbuffers -j $(nproc) flatc - # Build actual image used for CI pipeline FROM rust:slim-buster -COPY --from=flatc /usr/local/src/flatbuffers/flatc /usr/bin/flatc # make Apt non-interactive RUN echo 'APT::Get::Assume-Yes "true";' > /etc/apt/apt.conf.d/90ci \ && echo 'DPkg::Options "--force-confnew";' >> /etc/apt/apt.conf.d/90ci @@ -39,8 +23,7 @@ RUN apt-get update \ && apt-get install -y \ git locales sudo openssh-client ca-certificates tar gzip parallel \ unzip zip bzip2 gnupg curl make pkg-config libssl-dev \ - musl musl-dev musl-tools clang llvm \ - jq \ + jq clang lld \ --no-install-recommends \ && apt-get clean autoclean \ && apt-get autoremove --yes \ diff --git a/docker/Dockerfile.iox b/docker/Dockerfile.iox index e62d8f396d..2e13c4ebf8 100644 --- a/docker/Dockerfile.iox +++ b/docker/Dockerfile.iox @@ -4,7 +4,7 @@ FROM debian:buster-slim RUN apt-get update \ - && apt-get install -y libssl1.1 libgcc1 libc6 \ + && apt-get install -y libssl1.1 libgcc1 libc6 ca-certificates --no-install-recommends \ && rm -rf /var/lib/{apt,dpkg,cache,log} RUN groupadd -g 1500 rust \ @@ -20,3 +20,5 @@ COPY target/release/influxdb_iox /usr/bin/influxdb_iox EXPOSE 8080 8082 ENTRYPOINT ["influxdb_iox"] + +CMD ["run"] diff --git a/docs/README.md b/docs/README.md index a046bf8a60..c14e8f3e52 100644 --- a/docs/README.md +++ b/docs/README.md @@ -4,6 +4,15 @@ This directory contains internal design documentation of potential interest for those who wish to understand how the code works. It is not intended to be general user facing documentation +## IOx Tech Talks + +We hold monthly Tech Talks that explain the project's technical underpinnings. You can register for the [InfluxDB IOx Tech Talks here](https://www.influxdata.com/community-showcase/influxdb-tech-talks/), or you can find links to previous sessions below: + +* December 2020: Rusty Introduction to Apache Arrow [recording](https://www.youtube.com/watch?v=dQFjKa9vKhM) +* Jan 2021: Data Lifecycle in InfluxDB IOx & How it Uses Object Storage for Persistence [recording](https://www.youtube.com/watch?v=KwdPifHC1Gc) +* February 2021: Intro to the InfluxDB IOx Read Buffer [recording](https://www.youtube.com/watch?v=KslD31VNqPU) [slides](https://www.slideshare.net/influxdata/influxdb-iox-tech-talks-intro-to-the-influxdb-iox-read-buffer-a-readoptimized-inmemory-query-execution-engine) +* March 2021: Query Engine Design and the Rust-Based DataFusion in Apache Arrow [recording](https://www.youtube.com/watch?v=K6eCAVEk4kU) [slides](https://www.slideshare.net/influxdata/influxdb-iox-tech-talks-query-engine-design-and-the-rustbased-datafusion-in-apache-arrow-244161934) + ## Table of Contents: * Rust style and Idiom guide: [style_guide.md](style_guide.md) @@ -13,3 +22,5 @@ not intended to be general user facing documentation * Thoughts on using multiple cores: [multi_core_tasks.md](multi_core_tasks.md) * [Query Engine Docs](../query/README.md) * [Testing documentation](testing.md) for developers of IOx +* [Regenerating Flatbuffers code](regenerating_flatbuffers.md) when updating the version of the `flatbuffers` crate + diff --git a/docs/env.example b/docs/env.example index ed4f28a5c6..dc250a3530 100644 --- a/docs/env.example +++ b/docs/env.example @@ -7,7 +7,7 @@ # The full list of available configuration values can be found by in # the command line help (e.g. `env: INFLUXDB_IOX_DB_DIR=`): # -# ./influxdb_iox server --help +# ./influxdb_iox run --help # # # The identifier for the server. Used for writing to object storage and as diff --git a/docs/regenerating_flatbuffers.md b/docs/regenerating_flatbuffers.md new file mode 100644 index 0000000000..1c5f8a07f5 --- /dev/null +++ b/docs/regenerating_flatbuffers.md @@ -0,0 +1,7 @@ +# Regenerating Flatbuffers code + +When updating the version of the [flatbuffers](https://crates.io/crates/flatbuffers) Rust crate used as a dependency in the IOx workspace, the generated Rust code in `generated_types/src/wal_generated.rs` also needs to be updated in sync. + +To update the generated code, edit `generated_types/regenerate-flatbuffers.sh` and set the `FB_COMMIT` variable at the top of the file to the commit SHA of the same commit in the [flatbuffers repository](https://github.com/google/flatbuffers) where the `flatbuffers` Rust crate version was updated. This ensures we'll be [using the same version of `flatc` that the crate was tested with](https://github.com/google/flatbuffers/issues/6199#issuecomment-714562121). + +Then run the `generated_types/regenerate-flatbuffers.sh` script and check in any changes. Check the whole project builds. diff --git a/docs/testing.md b/docs/testing.md index 30c936dfd9..9e424127ec 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -21,14 +21,14 @@ of the object stores, the relevant tests will run. ### Configuration differences when running the tests -When running `influxdb_iox server`, you can pick one object store to use. When running the tests, +When running `influxdb_iox run`, you can pick one object store to use. When running the tests, you can run them against all the possible object stores. There's still only one `INFLUXDB_IOX_BUCKET` variable, though, so that will set the bucket name for all configured object stores. Use the same bucket name when setting up the different services. Other than possibly configuring multiple object stores, configuring the tests to use the object store services is the same as configuring the server to use an object store service. See the output -of `influxdb_iox server --help` for instructions. +of `influxdb_iox run --help` for instructions. ## InfluxDB IOx Client diff --git a/generated_types/.gitignore b/generated_types/.gitignore new file mode 100644 index 0000000000..3954622303 --- /dev/null +++ b/generated_types/.gitignore @@ -0,0 +1,2 @@ +.flatbuffers + diff --git a/generated_types/Cargo.toml b/generated_types/Cargo.toml index f961c86610..af4a631039 100644 --- a/generated_types/Cargo.toml +++ b/generated_types/Cargo.toml @@ -5,12 +5,14 @@ authors = ["Paul Dix "] edition = "2018" [dependencies] # In alphabetical order -flatbuffers = "0.6.1" +# See docs/regenerating_flatbuffers.md about updating generated code when updating the +# version of the flatbuffers crate +flatbuffers = "0.8" futures = "0.3.1" prost = "0.7" prost-types = "0.7" tonic = "0.4" - +tracing = "0.1" google_types = { path = "../google_types" } [build-dependencies] # In alphabetical order diff --git a/generated_types/build.rs b/generated_types/build.rs index 57261e1558..cb5782aa69 100644 --- a/generated_types/build.rs +++ b/generated_types/build.rs @@ -1,10 +1,6 @@ -//! Compiles Protocol Buffers and FlatBuffers schema definitions into -//! native Rust types. +//! Compiles Protocol Buffers into native Rust types. -use std::{ - path::{Path, PathBuf}, - process::Command, -}; +use std::path::{Path, PathBuf}; type Error = Box; type Result = std::result::Result; @@ -13,7 +9,6 @@ fn main() -> Result<()> { let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("protos"); generate_grpc_types(&root)?; - generate_wal_types(&root)?; Ok(()) } @@ -28,7 +23,7 @@ fn generate_grpc_types(root: &Path) -> Result<()> { let storage_path = root.join("influxdata/platform/storage"); let idpe_path = root.join("com/github/influxdata/idpe/storage/read"); let management_path = root.join("influxdata/iox/management/v1"); - let grpc_path = root.join("grpc/health/v1"); + let write_path = root.join("influxdata/iox/write/v1"); let proto_files = vec![ storage_path.join("test.proto"), @@ -39,8 +34,16 @@ fn generate_grpc_types(root: &Path) -> Result<()> { idpe_path.join("source.proto"), management_path.join("base_types.proto"), management_path.join("database_rules.proto"), + management_path.join("chunk.proto"), + management_path.join("partition.proto"), management_path.join("service.proto"), - grpc_path.join("service.proto"), + management_path.join("shard.proto"), + management_path.join("jobs.proto"), + write_path.join("service.proto"), + root.join("grpc/health/v1/service.proto"), + root.join("google/longrunning/operations.proto"), + root.join("google/rpc/error_details.proto"), + root.join("google/rpc/status.proto"), ]; // Tell cargo to recompile if any of these proto files are changed @@ -52,36 +55,10 @@ fn generate_grpc_types(root: &Path) -> Result<()> { config .compile_well_known_types() - .extern_path(".google", "::google_types"); + .disable_comments(&[".google"]) + .extern_path(".google.protobuf", "::google_types::protobuf"); tonic_build::configure().compile_with_config(config, &proto_files, &[root.into()])?; Ok(()) } - -/// Schema used in the WAL -/// -/// Creates `wal_generated.rs` -fn generate_wal_types(root: &Path) -> Result<()> { - let wal_file = root.join("wal.fbs"); - - println!("cargo:rerun-if-changed={}", wal_file.display()); - let out_dir: PathBuf = std::env::var_os("OUT_DIR") - .expect("Could not determine `OUT_DIR`") - .into(); - - let status = Command::new("flatc") - .arg("--rust") - .arg("-o") - .arg(&out_dir) - .arg(wal_file) - .status(); - - match status { - Ok(status) if !status.success() => panic!("`flatc` failed to compile the .fbs to Rust"), - Ok(_status) => {} // Successfully compiled - Err(err) => panic!("Could not execute `flatc`: {}", err), - } - - Ok(()) -} diff --git a/generated_types/protos/google/api/annotations.proto b/generated_types/protos/google/api/annotations.proto new file mode 100644 index 0000000000..18dcf20990 --- /dev/null +++ b/generated_types/protos/google/api/annotations.proto @@ -0,0 +1,31 @@ +// Copyright (c) 2015, Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.api; + +import "google/api/http.proto"; +import "google/protobuf/descriptor.proto"; + +option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations"; +option java_multiple_files = true; +option java_outer_classname = "AnnotationsProto"; +option java_package = "com.google.api"; +option objc_class_prefix = "GAPI"; + +extend google.protobuf.MethodOptions { + // See `HttpRule`. + HttpRule http = 72295728; +} \ No newline at end of file diff --git a/generated_types/protos/google/api/client.proto b/generated_types/protos/google/api/client.proto new file mode 100644 index 0000000000..2102623d30 --- /dev/null +++ b/generated_types/protos/google/api/client.proto @@ -0,0 +1,99 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.api; + +import "google/protobuf/descriptor.proto"; + +option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations"; +option java_multiple_files = true; +option java_outer_classname = "ClientProto"; +option java_package = "com.google.api"; +option objc_class_prefix = "GAPI"; + +extend google.protobuf.MethodOptions { + // A definition of a client library method signature. + // + // In client libraries, each proto RPC corresponds to one or more methods + // which the end user is able to call, and calls the underlying RPC. + // Normally, this method receives a single argument (a struct or instance + // corresponding to the RPC request object). Defining this field will + // add one or more overloads providing flattened or simpler method signatures + // in some languages. + // + // The fields on the method signature are provided as a comma-separated + // string. + // + // For example, the proto RPC and annotation: + // + // rpc CreateSubscription(CreateSubscriptionRequest) + // returns (Subscription) { + // option (google.api.method_signature) = "name,topic"; + // } + // + // Would add the following Java overload (in addition to the method accepting + // the request object): + // + // public final Subscription createSubscription(String name, String topic) + // + // The following backwards-compatibility guidelines apply: + // + // * Adding this annotation to an unannotated method is backwards + // compatible. + // * Adding this annotation to a method which already has existing + // method signature annotations is backwards compatible if and only if + // the new method signature annotation is last in the sequence. + // * Modifying or removing an existing method signature annotation is + // a breaking change. + // * Re-ordering existing method signature annotations is a breaking + // change. + repeated string method_signature = 1051; +} + +extend google.protobuf.ServiceOptions { + // The hostname for this service. + // This should be specified with no prefix or protocol. + // + // Example: + // + // service Foo { + // option (google.api.default_host) = "foo.googleapi.com"; + // ... + // } + string default_host = 1049; + + // OAuth scopes needed for the client. + // + // Example: + // + // service Foo { + // option (google.api.oauth_scopes) = \ + // "https://www.googleapis.com/auth/cloud-platform"; + // ... + // } + // + // If there is more than one scope, use a comma-separated string: + // + // Example: + // + // service Foo { + // option (google.api.oauth_scopes) = \ + // "https://www.googleapis.com/auth/cloud-platform," + // "https://www.googleapis.com/auth/monitoring"; + // ... + // } + string oauth_scopes = 1050; +} diff --git a/generated_types/protos/google/api/http.proto b/generated_types/protos/google/api/http.proto new file mode 100644 index 0000000000..69460cf791 --- /dev/null +++ b/generated_types/protos/google/api/http.proto @@ -0,0 +1,375 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.api; + +option cc_enable_arenas = true; +option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations"; +option java_multiple_files = true; +option java_outer_classname = "HttpProto"; +option java_package = "com.google.api"; +option objc_class_prefix = "GAPI"; + +// Defines the HTTP configuration for an API service. It contains a list of +// [HttpRule][google.api.HttpRule], each specifying the mapping of an RPC method +// to one or more HTTP REST API methods. +message Http { + // A list of HTTP configuration rules that apply to individual API methods. + // + // **NOTE:** All service configuration rules follow "last one wins" order. + repeated HttpRule rules = 1; + + // When set to true, URL path parameters will be fully URI-decoded except in + // cases of single segment matches in reserved expansion, where "%2F" will be + // left encoded. + // + // The default behavior is to not decode RFC 6570 reserved characters in multi + // segment matches. + bool fully_decode_reserved_expansion = 2; +} + +// # gRPC Transcoding +// +// gRPC Transcoding is a feature for mapping between a gRPC method and one or +// more HTTP REST endpoints. It allows developers to build a single API service +// that supports both gRPC APIs and REST APIs. Many systems, including [Google +// APIs](https://github.com/googleapis/googleapis), +// [Cloud Endpoints](https://cloud.google.com/endpoints), [gRPC +// Gateway](https://github.com/grpc-ecosystem/grpc-gateway), +// and [Envoy](https://github.com/envoyproxy/envoy) proxy support this feature +// and use it for large scale production services. +// +// `HttpRule` defines the schema of the gRPC/REST mapping. The mapping specifies +// how different portions of the gRPC request message are mapped to the URL +// path, URL query parameters, and HTTP request body. It also controls how the +// gRPC response message is mapped to the HTTP response body. `HttpRule` is +// typically specified as an `google.api.http` annotation on the gRPC method. +// +// Each mapping specifies a URL path template and an HTTP method. The path +// template may refer to one or more fields in the gRPC request message, as long +// as each field is a non-repeated field with a primitive (non-message) type. +// The path template controls how fields of the request message are mapped to +// the URL path. +// +// Example: +// +// service Messaging { +// rpc GetMessage(GetMessageRequest) returns (Message) { +// option (google.api.http) = { +// get: "/v1/{name=messages/*}" +// }; +// } +// } +// message GetMessageRequest { +// string name = 1; // Mapped to URL path. +// } +// message Message { +// string text = 1; // The resource content. +// } +// +// This enables an HTTP REST to gRPC mapping as below: +// +// HTTP | gRPC +// -----|----- +// `GET /v1/messages/123456` | `GetMessage(name: "messages/123456")` +// +// Any fields in the request message which are not bound by the path template +// automatically become HTTP query parameters if there is no HTTP request body. +// For example: +// +// service Messaging { +// rpc GetMessage(GetMessageRequest) returns (Message) { +// option (google.api.http) = { +// get:"/v1/messages/{message_id}" +// }; +// } +// } +// message GetMessageRequest { +// message SubMessage { +// string subfield = 1; +// } +// string message_id = 1; // Mapped to URL path. +// int64 revision = 2; // Mapped to URL query parameter `revision`. +// SubMessage sub = 3; // Mapped to URL query parameter `sub.subfield`. +// } +// +// This enables a HTTP JSON to RPC mapping as below: +// +// HTTP | gRPC +// -----|----- +// `GET /v1/messages/123456?revision=2&sub.subfield=foo` | +// `GetMessage(message_id: "123456" revision: 2 sub: SubMessage(subfield: +// "foo"))` +// +// Note that fields which are mapped to URL query parameters must have a +// primitive type or a repeated primitive type or a non-repeated message type. +// In the case of a repeated type, the parameter can be repeated in the URL +// as `...?param=A¶m=B`. In the case of a message type, each field of the +// message is mapped to a separate parameter, such as +// `...?foo.a=A&foo.b=B&foo.c=C`. +// +// For HTTP methods that allow a request body, the `body` field +// specifies the mapping. Consider a REST update method on the +// message resource collection: +// +// service Messaging { +// rpc UpdateMessage(UpdateMessageRequest) returns (Message) { +// option (google.api.http) = { +// patch: "/v1/messages/{message_id}" +// body: "message" +// }; +// } +// } +// message UpdateMessageRequest { +// string message_id = 1; // mapped to the URL +// Message message = 2; // mapped to the body +// } +// +// The following HTTP JSON to RPC mapping is enabled, where the +// representation of the JSON in the request body is determined by +// protos JSON encoding: +// +// HTTP | gRPC +// -----|----- +// `PATCH /v1/messages/123456 { "text": "Hi!" }` | `UpdateMessage(message_id: +// "123456" message { text: "Hi!" })` +// +// The special name `*` can be used in the body mapping to define that +// every field not bound by the path template should be mapped to the +// request body. This enables the following alternative definition of +// the update method: +// +// service Messaging { +// rpc UpdateMessage(Message) returns (Message) { +// option (google.api.http) = { +// patch: "/v1/messages/{message_id}" +// body: "*" +// }; +// } +// } +// message Message { +// string message_id = 1; +// string text = 2; +// } +// +// +// The following HTTP JSON to RPC mapping is enabled: +// +// HTTP | gRPC +// -----|----- +// `PATCH /v1/messages/123456 { "text": "Hi!" }` | `UpdateMessage(message_id: +// "123456" text: "Hi!")` +// +// Note that when using `*` in the body mapping, it is not possible to +// have HTTP parameters, as all fields not bound by the path end in +// the body. This makes this option more rarely used in practice when +// defining REST APIs. The common usage of `*` is in custom methods +// which don't use the URL at all for transferring data. +// +// It is possible to define multiple HTTP methods for one RPC by using +// the `additional_bindings` option. Example: +// +// service Messaging { +// rpc GetMessage(GetMessageRequest) returns (Message) { +// option (google.api.http) = { +// get: "/v1/messages/{message_id}" +// additional_bindings { +// get: "/v1/users/{user_id}/messages/{message_id}" +// } +// }; +// } +// } +// message GetMessageRequest { +// string message_id = 1; +// string user_id = 2; +// } +// +// This enables the following two alternative HTTP JSON to RPC mappings: +// +// HTTP | gRPC +// -----|----- +// `GET /v1/messages/123456` | `GetMessage(message_id: "123456")` +// `GET /v1/users/me/messages/123456` | `GetMessage(user_id: "me" message_id: +// "123456")` +// +// ## Rules for HTTP mapping +// +// 1. Leaf request fields (recursive expansion nested messages in the request +// message) are classified into three categories: +// - Fields referred by the path template. They are passed via the URL path. +// - Fields referred by the [HttpRule.body][google.api.HttpRule.body]. They are passed via the HTTP +// request body. +// - All other fields are passed via the URL query parameters, and the +// parameter name is the field path in the request message. A repeated +// field can be represented as multiple query parameters under the same +// name. +// 2. If [HttpRule.body][google.api.HttpRule.body] is "*", there is no URL query parameter, all fields +// are passed via URL path and HTTP request body. +// 3. If [HttpRule.body][google.api.HttpRule.body] is omitted, there is no HTTP request body, all +// fields are passed via URL path and URL query parameters. +// +// ### Path template syntax +// +// Template = "/" Segments [ Verb ] ; +// Segments = Segment { "/" Segment } ; +// Segment = "*" | "**" | LITERAL | Variable ; +// Variable = "{" FieldPath [ "=" Segments ] "}" ; +// FieldPath = IDENT { "." IDENT } ; +// Verb = ":" LITERAL ; +// +// The syntax `*` matches a single URL path segment. The syntax `**` matches +// zero or more URL path segments, which must be the last part of the URL path +// except the `Verb`. +// +// The syntax `Variable` matches part of the URL path as specified by its +// template. A variable template must not contain other variables. If a variable +// matches a single path segment, its template may be omitted, e.g. `{var}` +// is equivalent to `{var=*}`. +// +// The syntax `LITERAL` matches literal text in the URL path. If the `LITERAL` +// contains any reserved character, such characters should be percent-encoded +// before the matching. +// +// If a variable contains exactly one path segment, such as `"{var}"` or +// `"{var=*}"`, when such a variable is expanded into a URL path on the client +// side, all characters except `[-_.~0-9a-zA-Z]` are percent-encoded. The +// server side does the reverse decoding. Such variables show up in the +// [Discovery +// Document](https://developers.google.com/discovery/v1/reference/apis) as +// `{var}`. +// +// If a variable contains multiple path segments, such as `"{var=foo/*}"` +// or `"{var=**}"`, when such a variable is expanded into a URL path on the +// client side, all characters except `[-_.~/0-9a-zA-Z]` are percent-encoded. +// The server side does the reverse decoding, except "%2F" and "%2f" are left +// unchanged. Such variables show up in the +// [Discovery +// Document](https://developers.google.com/discovery/v1/reference/apis) as +// `{+var}`. +// +// ## Using gRPC API Service Configuration +// +// gRPC API Service Configuration (service config) is a configuration language +// for configuring a gRPC service to become a user-facing product. The +// service config is simply the YAML representation of the `google.api.Service` +// proto message. +// +// As an alternative to annotating your proto file, you can configure gRPC +// transcoding in your service config YAML files. You do this by specifying a +// `HttpRule` that maps the gRPC method to a REST endpoint, achieving the same +// effect as the proto annotation. This can be particularly useful if you +// have a proto that is reused in multiple services. Note that any transcoding +// specified in the service config will override any matching transcoding +// configuration in the proto. +// +// Example: +// +// http: +// rules: +// # Selects a gRPC method and applies HttpRule to it. +// - selector: example.v1.Messaging.GetMessage +// get: /v1/messages/{message_id}/{sub.subfield} +// +// ## Special notes +// +// When gRPC Transcoding is used to map a gRPC to JSON REST endpoints, the +// proto to JSON conversion must follow the [proto3 +// specification](https://developers.google.com/protocol-buffers/docs/proto3#json). +// +// While the single segment variable follows the semantics of +// [RFC 6570](https://tools.ietf.org/html/rfc6570) Section 3.2.2 Simple String +// Expansion, the multi segment variable **does not** follow RFC 6570 Section +// 3.2.3 Reserved Expansion. The reason is that the Reserved Expansion +// does not expand special characters like `?` and `#`, which would lead +// to invalid URLs. As the result, gRPC Transcoding uses a custom encoding +// for multi segment variables. +// +// The path variables **must not** refer to any repeated or mapped field, +// because client libraries are not capable of handling such variable expansion. +// +// The path variables **must not** capture the leading "/" character. The reason +// is that the most common use case "{var}" does not capture the leading "/" +// character. For consistency, all path variables must share the same behavior. +// +// Repeated message fields must not be mapped to URL query parameters, because +// no client library can support such complicated mapping. +// +// If an API needs to use a JSON array for request or response body, it can map +// the request or response body to a repeated field. However, some gRPC +// Transcoding implementations may not support this feature. +message HttpRule { + // Selects a method to which this rule applies. + // + // Refer to [selector][google.api.DocumentationRule.selector] for syntax details. + string selector = 1; + + // Determines the URL pattern is matched by this rules. This pattern can be + // used with any of the {get|put|post|delete|patch} methods. A custom method + // can be defined using the 'custom' field. + oneof pattern { + // Maps to HTTP GET. Used for listing and getting information about + // resources. + string get = 2; + + // Maps to HTTP PUT. Used for replacing a resource. + string put = 3; + + // Maps to HTTP POST. Used for creating a resource or performing an action. + string post = 4; + + // Maps to HTTP DELETE. Used for deleting a resource. + string delete = 5; + + // Maps to HTTP PATCH. Used for updating a resource. + string patch = 6; + + // The custom pattern is used for specifying an HTTP method that is not + // included in the `pattern` field, such as HEAD, or "*" to leave the + // HTTP method unspecified for this rule. The wild-card rule is useful + // for services that provide content to Web (HTML) clients. + CustomHttpPattern custom = 8; + } + + // The name of the request field whose value is mapped to the HTTP request + // body, or `*` for mapping all request fields not captured by the path + // pattern to the HTTP body, or omitted for not having any HTTP request body. + // + // NOTE: the referred field must be present at the top-level of the request + // message type. + string body = 7; + + // Optional. The name of the response field whose value is mapped to the HTTP + // response body. When omitted, the entire response message will be used + // as the HTTP response body. + // + // NOTE: The referred field must be present at the top-level of the response + // message type. + string response_body = 12; + + // Additional HTTP bindings for the selector. Nested bindings must + // not contain an `additional_bindings` field themselves (that is, + // the nesting may only be one level deep). + repeated HttpRule additional_bindings = 11; +} + +// A custom pattern is used for defining custom HTTP verb. +message CustomHttpPattern { + // The name of this custom HTTP verb. + string kind = 1; + + // The path matched by this custom verb. + string path = 2; +} diff --git a/generated_types/protos/google/longrunning/operations.proto b/generated_types/protos/google/longrunning/operations.proto new file mode 100644 index 0000000000..c1fdc6f529 --- /dev/null +++ b/generated_types/protos/google/longrunning/operations.proto @@ -0,0 +1,247 @@ +// Copyright 2020 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package google.longrunning; + +import "google/api/annotations.proto"; +import "google/api/client.proto"; +import "google/protobuf/any.proto"; +import "google/protobuf/duration.proto"; +import "google/protobuf/empty.proto"; +import "google/rpc/status.proto"; +import "google/protobuf/descriptor.proto"; + +option cc_enable_arenas = true; +option csharp_namespace = "Google.LongRunning"; +option go_package = "google.golang.org/genproto/googleapis/longrunning;longrunning"; +option java_multiple_files = true; +option java_outer_classname = "OperationsProto"; +option java_package = "com.google.longrunning"; +option php_namespace = "Google\\LongRunning"; + +extend google.protobuf.MethodOptions { + // Additional information regarding long-running operations. + // In particular, this specifies the types that are returned from + // long-running operations. + // + // Required for methods that return `google.longrunning.Operation`; invalid + // otherwise. + google.longrunning.OperationInfo operation_info = 1049; +} + +// Manages long-running operations with an API service. +// +// When an API method normally takes long time to complete, it can be designed +// to return [Operation][google.longrunning.Operation] to the client, and the client can use this +// interface to receive the real response asynchronously by polling the +// operation resource, or pass the operation resource to another API (such as +// Google Cloud Pub/Sub API) to receive the response. Any API service that +// returns long-running operations should implement the `Operations` interface +// so developers can have a consistent client experience. +service Operations { + option (google.api.default_host) = "longrunning.googleapis.com"; + + // Lists operations that match the specified filter in the request. If the + // server doesn't support this method, it returns `UNIMPLEMENTED`. + // + // NOTE: the `name` binding allows API services to override the binding + // to use different resource name schemes, such as `users/*/operations`. To + // override the binding, API services can add a binding such as + // `"/v1/{name=users/*}/operations"` to their service configuration. + // For backwards compatibility, the default name includes the operations + // collection id, however overriding users must ensure the name binding + // is the parent resource, without the operations collection id. + rpc ListOperations(ListOperationsRequest) returns (ListOperationsResponse) { + option (google.api.http) = { + get: "/v1/{name=operations}" + }; + option (google.api.method_signature) = "name,filter"; + } + + // Gets the latest state of a long-running operation. Clients can use this + // method to poll the operation result at intervals as recommended by the API + // service. + rpc GetOperation(GetOperationRequest) returns (Operation) { + option (google.api.http) = { + get: "/v1/{name=operations/**}" + }; + option (google.api.method_signature) = "name"; + } + + // Deletes a long-running operation. This method indicates that the client is + // no longer interested in the operation result. It does not cancel the + // operation. If the server doesn't support this method, it returns + // `google.rpc.Code.UNIMPLEMENTED`. + rpc DeleteOperation(DeleteOperationRequest) returns (google.protobuf.Empty) { + option (google.api.http) = { + delete: "/v1/{name=operations/**}" + }; + option (google.api.method_signature) = "name"; + } + + // Starts asynchronous cancellation on a long-running operation. The server + // makes a best effort to cancel the operation, but success is not + // guaranteed. If the server doesn't support this method, it returns + // `google.rpc.Code.UNIMPLEMENTED`. Clients can use + // [Operations.GetOperation][google.longrunning.Operations.GetOperation] or + // other methods to check whether the cancellation succeeded or whether the + // operation completed despite cancellation. On successful cancellation, + // the operation is not deleted; instead, it becomes an operation with + // an [Operation.error][google.longrunning.Operation.error] value with a [google.rpc.Status.code][google.rpc.Status.code] of 1, + // corresponding to `Code.CANCELLED`. + rpc CancelOperation(CancelOperationRequest) returns (google.protobuf.Empty) { + option (google.api.http) = { + post: "/v1/{name=operations/**}:cancel" + body: "*" + }; + option (google.api.method_signature) = "name"; + } + + // Waits until the specified long-running operation is done or reaches at most + // a specified timeout, returning the latest state. If the operation is + // already done, the latest state is immediately returned. If the timeout + // specified is greater than the default HTTP/RPC timeout, the HTTP/RPC + // timeout is used. If the server does not support this method, it returns + // `google.rpc.Code.UNIMPLEMENTED`. + // Note that this method is on a best-effort basis. It may return the latest + // state before the specified timeout (including immediately), meaning even an + // immediate response is no guarantee that the operation is done. + rpc WaitOperation(WaitOperationRequest) returns (Operation) { + } +} + +// This resource represents a long-running operation that is the result of a +// network API call. +message Operation { + // The server-assigned name, which is only unique within the same service that + // originally returns it. If you use the default HTTP mapping, the + // `name` should be a resource name ending with `operations/{unique_id}`. + string name = 1; + + // Service-specific metadata associated with the operation. It typically + // contains progress information and common metadata such as create time. + // Some services might not provide such metadata. Any method that returns a + // long-running operation should document the metadata type, if any. + google.protobuf.Any metadata = 2; + + // If the value is `false`, it means the operation is still in progress. + // If `true`, the operation is completed, and either `error` or `response` is + // available. + bool done = 3; + + // The operation result, which can be either an `error` or a valid `response`. + // If `done` == `false`, neither `error` nor `response` is set. + // If `done` == `true`, exactly one of `error` or `response` is set. + oneof result { + // The error result of the operation in case of failure or cancellation. + google.rpc.Status error = 4; + + // The normal response of the operation in case of success. If the original + // method returns no data on success, such as `Delete`, the response is + // `google.protobuf.Empty`. If the original method is standard + // `Get`/`Create`/`Update`, the response should be the resource. For other + // methods, the response should have the type `XxxResponse`, where `Xxx` + // is the original method name. For example, if the original method name + // is `TakeSnapshot()`, the inferred response type is + // `TakeSnapshotResponse`. + google.protobuf.Any response = 5; + } +} + +// The request message for [Operations.GetOperation][google.longrunning.Operations.GetOperation]. +message GetOperationRequest { + // The name of the operation resource. + string name = 1; +} + +// The request message for [Operations.ListOperations][google.longrunning.Operations.ListOperations]. +message ListOperationsRequest { + // The name of the operation's parent resource. + string name = 4; + + // The standard list filter. + string filter = 1; + + // The standard list page size. + int32 page_size = 2; + + // The standard list page token. + string page_token = 3; +} + +// The response message for [Operations.ListOperations][google.longrunning.Operations.ListOperations]. +message ListOperationsResponse { + // A list of operations that matches the specified filter in the request. + repeated Operation operations = 1; + + // The standard List next-page token. + string next_page_token = 2; +} + +// The request message for [Operations.CancelOperation][google.longrunning.Operations.CancelOperation]. +message CancelOperationRequest { + // The name of the operation resource to be cancelled. + string name = 1; +} + +// The request message for [Operations.DeleteOperation][google.longrunning.Operations.DeleteOperation]. +message DeleteOperationRequest { + // The name of the operation resource to be deleted. + string name = 1; +} + +// The request message for [Operations.WaitOperation][google.longrunning.Operations.WaitOperation]. +message WaitOperationRequest { + // The name of the operation resource to wait on. + string name = 1; + + // The maximum duration to wait before timing out. If left blank, the wait + // will be at most the time permitted by the underlying HTTP/RPC protocol. + // If RPC context deadline is also specified, the shorter one will be used. + google.protobuf.Duration timeout = 2; +} + +// A message representing the message types used by a long-running operation. +// +// Example: +// +// rpc LongRunningRecognize(LongRunningRecognizeRequest) +// returns (google.longrunning.Operation) { +// option (google.longrunning.operation_info) = { +// response_type: "LongRunningRecognizeResponse" +// metadata_type: "LongRunningRecognizeMetadata" +// }; +// } +message OperationInfo { + // Required. The message name of the primary return type for this + // long-running operation. + // This type will be used to deserialize the LRO's response. + // + // If the response is in a different package from the rpc, a fully-qualified + // message name must be used (e.g. `google.protobuf.Struct`). + // + // Note: Altering this value constitutes a breaking change. + string response_type = 1; + + // Required. The message name of the metadata type for this long-running + // operation. + // + // If the response is in a different package from the rpc, a fully-qualified + // message name must be used (e.g. `google.protobuf.Struct`). + // + // Note: Altering this value constitutes a breaking change. + string metadata_type = 2; +} diff --git a/google_types/protos/google/rpc/error_details.proto b/generated_types/protos/google/rpc/error_details.proto similarity index 100% rename from google_types/protos/google/rpc/error_details.proto rename to generated_types/protos/google/rpc/error_details.proto diff --git a/google_types/protos/google/rpc/status.proto b/generated_types/protos/google/rpc/status.proto similarity index 100% rename from google_types/protos/google/rpc/status.proto rename to generated_types/protos/google/rpc/status.proto diff --git a/generated_types/protos/influxdata/iox/management/v1/base_types.proto b/generated_types/protos/influxdata/iox/management/v1/base_types.proto index 90398b778a..53b9dc73d2 100644 --- a/generated_types/protos/influxdata/iox/management/v1/base_types.proto +++ b/generated_types/protos/influxdata/iox/management/v1/base_types.proto @@ -21,10 +21,3 @@ enum ColumnType { COLUMN_TYPE_STRING = 4; COLUMN_TYPE_BOOL = 5; } - -message HostGroup { - string id = 1; - - // connection strings for remote hosts. - repeated string hosts = 2; -} diff --git a/generated_types/protos/influxdata/iox/management/v1/chunk.proto b/generated_types/protos/influxdata/iox/management/v1/chunk.proto new file mode 100644 index 0000000000..23b30317b2 --- /dev/null +++ b/generated_types/protos/influxdata/iox/management/v1/chunk.proto @@ -0,0 +1,37 @@ +syntax = "proto3"; +package influxdata.iox.management.v1; + + + // Which storage system is a chunk located in? +enum ChunkStorage { + // Not currently returned + CHUNK_STORAGE_UNSPECIFIED = 0; + + // The chunk is still open for new writes, in the Mutable Buffer + CHUNK_STORAGE_OPEN_MUTABLE_BUFFER = 1; + + // The chunk is no longer open for writes, in the Mutable Buffer + CHUNK_STORAGE_CLOSED_MUTABLE_BUFFER = 2; + + // The chunk is in the Read Buffer (where it can not be mutated) + CHUNK_STORAGE_READ_BUFFER = 3; + + // The chunk is stored in Object Storage (where it can not be mutated) + CHUNK_STORAGE_OBJECT_STORE = 4; +} + +// `Chunk` represents part of a partition of data in a database. +// A chunk can contain one or more tables. +message Chunk { + // The partitition key of this chunk + string partition_key = 1; + + // The id of this chunk + uint32 id = 2; + + // Which storage system the chunk is located in + ChunkStorage storage = 3; + + // The total estimated size of this chunk, in bytes + uint64 estimated_bytes = 4; +} diff --git a/generated_types/protos/influxdata/iox/management/v1/database_rules.proto b/generated_types/protos/influxdata/iox/management/v1/database_rules.proto index df953f4224..3a7fd2c566 100644 --- a/generated_types/protos/influxdata/iox/management/v1/database_rules.proto +++ b/generated_types/protos/influxdata/iox/management/v1/database_rules.proto @@ -31,90 +31,6 @@ message PartitionTemplate { repeated Part parts = 1; } -message Matcher { - // A query predicate to filter rows - string predicate = 1; - // Restrict selection to a specific table or tables specified by a regex - oneof table_matcher { - google.protobuf.Empty all = 2; - string table = 3; - string regex = 4; - } -} - -message ReplicationConfig { - // The set of host groups that data should be replicated to. Which host a - // write goes to within a host group is determined by consistent hashing of - // the partition key. We'd use this to create a host group per - // availability zone, so you might have 5 availability zones with 2 - // hosts in each. Replication will ensure that N of those zones get a - // write. For each zone, only a single host needs to get the write. - // Replication is for ensuring a write exists across multiple hosts - // before returning success. Its purpose is to ensure write durability, - // rather than write availability for query (this is covered by - // subscriptions). - repeated string replications = 1; - - // The minimum number of host groups to replicate a write to before success - // is returned. This can be overridden on a per request basis. - // Replication will continue to write to the other host groups in the - // background. - uint32 replication_count = 2; - - // How long the replication queue can get before either rejecting writes or - // dropping missed writes. The queue is kept in memory on a - // per-database basis. A queue size of zero means it will only try to - // replicate synchronously and drop any failures. - uint64 replication_queue_max_size = 3; -} - -message SubscriptionConfig { - message Subscription { - string name = 1; - string host_group_id = 2; - Matcher matcher = 3; - } - - // `subscriptions` are used for query servers to get data via either push - // or pull as it arrives. They are separate from replication as they - // have a different purpose. They're for query servers or other clients - // that want to subscribe to some subset of data being written in. This - // could either be specific partitions, ranges of partitions, tables, or - // rows matching some predicate. - repeated Subscription subscriptions = 1; -} - -message QueryConfig { - // If set to `true`, this server should answer queries from one or more of - // of its local write buffer and any read-only partitions that it knows - // about. In this case, results will be merged with any others from the - // remote goups or read-only partitions. - bool query_local = 1; - - // Set `primary` to a host group if remote servers should be - // issued queries for this database. All hosts in the group should be - // queried with this server acting as the coordinator that merges - // results together. - string primary = 2; - - // If a specific host in the primary group is unavailable, - // another host in the same position from a secondary group should be - // queried. For example, imagine we've partitioned the data in this DB into - // 4 partitions and we are replicating the data across 3 availability - // zones. We have 4 hosts in each of those AZs, thus they each have 1 - // partition. We'd set the primary group to be the 4 hosts in the same - // AZ as this one, and the secondary groups as the hosts in the other 2 AZs. - repeated string secondaries = 3; - - // Use `readOnlyPartitions` when a server should answer queries for - // partitions that come from object storage. This can be used to start - // up a new query server to handle queries by pointing it at a - // collection of partitions and then telling it to also pull - // data from the replication servers (writes that haven't been snapshotted - // into a partition). - repeated string read_only_partitions = 4; -} - message WalBufferConfig { enum Rollover { ROLLOVER_UNSPECIFIED = 0; @@ -231,15 +147,6 @@ message DatabaseRules { // Template that generates a partition key for each row inserted into the database PartitionTemplate partition_template = 2; - // Synchronous replication configuration for this database - ReplicationConfig replication_config = 3; - - // Asynchronous pull-based subscription configuration for this database - SubscriptionConfig subscription_config = 4; - - // Query configuration for this database - QueryConfig query_config = 5; - // WAL configuration for this database WalBufferConfig wal_buffer_config = 6; diff --git a/generated_types/protos/influxdata/iox/management/v1/jobs.proto b/generated_types/protos/influxdata/iox/management/v1/jobs.proto new file mode 100644 index 0000000000..a04ecc259f --- /dev/null +++ b/generated_types/protos/influxdata/iox/management/v1/jobs.proto @@ -0,0 +1,47 @@ +syntax = "proto3"; +package influxdata.iox.management.v1; + +message OperationMetadata { + // How many nanoseconds of CPU time have been spent on this job so far? + uint64 cpu_nanos = 1; + + // How many nanoseconds has it been since the job was submitted + uint64 wall_nanos = 2; + + // How many total tasks does this job have currently + uint64 task_count = 3; + + // How many tasks for this job are still pending + uint64 pending_count = 4; + + // What kind of job is it? + oneof job { + Dummy dummy = 5; + PersistSegment persist_segment = 6; + CloseChunk close_chunk = 7; + } +} + +// A job that simply sleeps for a specified time and then returns success +message Dummy { + // How long the job should sleep for before returning + repeated uint64 nanos = 1; +} + +// A job that persists a WAL segment to object store +message PersistSegment { + uint32 writer_id = 1; + uint64 segment_id = 2; +} + +// Move a chunk from mutable buffer to read buffer +message CloseChunk { + // name of the database + string db_name = 1; + + // partition key + string partition_key = 2; + + // chunk_id + uint32 chunk_id = 3; +} diff --git a/generated_types/protos/influxdata/iox/management/v1/partition.proto b/generated_types/protos/influxdata/iox/management/v1/partition.proto new file mode 100644 index 0000000000..9498cf6f8a --- /dev/null +++ b/generated_types/protos/influxdata/iox/management/v1/partition.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; +package influxdata.iox.management.v1; + + +// `Partition` is comprised of data in one or more chunks +// +// TODO: add additional information to this structure (e.g. partition +// names, stats, etc) +message Partition { + // The partitition key of this partition + string key = 1; +} diff --git a/generated_types/protos/influxdata/iox/management/v1/service.proto b/generated_types/protos/influxdata/iox/management/v1/service.proto index 73735431c1..d02999bad3 100644 --- a/generated_types/protos/influxdata/iox/management/v1/service.proto +++ b/generated_types/protos/influxdata/iox/management/v1/service.proto @@ -1,8 +1,10 @@ syntax = "proto3"; package influxdata.iox.management.v1; -import "google/protobuf/empty.proto"; +import "google/longrunning/operations.proto"; import "influxdata/iox/management/v1/database_rules.proto"; +import "influxdata/iox/management/v1/chunk.proto"; +import "influxdata/iox/management/v1/partition.proto"; service ManagementService { rpc GetWriterId(GetWriterIdRequest) returns (GetWriterIdResponse); @@ -14,6 +16,43 @@ service ManagementService { rpc GetDatabase(GetDatabaseRequest) returns (GetDatabaseResponse); rpc CreateDatabase(CreateDatabaseRequest) returns (CreateDatabaseResponse); + + // List chunks available on this database + rpc ListChunks(ListChunksRequest) returns (ListChunksResponse); + + // List remote IOx servers we know about. + rpc ListRemotes(ListRemotesRequest) returns (ListRemotesResponse); + + // Update information about a remote IOx server (upsert). + rpc UpdateRemote(UpdateRemoteRequest) returns (UpdateRemoteResponse); + + // Delete a reference to remote IOx server. + rpc DeleteRemote(DeleteRemoteRequest) returns (DeleteRemoteResponse); + + // Creates a dummy job that for each value of the nanos field + // spawns a task that sleeps for that number of nanoseconds before returning + rpc CreateDummyJob(CreateDummyJobRequest) returns (CreateDummyJobResponse) { + option (google.longrunning.operation_info) = { + response_type: "google.protobuf.Empty" + metadata_type: "OperationMetadata" + }; + } + + // List partitions in a database + rpc ListPartitions(ListPartitionsRequest) returns (ListPartitionsResponse); + + // Get detail information about a partition + rpc GetPartition(GetPartitionRequest) returns (GetPartitionResponse); + + // List chunks in a partition + rpc ListPartitionChunks(ListPartitionChunksRequest) returns (ListPartitionChunksResponse); + + // Create a new chunk in the mutable buffer + rpc NewPartitionChunk(NewPartitionChunkRequest) returns (NewPartitionChunkResponse); + + // Close a chunk and move it to the read buffer + rpc ClosePartitionChunk(ClosePartitionChunkRequest) returns (ClosePartitionChunkResponse); + } message GetWriterIdRequest {} @@ -47,3 +86,121 @@ message CreateDatabaseRequest { } message CreateDatabaseResponse {} + +message ListChunksRequest { + // the name of the database + string db_name = 1; +} + +message ListChunksResponse { + repeated Chunk chunks = 1; +} + +message CreateDummyJobRequest { + repeated uint64 nanos = 1; +} + +message CreateDummyJobResponse { + google.longrunning.Operation operation = 1; +} + +message ListRemotesRequest {} + +message ListRemotesResponse { + repeated Remote remotes = 1; +} + +// This resource represents a remote IOx server. +message Remote { + // The writer ID associated with a remote IOx server. + uint32 id = 1; + + // The address of the remote IOx server gRPC endpoint. + string connection_string = 2; +} + +// Updates information about a remote IOx server. +// +// If a remote for a given `id` already exists, it is updated in place. +message UpdateRemoteRequest { + // If omitted, the remote associated with `id` will be removed. + Remote remote = 1; + + // TODO(#917): add an optional flag to test the connection or not before adding it. +} + +message UpdateRemoteResponse {} + +message DeleteRemoteRequest{ + uint32 id = 1; +} + +message DeleteRemoteResponse {} + +// Request to list all partitions from a named database +message ListPartitionsRequest { + // the name of the database + string db_name = 1; +} + +message ListPartitionsResponse { + // All partitions in a database + repeated Partition partitions = 1; +} + +// Request to list all chunks in a specific partitions from a named database +message ListPartitionChunksRequest { + // the name of the database + string db_name = 1; + + // the partition key + string partition_key = 2; +} + +message GetPartitionResponse { + // Detailed information about a partition + Partition partition = 1; +} + +message ListPartitionChunksResponse { + // All chunks in a partition + repeated Chunk chunks = 1; +} + +// Request to get details of a specific partition from a named database +message GetPartitionRequest { + // the name of the database + string db_name = 1; + + // the partition key + string partition_key = 2; +} + +// Request that a new chunk for writing is created in the mutable buffer +message NewPartitionChunkRequest { + // the name of the database + string db_name = 1; + + // the partition key + string partition_key = 2; +} + +message NewPartitionChunkResponse { +} + +// Request that a chunk be closed and moved to the read buffer +message ClosePartitionChunkRequest { + // the name of the database + string db_name = 1; + + // the partition key + string partition_key = 2; + + // the chunk id + uint32 chunk_id = 3; +} + +message ClosePartitionChunkResponse { + // The operation that tracks the work for migrating the chunk + google.longrunning.Operation operation = 1; +} diff --git a/generated_types/protos/influxdata/iox/management/v1/shard.proto b/generated_types/protos/influxdata/iox/management/v1/shard.proto new file mode 100644 index 0000000000..eb081287d3 --- /dev/null +++ b/generated_types/protos/influxdata/iox/management/v1/shard.proto @@ -0,0 +1,68 @@ +syntax = "proto3"; +package influxdata.iox.management.v1; + +// NOTE: documentation is manually synced from data_types/src/database_rules.rs + +// `ShardConfig` defines rules for assigning a line/row to an individual +// host or a group of hosts. A shard +// is a logical concept, but the usage is meant to split data into +// mutually exclusive areas. The rough order of organization is: +// database -> shard -> partition -> chunk. For example, you could shard +// based on table name and assign to 1 of 10 shards. Within each +// shard you would have partitions, which would likely be based off time. +// This makes it possible to horizontally scale out writes. +message ShardConfig { + /// An optional matcher. If there is a match, the route will be evaluated to + /// the given targets, otherwise the hash ring will be evaluated. This is + /// useful for overriding the hashring function on some hot spot. For + /// example, if you use the table name as the input to the hash function + /// and your ring has 4 slots. If two tables that are very hot get + /// assigned to the same slot you can override that by putting in a + /// specific matcher to pull that table over to a different node. + MatcherToTargets specific_targets = 1; + + /// An optional default hasher which will route to one in a collection of + /// nodes. + HashRing hash_ring = 2; + + /// If set to true the router will ignore any errors sent by the remote + /// targets in this route. That is, the write request will succeed + /// regardless of this route's success. + bool ignore_errors = 3; +} + +// Maps a matcher with specific target group. If the line/row matches +// it should be sent to the group. +message MatcherToTargets { + Matcher matcher = 1; + NodeGroup target = 2; +} + +/// A matcher is used to match routing rules or subscriptions on a row-by-row +/// (or line) basis. +message Matcher { + // if provided, match if the table name matches against the regex + string table_name_regex = 1; + // paul: what should we use for predicate matching here against a single row/line? + string predicate = 2; +} + +// A collection of IOx nodes +message NodeGroup { + message Node { + uint32 id = 1; + } + repeated Node nodes = 1; +} + +// HashRing is a rule for creating a hash key for a row and mapping that to +// an individual node on a ring. +message HashRing { + // If true the table name will be included in the hash key + bool table_name = 1; + // include the values of these columns in the hash key + repeated string columns = 2; + // ring of node groups. Each group holds a shard + repeated NodeGroup node_groups = 3; +} + diff --git a/generated_types/protos/influxdata/iox/write/v1/service.proto b/generated_types/protos/influxdata/iox/write/v1/service.proto new file mode 100644 index 0000000000..c0fa0a9cbf --- /dev/null +++ b/generated_types/protos/influxdata/iox/write/v1/service.proto @@ -0,0 +1,23 @@ +syntax = "proto3"; +package influxdata.iox.write.v1; + + +service WriteService { + // write data into a specific Database + rpc Write(WriteRequest) returns (WriteResponse); +} + +message WriteRequest { + // name of database into which to write + string db_name = 1; + + // data, in [LineProtocol] format + // + // [LineProtocol](https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/#data-types-and-format) + string lp_data = 2; +} + +message WriteResponse { + // how many lines were parsed and written into the database + uint64 lines_written = 1; +} diff --git a/generated_types/regenerate-flatbuffers.sh b/generated_types/regenerate-flatbuffers.sh new file mode 100755 index 0000000000..57fc2b8271 --- /dev/null +++ b/generated_types/regenerate-flatbuffers.sh @@ -0,0 +1,49 @@ +#!/bin/bash -e + +# The commit where the Rust `flatbuffers` crate version was changed to the version in `Cargo.lock` +# Update this, rerun this script, and check in the changes in the generated code when the +# `flatbuffers` crate version is updated. +FB_COMMIT="86401e078d0746d2381735415f8c2dfe849f3f52" + +# Change to the generated_types crate directory, where this script is located +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +pushd $DIR + +echo "Building flatc from source ..." + +FB_URL="https://github.com/google/flatbuffers" +FB_DIR=".flatbuffers" +FLATC="$FB_DIR/bazel-bin/flatc" + +if [ -z $(which bazel) ]; then + echo "bazel is required to build flatc" + exit 1 +fi + +echo "Bazel version: $(bazel version | head -1 | awk -F':' '{print $2}')" + +if [ ! -e $FB_DIR ]; then + echo "git clone $FB_URL ..." + git clone -b master --no-tag $FB_URL $FB_DIR +else + echo "git pull $FB_URL ..." + git -C $FB_DIR pull --ff-only +fi + +echo "hard reset to $FB_COMMIT" +git -C $FB_DIR reset --hard $FB_COMMIT + +pushd $FB_DIR +echo "run: bazel build :flatc ..." +bazel build :flatc +popd + +WAL_FBS="$DIR/protos/wal.fbs" +WAL_RS_DIR="$DIR/src" + +$FLATC --rust -o $WAL_RS_DIR $WAL_FBS + +cargo fmt +popd + +echo "DONE! Please run 'cargo test' and check in any changes." diff --git a/generated_types/src/google.rs b/generated_types/src/google.rs new file mode 100644 index 0000000000..75b204397e --- /dev/null +++ b/generated_types/src/google.rs @@ -0,0 +1,276 @@ +//! Protobuf types for errors from the google standards and +//! conversions to `tonic::Status` + +pub use google_types::*; + +pub mod rpc { + include!(concat!(env!("OUT_DIR"), "/google.rpc.rs")); +} + +pub mod longrunning { + include!(concat!(env!("OUT_DIR"), "/google.longrunning.rs")); +} + +use self::protobuf::Any; +use prost::{ + bytes::{Bytes, BytesMut}, + Message, +}; +use std::convert::{TryFrom, TryInto}; +use std::iter::FromIterator; +use tonic::Status; +use tracing::error; + +// A newtype struct to provide conversion into tonic::Status +struct EncodeError(prost::EncodeError); + +impl From for tonic::Status { + fn from(error: EncodeError) -> Self { + error!(error=%error.0, "failed to serialise error response details"); + tonic::Status::unknown(format!("failed to serialise server error: {}", error.0)) + } +} + +impl From for EncodeError { + fn from(e: prost::EncodeError) -> Self { + Self(e) + } +} + +fn encode_status(code: tonic::Code, message: String, details: Any) -> tonic::Status { + let mut buffer = BytesMut::new(); + + let status = rpc::Status { + code: code as i32, + message: message.clone(), + details: vec![details], + }; + + match status.encode(&mut buffer) { + Ok(_) => tonic::Status::with_details(code, message, buffer.freeze()), + Err(e) => EncodeError(e).into(), + } +} + +/// Error returned if a request field has an invalid value. Includes +/// machinery to add parent field names for context -- thus it will +/// report `rules.write_timeout` than simply `write_timeout`. +#[derive(Debug, Default, Clone)] +pub struct FieldViolation { + pub field: String, + pub description: String, +} + +impl FieldViolation { + pub fn required(field: impl Into) -> Self { + Self { + field: field.into(), + description: "Field is required".to_string(), + } + } + + /// Re-scopes this error as the child of another field + pub fn scope(self, field: impl Into) -> Self { + let field = if self.field.is_empty() { + field.into() + } else { + [field.into(), self.field].join(".") + }; + + Self { + field, + description: self.description, + } + } +} + +impl std::error::Error for FieldViolation {} + +impl std::fmt::Display for FieldViolation { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "Violation for field \"{}\": {}", + self.field, self.description + ) + } +} + +fn encode_bad_request(violation: Vec) -> Result { + let mut buffer = BytesMut::new(); + + rpc::BadRequest { + field_violations: violation + .into_iter() + .map(|f| rpc::bad_request::FieldViolation { + field: f.field, + description: f.description, + }) + .collect(), + } + .encode(&mut buffer)?; + + Ok(Any { + type_url: "type.googleapis.com/google.rpc.BadRequest".to_string(), + value: buffer.freeze(), + }) +} + +impl From for tonic::Status { + fn from(f: FieldViolation) -> Self { + let message = f.to_string(); + + match encode_bad_request(vec![f]) { + Ok(details) => encode_status(tonic::Code::InvalidArgument, message, details), + Err(e) => e.into(), + } + } +} + +#[derive(Debug, Default, Clone)] +pub struct InternalError {} + +impl From for tonic::Status { + fn from(_: InternalError) -> Self { + tonic::Status::new(tonic::Code::Internal, "Internal Error") + } +} + +#[derive(Debug, Default, Clone)] +pub struct AlreadyExists { + pub resource_type: String, + pub resource_name: String, + pub owner: String, + pub description: String, +} + +fn encode_resource_info( + resource_type: String, + resource_name: String, + owner: String, + description: String, +) -> Result { + let mut buffer = BytesMut::new(); + + rpc::ResourceInfo { + resource_type, + resource_name, + owner, + description, + } + .encode(&mut buffer)?; + + Ok(Any { + type_url: "type.googleapis.com/google.rpc.ResourceInfo".to_string(), + value: buffer.freeze(), + }) +} + +impl From for tonic::Status { + fn from(exists: AlreadyExists) -> Self { + let message = format!( + "Resource {}/{} already exists", + exists.resource_type, exists.resource_name + ); + match encode_resource_info( + exists.resource_type, + exists.resource_name, + exists.owner, + exists.description, + ) { + Ok(details) => encode_status(tonic::Code::AlreadyExists, message, details), + Err(e) => e.into(), + } + } +} + +#[derive(Debug, Default, Clone)] +pub struct NotFound { + pub resource_type: String, + pub resource_name: String, + pub owner: String, + pub description: String, +} + +impl From for tonic::Status { + fn from(not_found: NotFound) -> Self { + let message = format!( + "Resource {}/{} not found", + not_found.resource_type, not_found.resource_name + ); + match encode_resource_info( + not_found.resource_type, + not_found.resource_name, + not_found.owner, + not_found.description, + ) { + Ok(details) => encode_status(tonic::Code::NotFound, message, details), + Err(e) => e.into(), + } + } +} + +#[derive(Debug, Default, Clone)] +pub struct PreconditionViolation { + pub category: String, + pub subject: String, + pub description: String, +} + +fn encode_precondition_failure(violations: Vec) -> Result { + use rpc::precondition_failure::Violation; + + let mut buffer = BytesMut::new(); + + rpc::PreconditionFailure { + violations: violations + .into_iter() + .map(|x| Violation { + r#type: x.category, + subject: x.subject, + description: x.description, + }) + .collect(), + } + .encode(&mut buffer)?; + + Ok(Any { + type_url: "type.googleapis.com/google.rpc.PreconditionFailure".to_string(), + value: buffer.freeze(), + }) +} + +impl From for tonic::Status { + fn from(violation: PreconditionViolation) -> Self { + let message = format!( + "Precondition violation {} - {}: {}", + violation.subject, violation.category, violation.description + ); + match encode_precondition_failure(vec![violation]) { + Ok(details) => encode_status(tonic::Code::FailedPrecondition, message, details), + Err(e) => e.into(), + } + } +} + +/// An extension trait that adds the ability to convert an error +/// that can be converted to a String to a FieldViolation +pub trait FieldViolationExt { + type Output; + + fn field(self, field: &'static str) -> Result; +} + +impl FieldViolationExt for Result +where + E: ToString, +{ + type Output = T; + + fn field(self, field: &'static str) -> Result { + self.map_err(|e| FieldViolation { + field: field.to_string(), + description: e.to_string(), + }) + } +} diff --git a/generated_types/src/lib.rs b/generated_types/src/lib.rs index 0cb21817ce..2dafa8de12 100644 --- a/generated_types/src/lib.rs +++ b/generated_types/src/lib.rs @@ -9,61 +9,74 @@ clippy::clone_on_ref_ptr )] -mod pb { - pub mod influxdata { - pub mod platform { - pub mod storage { - include!(concat!(env!("OUT_DIR"), "/influxdata.platform.storage.rs")); +/// This module imports the generated protobuf code into a Rust module +/// hierarchy that matches the namespace hierarchy of the protobuf +/// definitions +pub mod influxdata { + pub mod platform { + pub mod storage { + include!(concat!(env!("OUT_DIR"), "/influxdata.platform.storage.rs")); - // Can't implement `Default` because `prost::Message` implements `Default` - impl TimestampRange { - pub fn max() -> Self { - TimestampRange { - start: std::i64::MIN, - end: std::i64::MAX, - } - } - } - } - } - - pub mod iox { - pub mod management { - pub mod v1 { - include!(concat!(env!("OUT_DIR"), "/influxdata.iox.management.v1.rs")); - } - } - } - } - - pub mod com { - pub mod github { - pub mod influxdata { - pub mod idpe { - pub mod storage { - pub mod read { - include!(concat!( - env!("OUT_DIR"), - "/com.github.influxdata.idpe.storage.read.rs" - )); - } + // Can't implement `Default` because `prost::Message` implements `Default` + impl TimestampRange { + pub fn max() -> Self { + TimestampRange { + start: std::i64::MIN, + end: std::i64::MAX, } } } } } - // Needed because of https://github.com/hyperium/tonic/issues/471 - pub mod grpc { - pub mod health { + pub mod iox { + pub mod management { pub mod v1 { - include!(concat!(env!("OUT_DIR"), "/grpc.health.v1.rs")); + /// Operation metadata type + pub const OPERATION_METADATA: &str = + "influxdata.iox.management.v1.OperationMetadata"; + + include!(concat!(env!("OUT_DIR"), "/influxdata.iox.management.v1.rs")); + } + } + + pub mod write { + pub mod v1 { + include!(concat!(env!("OUT_DIR"), "/influxdata.iox.write.v1.rs")); } } } } -include!(concat!(env!("OUT_DIR"), "/wal_generated.rs")); +pub mod com { + pub mod github { + pub mod influxdata { + pub mod idpe { + pub mod storage { + pub mod read { + include!(concat!( + env!("OUT_DIR"), + "/com.github.influxdata.idpe.storage.read.rs" + )); + } + } + } + } + } +} + +// Needed because of https://github.com/hyperium/tonic/issues/471 +pub mod grpc { + pub mod health { + pub mod v1 { + include!(concat!(env!("OUT_DIR"), "/grpc.health.v1.rs")); + } + } +} + +/// Generated Flatbuffers code for working with the write-ahead log +pub mod wal_generated; +pub use wal_generated::wal; /// gRPC Storage Service pub const STORAGE_SERVICE: &str = "influxdata.platform.storage.Storage"; @@ -71,9 +84,62 @@ pub const STORAGE_SERVICE: &str = "influxdata.platform.storage.Storage"; pub const IOX_TESTING_SERVICE: &str = "influxdata.platform.storage.IOxTesting"; /// gRPC Arrow Flight Service pub const ARROW_SERVICE: &str = "arrow.flight.protocol.FlightService"; +/// The type prefix for any types +pub const ANY_TYPE_PREFIX: &str = "type.googleapis.com"; -pub use pb::com::github::influxdata::idpe::storage::read::*; -pub use pb::influxdata::platform::storage::*; +/// Returns the protobuf URL usable with a google.protobuf.Any message +/// This is the full Protobuf package and message name prefixed by +/// "type.googleapis.com/" +pub fn protobuf_type_url(protobuf_type: &str) -> String { + format!("{}/{}", ANY_TYPE_PREFIX, protobuf_type) +} -pub use google_types as google; -pub use pb::{grpc, influxdata}; +/// Compares the protobuf type URL found within a google.protobuf.Any +/// message to an expected Protobuf package and message name +/// +/// i.e. strips off the "type.googleapis.com/" prefix from `url` +/// and compares the result with `protobuf_type` +/// +/// ``` +/// use generated_types::protobuf_type_url_eq; +/// assert!(protobuf_type_url_eq("type.googleapis.com/google.protobuf.Empty", "google.protobuf.Empty")); +/// assert!(!protobuf_type_url_eq("type.googleapis.com/google.protobuf.Empty", "something.else")); +/// ``` +pub fn protobuf_type_url_eq(url: &str, protobuf_type: &str) -> bool { + let mut split = url.splitn(2, '/'); + match (split.next(), split.next()) { + (Some(ANY_TYPE_PREFIX), Some(t)) => t == protobuf_type, + _ => false, + } +} + +pub use com::github::influxdata::idpe::storage::read::*; +pub use influxdata::platform::storage::*; + +pub mod google; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_protobuf_type_url() { + use influxdata::iox::management::v1::OPERATION_METADATA; + + let t = protobuf_type_url(OPERATION_METADATA); + + assert_eq!( + &t, + "type.googleapis.com/influxdata.iox.management.v1.OperationMetadata" + ); + + assert!(protobuf_type_url_eq(&t, OPERATION_METADATA)); + assert!(!protobuf_type_url_eq(&t, "foo")); + + // The URL must start with the type.googleapis.com prefix + assert!(!protobuf_type_url_eq( + OPERATION_METADATA, + OPERATION_METADATA + )); + } +} diff --git a/generated_types/src/wal_generated.rs b/generated_types/src/wal_generated.rs new file mode 100644 index 0000000000..10161fc37e --- /dev/null +++ b/generated_types/src/wal_generated.rs @@ -0,0 +1,3220 @@ +// automatically generated by the FlatBuffers compiler, do not modify + +use std::cmp::Ordering; +use std::mem; + +extern crate flatbuffers; +use self::flatbuffers::EndianScalar; + +#[allow(unused_imports, dead_code)] +pub mod wal { + + use std::cmp::Ordering; + use std::mem; + + extern crate flatbuffers; + use self::flatbuffers::EndianScalar; + + #[deprecated( + since = "2.0.0", + note = "Use associated constants instead. This will no longer be generated in 2021." + )] + pub const ENUM_MIN_POINT_VALUE: u8 = 0; + #[deprecated( + since = "2.0.0", + note = "Use associated constants instead. This will no longer be generated in 2021." + )] + pub const ENUM_MAX_POINT_VALUE: u8 = 5; + #[deprecated( + since = "2.0.0", + note = "Use associated constants instead. This will no longer be generated in 2021." + )] + #[allow(non_camel_case_types)] + pub const ENUM_VALUES_POINT_VALUE: [PointValue; 6] = [ + PointValue::NONE, + PointValue::I64Value, + PointValue::U64Value, + PointValue::F64Value, + PointValue::BoolValue, + PointValue::StringValue, + ]; + + #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)] + #[repr(transparent)] + pub struct PointValue(pub u8); + #[allow(non_upper_case_globals)] + impl PointValue { + pub const NONE: Self = Self(0); + pub const I64Value: Self = Self(1); + pub const U64Value: Self = Self(2); + pub const F64Value: Self = Self(3); + pub const BoolValue: Self = Self(4); + pub const StringValue: Self = Self(5); + + pub const ENUM_MIN: u8 = 0; + pub const ENUM_MAX: u8 = 5; + pub const ENUM_VALUES: &'static [Self] = &[ + Self::NONE, + Self::I64Value, + Self::U64Value, + Self::F64Value, + Self::BoolValue, + Self::StringValue, + ]; + /// Returns the variant's name or "" if unknown. + pub fn variant_name(self) -> Option<&'static str> { + match self { + Self::NONE => Some("NONE"), + Self::I64Value => Some("I64Value"), + Self::U64Value => Some("U64Value"), + Self::F64Value => Some("F64Value"), + Self::BoolValue => Some("BoolValue"), + Self::StringValue => Some("StringValue"), + _ => None, + } + } + } + impl std::fmt::Debug for PointValue { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + if let Some(name) = self.variant_name() { + f.write_str(name) + } else { + f.write_fmt(format_args!("", self.0)) + } + } + } + impl<'a> flatbuffers::Follow<'a> for PointValue { + type Inner = Self; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + let b = flatbuffers::read_scalar_at::(buf, loc); + Self(b) + } + } + + impl flatbuffers::Push for PointValue { + type Output = PointValue; + #[inline] + fn push(&self, dst: &mut [u8], _rest: &[u8]) { + flatbuffers::emplace_scalar::(dst, self.0); + } + } + + impl flatbuffers::EndianScalar for PointValue { + #[inline] + fn to_little_endian(self) -> Self { + let b = u8::to_le(self.0); + Self(b) + } + #[inline] + fn from_little_endian(self) -> Self { + let b = u8::from_le(self.0); + Self(b) + } + } + + impl<'a> flatbuffers::Verifiable for PointValue { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + u8::run_verifier(v, pos) + } + } + + impl flatbuffers::SimpleToVerifyInSlice for PointValue {} + pub struct PointValueUnionTableOffset {} + + #[deprecated( + since = "2.0.0", + note = "Use associated constants instead. This will no longer be generated in 2021." + )] + pub const ENUM_MIN_COLUMN_TYPE: i8 = 0; + #[deprecated( + since = "2.0.0", + note = "Use associated constants instead. This will no longer be generated in 2021." + )] + pub const ENUM_MAX_COLUMN_TYPE: i8 = 5; + #[deprecated( + since = "2.0.0", + note = "Use associated constants instead. This will no longer be generated in 2021." + )] + #[allow(non_camel_case_types)] + pub const ENUM_VALUES_COLUMN_TYPE: [ColumnType; 6] = [ + ColumnType::I64, + ColumnType::U64, + ColumnType::F64, + ColumnType::Tag, + ColumnType::String, + ColumnType::Bool, + ]; + + #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)] + #[repr(transparent)] + pub struct ColumnType(pub i8); + #[allow(non_upper_case_globals)] + impl ColumnType { + pub const I64: Self = Self(0); + pub const U64: Self = Self(1); + pub const F64: Self = Self(2); + pub const Tag: Self = Self(3); + pub const String: Self = Self(4); + pub const Bool: Self = Self(5); + + pub const ENUM_MIN: i8 = 0; + pub const ENUM_MAX: i8 = 5; + pub const ENUM_VALUES: &'static [Self] = &[ + Self::I64, + Self::U64, + Self::F64, + Self::Tag, + Self::String, + Self::Bool, + ]; + /// Returns the variant's name or "" if unknown. + pub fn variant_name(self) -> Option<&'static str> { + match self { + Self::I64 => Some("I64"), + Self::U64 => Some("U64"), + Self::F64 => Some("F64"), + Self::Tag => Some("Tag"), + Self::String => Some("String"), + Self::Bool => Some("Bool"), + _ => None, + } + } + } + impl std::fmt::Debug for ColumnType { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + if let Some(name) = self.variant_name() { + f.write_str(name) + } else { + f.write_fmt(format_args!("", self.0)) + } + } + } + impl<'a> flatbuffers::Follow<'a> for ColumnType { + type Inner = Self; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + let b = flatbuffers::read_scalar_at::(buf, loc); + Self(b) + } + } + + impl flatbuffers::Push for ColumnType { + type Output = ColumnType; + #[inline] + fn push(&self, dst: &mut [u8], _rest: &[u8]) { + flatbuffers::emplace_scalar::(dst, self.0); + } + } + + impl flatbuffers::EndianScalar for ColumnType { + #[inline] + fn to_little_endian(self) -> Self { + let b = i8::to_le(self.0); + Self(b) + } + #[inline] + fn from_little_endian(self) -> Self { + let b = i8::from_le(self.0); + Self(b) + } + } + + impl<'a> flatbuffers::Verifiable for ColumnType { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + i8::run_verifier(v, pos) + } + } + + impl flatbuffers::SimpleToVerifyInSlice for ColumnType {} + #[deprecated( + since = "2.0.0", + note = "Use associated constants instead. This will no longer be generated in 2021." + )] + pub const ENUM_MIN_COLUMN_VALUE: u8 = 0; + #[deprecated( + since = "2.0.0", + note = "Use associated constants instead. This will no longer be generated in 2021." + )] + pub const ENUM_MAX_COLUMN_VALUE: u8 = 6; + #[deprecated( + since = "2.0.0", + note = "Use associated constants instead. This will no longer be generated in 2021." + )] + #[allow(non_camel_case_types)] + pub const ENUM_VALUES_COLUMN_VALUE: [ColumnValue; 7] = [ + ColumnValue::NONE, + ColumnValue::TagValue, + ColumnValue::I64Value, + ColumnValue::U64Value, + ColumnValue::F64Value, + ColumnValue::BoolValue, + ColumnValue::StringValue, + ]; + + #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)] + #[repr(transparent)] + pub struct ColumnValue(pub u8); + #[allow(non_upper_case_globals)] + impl ColumnValue { + pub const NONE: Self = Self(0); + pub const TagValue: Self = Self(1); + pub const I64Value: Self = Self(2); + pub const U64Value: Self = Self(3); + pub const F64Value: Self = Self(4); + pub const BoolValue: Self = Self(5); + pub const StringValue: Self = Self(6); + + pub const ENUM_MIN: u8 = 0; + pub const ENUM_MAX: u8 = 6; + pub const ENUM_VALUES: &'static [Self] = &[ + Self::NONE, + Self::TagValue, + Self::I64Value, + Self::U64Value, + Self::F64Value, + Self::BoolValue, + Self::StringValue, + ]; + /// Returns the variant's name or "" if unknown. + pub fn variant_name(self) -> Option<&'static str> { + match self { + Self::NONE => Some("NONE"), + Self::TagValue => Some("TagValue"), + Self::I64Value => Some("I64Value"), + Self::U64Value => Some("U64Value"), + Self::F64Value => Some("F64Value"), + Self::BoolValue => Some("BoolValue"), + Self::StringValue => Some("StringValue"), + _ => None, + } + } + } + impl std::fmt::Debug for ColumnValue { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + if let Some(name) = self.variant_name() { + f.write_str(name) + } else { + f.write_fmt(format_args!("", self.0)) + } + } + } + impl<'a> flatbuffers::Follow<'a> for ColumnValue { + type Inner = Self; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + let b = flatbuffers::read_scalar_at::(buf, loc); + Self(b) + } + } + + impl flatbuffers::Push for ColumnValue { + type Output = ColumnValue; + #[inline] + fn push(&self, dst: &mut [u8], _rest: &[u8]) { + flatbuffers::emplace_scalar::(dst, self.0); + } + } + + impl flatbuffers::EndianScalar for ColumnValue { + #[inline] + fn to_little_endian(self) -> Self { + let b = u8::to_le(self.0); + Self(b) + } + #[inline] + fn from_little_endian(self) -> Self { + let b = u8::from_le(self.0); + Self(b) + } + } + + impl<'a> flatbuffers::Verifiable for ColumnValue { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + u8::run_verifier(v, pos) + } + } + + impl flatbuffers::SimpleToVerifyInSlice for ColumnValue {} + pub struct ColumnValueUnionTableOffset {} + + pub enum EntryOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct Entry<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for Entry<'a> { + type Inner = Entry<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf, loc }, + } + } + } + + impl<'a> Entry<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + Entry { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args EntryArgs<'args>, + ) -> flatbuffers::WIPOffset> { + let mut builder = EntryBuilder::new(_fbb); + if let Some(x) = args.entry_type { + builder.add_entry_type(x); + } + builder.finish() + } + + pub const VT_ENTRY_TYPE: flatbuffers::VOffsetT = 4; + + #[inline] + pub fn entry_type(&self) -> Option> { + self._tab + .get::>(Entry::VT_ENTRY_TYPE, None) + } + } + + impl flatbuffers::Verifiable for Entry<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::>( + &"entry_type", + Self::VT_ENTRY_TYPE, + false, + )? + .finish(); + Ok(()) + } + } + pub struct EntryArgs<'a> { + pub entry_type: Option>>, + } + impl<'a> Default for EntryArgs<'a> { + #[inline] + fn default() -> Self { + EntryArgs { entry_type: None } + } + } + pub struct EntryBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b> EntryBuilder<'a, 'b> { + #[inline] + pub fn add_entry_type(&mut self, entry_type: flatbuffers::WIPOffset>) { + self.fbb_ + .push_slot_always::>( + Entry::VT_ENTRY_TYPE, + entry_type, + ); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> EntryBuilder<'a, 'b> { + let start = _fbb.start_table(); + EntryBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl std::fmt::Debug for Entry<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("Entry"); + ds.field("entry_type", &self.entry_type()); + ds.finish() + } + } + pub enum EntryTypeOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct EntryType<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for EntryType<'a> { + type Inner = EntryType<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf, loc }, + } + } + } + + impl<'a> EntryType<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + EntryType { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args EntryTypeArgs<'args>, + ) -> flatbuffers::WIPOffset> { + let mut builder = EntryTypeBuilder::new(_fbb); + if let Some(x) = args.delete { + builder.add_delete(x); + } + if let Some(x) = args.write { + builder.add_write(x); + } + builder.finish() + } + + pub const VT_WRITE: flatbuffers::VOffsetT = 4; + pub const VT_DELETE: flatbuffers::VOffsetT = 6; + + #[inline] + pub fn write(&self) -> Option> { + self._tab + .get::>(EntryType::VT_WRITE, None) + } + #[inline] + pub fn delete(&self) -> Option> { + self._tab + .get::>(EntryType::VT_DELETE, None) + } + } + + impl flatbuffers::Verifiable for EntryType<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::>( + &"write", + Self::VT_WRITE, + false, + )? + .visit_field::>( + &"delete", + Self::VT_DELETE, + false, + )? + .finish(); + Ok(()) + } + } + pub struct EntryTypeArgs<'a> { + pub write: Option>>, + pub delete: Option>>, + } + impl<'a> Default for EntryTypeArgs<'a> { + #[inline] + fn default() -> Self { + EntryTypeArgs { + write: None, + delete: None, + } + } + } + pub struct EntryTypeBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b> EntryTypeBuilder<'a, 'b> { + #[inline] + pub fn add_write(&mut self, write: flatbuffers::WIPOffset>) { + self.fbb_ + .push_slot_always::>(EntryType::VT_WRITE, write); + } + #[inline] + pub fn add_delete(&mut self, delete: flatbuffers::WIPOffset>) { + self.fbb_ + .push_slot_always::>(EntryType::VT_DELETE, delete); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> EntryTypeBuilder<'a, 'b> { + let start = _fbb.start_table(); + EntryTypeBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl std::fmt::Debug for EntryType<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("EntryType"); + ds.field("write", &self.write()); + ds.field("delete", &self.delete()); + ds.finish() + } + } + pub enum WriteOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct Write<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for Write<'a> { + type Inner = Write<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf, loc }, + } + } + } + + impl<'a> Write<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + Write { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args WriteArgs<'args>, + ) -> flatbuffers::WIPOffset> { + let mut builder = WriteBuilder::new(_fbb); + if let Some(x) = args.points { + builder.add_points(x); + } + builder.finish() + } + + pub const VT_POINTS: flatbuffers::VOffsetT = 4; + + #[inline] + pub fn points( + &self, + ) -> Option>>> { + self._tab.get::>, + >>(Write::VT_POINTS, None) + } + } + + impl flatbuffers::Verifiable for Write<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::>, + >>(&"points", Self::VT_POINTS, false)? + .finish(); + Ok(()) + } + } + pub struct WriteArgs<'a> { + pub points: Option< + flatbuffers::WIPOffset< + flatbuffers::Vector<'a, flatbuffers::ForwardsUOffset>>, + >, + >, + } + impl<'a> Default for WriteArgs<'a> { + #[inline] + fn default() -> Self { + WriteArgs { points: None } + } + } + pub struct WriteBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b> WriteBuilder<'a, 'b> { + #[inline] + pub fn add_points( + &mut self, + points: flatbuffers::WIPOffset< + flatbuffers::Vector<'b, flatbuffers::ForwardsUOffset>>, + >, + ) { + self.fbb_ + .push_slot_always::>(Write::VT_POINTS, points); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> WriteBuilder<'a, 'b> { + let start = _fbb.start_table(); + WriteBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl std::fmt::Debug for Write<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("Write"); + ds.field("points", &self.points()); + ds.finish() + } + } + pub enum I64ValueOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct I64Value<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for I64Value<'a> { + type Inner = I64Value<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf, loc }, + } + } + } + + impl<'a> I64Value<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + I64Value { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args I64ValueArgs, + ) -> flatbuffers::WIPOffset> { + let mut builder = I64ValueBuilder::new(_fbb); + builder.add_value(args.value); + builder.finish() + } + + pub const VT_VALUE: flatbuffers::VOffsetT = 4; + + #[inline] + pub fn value(&self) -> i64 { + self._tab.get::(I64Value::VT_VALUE, Some(0)).unwrap() + } + } + + impl flatbuffers::Verifiable for I64Value<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::(&"value", Self::VT_VALUE, false)? + .finish(); + Ok(()) + } + } + pub struct I64ValueArgs { + pub value: i64, + } + impl<'a> Default for I64ValueArgs { + #[inline] + fn default() -> Self { + I64ValueArgs { value: 0 } + } + } + pub struct I64ValueBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b> I64ValueBuilder<'a, 'b> { + #[inline] + pub fn add_value(&mut self, value: i64) { + self.fbb_.push_slot::(I64Value::VT_VALUE, value, 0); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> I64ValueBuilder<'a, 'b> { + let start = _fbb.start_table(); + I64ValueBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl std::fmt::Debug for I64Value<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("I64Value"); + ds.field("value", &self.value()); + ds.finish() + } + } + pub enum U64ValueOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct U64Value<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for U64Value<'a> { + type Inner = U64Value<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf, loc }, + } + } + } + + impl<'a> U64Value<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + U64Value { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args U64ValueArgs, + ) -> flatbuffers::WIPOffset> { + let mut builder = U64ValueBuilder::new(_fbb); + builder.add_value(args.value); + builder.finish() + } + + pub const VT_VALUE: flatbuffers::VOffsetT = 4; + + #[inline] + pub fn value(&self) -> u64 { + self._tab.get::(U64Value::VT_VALUE, Some(0)).unwrap() + } + } + + impl flatbuffers::Verifiable for U64Value<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::(&"value", Self::VT_VALUE, false)? + .finish(); + Ok(()) + } + } + pub struct U64ValueArgs { + pub value: u64, + } + impl<'a> Default for U64ValueArgs { + #[inline] + fn default() -> Self { + U64ValueArgs { value: 0 } + } + } + pub struct U64ValueBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b> U64ValueBuilder<'a, 'b> { + #[inline] + pub fn add_value(&mut self, value: u64) { + self.fbb_.push_slot::(U64Value::VT_VALUE, value, 0); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> U64ValueBuilder<'a, 'b> { + let start = _fbb.start_table(); + U64ValueBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl std::fmt::Debug for U64Value<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("U64Value"); + ds.field("value", &self.value()); + ds.finish() + } + } + pub enum F64ValueOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct F64Value<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for F64Value<'a> { + type Inner = F64Value<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf, loc }, + } + } + } + + impl<'a> F64Value<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + F64Value { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args F64ValueArgs, + ) -> flatbuffers::WIPOffset> { + let mut builder = F64ValueBuilder::new(_fbb); + builder.add_value(args.value); + builder.finish() + } + + pub const VT_VALUE: flatbuffers::VOffsetT = 4; + + #[inline] + pub fn value(&self) -> f64 { + self._tab.get::(F64Value::VT_VALUE, Some(0.0)).unwrap() + } + } + + impl flatbuffers::Verifiable for F64Value<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::(&"value", Self::VT_VALUE, false)? + .finish(); + Ok(()) + } + } + pub struct F64ValueArgs { + pub value: f64, + } + impl<'a> Default for F64ValueArgs { + #[inline] + fn default() -> Self { + F64ValueArgs { value: 0.0 } + } + } + pub struct F64ValueBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b> F64ValueBuilder<'a, 'b> { + #[inline] + pub fn add_value(&mut self, value: f64) { + self.fbb_.push_slot::(F64Value::VT_VALUE, value, 0.0); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> F64ValueBuilder<'a, 'b> { + let start = _fbb.start_table(); + F64ValueBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl std::fmt::Debug for F64Value<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("F64Value"); + ds.field("value", &self.value()); + ds.finish() + } + } + pub enum BoolValueOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct BoolValue<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for BoolValue<'a> { + type Inner = BoolValue<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf, loc }, + } + } + } + + impl<'a> BoolValue<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + BoolValue { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args BoolValueArgs, + ) -> flatbuffers::WIPOffset> { + let mut builder = BoolValueBuilder::new(_fbb); + builder.add_value(args.value); + builder.finish() + } + + pub const VT_VALUE: flatbuffers::VOffsetT = 4; + + #[inline] + pub fn value(&self) -> bool { + self._tab + .get::(BoolValue::VT_VALUE, Some(false)) + .unwrap() + } + } + + impl flatbuffers::Verifiable for BoolValue<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::(&"value", Self::VT_VALUE, false)? + .finish(); + Ok(()) + } + } + pub struct BoolValueArgs { + pub value: bool, + } + impl<'a> Default for BoolValueArgs { + #[inline] + fn default() -> Self { + BoolValueArgs { value: false } + } + } + pub struct BoolValueBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b> BoolValueBuilder<'a, 'b> { + #[inline] + pub fn add_value(&mut self, value: bool) { + self.fbb_ + .push_slot::(BoolValue::VT_VALUE, value, false); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> BoolValueBuilder<'a, 'b> { + let start = _fbb.start_table(); + BoolValueBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl std::fmt::Debug for BoolValue<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("BoolValue"); + ds.field("value", &self.value()); + ds.finish() + } + } + pub enum StringValueOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct StringValue<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for StringValue<'a> { + type Inner = StringValue<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf, loc }, + } + } + } + + impl<'a> StringValue<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + StringValue { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args StringValueArgs<'args>, + ) -> flatbuffers::WIPOffset> { + let mut builder = StringValueBuilder::new(_fbb); + if let Some(x) = args.value { + builder.add_value(x); + } + builder.finish() + } + + pub const VT_VALUE: flatbuffers::VOffsetT = 4; + + #[inline] + pub fn value(&self) -> Option<&'a str> { + self._tab + .get::>(StringValue::VT_VALUE, None) + } + } + + impl flatbuffers::Verifiable for StringValue<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::>(&"value", Self::VT_VALUE, false)? + .finish(); + Ok(()) + } + } + pub struct StringValueArgs<'a> { + pub value: Option>, + } + impl<'a> Default for StringValueArgs<'a> { + #[inline] + fn default() -> Self { + StringValueArgs { value: None } + } + } + pub struct StringValueBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b> StringValueBuilder<'a, 'b> { + #[inline] + pub fn add_value(&mut self, value: flatbuffers::WIPOffset<&'b str>) { + self.fbb_ + .push_slot_always::>(StringValue::VT_VALUE, value); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> StringValueBuilder<'a, 'b> { + let start = _fbb.start_table(); + StringValueBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl std::fmt::Debug for StringValue<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("StringValue"); + ds.field("value", &self.value()); + ds.finish() + } + } + pub enum PointOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct Point<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for Point<'a> { + type Inner = Point<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf, loc }, + } + } + } + + impl<'a> Point<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + Point { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args PointArgs<'args>, + ) -> flatbuffers::WIPOffset> { + let mut builder = PointBuilder::new(_fbb); + builder.add_time(args.time); + if let Some(x) = args.value { + builder.add_value(x); + } + if let Some(x) = args.key { + builder.add_key(x); + } + builder.add_value_type(args.value_type); + builder.finish() + } + + pub const VT_KEY: flatbuffers::VOffsetT = 4; + pub const VT_TIME: flatbuffers::VOffsetT = 6; + pub const VT_VALUE_TYPE: flatbuffers::VOffsetT = 8; + pub const VT_VALUE: flatbuffers::VOffsetT = 10; + + #[inline] + pub fn key(&self) -> Option<&'a str> { + self._tab + .get::>(Point::VT_KEY, None) + } + #[inline] + pub fn time(&self) -> i64 { + self._tab.get::(Point::VT_TIME, Some(0)).unwrap() + } + #[inline] + pub fn value_type(&self) -> PointValue { + self._tab + .get::(Point::VT_VALUE_TYPE, Some(PointValue::NONE)) + .unwrap() + } + #[inline] + pub fn value(&self) -> Option> { + self._tab + .get::>>(Point::VT_VALUE, None) + } + #[inline] + #[allow(non_snake_case)] + pub fn value_as_i64value(&self) -> Option> { + if self.value_type() == PointValue::I64Value { + self.value().map(I64Value::init_from_table) + } else { + None + } + } + + #[inline] + #[allow(non_snake_case)] + pub fn value_as_u64value(&self) -> Option> { + if self.value_type() == PointValue::U64Value { + self.value().map(U64Value::init_from_table) + } else { + None + } + } + + #[inline] + #[allow(non_snake_case)] + pub fn value_as_f64value(&self) -> Option> { + if self.value_type() == PointValue::F64Value { + self.value().map(F64Value::init_from_table) + } else { + None + } + } + + #[inline] + #[allow(non_snake_case)] + pub fn value_as_bool_value(&self) -> Option> { + if self.value_type() == PointValue::BoolValue { + self.value().map(BoolValue::init_from_table) + } else { + None + } + } + + #[inline] + #[allow(non_snake_case)] + pub fn value_as_string_value(&self) -> Option> { + if self.value_type() == PointValue::StringValue { + self.value().map(StringValue::init_from_table) + } else { + None + } + } + } + + impl flatbuffers::Verifiable for Point<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::>(&"key", Self::VT_KEY, false)? + .visit_field::(&"time", Self::VT_TIME, false)? + .visit_union::( + &"value_type", + Self::VT_VALUE_TYPE, + &"value", + Self::VT_VALUE, + false, + |key, v, pos| match key { + PointValue::I64Value => v + .verify_union_variant::>( + "PointValue::I64Value", + pos, + ), + PointValue::U64Value => v + .verify_union_variant::>( + "PointValue::U64Value", + pos, + ), + PointValue::F64Value => v + .verify_union_variant::>( + "PointValue::F64Value", + pos, + ), + PointValue::BoolValue => v + .verify_union_variant::>( + "PointValue::BoolValue", + pos, + ), + PointValue::StringValue => v + .verify_union_variant::>( + "PointValue::StringValue", + pos, + ), + _ => Ok(()), + }, + )? + .finish(); + Ok(()) + } + } + pub struct PointArgs<'a> { + pub key: Option>, + pub time: i64, + pub value_type: PointValue, + pub value: Option>, + } + impl<'a> Default for PointArgs<'a> { + #[inline] + fn default() -> Self { + PointArgs { + key: None, + time: 0, + value_type: PointValue::NONE, + value: None, + } + } + } + pub struct PointBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b> PointBuilder<'a, 'b> { + #[inline] + pub fn add_key(&mut self, key: flatbuffers::WIPOffset<&'b str>) { + self.fbb_ + .push_slot_always::>(Point::VT_KEY, key); + } + #[inline] + pub fn add_time(&mut self, time: i64) { + self.fbb_.push_slot::(Point::VT_TIME, time, 0); + } + #[inline] + pub fn add_value_type(&mut self, value_type: PointValue) { + self.fbb_ + .push_slot::(Point::VT_VALUE_TYPE, value_type, PointValue::NONE); + } + #[inline] + pub fn add_value(&mut self, value: flatbuffers::WIPOffset) { + self.fbb_ + .push_slot_always::>(Point::VT_VALUE, value); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> PointBuilder<'a, 'b> { + let start = _fbb.start_table(); + PointBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl std::fmt::Debug for Point<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("Point"); + ds.field("key", &self.key()); + ds.field("time", &self.time()); + ds.field("value_type", &self.value_type()); + match self.value_type() { + PointValue::I64Value => { + if let Some(x) = self.value_as_i64value() { + ds.field("value", &x) + } else { + ds.field( + "value", + &"InvalidFlatbuffer: Union discriminant does not match value.", + ) + } + } + PointValue::U64Value => { + if let Some(x) = self.value_as_u64value() { + ds.field("value", &x) + } else { + ds.field( + "value", + &"InvalidFlatbuffer: Union discriminant does not match value.", + ) + } + } + PointValue::F64Value => { + if let Some(x) = self.value_as_f64value() { + ds.field("value", &x) + } else { + ds.field( + "value", + &"InvalidFlatbuffer: Union discriminant does not match value.", + ) + } + } + PointValue::BoolValue => { + if let Some(x) = self.value_as_bool_value() { + ds.field("value", &x) + } else { + ds.field( + "value", + &"InvalidFlatbuffer: Union discriminant does not match value.", + ) + } + } + PointValue::StringValue => { + if let Some(x) = self.value_as_string_value() { + ds.field("value", &x) + } else { + ds.field( + "value", + &"InvalidFlatbuffer: Union discriminant does not match value.", + ) + } + } + _ => { + let x: Option<()> = None; + ds.field("value", &x) + } + }; + ds.finish() + } + } + pub enum DeleteOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct Delete<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for Delete<'a> { + type Inner = Delete<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf, loc }, + } + } + } + + impl<'a> Delete<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + Delete { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args DeleteArgs<'args>, + ) -> flatbuffers::WIPOffset> { + let mut builder = DeleteBuilder::new(_fbb); + builder.add_stop_time(args.stop_time); + builder.add_start_time(args.start_time); + if let Some(x) = args.predicate { + builder.add_predicate(x); + } + builder.finish() + } + + pub const VT_PREDICATE: flatbuffers::VOffsetT = 4; + pub const VT_START_TIME: flatbuffers::VOffsetT = 6; + pub const VT_STOP_TIME: flatbuffers::VOffsetT = 8; + + #[inline] + pub fn predicate(&self) -> Option<&'a str> { + self._tab + .get::>(Delete::VT_PREDICATE, None) + } + #[inline] + pub fn start_time(&self) -> i64 { + self._tab + .get::(Delete::VT_START_TIME, Some(0)) + .unwrap() + } + #[inline] + pub fn stop_time(&self) -> i64 { + self._tab.get::(Delete::VT_STOP_TIME, Some(0)).unwrap() + } + } + + impl flatbuffers::Verifiable for Delete<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::>( + &"predicate", + Self::VT_PREDICATE, + false, + )? + .visit_field::(&"start_time", Self::VT_START_TIME, false)? + .visit_field::(&"stop_time", Self::VT_STOP_TIME, false)? + .finish(); + Ok(()) + } + } + pub struct DeleteArgs<'a> { + pub predicate: Option>, + pub start_time: i64, + pub stop_time: i64, + } + impl<'a> Default for DeleteArgs<'a> { + #[inline] + fn default() -> Self { + DeleteArgs { + predicate: None, + start_time: 0, + stop_time: 0, + } + } + } + pub struct DeleteBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b> DeleteBuilder<'a, 'b> { + #[inline] + pub fn add_predicate(&mut self, predicate: flatbuffers::WIPOffset<&'b str>) { + self.fbb_ + .push_slot_always::>(Delete::VT_PREDICATE, predicate); + } + #[inline] + pub fn add_start_time(&mut self, start_time: i64) { + self.fbb_ + .push_slot::(Delete::VT_START_TIME, start_time, 0); + } + #[inline] + pub fn add_stop_time(&mut self, stop_time: i64) { + self.fbb_ + .push_slot::(Delete::VT_STOP_TIME, stop_time, 0); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> DeleteBuilder<'a, 'b> { + let start = _fbb.start_table(); + DeleteBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl std::fmt::Debug for Delete<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("Delete"); + ds.field("predicate", &self.predicate()); + ds.field("start_time", &self.start_time()); + ds.field("stop_time", &self.stop_time()); + ds.finish() + } + } + pub enum SegmentOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct Segment<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for Segment<'a> { + type Inner = Segment<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf, loc }, + } + } + } + + impl<'a> Segment<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + Segment { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args SegmentArgs<'args>, + ) -> flatbuffers::WIPOffset> { + let mut builder = SegmentBuilder::new(_fbb); + builder.add_id(args.id); + if let Some(x) = args.writes { + builder.add_writes(x); + } + builder.add_writer_id(args.writer_id); + builder.finish() + } + + pub const VT_ID: flatbuffers::VOffsetT = 4; + pub const VT_WRITER_ID: flatbuffers::VOffsetT = 6; + pub const VT_WRITES: flatbuffers::VOffsetT = 8; + + #[inline] + pub fn id(&self) -> u64 { + self._tab.get::(Segment::VT_ID, Some(0)).unwrap() + } + #[inline] + pub fn writer_id(&self) -> u32 { + self._tab + .get::(Segment::VT_WRITER_ID, Some(0)) + .unwrap() + } + #[inline] + pub fn writes( + &self, + ) -> Option>>> + { + self._tab.get::>, + >>(Segment::VT_WRITES, None) + } + } + + impl flatbuffers::Verifiable for Segment<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::(&"id", Self::VT_ID, false)? + .visit_field::(&"writer_id", Self::VT_WRITER_ID, false)? + .visit_field::>, + >>(&"writes", Self::VT_WRITES, false)? + .finish(); + Ok(()) + } + } + pub struct SegmentArgs<'a> { + pub id: u64, + pub writer_id: u32, + pub writes: Option< + flatbuffers::WIPOffset< + flatbuffers::Vector<'a, flatbuffers::ForwardsUOffset>>, + >, + >, + } + impl<'a> Default for SegmentArgs<'a> { + #[inline] + fn default() -> Self { + SegmentArgs { + id: 0, + writer_id: 0, + writes: None, + } + } + } + pub struct SegmentBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b> SegmentBuilder<'a, 'b> { + #[inline] + pub fn add_id(&mut self, id: u64) { + self.fbb_.push_slot::(Segment::VT_ID, id, 0); + } + #[inline] + pub fn add_writer_id(&mut self, writer_id: u32) { + self.fbb_ + .push_slot::(Segment::VT_WRITER_ID, writer_id, 0); + } + #[inline] + pub fn add_writes( + &mut self, + writes: flatbuffers::WIPOffset< + flatbuffers::Vector<'b, flatbuffers::ForwardsUOffset>>, + >, + ) { + self.fbb_ + .push_slot_always::>(Segment::VT_WRITES, writes); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> SegmentBuilder<'a, 'b> { + let start = _fbb.start_table(); + SegmentBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl std::fmt::Debug for Segment<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("Segment"); + ds.field("id", &self.id()); + ds.field("writer_id", &self.writer_id()); + ds.field("writes", &self.writes()); + ds.finish() + } + } + pub enum ReplicatedWriteDataOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct ReplicatedWriteData<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for ReplicatedWriteData<'a> { + type Inner = ReplicatedWriteData<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf, loc }, + } + } + } + + impl<'a> ReplicatedWriteData<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + ReplicatedWriteData { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args ReplicatedWriteDataArgs<'args>, + ) -> flatbuffers::WIPOffset> { + let mut builder = ReplicatedWriteDataBuilder::new(_fbb); + if let Some(x) = args.payload { + builder.add_payload(x); + } + builder.finish() + } + + pub const VT_PAYLOAD: flatbuffers::VOffsetT = 4; + + #[inline] + pub fn payload(&self) -> Option<&'a [u8]> { + self._tab + .get::>>( + ReplicatedWriteData::VT_PAYLOAD, + None, + ) + .map(|v| v.safe_slice()) + } + } + + impl flatbuffers::Verifiable for ReplicatedWriteData<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::>>( + &"payload", + Self::VT_PAYLOAD, + false, + )? + .finish(); + Ok(()) + } + } + pub struct ReplicatedWriteDataArgs<'a> { + pub payload: Option>>, + } + impl<'a> Default for ReplicatedWriteDataArgs<'a> { + #[inline] + fn default() -> Self { + ReplicatedWriteDataArgs { payload: None } + } + } + pub struct ReplicatedWriteDataBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b> ReplicatedWriteDataBuilder<'a, 'b> { + #[inline] + pub fn add_payload( + &mut self, + payload: flatbuffers::WIPOffset>, + ) { + self.fbb_.push_slot_always::>( + ReplicatedWriteData::VT_PAYLOAD, + payload, + ); + } + #[inline] + pub fn new( + _fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>, + ) -> ReplicatedWriteDataBuilder<'a, 'b> { + let start = _fbb.start_table(); + ReplicatedWriteDataBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl std::fmt::Debug for ReplicatedWriteData<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("ReplicatedWriteData"); + ds.field("payload", &self.payload()); + ds.finish() + } + } + pub enum WriterSummaryOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct WriterSummary<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for WriterSummary<'a> { + type Inner = WriterSummary<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf, loc }, + } + } + } + + impl<'a> WriterSummary<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + WriterSummary { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args WriterSummaryArgs, + ) -> flatbuffers::WIPOffset> { + let mut builder = WriterSummaryBuilder::new(_fbb); + builder.add_end_sequence(args.end_sequence); + builder.add_start_sequence(args.start_sequence); + builder.add_writer_id(args.writer_id); + builder.add_missing_sequences(args.missing_sequences); + builder.finish() + } + + pub const VT_WRITER_ID: flatbuffers::VOffsetT = 4; + pub const VT_START_SEQUENCE: flatbuffers::VOffsetT = 6; + pub const VT_END_SEQUENCE: flatbuffers::VOffsetT = 8; + pub const VT_MISSING_SEQUENCES: flatbuffers::VOffsetT = 10; + + #[inline] + pub fn writer_id(&self) -> u64 { + self._tab + .get::(WriterSummary::VT_WRITER_ID, Some(0)) + .unwrap() + } + #[inline] + pub fn start_sequence(&self) -> u64 { + self._tab + .get::(WriterSummary::VT_START_SEQUENCE, Some(0)) + .unwrap() + } + #[inline] + pub fn end_sequence(&self) -> u64 { + self._tab + .get::(WriterSummary::VT_END_SEQUENCE, Some(0)) + .unwrap() + } + #[inline] + pub fn missing_sequences(&self) -> bool { + self._tab + .get::(WriterSummary::VT_MISSING_SEQUENCES, Some(false)) + .unwrap() + } + } + + impl flatbuffers::Verifiable for WriterSummary<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::(&"writer_id", Self::VT_WRITER_ID, false)? + .visit_field::(&"start_sequence", Self::VT_START_SEQUENCE, false)? + .visit_field::(&"end_sequence", Self::VT_END_SEQUENCE, false)? + .visit_field::(&"missing_sequences", Self::VT_MISSING_SEQUENCES, false)? + .finish(); + Ok(()) + } + } + pub struct WriterSummaryArgs { + pub writer_id: u64, + pub start_sequence: u64, + pub end_sequence: u64, + pub missing_sequences: bool, + } + impl<'a> Default for WriterSummaryArgs { + #[inline] + fn default() -> Self { + WriterSummaryArgs { + writer_id: 0, + start_sequence: 0, + end_sequence: 0, + missing_sequences: false, + } + } + } + pub struct WriterSummaryBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b> WriterSummaryBuilder<'a, 'b> { + #[inline] + pub fn add_writer_id(&mut self, writer_id: u64) { + self.fbb_ + .push_slot::(WriterSummary::VT_WRITER_ID, writer_id, 0); + } + #[inline] + pub fn add_start_sequence(&mut self, start_sequence: u64) { + self.fbb_ + .push_slot::(WriterSummary::VT_START_SEQUENCE, start_sequence, 0); + } + #[inline] + pub fn add_end_sequence(&mut self, end_sequence: u64) { + self.fbb_ + .push_slot::(WriterSummary::VT_END_SEQUENCE, end_sequence, 0); + } + #[inline] + pub fn add_missing_sequences(&mut self, missing_sequences: bool) { + self.fbb_.push_slot::( + WriterSummary::VT_MISSING_SEQUENCES, + missing_sequences, + false, + ); + } + #[inline] + pub fn new( + _fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>, + ) -> WriterSummaryBuilder<'a, 'b> { + let start = _fbb.start_table(); + WriterSummaryBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl std::fmt::Debug for WriterSummary<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("WriterSummary"); + ds.field("writer_id", &self.writer_id()); + ds.field("start_sequence", &self.start_sequence()); + ds.field("end_sequence", &self.end_sequence()); + ds.field("missing_sequences", &self.missing_sequences()); + ds.finish() + } + } + pub enum ReplicatedWriteOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct ReplicatedWrite<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for ReplicatedWrite<'a> { + type Inner = ReplicatedWrite<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf, loc }, + } + } + } + + impl<'a> ReplicatedWrite<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + ReplicatedWrite { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args ReplicatedWriteArgs<'args>, + ) -> flatbuffers::WIPOffset> { + let mut builder = ReplicatedWriteBuilder::new(_fbb); + builder.add_sequence(args.sequence); + if let Some(x) = args.payload { + builder.add_payload(x); + } + builder.add_checksum(args.checksum); + builder.add_writer(args.writer); + builder.finish() + } + + pub const VT_WRITER: flatbuffers::VOffsetT = 4; + pub const VT_SEQUENCE: flatbuffers::VOffsetT = 6; + pub const VT_CHECKSUM: flatbuffers::VOffsetT = 8; + pub const VT_PAYLOAD: flatbuffers::VOffsetT = 10; + + #[inline] + pub fn writer(&self) -> u32 { + self._tab + .get::(ReplicatedWrite::VT_WRITER, Some(0)) + .unwrap() + } + #[inline] + pub fn sequence(&self) -> u64 { + self._tab + .get::(ReplicatedWrite::VT_SEQUENCE, Some(0)) + .unwrap() + } + #[inline] + pub fn checksum(&self) -> u32 { + self._tab + .get::(ReplicatedWrite::VT_CHECKSUM, Some(0)) + .unwrap() + } + #[inline] + pub fn payload(&self) -> Option<&'a [u8]> { + self._tab + .get::>>( + ReplicatedWrite::VT_PAYLOAD, + None, + ) + .map(|v| v.safe_slice()) + } + } + + impl flatbuffers::Verifiable for ReplicatedWrite<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::(&"writer", Self::VT_WRITER, false)? + .visit_field::(&"sequence", Self::VT_SEQUENCE, false)? + .visit_field::(&"checksum", Self::VT_CHECKSUM, false)? + .visit_field::>>( + &"payload", + Self::VT_PAYLOAD, + false, + )? + .finish(); + Ok(()) + } + } + pub struct ReplicatedWriteArgs<'a> { + pub writer: u32, + pub sequence: u64, + pub checksum: u32, + pub payload: Option>>, + } + impl<'a> Default for ReplicatedWriteArgs<'a> { + #[inline] + fn default() -> Self { + ReplicatedWriteArgs { + writer: 0, + sequence: 0, + checksum: 0, + payload: None, + } + } + } + pub struct ReplicatedWriteBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b> ReplicatedWriteBuilder<'a, 'b> { + #[inline] + pub fn add_writer(&mut self, writer: u32) { + self.fbb_ + .push_slot::(ReplicatedWrite::VT_WRITER, writer, 0); + } + #[inline] + pub fn add_sequence(&mut self, sequence: u64) { + self.fbb_ + .push_slot::(ReplicatedWrite::VT_SEQUENCE, sequence, 0); + } + #[inline] + pub fn add_checksum(&mut self, checksum: u32) { + self.fbb_ + .push_slot::(ReplicatedWrite::VT_CHECKSUM, checksum, 0); + } + #[inline] + pub fn add_payload( + &mut self, + payload: flatbuffers::WIPOffset>, + ) { + self.fbb_.push_slot_always::>( + ReplicatedWrite::VT_PAYLOAD, + payload, + ); + } + #[inline] + pub fn new( + _fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>, + ) -> ReplicatedWriteBuilder<'a, 'b> { + let start = _fbb.start_table(); + ReplicatedWriteBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl std::fmt::Debug for ReplicatedWrite<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("ReplicatedWrite"); + ds.field("writer", &self.writer()); + ds.field("sequence", &self.sequence()); + ds.field("checksum", &self.checksum()); + ds.field("payload", &self.payload()); + ds.finish() + } + } + pub enum WriteBufferBatchOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct WriteBufferBatch<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for WriteBufferBatch<'a> { + type Inner = WriteBufferBatch<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf, loc }, + } + } + } + + impl<'a> WriteBufferBatch<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + WriteBufferBatch { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args WriteBufferBatchArgs<'args>, + ) -> flatbuffers::WIPOffset> { + let mut builder = WriteBufferBatchBuilder::new(_fbb); + if let Some(x) = args.entries { + builder.add_entries(x); + } + builder.finish() + } + + pub const VT_ENTRIES: flatbuffers::VOffsetT = 4; + + #[inline] + pub fn entries( + &self, + ) -> Option>>> + { + self._tab.get::>, + >>(WriteBufferBatch::VT_ENTRIES, None) + } + } + + impl flatbuffers::Verifiable for WriteBufferBatch<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::>, + >>(&"entries", Self::VT_ENTRIES, false)? + .finish(); + Ok(()) + } + } + pub struct WriteBufferBatchArgs<'a> { + pub entries: Option< + flatbuffers::WIPOffset< + flatbuffers::Vector<'a, flatbuffers::ForwardsUOffset>>, + >, + >, + } + impl<'a> Default for WriteBufferBatchArgs<'a> { + #[inline] + fn default() -> Self { + WriteBufferBatchArgs { entries: None } + } + } + pub struct WriteBufferBatchBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b> WriteBufferBatchBuilder<'a, 'b> { + #[inline] + pub fn add_entries( + &mut self, + entries: flatbuffers::WIPOffset< + flatbuffers::Vector<'b, flatbuffers::ForwardsUOffset>>, + >, + ) { + self.fbb_.push_slot_always::>( + WriteBufferBatch::VT_ENTRIES, + entries, + ); + } + #[inline] + pub fn new( + _fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>, + ) -> WriteBufferBatchBuilder<'a, 'b> { + let start = _fbb.start_table(); + WriteBufferBatchBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl std::fmt::Debug for WriteBufferBatch<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("WriteBufferBatch"); + ds.field("entries", &self.entries()); + ds.finish() + } + } + pub enum WriteBufferEntryOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct WriteBufferEntry<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for WriteBufferEntry<'a> { + type Inner = WriteBufferEntry<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf, loc }, + } + } + } + + impl<'a> WriteBufferEntry<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + WriteBufferEntry { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args WriteBufferEntryArgs<'args>, + ) -> flatbuffers::WIPOffset> { + let mut builder = WriteBufferEntryBuilder::new(_fbb); + if let Some(x) = args.delete { + builder.add_delete(x); + } + if let Some(x) = args.table_batches { + builder.add_table_batches(x); + } + if let Some(x) = args.partition_key { + builder.add_partition_key(x); + } + builder.finish() + } + + pub const VT_PARTITION_KEY: flatbuffers::VOffsetT = 4; + pub const VT_TABLE_BATCHES: flatbuffers::VOffsetT = 6; + pub const VT_DELETE: flatbuffers::VOffsetT = 8; + + #[inline] + pub fn partition_key(&self) -> Option<&'a str> { + self._tab + .get::>(WriteBufferEntry::VT_PARTITION_KEY, None) + } + #[inline] + pub fn table_batches( + &self, + ) -> Option>>> + { + self._tab.get::>, + >>(WriteBufferEntry::VT_TABLE_BATCHES, None) + } + #[inline] + pub fn delete(&self) -> Option> { + self._tab + .get::>( + WriteBufferEntry::VT_DELETE, + None, + ) + } + } + + impl flatbuffers::Verifiable for WriteBufferEntry<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::>( + &"partition_key", + Self::VT_PARTITION_KEY, + false, + )? + .visit_field::>, + >>(&"table_batches", Self::VT_TABLE_BATCHES, false)? + .visit_field::>( + &"delete", + Self::VT_DELETE, + false, + )? + .finish(); + Ok(()) + } + } + pub struct WriteBufferEntryArgs<'a> { + pub partition_key: Option>, + pub table_batches: Option< + flatbuffers::WIPOffset< + flatbuffers::Vector<'a, flatbuffers::ForwardsUOffset>>, + >, + >, + pub delete: Option>>, + } + impl<'a> Default for WriteBufferEntryArgs<'a> { + #[inline] + fn default() -> Self { + WriteBufferEntryArgs { + partition_key: None, + table_batches: None, + delete: None, + } + } + } + pub struct WriteBufferEntryBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b> WriteBufferEntryBuilder<'a, 'b> { + #[inline] + pub fn add_partition_key(&mut self, partition_key: flatbuffers::WIPOffset<&'b str>) { + self.fbb_.push_slot_always::>( + WriteBufferEntry::VT_PARTITION_KEY, + partition_key, + ); + } + #[inline] + pub fn add_table_batches( + &mut self, + table_batches: flatbuffers::WIPOffset< + flatbuffers::Vector<'b, flatbuffers::ForwardsUOffset>>, + >, + ) { + self.fbb_.push_slot_always::>( + WriteBufferEntry::VT_TABLE_BATCHES, + table_batches, + ); + } + #[inline] + pub fn add_delete(&mut self, delete: flatbuffers::WIPOffset>) { + self.fbb_ + .push_slot_always::>( + WriteBufferEntry::VT_DELETE, + delete, + ); + } + #[inline] + pub fn new( + _fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>, + ) -> WriteBufferEntryBuilder<'a, 'b> { + let start = _fbb.start_table(); + WriteBufferEntryBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl std::fmt::Debug for WriteBufferEntry<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("WriteBufferEntry"); + ds.field("partition_key", &self.partition_key()); + ds.field("table_batches", &self.table_batches()); + ds.field("delete", &self.delete()); + ds.finish() + } + } + pub enum TableWriteBatchOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct TableWriteBatch<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for TableWriteBatch<'a> { + type Inner = TableWriteBatch<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf, loc }, + } + } + } + + impl<'a> TableWriteBatch<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + TableWriteBatch { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args TableWriteBatchArgs<'args>, + ) -> flatbuffers::WIPOffset> { + let mut builder = TableWriteBatchBuilder::new(_fbb); + if let Some(x) = args.rows { + builder.add_rows(x); + } + if let Some(x) = args.name { + builder.add_name(x); + } + builder.finish() + } + + pub const VT_NAME: flatbuffers::VOffsetT = 4; + pub const VT_ROWS: flatbuffers::VOffsetT = 6; + + #[inline] + pub fn name(&self) -> Option<&'a str> { + self._tab + .get::>(TableWriteBatch::VT_NAME, None) + } + #[inline] + pub fn rows( + &self, + ) -> Option>>> { + self._tab.get::>, + >>(TableWriteBatch::VT_ROWS, None) + } + } + + impl flatbuffers::Verifiable for TableWriteBatch<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::>(&"name", Self::VT_NAME, false)? + .visit_field::>, + >>(&"rows", Self::VT_ROWS, false)? + .finish(); + Ok(()) + } + } + pub struct TableWriteBatchArgs<'a> { + pub name: Option>, + pub rows: Option< + flatbuffers::WIPOffset>>>, + >, + } + impl<'a> Default for TableWriteBatchArgs<'a> { + #[inline] + fn default() -> Self { + TableWriteBatchArgs { + name: None, + rows: None, + } + } + } + pub struct TableWriteBatchBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b> TableWriteBatchBuilder<'a, 'b> { + #[inline] + pub fn add_name(&mut self, name: flatbuffers::WIPOffset<&'b str>) { + self.fbb_ + .push_slot_always::>(TableWriteBatch::VT_NAME, name); + } + #[inline] + pub fn add_rows( + &mut self, + rows: flatbuffers::WIPOffset< + flatbuffers::Vector<'b, flatbuffers::ForwardsUOffset>>, + >, + ) { + self.fbb_ + .push_slot_always::>(TableWriteBatch::VT_ROWS, rows); + } + #[inline] + pub fn new( + _fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>, + ) -> TableWriteBatchBuilder<'a, 'b> { + let start = _fbb.start_table(); + TableWriteBatchBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl std::fmt::Debug for TableWriteBatch<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("TableWriteBatch"); + ds.field("name", &self.name()); + ds.field("rows", &self.rows()); + ds.finish() + } + } + pub enum RowOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct Row<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for Row<'a> { + type Inner = Row<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf, loc }, + } + } + } + + impl<'a> Row<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + Row { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args RowArgs<'args>, + ) -> flatbuffers::WIPOffset> { + let mut builder = RowBuilder::new(_fbb); + if let Some(x) = args.values { + builder.add_values(x); + } + builder.finish() + } + + pub const VT_VALUES: flatbuffers::VOffsetT = 4; + + #[inline] + pub fn values( + &self, + ) -> Option>>> { + self._tab.get::>, + >>(Row::VT_VALUES, None) + } + } + + impl flatbuffers::Verifiable for Row<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::>, + >>(&"values", Self::VT_VALUES, false)? + .finish(); + Ok(()) + } + } + pub struct RowArgs<'a> { + pub values: Option< + flatbuffers::WIPOffset< + flatbuffers::Vector<'a, flatbuffers::ForwardsUOffset>>, + >, + >, + } + impl<'a> Default for RowArgs<'a> { + #[inline] + fn default() -> Self { + RowArgs { values: None } + } + } + pub struct RowBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b> RowBuilder<'a, 'b> { + #[inline] + pub fn add_values( + &mut self, + values: flatbuffers::WIPOffset< + flatbuffers::Vector<'b, flatbuffers::ForwardsUOffset>>, + >, + ) { + self.fbb_ + .push_slot_always::>(Row::VT_VALUES, values); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> RowBuilder<'a, 'b> { + let start = _fbb.start_table(); + RowBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl std::fmt::Debug for Row<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("Row"); + ds.field("values", &self.values()); + ds.finish() + } + } + pub enum TagValueOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct TagValue<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for TagValue<'a> { + type Inner = TagValue<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf, loc }, + } + } + } + + impl<'a> TagValue<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + TagValue { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args TagValueArgs<'args>, + ) -> flatbuffers::WIPOffset> { + let mut builder = TagValueBuilder::new(_fbb); + if let Some(x) = args.value { + builder.add_value(x); + } + builder.finish() + } + + pub const VT_VALUE: flatbuffers::VOffsetT = 4; + + #[inline] + pub fn value(&self) -> Option<&'a str> { + self._tab + .get::>(TagValue::VT_VALUE, None) + } + } + + impl flatbuffers::Verifiable for TagValue<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::>(&"value", Self::VT_VALUE, false)? + .finish(); + Ok(()) + } + } + pub struct TagValueArgs<'a> { + pub value: Option>, + } + impl<'a> Default for TagValueArgs<'a> { + #[inline] + fn default() -> Self { + TagValueArgs { value: None } + } + } + pub struct TagValueBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b> TagValueBuilder<'a, 'b> { + #[inline] + pub fn add_value(&mut self, value: flatbuffers::WIPOffset<&'b str>) { + self.fbb_ + .push_slot_always::>(TagValue::VT_VALUE, value); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> TagValueBuilder<'a, 'b> { + let start = _fbb.start_table(); + TagValueBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl std::fmt::Debug for TagValue<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("TagValue"); + ds.field("value", &self.value()); + ds.finish() + } + } + pub enum ValueOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct Value<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for Value<'a> { + type Inner = Value<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf, loc }, + } + } + } + + impl<'a> Value<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + Value { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args ValueArgs<'args>, + ) -> flatbuffers::WIPOffset> { + let mut builder = ValueBuilder::new(_fbb); + if let Some(x) = args.value { + builder.add_value(x); + } + if let Some(x) = args.column { + builder.add_column(x); + } + builder.add_value_type(args.value_type); + builder.finish() + } + + pub const VT_COLUMN: flatbuffers::VOffsetT = 4; + pub const VT_VALUE_TYPE: flatbuffers::VOffsetT = 6; + pub const VT_VALUE: flatbuffers::VOffsetT = 8; + + #[inline] + pub fn column(&self) -> Option<&'a str> { + self._tab + .get::>(Value::VT_COLUMN, None) + } + #[inline] + pub fn value_type(&self) -> ColumnValue { + self._tab + .get::(Value::VT_VALUE_TYPE, Some(ColumnValue::NONE)) + .unwrap() + } + #[inline] + pub fn value(&self) -> Option> { + self._tab + .get::>>(Value::VT_VALUE, None) + } + #[inline] + #[allow(non_snake_case)] + pub fn value_as_tag_value(&self) -> Option> { + if self.value_type() == ColumnValue::TagValue { + self.value().map(TagValue::init_from_table) + } else { + None + } + } + + #[inline] + #[allow(non_snake_case)] + pub fn value_as_i64value(&self) -> Option> { + if self.value_type() == ColumnValue::I64Value { + self.value().map(I64Value::init_from_table) + } else { + None + } + } + + #[inline] + #[allow(non_snake_case)] + pub fn value_as_u64value(&self) -> Option> { + if self.value_type() == ColumnValue::U64Value { + self.value().map(U64Value::init_from_table) + } else { + None + } + } + + #[inline] + #[allow(non_snake_case)] + pub fn value_as_f64value(&self) -> Option> { + if self.value_type() == ColumnValue::F64Value { + self.value().map(F64Value::init_from_table) + } else { + None + } + } + + #[inline] + #[allow(non_snake_case)] + pub fn value_as_bool_value(&self) -> Option> { + if self.value_type() == ColumnValue::BoolValue { + self.value().map(BoolValue::init_from_table) + } else { + None + } + } + + #[inline] + #[allow(non_snake_case)] + pub fn value_as_string_value(&self) -> Option> { + if self.value_type() == ColumnValue::StringValue { + self.value().map(StringValue::init_from_table) + } else { + None + } + } + } + + impl flatbuffers::Verifiable for Value<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::>( + &"column", + Self::VT_COLUMN, + false, + )? + .visit_union::( + &"value_type", + Self::VT_VALUE_TYPE, + &"value", + Self::VT_VALUE, + false, + |key, v, pos| match key { + ColumnValue::TagValue => v + .verify_union_variant::>( + "ColumnValue::TagValue", + pos, + ), + ColumnValue::I64Value => v + .verify_union_variant::>( + "ColumnValue::I64Value", + pos, + ), + ColumnValue::U64Value => v + .verify_union_variant::>( + "ColumnValue::U64Value", + pos, + ), + ColumnValue::F64Value => v + .verify_union_variant::>( + "ColumnValue::F64Value", + pos, + ), + ColumnValue::BoolValue => v + .verify_union_variant::>( + "ColumnValue::BoolValue", + pos, + ), + ColumnValue::StringValue => v + .verify_union_variant::>( + "ColumnValue::StringValue", + pos, + ), + _ => Ok(()), + }, + )? + .finish(); + Ok(()) + } + } + pub struct ValueArgs<'a> { + pub column: Option>, + pub value_type: ColumnValue, + pub value: Option>, + } + impl<'a> Default for ValueArgs<'a> { + #[inline] + fn default() -> Self { + ValueArgs { + column: None, + value_type: ColumnValue::NONE, + value: None, + } + } + } + pub struct ValueBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b> ValueBuilder<'a, 'b> { + #[inline] + pub fn add_column(&mut self, column: flatbuffers::WIPOffset<&'b str>) { + self.fbb_ + .push_slot_always::>(Value::VT_COLUMN, column); + } + #[inline] + pub fn add_value_type(&mut self, value_type: ColumnValue) { + self.fbb_ + .push_slot::(Value::VT_VALUE_TYPE, value_type, ColumnValue::NONE); + } + #[inline] + pub fn add_value(&mut self, value: flatbuffers::WIPOffset) { + self.fbb_ + .push_slot_always::>(Value::VT_VALUE, value); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>) -> ValueBuilder<'a, 'b> { + let start = _fbb.start_table(); + ValueBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl std::fmt::Debug for Value<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("Value"); + ds.field("column", &self.column()); + ds.field("value_type", &self.value_type()); + match self.value_type() { + ColumnValue::TagValue => { + if let Some(x) = self.value_as_tag_value() { + ds.field("value", &x) + } else { + ds.field( + "value", + &"InvalidFlatbuffer: Union discriminant does not match value.", + ) + } + } + ColumnValue::I64Value => { + if let Some(x) = self.value_as_i64value() { + ds.field("value", &x) + } else { + ds.field( + "value", + &"InvalidFlatbuffer: Union discriminant does not match value.", + ) + } + } + ColumnValue::U64Value => { + if let Some(x) = self.value_as_u64value() { + ds.field("value", &x) + } else { + ds.field( + "value", + &"InvalidFlatbuffer: Union discriminant does not match value.", + ) + } + } + ColumnValue::F64Value => { + if let Some(x) = self.value_as_f64value() { + ds.field("value", &x) + } else { + ds.field( + "value", + &"InvalidFlatbuffer: Union discriminant does not match value.", + ) + } + } + ColumnValue::BoolValue => { + if let Some(x) = self.value_as_bool_value() { + ds.field("value", &x) + } else { + ds.field( + "value", + &"InvalidFlatbuffer: Union discriminant does not match value.", + ) + } + } + ColumnValue::StringValue => { + if let Some(x) = self.value_as_string_value() { + ds.field("value", &x) + } else { + ds.field( + "value", + &"InvalidFlatbuffer: Union discriminant does not match value.", + ) + } + } + _ => { + let x: Option<()> = None; + ds.field("value", &x) + } + }; + ds.finish() + } + } + pub enum WriteBufferDeleteOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct WriteBufferDelete<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for WriteBufferDelete<'a> { + type Inner = WriteBufferDelete<'a>; + #[inline] + fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table { buf, loc }, + } + } + } + + impl<'a> WriteBufferDelete<'a> { + #[inline] + pub fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + WriteBufferDelete { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr>, + args: &'args WriteBufferDeleteArgs<'args>, + ) -> flatbuffers::WIPOffset> { + let mut builder = WriteBufferDeleteBuilder::new(_fbb); + if let Some(x) = args.predicate { + builder.add_predicate(x); + } + if let Some(x) = args.table_name { + builder.add_table_name(x); + } + builder.finish() + } + + pub const VT_TABLE_NAME: flatbuffers::VOffsetT = 4; + pub const VT_PREDICATE: flatbuffers::VOffsetT = 6; + + #[inline] + pub fn table_name(&self) -> Option<&'a str> { + self._tab + .get::>(WriteBufferDelete::VT_TABLE_NAME, None) + } + #[inline] + pub fn predicate(&self) -> Option<&'a str> { + self._tab + .get::>(WriteBufferDelete::VT_PREDICATE, None) + } + } + + impl flatbuffers::Verifiable for WriteBufferDelete<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, + pos: usize, + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::>( + &"table_name", + Self::VT_TABLE_NAME, + false, + )? + .visit_field::>( + &"predicate", + Self::VT_PREDICATE, + false, + )? + .finish(); + Ok(()) + } + } + pub struct WriteBufferDeleteArgs<'a> { + pub table_name: Option>, + pub predicate: Option>, + } + impl<'a> Default for WriteBufferDeleteArgs<'a> { + #[inline] + fn default() -> Self { + WriteBufferDeleteArgs { + table_name: None, + predicate: None, + } + } + } + pub struct WriteBufferDeleteBuilder<'a: 'b, 'b> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b> WriteBufferDeleteBuilder<'a, 'b> { + #[inline] + pub fn add_table_name(&mut self, table_name: flatbuffers::WIPOffset<&'b str>) { + self.fbb_.push_slot_always::>( + WriteBufferDelete::VT_TABLE_NAME, + table_name, + ); + } + #[inline] + pub fn add_predicate(&mut self, predicate: flatbuffers::WIPOffset<&'b str>) { + self.fbb_.push_slot_always::>( + WriteBufferDelete::VT_PREDICATE, + predicate, + ); + } + #[inline] + pub fn new( + _fbb: &'b mut flatbuffers::FlatBufferBuilder<'a>, + ) -> WriteBufferDeleteBuilder<'a, 'b> { + let start = _fbb.start_table(); + WriteBufferDeleteBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl std::fmt::Debug for WriteBufferDelete<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("WriteBufferDelete"); + ds.field("table_name", &self.table_name()); + ds.field("predicate", &self.predicate()); + ds.finish() + } + } +} // pub mod wal diff --git a/google_types/Cargo.toml b/google_types/Cargo.toml index bfa9070899..ec72717045 100644 --- a/google_types/Cargo.toml +++ b/google_types/Cargo.toml @@ -7,9 +7,6 @@ edition = "2018" [dependencies] # In alphabetical order prost = "0.7" -prost-types = "0.7" -tonic = "0.4" -tracing = { version = "0.1" } [build-dependencies] # In alphabetical order prost-build = "0.7" diff --git a/google_types/build.rs b/google_types/build.rs index 5d30200d49..0112e26e84 100644 --- a/google_types/build.rs +++ b/google_types/build.rs @@ -9,11 +9,7 @@ type Result = std::result::Result; fn main() -> Result<()> { let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("protos"); - let proto_files = vec![ - root.join("google/rpc/error_details.proto"), - root.join("google/rpc/status.proto"), - root.join("google/protobuf/types.proto"), - ]; + let proto_files = vec![root.join("google/protobuf/types.proto")]; // Tell cargo to recompile if any of these proto files are changed for proto_file in &proto_files { diff --git a/google_types/src/lib.rs b/google_types/src/lib.rs index 74ad9b76e2..6a41539c3d 100644 --- a/google_types/src/lib.rs +++ b/google_types/src/lib.rs @@ -36,260 +36,7 @@ mod pb { } } } - - pub mod rpc { - include!(concat!(env!("OUT_DIR"), "/google.rpc.rs")); - } } } pub use pb::google::*; - -use pb::google::protobuf::Any; -use prost::{ - bytes::{Bytes, BytesMut}, - Message, -}; -use std::convert::{TryFrom, TryInto}; -use std::iter::FromIterator; -use tonic::Status; -use tracing::error; - -// A newtype struct to provide conversion into tonic::Status -struct EncodeError(prost::EncodeError); - -impl From for tonic::Status { - fn from(error: EncodeError) -> Self { - error!(error=%error.0, "failed to serialise error response details"); - tonic::Status::unknown(format!("failed to serialise server error: {}", error.0)) - } -} - -impl From for EncodeError { - fn from(e: prost::EncodeError) -> Self { - Self(e) - } -} - -fn encode_status(code: tonic::Code, message: String, details: Any) -> tonic::Status { - let mut buffer = BytesMut::new(); - - let status = pb::google::rpc::Status { - code: code as i32, - message: message.clone(), - details: vec![details], - }; - - match status.encode(&mut buffer) { - Ok(_) => tonic::Status::with_details(code, message, buffer.freeze()), - Err(e) => EncodeError(e).into(), - } -} - -#[derive(Debug, Default, Clone)] -pub struct FieldViolation { - pub field: String, - pub description: String, -} - -impl FieldViolation { - pub fn required(field: impl Into) -> Self { - Self { - field: field.into(), - description: "Field is required".to_string(), - } - } - - /// Re-scopes this error as the child of another field - pub fn scope(self, field: impl Into) -> Self { - let field = if self.field.is_empty() { - field.into() - } else { - [field.into(), self.field].join(".") - }; - - Self { - field, - description: self.description, - } - } -} - -fn encode_bad_request(violation: Vec) -> Result { - let mut buffer = BytesMut::new(); - - pb::google::rpc::BadRequest { - field_violations: violation - .into_iter() - .map(|f| pb::google::rpc::bad_request::FieldViolation { - field: f.field, - description: f.description, - }) - .collect(), - } - .encode(&mut buffer)?; - - Ok(Any { - type_url: "type.googleapis.com/google.rpc.BadRequest".to_string(), - value: buffer.freeze(), - }) -} - -impl From for tonic::Status { - fn from(f: FieldViolation) -> Self { - let message = format!("Violation for field \"{}\": {}", f.field, f.description); - - match encode_bad_request(vec![f]) { - Ok(details) => encode_status(tonic::Code::InvalidArgument, message, details), - Err(e) => e.into(), - } - } -} - -#[derive(Debug, Default, Clone)] -pub struct InternalError {} - -impl From for tonic::Status { - fn from(_: InternalError) -> Self { - tonic::Status::new(tonic::Code::Internal, "Internal Error") - } -} - -#[derive(Debug, Default, Clone)] -pub struct AlreadyExists { - pub resource_type: String, - pub resource_name: String, - pub owner: String, - pub description: String, -} - -fn encode_resource_info( - resource_type: String, - resource_name: String, - owner: String, - description: String, -) -> Result { - let mut buffer = BytesMut::new(); - - pb::google::rpc::ResourceInfo { - resource_type, - resource_name, - owner, - description, - } - .encode(&mut buffer)?; - - Ok(Any { - type_url: "type.googleapis.com/google.rpc.ResourceInfo".to_string(), - value: buffer.freeze(), - }) -} - -impl From for tonic::Status { - fn from(exists: AlreadyExists) -> Self { - let message = format!( - "Resource {}/{} already exists", - exists.resource_type, exists.resource_name - ); - match encode_resource_info( - exists.resource_type, - exists.resource_name, - exists.owner, - exists.description, - ) { - Ok(details) => encode_status(tonic::Code::AlreadyExists, message, details), - Err(e) => e.into(), - } - } -} - -#[derive(Debug, Default, Clone)] -pub struct NotFound { - pub resource_type: String, - pub resource_name: String, - pub owner: String, - pub description: String, -} - -impl From for tonic::Status { - fn from(not_found: NotFound) -> Self { - let message = format!( - "Resource {}/{} not found", - not_found.resource_type, not_found.resource_name - ); - match encode_resource_info( - not_found.resource_type, - not_found.resource_name, - not_found.owner, - not_found.description, - ) { - Ok(details) => encode_status(tonic::Code::NotFound, message, details), - Err(e) => e.into(), - } - } -} - -#[derive(Debug, Default, Clone)] -pub struct PreconditionViolation { - pub category: String, - pub subject: String, - pub description: String, -} - -fn encode_precondition_failure(violations: Vec) -> Result { - use pb::google::rpc::precondition_failure::Violation; - - let mut buffer = BytesMut::new(); - - pb::google::rpc::PreconditionFailure { - violations: violations - .into_iter() - .map(|x| Violation { - r#type: x.category, - subject: x.subject, - description: x.description, - }) - .collect(), - } - .encode(&mut buffer)?; - - Ok(Any { - type_url: "type.googleapis.com/google.rpc.PreconditionFailure".to_string(), - value: buffer.freeze(), - }) -} - -impl From for tonic::Status { - fn from(violation: PreconditionViolation) -> Self { - let message = format!( - "Precondition violation {} - {}: {}", - violation.subject, violation.category, violation.description - ); - match encode_precondition_failure(vec![violation]) { - Ok(details) => encode_status(tonic::Code::FailedPrecondition, message, details), - Err(e) => e.into(), - } - } -} - -/// An extension trait that adds the ability to convert an error -/// that can be converted to a String to a FieldViolation -pub trait FieldViolationExt { - type Output; - - fn field(self, field: &'static str) -> Result; -} - -impl FieldViolationExt for Result -where - E: ToString, -{ - type Output = T; - - fn field(self, field: &'static str) -> Result { - self.map_err(|e| FieldViolation { - field: field.to_string(), - description: e.to_string(), - }) - } -} diff --git a/influxdb2_client/README.md b/influxdb2_client/README.md new file mode 100644 index 0000000000..ce4842a8ea --- /dev/null +++ b/influxdb2_client/README.md @@ -0,0 +1,23 @@ +# InfluxDB V2 Client API + +This crate contains a work-in-progress implementation of a Rust client for the [InfluxDB 2.0 API](https://docs.influxdata.com/influxdb/v2.0/reference/api/). + +This client is not the Rust client for IOx. You can find that [here](../influxdb_iox_client). + +The InfluxDB IOx project plans to focus its efforts on the subset of the API which are most relevent to IOx, but we accept (welcome!) PRs for adding the other pieces of functionality. + + +## Design Notes + +When it makes sense, this client aims to mirror the [InfluxDB 2.x Go client API](https://github.com/influxdata/influxdb-client-go) + +## Contributing + +If you would like to contribute code you can do through GitHub by forking the repository and sending a pull request into the master branch. + + +## Future work + +- [ ] Publish as a crate on [crates.io](http://crates.io) + +If you would like to contribute code you can do through GitHub by forking the repository and sending a pull request into the main branch. diff --git a/influxdb2_client/examples/ready.rs b/influxdb2_client/examples/ready.rs new file mode 100644 index 0000000000..07d69c4d88 --- /dev/null +++ b/influxdb2_client/examples/ready.rs @@ -0,0 +1,11 @@ +#[tokio::main] +async fn main() -> Result<(), Box> { + let influx_url = "some-url"; + let token = "some-token"; + + let client = influxdb2_client::Client::new(influx_url, token); + + println!("{:?}", client.ready().await?); + + Ok(()) +} diff --git a/influxdb2_client/src/lib.rs b/influxdb2_client/src/lib.rs index 52a1b60e6a..f332a0a154 100644 --- a/influxdb2_client/src/lib.rs +++ b/influxdb2_client/src/lib.rs @@ -302,3 +302,5 @@ cpu,host=server01,region=us-west usage=0.87 Ok(()) } } + +mod ready; diff --git a/influxdb2_client/src/ready.rs b/influxdb2_client/src/ready.rs new file mode 100644 index 0000000000..fae9ecce94 --- /dev/null +++ b/influxdb2_client/src/ready.rs @@ -0,0 +1,51 @@ +use reqwest::{Method, StatusCode}; +use snafu::ResultExt; + +use super::{Client, Http, RequestError, ReqwestProcessing}; + +impl Client { + /// Get the readiness of an instance at startup + pub async fn ready(&self) -> Result { + let ready_url = format!("{}/ready", self.url); + let response = self + .request(Method::GET, &ready_url) + .send() + .await + .context(ReqwestProcessing)?; + + match response.status() { + StatusCode::OK => Ok(true), + _ => { + let status = response.status(); + let text = response.text().await.context(ReqwestProcessing)?; + Http { status, text }.fail()? + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use mockito::mock; + + type Error = Box; + type Result = std::result::Result; + + #[tokio::test] + async fn ready() -> Result { + let token = "some-token"; + + let mock_server = mock("GET", "/ready") + .match_header("Authorization", format!("Token {}", token).as_str()) + .create(); + + let client = Client::new(&mockito::server_url(), token); + + let _result = client.ready().await; + + mock_server.assert(); + + Ok(()) + } +} diff --git a/influxdb_iox_client/Cargo.toml b/influxdb_iox_client/Cargo.toml index 67e6274b8b..12f2383f44 100644 --- a/influxdb_iox_client/Cargo.toml +++ b/influxdb_iox_client/Cargo.toml @@ -6,6 +6,7 @@ edition = "2018" [features] flight = ["arrow_deps", "serde/derive", "serde_json", "futures-util"] +format = ["arrow_deps"] [dependencies] # Workspace dependencies, in alphabetical order @@ -23,5 +24,5 @@ tokio = { version = "1.0", features = ["macros"] } tonic = { version = "0.4.0" } [dev-dependencies] # In alphabetical order -rand = "0.8.1" +rand = "0.8.3" serde_json = "1.0" diff --git a/influxdb_iox_client/src/client.rs b/influxdb_iox_client/src/client.rs index 01addd884e..cf822b523e 100644 --- a/influxdb_iox_client/src/client.rs +++ b/influxdb_iox_client/src/client.rs @@ -4,6 +4,12 @@ pub mod health; /// Client for the management API pub mod management; +/// Client for the write API +pub mod write; + +/// Client for the operations API +pub mod operations; + #[cfg(feature = "flight")] /// Client for the flight API pub mod flight; diff --git a/influxdb_iox_client/src/client/management.rs b/influxdb_iox_client/src/client/management.rs index 2b7c2b1fee..0946609271 100644 --- a/influxdb_iox_client/src/client/management.rs +++ b/influxdb_iox_client/src/client/management.rs @@ -5,6 +5,7 @@ use thiserror::Error; use self::generated_types::{management_service_client::ManagementServiceClient, *}; use crate::connection::Connection; +use ::generated_types::google::longrunning::Operation; use std::convert::TryInto; /// Re-export generated_types @@ -80,8 +81,111 @@ pub enum GetDatabaseError { ServerError(tonic::Status), } +/// Errors returned by Client::list_chunks +#[derive(Debug, Error)] +pub enum ListChunksError { + /// Client received an unexpected error from the server + #[error("Unexpected server error: {}: {}", .0.code(), .0.message())] + ServerError(tonic::Status), +} + +/// Errors returned by Client::list_remotes +#[derive(Debug, Error)] +pub enum ListRemotesError { + /// Client received an unexpected error from the server + #[error("Unexpected server error: {}: {}", .0.code(), .0.message())] + ServerError(tonic::Status), +} + +/// Errors returned by Client::update_remote +#[derive(Debug, Error)] +pub enum UpdateRemoteError { + /// Client received an unexpected error from the server + #[error("Unexpected server error: {}: {}", .0.code(), .0.message())] + ServerError(tonic::Status), +} + +/// Errors returned by Client::create_dummy_job +#[derive(Debug, Error)] +pub enum CreateDummyJobError { + /// Response contained no payload + #[error("Server returned an empty response")] + EmptyResponse, + + /// Client received an unexpected error from the server + #[error("Unexpected server error: {}: {}", .0.code(), .0.message())] + ServerError(tonic::Status), +} + +/// Errors returned by Client::list_partitions +#[derive(Debug, Error)] +pub enum ListPartitionsError { + /// Database not found + #[error("Database not found")] + DatabaseNotFound, + + /// Client received an unexpected error from the server + #[error("Unexpected server error: {}: {}", .0.code(), .0.message())] + ServerError(tonic::Status), +} + +/// Errors returned by Client::get_partition +#[derive(Debug, Error)] +pub enum GetPartitionError { + /// Database not found + #[error("Database not found")] + DatabaseNotFound, + + /// Partition not found + #[error("Partition not found")] + PartitionNotFound, + + /// Client received an unexpected error from the server + #[error("Unexpected server error: {}: {}", .0.code(), .0.message())] + ServerError(tonic::Status), +} + +/// Errors returned by Client::list_partition_chunks +#[derive(Debug, Error)] +pub enum ListPartitionChunksError { + /// Client received an unexpected error from the server + #[error("Unexpected server error: {}: {}", .0.code(), .0.message())] + ServerError(tonic::Status), +} + +/// Errors returned by Client::new_partition_chunk +#[derive(Debug, Error)] +pub enum NewPartitionChunkError { + /// Database not found + #[error("Database not found")] + DatabaseNotFound, + + /// Client received an unexpected error from the server + #[error("Unexpected server error: {}: {}", .0.code(), .0.message())] + ServerError(tonic::Status), +} + +/// Errors returned by Client::close_partition_chunk +#[derive(Debug, Error)] +pub enum ClosePartitionChunkError { + /// Database not found + #[error("Database not found")] + DatabaseNotFound, + + /// Response contained no payload + #[error("Server returned an empty response")] + EmptyResponse, + + /// Client received an unexpected error from the server + #[error("Unexpected server error: {}: {}", .0.code(), .0.message())] + ServerError(tonic::Status), +} + /// An IOx Management API client. /// +/// This client wraps the underlying `tonic` generated client with a +/// more ergonomic interface. +/// /// ```no_run /// #[tokio::main] /// # async fn main() { @@ -198,4 +302,196 @@ impl Client { .ok_or(GetDatabaseError::EmptyResponse)?; Ok(rules) } + + /// List chunks in a database. + pub async fn list_chunks( + &mut self, + db_name: impl Into, + ) -> Result, ListChunksError> { + let db_name = db_name.into(); + + let response = self + .inner + .list_chunks(ListChunksRequest { db_name }) + .await + .map_err(ListChunksError::ServerError)?; + Ok(response.into_inner().chunks) + } + + /// List remotes. + pub async fn list_remotes(&mut self) -> Result, ListRemotesError> { + let response = self + .inner + .list_remotes(ListRemotesRequest {}) + .await + .map_err(ListRemotesError::ServerError)?; + Ok(response.into_inner().remotes) + } + + /// Update remote + pub async fn update_remote( + &mut self, + id: u32, + connection_string: impl Into, + ) -> Result<(), UpdateRemoteError> { + self.inner + .update_remote(UpdateRemoteRequest { + remote: Some(generated_types::Remote { + id, + connection_string: connection_string.into(), + }), + }) + .await + .map_err(UpdateRemoteError::ServerError)?; + Ok(()) + } + + /// Delete remote + pub async fn delete_remote(&mut self, id: u32) -> Result<(), UpdateRemoteError> { + self.inner + .delete_remote(DeleteRemoteRequest { id }) + .await + .map_err(UpdateRemoteError::ServerError)?; + Ok(()) + } + + /// List all partitions of the database + pub async fn list_partitions( + &mut self, + db_name: impl Into, + ) -> Result, ListPartitionsError> { + let db_name = db_name.into(); + let response = self + .inner + .list_partitions(ListPartitionsRequest { db_name }) + .await + .map_err(|status| match status.code() { + tonic::Code::NotFound => ListPartitionsError::DatabaseNotFound, + _ => ListPartitionsError::ServerError(status), + })?; + + let ListPartitionsResponse { partitions } = response.into_inner(); + + Ok(partitions) + } + + /// Get details about a specific partition + pub async fn get_partition( + &mut self, + db_name: impl Into, + partition_key: impl Into, + ) -> Result { + let db_name = db_name.into(); + let partition_key = partition_key.into(); + + let response = self + .inner + .get_partition(GetPartitionRequest { + db_name, + partition_key, + }) + .await + .map_err(|status| match status.code() { + tonic::Code::NotFound => GetPartitionError::DatabaseNotFound, + _ => GetPartitionError::ServerError(status), + })?; + + let GetPartitionResponse { partition } = response.into_inner(); + + partition.ok_or(GetPartitionError::PartitionNotFound) + } + + /// List chunks in a partition + pub async fn list_partition_chunks( + &mut self, + db_name: impl Into, + partition_key: impl Into, + ) -> Result, ListPartitionChunksError> { + let db_name = db_name.into(); + let partition_key = partition_key.into(); + + let response = self + .inner + .list_partition_chunks(ListPartitionChunksRequest { + db_name, + partition_key, + }) + .await + .map_err(ListPartitionChunksError::ServerError)?; + Ok(response.into_inner().chunks) + } + + /// Create a new chunk in a partittion + pub async fn new_partition_chunk( + &mut self, + db_name: impl Into, + partition_key: impl Into, + ) -> Result<(), NewPartitionChunkError> { + let db_name = db_name.into(); + let partition_key = partition_key.into(); + + self.inner + .new_partition_chunk(NewPartitionChunkRequest { + db_name, + partition_key, + }) + .await + .map_err(|status| match status.code() { + tonic::Code::NotFound => NewPartitionChunkError::DatabaseNotFound, + _ => NewPartitionChunkError::ServerError(status), + })?; + + Ok(()) + } + + /// Creates a dummy job that for each value of the nanos field + /// spawns a task that sleeps for that number of nanoseconds before + /// returning + pub async fn create_dummy_job( + &mut self, + nanos: Vec, + ) -> Result { + let response = self + .inner + .create_dummy_job(CreateDummyJobRequest { nanos }) + .await + .map_err(CreateDummyJobError::ServerError)?; + + Ok(response + .into_inner() + .operation + .ok_or(CreateDummyJobError::EmptyResponse)?) + } + + /// Closes the specified chunk in the specified partition and + /// begins it moving to the read buffer. + /// + /// Returns the job tracking the data's movement + pub async fn close_partition_chunk( + &mut self, + db_name: impl Into, + partition_key: impl Into, + chunk_id: u32, + ) -> Result { + let db_name = db_name.into(); + let partition_key = partition_key.into(); + + let response = self + .inner + .close_partition_chunk(ClosePartitionChunkRequest { + db_name, + partition_key, + chunk_id, + }) + .await + .map_err(|status| match status.code() { + tonic::Code::NotFound => ClosePartitionChunkError::DatabaseNotFound, + _ => ClosePartitionChunkError::ServerError(status), + })?; + + Ok(response + .into_inner() + .operation + .ok_or(ClosePartitionChunkError::EmptyResponse)?) + } } diff --git a/influxdb_iox_client/src/client/operations.rs b/influxdb_iox_client/src/client/operations.rs new file mode 100644 index 0000000000..462baeb0b3 --- /dev/null +++ b/influxdb_iox_client/src/client/operations.rs @@ -0,0 +1,125 @@ +use thiserror::Error; + +use ::generated_types::google::FieldViolation; + +use crate::connection::Connection; + +use self::generated_types::{operations_client::OperationsClient, *}; + +/// Re-export generated_types +pub mod generated_types { + pub use generated_types::google::longrunning::*; +} + +/// Error type for the operations Client +#[derive(Debug, Error)] +pub enum Error { + /// Client received an invalid response + #[error("Invalid server response: {}", .0)] + InvalidResponse(#[from] FieldViolation), + + /// Operation was not found + #[error("Operation not found: {}", .0)] + NotFound(usize), + + /// Client received an unexpected error from the server + #[error("Unexpected server error: {}: {}", .0.code(), .0.message())] + ServerError(tonic::Status), +} + +/// Result type for the operations Client +pub type Result = std::result::Result; + +/// An IOx Long Running Operations API client. +/// +/// ```no_run +/// #[tokio::main] +/// # async fn main() { +/// use influxdb_iox_client::{ +/// operations::Client, +/// connection::Builder, +/// }; +/// +/// let mut connection = Builder::default() +/// .build("http://127.0.0.1:8082") +/// .await +/// .unwrap(); +/// +/// let mut client = Client::new(connection); +/// # } +/// ``` +#[derive(Debug, Clone)] +pub struct Client { + inner: OperationsClient, +} + +impl Client { + /// Creates a new client with the provided connection + pub fn new(channel: tonic::transport::Channel) -> Self { + Self { + inner: OperationsClient::new(channel), + } + } + + /// Get information about all operations + pub async fn list_operations(&mut self) -> Result> { + Ok(self + .inner + .list_operations(ListOperationsRequest::default()) + .await + .map_err(Error::ServerError)? + .into_inner() + .operations) + } + + /// Get information about a specific operation + pub async fn get_operation(&mut self, id: usize) -> Result { + Ok(self + .inner + .get_operation(GetOperationRequest { + name: id.to_string(), + }) + .await + .map_err(|e| match e.code() { + tonic::Code::NotFound => Error::NotFound(id), + _ => Error::ServerError(e), + })? + .into_inner()) + } + + /// Cancel a given operation + pub async fn cancel_operation(&mut self, id: usize) -> Result<()> { + self.inner + .cancel_operation(CancelOperationRequest { + name: id.to_string(), + }) + .await + .map_err(|e| match e.code() { + tonic::Code::NotFound => Error::NotFound(id), + _ => Error::ServerError(e), + })?; + + Ok(()) + } + + /// Waits until an operation completes, or the timeout expires, and + /// returns the latest operation metadata + pub async fn wait_operation( + &mut self, + id: usize, + timeout: Option, + ) -> Result { + Ok(self + .inner + .wait_operation(WaitOperationRequest { + name: id.to_string(), + timeout: timeout.map(Into::into), + }) + .await + .map_err(|e| match e.code() { + tonic::Code::NotFound => Error::NotFound(id), + _ => Error::ServerError(e), + })? + .into_inner()) + } +} diff --git a/influxdb_iox_client/src/client/write.rs b/influxdb_iox_client/src/client/write.rs new file mode 100644 index 0000000000..4f57e0bd06 --- /dev/null +++ b/influxdb_iox_client/src/client/write.rs @@ -0,0 +1,77 @@ +use thiserror::Error; + +use self::generated_types::{write_service_client::WriteServiceClient, *}; + +use crate::connection::Connection; + +/// Re-export generated_types +pub mod generated_types { + pub use generated_types::influxdata::iox::write::v1::*; +} + +/// Errors returned by Client::write_data +#[derive(Debug, Error)] +pub enum WriteError { + /// Client received an unexpected error from the server + #[error("Unexpected server error: {}: {}", .0.code(), .0.message())] + ServerError(tonic::Status), +} + +/// An IOx Write API client. +/// +/// ```no_run +/// #[tokio::main] +/// # async fn main() { +/// use influxdb_iox_client::{ +/// write::Client, +/// connection::Builder, +/// }; +/// +/// let mut connection = Builder::default() +/// .build("http://127.0.0.1:8082") +/// .await +/// .unwrap(); +/// +/// let mut client = Client::new(connection); +/// +/// // write a line of line procol data +/// client +/// .write("bananas", "cpu,region=west user=23.2 100") +/// .await +/// .expect("failed to create database"); +/// # } +/// ``` +#[derive(Debug, Clone)] +pub struct Client { + inner: WriteServiceClient, +} + +impl Client { + /// Creates a new client with the provided connection + pub fn new(channel: tonic::transport::Channel) -> Self { + Self { + inner: WriteServiceClient::new(channel), + } + } + + /// Write the [LineProtocol] formatted data in `lp_data` to + /// database `name`. Returns the number of lines which were parsed + /// and written to the database + /// + /// [LineProtocol](https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/#data-types-and-format) + pub async fn write( + &mut self, + db_name: impl Into, + lp_data: impl Into, + ) -> Result { + let db_name = db_name.into(); + let lp_data = lp_data.into(); + let response = self + .inner + .write(WriteRequest { db_name, lp_data }) + .await + .map_err(WriteError::ServerError)?; + + Ok(response.into_inner().lines_written as usize) + } +} diff --git a/influxdb_iox_client/src/format.rs b/influxdb_iox_client/src/format.rs new file mode 100644 index 0000000000..4e05efe7fa --- /dev/null +++ b/influxdb_iox_client/src/format.rs @@ -0,0 +1,217 @@ +//! Output formatting utilities for Arrow record batches + +use std::{fmt::Display, str::FromStr}; + +use thiserror::Error; + +use arrow_deps::arrow::{ + self, csv::WriterBuilder, error::ArrowError, json::ArrayWriter, record_batch::RecordBatch, +}; + +/// Error type for results formatting +#[derive(Debug, Error)] +pub enum Error { + /// Unknown formatting type + #[error("Unknown format type: {}. Expected one of 'pretty', 'csv' or 'json'", .0)] + Invalid(String), + + /// Error pretty printing + #[error("Arrow pretty printing error: {}", .0)] + PrettyArrow(ArrowError), + + /// Error during CSV conversion + #[error("Arrow csv printing error: {}", .0)] + CsvArrow(ArrowError), + + /// Error during JSON conversion + #[error("Arrow json printing error: {}", .0)] + JsonArrow(ArrowError), + + /// Error converting CSV output to utf-8 + #[error("Error converting CSV output to UTF-8: {}", .0)] + CsvUtf8(std::string::FromUtf8Error), + + /// Error converting JSON output to utf-8 + #[error("Error converting JSON output to UTF-8: {}", .0)] + JsonUtf8(std::string::FromUtf8Error), +} +type Result = std::result::Result; + +#[derive(Debug, Copy, Clone, PartialEq)] +/// Requested output format for the query endpoint +pub enum QueryOutputFormat { + /// Arrow pretty printer format (default) + Pretty, + /// Comma separated values + CSV, + /// Arrow JSON format + JSON, +} + +impl Display for QueryOutputFormat { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + QueryOutputFormat::Pretty => write!(f, "pretty"), + QueryOutputFormat::CSV => write!(f, "csv"), + QueryOutputFormat::JSON => write!(f, "json"), + } + } +} + +impl Default for QueryOutputFormat { + fn default() -> Self { + Self::Pretty + } +} + +impl FromStr for QueryOutputFormat { + type Err = Error; + + fn from_str(s: &str) -> Result { + match s.to_ascii_lowercase().as_str() { + "pretty" => Ok(Self::Pretty), + "csv" => Ok(Self::CSV), + "json" => Ok(Self::JSON), + _ => Err(Error::Invalid(s.to_string())), + } + } +} + +impl QueryOutputFormat { + /// Return the Mcontent-type of this format + pub fn content_type(&self) -> &'static str { + match self { + Self::Pretty => "text/plain", + Self::CSV => "text/csv", + Self::JSON => "application/json", + } + } +} + +impl QueryOutputFormat { + /// Format the [`RecordBatch`]es into a String in one of the + /// following formats: + /// + /// Pretty: + /// ```text + /// +----------------+--------------+-------+-----------------+------------+ + /// | bottom_degrees | location | state | surface_degrees | time | + /// +----------------+--------------+-------+-----------------+------------+ + /// | 50.4 | santa_monica | CA | 65.2 | 1568756160 | + /// +----------------+--------------+-------+-----------------+------------+ + /// ``` + /// + /// CSV: + /// ```text + /// bottom_degrees,location,state,surface_degrees,time + /// 50.4,santa_monica,CA,65.2,1568756160 + /// ``` + /// + /// JSON: + /// + /// Example (newline + whitespace added for clarity): + /// ```text + /// [ + /// {"bottom_degrees":50.4,"location":"santa_monica","state":"CA","surface_degrees":65.2,"time":1568756160}, + /// {"location":"Boston","state":"MA","surface_degrees":50.2,"time":1568756160} + /// ] + /// ``` + pub fn format(&self, batches: &[RecordBatch]) -> Result { + match self { + Self::Pretty => batches_to_pretty(&batches), + Self::CSV => batches_to_csv(&batches), + Self::JSON => batches_to_json(&batches), + } + } +} + +fn batches_to_pretty(batches: &[RecordBatch]) -> Result { + arrow::util::pretty::pretty_format_batches(batches).map_err(Error::PrettyArrow) +} + +fn batches_to_csv(batches: &[RecordBatch]) -> Result { + let mut bytes = vec![]; + + { + let mut writer = WriterBuilder::new().has_headers(true).build(&mut bytes); + + for batch in batches { + writer.write(batch).map_err(Error::CsvArrow)?; + } + } + let csv = String::from_utf8(bytes).map_err(Error::CsvUtf8)?; + Ok(csv) +} + +fn batches_to_json(batches: &[RecordBatch]) -> Result { + let mut bytes = vec![]; + + { + let mut writer = ArrayWriter::new(&mut bytes); + writer.write_batches(batches).map_err(Error::CsvArrow)?; + + writer.finish().map_err(Error::CsvArrow)?; + } + + let json = String::from_utf8(bytes).map_err(Error::JsonUtf8)?; + + Ok(json) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_from_str() { + assert_eq!( + QueryOutputFormat::from_str("pretty").unwrap(), + QueryOutputFormat::Pretty + ); + assert_eq!( + QueryOutputFormat::from_str("pRetty").unwrap(), + QueryOutputFormat::Pretty + ); + + assert_eq!( + QueryOutputFormat::from_str("csv").unwrap(), + QueryOutputFormat::CSV + ); + assert_eq!( + QueryOutputFormat::from_str("CSV").unwrap(), + QueryOutputFormat::CSV + ); + + assert_eq!( + QueryOutputFormat::from_str("json").unwrap(), + QueryOutputFormat::JSON + ); + assert_eq!( + QueryOutputFormat::from_str("JSON").unwrap(), + QueryOutputFormat::JSON + ); + + assert_eq!( + QueryOutputFormat::from_str("un").unwrap_err().to_string(), + "Unknown format type: un. Expected one of 'pretty', 'csv' or 'json'" + ); + } + + #[test] + fn test_from_roundtrip() { + assert_eq!( + QueryOutputFormat::from_str(&QueryOutputFormat::Pretty.to_string()).unwrap(), + QueryOutputFormat::Pretty + ); + + assert_eq!( + QueryOutputFormat::from_str(&QueryOutputFormat::CSV.to_string()).unwrap(), + QueryOutputFormat::CSV + ); + + assert_eq!( + QueryOutputFormat::from_str(&QueryOutputFormat::JSON.to_string()).unwrap(), + QueryOutputFormat::JSON + ); + } +} diff --git a/influxdb_iox_client/src/lib.rs b/influxdb_iox_client/src/lib.rs index 5686cfab00..1f2ab0dc45 100644 --- a/influxdb_iox_client/src/lib.rs +++ b/influxdb_iox_client/src/lib.rs @@ -8,12 +8,15 @@ )] #![allow(clippy::missing_docs_in_private_items)] -pub use client::{health, management}; +pub use generated_types::{protobuf_type_url, protobuf_type_url_eq}; -#[cfg(feature = "flight")] -pub use client::flight; +pub use client::*; /// Builder for constructing connections for use with the various gRPC clients pub mod connection; +#[cfg(feature = "format")] +/// Output formatting utilities +pub mod format; + mod client; diff --git a/influxdb_line_protocol/src/lib.rs b/influxdb_line_protocol/src/lib.rs index 6b0629bb46..89ab451edb 100644 --- a/influxdb_line_protocol/src/lib.rs +++ b/influxdb_line_protocol/src/lib.rs @@ -50,6 +50,12 @@ pub enum Error { value: String, }, + #[snafu(display(r#"Unable to parse unsigned integer value '{}'"#, value))] + UIntegerValueInvalid { + source: std::num::ParseIntError, + value: String, + }, + #[snafu(display(r#"Unable to parse floating-point value '{}'"#, value))] FloatValueInvalid { source: std::num::ParseFloatError, @@ -333,10 +339,11 @@ pub type FieldSet<'a> = SmallVec<[(EscapedStr<'a>, FieldValue<'a>); 4]>; pub type TagSet<'a> = SmallVec<[(EscapedStr<'a>, EscapedStr<'a>); 8]>; /// Allowed types of Fields in a `ParsedLine`. One of the types described in -/// https://docs.influxdata.com/influxdb/v1.8/write_protocols/line_protocol_tutorial/#data-types +/// https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/#data-types-and-format #[derive(Debug, Clone, PartialEq)] pub enum FieldValue<'a> { I64(i64), + U64(u64), F64(f64), String(EscapedStr<'a>), Boolean(bool), @@ -349,6 +356,7 @@ impl<'a> Display for FieldValue<'a> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::I64(v) => write!(f, "{}i", v), + Self::U64(v) => write!(f, "{}u", v), Self::F64(v) => write!(f, "{}", v), Self::String(v) => escape_and_write_value(f, v, FIELD_VALUE_STRING_DELIMITERS), Self::Boolean(v) => write!(f, "{}", v), @@ -644,47 +652,76 @@ fn field_key(i: &str) -> IResult<&str, EscapedStr<'_>> { fn field_value(i: &str) -> IResult<&str, FieldValue<'_>> { let int = map(field_integer_value, FieldValue::I64); + let uint = map(field_uinteger_value, FieldValue::U64); let float = map(field_float_value, FieldValue::F64); let string = map(field_string_value, FieldValue::String); let boolv = map(field_bool_value, FieldValue::Boolean); - alt((int, float, string, boolv))(i) + alt((int, uint, float, string, boolv))(i) } fn field_integer_value(i: &str) -> IResult<&str, i64> { - let tagged_value = terminated(integral_value_common, tag("i")); + let tagged_value = terminated(integral_value_signed, tag("i")); map_fail(tagged_value, |value| { value.parse().context(IntegerValueInvalid { value }) })(i) } +fn field_uinteger_value(i: &str) -> IResult<&str, u64> { + let tagged_value = terminated(digit1, tag("u")); + map_fail(tagged_value, |value| { + value.parse().context(UIntegerValueInvalid { value }) + })(i) +} + fn field_float_value(i: &str) -> IResult<&str, f64> { - let value = alt((field_float_value_with_decimal, field_float_value_no_decimal)); + let value = alt(( + field_float_value_with_exponential_and_decimal, + field_float_value_with_exponential_no_decimal, + field_float_value_with_decimal, + field_float_value_no_decimal, + )); map_fail(value, |value| { value.parse().context(FloatValueInvalid { value }) })(i) } fn field_float_value_with_decimal(i: &str) -> IResult<&str, &str> { - recognize(separated_pair(integral_value_common, tag("."), digit1))(i) + recognize(separated_pair(integral_value_signed, tag("."), digit1))(i) +} + +fn field_float_value_with_exponential_and_decimal(i: &str) -> IResult<&str, &str> { + recognize(separated_pair( + integral_value_signed, + tag("."), + exponential_value, + ))(i) +} + +fn field_float_value_with_exponential_no_decimal(i: &str) -> IResult<&str, &str> { + exponential_value(i) +} + +fn exponential_value(i: &str) -> IResult<&str, &str> { + recognize(separated_pair(digit1, tag("e+"), digit1))(i) } fn field_float_value_no_decimal(i: &str) -> IResult<&str, &str> { - integral_value_common(i) + integral_value_signed(i) } -fn integral_value_common(i: &str) -> IResult<&str, &str> { +fn integral_value_signed(i: &str) -> IResult<&str, &str> { recognize(preceded(opt(tag("-")), digit1))(i) } fn timestamp(i: &str) -> IResult<&str, i64> { - map_fail(integral_value_common, |value| { + map_fail(integral_value_signed, |value| { value.parse().context(TimestampValueInvalid { value }) })(i) } fn field_string_value(i: &str) -> IResult<&str, EscapedStr<'_>> { - // https://docs.influxdata.com/influxdb/v1.8/write_protocols/line_protocol_tutorial/#data-types + // https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/#data-types-and-format // For string field values, backslash is only used to escape itself(\) or double // quotes. let string_data = alt(( @@ -707,7 +744,7 @@ fn field_string_value(i: &str) -> IResult<&str, EscapedStr<'_>> { } fn field_bool_value(i: &str) -> IResult<&str, bool> { - // https://docs.influxdata.com/influxdb/v1.8/write_protocols/line_protocol_tutorial/#data-types + // https://docs.influxdata.com/influxdb/v2.0/reference/syntax/line-protocol/#data-types-and-format // "specify TRUE with t, T, true, True, or TRUE. Specify FALSE with f, F, false, // False, or FALSE alt(( @@ -1037,6 +1074,13 @@ mod test { } } + fn unwrap_u64(&self) -> u64 { + match self { + Self::U64(v) => *v, + _ => panic!("field was not an u64"), + } + } + fn unwrap_f64(&self) -> f64 { match self { Self::F64(v) => *v, @@ -1202,6 +1246,19 @@ mod test { Ok(()) } + #[test] + fn parse_single_field_unteger() -> Result { + let input = "foo asdf=23u 1234"; + let vals = parse(input)?; + + assert_eq!(vals[0].series.measurement, "foo"); + assert_eq!(vals[0].timestamp, Some(1234)); + assert_eq!(vals[0].field_set[0].0, "asdf"); + assert_eq!(vals[0].field_set[0].1.unwrap_u64(), 23); + + Ok(()) + } + #[test] fn parse_single_field_float_no_decimal() -> Result { let input = "foo asdf=44 546"; @@ -1340,6 +1397,23 @@ mod test { Ok(()) } + #[test] + fn parse_two_fields_unteger() -> Result { + let input = "foo asdf=23u,bar=5u 1234"; + let vals = parse(input)?; + + assert_eq!(vals[0].series.measurement, "foo"); + assert_eq!(vals[0].timestamp, Some(1234)); + + assert_eq!(vals[0].field_set[0].0, "asdf"); + assert_eq!(vals[0].field_set[0].1.unwrap_u64(), 23); + + assert_eq!(vals[0].field_set[1].0, "bar"); + assert_eq!(vals[0].field_set[1].1.unwrap_u64(), 5); + + Ok(()) + } + #[test] fn parse_two_fields_float() -> Result { let input = "foo asdf=23.1,bar=5 1234"; @@ -1365,7 +1439,7 @@ mod test { #[test] fn parse_mixed_field_types() -> Result { - let input = r#"foo asdf=23.1,bar=5i,baz="the string",frab=false 1234"#; + let input = r#"foo asdf=23.1,bar=-5i,qux=9u,baz="the string",frab=false 1234"#; let vals = parse(input)?; assert_eq!(vals[0].series.measurement, "foo"); @@ -1378,13 +1452,16 @@ mod test { )); assert_eq!(vals[0].field_set[1].0, "bar"); - assert_eq!(vals[0].field_set[1].1.unwrap_i64(), 5); + assert_eq!(vals[0].field_set[1].1.unwrap_i64(), -5); - assert_eq!(vals[0].field_set[2].0, "baz"); - assert_eq!(vals[0].field_set[2].1.unwrap_string(), "the string"); + assert_eq!(vals[0].field_set[2].0, "qux"); + assert_eq!(vals[0].field_set[2].1.unwrap_u64(), 9); - assert_eq!(vals[0].field_set[3].0, "frab"); - assert_eq!(vals[0].field_set[3].1.unwrap_bool(), false); + assert_eq!(vals[0].field_set[3].0, "baz"); + assert_eq!(vals[0].field_set[3].1.unwrap_string(), "the string"); + + assert_eq!(vals[0].field_set[4].0, "frab"); + assert_eq!(vals[0].field_set[4].1.unwrap_bool(), false); Ok(()) } @@ -1400,6 +1477,49 @@ mod test { Ok(()) } + #[test] + fn parse_negative_uinteger() -> Result { + let input = "m0 field=-1u 99"; + let parsed = parse(input); + + assert!( + matches!(parsed, Err(super::Error::CannotParseEntireLine { .. })), + "Wrong error: {:?}", + parsed, + ); + + Ok(()) + } + + #[test] + fn parse_scientific_float() -> Result { + let input = "m0 field=-1.234456e+06 1615869152385000000"; + let vals = parse(input)?; + assert_eq!(vals.len(), 1); + + let input = "m0 field=1.234456e+06 1615869152385000000"; + let vals = parse(input)?; + assert_eq!(vals.len(), 1); + + let input = "m0 field=-1.234456e06 1615869152385000000"; + let parsed = parse(input); + assert!( + matches!(parsed, Err(super::Error::CannotParseEntireLine { .. })), + "Wrong error: {:?}", + parsed, + ); + + let input = "m0 field=1.234456e06 1615869152385000000"; + let parsed = parse(input); + assert!( + matches!(parsed, Err(super::Error::CannotParseEntireLine { .. })), + "Wrong error: {:?}", + parsed, + ); + + Ok(()) + } + #[test] fn parse_negative_float() -> Result { let input = "m0 field2=-1 99"; @@ -1428,6 +1548,20 @@ mod test { Ok(()) } + #[test] + fn parse_out_of_range_uinteger() -> Result { + let input = "m0 field=99999999999999999999999999999999u 99"; + let parsed = parse(input); + + assert!( + matches!(parsed, Err(super::Error::UIntegerValueInvalid { .. })), + "Wrong error: {:?}", + parsed, + ); + + Ok(()) + } + #[test] fn parse_out_of_range_float() -> Result { let input = format!("m0 field={val}.{val} 99", val = "9".repeat(200)); @@ -1913,7 +2047,8 @@ her"#, #[test] fn field_value_display() -> Result { - assert_eq!(FieldValue::I64(42).to_string(), "42i"); + assert_eq!(FieldValue::I64(-42).to_string(), "-42i"); + assert_eq!(FieldValue::U64(42).to_string(), "42u"); assert_eq!(FieldValue::F64(42.11).to_string(), "42.11"); assert_eq!( FieldValue::String(EscapedStr::from("foo")).to_string(), diff --git a/influxdb_tsm/Cargo.toml b/influxdb_tsm/Cargo.toml index bfe4a7251b..1355210456 100644 --- a/influxdb_tsm/Cargo.toml +++ b/influxdb_tsm/Cargo.toml @@ -13,5 +13,5 @@ tracing = "0.1" [dev-dependencies] # In alphabetical order flate2 = "1.0" hex = "0.4.2" -rand = "0.7.2" +rand = "0.8.3" test_helpers = { path = "../test_helpers" } diff --git a/influxdb_tsm/src/encoders/simple8b.rs b/influxdb_tsm/src/encoders/simple8b.rs index b6e1297fe5..131050923f 100644 --- a/influxdb_tsm/src/encoders/simple8b.rs +++ b/influxdb_tsm/src/encoders/simple8b.rs @@ -381,7 +381,7 @@ mod tests { let mut a = Vec::with_capacity(n as usize); for i in 0..n { let top_bit = (i & 1) << (bits - 1); - let v = rng.gen_range(0, max) | top_bit; + let v = rng.gen_range(0..max) | top_bit; assert!(v < max); a.push(v); } diff --git a/ingest/Cargo.toml b/ingest/Cargo.toml index b43ce27c94..743a283a24 100644 --- a/ingest/Cargo.toml +++ b/ingest/Cargo.toml @@ -6,9 +6,9 @@ edition = "2018" [dependencies] # In alphabetical order arrow_deps = { path = "../arrow_deps" } -data_types = { path = "../data_types" } influxdb_line_protocol = { path = "../influxdb_line_protocol" } influxdb_tsm = { path = "../influxdb_tsm" } +internal_types = { path = "../internal_types" } packers = { path = "../packers" } snafu = "0.6.2" tracing = "0.1" diff --git a/ingest/src/lib.rs b/ingest/src/lib.rs index 37f45a1c82..a07b7187d5 100644 --- a/ingest/src/lib.rs +++ b/ingest/src/lib.rs @@ -11,16 +11,15 @@ clippy::clone_on_ref_ptr )] -use data_types::{ - schema::{builder::InfluxSchemaBuilder, InfluxFieldType, Schema}, - TIME_COLUMN_NAME, -}; use influxdb_line_protocol::{FieldValue, ParsedLine}; use influxdb_tsm::{ mapper::{ColumnData, MeasurementTable, TSMMeasurementMapper}, reader::{BlockDecoder, TSMBlockReader, TSMIndexReader}, BlockType, TSMError, }; +use internal_types::schema::{ + builder::InfluxSchemaBuilder, InfluxFieldType, Schema, TIME_COLUMN_NAME, +}; use packers::{ ByteArray, Error as TableError, IOxTableWriter, IOxTableWriterSource, Packer, Packers, }; @@ -75,7 +74,7 @@ pub enum Error { #[snafu(display(r#"Error building schema: {}"#, source))] BuildingSchema { - source: data_types::schema::builder::Error, + source: internal_types::schema::builder::Error, }, #[snafu(display(r#"Error writing to TableWriter: {}"#, source))] @@ -96,8 +95,8 @@ pub enum Error { CouldNotFindColumn, } -impl From for Error { - fn from(source: data_types::schema::builder::Error) -> Self { +impl From for Error { + fn from(source: internal_types::schema::builder::Error) -> Self { Self::BuildingSchema { source } } } @@ -310,6 +309,7 @@ impl<'a> MeasurementSampler<'a> { let field_type = match field_value { FieldValue::F64(_) => InfluxFieldType::Float, FieldValue::I64(_) => InfluxFieldType::Integer, + FieldValue::U64(_) => InfluxFieldType::UInteger, FieldValue::String(_) => InfluxFieldType::String, FieldValue::Boolean(_) => InfluxFieldType::Boolean, }; @@ -474,6 +474,9 @@ fn pack_lines<'a>(schema: &Schema, lines: &[ParsedLine<'a>]) -> Vec { FieldValue::I64(i) => { packer.i64_packer_mut().push(i); } + FieldValue::U64(i) => { + packer.u64_packer_mut().push(i); + } FieldValue::String(ref s) => { packer.bytes_packer_mut().push(ByteArray::from(s.as_str())); } @@ -816,7 +819,8 @@ impl TSMFileConverter { mut block_reader: impl BlockDecoder, m: &mut MeasurementTable, ) -> Result<(Schema, Vec), Error> { - let mut builder = data_types::schema::builder::SchemaBuilder::new().measurement(&m.name); + let mut builder = + internal_types::schema::builder::SchemaBuilder::new().measurement(&m.name); let mut packed_columns: Vec = Vec::new(); let mut tks = Vec::new(); @@ -1095,11 +1099,11 @@ impl std::fmt::Debug for TSMFileConverter { #[cfg(test)] mod tests { use super::*; - use data_types::{assert_column_eq, schema::InfluxColumnType}; use influxdb_tsm::{ reader::{BlockData, MockBlockDecoder}, Block, }; + use internal_types::{assert_column_eq, schema::InfluxColumnType}; use packers::{Error as TableError, IOxTableWriter, IOxTableWriterSource, Packers}; use test_helpers::approximately_equal; diff --git a/ingest/src/parquet/writer.rs b/ingest/src/parquet/writer.rs index e83a7b9827..ab5046fd7c 100644 --- a/ingest/src/parquet/writer.rs +++ b/ingest/src/parquet/writer.rs @@ -1,7 +1,7 @@ //! This module contains the code to write table data to parquet use arrow_deps::parquet::{ self, - basic::{Compression, Encoding, LogicalType, Repetition, Type as PhysicalType}, + basic::{Compression, ConvertedType, Encoding, Repetition, Type as PhysicalType}, errors::ParquetError, file::{ properties::{WriterProperties, WriterPropertiesBuilder}, @@ -9,7 +9,7 @@ use arrow_deps::parquet::{ }, schema::types::{ColumnPath, Type}, }; -use data_types::schema::{InfluxColumnType, InfluxFieldType, Schema}; +use internal_types::schema::{InfluxColumnType, InfluxFieldType, Schema}; use parquet::file::writer::ParquetWriter; use snafu::{OptionExt, ResultExt, Snafu}; use std::{ @@ -97,7 +97,7 @@ where /// /// ``` /// # use std::fs; - /// # use data_types::schema::{builder::SchemaBuilder, InfluxFieldType}; + /// # use internal_types::schema::{builder::SchemaBuilder, InfluxFieldType}; /// # use packers::IOxTableWriter; /// # use packers::{Packer, Packers}; /// # use ingest::parquet::writer::{IOxParquetTableWriter, CompressionLevel}; @@ -297,19 +297,19 @@ fn convert_to_parquet_schema(schema: &Schema) -> Result (PhysicalType::BYTE_ARRAY, Some(LogicalType::UTF8)), + Some(InfluxColumnType::Tag) => (PhysicalType::BYTE_ARRAY, Some(ConvertedType::UTF8)), Some(InfluxColumnType::Field(InfluxFieldType::Boolean)) => { (PhysicalType::BOOLEAN, None) } Some(InfluxColumnType::Field(InfluxFieldType::Float)) => (PhysicalType::DOUBLE, None), Some(InfluxColumnType::Field(InfluxFieldType::Integer)) => { - (PhysicalType::INT64, Some(LogicalType::UINT_64)) + (PhysicalType::INT64, Some(ConvertedType::UINT_64)) } Some(InfluxColumnType::Field(InfluxFieldType::UInteger)) => { - (PhysicalType::INT64, Some(LogicalType::UINT_64)) + (PhysicalType::INT64, Some(ConvertedType::UINT_64)) } Some(InfluxColumnType::Field(InfluxFieldType::String)) => { - (PhysicalType::BYTE_ARRAY, Some(LogicalType::UTF8)) + (PhysicalType::BYTE_ARRAY, Some(ConvertedType::UTF8)) } Some(InfluxColumnType::Timestamp) => { // At the time of writing, the underlying rust parquet @@ -325,7 +325,7 @@ fn convert_to_parquet_schema(schema: &Schema) -> Result { return UnsupportedDataType { @@ -340,7 +340,7 @@ fn convert_to_parquet_schema(schema: &Schema) -> Result"] +edition = "2018" +description = "InfluxDB IOx internal types, shared between IOx instances" +readme = "README.md" + +[dependencies] +arrow_deps = { path = "../arrow_deps" } +crc32fast = "1.2.0" +chrono = { version = "0.4", features = ["serde"] } +data_types = { path = "../data_types" } +# See docs/regenerating_flatbuffers.md about updating generated code when updating the +# version of the flatbuffers crate +flatbuffers = "0.8" +generated_types = { path = "../generated_types" } +influxdb_line_protocol = { path = "../influxdb_line_protocol" } +ouroboros = "0.8.3" +snafu = "0.6" +tracing = "0.1" + +[dev-dependencies] # In alphabetical order +criterion = "0.3" + +[[bench]] +name = "benchmark" +harness = false diff --git a/internal_types/README.md b/internal_types/README.md new file mode 100644 index 0000000000..5ee9876ad8 --- /dev/null +++ b/internal_types/README.md @@ -0,0 +1,7 @@ +# Internal Types + +This crate contains InfluxDB IOx "internal" types which are shared +across crates and internally between IOx instances, but not exposed +externally to clients + +*Internal* in this case means that changing the structs is designed not to require additional coordination / organization with clients. diff --git a/data_types/benches/benchmark.rs b/internal_types/benches/benchmark.rs similarity index 97% rename from data_types/benches/benchmark.rs rename to internal_types/benches/benchmark.rs index 9914d41e57..abccc96f06 100644 --- a/data_types/benches/benchmark.rs +++ b/internal_types/benches/benchmark.rs @@ -1,12 +1,15 @@ use criterion::measurement::WallTime; use criterion::{criterion_group, criterion_main, Bencher, BenchmarkId, Criterion, Throughput}; -use data_types::data::{lines_to_replicated_write as lines_to_rw, ReplicatedWrite}; use data_types::database_rules::{DatabaseRules, PartitionTemplate, TemplatePart}; use generated_types::wal as wb; use influxdb_line_protocol::{parse_lines, ParsedLine}; -use std::collections::{BTreeMap, BTreeSet}; -use std::fmt; -use std::time::Duration; +use internal_types::data::{lines_to_replicated_write as lines_to_rw, ReplicatedWrite}; +use std::{ + collections::{BTreeMap, BTreeSet}, + convert::TryFrom, + fmt, + time::Duration, +}; const NEXT_ENTRY_NS: i64 = 1_000_000_000; const STARTING_TIMESTAMP_NS: i64 = 0; @@ -61,7 +64,7 @@ fn replicated_write_into_bytes(c: &mut Criterion) { assert_eq!(write.entry_count(), config.partition_count); b.iter(|| { - let _ = write.bytes().len(); + let _ = write.data().len(); }); }, ); @@ -73,7 +76,7 @@ fn bytes_into_struct(c: &mut Criterion) { run_group("bytes_into_struct", c, |lines, rules, config, b| { let write = lines_to_rw(0, 0, &lines, rules); assert_eq!(write.entry_count(), config.partition_count); - let data = write.bytes(); + let data = write.data(); b.iter(|| { let mut db = Db::default(); @@ -160,7 +163,7 @@ struct Db { impl Db { fn deserialize_write(&mut self, data: &[u8]) { - let write = ReplicatedWrite::from(data); + let write = ReplicatedWrite::try_from(data.to_vec()).unwrap(); if let Some(batch) = write.write_buffer_batch() { if let Some(entries) = batch.entries() { diff --git a/data_types/src/data.rs b/internal_types/src/data.rs similarity index 81% rename from data_types/src/data.rs rename to internal_types/src/data.rs index f511c5f47f..56b69111d0 100644 --- a/data_types/src/data.rs +++ b/internal_types/src/data.rs @@ -1,92 +1,103 @@ //! This module contains helper methods for constructing replicated writes //! based on `DatabaseRules`. -use crate::database_rules::Partitioner; -use crate::TIME_COLUMN_NAME; +use crate::schema::TIME_COLUMN_NAME; +use data_types::database_rules::Partitioner; use generated_types::wal as wb; use influxdb_line_protocol::{FieldValue, ParsedLine}; -use std::{collections::BTreeMap, fmt}; +use std::{collections::BTreeMap, convert::TryFrom, fmt}; use chrono::Utc; use crc32fast::Hasher; use flatbuffers::FlatBufferBuilder; +use ouroboros::self_referencing; pub fn type_description(value: wb::ColumnValue) -> &'static str { - use wb::ColumnValue::*; - match value { - NONE => "none", - TagValue => "tag", - I64Value => "i64", - U64Value => "u64", - F64Value => "f64", - BoolValue => "bool", - StringValue => "String", + wb::ColumnValue::TagValue => "tag", + wb::ColumnValue::I64Value => "i64", + wb::ColumnValue::U64Value => "u64", + wb::ColumnValue::F64Value => "f64", + wb::ColumnValue::BoolValue => "bool", + wb::ColumnValue::StringValue => "String", + wb::ColumnValue::NONE => "none", + _ => "none", } } /// A friendlier wrapper to help deal with the Flatbuffers write data -#[derive(Debug, Default, Clone, PartialEq)] +#[self_referencing] +#[derive(Debug, Clone, PartialEq)] pub struct ReplicatedWrite { - pub data: Vec, + data: Vec, + #[borrows(data)] + #[covariant] + fb: wb::ReplicatedWrite<'this>, + #[borrows(data)] + #[covariant] + write_buffer_batch: Option>, } impl ReplicatedWrite { - /// Returns the Flatbuffers struct represented by the raw bytes. - pub fn to_fb(&self) -> wb::ReplicatedWrite<'_> { - flatbuffers::get_root::>(&self.data) - } - /// Returns the Flatbuffers struct for the WriteBufferBatch in the raw bytes /// of the payload of the ReplicatedWrite. - pub fn write_buffer_batch(&self) -> Option> { - match self.to_fb().payload() { - Some(d) => Some(flatbuffers::get_root::>(&d)), - None => None, - } + pub fn write_buffer_batch(&self) -> Option<&wb::WriteBufferBatch<'_>> { + self.borrow_write_buffer_batch().as_ref() + } + + /// Returns the Flatbuffers struct for the ReplicatedWrite + pub fn fb(&self) -> &wb::ReplicatedWrite<'_> { + self.borrow_fb() } /// Returns true if this replicated write matches the writer and sequence. pub fn equal_to_writer_and_sequence(&self, writer_id: u32, sequence_number: u64) -> bool { - let fb = self.to_fb(); - fb.writer() == writer_id && fb.sequence() == sequence_number + self.fb().writer() == writer_id && self.fb().sequence() == sequence_number } /// Returns the writer id and sequence number pub fn writer_and_sequence(&self) -> (u32, u64) { - let fb = self.to_fb(); - (fb.writer(), fb.sequence()) + (self.fb().writer(), self.fb().sequence()) } - /// Returns the serialized bytes for the write. (used for benchmarking) - pub fn bytes(&self) -> &Vec { - &self.data + /// Returns the serialized bytes for the write + pub fn data(&self) -> &[u8] { + self.borrow_data() } /// Returns the number of write buffer entries in this replicated write pub fn entry_count(&self) -> usize { - if let Some(batch) = self.write_buffer_batch() { - if let Some(entries) = batch.entries() { - return entries.len(); - } - } - - 0 + self.write_buffer_batch() + .map_or(0, |wbb| wbb.entries().map_or(0, |entries| entries.len())) } } -impl From<&[u8]> for ReplicatedWrite { - fn from(data: &[u8]) -> Self { - Self { - data: Vec::from(data), +impl TryFrom> for ReplicatedWrite { + type Error = flatbuffers::InvalidFlatbuffer; + + fn try_from(data: Vec) -> Result { + ReplicatedWriteTryBuilder { + data, + fb_builder: |data| flatbuffers::root::>(data), + write_buffer_batch_builder: |data| match flatbuffers::root::>( + data, + )? + .payload() + { + Some(payload) => Ok(Some(flatbuffers::root::>( + &payload, + )?)), + None => Ok(None), + }, } + .try_build() } } impl fmt::Display for ReplicatedWrite { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let fb = self.to_fb(); + let fb = self.fb(); write!( f, "\nwriter:{}, sequence:{}, checksum:{}\n", @@ -143,6 +154,7 @@ impl fmt::Display for ReplicatedWrite { .unwrap_or("") .to_string(), wb::ColumnValue::NONE => "".to_string(), + _ => "".to_string(), }; write!(f, " {}:{}", value.column().unwrap_or(""), val)?; } @@ -192,9 +204,8 @@ pub fn lines_to_replicated_write( fbb.finish(write, None); let (mut data, idx) = fbb.collapse(); - ReplicatedWrite { - data: data.split_off(idx), - } + ReplicatedWrite::try_from(data.split_off(idx)) + .expect("Flatbuffer data just constructed should be valid") } pub fn split_lines_into_write_entry_partitions( @@ -317,6 +328,7 @@ fn add_line<'a>( for (column, value) in &line.field_set { let val = match value { FieldValue::I64(v) => add_i64_value(fbb, column.as_str(), *v), + FieldValue::U64(v) => add_u64_value(fbb, column.as_str(), *v), FieldValue::F64(v) => add_f64_value(fbb, column.as_str(), *v), FieldValue::Boolean(v) => add_bool_value(fbb, column.as_str(), *v), FieldValue::String(v) => add_string_value(fbb, column.as_str(), v.as_str()), @@ -393,6 +405,16 @@ fn add_i64_value<'a>( add_value(fbb, column, wb::ColumnValue::I64Value, iv.as_union_value()) } +fn add_u64_value<'a>( + fbb: &mut FlatBufferBuilder<'a>, + column: &str, + value: u64, +) -> flatbuffers::WIPOffset> { + let iv = wb::U64Value::create(fbb, &wb::U64ValueArgs { value }); + + add_value(fbb, column, wb::ColumnValue::U64Value, iv.as_union_value()) +} + fn add_bool_value<'a>( fbb: &mut FlatBufferBuilder<'a>, column: &str, diff --git a/internal_types/src/lib.rs b/internal_types/src/lib.rs new file mode 100644 index 0000000000..ecaf7f6243 --- /dev/null +++ b/internal_types/src/lib.rs @@ -0,0 +1,11 @@ +#![deny(rust_2018_idioms)] +#![warn( + missing_debug_implementations, + clippy::explicit_iter_loop, + clippy::use_self, + clippy::clone_on_ref_ptr +)] + +pub mod data; +pub mod schema; +pub mod selection; diff --git a/data_types/src/schema.rs b/internal_types/src/schema.rs similarity index 100% rename from data_types/src/schema.rs rename to internal_types/src/schema.rs diff --git a/data_types/src/schema/builder.rs b/internal_types/src/schema/builder.rs similarity index 99% rename from data_types/src/schema/builder.rs rename to internal_types/src/schema/builder.rs index 8a04f044c5..8746d3ec93 100644 --- a/data_types/src/schema/builder.rs +++ b/internal_types/src/schema/builder.rs @@ -140,7 +140,7 @@ impl SchemaBuilder { /// schema validation happens at this time. /// ``` - /// use data_types::schema::{builder::SchemaBuilder, InfluxColumnType, InfluxFieldType}; + /// use internal_types::schema::{builder::SchemaBuilder, InfluxColumnType, InfluxFieldType}; /// /// let schema = SchemaBuilder::new() /// .tag("region") diff --git a/data_types/src/selection.rs b/internal_types/src/selection.rs similarity index 100% rename from data_types/src/selection.rs rename to internal_types/src/selection.rs diff --git a/mem_qe/Cargo.toml b/mem_qe/Cargo.toml index dc36751890..23ff477433 100644 --- a/mem_qe/Cargo.toml +++ b/mem_qe/Cargo.toml @@ -8,8 +8,8 @@ edition = "2018" arrow_deps = { path = "../arrow_deps" } chrono = "0.4" croaring = "0.4.5" -crossbeam = "0.7.3" -env_logger = "0.7.1" +crossbeam = "0.8" +env_logger = "0.8.3" human_format = "1.0.3" packers = { path = "../packers" } snafu = "0.6.8" diff --git a/mutable_buffer/Cargo.toml b/mutable_buffer/Cargo.toml index 46bb17eeb3..57c3c0e81b 100644 --- a/mutable_buffer/Cargo.toml +++ b/mutable_buffer/Cargo.toml @@ -18,8 +18,11 @@ arrow_deps = { path = "../arrow_deps" } async-trait = "0.1" chrono = "0.4" data_types = { path = "../data_types" } -flatbuffers = "0.6.1" +# See docs/regenerating_flatbuffers.md about updating generated code when updating the +# version of the flatbuffers crate +flatbuffers = "0.8" generated_types = { path = "../generated_types" } +internal_types = { path = "../internal_types" } influxdb_line_protocol = { path = "../influxdb_line_protocol" } snafu = "0.6.2" string-interner = "0.12.2" diff --git a/mutable_buffer/src/chunk.rs b/mutable_buffer/src/chunk.rs index 20713c8c01..6f4ebc1a27 100644 --- a/mutable_buffer/src/chunk.rs +++ b/mutable_buffer/src/chunk.rs @@ -9,7 +9,8 @@ use chrono::{DateTime, Utc}; use generated_types::wal as wb; use std::collections::{BTreeSet, HashMap}; -use data_types::{partition_metadata::TableSummary, schema::Schema, selection::Selection}; +use data_types::partition_metadata::TableSummary; +use internal_types::{schema::Schema, selection::Selection}; use crate::{ column::Column, diff --git a/mutable_buffer/src/column.rs b/mutable_buffer/src/column.rs index ab6e2e9991..89a88e111a 100644 --- a/mutable_buffer/src/column.rs +++ b/mutable_buffer/src/column.rs @@ -2,9 +2,9 @@ use generated_types::wal as wb; use snafu::Snafu; use crate::dictionary::Dictionary; -use data_types::{data::type_description, partition_metadata::StatValues}; - use arrow_deps::arrow::datatypes::DataType as ArrowDataType; +use data_types::partition_metadata::StatValues; +use internal_types::data::type_description; use std::mem; @@ -34,6 +34,7 @@ pub type Result = std::result::Result; pub enum Column { F64(Vec>, StatValues), I64(Vec>, StatValues), + U64(Vec>, StatValues), String(Vec>, StatValues), Bool(Vec>, StatValues), Tag(Vec>, StatValues), @@ -45,10 +46,8 @@ impl Column { capacity: usize, value: wb::Value<'_>, ) -> Result { - use wb::ColumnValue::*; - Ok(match value.value_type() { - F64Value => { + wb::ColumnValue::F64Value => { let val = value .value_as_f64value() .expect("f64 value should be present") @@ -57,7 +56,7 @@ impl Column { vals.push(Some(val)); Self::F64(vals, StatValues::new(val)) } - I64Value => { + wb::ColumnValue::I64Value => { let val = value .value_as_i64value() .expect("i64 value should be present") @@ -66,7 +65,16 @@ impl Column { vals.push(Some(val)); Self::I64(vals, StatValues::new(val)) } - StringValue => { + wb::ColumnValue::U64Value => { + let val = value + .value_as_u64value() + .expect("u64 value should be present") + .value(); + let mut vals = vec![None; capacity]; + vals.push(Some(val)); + Self::U64(vals, StatValues::new(val)) + } + wb::ColumnValue::StringValue => { let val = value .value_as_string_value() .expect("string value should be present") @@ -76,7 +84,7 @@ impl Column { vals.push(Some(val.to_string())); Self::String(vals, StatValues::new(val.to_string())) } - BoolValue => { + wb::ColumnValue::BoolValue => { let val = value .value_as_bool_value() .expect("bool value should be present") @@ -85,7 +93,7 @@ impl Column { vals.push(Some(val)); Self::Bool(vals, StatValues::new(val)) } - TagValue => { + wb::ColumnValue::TagValue => { let val = value .value_as_tag_value() .expect("tag value should be present") @@ -109,6 +117,7 @@ impl Column { match self { Self::F64(v, _) => v.len(), Self::I64(v, _) => v.len(), + Self::U64(v, _) => v.len(), Self::String(v, _) => v.len(), Self::Bool(v, _) => v.len(), Self::Tag(v, _) => v.len(), @@ -123,6 +132,7 @@ impl Column { match self { Self::F64(_, _) => "f64", Self::I64(_, _) => "i64", + Self::U64(_, _) => "u64", Self::String(_, _) => "String", Self::Bool(_, _) => "bool", Self::Tag(_, _) => "tag", @@ -134,6 +144,7 @@ impl Column { match self { Self::F64(..) => ArrowDataType::Float64, Self::I64(..) => ArrowDataType::Int64, + Self::U64(..) => ArrowDataType::UInt64, Self::String(..) => ArrowDataType::Utf8, Self::Bool(..) => ArrowDataType::Boolean, Self::Tag(..) => ArrowDataType::Utf8, @@ -179,6 +190,15 @@ impl Column { } None => false, }, + Self::U64(vals, stats) => match value.value_as_u64value() { + Some(u64_val) => { + let u64_val = u64_val.value(); + vals.push(Some(u64_val)); + stats.update(u64_val); + true + } + None => false, + }, Self::F64(vals, stats) => match value.value_as_f64value() { Some(f64_val) => { let f64_val = f64_val.value(); @@ -216,6 +236,11 @@ impl Column { v.push(None); } } + Self::U64(v, _) => { + if v.len() == len { + v.push(None); + } + } Self::String(v, _) => { if v.len() == len { v.push(None); @@ -290,6 +315,9 @@ impl Column { Self::I64(v, stats) => { mem::size_of::>() * v.len() + mem::size_of_val(&stats) } + Self::U64(v, stats) => { + mem::size_of::>() * v.len() + mem::size_of_val(&stats) + } Self::Bool(v, stats) => { mem::size_of::>() * v.len() + mem::size_of_val(&stats) } diff --git a/mutable_buffer/src/database.rs b/mutable_buffer/src/database.rs index 11e0067ee0..ba45dcd57e 100644 --- a/mutable_buffer/src/database.rs +++ b/mutable_buffer/src/database.rs @@ -1,8 +1,6 @@ -use data_types::{ - data::ReplicatedWrite, - database_rules::{PartitionSort, PartitionSortRules}, -}; +use data_types::database_rules::{PartitionSort, PartitionSortRules}; use generated_types::wal; +use internal_types::data::ReplicatedWrite; use crate::{chunk::Chunk, partition::Partition}; @@ -79,6 +77,14 @@ impl MutableBufferDb { } } + /// returns the id of the current open chunk in the specified partition + pub fn open_chunk_id(&self, partition_key: &str) -> u32 { + let partition = self.get_partition(partition_key); + let partition = partition.read().expect("mutex poisoned"); + + partition.open_chunk_id() + } + /// Directs the writes from batch into the appropriate partitions fn write_entries_to_partitions(&self, batch: &wal::WriteBufferBatch<'_>) -> Result<()> { if let Some(entries) = batch.entries() { @@ -96,7 +102,8 @@ impl MutableBufferDb { Ok(()) } - /// Rolls over the active chunk in this partititon + /// Rolls over the active chunk in this partititon. Returns the + /// previously open (now closed) Chunk pub fn rollover_partition(&self, partition_key: &str) -> Result> { let partition = self.get_partition(partition_key); let mut partition = partition.write().expect("mutex poisoned"); @@ -181,7 +188,7 @@ impl MutableBufferDb { Some(b) => self.write_entries_to_partitions(&b)?, None => { return MissingPayload { - writer: write.to_fb().writer(), + writer: write.fb().writer(), } .fail() } @@ -240,12 +247,10 @@ impl MutableBufferDb { mod tests { use super::*; use chrono::{DateTime, Utc}; - use data_types::{ - data::lines_to_replicated_write, database_rules::Partitioner, selection::Selection, - }; + use data_types::database_rules::{Order, Partitioner}; + use internal_types::{data::lines_to_replicated_write, selection::Selection}; use arrow_deps::arrow::array::{Array, StringArray}; - use data_types::database_rules::Order; use influxdb_line_protocol::{parse_lines, ParsedLine}; type TestError = Box; diff --git a/mutable_buffer/src/partition.rs b/mutable_buffer/src/partition.rs index e64036056a..ce9f499c52 100644 --- a/mutable_buffer/src/partition.rs +++ b/mutable_buffer/src/partition.rs @@ -107,6 +107,11 @@ impl Partition { } } + /// returns the id of the current open chunk in this partition + pub(crate) fn open_chunk_id(&self) -> u32 { + self.open_chunk.id() + } + /// write data to the open chunk pub fn write_entry(&mut self, entry: &wb::WriteBufferEntry<'_>) -> Result<()> { assert_eq!( @@ -173,6 +178,8 @@ impl Partition { /// /// Queries will continue to see data in the specified chunk until /// it is dropped. + /// + /// Returns the previously open (now closed) Chunk pub fn rollover_chunk(&mut self) -> Arc { let chunk_id = self.id_generator; self.id_generator += 1; @@ -295,10 +302,8 @@ impl<'a> Iterator for ChunkIter<'a> { mod tests { use super::*; use chrono::Utc; - use data_types::{ - data::split_lines_into_write_entry_partitions, partition_metadata::PartitionSummary, - selection::Selection, - }; + use data_types::partition_metadata::PartitionSummary; + use internal_types::{data::split_lines_into_write_entry_partitions, selection::Selection}; use arrow_deps::{ arrow::record_batch::RecordBatch, assert_table_eq, test_util::sort_record_batch, @@ -924,7 +929,7 @@ mod tests { let lines: Vec<_> = parse_lines(&lp_string).map(|l| l.unwrap()).collect(); let data = split_lines_into_write_entry_partitions(|_| partition.key().into(), &lines); - let batch = flatbuffers::get_root::>(&data); + let batch = flatbuffers::root::>(&data).unwrap(); let entries = batch.entries().unwrap(); for entry in entries { diff --git a/mutable_buffer/src/pred.rs b/mutable_buffer/src/pred.rs index 869840e302..24a7db3074 100644 --- a/mutable_buffer/src/pred.rs +++ b/mutable_buffer/src/pred.rs @@ -10,7 +10,8 @@ use arrow_deps::{ }, util::{make_range_expr, AndExprBuilder}, }; -use data_types::{timestamp::TimestampRange, TIME_COLUMN_NAME}; +use data_types::timestamp::TimestampRange; +use internal_types::schema::TIME_COLUMN_NAME; //use snafu::{OptionExt, ResultExt, Snafu}; use snafu::{ensure, ResultExt, Snafu}; diff --git a/mutable_buffer/src/table.rs b/mutable_buffer/src/table.rs index 9174a1c99d..57d0178346 100644 --- a/mutable_buffer/src/table.rs +++ b/mutable_buffer/src/table.rs @@ -12,18 +12,20 @@ use crate::{ dictionary::{Dictionary, Error as DictionaryError}, pred::{ChunkIdSet, ChunkPredicate}, }; -use data_types::{ - partition_metadata::{ColumnSummary, Statistics}, - schema::{builder::SchemaBuilder, Schema}, +use data_types::partition_metadata::{ColumnSummary, Statistics}; +use internal_types::{ + schema::{builder::SchemaBuilder, Schema, TIME_COLUMN_NAME}, selection::Selection, - TIME_COLUMN_NAME, }; + use snafu::{OptionExt, ResultExt, Snafu}; use arrow_deps::{ arrow, arrow::{ - array::{ArrayRef, BooleanBuilder, Float64Builder, Int64Builder, StringBuilder}, + array::{ + ArrayRef, BooleanBuilder, Float64Builder, Int64Builder, StringBuilder, UInt64Builder, + }, datatypes::DataType as ArrowDataType, record_batch::RecordBatch, }, @@ -82,7 +84,7 @@ pub enum Error { #[snafu(display("Internal error converting schema: {}", source))] InternalSchema { - source: data_types::schema::builder::Error, + source: internal_types::schema::builder::Error, }, #[snafu(display( @@ -326,6 +328,7 @@ impl Table { schema_builder.field(column_name, ArrowDataType::Int64) } } + Column::U64(_, _) => schema_builder.field(column_name, ArrowDataType::UInt64), Column::Bool(_, _) => schema_builder.field(column_name, ArrowDataType::Boolean), }; } @@ -399,6 +402,15 @@ impl Table { Arc::new(builder.finish()) as ArrayRef } + Column::U64(vals, _) => { + let mut builder = UInt64Builder::new(vals.len()); + + for v in vals { + builder.append_option(*v).context(ArrowError {})?; + } + + Arc::new(builder.finish()) as ArrayRef + } Column::Bool(vals, _) => { let mut builder = BooleanBuilder::new(vals.len()); @@ -504,6 +516,7 @@ impl Table { match column { Column::F64(v, _) => self.column_value_matches_predicate(v, chunk_predicate), Column::I64(v, _) => self.column_value_matches_predicate(v, chunk_predicate), + Column::U64(v, _) => self.column_value_matches_predicate(v, chunk_predicate), Column::String(v, _) => self.column_value_matches_predicate(v, chunk_predicate), Column::Bool(v, _) => self.column_value_matches_predicate(v, chunk_predicate), Column::Tag(v, _) => self.column_value_matches_predicate(v, chunk_predicate), @@ -545,6 +558,7 @@ impl Table { let stats = match c { Column::F64(_, stats) => Statistics::F64(stats.clone()), Column::I64(_, stats) => Statistics::I64(stats.clone()), + Column::U64(_, stats) => Statistics::U64(stats.clone()), Column::Bool(_, stats) => Statistics::Bool(stats.clone()), Column::String(_, stats) | Column::Tag(_, stats) => { Statistics::String(stats.clone()) @@ -583,8 +597,8 @@ impl<'a> TableColSelection<'a> { #[cfg(test)] mod tests { - use data_types::data::split_lines_into_write_entry_partitions; use influxdb_line_protocol::{parse_lines, ParsedLine}; + use internal_types::data::split_lines_into_write_entry_partitions; use super::*; @@ -736,7 +750,7 @@ mod tests { let mut table = Table::new(dictionary.lookup_value_or_insert("table_name")); let lp_lines = vec![ - "h2o,state=MA,city=Boston float_field=70.4,int_field=8i,bool_field=t,string_field=\"foo\" 100", + "h2o,state=MA,city=Boston float_field=70.4,int_field=8i,uint_field=42u,bool_field=t,string_field=\"foo\" 100", ]; write_lines_to_table(&mut table, dictionary, lp_lines); @@ -751,6 +765,7 @@ mod tests { .tag("state") .field("string_field", ArrowDataType::Utf8) .timestamp() + .field("uint_field", ArrowDataType::UInt64) .build() .unwrap(); @@ -793,7 +808,7 @@ mod tests { let data = split_lines_into_write_entry_partitions(chunk_key_func, &lines); - let batch = flatbuffers::get_root::>(&data); + let batch = flatbuffers::root::>(&data).unwrap(); let entries = batch.entries().expect("at least one entry"); for entry in entries { diff --git a/object_store/src/aws.rs b/object_store/src/aws.rs index 040acdf49d..16a6cb47af 100644 --- a/object_store/src/aws.rs +++ b/object_store/src/aws.rs @@ -13,7 +13,7 @@ use futures::{ Stream, StreamExt, TryStreamExt, }; use rusoto_core::ByteStream; -use rusoto_credential::StaticProvider; +use rusoto_credential::{InstanceMetadataProvider, StaticProvider}; use rusoto_s3::S3; use snafu::{futures::TryStreamExt as _, OptionExt, ResultExt, Snafu}; use std::convert::TryFrom; @@ -108,6 +108,12 @@ pub enum Error { region: String, source: rusoto_core::region::ParseRegionError, }, + + #[snafu(display("Missing aws-access-key"))] + MissingAccessKey, + + #[snafu(display("Missing aws-secret-access-key"))] + MissingSecretAccessKey, } /// Configuration for connecting to [Amazon S3](https://aws.amazon.com/s3/). @@ -285,8 +291,8 @@ impl AmazonS3 { /// Configure a connection to Amazon S3 using the specified credentials in /// the specified Amazon region and bucket pub fn new( - access_key_id: impl Into, - secret_access_key: impl Into, + access_key_id: Option>, + secret_access_key: Option>, region: impl Into, bucket_name: impl Into, ) -> Result { @@ -296,11 +302,22 @@ impl AmazonS3 { let http_client = rusoto_core::request::HttpClient::new() .expect("Current implementation of rusoto_core has no way for this to fail"); - let credentials_provider = - StaticProvider::new_minimal(access_key_id.into(), secret_access_key.into()); + let client = match (access_key_id, secret_access_key) { + (Some(access_key_id), Some(secret_access_key)) => { + let credentials_provider = + StaticProvider::new_minimal(access_key_id.into(), secret_access_key.into()); + rusoto_s3::S3Client::new_with(http_client, credentials_provider, region) + } + (None, Some(_)) => return Err(Error::MissingAccessKey), + (Some(_), None) => return Err(Error::MissingSecretAccessKey), + _ => { + let credentials_provider = InstanceMetadataProvider::new(); + rusoto_s3::S3Client::new_with(http_client, credentials_provider, region) + } + }; Ok(Self { - client: rusoto_s3::S3Client::new_with(http_client, credentials_provider, region), + client, bucket_name: bucket_name.into(), }) } @@ -502,8 +519,8 @@ mod tests { let config = maybe_skip_integration!(); let integration = ObjectStore::new_amazon_s3( AmazonS3::new( - config.access_key_id, - config.secret_access_key, + Some(config.access_key_id), + Some(config.secret_access_key), config.region, config.bucket, ) @@ -524,8 +541,8 @@ mod tests { let integration = ObjectStore::new_amazon_s3( AmazonS3::new( - config.access_key_id, - config.secret_access_key, + Some(config.access_key_id), + Some(config.secret_access_key), config.region, &config.bucket, ) @@ -556,8 +573,8 @@ mod tests { let config = maybe_skip_integration!(); let integration = ObjectStore::new_amazon_s3( AmazonS3::new( - config.access_key_id, - config.secret_access_key, + Some(config.access_key_id), + Some(config.secret_access_key), config.region, &config.bucket, ) @@ -599,8 +616,8 @@ mod tests { let integration = ObjectStore::new_amazon_s3( AmazonS3::new( - config.access_key_id, - config.secret_access_key, + Some(config.access_key_id), + Some(config.secret_access_key), config.region, &config.bucket, ) @@ -637,8 +654,8 @@ mod tests { let integration = ObjectStore::new_amazon_s3( AmazonS3::new( - config.access_key_id, - config.secret_access_key, + Some(config.access_key_id), + Some(config.secret_access_key), config.region, &config.bucket, ) @@ -685,8 +702,8 @@ mod tests { let integration = ObjectStore::new_amazon_s3( AmazonS3::new( - config.access_key_id, - config.secret_access_key, + Some(config.access_key_id), + Some(config.secret_access_key), config.region, &config.bucket, ) @@ -731,8 +748,8 @@ mod tests { let config = maybe_skip_integration!(); let integration = ObjectStore::new_amazon_s3( AmazonS3::new( - config.access_key_id, - config.secret_access_key, + Some(config.access_key_id), + Some(config.secret_access_key), config.region, config.bucket, ) @@ -757,8 +774,8 @@ mod tests { let integration = ObjectStore::new_amazon_s3( AmazonS3::new( - config.access_key_id, - config.secret_access_key, + Some(config.access_key_id), + Some(config.secret_access_key), config.region, &config.bucket, ) @@ -795,8 +812,8 @@ mod tests { let integration = ObjectStore::new_amazon_s3( AmazonS3::new( - config.access_key_id, - config.secret_access_key, + Some(config.access_key_id), + Some(config.secret_access_key), config.region, &config.bucket, ) diff --git a/packers/Cargo.toml b/packers/Cargo.toml index 94a737d040..627ac781b6 100644 --- a/packers/Cargo.toml +++ b/packers/Cargo.toml @@ -6,12 +6,12 @@ edition = "2018" [dependencies] # In alphabetical order arrow_deps = { path = "../arrow_deps" } -data_types = { path = "../data_types" } human_format = "1.0.3" influxdb_tsm = { path = "../influxdb_tsm" } +internal_types = { path = "../internal_types" } snafu = "0.6.2" tracing = "0.1" [dev-dependencies] # In alphabetical order -rand = "0.7.3" +rand = "0.8.3" test_helpers = { path = "../test_helpers" } diff --git a/packers/src/lib.rs b/packers/src/lib.rs index e3197acf80..5a82ec9bdb 100644 --- a/packers/src/lib.rs +++ b/packers/src/lib.rs @@ -15,7 +15,7 @@ use snafu::Snafu; pub use crate::packers::{Packer, Packers}; pub use arrow_deps::parquet::data_type::ByteArray; -use data_types::schema::Schema; +use internal_types::schema::Schema; use std::borrow::Cow; diff --git a/packers/src/packers.rs b/packers/src/packers.rs index dbbf86a0f2..64d9767943 100644 --- a/packers/src/packers.rs +++ b/packers/src/packers.rs @@ -10,7 +10,7 @@ use std::iter; use std::slice::Chunks; use arrow_deps::parquet::data_type::ByteArray; -use data_types::schema::{InfluxColumnType, InfluxFieldType}; +use internal_types::schema::{InfluxColumnType, InfluxFieldType}; use std::default::Default; // NOTE: See https://blog.twitter.com/engineering/en_us/a/2013/dremel-made-simple-with-parquet.html @@ -20,6 +20,7 @@ use std::default::Default; pub enum Packers { Float(Packer), Integer(Packer), + UInteger(Packer), Bytes(Packer), String(Packer), Boolean(Packer), @@ -52,6 +53,7 @@ impl<'a> Packers { match self { Self::Float(p) => PackerChunker::Float(p.values.chunks(chunk_size)), Self::Integer(p) => PackerChunker::Integer(p.values.chunks(chunk_size)), + Self::UInteger(p) => PackerChunker::UInteger(p.values.chunks(chunk_size)), Self::Bytes(p) => PackerChunker::Bytes(p.values.chunks(chunk_size)), Self::String(p) => PackerChunker::String(p.values.chunks(chunk_size)), Self::Boolean(p) => PackerChunker::Boolean(p.values.chunks(chunk_size)), @@ -69,6 +71,7 @@ impl<'a> Packers { match self { Self::Float(p) => p.reserve_exact(additional), Self::Integer(p) => p.reserve_exact(additional), + Self::UInteger(p) => p.reserve_exact(additional), Self::Bytes(p) => p.reserve_exact(additional), Self::String(p) => p.reserve_exact(additional), Self::Boolean(p) => p.reserve_exact(additional), @@ -79,6 +82,7 @@ impl<'a> Packers { match self { Self::Float(p) => p.push_option(None), Self::Integer(p) => p.push_option(None), + Self::UInteger(p) => p.push_option(None), Self::Bytes(p) => p.push_option(None), Self::String(p) => p.push_option(None), Self::Boolean(p) => p.push_option(None), @@ -90,6 +94,7 @@ impl<'a> Packers { match self { Self::Float(p) => p.swap(a, b), Self::Integer(p) => p.swap(a, b), + Self::UInteger(p) => p.swap(a, b), Self::Bytes(p) => p.swap(a, b), Self::String(p) => p.swap(a, b), Self::Boolean(p) => p.swap(a, b), @@ -101,6 +106,7 @@ impl<'a> Packers { match self { Self::Float(p) => p.num_rows(), Self::Integer(p) => p.num_rows(), + Self::UInteger(p) => p.num_rows(), Self::Bytes(p) => p.num_rows(), Self::String(p) => p.num_rows(), Self::Boolean(p) => p.num_rows(), @@ -114,6 +120,7 @@ impl<'a> Packers { match self { Self::Float(p) => p.is_null(row), Self::Integer(p) => p.is_null(row), + Self::UInteger(p) => p.is_null(row), Self::Bytes(p) => p.is_null(row), Self::String(p) => p.is_null(row), Self::Boolean(p) => p.is_null(row), @@ -124,6 +131,7 @@ impl<'a> Packers { typed_packer_accessors! { (f64_packer, f64_packer_mut, f64, Float), (i64_packer, i64_packer_mut, i64, Integer), + (u64_packer, u64_packer_mut, u64, UInteger), (bytes_packer, bytes_packer_mut, ByteArray, Bytes), (str_packer, str_packer_mut, String, String), (bool_packer, bool_packer_mut, bool, Boolean), @@ -245,6 +253,7 @@ impl std::convert::From>>> for Packers { pub enum PackerChunker<'a> { Float(Chunks<'a, Option>), Integer(Chunks<'a, Option>), + UInteger(Chunks<'a, Option>), Bytes(Chunks<'a, Option>), String(Chunks<'a, Option>), Boolean(Chunks<'a, Option>), @@ -523,6 +532,7 @@ mod test { let mut packers: Vec = Vec::new(); packers.push(Packers::Float(Packer::new())); packers.push(Packers::Integer(Packer::new())); + packers.push(Packers::UInteger(Packer::new())); packers.push(Packers::Boolean(Packer::new())); packers.get_mut(0).unwrap().f64_packer_mut().push(22.033); diff --git a/packers/src/sorter.rs b/packers/src/sorter.rs index 6b6bf49f7c..016a72f67f 100644 --- a/packers/src/sorter.rs +++ b/packers/src/sorter.rs @@ -387,7 +387,7 @@ mod test { for _ in 0..250 { let packer: Packer = Packer::from( (0..1000) - .map(|_| rng.gen_range(0, 20)) + .map(|_| rng.gen_range(0..20)) .collect::>(), ); let mut packers = vec![Packers::Integer(packer)]; @@ -410,7 +410,7 @@ mod test { for _ in 0..250 { let packer: Packer = Packer::from( (0..1000) - .map(|_| format!("{:?}", rng.gen_range(0, 20))) + .map(|_| format!("{:?}", rng.gen_range(0..20))) .collect::>(), ); let mut packers = vec![Packers::String(packer)]; diff --git a/query/Cargo.toml b/query/Cargo.toml index ce25b22fb2..2276bf995d 100644 --- a/query/Cargo.toml +++ b/query/Cargo.toml @@ -21,9 +21,10 @@ croaring = "0.4.5" data_types = { path = "../data_types" } futures = "0.3.7" influxdb_line_protocol = { path = "../influxdb_line_protocol" } +internal_types = { path = "../internal_types" } parking_lot = "0.11.1" snafu = "0.6.2" -sqlparser = "0.6.1" +sqlparser = "0.8.0" tokio = { version = "1.0", features = ["macros"] } tokio-stream = "0.1.2" tracing = "0.1" diff --git a/query/src/exec/field.rs b/query/src/exec/field.rs index a9ef1154eb..b21f42d757 100644 --- a/query/src/exec/field.rs +++ b/query/src/exec/field.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use arrow_deps::arrow::{self, datatypes::SchemaRef}; -use data_types::TIME_COLUMN_NAME; +use internal_types::schema::TIME_COLUMN_NAME; use snafu::{ResultExt, Snafu}; #[derive(Debug, Snafu)] diff --git a/query/src/exec/fieldlist.rs b/query/src/exec/fieldlist.rs index e0b846aa0c..4c30928e77 100644 --- a/query/src/exec/fieldlist.rs +++ b/query/src/exec/fieldlist.rs @@ -9,7 +9,7 @@ use arrow_deps::arrow::{ datatypes::{DataType, SchemaRef}, record_batch::RecordBatch, }; -use data_types::TIME_COLUMN_NAME; +use internal_types::schema::TIME_COLUMN_NAME; use snafu::{ensure, ResultExt, Snafu}; diff --git a/query/src/frontend/influxrpc.rs b/query/src/frontend/influxrpc.rs index 86853b4a94..590233ae8b 100644 --- a/query/src/frontend/influxrpc.rs +++ b/query/src/frontend/influxrpc.rs @@ -14,10 +14,9 @@ use arrow_deps::{ }, util::IntoExpr, }; -use data_types::{ - schema::{InfluxColumnType, Schema}, +use internal_types::{ + schema::{InfluxColumnType, Schema, TIME_COLUMN_NAME}, selection::Selection, - TIME_COLUMN_NAME, }; use snafu::{ensure, OptionExt, ResultExt, Snafu}; use tracing::debug; diff --git a/query/src/frontend/sql.rs b/query/src/frontend/sql.rs index ce305c21a0..f30788236b 100644 --- a/query/src/frontend/sql.rs +++ b/query/src/frontend/sql.rs @@ -4,7 +4,7 @@ use snafu::{ResultExt, Snafu}; use crate::{exec::Executor, provider::ProviderBuilder, Database, PartitionChunk}; use arrow_deps::datafusion::{error::DataFusionError, physical_plan::ExecutionPlan}; -use data_types::selection::Selection; +use internal_types::selection::Selection; #[derive(Debug, Snafu)] pub enum Error { diff --git a/query/src/lib.rs b/query/src/lib.rs index a10eaef99f..446f2297f8 100644 --- a/query/src/lib.rs +++ b/query/src/lib.rs @@ -8,10 +8,9 @@ use arrow_deps::datafusion::physical_plan::SendableRecordBatchStream; use async_trait::async_trait; -use data_types::{ - data::ReplicatedWrite, partition_metadata::TableSummary, schema::Schema, selection::Selection, -}; +use data_types::{chunk::ChunkSummary, partition_metadata::TableSummary}; use exec::{stringset::StringSet, Executor}; +use internal_types::{data::ReplicatedWrite, schema::Schema, selection::Selection}; use std::{fmt::Debug, sync::Arc}; @@ -49,6 +48,9 @@ pub trait Database: Debug + Send + Sync { /// covering set means that together the chunks make up a single /// complete copy of the data being queried. fn chunks(&self, partition_key: &str) -> Vec>; + + /// Return a summary of all chunks in this database, in all partitions + fn chunk_summaries(&self) -> Result, Self::Error>; } /// Collection of data that shares the same partition key @@ -60,7 +62,7 @@ pub trait PartitionChunk: Debug + Send + Sync { /// particular partition. fn id(&self) -> u32; - /// returns the partition metadata stats for every table in the partition + /// returns the partition metadata stats for every table in the chunk fn table_stats(&self) -> Result, Self::Error>; /// Returns true if this chunk *might* have data that passes the @@ -155,11 +157,11 @@ pub trait DatabaseStore: Debug + Send + Sync { type Error: std::error::Error + Send + Sync + 'static; /// List the database names. - async fn db_names_sorted(&self) -> Vec; + fn db_names_sorted(&self) -> Vec; /// Retrieve the database specified by `name` returning None if no /// such database exists - async fn db(&self, name: &str) -> Option>; + fn db(&self, name: &str) -> Option>; /// Retrieve the database specified by `name`, creating it if it /// doesn't exist. diff --git a/query/src/plan/stringset.rs b/query/src/plan/stringset.rs index e2b9a60962..73cc850854 100644 --- a/query/src/plan/stringset.rs +++ b/query/src/plan/stringset.rs @@ -1,7 +1,10 @@ use std::sync::Arc; use arrow_deps::{datafusion::logical_plan::LogicalPlan, util::str_iter_to_batch}; -use data_types::TABLE_NAMES_COLUMN_NAME; + +/// The name of the column containing table names returned by a call to +/// `table_names`. +const TABLE_NAMES_COLUMN_NAME: &str = "table"; use crate::{ exec::stringset::{StringSet, StringSetRef}, diff --git a/query/src/predicate.rs b/query/src/predicate.rs index e2b6759ab5..d02b8dec2a 100644 --- a/query/src/predicate.rs +++ b/query/src/predicate.rs @@ -9,7 +9,8 @@ use arrow_deps::{ datafusion::logical_plan::Expr, util::{make_range_expr, AndExprBuilder}, }; -use data_types::{timestamp::TimestampRange, TIME_COLUMN_NAME}; +use data_types::timestamp::TimestampRange; +use internal_types::schema::TIME_COLUMN_NAME; /// This `Predicate` represents the empty predicate (aka that /// evaluates to true for all rows). diff --git a/query/src/provider.rs b/query/src/provider.rs index 620555536a..5a6e8065e6 100644 --- a/query/src/provider.rs +++ b/query/src/provider.rs @@ -14,7 +14,7 @@ use arrow_deps::{ physical_plan::ExecutionPlan, }, }; -use data_types::schema::{builder::SchemaMerger, Schema}; +use internal_types::schema::{builder::SchemaMerger, Schema}; use crate::{predicate::Predicate, util::project_schema, PartitionChunk}; @@ -29,7 +29,7 @@ pub enum Error { #[snafu(display("Chunk schema not compatible for table '{}': {}", table_name, source))] ChunkSchemaNotCompatible { table_name: String, - source: data_types::schema::builder::Error, + source: internal_types::schema::builder::Error, }, #[snafu(display( @@ -39,7 +39,7 @@ pub enum Error { ))] InternalNoChunks { table_name: String, - source: data_types::schema::builder::Error, + source: internal_types::schema::builder::Error, }, #[snafu(display("Internal error: No rows found in table '{}'", table_name))] @@ -193,6 +193,7 @@ impl TableProvider for ChunkTableProvider { projection: &Option>, _batch_size: usize, _filters: &[Expr], + _limit: Option, ) -> std::result::Result, DataFusionError> { // TODO Here is where predicate pushdown will happen. To make // predicate push down happen, the provider need need to diff --git a/query/src/provider/physical.rs b/query/src/provider/physical.rs index c17e961450..3342711d7a 100644 --- a/query/src/provider/physical.rs +++ b/query/src/provider/physical.rs @@ -9,7 +9,7 @@ use arrow_deps::{ physical_plan::{ExecutionPlan, Partitioning, SendableRecordBatchStream}, }, }; -use data_types::{schema::Schema, selection::Selection}; +use internal_types::{schema::Schema, selection::Selection}; use crate::{predicate::Predicate, PartitionChunk}; diff --git a/query/src/test.rs b/query/src/test.rs index 7fd877f24e..f46740ac18 100644 --- a/query/src/test.rs +++ b/query/src/test.rs @@ -18,16 +18,16 @@ use crate::{ Database, DatabaseStore, PartitionChunk, Predicate, }; -use data_types::{ +use data_types::database_rules::{DatabaseRules, PartitionTemplate, TemplatePart}; +use influxdb_line_protocol::{parse_lines, ParsedLine}; +use internal_types::{ data::{lines_to_replicated_write, ReplicatedWrite}, - database_rules::{DatabaseRules, PartitionTemplate, TemplatePart}, schema::{ builder::{SchemaBuilder, SchemaMerger}, Schema, }, selection::Selection, }; -use influxdb_line_protocol::{parse_lines, ParsedLine}; use async_trait::async_trait; use chrono::{DateTime, Utc}; @@ -154,6 +154,10 @@ impl Database for TestDatabase { vec![] } } + + fn chunk_summaries(&self) -> Result, Self::Error> { + unimplemented!("summaries not implemented TestDatabase") + } } #[derive(Debug, Default)] @@ -474,14 +478,14 @@ impl DatabaseStore for TestDatabaseStore { type Error = TestError; /// List the database names. - async fn db_names_sorted(&self) -> Vec { + fn db_names_sorted(&self) -> Vec { let databases = self.databases.lock(); databases.keys().cloned().collect() } /// Retrieve the database specified name - async fn db(&self, name: &str) -> Option> { + fn db(&self, name: &str) -> Option> { let databases = self.databases.lock(); databases.get(name).cloned() diff --git a/query/src/util.rs b/query/src/util.rs index 19244482d0..53bd71cc78 100644 --- a/query/src/util.rs +++ b/query/src/util.rs @@ -11,7 +11,7 @@ use arrow_deps::{ optimizer::utils::expr_to_column_names, }, }; -use data_types::schema::Schema; +use internal_types::schema::Schema; /// Create a logical plan that produces the record batch pub fn make_scan_plan(batch: RecordBatch) -> std::result::Result { @@ -57,7 +57,7 @@ pub fn schema_has_all_expr_columns(schema: &Schema, expr: &Expr) -> bool { #[cfg(test)] mod tests { use arrow_deps::datafusion::prelude::*; - use data_types::schema::builder::SchemaBuilder; + use internal_types::schema::builder::SchemaBuilder; use super::*; diff --git a/read_buffer/Cargo.toml b/read_buffer/Cargo.toml index 7e5a52332d..61d37b35b0 100644 --- a/read_buffer/Cargo.toml +++ b/read_buffer/Cargo.toml @@ -13,9 +13,9 @@ edition = "2018" [dependencies] # In alphabetical order arrow_deps = { path = "../arrow_deps" } croaring = "0.4.5" -data_types = { path = "../data_types" } either = "1.6.1" hashbrown = "0.9.1" +internal_types = { path = "../internal_types" } itertools = "0.9.0" packers = { path = "../packers" } permutation = "0.2.5" @@ -23,8 +23,8 @@ snafu = "0.6" [dev-dependencies] # In alphabetical order criterion = "0.3.3" -rand = "0.7.3" -rand_distr = "0.3.0" +rand = "0.8.3" +rand_distr = "0.4.0" [[bench]] name = "database" diff --git a/read_buffer/benches/database.rs b/read_buffer/benches/database.rs index 1bef6a222c..975e550b1f 100644 --- a/read_buffer/benches/database.rs +++ b/read_buffer/benches/database.rs @@ -6,7 +6,7 @@ use arrow_deps::arrow::{ array::{ArrayRef, Int64Array, StringArray}, record_batch::RecordBatch, }; -use data_types::schema::builder::SchemaBuilder; +use internal_types::schema::builder::SchemaBuilder; use read_buffer::{BinaryExpr, Database, Predicate}; const BASE_TIME: i64 = 1351700038292387000_i64; diff --git a/read_buffer/benches/dictionary.rs b/read_buffer/benches/dictionary.rs index bb3aff61bc..6558bd0a2b 100644 --- a/read_buffer/benches/dictionary.rs +++ b/read_buffer/benches/dictionary.rs @@ -68,16 +68,16 @@ fn benchmark_select( let value = match location { Location::Start => { // find a value in the column close to the beginning. - &col_data[rng.gen_range(0, col_data.len() / 20)] // something in first 5% + &col_data[rng.gen_range(0..col_data.len() / 20)] // something in first 5% } Location::Middle => { // find a value in the column somewhere in the middle let fifth = col_data.len() / 5; - &col_data[rng.gen_range(2 * fifth, 3 * fifth)] // something in middle fifth + &col_data[rng.gen_range(2 * fifth..3 * fifth)] // something in middle fifth } Location::End => { &col_data - [rng.gen_range(col_data.len() - (col_data.len() / 9), col_data.len())] + [rng.gen_range(col_data.len() - (col_data.len() / 9)..col_data.len())] } // something in the last ~10% }; @@ -139,7 +139,10 @@ fn generate_column(rows: usize, rows_per_value: usize, rng: &mut ThreadRng) -> V for _ in 0..distinct_values { let value = format!( "value-{}", - rng.sample_iter(&Alphanumeric).take(8).collect::() + rng.sample_iter(&Alphanumeric) + .map(char::from) + .take(8) + .collect::() ); col.extend(std::iter::repeat(value).take(rows_per_value)); } diff --git a/read_buffer/benches/row_group.rs b/read_buffer/benches/row_group.rs index c8c141aa25..a19d4915a9 100644 --- a/read_buffer/benches/row_group.rs +++ b/read_buffer/benches/row_group.rs @@ -35,7 +35,7 @@ fn read_group_predicate_all_time(c: &mut Criterion, row_group: &RowGroup, rng: & &time_pred, // grouping columns and expected cardinality vec![ - (vec!["env"], 20), + (vec!["env"], 2), (vec!["env", "data_centre"], 20), (vec!["data_centre", "cluster"], 200), (vec!["cluster", "node_id"], 2000), @@ -358,13 +358,13 @@ fn generate_trace_for_row_group( let duration_idx = 9; let time_idx = 10; - let env_value = rng.gen_range(0_u8, 2); + let env_value = rng.gen_range(0_u8..2); let env = format!("env-{:?}", env_value); // cardinality of 2. - let data_centre_value = rng.gen_range(0_u8, 10); + let data_centre_value = rng.gen_range(0_u8..10); let data_centre = format!("data_centre-{:?}-{:?}", env_value, data_centre_value); // cardinality of 2 * 10 = 20 - let cluster_value = rng.gen_range(0_u8, 10); + let cluster_value = rng.gen_range(0_u8..10); let cluster = format!( "cluster-{:?}-{:?}-{:?}", env_value, @@ -373,7 +373,7 @@ fn generate_trace_for_row_group( ); // user id is dependent on the cluster - let user_id_value = rng.gen_range(0_u32, 1000); + let user_id_value = rng.gen_range(0_u32..1000); let user_id = format!( "uid-{:?}-{:?}-{:?}-{:?}", env_value, @@ -382,7 +382,7 @@ fn generate_trace_for_row_group( user_id_value // cardinality of 2 * 10 * 10 * 1000 = 200,000 ); - let request_id_value = rng.gen_range(0_u32, 10); + let request_id_value = rng.gen_range(0_u32..10); let request_id = format!( "rid-{:?}-{:?}-{:?}-{:?}-{:?}", env_value, @@ -392,7 +392,11 @@ fn generate_trace_for_row_group( request_id_value // cardinality of 2 * 10 * 10 * 1000 * 10 = 2,000,000 ); - let trace_id = rng.sample_iter(&Alphanumeric).take(8).collect::(); + let trace_id = rng + .sample_iter(&Alphanumeric) + .map(char::from) + .take(8) + .collect::(); // the trace should move across hosts, which in this setup would be nodes // and pods. @@ -401,13 +405,13 @@ fn generate_trace_for_row_group( for _ in 0..spans_per_trace { // these values are not the same for each span so need to be generated // separately. - let node_id = rng.gen_range(0, 10); // cardinality is 2 * 10 * 10 * 10 = 2,000 + let node_id = rng.gen_range(0..10); // cardinality is 2 * 10 * 10 * 10 = 2,000 column_packers[pod_id_idx].str_packer_mut().push(format!( "pod_id-{}-{}-{}", node_id_prefix, node_id, - rng.gen_range(0, 10) // cardinality is 2 * 10 * 10 * 10 * 10 = 20,000 + rng.gen_range(0..10) // cardinality is 2 * 10 * 10 * 10 * 10 = 20,000 )); column_packers[node_id_idx] @@ -415,9 +419,12 @@ fn generate_trace_for_row_group( .push(format!("node_id-{}-{}", node_id_prefix, node_id)); // randomly generate a span_id - column_packers[span_id_idx] - .str_packer_mut() - .push(rng.sample_iter(&Alphanumeric).take(8).collect::()); + column_packers[span_id_idx].str_packer_mut().push( + rng.sample_iter(&Alphanumeric) + .map(char::from) + .take(8) + .collect::(), + ); // randomly generate some duration times in milliseconds. column_packers[duration_idx].i64_packer_mut().push( diff --git a/read_buffer/src/chunk.rs b/read_buffer/src/chunk.rs index 30bccdb8eb..e02d999e42 100644 --- a/read_buffer/src/chunk.rs +++ b/read_buffer/src/chunk.rs @@ -3,8 +3,8 @@ use std::{ sync::RwLock, }; -use data_types::selection::Selection; -use snafu::{ResultExt, Snafu}; +use internal_types::selection::Selection; +use snafu::{OptionExt, ResultExt, Snafu}; use crate::row_group::RowGroup; use crate::row_group::{ColumnName, Predicate}; @@ -184,9 +184,7 @@ impl Chunk { let table = chunk_data .data .get(table_name) - .ok_or(Error::TableNotFound { - table_name: table_name.to_owned(), - })?; + .context(TableNotFound { table_name })?; Ok(table.read_filter(select_columns, predicate)) } @@ -195,7 +193,7 @@ impl Chunk { /// columns, optionally filtered by the provided predicate. Results are /// merged across all row groups within the returned table. /// - /// Returns `None` if the table no longer exists within the chunk. + /// Returns an error if the specified table does not exist. /// /// Note: `read_aggregate` currently only supports grouping on "tag" /// columns. @@ -205,17 +203,18 @@ impl Chunk { predicate: Predicate, group_columns: &Selection<'_>, aggregates: &[(ColumnName<'_>, AggregateType)], - ) -> Option { + ) -> Result { // read lock on chunk. let chunk_data = self.chunk_data.read().unwrap(); - // Lookup table by name and dispatch execution. - // - // TODO(edd): this should return an error - chunk_data + let table = chunk_data .data .get(table_name) - .map(|table| table.read_aggregate(predicate, group_columns, aggregates)) + .context(TableNotFound { table_name })?; + + table + .read_aggregate(predicate, group_columns, aggregates) + .context(TableError) } // diff --git a/read_buffer/src/column.rs b/read_buffer/src/column.rs index 8bf66bddc1..e2fddb5df0 100644 --- a/read_buffer/src/column.rs +++ b/read_buffer/src/column.rs @@ -330,7 +330,7 @@ impl Column { // Check the column for all rows that satisfy the predicate. let row_ids = match &self { - Self::String(_, data) => data.row_ids_filter(op, value.string(), dst), + Self::String(_, data) => data.row_ids_filter(op, value.str(), dst), Self::Float(_, data) => data.row_ids_filter(op, value.scalar(), dst), Self::Integer(_, data) => data.row_ids_filter(op, value.scalar(), dst), Self::Unsigned(_, data) => data.row_ids_filter(op, value.scalar(), dst), @@ -1291,8 +1291,6 @@ mod test { use super::*; use arrow_deps::arrow::array::{Int64Array, StringArray}; - use crate::value::AggregateResult; - #[test] fn row_ids_intersect() { let mut row_ids = RowIDs::new_bitmap(); @@ -2190,74 +2188,6 @@ mod test { assert_eq!(col.count(&[0, 2][..]), 0); } - #[test] - fn aggregate_result() { - let mut res = AggregateResult::Count(0); - res.update(Value::Null); - assert!(matches!(res, AggregateResult::Count(0))); - res.update(Value::String("hello")); - assert!(matches!(res, AggregateResult::Count(1))); - - let mut res = AggregateResult::Min(Value::Null); - res.update(Value::String("Dance Yrself Clean")); - assert!(matches!( - res, - AggregateResult::Min(Value::String("Dance Yrself Clean")) - )); - res.update(Value::String("All My Friends")); - assert!(matches!( - res, - AggregateResult::Min(Value::String("All My Friends")) - )); - res.update(Value::String("Dance Yrself Clean")); - assert!(matches!( - res, - AggregateResult::Min(Value::String("All My Friends")) - )); - res.update(Value::Null); - assert!(matches!( - res, - AggregateResult::Min(Value::String("All My Friends")) - )); - - let mut res = AggregateResult::Max(Value::Null); - res.update(Value::Scalar(Scalar::I64(20))); - assert!(matches!( - res, - AggregateResult::Max(Value::Scalar(Scalar::I64(20))) - )); - res.update(Value::Scalar(Scalar::I64(39))); - assert!(matches!( - res, - AggregateResult::Max(Value::Scalar(Scalar::I64(39))) - )); - res.update(Value::Scalar(Scalar::I64(20))); - assert!(matches!( - res, - AggregateResult::Max(Value::Scalar(Scalar::I64(39))) - )); - res.update(Value::Null); - assert!(matches!( - res, - AggregateResult::Max(Value::Scalar(Scalar::I64(39))) - )); - - let mut res = AggregateResult::Sum(Scalar::Null); - res.update(Value::Null); - assert!(matches!(res, AggregateResult::Sum(Scalar::Null))); - res.update(Value::Scalar(Scalar::Null)); - assert!(matches!(res, AggregateResult::Sum(Scalar::Null))); - - res.update(Value::Scalar(Scalar::I64(20))); - assert!(matches!(res, AggregateResult::Sum(Scalar::I64(20)))); - - res.update(Value::Scalar(Scalar::I64(-5))); - assert!(matches!(res, AggregateResult::Sum(Scalar::I64(15)))); - - res.update(Value::Scalar(Scalar::Null)); - assert!(matches!(res, AggregateResult::Sum(Scalar::I64(15)))); - } - #[test] fn has_non_null_value() { // Check each column type is wired up. Actual logic is tested in encoders. diff --git a/read_buffer/src/lib.rs b/read_buffer/src/lib.rs index b8d9f13751..23d2372ee6 100644 --- a/read_buffer/src/lib.rs +++ b/read_buffer/src/lib.rs @@ -16,7 +16,7 @@ use std::{ }; use arrow_deps::arrow::record_batch::RecordBatch; -use data_types::{ +use internal_types::{ schema::{builder::SchemaMerger, Schema}, selection::Selection, }; @@ -40,7 +40,7 @@ pub enum Error { // TODO add more context / helpful error here #[snafu(display("Error building unioned read buffer schema for chunks: {}", source))] BuildingSchema { - source: data_types::schema::builder::Error, + source: internal_types::schema::builder::Error, }, #[snafu(display("partition key does not exist: {}", key))] @@ -177,6 +177,25 @@ impl Database { .unwrap_or_default() } + /// Returns the total estimated size in bytes for the chunks in the + /// specified partition. Returns None if there is no such partition + pub fn chunks_size<'a>( + &self, + partition_key: &str, + chunk_ids: impl IntoIterator, + ) -> Option { + let partition_data = self.data.read().unwrap(); + + let partition = partition_data.partitions.get(partition_key); + + partition.map(|partition| { + chunk_ids + .into_iter() + .map(|chunk_id| partition.chunk_size(*chunk_id)) + .sum::() + }) + } + /// Returns the total estimated size in bytes of the database. pub fn size(&self) -> u64 { let base_size = std::mem::size_of::(); @@ -344,11 +363,11 @@ impl Database { // Get all relevant row groups for this chunk's table. This // is cheap because it doesn't execute the read operation, // but just gets references to the needed to data to do so. - if let Some(table_results) = - chunk.read_aggregate(table_name, predicate.clone(), &group_columns, &aggregates) - { - chunk_table_results.push(table_results); - } + let table_results = chunk + .read_aggregate(table_name, predicate.clone(), &group_columns, &aggregates) + .context(ChunkError)?; + + chunk_table_results.push(table_results); } Ok(ReadAggregateResults::new(chunk_table_results)) @@ -662,6 +681,16 @@ impl Partition { .map(|chunk| std::mem::size_of::() as u64 + chunk.size()) .sum::() } + + /// The total estimated size in bytes of the specified chunk id + pub fn chunk_size(&self, chunk_id: u32) -> u64 { + let chunk_data = self.data.read().unwrap(); + chunk_data + .chunks + .get(&chunk_id) + .map(|chunk| chunk.size()) + .unwrap_or(0) // treat unknown chunks as zero size + } } /// ReadFilterResults implements ... @@ -813,7 +842,7 @@ mod test { }, datatypes::DataType::{Boolean, Float64, Int64, UInt64, Utf8}, }; - use data_types::schema::builder::SchemaBuilder; + use internal_types::schema::builder::SchemaBuilder; use crate::value::Values; diff --git a/read_buffer/src/row_group.rs b/read_buffer/src/row_group.rs index 07b94fddea..2a4f112bb0 100644 --- a/read_buffer/src/row_group.rs +++ b/read_buffer/src/row_group.rs @@ -15,20 +15,18 @@ use crate::column::{cmp::Operator, Column, RowIDs, RowIDsOption}; use crate::schema; use crate::schema::{AggregateType, LogicalDataType, ResultSchema}; use crate::value::{ - AggregateResult, EncodedValues, OwnedValue, Scalar, Value, Values, ValuesIterator, + AggregateVec, EncodedValues, OwnedValue, Scalar, Value, Values, ValuesIterator, }; use arrow_deps::arrow::record_batch::RecordBatch; use arrow_deps::{ arrow, datafusion::logical_plan::Expr as DfExpr, datafusion::scalar::ScalarValue as DFScalarValue, }; -use data_types::{ - schema::{InfluxColumnType, Schema}, - selection::Selection, -}; +use internal_types::schema::{InfluxColumnType, Schema}; +use internal_types::selection::Selection; /// The name used for a timestamp column. -pub const TIME_COLUMN_NAME: &str = data_types::TIME_COLUMN_NAME; +pub const TIME_COLUMN_NAME: &str = internal_types::schema::TIME_COLUMN_NAME; #[derive(Debug, Snafu)] pub enum Error { @@ -39,7 +37,7 @@ pub enum Error { #[snafu(display("schema conversion error: {}", source))] SchemaError { - source: data_types::schema::builder::Error, + source: internal_types::schema::builder::Error, }, #[snafu(display("unsupported operation: {}", msg))] @@ -496,17 +494,6 @@ impl RowGroup { aggregate_columns_data.push(column_values); } - // If there is a single group column then we can use an optimised - // approach for building group keys - if group_columns.len() == 1 { - self.read_group_single_group_column( - &mut result, - &groupby_encoded_ids[0], - aggregate_columns_data, - ); - return result; - } - // Perform the group by using a hashmap self.read_group_with_hashing(&mut result, &groupby_encoded_ids, aggregate_columns_data); result @@ -527,11 +514,11 @@ impl RowGroup { // single 128-bit integer as the group key. If grouping is on more than // four columns then a fallback to using an vector as a key will happen. if dst.schema.group_columns.len() <= 4 { - self.read_group_hash_with_u128_key(dst, &groupby_encoded_ids, &aggregate_columns_data); + self.read_group_hash_with_u128_key(dst, &groupby_encoded_ids, aggregate_columns_data); return; } - self.read_group_hash_with_vec_key(dst, &groupby_encoded_ids, &aggregate_columns_data); + self.read_group_hash_with_vec_key(dst, &groupby_encoded_ids, aggregate_columns_data); } // This function is used with `read_group_hash` when the number of columns @@ -541,71 +528,101 @@ impl RowGroup { &'a self, dst: &mut ReadAggregateResult<'a>, groupby_encoded_ids: &[Vec], - aggregate_columns_data: &[Values<'a>], + aggregate_input_columns: Vec>, ) { - // Now begin building the group keys. - let mut groups: HashMap, Vec>> = HashMap::default(); let total_rows = groupby_encoded_ids[0].len(); assert!(groupby_encoded_ids.iter().all(|x| x.len() == total_rows)); - // key_buf will be used as a temporary buffer for group keys, which are - // themselves integers. - let mut key_buf = vec![0; dst.schema.group_columns.len()]; + // These vectors will hold the decoded values of each part of each + // group key. They are the output columns of the input columns used for + // the grouping operation. + let mut group_cols_out = vec![vec![]; groupby_encoded_ids.len()]; + // Each of these vectors will be used to store each aggregate row-value + // for a specific aggregate result column. + let mut agg_cols_out = dst + .schema + .aggregate_columns + .iter() + .map(|(_, agg_type, data_type)| AggregateVec::from((agg_type, data_type))) + .collect::>(); + + // Maps each group key to an ordinal offset on output columns. This + // offset is used to update aggregate values for each group key and to + // store the decoded representations of the group keys themselves in + // the associated output columns. + let mut group_keys: HashMap, usize> = HashMap::default(); + + // reference back to underlying group columns for fetching decoded group + // key values. + let input_group_columns = dst + .schema + .group_column_names_iter() + .map(|name| self.column_by_name(name)) + .collect::>(); + + // key_buf will be used as a temporary buffer for group keys represented + // as a `Vec`. + let mut key_buf = vec![0; dst.schema.group_columns.len()]; + let mut next_ordinal_id = 0; // assign a position for each group key in output columns. for row in 0..total_rows { // update the group key buffer with the group key for this row for (j, col_ids) in groupby_encoded_ids.iter().enumerate() { key_buf[j] = col_ids[row]; } - match groups.raw_entry_mut().from_key(&key_buf) { - // aggregates for this group key are already present. Update - // them - hash_map::RawEntryMut::Occupied(mut entry) => { - for (i, values) in aggregate_columns_data.iter().enumerate() { - entry.get_mut()[i].update(values.value(row)); + match group_keys.raw_entry_mut().from_key(&key_buf) { + hash_map::RawEntryMut::Occupied(entry) => { + let ordinal_id = entry.get(); + + // Update each aggregate column at this ordinal offset + // with the values present in the input columns at the + // current row. + for (agg_col_i, aggregate_result) in agg_cols_out.iter_mut().enumerate() { + aggregate_result.update( + &aggregate_input_columns[agg_col_i], + row, + *ordinal_id, + ) } } // group key does not exist, so create it. hash_map::RawEntryMut::Vacant(entry) => { - let mut group_key_aggs = Vec::with_capacity(dst.schema.aggregate_columns.len()); - for (_, agg_type, _) in &dst.schema.aggregate_columns { - group_key_aggs.push(AggregateResult::from(agg_type)); + // Update each aggregate column at this ordinal offset + // with the values present in the input columns at the + // current row. + for (agg_col_i, aggregate_result) in agg_cols_out.iter_mut().enumerate() { + aggregate_result.update( + &aggregate_input_columns[agg_col_i], + row, + next_ordinal_id, + ) } - for (i, values) in aggregate_columns_data.iter().enumerate() { - group_key_aggs[i].update(values.value(row)); + // Add decoded group key values to the output group columns. + for (group_col_i, group_key_col) in group_cols_out.iter_mut().enumerate() { + if group_key_col.len() >= next_ordinal_id { + group_key_col.resize(next_ordinal_id + 1, None); + } + let decoded_value = input_group_columns[group_col_i] + .decode_id(groupby_encoded_ids[group_col_i][row]); + group_key_col[next_ordinal_id] = match decoded_value { + Value::Null => None, + Value::String(s) => Some(s), + _ => panic!("currently unsupported group column"), + }; } - entry.insert(key_buf.clone(), group_key_aggs); + // update the hashmap with the encoded group key and the + // associated ordinal offset. + entry.insert(key_buf.clone(), next_ordinal_id); + next_ordinal_id += 1; } } } - // Finally, build results set. Each encoded group key needs to be - // materialised into a logical group key - let columns = dst - .schema - .group_column_names_iter() - .map(|name| self.column_by_name(name)) - .collect::>(); - let mut group_key_vec: Vec> = Vec::with_capacity(groups.len()); - let mut aggregate_vec: Vec> = Vec::with_capacity(groups.len()); - - for (group_key, aggs) in groups.into_iter() { - let mut logical_key = Vec::with_capacity(group_key.len()); - for (col_idx, &encoded_id) in group_key.iter().enumerate() { - // TODO(edd): address the cast to u32 - logical_key.push(columns[col_idx].decode_id(encoded_id as u32)); - } - - group_key_vec.push(GroupKey(logical_key)); - aggregate_vec.push(AggregateResults(aggs)); - } - - // update results - dst.group_keys = group_key_vec; - dst.aggregates = aggregate_vec; + dst.group_key_cols = group_cols_out; + dst.aggregate_cols = agg_cols_out; } // This function is similar to `read_group_hash_with_vec_key` in that it @@ -619,74 +636,101 @@ impl RowGroup { &'a self, dst: &mut ReadAggregateResult<'a>, groupby_encoded_ids: &[Vec], - aggregate_columns_data: &[Values<'a>], + aggregate_input_columns: Vec>, ) { let total_rows = groupby_encoded_ids[0].len(); assert!(groupby_encoded_ids.iter().all(|x| x.len() == total_rows)); assert!(dst.schema.group_columns.len() <= 4); - // Now begin building the group keys. - let mut groups: HashMap>> = HashMap::default(); + // These vectors will hold the decoded values of each part of each + // group key. They are the output columns derived from the input + // grouping columns. + let mut group_cols_out: Vec>>> = vec![]; + group_cols_out.resize(groupby_encoded_ids.len(), vec![]); - for row in 0..groupby_encoded_ids[0].len() { - // pack each column's encoded value for the row into a packed group - // key. + // Each of these vectors will be used to store each aggregate row-value + // for a specific aggregate result column. + let mut agg_cols_out = dst + .schema + .aggregate_columns + .iter() + .map(|(_, agg_type, data_type)| AggregateVec::from((agg_type, data_type))) + .collect::>(); + + // Maps each group key to an ordinal offset on output columns. This + // offset is used to update aggregate values for each group key and to + // store the decoded representations of the group keys themselves in + // the associated output columns. + let mut group_keys: HashMap = HashMap::default(); + + // reference back to underlying group columns for fetching decoded group + // key values. + let input_group_columns = dst + .schema + .group_column_names_iter() + .map(|name| self.column_by_name(name)) + .collect::>(); + + let mut next_ordinal_id = 0; // assign a position for each group key in output columns. + for row in 0..total_rows { + // pack each column's encoded value for the row into a packed + // group key. let mut group_key_packed = 0_u128; for (i, col_ids) in groupby_encoded_ids.iter().enumerate() { group_key_packed = pack_u32_in_u128(group_key_packed, col_ids[row], i); } - match groups.raw_entry_mut().from_key(&group_key_packed) { - // aggregates for this group key are already present. Update - // them - hash_map::RawEntryMut::Occupied(mut entry) => { - for (i, values) in aggregate_columns_data.iter().enumerate() { - entry.get_mut()[i].update(values.value(row)); + match group_keys.raw_entry_mut().from_key(&group_key_packed) { + hash_map::RawEntryMut::Occupied(entry) => { + let ordinal_id = entry.get(); + + // Update each aggregate column at this ordinal offset + // with the values present in the input columns at the + // current row. + for (agg_col_i, aggregate_result) in agg_cols_out.iter_mut().enumerate() { + aggregate_result.update( + &aggregate_input_columns[agg_col_i], + row, + *ordinal_id, + ) } } - // group key does not exist, so create it. hash_map::RawEntryMut::Vacant(entry) => { - let mut group_key_aggs = Vec::with_capacity(dst.schema.aggregate_columns.len()); - for (_, agg_type, _) in &dst.schema.aggregate_columns { - group_key_aggs.push(AggregateResult::from(agg_type)); + // Update each aggregate column at this ordinal offset + // with the values present in the input columns at the + // current row. + for (agg_col_i, aggregate_result) in agg_cols_out.iter_mut().enumerate() { + aggregate_result.update( + &aggregate_input_columns[agg_col_i], + row, + next_ordinal_id, + ) } - for (i, values) in aggregate_columns_data.iter().enumerate() { - group_key_aggs[i].update(values.value(row)); + // Add decoded group key values to the output group columns. + for (group_col_i, group_key_col) in group_cols_out.iter_mut().enumerate() { + if group_key_col.len() >= next_ordinal_id { + group_key_col.resize(next_ordinal_id + 1, None); + } + let decoded_value = input_group_columns[group_col_i] + .decode_id(groupby_encoded_ids[group_col_i][row]); + group_key_col[next_ordinal_id] = match decoded_value { + Value::Null => None, + Value::String(s) => Some(s), + _ => panic!("currently unsupported group column"), + }; } - entry.insert(group_key_packed, group_key_aggs); + // update the hashmap with the encoded group key and the + // associated ordinal offset. + entry.insert(group_key_packed, next_ordinal_id); + next_ordinal_id += 1; } } } - // Finally, build results set. Each encoded group key needs to be - // materialised into a logical group key - let columns = dst - .schema - .group_column_names_iter() - .map(|name| self.column_by_name(name)) - .collect::>(); - let mut group_key_vec: Vec> = Vec::with_capacity(groups.len()); - let mut aggregate_vec: Vec> = Vec::with_capacity(groups.len()); - - for (group_key_packed, aggs) in groups.into_iter() { - let mut logical_key = Vec::with_capacity(columns.len()); - - // Unpack the appropriate encoded id for each column from the packed - // group key, then materialise the logical value for that id and add - // it to the materialised group key (`logical_key`). - for (col_idx, column) in columns.iter().enumerate() { - let encoded_id = (group_key_packed >> (col_idx * 32)) as u32; - logical_key.push(column.decode_id(encoded_id)); - } - - group_key_vec.push(GroupKey(logical_key)); - aggregate_vec.push(AggregateResults(aggs)); - } - - dst.group_keys = group_key_vec; - dst.aggregates = aggregate_vec; + dst.group_key_cols = group_cols_out; + dst.aggregate_cols = agg_cols_out; } // Optimised `read_group` method when there are no predicates and all the @@ -695,20 +739,24 @@ impl RowGroup { // In this case all the grouping columns pre-computed bitsets for each // distinct value. fn read_group_all_rows_all_rle<'a>(&'a self, dst: &mut ReadAggregateResult<'a>) { - let group_columns = dst + // References to the columns to be used as input for producing the + // output aggregates. + let input_group_columns = dst .schema .group_column_names_iter() .map(|name| self.column_by_name(name)) .collect::>(); - let aggregate_columns_typ = dst + // References to the columns to be used as input for producing the + // output aggregates. Also returns the required aggregate type. + let input_aggregate_columns = dst .schema .aggregate_columns .iter() .map(|(col_type, agg_type, _)| (self.column_by_name(col_type.as_str()), *agg_type)) .collect::>(); - let encoded_groups = dst + let groupby_encoded_ids = dst .schema .group_column_names_iter() .map(|col_type| { @@ -718,6 +766,23 @@ impl RowGroup { }) .collect::>(); + // These vectors will hold the decoded values of each part of each + // group key. They are the output columns derived from the input + // grouping columns. + let mut group_cols_out: Vec>>> = vec![]; + group_cols_out.resize(groupby_encoded_ids.len(), vec![]); + + // Each of these vectors will be used to store each aggregate row-value + // for a specific aggregate result column. + let mut agg_cols_out = dst + .schema + .aggregate_columns + .iter() + .map(|(_, agg_type, data_type)| AggregateVec::from((agg_type, data_type))) + .collect::>(); + + let mut output_rows = 0; + // multi_cartesian_product will create the cartesian product of all // grouping-column values. This is likely going to be more group keys // than there exists row-data for, so don't materialise them yet... @@ -743,129 +808,85 @@ impl RowGroup { // [0, 3], [1, 3], [2, 3], [2, 4], [3, 2], [4, 1] // // We figure out which group keys have data and which don't in the loop - // below, by intersecting bitsets for each id and checking for non-empty - // sets. - let group_keys = encoded_groups + // below, by intersecting row_id bitsets for each encoded id, and + // checking for non-empty sets. + let candidate_group_keys = groupby_encoded_ids .iter() .map(|ids| (0..ids.len())) .multi_cartesian_product(); // Let's figure out which of the candidate group keys are actually group // keys with data. - 'outer: for group_key in group_keys { - let mut aggregate_row_ids = - Cow::Borrowed(encoded_groups[0][group_key[0]].unwrap_bitmap()); + 'outer: for group_key_buf in candidate_group_keys { + let mut group_key_row_ids = + Cow::Borrowed(groupby_encoded_ids[0][group_key_buf[0]].unwrap_bitmap()); - if aggregate_row_ids.is_empty() { + if group_key_row_ids.is_empty() { continue; } - for i in 1..group_key.len() { - let other = encoded_groups[i][group_key[i]].unwrap_bitmap(); + for i in 1..group_key_buf.len() { + let other = groupby_encoded_ids[i][group_key_buf[i]].unwrap_bitmap(); - if aggregate_row_ids.and_cardinality(other) > 0 { - aggregate_row_ids = Cow::Owned(aggregate_row_ids.and(other)); + if group_key_row_ids.and_cardinality(other) > 0 { + group_key_row_ids = Cow::Owned(group_key_row_ids.and(other)); } else { continue 'outer; } } - // This group key has some matching row ids. Materialise the group - // key and calculate the aggregates. + // There exist rows for this group key combination. Materialise the + // group key and calculate the aggregates for this key using set + // of row IDs. + output_rows += 1; - // TODO(edd): given these RLE columns should have low cardinality - // there should be a reasonably low group key cardinality. It could - // be safe to use `small_vec` here without blowing the stack up. - let mut material_key = Vec::with_capacity(group_key.len()); - for (col_idx, &encoded_id) in group_key.iter().enumerate() { - material_key.push(group_columns[col_idx].decode_id(encoded_id as u32)); - } - dst.group_keys.push(GroupKey(material_key)); + // Add decoded group key values to the output group columns. + for (group_col_i, col) in group_cols_out.iter_mut().enumerate() { + let decoded_value = + input_group_columns[group_col_i].decode_id(group_key_buf[group_col_i] as u32); - let mut aggregates = Vec::with_capacity(aggregate_columns_typ.len()); - for (agg_col, typ) in &aggregate_columns_typ { - aggregates.push(match typ { - AggregateType::Count => { - AggregateResult::Count(agg_col.count(&aggregate_row_ids.to_vec()) as u64) - } - AggregateType::First => todo!(), - AggregateType::Last => todo!(), - AggregateType::Min => { - AggregateResult::Min(agg_col.min(&aggregate_row_ids.to_vec())) - } - AggregateType::Max => { - AggregateResult::Max(agg_col.max(&aggregate_row_ids.to_vec())) - } - AggregateType::Sum => { - AggregateResult::Sum(agg_col.sum(&aggregate_row_ids.to_vec())) - } + col.push(match decoded_value { + Value::Null => None, + Value::String(s) => Some(s), + _ => panic!("currently unsupported group column"), }); } - dst.aggregates.push(AggregateResults(aggregates)); - } - } - // Optimised `read_group` path for queries where only a single column is - // being grouped on. In this case building a hash table is not necessary, - // and the group keys can be used as indexes into a vector whose values - // contain aggregates. As rows are processed these aggregates can be updated - // in constant time. - fn read_group_single_group_column<'a>( - &'a self, - dst: &mut ReadAggregateResult<'a>, - groupby_encoded_ids: &[u32], - aggregate_columns_data: Vec>, - ) { - assert_eq!(dst.schema().group_columns.len(), 1); - let column = self.column_by_name(dst.schema.group_column_names_iter().next().unwrap()); - - // Allocate a vector to hold aggregates that can be updated as rows are - // processed. An extra group is required because encoded ids are - // 0-indexed. - let required_groups = groupby_encoded_ids.iter().max().unwrap() + 1; - let mut groups: Vec>>> = - vec![None; required_groups as usize]; - - for (row, encoded_id) in groupby_encoded_ids.iter().enumerate() { - let idx = *encoded_id as usize; - match &mut groups[idx] { - Some(group_key_aggs) => { - // Update all aggregates for the group key - for (i, values) in aggregate_columns_data.iter().enumerate() { - group_key_aggs[i].update(values.value(row)); + // Calculate an aggregate from each input aggregate column and + // set it at the relevant offset in the output column. + for (agg_col_i, (agg_col, typ)) in input_aggregate_columns.iter().enumerate() { + match typ { + AggregateType::Count => { + let agg = agg_col.count(&group_key_row_ids.to_vec()) as u64; + agg_cols_out[agg_col_i].push(Value::Scalar(Scalar::U64(agg))) } - } - None => { - let mut group_key_aggs = dst - .schema - .aggregate_columns - .iter() - .map(|(_, agg_type, _)| AggregateResult::from(agg_type)) - .collect::>(); - - for (i, values) in aggregate_columns_data.iter().enumerate() { - group_key_aggs[i].update(values.value(row)); + AggregateType::First => {} + AggregateType::Last => {} + AggregateType::Min => { + let agg = agg_col.min(&group_key_row_ids.to_vec()); + agg_cols_out[agg_col_i].push(agg); + } + AggregateType::Max => { + let agg = agg_col.max(&group_key_row_ids.to_vec()); + agg_cols_out[agg_col_i].push(agg); + } + AggregateType::Sum => { + let agg = agg_col.sum(&group_key_row_ids.to_vec()); + agg_cols_out[agg_col_i].push(Value::Scalar(agg)); } - - groups[idx] = Some(group_key_aggs); } } } - // Finally, build results set. Each encoded group key needs to be - // materialised into a logical group key - let mut group_key_vec: Vec> = Vec::with_capacity(groups.len()); - let mut aggregate_vec = Vec::with_capacity(groups.len()); - - for (group_key, aggs) in groups.into_iter().enumerate() { - if let Some(aggs) = aggs { - group_key_vec.push(GroupKey(vec![column.decode_id(group_key as u32)])); - aggregate_vec.push(AggregateResults(aggs)); - } + for col in &group_cols_out { + assert_eq!(col.len(), output_rows); + } + for col in &agg_cols_out { + assert_eq!(col.len(), output_rows); } - dst.group_keys = group_key_vec; - dst.aggregates = aggregate_vec; + dst.group_key_cols = group_cols_out; + dst.aggregate_cols = agg_cols_out; } // Optimised `read_group` method for cases where the columns being grouped @@ -873,7 +894,7 @@ impl RowGroup { // // In this case the rows are already in "group key order" and the aggregates // can be calculated by reading the rows in order. - fn read_group_sorted_stream( + fn _read_group_sorted_stream( &self, _predicates: &Predicate, _group_column: ColumnName<'_>, @@ -884,13 +905,6 @@ impl RowGroup { // Applies aggregates on multiple columns with an optional predicate. fn aggregate_columns<'a>(&'a self, predicate: &Predicate, dst: &mut ReadAggregateResult<'a>) { - let aggregate_columns = dst - .schema - .aggregate_columns - .iter() - .map(|(col_type, agg_type, _)| (self.column_by_name(col_type.as_str()), *agg_type)) - .collect::>(); - let row_ids = match predicate.is_empty() { true => { // TODO(edd): PERF - teach each column encoding how to produce @@ -910,26 +924,30 @@ impl RowGroup { }, }; - // the single row that will store the aggregate column values. - let mut aggregate_row = vec![]; - for (col, agg_type) in aggregate_columns { - match agg_type { - AggregateType::Count => { - aggregate_row.push(AggregateResult::Count(col.count(&row_ids) as u64)); + dst.aggregate_cols = dst + .schema + .aggregate_columns + .iter() + .map(|(col_type, agg_type, data_type)| { + let col = self.column_by_name(col_type.as_str()); // input aggregate column + let mut agg_vec = AggregateVec::from((agg_type, data_type)); + + // produce single aggregate for the input column subject to a + // predicate filter. + match agg_type { + AggregateType::Count => { + let value = Value::Scalar(Scalar::U64(col.count(&row_ids) as u64)); + agg_vec.push(value); + } + AggregateType::First => unimplemented!("First not yet implemented"), + AggregateType::Last => unimplemented!("Last not yet implemented"), + AggregateType::Min => agg_vec.push(col.min(&row_ids)), + AggregateType::Max => agg_vec.push(col.max(&row_ids)), + AggregateType::Sum => agg_vec.push(Value::Scalar(col.sum(&row_ids))), } - AggregateType::Sum => { - aggregate_row.push(AggregateResult::Sum(col.sum(&row_ids))); - } - AggregateType::Min => { - aggregate_row.push(AggregateResult::Min(col.min(&row_ids))); - } - AggregateType::Max => { - aggregate_row.push(AggregateResult::Max(col.max(&row_ids))); - } - _ => unimplemented!("Other aggregates are not yet supported"), - } - } - dst.aggregates.push(AggregateResults(aggregate_row)); // write the row + agg_vec + }) + .collect::>(); } /// Given the predicate (which may be empty), determine a set of rows @@ -1128,6 +1146,7 @@ fn pack_u32_in_u128(packed_value: u128, encoded_id: u32, pos: usize) -> u128 { // Given a packed encoded group key, unpacks them into `n` individual `u32` // group keys, and stores them in `dst`. It is the caller's responsibility to // ensure n <= 4. +#[cfg(test)] fn unpack_u128_group_key(group_key_packed: u128, n: usize, mut dst: Vec) -> Vec { dst.resize(n, 0); @@ -1346,83 +1365,6 @@ impl TryFrom<&DfExpr> for BinaryExpr { } } -// A GroupKey is an ordered collection of row values. The order determines which -// columns the values originated from. -#[derive(PartialEq, PartialOrd, Clone)] -pub struct GroupKey<'row_group>(Vec>); - -impl GroupKey<'_> { - fn len(&self) -> usize { - self.0.len() - } -} - -impl<'a> From>> for GroupKey<'a> { - fn from(values: Vec>) -> Self { - Self(values) - } -} - -impl Eq for GroupKey<'_> {} - -// Implementing the `Ord` trait on `GroupKey` means that collections of group -// keys become sortable. This is typically useful for test because depending on -// the implementation, group keys are not always emitted in sorted order. -// -// To be compared, group keys *must* have the same length, or `cmp` will panic. -// They will be ordered as follows: -// -// [foo, zoo, zoo], [foo, bar, zoo], [bar, bar, bar], [bar, bar, zoo], -// -// becomes: -// -// [bar, bar, bar], [bar, bar, zoo], [foo, bar, zoo], [foo, zoo, zoo], -// -// Be careful sorting group keys in result sets, because other columns -// associated with the group keys won't be sorted unless the correct `sort` -// methods are used on the result set implementations. -impl Ord for GroupKey<'_> { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - // two group keys must have same length - assert_eq!(self.0.len(), other.0.len()); - - let cols = self.0.len(); - for i in 0..cols { - match self.0[i].partial_cmp(&other.0[i]) { - Some(ord) => return ord, - None => continue, - } - } - - std::cmp::Ordering::Equal - } -} - -#[derive(Debug, PartialEq, Clone)] -pub struct AggregateResults<'row_group>(Vec>); - -impl<'row_group> AggregateResults<'row_group> { - fn len(&self) -> usize { - self.0.len() - } - - fn merge(&mut self, other: &AggregateResults<'row_group>) { - assert_eq!(self.0.len(), other.len()); - for (i, agg) in self.0.iter_mut().enumerate() { - agg.merge(&other.0[i]); - } - } -} - -impl<'a> IntoIterator for AggregateResults<'a> { - type Item = AggregateResult<'a>; - type IntoIter = std::vec::IntoIter; - - fn into_iter(self) -> Self::IntoIter { - self.0.into_iter() - } -} - // A representation of a column name. pub type ColumnName<'a> = &'a str; @@ -1571,11 +1513,6 @@ impl MetaData { self.columns_size += column_size; } - // Returns meta information about the column. - fn column_meta(&self, name: ColumnName<'_>) -> &ColumnMeta { - self.columns.get(name).unwrap() - } - // Extract schema information for a set of columns. fn schema_for_column_names( &self, @@ -1638,7 +1575,7 @@ impl TryFrom> for RecordBatch { type Error = Error; fn try_from(result: ReadFilterResult<'_>) -> Result { - let schema = data_types::schema::Schema::try_from(result.schema()) + let schema = internal_types::schema::Schema::try_from(result.schema()) .map_err(|source| Error::SchemaError { source })?; let arrow_schema: arrow_deps::arrow::datatypes::SchemaRef = schema.into(); @@ -1705,58 +1642,68 @@ pub struct ReadAggregateResult<'row_group> { // a schema describing the columns in the results and their types. pub(crate) schema: ResultSchema, - // row-wise collection of group keys. Each group key contains column-wise - // values for each of the groupby_columns. - pub(crate) group_keys: Vec>, + // The collection of columns forming the group keys. + pub(crate) group_key_cols: Vec>>, - // row-wise collection of aggregates. Each aggregate contains column-wise - // values for each of the aggregate_columns. - pub(crate) aggregates: Vec>, + // The collection of aggregate columns. Each value in each column is an + // aggregate associated with the group key built from values in the group + // columns and the same ordinal position. + pub(crate) aggregate_cols: Vec, pub(crate) group_keys_sorted: bool, } impl<'row_group> ReadAggregateResult<'row_group> { - fn with_capacity(schema: ResultSchema, capacity: usize) -> Self { + pub fn new(schema: ResultSchema) -> Self { Self { schema, - group_keys: Vec::with_capacity(capacity), - aggregates: Vec::with_capacity(capacity), ..Default::default() } } + /// A `ReadAggregateResult` is empty if there are no aggregate columns. pub fn is_empty(&self) -> bool { - self.aggregates.is_empty() + self.aggregate_cols.is_empty() } pub fn schema(&self) -> &ResultSchema { &self.schema } - // The number of rows in the result. + /// The number of rows in the result. pub fn rows(&self) -> usize { - self.aggregates.len() + if self.aggregate_cols.is_empty() { + return 0; + } + self.aggregate_cols[0].len() } - // The number of distinct group keys in the result. + // The number of distinct group keys in the result. Not the same as `rows()` + // because a `ReadAggregateResult` can have no group keys and have a single + // aggregate row. pub fn cardinality(&self) -> usize { - self.group_keys.len() + if self.group_key_cols.is_empty() { + return 0; + } + self.group_key_cols[0].len() } // Is this result for a grouped aggregate? pub fn is_grouped_aggregate(&self) -> bool { - !self.group_keys.is_empty() + !self.group_key_cols.is_empty() + } + + // The number of grouping columns. + pub fn group_key_columns(&self) -> usize { + self.group_key_cols.len() } // Whether or not the rows in the results are sorted by group keys or not. pub fn group_keys_sorted(&self) -> bool { - self.group_keys.is_empty() || self.group_keys_sorted + self.group_key_cols.is_empty() || self.group_keys_sorted } /// Merges `other` and self, returning a new set of results. - /// - /// NOTE: This is slow! Not expected to be the final type of implementation pub fn merge( mut self, mut other: ReadAggregateResult<'row_group>, @@ -1780,194 +1727,367 @@ impl<'row_group> ReadAggregateResult<'row_group> { other.sort(); } - let self_group_keys = self.cardinality(); - let self_len = self.rows(); - let other_len = other.rows(); - let mut result = Self::with_capacity(self.schema, self_len.max(other_len)); + let mut result = Self::new(self.schema.clone()); + // Allocate output grouping columns + result + .group_key_cols + .resize(result.schema.group_columns.len(), vec![]); + + // Allocate output aggregate columns + result.aggregate_cols = result + .schema + .aggregate_columns + .iter() + .map(|(_, agg_type, data_type)| AggregateVec::from((agg_type, data_type))) + .collect::>(); + + let mut self_i = 0; + let mut other_i = 0; + while self_i < self.rows() || other_i < other.rows() { + if self_i == self.rows() { + // drained self, add the rest of other's group key columns + for (col_i, col) in result.group_key_cols.iter_mut().enumerate() { + col.extend(other.group_key_cols[col_i].iter().skip(other_i)); + } + + // add the rest of other's aggregate columns + // + // N.B - by checking the data type of the aggregate columns here + // we can do type checking on a column basis (once) rather than + // for each row. This allows us to extract an aggregate vec + // and an iterator of the same type to extend the aggregate vec. + for (col_i, (_, _, data_type)) in result.schema.aggregate_columns.iter().enumerate() + { + match data_type { + LogicalDataType::Integer => { + let arr = other.aggregate_cols.remove(0); + result.aggregate_cols[col_i] + .extend_with_i64(arr.take_as_i64().into_iter()); + } + LogicalDataType::Unsigned => { + let arr = other.aggregate_cols.remove(0); + result.aggregate_cols[col_i] + .extend_with_u64(arr.take_as_u64().into_iter()); + } + LogicalDataType::Float => { + let arr = other.aggregate_cols.remove(0); + result.aggregate_cols[col_i] + .extend_with_f64(arr.take_as_f64().into_iter()); + } + LogicalDataType::String => { + let arr = other.aggregate_cols.remove(0); + result.aggregate_cols[col_i] + .extend_with_str(arr.take_as_str().into_iter()); + } + LogicalDataType::Binary => { + let arr = other.aggregate_cols.remove(0); + result.aggregate_cols[col_i] + .extend_with_bytes(arr.take_as_bytes().into_iter()); + } + LogicalDataType::Boolean => { + let arr = other.aggregate_cols.remove(0); + result.aggregate_cols[col_i] + .extend_with_bool(arr.take_as_bool().into_iter()); + } + } + } - let mut i: usize = 0; - let mut j: usize = 0; - while i < self_len || j < other_len { - if i >= self_len { - // drained self, add the rest of other - result - .group_keys - .extend(other.group_keys.iter().skip(j).cloned()); - result - .aggregates - .extend(other.aggregates.iter().skip(j).cloned()); return result; - } else if j >= other_len { - // drained other, add the rest of self - result - .group_keys - .extend(self.group_keys.iter().skip(j).cloned()); - result - .aggregates - .extend(self.aggregates.iter().skip(j).cloned()); + } else if other_i == other.rows() { + // drained other, add the rest of self's group key columns + for (col_i, col) in result.group_key_cols.iter_mut().enumerate() { + col.extend(self.group_key_cols[col_i].iter().skip(self_i)); + } + + // add the rest of self's aggregate columns + for (col_i, (_, _, data_type)) in result.schema.aggregate_columns.iter().enumerate() + { + match data_type { + LogicalDataType::Integer => { + let arr = self.aggregate_cols.remove(0); + result.aggregate_cols[col_i] + .extend_with_i64(arr.take_as_i64().into_iter()); + } + LogicalDataType::Unsigned => { + let arr = self.aggregate_cols.remove(0); + result.aggregate_cols[col_i] + .extend_with_u64(arr.take_as_u64().into_iter()); + } + LogicalDataType::Float => { + let arr = self.aggregate_cols.remove(0); + result.aggregate_cols[col_i] + .extend_with_f64(arr.take_as_f64().into_iter()); + } + LogicalDataType::String => { + let arr = self.aggregate_cols.remove(0); + result.aggregate_cols[col_i] + .extend_with_str(arr.take_as_str().into_iter()); + } + LogicalDataType::Binary => { + let arr = self.aggregate_cols.remove(0); + result.aggregate_cols[col_i] + .extend_with_bytes(arr.take_as_bytes().into_iter()); + } + LogicalDataType::Boolean => { + let arr = self.aggregate_cols.remove(0); + result.aggregate_cols[col_i] + .extend_with_bool(arr.take_as_bool().into_iter()); + } + } + } + return result; } - // just merge the aggregate if there are no group keys - if self_group_keys == 0 { - assert!((self_len == other_len) && self_len == 1); // there should be a single aggregate row - self.aggregates[i].merge(&other.aggregates[j]); - result.aggregates.push(self.aggregates[i].clone()); - return result; + // compare the next row in self and other and determine if there is + // a clear lexicographic order. + let mut ord = Ordering::Equal; + for i in 0..result.schema.group_columns.len() { + match self.group_key_cols[i][self_i].partial_cmp(&other.group_key_cols[i][other_i]) + { + Some(o) => { + ord = o; + if !matches!(ord, Ordering::Equal) { + break; + } + } + None => continue, + } } - // there are group keys so merge them. - match self.group_keys[i].cmp(&other.group_keys[j]) { + match ord { Ordering::Less => { - result.group_keys.push(self.group_keys[i].clone()); - result.aggregates.push(self.aggregates[i].clone()); - i += 1; + // move the next row for each of self's columns onto result. + for (col_i, col) in result.group_key_cols.iter_mut().enumerate() { + col.push(self.group_key_cols[col_i][self_i]); + } + for (col_i, col) in result.aggregate_cols.iter_mut().enumerate() { + col.push(self.aggregate_cols[col_i].value(self_i)); + } + self_i += 1; } Ordering::Equal => { - // merge aggregates - self.aggregates[i].merge(&other.aggregates[j]); - result.group_keys.push(self.group_keys[i].clone()); - result.aggregates.push(self.aggregates[i].clone()); - i += 1; - j += 1; + // move the next row for each of self's columns onto result. + for (col_i, col) in result.group_key_cols.iter_mut().enumerate() { + col.push(self.group_key_cols[col_i][self_i]); + } + + // merge all the aggregates for this group key. + for (col_i, col) in result.aggregate_cols.iter_mut().enumerate() { + let self_value = self.aggregate_cols[col_i].value(self_i); + let other_value = other.aggregate_cols[col_i].value(other_i); + let (_, agg_type, _) = &self.schema.aggregate_columns[col_i]; + col.push(match agg_type { + AggregateType::Count => self_value + other_value, + AggregateType::Min => match self_value.partial_cmp(&other_value) { + Some(ord) => match ord { + Ordering::Less => self_value, + Ordering::Equal => self_value, + Ordering::Greater => other_value, + }, + None => self_value, + }, + AggregateType::Max => match self_value.partial_cmp(&other_value) { + Some(ord) => match ord { + Ordering::Less => other_value, + Ordering::Equal => other_value, + Ordering::Greater => self_value, + }, + None => self_value, + }, + AggregateType::Sum => self_value + other_value, + _ => unimplemented!("first/last not implemented"), + }); + } + self_i += 1; + other_i += 1; } Ordering::Greater => { - result.group_keys.push(other.group_keys[j].clone()); - result.aggregates.push(other.aggregates[j].clone()); - j += 1; + // move the next row for each of other's columns onto result. + for (col_i, col) in result.group_key_cols.iter_mut().enumerate() { + col.push(other.group_key_cols[col_i][other_i]); + } + for (col_i, col) in result.aggregate_cols.iter_mut().enumerate() { + col.push(other.aggregate_cols[col_i].value(other_i)); + } + other_i += 1; } } } - result } - /// Executes a mutable sort of the rows in the result set based on the - /// lexicographic order of each group key column. - /// - /// TODO(edd): this has really poor performance. It clones the underlying - /// vectors rather than sorting them in place. + // Executes a mutable sort of the results based on the lexicographic order + // of each group key columns. + // + // Given these group key columns: + // + // [foo [zoo [zoo + // foo bar zoo + // bar bar bar + // bar] bar] zoo] + // + // `sort` would result them becoming: + // + // [bar [bar [bar + // bar bar zoo + // foo bar zoo + // foo] zoo] zoo] + // + // The same permutation is also applied to the aggregate columns. + // pub fn sort(&mut self) { - // The permutation crate lets you execute a sort on anything implements - // `Ord` and return the sort order, which can then be applied to other - // columns. - let perm = permutation::sort(self.group_keys.as_slice()); - self.group_keys = perm.apply_slice(self.group_keys.as_slice()); - self.aggregates = perm.apply_slice(self.aggregates.as_slice()); + if self.group_keys_sorted { + return; + } + + // Create a vector of group keys, which allows us to determine a + // permutation by which we should sort all columns. + let mut group_keys = (0..self.rows()) + .map(|i| GroupKey::new(&self.group_key_cols, i)) + .collect::>(); + + // sort the vector of group keys, which will give us a permutation + // that we can apply to all of the columns. + group_keys.sort_unstable_by(|a, b| { + let cols = a.len(); + for i in 0..cols { + match a.columns[i][a.row_offset].partial_cmp(&b.columns[i][b.row_offset]) { + Some(ord) => { + if matches!(ord, Ordering::Equal) { + continue; + } + return ord; + } + None => continue, + } + } + + std::cmp::Ordering::Equal + }); + + // Now create a permutation by looking at how the row_offsets have been + // ordered in the `group_keys` array. + let perm = permutation::Permutation::from_vec( + group_keys + .iter() + .map(|gk| gk.row_offset) + .collect::>(), + ); + assert_eq!(perm.len(), self.rows()); + + // Apply that permutation to all of the columns. + for col in self.group_key_cols.iter_mut() { + *col = perm.apply_slice(col.as_slice()); + } + + for col in self.aggregate_cols.iter_mut() { + col.sort_with_permutation(&perm); + } + self.group_keys_sorted = true; } +} - pub fn add_row( - &mut self, - group_key: Vec>, - aggregates: Vec>, - ) { - self.group_keys.push(GroupKey(group_key)); - self.aggregates.push(AggregateResults(aggregates)); +// The `GroupKey` struct is a wrapper over a specific row of data in grouping +// columns. +// +// Rather than pivot the columns into a row-wise orientation to sort them, we +// can effectively sort a projection across them (`row_offset`) storing +// `GroupKey`s in a vector and sorting that. +struct GroupKey<'a> { + columns: &'a [Vec>], + row_offset: usize, +} + +impl<'a> GroupKey<'a> { + fn new(columns: &'a [Vec>], offset: usize) -> Self { + Self { + columns, + row_offset: offset, + } + } + + // The number of columns comprising the `GroupKey`. + fn len(&self) -> usize { + self.columns.len() } } impl TryFrom> for RecordBatch { type Error = Error; - fn try_from(result: ReadAggregateResult<'_>) -> Result { - let schema = data_types::schema::Schema::try_from(result.schema()) + fn try_from(mut result: ReadAggregateResult<'_>) -> Result { + let schema = internal_types::schema::Schema::try_from(result.schema()) .map_err(|source| Error::SchemaError { source })?; let arrow_schema: arrow_deps::arrow::datatypes::SchemaRef = schema.into(); - // Build the columns for the group keys. This involves pivoting the - // row-wise group keys into column-wise data. - let mut group_column_builders = (0..result.schema.group_columns.len()) - .map(|_| { - arrow::array::StringBuilder::with_capacity( - result.cardinality(), - result.cardinality() * 8, // arbitrarily picked for now - ) - }) - .collect::>(); - - // build each column for a group key value row by row. - for gk in result.group_keys.iter() { - for (i, v) in gk.0.iter().enumerate() { - group_column_builders[i] - .append_value(v.string()) - .map_err(|source| Error::ArrowError { source })?; - } - } - // Add the group columns to the set of column data for the record batch. let mut columns: Vec> = Vec::with_capacity(result.schema.len()); - for col in group_column_builders.iter_mut() { - columns.push(Arc::new(col.finish())); + + for (_, data_type) in &result.schema.group_columns { + match data_type { + LogicalDataType::String => { + columns.push(Arc::new(array::StringArray::from( + result.group_key_cols.remove(0), // move column out of result + ))); + } + _ => panic!("only String currently supported as group column"), + } } - // For the aggregate columns, build one column at a time, repeatedly - // iterating rows until all columns have been built. - // - // TODO(edd): I don't like this *at all*. I'm going to refactor the way - // aggregates are produced. - for (i, (_, _, data_type)) in result.schema.aggregate_columns.iter().enumerate() { + for (_, _, data_type) in &result.schema.aggregate_columns { match data_type { LogicalDataType::Integer => { - let mut builder = array::Int64Builder::new(result.cardinality()); - for agg_row in &result.aggregates { - builder - .append_option(agg_row.0[i].try_as_i64_scalar()) - .context(ArrowError)?; - } - columns.push(Arc::new(builder.finish())); + columns.push(Arc::new(array::Int64Array::from( + result.aggregate_cols.remove(0).take_as_i64(), + ))); } LogicalDataType::Unsigned => { - let mut builder = array::UInt64Builder::new(result.cardinality()); - for agg_row in &result.aggregates { - builder - .append_option(agg_row.0[i].try_as_u64_scalar()) - .context(ArrowError)?; - } - columns.push(Arc::new(builder.finish())); + columns.push(Arc::new(array::UInt64Array::from( + result.aggregate_cols.remove(0).take_as_u64(), + ))); } LogicalDataType::Float => { - let mut builder = array::Float64Builder::new(result.cardinality()); - for agg_row in &result.aggregates { - builder - .append_option(agg_row.0[i].try_as_f64_scalar()) - .context(ArrowError)?; - } - columns.push(Arc::new(builder.finish())); + columns.push(Arc::new(array::Float64Array::from( + result.aggregate_cols.remove(0).take_as_f64(), + ))); } LogicalDataType::String => { - let mut builder = array::StringBuilder::new(result.cardinality()); - for agg_row in &result.aggregates { - match agg_row.0[i].try_as_str() { - Some(s) => builder.append_value(s).context(ArrowError)?, - None => builder.append_null().context(ArrowError)?, - } - } - columns.push(Arc::new(builder.finish())); + columns.push(Arc::new(array::StringArray::from( + result + .aggregate_cols + .remove(0) + .take_as_str() + .iter() + .map(|x| x.as_deref()) + .collect::>(), + ))); } LogicalDataType::Binary => { - let mut builder = array::BinaryBuilder::new(result.cardinality()); - for agg_row in &result.aggregates { - match agg_row.0[i].try_as_bytes() { - Some(s) => builder.append_value(s).context(ArrowError)?, - None => builder.append_null().context(ArrowError)?, - } - } - columns.push(Arc::new(builder.finish())); + columns.push(Arc::new(array::BinaryArray::from( + result + .aggregate_cols + .remove(0) + .take_as_bytes() + .iter() + .map(|x| x.as_deref()) + .collect::>(), + ))); } LogicalDataType::Boolean => { - let mut builder = array::BooleanBuilder::new(result.cardinality()); - for agg_row in &result.aggregates { - builder - .append_option(agg_row.0[i].try_as_bool()) - .context(ArrowError)?; - } - columns.push(Arc::new(builder.finish())); + columns.push(Arc::new(array::BooleanArray::from( + result.aggregate_cols.remove(0).take_as_bool(), + ))); } } } + // everything has been moved and copied into record batch. + assert!(result.group_key_cols.is_empty()); + assert!(result.aggregate_cols.is_empty()); + // try_new only returns an error if the schema is invalid or the number // of rows on columns differ. We have full control over both so there // should never be an error to return... @@ -1979,8 +2099,8 @@ impl TryFrom> for RecordBatch { impl PartialEq for ReadAggregateResult<'_> { fn eq(&self, other: &Self) -> bool { self.schema() == other.schema() - && self.group_keys == other.group_keys - && self.aggregates == other.aggregates + && self.group_key_cols == other.group_key_cols + && self.aggregate_cols == other.aggregate_cols } } @@ -2005,24 +2125,27 @@ impl std::fmt::Display for &ReadAggregateResult<'_> { } // There may or may not be group keys - let expected_rows = self.aggregates.len(); + let expected_rows = self.rows(); for row in 0..expected_rows { if row > 0 { writeln!(f)?; } // write row for group by columns - if !self.group_keys.is_empty() { - for value in self.group_keys[row].0.iter() { - write!(f, "{},", value)?; + if self.is_grouped_aggregate() { + for col in &self.group_key_cols { + match col[row] { + Some(v) => write!(f, "{},", v)?, + None => write!(f, "NULL,")?, + } } } // write row for aggregate columns - for (col_i, agg) in self.aggregates[row].0.iter().enumerate() { - write!(f, "{}", agg)?; - if col_i < self.aggregates[row].0.len() - 1 { - write!(f, ",")?; + for (i, col) in self.aggregate_cols.iter().enumerate() { + col.write_value(row, f)?; + if i < self.aggregate_cols.len() - 1 { + write!(f, ",")? } } } @@ -2225,7 +2348,7 @@ west,4 } #[test] - fn read_group() { + fn read_aggregate() { let mut columns = BTreeMap::new(); let tc = ColumnType::Time(Column::from(&[1_i64, 2, 3, 4, 5, 6][..])); columns.insert("time".to_string(), tc); @@ -2269,20 +2392,20 @@ west,4 // test queries with no predicates and grouping on low cardinality // columns - read_group_all_rows_all_rle(&row_group); + read_aggregate_all_rows_all_rle(&row_group); // test row group queries that group on fewer than five columns. - read_group_hash_u128_key(&row_group); + read_aggregate_hash_u128_key(&row_group); // test row group queries that use a vector-based group key. - read_group_hash_vec_key(&row_group); + read_aggregate_hash_vec_key(&row_group); // test row group queries that only group on one column. - read_group_single_groupby_column(&row_group); + read_aggregate_single_groupby_column(&row_group); } // the read_group path where grouping is on fewer than five columns. - fn read_group_hash_u128_key(row_group: &RowGroup) { + fn read_aggregate_hash_u128_key(row_group: &RowGroup) { let cases = vec![ ( Predicate::with_time_range(&[], 0, 7), // all time but without explicit pred @@ -2354,7 +2477,7 @@ west,prod,POST,4 // the read_group path where grouping is on five or more columns. This will // ensure that the `read_group_hash_with_vec_key` path is exercised. - fn read_group_hash_vec_key(row_group: &RowGroup) { + fn read_aggregate_hash_vec_key(row_group: &RowGroup) { let cases = vec![( Predicate::with_time_range(&[], 0, 7), // all time but with explicit pred vec!["region", "method", "env", "letters", "numbers"], @@ -2377,7 +2500,7 @@ west,POST,prod,Bravo,two,203 } // the read_group path where grouping is on a single column. - fn read_group_single_groupby_column(row_group: &RowGroup) { + fn read_aggregate_single_groupby_column(row_group: &RowGroup) { let cases = vec![( Predicate::with_time_range(&[], 0, 7), // all time but with explicit pred vec!["method"], @@ -2396,7 +2519,7 @@ PUT,203 } } - fn read_group_all_rows_all_rle(row_group: &RowGroup) { + fn read_aggregate_all_rows_all_rle(row_group: &RowGroup) { let cases = vec![ ( Predicate::default(), @@ -2457,7 +2580,7 @@ west,POST,304,101,203 } #[test] - fn row_group_could_satisfy_predicate() { + fn row_aggregate_could_satisfy_predicate() { let mut columns = BTreeMap::new(); let tc = ColumnType::Time(Column::from(&[1_i64, 2, 3, 4, 5, 6][..])); columns.insert("time".to_string(), tc); @@ -2513,7 +2636,7 @@ west,POST,304,101,203 } #[test] - fn row_group_satisfies_predicate() { + fn row_aggregate_satisfies_predicate() { let mut columns = BTreeMap::new(); let tc = ColumnType::Time(Column::from(&[1_i64, 2, 3, 4, 5, 6][..])); columns.insert("time".to_string(), tc); @@ -2602,7 +2725,7 @@ west,POST,304,101,203 } #[test] - fn read_group_result_display() { + fn read_aggregate_result_display() { let mut result = ReadAggregateResult { schema: ResultSchema { select_columns: vec![], @@ -2629,34 +2752,25 @@ west,POST,304,101,203 ), ], }, - group_keys: vec![ - GroupKey(vec![Value::String("east"), Value::String("host-a")]), - GroupKey(vec![Value::String("east"), Value::String("host-b")]), - GroupKey(vec![Value::String("west"), Value::String("host-a")]), - GroupKey(vec![Value::String("west"), Value::String("host-c")]), - GroupKey(vec![Value::String("west"), Value::String("host-d")]), + group_key_cols: vec![ + vec![ + Some("east"), + Some("east"), + Some("west"), + Some("west"), + Some("west"), + ], + vec![ + Some("host-a"), + Some("host-b"), + Some("host-a"), + Some("host-c"), + Some("host-d"), + ], ], - aggregates: vec![ - AggregateResults(vec![ - AggregateResult::Sum(Scalar::I64(10)), - AggregateResult::Count(3), - ]), - AggregateResults(vec![ - AggregateResult::Sum(Scalar::I64(20)), - AggregateResult::Count(4), - ]), - AggregateResults(vec![ - AggregateResult::Sum(Scalar::I64(25)), - AggregateResult::Count(3), - ]), - AggregateResults(vec![ - AggregateResult::Sum(Scalar::I64(21)), - AggregateResult::Count(1), - ]), - AggregateResults(vec![ - AggregateResult::Sum(Scalar::I64(11)), - AggregateResult::Count(9), - ]), + aggregate_cols: vec![ + AggregateVec::SumI64(vec![Some(10), Some(20), Some(25), Some(21), Some(11)]), + AggregateVec::Count(vec![Some(3), Some(4), Some(3), Some(1), Some(9)]), ], group_keys_sorted: false, }; @@ -2686,7 +2800,7 @@ west,host-d,11,9 // results don't have to have group keys. result.schema.group_columns = vec![]; - result.group_keys = vec![]; + result.group_key_cols = vec![]; // Debug implementation assert_eq!( @@ -2713,7 +2827,71 @@ west,host-d,11,9 } #[test] - fn read_group_result_merge() { + fn read_aggregate_result_sort() { + let mut result = ReadAggregateResult { + schema: ResultSchema::default(), // schema not needed for sorting. + group_key_cols: vec![ + vec![ + Some("east"), + Some("west"), + Some("west"), + Some("east"), + Some("west"), + ], + vec![ + Some("host-a"), + Some("host-c"), + Some("host-a"), + Some("host-d"), + Some("host-b"), + ], + ], + aggregate_cols: vec![ + AggregateVec::SumI64(vec![Some(10), Some(20), Some(25), Some(21), Some(11)]), + AggregateVec::Count(vec![Some(3), Some(4), Some(3), Some(1), Some(9)]), + ], + group_keys_sorted: false, + }; + + result.sort(); + + // Debug implementation + assert_eq!( + format!("{}", &result), + "east,host-a,10,3 +east,host-d,21,1 +west,host-a,25,3 +west,host-b,11,9 +west,host-c,20,4 +" + ); + + let mut result = ReadAggregateResult { + schema: ResultSchema::default(), + group_key_cols: vec![ + vec![Some("west"), Some("east"), Some("north")], + vec![Some("host-c"), Some("host-c"), Some("host-c")], + vec![Some("pro"), Some("stag"), Some("dev")], + ], + aggregate_cols: vec![ + AggregateVec::SumI64(vec![Some(10), Some(20), Some(-5)]), + AggregateVec::Count(vec![Some(6), Some(8), Some(2)]), + ], + ..Default::default() + }; + result.sort(); + + assert_eq!( + format!("{}", &result), + "east,host-c,stag,20,8 +north,host-c,dev,-5,2 +west,host-c,pro,10,6 +" + ); + } + + #[test] + fn read_aggregate_result_merge() { let schema = ResultSchema { group_columns: vec![ ( @@ -2745,24 +2923,18 @@ west,host-d,11,9 ..Default::default() }; - let mut other_result = ReadAggregateResult { + let other_result = ReadAggregateResult { schema: schema.clone(), + group_key_cols: vec![ + vec![Some("east"), Some("east")], + vec![Some("host-a"), Some("host-b")], + ], + aggregate_cols: vec![ + AggregateVec::SumI64(vec![Some(10), Some(20)]), + AggregateVec::Count(vec![Some(3), Some(4)]), + ], ..Default::default() }; - other_result.add_row( - vec![Value::String("east"), Value::String("host-a")], - vec![ - AggregateResult::Sum(Scalar::I64(10)), - AggregateResult::Count(3), - ], - ); - other_result.add_row( - vec![Value::String("east"), Value::String("host-b")], - vec![ - AggregateResult::Sum(Scalar::I64(20)), - AggregateResult::Count(4), - ], - ); // merging something into nothing results in having a copy of something. result = result.merge(other_result.clone()); @@ -2775,19 +2947,13 @@ west,host-d,11,9 result, ReadAggregateResult { schema: schema.clone(), - group_keys: vec![ - GroupKey(vec![Value::String("east"), Value::String("host-a")]), - GroupKey(vec![Value::String("east"), Value::String("host-b")]), + group_key_cols: vec![ + vec![Some("east"), Some("east")], + vec![Some("host-a"), Some("host-b")], ], - aggregates: vec![ - AggregateResults(vec![ - AggregateResult::Sum(Scalar::I64(20)), - AggregateResult::Count(6), - ]), - AggregateResults(vec![ - AggregateResult::Sum(Scalar::I64(40)), - AggregateResult::Count(8), - ]), + aggregate_cols: vec![ + AggregateVec::SumI64(vec![Some(20), Some(40)]), + AggregateVec::Count(vec![Some(6), Some(8)]), ], ..Default::default() } @@ -2795,41 +2961,28 @@ west,host-d,11,9 // merging a result in with different group keys merges those group // keys in. - let mut other_result = ReadAggregateResult { + let other_result = ReadAggregateResult { schema: schema.clone(), + group_key_cols: vec![vec![Some("north")], vec![Some("host-a")]], + aggregate_cols: vec![ + AggregateVec::SumI64(vec![Some(-5)]), + AggregateVec::Count(vec![Some(2)]), + ], ..Default::default() }; - other_result.add_row( - vec![Value::String("north"), Value::String("host-a")], - vec![ - AggregateResult::Sum(Scalar::I64(-5)), - AggregateResult::Count(2), - ], - ); result = result.merge(other_result.clone()); assert_eq!( result, ReadAggregateResult { schema: schema.clone(), - group_keys: vec![ - GroupKey(vec![Value::String("east"), Value::String("host-a")]), - GroupKey(vec![Value::String("east"), Value::String("host-b")]), - GroupKey(vec![Value::String("north"), Value::String("host-a")]), + group_key_cols: vec![ + vec![Some("east"), Some("east"), Some("north")], + vec![Some("host-a"), Some("host-b"), Some("host-a")], ], - aggregates: vec![ - AggregateResults(vec![ - AggregateResult::Sum(Scalar::I64(20)), - AggregateResult::Count(6), - ]), - AggregateResults(vec![ - AggregateResult::Sum(Scalar::I64(40)), - AggregateResult::Count(8), - ]), - AggregateResults(vec![ - AggregateResult::Sum(Scalar::I64(-5)), - AggregateResult::Count(2), - ]), + aggregate_cols: vec![ + AggregateVec::SumI64(vec![Some(20), Some(40), Some(-5)]), + AggregateVec::Count(vec![Some(6), Some(8), Some(2)]), ], ..Default::default() } @@ -2846,24 +2999,13 @@ west,host-d,11,9 result, ReadAggregateResult { schema, - group_keys: vec![ - GroupKey(vec![Value::String("east"), Value::String("host-a")]), - GroupKey(vec![Value::String("east"), Value::String("host-b")]), - GroupKey(vec![Value::String("north"), Value::String("host-a")]), + group_key_cols: vec![ + vec![Some("east"), Some("east"), Some("north")], + vec![Some("host-a"), Some("host-b"), Some("host-a")], ], - aggregates: vec![ - AggregateResults(vec![ - AggregateResult::Sum(Scalar::I64(20)), - AggregateResult::Count(6), - ]), - AggregateResults(vec![ - AggregateResult::Sum(Scalar::I64(40)), - AggregateResult::Count(8), - ]), - AggregateResults(vec![ - AggregateResult::Sum(Scalar::I64(-5)), - AggregateResult::Count(2), - ]), + aggregate_cols: vec![ + AggregateVec::SumI64(vec![Some(20), Some(40), Some(-5)]), + AggregateVec::Count(vec![Some(6), Some(8), Some(2)]), ], ..Default::default() } diff --git a/read_buffer/src/schema.rs b/read_buffer/src/schema.rs index 895a1b6ee7..a9964356b1 100644 --- a/read_buffer/src/schema.rs +++ b/read_buffer/src/schema.rs @@ -1,7 +1,7 @@ use std::{convert::TryFrom, fmt::Display}; use arrow_deps::arrow; -use data_types::schema::InfluxFieldType; +use internal_types::schema::InfluxFieldType; /// A schema that is used to track the names and semantics of columns returned /// in results out of various operations on a row group. @@ -50,6 +50,7 @@ impl ResultSchema { self.len() == 0 } + /// The total number of columns the schema represents. pub fn len(&self) -> usize { self.select_columns.len() + self.group_columns.len() + self.aggregate_columns.len() } @@ -96,11 +97,11 @@ impl Display for ResultSchema { } } -impl TryFrom<&ResultSchema> for data_types::schema::Schema { - type Error = data_types::schema::builder::Error; +impl TryFrom<&ResultSchema> for internal_types::schema::Schema { + type Error = internal_types::schema::builder::Error; fn try_from(rs: &ResultSchema) -> Result { - let mut builder = data_types::schema::builder::SchemaBuilder::new(); + let mut builder = internal_types::schema::builder::SchemaBuilder::new(); for (col_type, data_type) in &rs.select_columns { match col_type { ColumnType::Tag(name) => builder = builder.tag(name.as_str()), diff --git a/read_buffer/src/table.rs b/read_buffer/src/table.rs index e80fae23fe..7e68a13857 100644 --- a/read_buffer/src/table.rs +++ b/read_buffer/src/table.rs @@ -7,12 +7,12 @@ use std::{ }; use arrow_deps::arrow::record_batch::RecordBatch; -use data_types::selection::Selection; +use internal_types::selection::Selection; use snafu::{ensure, Snafu}; -use crate::row_group::{self, ColumnName, GroupKey, Predicate, RowGroup}; +use crate::row_group::{self, ColumnName, Predicate, RowGroup}; use crate::schema::{AggregateType, ColumnType, LogicalDataType, ResultSchema}; -use crate::value::{AggregateResult, Scalar, Value}; +use crate::value::Value; #[derive(Debug, Snafu)] pub enum Error { #[snafu(display("cannot drop last row group in table; drop table"))] @@ -94,6 +94,8 @@ impl Table { row_groups.data.push(Arc::new(rg)); } + /// TODO(edd): wire up + /// /// Remove the row group at `position` from table, returning an error if the /// caller has attempted to drop the last row group. /// @@ -226,7 +228,7 @@ impl Table { predicate: Predicate, group_columns: &'input Selection<'_>, aggregates: &'input [(ColumnName<'input>, AggregateType)], - ) -> ReadAggregateResults { + ) -> Result { let (meta, row_groups) = self.filter_row_groups(&predicate); // Filter out any column names that we do not have data for. @@ -239,13 +241,24 @@ impl Table { ..ResultSchema::default() }; + // Check all grouping columns are valid for grouping operation. + for (ct, _) in &schema.group_columns { + ensure!( + matches!(ct, ColumnType::Tag(_)), + UnsupportedColumnOperation { + msg: format!("column type must be ColumnType::Tag, got {:?}", ct), + column_name: ct.as_str().to_string(), + }, + ) + } + // return the iterator to build the results. - ReadAggregateResults { + Ok(ReadAggregateResults { schema, predicate, row_groups, ..Default::default() - } + }) } /// Returns aggregates segmented by grouping keys and windowed by time. @@ -273,76 +286,15 @@ impl Table { _group_columns: Vec>, _aggregates: Vec<(ColumnName<'a>, AggregateType)>, _window: i64, - ) -> BTreeMap, Vec<(ColumnName<'a>, AggregateResult<'_>)>> { + ) -> BTreeMap, Vec<(ColumnName<'a>, ReadAggregateResults)>> { // identify segments where time range and predicates match could match // using segment meta data, and then execute against those segments and // merge results. todo!() } - // Perform aggregates without any grouping. Filtering on optional predicates - // and time range is still supported. - fn read_aggregate_no_group<'a>( - &self, - time_range: (i64, i64), - predicates: &[(&str, &str)], - aggregates: Vec<(ColumnName<'a>, AggregateType)>, - ) -> Vec<(ColumnName<'a>, AggregateResult<'_>)> { - // The fast path where there are no predicates or a time range to apply. - // We just want the equivalent of column statistics. - if predicates.is_empty() { - let mut results = Vec::with_capacity(aggregates.len()); - for (col_name, agg_type) in &aggregates { - match agg_type { - AggregateType::Count => { - results.push(( - col_name, - AggregateResult::Count(self.count(col_name, time_range)), - )); - } - AggregateType::First => { - results.push(( - col_name, - AggregateResult::First(self.first(col_name, time_range.0)), - )); - } - AggregateType::Last => { - results.push(( - col_name, - AggregateResult::Last(self.last(col_name, time_range.1)), - )); - } - AggregateType::Min => { - results.push(( - col_name, - AggregateResult::Min(self.min(col_name, time_range)), - )); - } - AggregateType::Max => { - results.push(( - col_name, - AggregateResult::Max(self.max(col_name, time_range)), - )); - } - AggregateType::Sum => { - let res = match self.sum(col_name, time_range) { - Some(x) => x, - None => Scalar::Null, - }; - - results.push((col_name, AggregateResult::Sum(res))); - } - } - } - } - - // Otherwise we have predicates so for each segment we will execute a - // generalised aggregation method and build up the result set. - todo!(); - } - // - // ---- Fast-path aggregations on single columns. + // ---- Fast-path first/last selectors. // // Returns the first value for the specified column across the table @@ -387,44 +339,6 @@ impl Table { todo!(); } - /// The minimum non-null value in the column for the table. - fn min(&self, _column_name: &str, _time_range: (i64, i64)) -> Value<'_> { - // Loop over segments, skipping any that don't satisfy the time range. - // Any segments completely overlapped can have a candidate min taken - // directly from their zone map. Partially overlapped segments will be - // read using the appropriate execution API. - // - // Return the min of minimums. - todo!(); - } - - /// The maximum non-null value in the column for the table. - fn max(&self, _column_name: &str, _time_range: (i64, i64)) -> Value<'_> { - // Loop over segments, skipping any that don't satisfy the time range. - // Any segments completely overlapped can have a candidate max taken - // directly from their zone map. Partially overlapped segments will be - // read using the appropriate execution API. - // - // Return the max of maximums. - todo!(); - } - - /// The number of non-null values in the column for the table. - fn count(&self, _column_name: &str, _time_range: (i64, i64)) -> u64 { - // Loop over segments, skipping any that don't satisfy the time range. - // Execute appropriate aggregation call on each segment and aggregate - // the results. - todo!(); - } - - /// The total sum of non-null values in the column for the table. - fn sum(&self, _column_name: &str, _time_range: (i64, i64)) -> Option { - // Loop over segments, skipping any that don't satisfy the time range. - // Execute appropriate aggregation call on each segment and aggregate - // the results. - todo!(); - } - // // ---- Schema API queries // @@ -500,36 +414,6 @@ impl Table { Ok(dst) } - /// Determines if this table could satisfy the provided predicate. - /// - /// `false` is proof that no row within this table would match the - /// predicate, whilst `true` indicates one or more rows *might* match the - /// predicate. - fn could_satisfy_predicate(&self, predicate: &Predicate) -> bool { - // Get a snapshot of the table data under a read lock. - let (meta, row_groups) = { - let table_data = self.table_data.read().unwrap(); - (Arc::clone(&table_data.meta), table_data.data.to_vec()) - }; - - // if the table doesn't have a column for one of the predicate's - // expressions then the table cannot satisfy the predicate. - if !predicate - .iter() - .all(|expr| meta.columns.contains_key(expr.column())) - { - return false; - } - - // If there is a single row group in the table that could satisfy the - // predicate then the table itself could satisfy the predicate so return - // true. If none of the row groups could match then return false. - let exprs = predicate.expressions(); - row_groups - .iter() - .any(|row_group| row_group.could_satisfy_conjunctive_binary_expressions(exprs)) - } - /// Determines if this table contains one or more rows that satisfy the /// predicate. pub fn satisfies_predicate(&self, predicate: &Predicate) -> bool { @@ -942,7 +826,7 @@ mod test { use crate::row_group::{BinaryExpr, ColumnType, ReadAggregateResult}; use crate::schema; use crate::schema::LogicalDataType; - use crate::value::{OwnedValue, Scalar}; + use crate::value::{AggregateVec, OwnedValue, Scalar}; #[test] fn meta_data_update_with() { @@ -1191,11 +1075,13 @@ mod test { table.add_row_group(rg); // no predicate aggregate - let mut results = table.read_aggregate( - Predicate::default(), - &Selection::Some(&[]), - &[("time", AggregateType::Count), ("time", AggregateType::Sum)], - ); + let mut results = table + .read_aggregate( + Predicate::default(), + &Selection::Some(&[]), + &[("time", AggregateType::Count), ("time", AggregateType::Sum)], + ) + .unwrap(); // check the column result schema let exp_schema = ResultSchema { @@ -1222,22 +1108,36 @@ mod test { assert!(matches!(results.next_merged_result(), None)); // apply a predicate - let mut results = table.read_aggregate( - Predicate::new(vec![BinaryExpr::from(("region", "=", "west"))]), - &Selection::Some(&[]), - &[("time", AggregateType::Count), ("time", AggregateType::Sum)], - ); + let mut results = table + .read_aggregate( + Predicate::new(vec![BinaryExpr::from(("region", "=", "west"))]), + &Selection::Some(&[]), + &[("time", AggregateType::Count), ("time", AggregateType::Sum)], + ) + .unwrap(); assert_eq!( DisplayReadAggregateResults(vec![results.next_merged_result().unwrap()]).to_string(), "time_count,time_sum\n2,300\n", ); assert!(matches!(results.next_merged_result(), None)); + + // group on wrong columns. + let results = table.read_aggregate( + Predicate::new(vec![BinaryExpr::from(("region", "=", "west"))]), + &Selection::Some(&["time"]), + &[("min", AggregateType::Min)], + ); + + assert!(matches!( + &results, + Err(Error::UnsupportedColumnOperation { .. }) + ),); } #[test] fn read_aggregate_result_display() { - let mut result_a = ReadAggregateResult { + let result_a = ReadAggregateResult { schema: ResultSchema { select_columns: vec![], group_columns: vec![ @@ -1256,14 +1156,12 @@ mod test { LogicalDataType::Integer, )], }, + group_key_cols: vec![vec![Some("east")], vec![Some("host-a")]], + aggregate_cols: vec![AggregateVec::SumI64(vec![Some(10)])], ..ReadAggregateResult::default() }; - result_a.add_row( - vec![Value::String("east"), Value::String("host-a")], - vec![AggregateResult::Sum(Scalar::I64(10))], - ); - let mut result_b = ReadAggregateResult { + let result_b = ReadAggregateResult { schema: ResultSchema { select_columns: vec![], group_columns: vec![ @@ -1282,12 +1180,10 @@ mod test { LogicalDataType::Integer, )], }, + group_key_cols: vec![vec![Some("west")], vec![Some("host-b")]], + aggregate_cols: vec![AggregateVec::SumI64(vec![Some(100)])], ..Default::default() }; - result_b.add_row( - vec![Value::String("west"), Value::String("host-b")], - vec![AggregateResult::Sum(Scalar::I64(100))], - ); let results = DisplayReadAggregateResults(vec![result_a, result_b]); //Display implementation assert_eq!( diff --git a/read_buffer/src/value.rs b/read_buffer/src/value.rs index e30eebe9fe..c7adf11952 100644 --- a/read_buffer/src/value.rs +++ b/read_buffer/src/value.rs @@ -1,330 +1,946 @@ -use std::{collections::BTreeSet, convert::TryFrom}; +use std::{convert::TryFrom, fmt::Formatter}; use std::{mem::size_of, sync::Arc}; use arrow_deps::arrow; -use crate::AggregateType; +use crate::{AggregateType, LogicalDataType}; -/// These variants hold aggregates, which are the results of applying aggregates -/// to column data. -#[derive(Debug, Copy, Clone, PartialEq)] -pub enum AggregateResult<'a> { - // Any type of column can have rows counted. NULL values do not contribute - // to the count. If all rows are NULL then count will be `0`. - Count(u64), +#[derive(Clone, PartialEq, Debug)] +/// A type that holds aggregates where each variant encodes the underlying data +/// type and aggregate type for a vector of data. An `AggregateVec` can be +/// updated on a value-by-value basis and new values can be appended. +/// +/// The type is structured this way to improve the performance of aggregations +/// in the read buffer by reducing the number of matches (branches) needed per +/// row. +pub enum AggregateVec { + Count(Vec>), - // Only numerical columns with scalar values can be summed. NULL values do - // not contribute to the sum, but if all rows are NULL then the sum is - // itself NULL (represented by `None`). - Sum(Scalar), + SumI64(Vec>), + SumU64(Vec>), + SumF64(Vec>), - // The minimum value in the column data. - Min(Value<'a>), + MinU64(Vec>), + MinI64(Vec>), + MinF64(Vec>), + MinString(Vec>), + MinBytes(Vec>>), + MinBool(Vec>), - // The maximum value in the column data. - Max(Value<'a>), + MaxU64(Vec>), + MaxI64(Vec>), + MaxF64(Vec>), + MaxString(Vec>), + MaxBytes(Vec>>), + MaxBool(Vec>), - // The first value in the column data and the corresponding timestamp. - First(Option<(i64, Value<'a>)>), + FirstU64((Vec>, Vec>)), + FirstI64((Vec>, Vec>)), + FirstF64((Vec>, Vec>)), + FirstString((Vec>, Vec>)), + FirstBytes((Vec>>, Vec>)), + FirstBool((Vec>, Vec>)), - // The last value in the column data and the corresponding timestamp. - Last(Option<(i64, Value<'a>)>), + LastU64((Vec>, Vec>)), + LastI64((Vec>, Vec>)), + LastF64((Vec>, Vec>)), + LastString((Vec>, Vec>)), + LastBytes((Vec>>, Vec>)), + LastBool((Vec>, Vec>)), } -#[allow(unused_assignments)] -impl<'a> AggregateResult<'a> { - pub fn update(&mut self, other: Value<'a>) { - if other.is_null() { - // a NULL value has no effect on aggregates +impl AggregateVec { + pub fn len(&self) -> usize { + match self { + Self::Count(arr) => arr.len(), + Self::SumI64(arr) => arr.len(), + Self::SumU64(arr) => arr.len(), + Self::SumF64(arr) => arr.len(), + Self::MinU64(arr) => arr.len(), + Self::MinI64(arr) => arr.len(), + Self::MinF64(arr) => arr.len(), + Self::MinString(arr) => arr.len(), + Self::MinBytes(arr) => arr.len(), + Self::MinBool(arr) => arr.len(), + Self::MaxU64(arr) => arr.len(), + Self::MaxI64(arr) => arr.len(), + Self::MaxF64(arr) => arr.len(), + Self::MaxString(arr) => arr.len(), + Self::MaxBytes(arr) => arr.len(), + Self::MaxBool(arr) => arr.len(), + Self::FirstU64((arr, _)) => arr.len(), + Self::FirstI64((arr, _)) => arr.len(), + Self::FirstF64((arr, _)) => arr.len(), + Self::FirstString((arr, _)) => arr.len(), + Self::FirstBytes((arr, _)) => arr.len(), + Self::FirstBool((arr, _)) => arr.len(), + Self::LastU64((arr, _)) => arr.len(), + Self::LastI64((arr, _)) => arr.len(), + Self::LastF64((arr, _)) => arr.len(), + Self::LastString((arr, _)) => arr.len(), + Self::LastBytes((arr, _)) => arr.len(), + Self::LastBool((arr, _)) => arr.len(), + } + } + + /// Returns the value specified by `offset`. + pub fn value(&self, offset: usize) -> Value<'_> { + match &self { + Self::Count(arr) => Value::from(arr[offset]), + Self::SumI64(arr) => Value::from(arr[offset]), + Self::SumU64(arr) => Value::from(arr[offset]), + Self::SumF64(arr) => Value::from(arr[offset]), + Self::MinU64(arr) => Value::from(arr[offset]), + Self::MinI64(arr) => Value::from(arr[offset]), + Self::MinF64(arr) => Value::from(arr[offset]), + Self::MinString(arr) => Value::from(arr[offset].as_deref()), + Self::MinBytes(arr) => Value::from(arr[offset].as_deref()), + Self::MinBool(arr) => Value::from(arr[offset]), + Self::MaxU64(arr) => Value::from(arr[offset]), + Self::MaxI64(arr) => Value::from(arr[offset]), + Self::MaxF64(arr) => Value::from(arr[offset]), + Self::MaxString(arr) => Value::from(arr[offset].as_deref()), + Self::MaxBytes(arr) => Value::from(arr[offset].as_deref()), + Self::MaxBool(arr) => Value::from(arr[offset]), + _ => unimplemented!("first/last not yet implemented"), + } + } + + /// Updates with a new value located in the provided input column help in + /// `Values`. + /// + /// Panics if the type of `Value` does not satisfy the aggregate type. + pub fn update(&mut self, values: &Values<'_>, row_id: usize, offset: usize) { + if values.is_null(row_id) { return; } match self { - Self::Count(v) => { - if !other.is_null() { - *v += 1; + Self::Count(arr) => { + if offset >= arr.len() { + arr.resize(offset + 1, None); + } + + *arr[offset].get_or_insert(0) += 1; + } + Self::SumI64(arr) => { + if offset >= arr.len() { + arr.resize(offset + 1, None); + } + + match &mut arr[offset] { + Some(v) => *v += values.value_i64(row_id), + None => arr[offset] = Some(values.value_i64(row_id)), } } - Self::Min(v) => match (&v, &other) { - (Value::Null, _) => { - // something is always smaller than NULL - *v = other; + Self::SumU64(arr) => { + if offset >= arr.len() { + arr.resize(offset + 1, None); } - (Value::String(_), Value::Null) => {} // do nothing - (Value::String(a), Value::String(b)) => { - if a.cmp(b) == std::cmp::Ordering::Greater { - *v = other; - } - } - (Value::String(a), Value::ByteArray(b)) => { - if a.as_bytes().cmp(b) == std::cmp::Ordering::Greater { - *v = other; - } - } - (Value::ByteArray(_), Value::Null) => {} // do nothing - (Value::ByteArray(a), Value::String(b)) => { - if a.cmp(&b.as_bytes()) == std::cmp::Ordering::Greater { - *v = other; - } - } - (Value::ByteArray(a), Value::ByteArray(b)) => { - if a.cmp(b) == std::cmp::Ordering::Greater { - *v = other; - } - } - (Value::Scalar(_), Value::Null) => {} // do nothing - (Value::Scalar(a), Value::Scalar(b)) => { - if a > b { - *v = other; - } - } - (_, _) => unreachable!("not a possible variant combination"), - }, - Self::Max(v) => match (&v, &other) { - (Value::Null, _) => { - // something is always larger than NULL - *v = other; - } - (Value::String(_), Value::Null) => {} // do nothing - (Value::String(a), Value::String(b)) => { - if a.cmp(b) == std::cmp::Ordering::Less { - *v = other; - } - } - (Value::String(a), Value::ByteArray(b)) => { - if a.as_bytes().cmp(b) == std::cmp::Ordering::Less { - *v = other; - } - } - (Value::ByteArray(_), Value::Null) => {} // do nothing - (Value::ByteArray(a), Value::String(b)) => { - if a.cmp(&b.as_bytes()) == std::cmp::Ordering::Less { - *v = other; - } - } - (Value::ByteArray(a), Value::ByteArray(b)) => { - if a.cmp(b) == std::cmp::Ordering::Less { - *v = other; - } - } - (Value::Scalar(_), Value::Null) => {} // do nothing - (Value::Scalar(a), Value::Scalar(b)) => { - if a < b { - *v = other; - } - } - (_, _) => unreachable!("not a possible variant combination"), - }, - Self::Sum(v) => match (&v, &other) { - (Scalar::Null, Value::Scalar(other_scalar)) => { - // NULL + something == something - *v = *other_scalar; - } - (_, Value::Scalar(b)) => *v += b, - (_, _) => unreachable!("not a possible variant combination"), - }, - _ => unimplemented!("First and Last aggregates not implemented yet"), - } - } - /// Merge `other` into `self` - pub fn merge(&mut self, other: &AggregateResult<'a>) { - match (self, other) { - (AggregateResult::Count(this), AggregateResult::Count(that)) => *this += *that, - (AggregateResult::Sum(this), AggregateResult::Sum(that)) => *this += that, - (AggregateResult::Min(this), AggregateResult::Min(that)) => { - if *this > *that { - *this = *that; + match &mut arr[offset] { + Some(v) => *v += values.value_u64(row_id), + None => arr[offset] = Some(values.value_u64(row_id)), } } - (AggregateResult::Max(this), AggregateResult::Max(that)) => { - if *this < *that { - *this = *that; + Self::SumF64(arr) => { + if offset >= arr.len() { + arr.resize(offset + 1, None); + } + + match &mut arr[offset] { + Some(v) => *v += values.value_f64(row_id), + None => arr[offset] = Some(values.value_f64(row_id)), } } - (a, b) => unimplemented!("merging {:?} into {:?} not yet implemented", b, a), + Self::MinU64(arr) => { + if offset >= arr.len() { + arr.resize(offset + 1, None); + } + + match &mut arr[offset] { + Some(v) => *v = (*v).min(values.value_u64(row_id)), + None => arr[offset] = Some(values.value_u64(row_id)), + } + } + Self::MinI64(arr) => { + if offset >= arr.len() { + arr.resize(offset + 1, None); + } + + match &mut arr[offset] { + Some(v) => *v = (*v).min(values.value_i64(row_id)), + None => arr[offset] = Some(values.value_i64(row_id)), + } + } + Self::MinF64(arr) => { + if offset >= arr.len() { + arr.resize(offset + 1, None); + } + + match &mut arr[offset] { + Some(v) => *v = (*v).min(values.value_f64(row_id)), + None => arr[offset] = Some(values.value_f64(row_id)), + } + } + Self::MinString(arr) => { + if offset >= arr.len() { + arr.resize(offset + 1, None); + } + + match &mut arr[offset] { + Some(v) => { + let other = values.value_str(row_id); + if other < v.as_str() { + *v = other.to_owned(); + } + } + None => arr[offset] = Some(values.value_str(row_id).to_owned()), + } + } + Self::MinBytes(arr) => { + if offset >= arr.len() { + arr.resize(offset + 1, None); + } + + match &mut arr[offset] { + Some(v) => { + let other = values.value_bytes(row_id); + if other < v.as_slice() { + *v = other.to_owned(); + } + } + None => arr[offset] = Some(values.value_bytes(row_id).to_owned()), + } + } + Self::MinBool(arr) => { + if offset >= arr.len() { + arr.resize(offset + 1, None); + } + + match &mut arr[offset] { + Some(v) => *v = (*v).min(values.value_bool(row_id)), + None => arr[offset] = Some(values.value_bool(row_id)), + } + } + Self::MaxU64(arr) => { + if offset >= arr.len() { + arr.resize(offset + 1, None); + } + + match &mut arr[offset] { + Some(v) => *v = (*v).max(values.value_u64(row_id)), + None => arr[offset] = Some(values.value_u64(row_id)), + } + } + Self::MaxI64(arr) => { + if offset >= arr.len() { + arr.resize(offset + 1, None); + } + + match &mut arr[offset] { + Some(v) => *v = (*v).max(values.value_i64(row_id)), + None => arr[offset] = Some(values.value_i64(row_id)), + } + } + Self::MaxF64(arr) => { + if offset >= arr.len() { + arr.resize(offset + 1, None); + } + + match &mut arr[offset] { + Some(v) => *v = (*v).max(values.value_f64(row_id)), + None => arr[offset] = Some(values.value_f64(row_id)), + } + } + Self::MaxString(arr) => { + if offset >= arr.len() { + arr.resize(offset + 1, None); + } + + match &mut arr[offset] { + Some(v) => { + let other = values.value_str(row_id); + if other > v.as_str() { + *v = other.to_owned(); + } + } + None => arr[offset] = Some(values.value_str(row_id).to_owned()), + } + } + Self::MaxBytes(arr) => { + if offset >= arr.len() { + arr.resize(offset + 1, None); + } + + match &mut arr[offset] { + Some(v) => { + let other = values.value_bytes(row_id); + if other > v.as_slice() { + *v = other.to_owned(); + } + } + None => arr[offset] = Some(values.value_bytes(row_id).to_owned()), + } + } + Self::MaxBool(arr) => { + if offset >= arr.len() { + arr.resize(offset + 1, None); + } + + match &mut arr[offset] { + Some(v) => *v = (*v).max(values.value_bool(row_id)), + None => arr[offset] = Some(values.value_bool(row_id)), + } + } + // TODO - implement first/last + _ => unimplemented!("aggregate update not implemented"), } } - pub fn try_as_str(&self) -> Option<&str> { - match &self { - AggregateResult::Min(v) => match v { - Value::Null => None, - Value::String(s) => Some(s), - v => panic!("cannot convert {:?} to &str", v), - }, - AggregateResult::Max(v) => match v { - Value::Null => None, - Value::String(s) => Some(s), - v => panic!("cannot convert {:?} to &str", v), - }, - AggregateResult::First(_) => panic!("cannot convert first tuple to &str"), - AggregateResult::Last(_) => panic!("cannot convert last tuple to &str"), - AggregateResult::Sum(v) => panic!("cannot convert {:?} to &str", v), - AggregateResult::Count(_) => panic!("cannot convert count to &str"), + /// Appends the provided value to the end of the aggregate vector. + /// Panics if the type of `Value` does not satisfy the aggregate type. + /// + /// Note: updating pushed first/last variants is not currently a supported + /// operation. + pub fn push(&mut self, value: Value<'_>) { + match self { + Self::Count(arr) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.u64())); + } + } + Self::SumI64(arr) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.i64())); + } + } + Self::SumU64(arr) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.u64())); + } + } + Self::SumF64(arr) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.f64())); + } + } + Self::MinU64(arr) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.u64())); + } + } + Self::MinI64(arr) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.i64())); + } + } + Self::MinF64(arr) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.f64())); + } + } + Self::MinString(arr) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.str().to_owned())); + } + } + Self::MinBytes(arr) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.bytes().to_owned())); + } + } + Self::MinBool(arr) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.bool())); + } + } + Self::MaxU64(arr) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.u64())); + } + } + Self::MaxI64(arr) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.i64())); + } + } + Self::MaxF64(arr) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.f64())); + } + } + Self::MaxString(arr) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.str().to_owned())); + } + } + Self::MaxBytes(arr) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.bytes().to_owned())); + } + } + Self::MaxBool(arr) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.bool())); + } + } + Self::FirstU64((arr, _)) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.u64())); + } + } + Self::FirstI64((arr, _)) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.i64())); + } + } + Self::FirstF64((arr, _)) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.f64())); + } + } + Self::FirstString((arr, _)) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.str().to_owned())); + } + } + Self::FirstBytes((arr, _)) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.bytes().to_owned())); + } + } + Self::FirstBool((arr, _)) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.bool())); + } + } + Self::LastU64((arr, _)) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.u64())); + } + } + Self::LastI64((arr, _)) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.i64())); + } + } + Self::LastF64((arr, _)) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.f64())); + } + } + Self::LastString((arr, _)) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.str().to_owned())); + } + } + Self::LastBytes((arr, _)) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.bytes().to_owned())); + } + } + Self::LastBool((arr, _)) => { + if value.is_null() { + arr.push(None); + } else { + arr.push(Some(value.bool())); + } + } } } - pub fn try_as_bytes(&self) -> Option<&[u8]> { - match &self { - AggregateResult::Min(v) => match v { - Value::Null => None, - Value::ByteArray(s) => Some(s), - v => panic!("cannot convert {:?} to &[u8]", v), + /// Writes a textual representation of the value specified by `offset` to + /// the provided formatter. + pub fn write_value(&self, offset: usize, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Self::Count(arr) => match arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, }, - AggregateResult::Max(v) => match v { - Value::Null => None, - Value::ByteArray(s) => Some(s), - v => panic!("cannot convert {:?} to &[u8]", v), + Self::SumI64(arr) => match arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, }, - AggregateResult::First(_) => panic!("cannot convert first tuple to &[u8]"), - AggregateResult::Last(_) => panic!("cannot convert last tuple to &[u8]"), - AggregateResult::Sum(v) => panic!("cannot convert {:?} to &[u8]", v), - AggregateResult::Count(_) => panic!("cannot convert count to &[u8]"), + Self::SumU64(arr) => match arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::SumF64(arr) => match arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::MinU64(arr) => match arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::MinI64(arr) => match arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::MinF64(arr) => match arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::MinString(arr) => match &arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::MinBytes(arr) => match &arr[offset] { + Some(v) => write!(f, "{:?}", v)?, + None => write!(f, "NULL")?, + }, + Self::MinBool(arr) => match arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::MaxU64(arr) => match arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::MaxI64(arr) => match arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::MaxF64(arr) => match arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::MaxString(arr) => match &arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::MaxBytes(arr) => match &arr[offset] { + Some(v) => write!(f, "{:?}", v)?, + None => write!(f, "NULL")?, + }, + Self::MaxBool(arr) => match arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::FirstU64((arr, _)) => match arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::FirstI64((arr, _)) => match arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::FirstF64((arr, _)) => match arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::FirstString((arr, _)) => match &arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::FirstBytes((arr, _)) => match &arr[offset] { + Some(v) => write!(f, "{:?}", v)?, + None => write!(f, "NULL")?, + }, + Self::FirstBool((arr, _)) => match arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::LastU64((arr, _)) => match arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::LastI64((arr, _)) => match arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::LastF64((arr, _)) => match arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::LastString((arr, _)) => match &arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + Self::LastBytes((arr, _)) => match &arr[offset] { + Some(v) => write!(f, "{:?}", v)?, + None => write!(f, "NULL")?, + }, + Self::LastBool((arr, _)) => match arr[offset] { + Some(v) => write!(f, "{}", v)?, + None => write!(f, "NULL")?, + }, + } + Ok(()) + } + + // Consumes self and returns the inner `Vec>`. + pub fn take_as_i64(self) -> Vec> { + match self { + Self::SumI64(arr) => arr, + Self::MinI64(arr) => arr, + Self::MaxI64(arr) => arr, + _ => panic!("cannot convert {} to Vec>", self), } } - pub fn try_as_bool(&self) -> Option { - match &self { - AggregateResult::Min(v) => match v { - Value::Null => None, - Value::Boolean(s) => Some(*s), - v => panic!("cannot convert {:?} to bool", v), - }, - AggregateResult::Max(v) => match v { - Value::Null => None, - Value::Boolean(s) => Some(*s), - v => panic!("cannot convert {:?} to bool", v), - }, - AggregateResult::First(_) => panic!("cannot convert first tuple to bool"), - AggregateResult::Last(_) => panic!("cannot convert last tuple to bool"), - AggregateResult::Sum(v) => panic!("cannot convert {:?} to bool", v), - AggregateResult::Count(_) => panic!("cannot convert count to bool"), + // Consumes self and returns the inner `Vec>`. + pub fn take_as_u64(self) -> Vec> { + match self { + Self::Count(arr) => arr, + Self::SumU64(arr) => arr, + Self::MinU64(arr) => arr, + Self::MaxU64(arr) => arr, + _ => panic!("cannot convert {} to Vec>", self), } } - pub fn try_as_i64_scalar(&self) -> Option { - match &self { - AggregateResult::Sum(v) => match v { - Scalar::Null => None, - Scalar::I64(v) => Some(*v), - v => panic!("cannot convert {:?} to i64", v), - }, - AggregateResult::Min(v) => match v { - Value::Null => None, - Value::Scalar(s) => match s { - Scalar::Null => None, - Scalar::I64(v) => Some(*v), - v => panic!("cannot convert {:?} to u64", v), - }, - v => panic!("cannot convert {:?} to i64", v), - }, - AggregateResult::Max(v) => match v { - Value::Null => None, - Value::Scalar(s) => match s { - Scalar::Null => None, - Scalar::I64(v) => Some(*v), - v => panic!("cannot convert {:?} to u64", v), - }, - v => panic!("cannot convert {:?} to i64", v), - }, - AggregateResult::First(_) => panic!("cannot convert first tuple to scalar"), - AggregateResult::Last(_) => panic!("cannot convert last tuple to scalar"), - AggregateResult::Count(_) => panic!("cannot represent count as i64"), + // Consumes self and returns the inner `Vec>`. + pub fn take_as_f64(self) -> Vec> { + match self { + Self::SumF64(arr) => arr, + Self::MinF64(arr) => arr, + Self::MaxF64(arr) => arr, + _ => panic!("cannot convert {} to Vec>", self), } } - pub fn try_as_u64_scalar(&self) -> Option { - match &self { - AggregateResult::Sum(v) => match v { - Scalar::Null => None, - Scalar::U64(v) => Some(*v), - v => panic!("cannot convert {:?} to u64", v), - }, - AggregateResult::Count(c) => Some(*c), - AggregateResult::Min(v) => match v { - Value::Null => None, - Value::Scalar(s) => match s { - Scalar::Null => None, - Scalar::U64(v) => Some(*v), - v => panic!("cannot convert {:?} to u64", v), - }, - v => panic!("cannot convert {:?} to u64", v), - }, - AggregateResult::Max(v) => match v { - Value::Null => None, - Value::Scalar(s) => match s { - Scalar::Null => None, - Scalar::U64(v) => Some(*v), - v => panic!("cannot convert {:?} to u64", v), - }, - v => panic!("cannot convert {:?} to u64", v), - }, - AggregateResult::First(_) => panic!("cannot convert first tuple to scalar"), - AggregateResult::Last(_) => panic!("cannot convert last tuple to scalar"), + // Consumes self and returns the inner `Vec>`. + pub fn take_as_str(self) -> Vec> { + match self { + Self::MinString(arr) => arr, + Self::MaxString(arr) => arr, + _ => panic!("cannot convert {} to Vec>", self), } } - pub fn try_as_f64_scalar(&self) -> Option { - match &self { - AggregateResult::Sum(v) => match v { - Scalar::Null => None, - Scalar::F64(v) => Some(*v), - v => panic!("cannot convert {:?} to f64", v), - }, - AggregateResult::Min(v) => match v { - Value::Null => None, - Value::Scalar(s) => match s { - Scalar::Null => None, - Scalar::F64(v) => Some(*v), - v => panic!("cannot convert {:?} to f64", v), - }, - v => panic!("cannot convert {:?} to f64", v), - }, - AggregateResult::Max(v) => match v { - Value::Null => None, - Value::Scalar(s) => match s { - Scalar::Null => None, - Scalar::F64(v) => Some(*v), - v => panic!("cannot convert {:?} to f64", v), - }, - v => panic!("cannot convert {:?} to f64", v), - }, - AggregateResult::First(_) => panic!("cannot convert first tuple to scalar"), - AggregateResult::Last(_) => panic!("cannot convert last tuple to scalar"), - AggregateResult::Count(_) => panic!("cannot represent count as f64"), + // Consumes self and returns the inner `Vec>>`. + pub fn take_as_bytes(self) -> Vec>> { + match self { + Self::MinBytes(arr) => arr, + Self::MaxBytes(arr) => arr, + _ => panic!("cannot convert {} to Vec>", self), + } + } + + // Consumes self and returns the inner `Vec>`. + pub fn take_as_bool(self) -> Vec> { + match self { + Self::MinBool(arr) => arr, + Self::MaxBool(arr) => arr, + _ => panic!("cannot convert {} to Vec", self), + } + } + + /// Extends the `AggregateVec` with the provided `Option` iterator. + pub fn extend_with_i64(&mut self, itr: impl Iterator>) { + match self { + Self::SumI64(arr) => { + arr.extend(itr); + } + Self::MinI64(arr) => { + arr.extend(itr); + } + Self::MaxI64(arr) => { + arr.extend(itr); + } + _ => panic!("unsupported iterator"), + } + } + + /// Extends the `AggregateVec` with the provided `Option` iterator. + pub fn extend_with_u64(&mut self, itr: impl Iterator>) { + match self { + Self::Count(arr) => { + arr.extend(itr); + } + Self::SumU64(arr) => { + arr.extend(itr); + } + Self::MinU64(arr) => { + arr.extend(itr); + } + Self::MaxU64(arr) => { + arr.extend(itr); + } + _ => panic!("unsupported iterator"), + } + } + + /// Extends the `AggregateVec` with the provided `Option` iterator. + pub fn extend_with_f64(&mut self, itr: impl Iterator>) { + match self { + Self::SumF64(arr) => { + arr.extend(itr); + } + Self::MinF64(arr) => { + arr.extend(itr); + } + Self::MaxF64(arr) => { + arr.extend(itr); + } + _ => panic!("unsupported iterator"), + } + } + + /// Extends the `AggregateVec` with the provided `Option` iterator. + pub fn extend_with_str(&mut self, itr: impl Iterator>) { + match self { + Self::MinString(arr) => { + arr.extend(itr); + } + Self::MaxString(arr) => { + arr.extend(itr); + } + _ => panic!("unsupported iterator"), + } + } + + /// Extends the `AggregateVec` with the provided `Option>` iterator. + pub fn extend_with_bytes(&mut self, itr: impl Iterator>>) { + match self { + Self::MinBytes(arr) => { + arr.extend(itr); + } + Self::MaxBytes(arr) => { + arr.extend(itr); + } + _ => panic!("unsupported iterator"), + } + } + + /// Extends the `AggregateVec` with the provided `Option` iterator. + pub fn extend_with_bool(&mut self, itr: impl Iterator>) { + match self { + Self::MinBool(arr) => { + arr.extend(itr); + } + Self::MaxBool(arr) => { + arr.extend(itr); + } + _ => panic!("unsupported iterator"), + } + } + + pub fn sort_with_permutation(&mut self, p: &permutation::Permutation) { + match self { + Self::Count(arr) => { + *arr = p.apply_slice(arr.as_slice()); + } + Self::SumI64(arr) => { + *arr = p.apply_slice(arr.as_slice()); + } + Self::SumU64(arr) => { + *arr = p.apply_slice(arr.as_slice()); + } + Self::SumF64(arr) => { + *arr = p.apply_slice(arr.as_slice()); + } + Self::MinU64(arr) => { + *arr = p.apply_slice(arr.as_slice()); + } + Self::MinI64(arr) => { + *arr = p.apply_slice(arr.as_slice()); + } + Self::MinF64(arr) => { + *arr = p.apply_slice(arr.as_slice()); + } + Self::MinString(arr) => { + *arr = p.apply_slice(arr.as_slice()); + } + Self::MinBytes(arr) => { + *arr = p.apply_slice(arr.as_slice()); + } + Self::MinBool(arr) => { + *arr = p.apply_slice(arr.as_slice()); + } + Self::MaxU64(arr) => { + *arr = p.apply_slice(arr.as_slice()); + } + Self::MaxI64(arr) => { + *arr = p.apply_slice(arr.as_slice()); + } + Self::MaxF64(arr) => { + *arr = p.apply_slice(arr.as_slice()); + } + Self::MaxString(arr) => { + *arr = p.apply_slice(arr.as_slice()); + } + Self::MaxBytes(arr) => { + *arr = p.apply_slice(arr.as_slice()); + } + Self::MaxBool(arr) => { + *arr = p.apply_slice(arr.as_slice()); + } + Self::FirstU64((arr, time)) => { + *arr = p.apply_slice(arr.as_slice()); + *time = p.apply_slice(time.as_slice()); + } + Self::FirstI64((arr, time)) => { + *arr = p.apply_slice(arr.as_slice()); + *time = p.apply_slice(time.as_slice()); + } + Self::FirstF64((arr, time)) => { + *arr = p.apply_slice(arr.as_slice()); + *time = p.apply_slice(time.as_slice()); + } + Self::FirstString((arr, time)) => { + *arr = p.apply_slice(arr.as_slice()); + *time = p.apply_slice(time.as_slice()); + } + Self::FirstBytes((arr, time)) => { + *arr = p.apply_slice(arr.as_slice()); + *time = p.apply_slice(time.as_slice()); + } + Self::FirstBool((arr, time)) => { + *arr = p.apply_slice(arr.as_slice()); + *time = p.apply_slice(time.as_slice()); + } + Self::LastU64((arr, time)) => { + *arr = p.apply_slice(arr.as_slice()); + *time = p.apply_slice(time.as_slice()); + } + Self::LastI64((arr, time)) => { + *arr = p.apply_slice(arr.as_slice()); + *time = p.apply_slice(time.as_slice()); + } + Self::LastF64((arr, time)) => { + *arr = p.apply_slice(arr.as_slice()); + *time = p.apply_slice(time.as_slice()); + } + Self::LastString((arr, time)) => { + *arr = p.apply_slice(arr.as_slice()); + *time = p.apply_slice(time.as_slice()); + } + Self::LastBytes((arr, time)) => { + *arr = p.apply_slice(arr.as_slice()); + *time = p.apply_slice(time.as_slice()); + } + Self::LastBool((arr, time)) => { + *arr = p.apply_slice(arr.as_slice()); + *time = p.apply_slice(time.as_slice()); + } } } } -impl From<&AggregateType> for AggregateResult<'_> { - fn from(typ: &AggregateType) -> Self { - match typ { - AggregateType::Count => Self::Count(0), - AggregateType::First => Self::First(None), - AggregateType::Last => Self::Last(None), - AggregateType::Min => Self::Min(Value::Null), - AggregateType::Max => Self::Max(Value::Null), - AggregateType::Sum => Self::Sum(Scalar::Null), - } - } -} - -impl std::fmt::Display for AggregateResult<'_> { +impl std::fmt::Display for AggregateVec { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - AggregateResult::Count(v) => write!(f, "{}", v), - AggregateResult::First(v) => match v { - Some((_, v)) => write!(f, "{}", v), - None => write!(f, "NULL"), - }, - AggregateResult::Last(v) => match v { - Some((_, v)) => write!(f, "{}", v), - None => write!(f, "NULL"), - }, - AggregateResult::Min(v) => write!(f, "{}", v), - AggregateResult::Max(v) => write!(f, "{}", v), - AggregateResult::Sum(v) => write!(f, "{}", v), + Self::Count(_) => write!(f, "Count"), + Self::SumI64(_) => write!(f, "Sum"), + Self::SumU64(_) => write!(f, "Sum"), + Self::SumF64(_) => write!(f, "Sum"), + Self::MinU64(_) => write!(f, "Min"), + Self::MinI64(_) => write!(f, "Min"), + Self::MinF64(_) => write!(f, "Min"), + Self::MinString(_) => write!(f, "Min"), + Self::MinBytes(_) => write!(f, "Min>"), + Self::MinBool(_) => write!(f, "Min"), + Self::MaxU64(_) => write!(f, "Max"), + Self::MaxI64(_) => write!(f, "Max"), + Self::MaxF64(_) => write!(f, "Max"), + Self::MaxString(_) => write!(f, "Max"), + Self::MaxBytes(_) => write!(f, "Max>"), + Self::MaxBool(_) => write!(f, "Max"), + Self::FirstU64(_) => write!(f, "First"), + Self::FirstI64(_) => write!(f, "First"), + Self::FirstF64(_) => write!(f, "First"), + Self::FirstString(_) => write!(f, "First"), + Self::FirstBytes(_) => write!(f, "First>"), + Self::FirstBool(_) => write!(f, "First"), + Self::LastU64(_) => write!(f, "Last"), + Self::LastI64(_) => write!(f, "Last"), + Self::LastF64(_) => write!(f, "Last"), + Self::LastString(_) => write!(f, "Last"), + Self::LastBytes(_) => write!(f, "Last>"), + Self::LastBool(_) => write!(f, "Last"), + } + } +} + +impl From<(&AggregateType, &LogicalDataType)> for AggregateVec { + fn from(v: (&AggregateType, &LogicalDataType)) -> Self { + match (v.0, v.1) { + (AggregateType::Count, _) => Self::Count(vec![]), + (AggregateType::First, LogicalDataType::Integer) => Self::FirstI64((vec![], vec![])), + (AggregateType::First, LogicalDataType::Unsigned) => Self::FirstU64((vec![], vec![])), + (AggregateType::First, LogicalDataType::Float) => Self::FirstF64((vec![], vec![])), + (AggregateType::First, LogicalDataType::String) => Self::FirstString((vec![], vec![])), + (AggregateType::First, LogicalDataType::Binary) => Self::FirstBytes((vec![], vec![])), + (AggregateType::First, LogicalDataType::Boolean) => Self::FirstBool((vec![], vec![])), + (AggregateType::Last, LogicalDataType::Integer) => Self::LastI64((vec![], vec![])), + (AggregateType::Last, LogicalDataType::Unsigned) => Self::LastU64((vec![], vec![])), + (AggregateType::Last, LogicalDataType::Float) => Self::LastF64((vec![], vec![])), + (AggregateType::Last, LogicalDataType::String) => Self::LastString((vec![], vec![])), + (AggregateType::Last, LogicalDataType::Binary) => Self::LastBytes((vec![], vec![])), + (AggregateType::Last, LogicalDataType::Boolean) => Self::LastBool((vec![], vec![])), + (AggregateType::Min, LogicalDataType::Integer) => Self::MinI64(vec![]), + (AggregateType::Min, LogicalDataType::Unsigned) => Self::MinU64(vec![]), + (AggregateType::Min, LogicalDataType::Float) => Self::MinF64(vec![]), + (AggregateType::Min, LogicalDataType::String) => Self::MinString(vec![]), + (AggregateType::Min, LogicalDataType::Binary) => Self::MinBytes(vec![]), + (AggregateType::Min, LogicalDataType::Boolean) => Self::MinBool(vec![]), + (AggregateType::Max, LogicalDataType::Integer) => Self::MaxI64(vec![]), + (AggregateType::Max, LogicalDataType::Unsigned) => Self::MaxU64(vec![]), + (AggregateType::Max, LogicalDataType::Float) => Self::MaxF64(vec![]), + (AggregateType::Max, LogicalDataType::String) => Self::MaxString(vec![]), + (AggregateType::Max, LogicalDataType::Binary) => Self::MaxBytes(vec![]), + (AggregateType::Max, LogicalDataType::Boolean) => Self::MaxBool(vec![]), + (AggregateType::Sum, LogicalDataType::Integer) => Self::SumI64(vec![]), + (AggregateType::Sum, LogicalDataType::Unsigned) => Self::SumU64(vec![]), + (AggregateType::Sum, LogicalDataType::Float) => Self::SumF64(vec![]), + (AggregateType::Sum, _) => unreachable!("unsupported SUM aggregates"), } } } @@ -457,6 +1073,26 @@ impl<'a> std::ops::AddAssign<&Scalar> for &mut Scalar { } } +impl std::ops::Add for Scalar { + type Output = Self; + + fn add(self, other: Self) -> Self { + match (self, other) { + (Self::Null, Self::Null) => Self::Null, + (Self::Null, Self::I64(_)) => other, + (Self::Null, Self::U64(_)) => other, + (Self::Null, Self::F64(_)) => other, + (Self::I64(_), Self::Null) => self, + (Self::I64(a), Self::I64(b)) => Self::I64(a + b), + (Self::U64(_), Self::Null) => self, + (Self::U64(a), Self::U64(b)) => Self::U64(a + b), + (Self::F64(_), Self::Null) => self, + (Self::F64(a), Self::F64(b)) => Self::F64(a + b), + (a, b) => panic!("{:?} + {:?} is an unsupported operation", a, b), + } + } +} + impl std::fmt::Display for Scalar { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { @@ -541,7 +1177,7 @@ pub enum Value<'a> { Scalar(Scalar), } -impl Value<'_> { +impl<'a> Value<'a> { pub fn is_null(&self) -> bool { matches!(self, Self::Null) } @@ -553,16 +1189,44 @@ impl Value<'_> { panic!("cannot unwrap Value to Scalar"); } - pub fn string(&self) -> &str { + pub fn i64(self) -> i64 { + if let Self::Scalar(Scalar::I64(v)) = self { + return v; + } + panic!("cannot unwrap Value to i64"); + } + + pub fn u64(self) -> u64 { + if let Self::Scalar(Scalar::U64(v)) = self { + return v; + } + panic!("cannot unwrap Value to u64"); + } + + pub fn f64(self) -> f64 { + if let Self::Scalar(Scalar::F64(v)) = self { + return v; + } + panic!("cannot unwrap Value to f64"); + } + + pub fn str(self) -> &'a str { if let Self::String(s) = self { return s; } panic!("cannot unwrap Value to String"); } - pub fn bool(&self) -> bool { + pub fn bytes(self) -> &'a [u8] { + if let Self::ByteArray(s) = self { + return s; + } + panic!("cannot unwrap Value to byte array"); + } + + pub fn bool(self) -> bool { if let Self::Boolean(b) = self { - return *b; + return b; } panic!("cannot unwrap Value to Scalar"); } @@ -591,6 +1255,45 @@ impl<'a> From<&'a str> for Value<'a> { } } +impl<'a> From> for Value<'a> { + fn from(v: Option<&'a str>) -> Self { + match v { + Some(s) => Self::String(s), + None => Self::Null, + } + } +} + +impl<'a> From<&'a [u8]> for Value<'a> { + fn from(v: &'a [u8]) -> Self { + Self::ByteArray(v) + } +} + +impl<'a> From> for Value<'a> { + fn from(v: Option<&'a [u8]>) -> Self { + match v { + Some(s) => Self::ByteArray(s), + None => Self::Null, + } + } +} + +impl<'a> From for Value<'a> { + fn from(v: bool) -> Self { + Self::Boolean(v) + } +} + +impl<'a> From> for Value<'a> { + fn from(v: Option) -> Self { + match v { + Some(s) => Self::Boolean(s), + None => Self::Null, + } + } +} + // Implementations of From trait for various concrete types. macro_rules! scalar_from_impls { ($(($variant:ident, $type:ident),)*) => { @@ -619,6 +1322,17 @@ scalar_from_impls! { (F64, f64), } +impl std::ops::Add for Value<'_> { + type Output = Self; + + fn add(self, other: Self) -> Self { + match (self, other) { + (Self::Scalar(a), Self::Scalar(b)) => Self::Scalar(a + b), + _ => panic!("unsupported operation on Value"), + } + } +} + /// Each variant is a typed vector of materialised values for a column. #[derive(Debug, PartialEq)] pub enum Values<'a> { @@ -659,6 +1373,20 @@ impl<'a> Values<'a> { self.len() == 0 } + pub fn is_null(&self, i: usize) -> bool { + match &self { + Self::String(c) => c[i].is_none(), + Self::F64(_) => false, + Self::I64(_) => false, + Self::U64(_) => false, + Self::Bool(c) => c[i].is_none(), + Self::ByteArray(c) => c[i].is_none(), + Self::I64N(c) => c[i].is_none(), + Self::U64N(c) => c[i].is_none(), + Self::F64N(c) => c[i].is_none(), + } + } + pub fn value(&self, i: usize) -> Value<'a> { match &self { Self::String(c) => match c[i] { @@ -690,6 +1418,57 @@ impl<'a> Values<'a> { }, } } + + // Returns a value as an i64. Panics if not possible. + fn value_i64(&self, i: usize) -> i64 { + match &self { + Values::I64(c) => c[i], + Values::I64N(c) => c[i].unwrap(), + _ => panic!("value cannot be returned as i64"), + } + } + + // Returns a value as an u64. Panics if not possible. + fn value_u64(&self, i: usize) -> u64 { + match &self { + Values::U64(c) => c[i], + Values::U64N(c) => c[i].unwrap(), + _ => panic!("value cannot be returned as u64"), + } + } + + // Returns a value as an f64. Panics if not possible. + fn value_f64(&self, i: usize) -> f64 { + match &self { + Values::F64(c) => c[i], + Values::F64N(c) => c[i].unwrap(), + _ => panic!("value cannot be returned as f64"), + } + } + + // Returns a value as a &str. Panics if not possible. + fn value_str(&self, i: usize) -> &'a str { + match &self { + Values::String(c) => c[i].unwrap(), + _ => panic!("value cannot be returned as &str"), + } + } + + // Returns a value as a binary array. Panics if not possible. + fn value_bytes(&self, i: usize) -> &'a [u8] { + match &self { + Values::ByteArray(c) => c[i].unwrap(), + _ => panic!("value cannot be returned as &str"), + } + } + + // Returns a value as a bool. Panics if not possible. + fn value_bool(&self, i: usize) -> bool { + match &self { + Values::Bool(c) => c[i].unwrap(), + _ => panic!("value cannot be returned as &str"), + } + } } /// Moves ownership of Values into an arrow `ArrayRef`. @@ -734,15 +1513,6 @@ impl<'a> Iterator for ValuesIterator<'a> { } } -#[derive(PartialEq, Debug)] -pub enum ValueSet<'a> { - // UTF-8 valid unicode strings - String(BTreeSet>), - - // Arbitrary collections of bytes - ByteArray(BTreeSet>), -} - #[derive(Debug, PartialEq)] /// A representation of encoded values for a column. pub enum EncodedValues { @@ -814,6 +1584,181 @@ impl EncodedValues { mod test { use super::*; + #[test] + fn aggregate_vec_update() { + // i64 + let values = Values::I64N(vec![Some(1), Some(2), Some(3), None, Some(-1), Some(2)]); + + let mut aggs = vec![ + AggregateVec::Count(vec![]), + AggregateVec::SumI64(vec![]), + AggregateVec::MinI64(vec![]), + AggregateVec::MaxI64(vec![]), + ]; + + for i in 0..values.len() { + for agg in &mut aggs { + agg.update(&values, i, 0); + } + } + + assert_eq!( + aggs, + vec![ + AggregateVec::Count(vec![Some(5)]), + AggregateVec::SumI64(vec![Some(7)]), + AggregateVec::MinI64(vec![Some(-1)]), + AggregateVec::MaxI64(vec![Some(3)]), + ] + ); + + // u64 + let values = Values::U64N(vec![Some(1), Some(2), Some(3), None, Some(0), Some(2)]); + + let mut aggs = vec![ + AggregateVec::Count(vec![]), + AggregateVec::SumU64(vec![]), + AggregateVec::MinU64(vec![]), + AggregateVec::MaxU64(vec![]), + ]; + + for i in 0..values.len() { + for agg in &mut aggs { + agg.update(&values, i, 0); + } + } + + assert_eq!( + aggs, + vec![ + AggregateVec::Count(vec![Some(5)]), + AggregateVec::SumU64(vec![Some(8)]), + AggregateVec::MinU64(vec![Some(0)]), + AggregateVec::MaxU64(vec![Some(3)]), + ] + ); + + // f64 + let values = Values::F64N(vec![ + Some(1.0), + Some(2.0), + Some(3.0), + None, + Some(0.0), + Some(2.0), + ]); + + let mut aggs = vec![ + AggregateVec::Count(vec![]), + AggregateVec::SumF64(vec![]), + AggregateVec::MinF64(vec![]), + AggregateVec::MaxF64(vec![]), + ]; + + for i in 0..values.len() { + for agg in &mut aggs { + agg.update(&values, i, 0); + } + } + + assert_eq!( + aggs, + vec![ + AggregateVec::Count(vec![Some(5)]), + AggregateVec::SumF64(vec![Some(8.0)]), + AggregateVec::MinF64(vec![Some(0.0)]), + AggregateVec::MaxF64(vec![Some(3.0)]), + ] + ); + + // string + let values = Values::String(vec![ + Some("Pop Song 89"), + Some("Orange Crush"), + Some("Stand"), + None, + ]); + + let mut aggs = vec![ + AggregateVec::Count(vec![]), + AggregateVec::MinString(vec![]), + AggregateVec::MaxString(vec![]), + ]; + + for i in 0..values.len() { + for agg in &mut aggs { + agg.update(&values, i, 0); + } + } + + assert_eq!( + aggs, + vec![ + AggregateVec::Count(vec![Some(3)]), + AggregateVec::MinString(vec![Some("Orange Crush".to_owned())]), + AggregateVec::MaxString(vec![Some("Stand".to_owned())]), + ] + ); + + // bytes + let arr = vec![ + "Pop Song 89".to_string(), + "Orange Crush".to_string(), + "Stand".to_string(), + ]; + let values = Values::ByteArray(vec![ + Some(arr[0].as_bytes()), + Some(arr[1].as_bytes()), + Some(arr[2].as_bytes()), + None, + ]); + + let mut aggs = vec![ + AggregateVec::Count(vec![]), + AggregateVec::MinBytes(vec![]), + AggregateVec::MaxBytes(vec![]), + ]; + + for i in 0..values.len() { + for agg in &mut aggs { + agg.update(&values, i, 0); + } + } + + assert_eq!( + aggs, + vec![ + AggregateVec::Count(vec![Some(3)]), + AggregateVec::MinBytes(vec![Some(arr[1].bytes().collect())]), + AggregateVec::MaxBytes(vec![Some(arr[2].bytes().collect())]), + ] + ); + + // bool + let values = Values::Bool(vec![Some(true), None, Some(false)]); + + let mut aggs = vec![ + AggregateVec::Count(vec![]), + AggregateVec::MinBool(vec![]), + AggregateVec::MaxBool(vec![]), + ]; + + for i in 0..values.len() { + for agg in &mut aggs { + agg.update(&values, i, 0); + } + } + + assert_eq!( + aggs, + vec![ + AggregateVec::Count(vec![Some(2)]), + AggregateVec::MinBool(vec![Some(false)]), + AggregateVec::MaxBool(vec![Some(true)]), + ] + ); + } + #[test] fn size() { let v1 = OwnedValue::Null; diff --git a/scripts/grpcurl b/scripts/grpcurl new file mode 100755 index 0000000000..36fa4a6ce6 --- /dev/null +++ b/scripts/grpcurl @@ -0,0 +1,22 @@ +#!/bin/bash +# +# This script is a convenience wrapper around grpcurl that passes all the known *.proto to it. +# +# Script self-destruction condition: +# Once tonic implements reflection this script will no longer be necessary. +# The reflection feature is tracked in https://github.com/hyperium/tonic/issues/165 +# and marked as closed and will likely be included in a tonic version > 0.4.0. +# + +set -eu -o pipefail + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +proto_dir="${SCRIPT_DIR}"/../generated_types/protos + +# bash 3.x (default on macos big-sur 🤦) has no readarray. +while IFS= read -r line; do + proto_flags+=("-proto" "$line") +done < <(find "${proto_dir}" -name '*.proto') + +grpcurl -import-path ./generated_types/protos "${proto_flags[@]}" "$@" diff --git a/server/Cargo.toml b/server/Cargo.toml index a82d9bd3e6..7c020bc999 100644 --- a/server/Cargo.toml +++ b/server/Cargo.toml @@ -11,10 +11,14 @@ bytes = "1.0" chrono = "0.4" crc32fast = "1.2.0" data_types = { path = "../data_types" } -flatbuffers = "0.6" +# See docs/regenerating_flatbuffers.md about updating generated code when updating the +# version of the flatbuffers crate +flatbuffers = "0.8" futures = "0.3.7" generated_types = { path = "../generated_types" } +hashbrown = "0.9.1" influxdb_line_protocol = { path = "../influxdb_line_protocol" } +internal_types = { path = "../internal_types" } mutable_buffer = { path = "../mutable_buffer" } object_store = { path = "../object_store" } parking_lot = "0.11.1" @@ -26,6 +30,7 @@ serde_json = "1.0" snafu = "0.6" snap = "1.0.0" tokio = { version = "1.0", features = ["macros", "time"] } +tokio-util = { version = "0.6.3" } tracing = "0.1" uuid = { version = "0.8", features = ["serde", "v4"] } diff --git a/server/src/buffer.rs b/server/src/buffer.rs index 550719b7b4..a2b5e56d78 100644 --- a/server/src/buffer.rs +++ b/server/src/buffer.rs @@ -1,11 +1,11 @@ //! This module contains code for managing the WAL buffer use data_types::{ - data::ReplicatedWrite, database_rules::{WalBufferRollover, WriterId}, DatabaseName, }; use generated_types::wal; +use internal_types::data::ReplicatedWrite; use object_store::{path::ObjectStorePath, ObjectStore, ObjectStoreApi}; use std::{ @@ -15,7 +15,7 @@ use std::{ sync::Arc, }; -use crate::tracker::{TrackedFutureExt, TrackerRegistry}; +use crate::tracker::{TrackedFutureExt, TrackerRegistration}; use bytes::Bytes; use chrono::{DateTime, Utc}; use crc32fast::Hasher; @@ -69,14 +69,16 @@ pub enum Error { #[snafu(display("checksum mismatch for segment"))] ChecksumMismatch, - #[snafu(display("the flatbuffers Segment is invalid"))] - InvalidFlatbuffersSegment, -} + #[snafu(display("the flatbuffers Segment is invalid: {}", source))] + InvalidFlatbuffersSegment { + source: flatbuffers::InvalidFlatbuffer, + }, -#[derive(Debug, Clone)] -pub struct SegmentPersistenceTask { - writer_id: u32, - location: object_store::path::Path, + #[snafu(display("the flatbuffers size is too small; only found {} bytes", bytes))] + FlatbuffersSegmentTooSmall { bytes: usize }, + + #[snafu(display("the flatbuffers Segment is missing an expected value for {}", field))] + FlatbuffersMissingField { field: String }, } pub type Result = std::result::Result; @@ -117,7 +119,7 @@ impl Buffer { /// by accepting the write, the oldest (first) of the closed segments /// will be dropped, if it is persisted. Otherwise, an error is returned. pub fn append(&mut self, write: Arc) -> Result>> { - let write_size = u64::try_from(write.data.len()) + let write_size = u64::try_from(write.data().len()) .expect("appended data must be less than a u64 in length"); while self.current_size + write_size > self.max_size { @@ -316,7 +318,7 @@ impl Segment { let (writer_id, sequence_number) = write.writer_and_sequence(); self.validate_and_update_sequence_summary(writer_id, sequence_number)?; - let size = write.data.len(); + let size = write.data().len(); let size = u64::try_from(size).expect("appended data must be less than a u64 in length"); self.size += size; @@ -376,7 +378,7 @@ impl Segment { /// the given object store location. pub fn persist_bytes_in_background( &self, - reg: &TrackerRegistry, + tracker: TrackerRegistration, writer_id: u32, db_name: &DatabaseName<'_>, store: Arc, @@ -385,11 +387,6 @@ impl Segment { let location = database_object_store_path(writer_id, db_name, &store); let location = object_store_path_for_segment(&location, self.id)?; - let task_meta = SegmentPersistenceTask { - writer_id, - location: location.clone(), - }; - let len = data.len(); let mut stream_data = std::io::Result::Ok(data.clone()); @@ -414,7 +411,7 @@ impl Segment { // TODO: Mark segment as persisted info!("persisted data to {}", location.display()); } - .track(reg, task_meta), + .track(tracker), ); Ok(()) @@ -429,7 +426,7 @@ impl Segment { .writes .iter() .map(|rw| { - let payload = fbb.create_vector_direct(&rw.data); + let payload = fbb.create_vector_direct(rw.data()); wal::ReplicatedWriteData::create( &mut fbb, &wal::ReplicatedWriteDataArgs { @@ -494,7 +491,7 @@ impl Segment { /// deserializes it into a Segment struct. pub fn from_file_bytes(data: &[u8]) -> Result { if data.len() < std::mem::size_of::() { - return Err(Error::InvalidFlatbuffersSegment); + return FlatbuffersSegmentTooSmall { bytes: data.len() }.fail(); } let (data, checksum) = data.split_at(data.len() - std::mem::size_of::()); @@ -512,15 +509,21 @@ impl Segment { .decompress_vec(data) .context(UnableToDecompressData)?; - let fb_segment = flatbuffers::get_root::>(&data); + // Use verified flatbuffer functionality here + let fb_segment = + flatbuffers::root::>(&data).context(InvalidFlatbuffersSegment)?; - let writes = fb_segment.writes().context(InvalidFlatbuffersSegment)?; + let writes = fb_segment + .writes() + .context(FlatbuffersMissingField { field: "writes" })?; let mut segment = Self::new_with_capacity(fb_segment.id(), writes.len()); for w in writes { - let data = w.payload().context(InvalidFlatbuffersSegment)?; - let rw = ReplicatedWrite { - data: data.to_vec(), - }; + let data = w + .payload() + .context(FlatbuffersMissingField { field: "payload" })? + .to_vec(); + let rw = ReplicatedWrite::try_from(data).context(InvalidFlatbuffersSegment)?; + segment.append(Arc::new(rw))?; } @@ -578,8 +581,9 @@ fn database_object_store_path( #[cfg(test)] mod tests { use super::*; - use data_types::{data::lines_to_replicated_write, database_rules::DatabaseRules}; + use data_types::database_rules::DatabaseRules; use influxdb_line_protocol::parse_lines; + use internal_types::data::lines_to_replicated_write; use object_store::memory::InMemory; #[test] @@ -589,7 +593,7 @@ mod tests { let mut buf = Buffer::new(max, segment, WalBufferRollover::ReturnError, false); let write = lp_to_replicated_write(1, 1, "cpu val=1 10"); - let size = write.data.len() as u64; + let size = write.data().len() as u64; assert_eq!(0, buf.size()); let segment = buf.append(write).unwrap(); assert_eq!(size, buf.size()); @@ -742,7 +746,7 @@ mod tests { fn all_writes_since() { let max = 1 << 63; let write = lp_to_replicated_write(1, 1, "cpu val=1 10"); - let segment = (write.data.len() + 1) as u64; + let segment = (write.data().len() + 1) as u64; let mut buf = Buffer::new(max, segment, WalBufferRollover::ReturnError, false); let segment = buf.append(write).unwrap(); @@ -799,7 +803,7 @@ mod tests { fn writes_since() { let max = 1 << 63; let write = lp_to_replicated_write(1, 1, "cpu val=1 10"); - let segment = (write.data.len() + 1) as u64; + let segment = (write.data().len() + 1) as u64; let mut buf = Buffer::new(max, segment, WalBufferRollover::ReturnError, false); let segment = buf.append(write).unwrap(); @@ -846,7 +850,7 @@ mod tests { fn returns_error_if_sequence_decreases() { let max = 1 << 63; let write = lp_to_replicated_write(1, 3, "cpu val=1 10"); - let segment = (write.data.len() + 1) as u64; + let segment = (write.data().len() + 1) as u64; let mut buf = Buffer::new(max, segment, WalBufferRollover::ReturnError, false); let segment = buf.append(write).unwrap(); diff --git a/server/src/config.rs b/server/src/config.rs index b7781734ac..4082203faa 100644 --- a/server/src/config.rs +++ b/server/src/config.rs @@ -1,29 +1,48 @@ -/// This module contains code for managing the configuration of the server. -use crate::{db::Db, Error, Result}; +use std::{ + collections::{BTreeMap, BTreeSet}, + sync::{Arc, RwLock}, +}; + use data_types::{ - database_rules::{DatabaseRules, HostGroup, HostGroupId}, + database_rules::{DatabaseRules, WriterId}, DatabaseName, }; use mutable_buffer::MutableBufferDb; use object_store::path::ObjectStorePath; use read_buffer::Database as ReadBufferDb; -use std::{ - collections::{BTreeMap, BTreeSet}, - sync::{Arc, RwLock}, -}; +/// This module contains code for managing the configuration of the server. +use crate::{db::Db, Error, JobRegistry, Result}; +use tokio::task::JoinHandle; +use tokio_util::sync::CancellationToken; +use tracing::{error, info, warn, Instrument}; pub(crate) const DB_RULES_FILE_NAME: &str = "rules.json"; -/// The Config tracks the configuration od databases and their rules along +/// The Config tracks the configuration of databases and their rules along /// with host groups for replication. It is used as an in-memory structure -/// that can be loaded incrementally from objet storage. -#[derive(Default, Debug)] +/// that can be loaded incrementally from object storage. +/// +/// drain() should be called prior to drop to ensure termination +/// of background worker tasks. They will be cancelled on drop +/// but they are effectively "detached" at that point, and they may not +/// run to completion if the tokio runtime is dropped +#[derive(Debug)] pub(crate) struct Config { + shutdown: CancellationToken, + jobs: Arc, state: RwLock, } impl Config { + pub(crate) fn new(jobs: Arc) -> Self { + Self { + shutdown: Default::default(), + state: Default::default(), + jobs, + } + } + pub(crate) fn create_db( &self, name: DatabaseName<'static>, @@ -45,7 +64,13 @@ impl Config { let read_buffer = ReadBufferDb::new(); let wal_buffer = rules.wal_buffer_config.as_ref().map(Into::into); - let db = Arc::new(Db::new(rules, mutable_buffer, read_buffer, wal_buffer)); + let db = Arc::new(Db::new( + rules, + mutable_buffer, + read_buffer, + wal_buffer, + Arc::clone(&self.jobs), + )); state.reservations.insert(name.clone()); Ok(CreateDatabaseHandle { @@ -57,19 +82,7 @@ impl Config { pub(crate) fn db(&self, name: &DatabaseName<'_>) -> Option> { let state = self.state.read().expect("mutex poisoned"); - state.databases.get(name).cloned() - } - - pub(crate) fn create_host_group(&self, host_group: HostGroup) { - let mut state = self.state.write().expect("mutex poisoned"); - state - .host_groups - .insert(host_group.id.clone(), Arc::new(host_group)); - } - - pub(crate) fn host_group(&self, host_group_id: &str) -> Option> { - let state = self.state.read().expect("mutex poisoned"); - state.host_groups.get(host_group_id).cloned() + state.databases.get(name).map(|x| Arc::clone(&x.db)) } pub(crate) fn db_names_sorted(&self) -> Vec> { @@ -77,19 +90,85 @@ impl Config { state.databases.keys().cloned().collect() } + pub(crate) fn remotes_sorted(&self) -> Vec<(WriterId, String)> { + let state = self.state.read().expect("mutex poisoned"); + state.remotes.iter().map(|(&a, b)| (a, b.clone())).collect() + } + + pub(crate) fn update_remote(&self, id: WriterId, addr: GRPCConnectionString) { + let mut state = self.state.write().expect("mutex poisoned"); + state.remotes.insert(id, addr); + } + + pub(crate) fn delete_remote(&self, id: WriterId) -> Option { + let mut state = self.state.write().expect("mutex poisoned"); + state.remotes.remove(&id) + } + fn commit(&self, name: &DatabaseName<'static>, db: Arc) { let mut state = self.state.write().expect("mutex poisoned"); let name = state .reservations .take(name) .expect("reservation doesn't exist"); - assert!(state.databases.insert(name, db).is_none()) + + if self.shutdown.is_cancelled() { + error!("server is shutting down"); + return; + } + + let shutdown = self.shutdown.child_token(); + let shutdown_captured = shutdown.clone(); + let db_captured = Arc::clone(&db); + let name_captured = name.clone(); + + let handle = Some(tokio::spawn(async move { + db_captured + .background_worker(shutdown_captured) + .instrument(tracing::info_span!("db_worker", database=%name_captured)) + .await + })); + + assert!(state + .databases + .insert( + name, + DatabaseState { + db, + handle, + shutdown + } + ) + .is_none()) } fn rollback(&self, name: &DatabaseName<'static>) { let mut state = self.state.write().expect("mutex poisoned"); state.reservations.remove(name); } + + /// Cancels and drains all background worker tasks + pub(crate) async fn drain(&self) { + info!("shutting down database background workers"); + + // This will cancel all background child tasks + self.shutdown.cancel(); + + let handles: Vec<_> = self + .state + .write() + .expect("mutex poisoned") + .databases + .iter_mut() + .filter_map(|(_, v)| v.join()) + .collect(); + + for handle in handles { + let _ = handle.await; + } + + info!("database background workers shutdown"); + } } pub fn object_store_path_for_database_config( @@ -102,14 +181,42 @@ pub fn object_store_path_for_database_config( path } +/// A gRPC connection string. +pub type GRPCConnectionString = String; + #[derive(Default, Debug)] struct ConfigState { reservations: BTreeSet>, - databases: BTreeMap, Arc>, - host_groups: BTreeMap>, + databases: BTreeMap, DatabaseState>, + /// Map between remote IOx server IDs and management API connection strings. + remotes: BTreeMap, } -/// CreateDatabaseHandle is retunred when a call is made to `create_db` on +#[derive(Debug)] +struct DatabaseState { + db: Arc, + handle: Option>, + shutdown: CancellationToken, +} + +impl DatabaseState { + fn join(&mut self) -> Option> { + self.handle.take() + } +} + +impl Drop for DatabaseState { + fn drop(&mut self) { + if self.handle.is_some() { + // Join should be called on `DatabaseState` prior to dropping, for example, by + // calling drain() on the owning `Config` + warn!("DatabaseState dropped without waiting for background task to complete"); + self.shutdown.cancel(); + } + } +} + +/// CreateDatabaseHandle is returned when a call is made to `create_db` on /// the Config struct. The handle can be used to hold a reservation for the /// database name. Calling `commit` on the handle will consume the struct /// and move the database from reserved to being in the config. @@ -140,13 +247,14 @@ impl<'a> Drop for CreateDatabaseHandle<'a> { #[cfg(test)] mod test { - use super::*; use object_store::{memory::InMemory, ObjectStore, ObjectStoreApi}; - #[test] - fn create_db() { + use super::*; + + #[tokio::test] + async fn create_db() { let name = DatabaseName::new("foo").unwrap(); - let config = Config::default(); + let config = Config::new(Arc::new(JobRegistry::new())); let rules = DatabaseRules::new(); { @@ -158,8 +266,45 @@ mod test { let db_reservation = config.create_db(name.clone(), rules).unwrap(); db_reservation.commit(); assert!(config.db(&name).is_some()); + assert_eq!(config.db_names_sorted(), vec![name.clone()]); - assert_eq!(config.db_names_sorted(), vec![name]); + tokio::time::sleep(tokio::time::Duration::from_millis(10)).await; + + assert!( + config + .db(&name) + .expect("expected database") + .worker_iterations() + > 0 + ); + + config.drain().await + } + + #[tokio::test] + async fn test_db_drop() { + let name = DatabaseName::new("foo").unwrap(); + let config = Config::new(Arc::new(JobRegistry::new())); + let rules = DatabaseRules::new(); + + let db_reservation = config.create_db(name.clone(), rules).unwrap(); + db_reservation.commit(); + + let token = config + .state + .read() + .expect("lock poisoned") + .databases + .get(&name) + .unwrap() + .shutdown + .clone(); + + // Drop config without calling drain + std::mem::drop(config); + + // This should cancel the the background task + assert!(token.is_cancelled()); } #[test] diff --git a/server/src/db.rs b/server/src/db.rs index b5ad60244b..fd6a6c723d 100644 --- a/server/src/db.rs +++ b/server/src/db.rs @@ -4,26 +4,26 @@ use std::{ collections::BTreeMap, sync::{ - atomic::{AtomicU64, Ordering}, + atomic::{AtomicU64, AtomicUsize, Ordering}, Arc, }, }; use async_trait::async_trait; -use data_types::{data::ReplicatedWrite, database_rules::DatabaseRules, selection::Selection}; -use mutable_buffer::MutableBufferDb; use parking_lot::Mutex; -use query::{Database, PartitionChunk}; -use read_buffer::Database as ReadBufferDb; -use serde::{Deserialize, Serialize}; use snafu::{OptionExt, ResultExt, Snafu}; - -use crate::buffer::Buffer; - use tracing::info; -mod chunk; pub(crate) use chunk::DBChunk; +use data_types::{chunk::ChunkSummary, database_rules::DatabaseRules}; +use internal_types::{data::ReplicatedWrite, selection::Selection}; +use mutable_buffer::MutableBufferDb; +use query::{Database, PartitionChunk}; +use read_buffer::Database as ReadBufferDb; + +use crate::{buffer::Buffer, JobRegistry}; + +mod chunk; pub mod pred; mod streams; @@ -43,6 +43,16 @@ pub enum Error { #[snafu(display("Cannot read to this database: no mutable buffer configured"))] DatabaseNotReadable {}, + #[snafu(display( + "Only closed chunks can be moved to read buffer. Chunk {} {} was open", + partition_key, + chunk_id + ))] + ChunkNotClosed { + partition_key: String, + chunk_id: u32, + }, + #[snafu(display("Error dropping data from mutable buffer: {}", source))] MutableBufferDrop { source: mutable_buffer::database::Error, @@ -70,39 +80,40 @@ pub type Result = std::result::Result; const STARTING_SEQUENCE: u64 = 1; -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug)] /// This is the main IOx Database object. It is the root object of any /// specific InfluxDB IOx instance pub struct Db { - #[serde(flatten)] pub rules: DatabaseRules, - #[serde(skip)] /// The (optional) mutable buffer stores incoming writes. If a /// database does not have a mutable buffer it can not accept /// writes (it is a read replica) pub mutable_buffer: Option, - #[serde(skip)] /// The read buffer holds chunk data in an in-memory optimized /// format. pub read_buffer: Arc, - #[serde(skip)] /// The wal buffer holds replicated writes in an append in-memory /// buffer. This buffer is used for sending data to subscribers /// and to persist segments in object storage for recovery. pub wal_buffer: Option>, - #[serde(skip)] + jobs: Arc, + sequence: AtomicU64, + + worker_iterations: AtomicUsize, } + impl Db { pub fn new( rules: DatabaseRules, mutable_buffer: Option, read_buffer: ReadBufferDb, wal_buffer: Option, + jobs: Arc, ) -> Self { let wal_buffer = wal_buffer.map(Mutex::new); let read_buffer = Arc::new(read_buffer); @@ -111,30 +122,48 @@ impl Db { mutable_buffer, read_buffer, wal_buffer, + jobs, sequence: AtomicU64::new(STARTING_SEQUENCE), + worker_iterations: AtomicUsize::new(0), } } - /// Rolls over the active chunk in the database's specified partition + /// Rolls over the active chunk in the database's specified + /// partition. Returns the previously open (now closed) Chunk pub async fn rollover_partition(&self, partition_key: &str) -> Result> { if let Some(local_store) = self.mutable_buffer.as_ref() { local_store .rollover_partition(partition_key) .context(RollingPartition) - .map(DBChunk::new_mb) + .map(|c| DBChunk::new_mb(c, partition_key, false)) } else { DatatbaseNotWriteable {}.fail() } } + /// Return true if the specified chunk is still open for new writes + pub fn is_open_chunk(&self, partition_key: &str, chunk_id: u32) -> bool { + if let Some(mutable_buffer) = self.mutable_buffer.as_ref() { + let open_chunk_id = mutable_buffer.open_chunk_id(partition_key); + open_chunk_id == chunk_id + } else { + false + } + } + // Return a list of all chunks in the mutable_buffer (that can // potentially be migrated into the read buffer or object store) pub fn mutable_buffer_chunks(&self, partition_key: &str) -> Vec> { let chunks = if let Some(mutable_buffer) = self.mutable_buffer.as_ref() { + let open_chunk_id = mutable_buffer.open_chunk_id(partition_key); + mutable_buffer .chunks(partition_key) .into_iter() - .map(DBChunk::new_mb) + .map(|c| { + let open = c.id() == open_chunk_id; + DBChunk::new_mb(c, partition_key, open) + }) .collect() } else { vec![] @@ -142,6 +171,19 @@ impl Db { chunks } + // Return the specified chunk in the mutable buffer + pub fn mutable_buffer_chunk( + &self, + partition_key: &str, + chunk_id: u32, + ) -> Result> { + self.mutable_buffer + .as_ref() + .context(DatatbaseNotWriteable)? + .get_chunk(partition_key, chunk_id) + .context(UnknownMutableBufferChunk { chunk_id }) + } + /// List chunks that are currently in the read buffer pub fn read_buffer_chunks(&self, partition_key: &str) -> Vec> { self.read_buffer @@ -162,7 +204,7 @@ impl Db { .as_ref() .context(DatatbaseNotWriteable)? .drop_chunk(partition_key, chunk_id) - .map(DBChunk::new_mb) + .map(|c| DBChunk::new_mb(c, partition_key, false)) .context(MutableBufferDrop) } @@ -202,12 +244,16 @@ impl Db { partition_key: &str, chunk_id: u32, ) -> Result> { - let mb_chunk = self - .mutable_buffer - .as_ref() - .context(DatatbaseNotWriteable)? - .get_chunk(partition_key, chunk_id) - .context(UnknownMutableBufferChunk { chunk_id })?; + let mb_chunk = self.mutable_buffer_chunk(partition_key, chunk_id)?; + + // Can't load an open chunk to the read buffer + if self.is_open_chunk(partition_key, chunk_id) { + return ChunkNotClosed { + partition_key, + chunk_id, + } + .fail(); + } let mut batches = Vec::new(); for stats in mb_chunk.table_stats().unwrap() { @@ -261,6 +307,40 @@ impl Db { Ok(()) } + + /// Return Summary information for chunks in the specified partition + pub fn partition_chunk_summaries( + &self, + partition_key: &str, + ) -> impl Iterator { + self.mutable_buffer_chunks(&partition_key) + .into_iter() + .chain(self.read_buffer_chunks(&partition_key).into_iter()) + .map(|c| c.summary()) + } + + /// Returns the number of iterations of the background worker loop + pub fn worker_iterations(&self) -> usize { + self.worker_iterations.load(Ordering::Relaxed) + } + + /// Background worker function + pub async fn background_worker(&self, shutdown: tokio_util::sync::CancellationToken) { + info!("started background worker"); + + let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(1)); + + while !shutdown.is_cancelled() { + self.worker_iterations.fetch_add(1, Ordering::Relaxed); + + tokio::select! { + _ = interval.tick() => {}, + _ = shutdown.cancelled() => break + } + } + + info!("finished background worker"); + } } impl PartialEq for Db { @@ -313,25 +393,36 @@ impl Database for Db { .partition_keys() .context(MutableBufferRead) } + + fn chunk_summaries(&self) -> Result> { + let summaries = self + .partition_keys()? + .into_iter() + .map(|partition_key| self.partition_chunk_summaries(&partition_key)) + .flatten() + .collect(); + Ok(summaries) + } } #[cfg(test)] mod tests { - use crate::query_tests::utils::make_db; - - use super::*; - use arrow_deps::{ arrow::record_batch::RecordBatch, assert_table_eq, datafusion::physical_plan::collect, }; - use data_types::database_rules::{ - MutableBufferConfig, Order, PartitionSort, PartitionSortRules, + use data_types::{ + chunk::ChunkStorage, + database_rules::{MutableBufferConfig, Order, PartitionSort, PartitionSortRules}, }; use query::{ exec::Executor, frontend::sql::SQLQueryPlanner, test::TestLPWriter, PartitionChunk, }; use test_helpers::assert_contains; + use crate::query_tests::utils::make_db; + + use super::*; + #[tokio::test] async fn write_no_mutable_buffer() { // Validate that writes are rejected if there is no mutable buffer @@ -409,6 +500,28 @@ mod tests { assert_table_eq!(&expected, &batches); } + #[tokio::test] + async fn no_load_open_chunk() { + // Test that data can not be loaded into the ReadBuffer while + // still open (no way to ensure that new data gets into the + // read buffer) + let db = make_db(); + let mut writer = TestLPWriter::default(); + writer.write_lp_string(&db, "cpu bar=1 10").await.unwrap(); + + let partition_key = "1970-01-01T00"; + let err = db + .load_chunk_to_read_buffer(partition_key, 0) + .await + .unwrap_err(); + + // it should be the same chunk! + assert_contains!( + err.to_string(), + "Only closed chunks can be moved to read buffer. Chunk 1970-01-01T00 0 was open" + ); + } + #[tokio::test] async fn read_from_read_buffer() { // Test that data can be loaded into the ReadBuffer @@ -510,6 +623,7 @@ mod tests { buffer_size: 300, ..Default::default() }; + let rules = DatabaseRules { mutable_buffer_config: Some(mbconf.clone()), ..Default::default() @@ -520,6 +634,7 @@ mod tests { Some(MutableBufferDb::new("foo")), read_buffer::Database::new(), None, // wal buffer + Arc::new(JobRegistry::new()), ); let mut writer = TestLPWriter::default(); @@ -559,6 +674,116 @@ mod tests { db.rules.mutable_buffer_config = Some(mbconf); } + #[tokio::test] + async fn partition_chunk_summaries() { + // Test that chunk id listing is hooked up + let db = make_db(); + let mut writer = TestLPWriter::default(); + + writer.write_lp_string(&db, "cpu bar=1 1").await.unwrap(); + db.rollover_partition("1970-01-01T00").await.unwrap(); + + // write into a separate partitiion + writer + .write_lp_string(&db, "cpu bar=1,baz2,frob=3 400000000000000") + .await + .unwrap(); + + print!("Partitions: {:?}", db.partition_keys().unwrap()); + + fn to_arc(s: &str) -> Arc { + Arc::new(s.to_string()) + } + + let mut chunk_summaries = db + .partition_chunk_summaries("1970-01-05T15") + .collect::>(); + + chunk_summaries.sort_unstable(); + + let expected = vec![ChunkSummary { + partition_key: to_arc("1970-01-05T15"), + id: 0, + storage: ChunkStorage::OpenMutableBuffer, + estimated_bytes: 107, + }]; + + assert_eq!( + expected, chunk_summaries, + "expected:\n{:#?}\n\nactual:{:#?}\n\n", + expected, chunk_summaries + ); + } + + #[tokio::test] + async fn chunk_summaries() { + // Test that chunk id listing is hooked up + let db = make_db(); + let mut writer = TestLPWriter::default(); + + // get three chunks: one open, one closed in mb and one close in rb + + writer.write_lp_string(&db, "cpu bar=1 1").await.unwrap(); + db.rollover_partition("1970-01-01T00").await.unwrap(); + + writer + .write_lp_string(&db, "cpu bar=1,baz=2 2") + .await + .unwrap(); + + // a fourth chunk in a different partition + writer + .write_lp_string(&db, "cpu bar=1,baz2,frob=3 400000000000000") + .await + .unwrap(); + + print!("Partitions: {:?}", db.partition_keys().unwrap()); + + db.load_chunk_to_read_buffer("1970-01-01T00", 0) + .await + .unwrap(); + + fn to_arc(s: &str) -> Arc { + Arc::new(s.to_string()) + } + + let mut chunk_summaries = db.chunk_summaries().expect("expected summary to return"); + chunk_summaries.sort_unstable(); + + let expected = vec![ + ChunkSummary { + partition_key: to_arc("1970-01-01T00"), + id: 0, + storage: ChunkStorage::ClosedMutableBuffer, + estimated_bytes: 70, + }, + ChunkSummary { + partition_key: to_arc("1970-01-01T00"), + id: 0, + storage: ChunkStorage::ReadBuffer, + estimated_bytes: 1221, + }, + ChunkSummary { + partition_key: to_arc("1970-01-01T00"), + id: 1, + storage: ChunkStorage::OpenMutableBuffer, + estimated_bytes: 101, + }, + ChunkSummary { + partition_key: to_arc("1970-01-05T15"), + id: 0, + storage: ChunkStorage::OpenMutableBuffer, + estimated_bytes: 107, + }, + ]; + + assert_eq!( + expected, chunk_summaries, + "expected:\n{:#?}\n\nactual:{:#?}\n\n", + expected, chunk_summaries + ); + } + // run a sql query against the database, returning the results as record batches async fn run_query(db: &Db, query: &str) -> Vec { let planner = SQLQueryPlanner::default(); diff --git a/server/src/db/chunk.rs b/server/src/db/chunk.rs index 185f033d32..be1cf8747d 100644 --- a/server/src/db/chunk.rs +++ b/server/src/db/chunk.rs @@ -1,5 +1,6 @@ use arrow_deps::datafusion::physical_plan::SendableRecordBatchStream; -use data_types::{schema::Schema, selection::Selection}; +use data_types::chunk::{ChunkStorage, ChunkSummary}; +use internal_types::{schema::Schema, selection::Selection}; use mutable_buffer::chunk::Chunk as MBChunk; use query::{exec::stringset::StringSet, predicate::Predicate, PartitionChunk}; use read_buffer::Database as ReadBufferDb; @@ -29,7 +30,9 @@ pub enum Error { }, #[snafu(display("Internal error restricting schema: {}", source))] - InternalSelectingSchema { source: data_types::schema::Error }, + InternalSelectingSchema { + source: internal_types::schema::Error, + }, #[snafu(display("Predicate conversion error: {}", source))] PredicateConversion { source: super::pred::Error }, @@ -58,10 +61,13 @@ pub type Result = std::result::Result; pub enum DBChunk { MutableBuffer { chunk: Arc, + partition_key: Arc, + /// is this chunk open for writing? + open: bool, }, ReadBuffer { db: Arc, - partition_key: String, + partition_key: Arc, chunk_id: u32, }, ParquetFile, // TODO add appropriate type here @@ -69,8 +75,17 @@ pub enum DBChunk { impl DBChunk { /// Create a new mutable buffer chunk - pub fn new_mb(chunk: Arc) -> Arc { - Arc::new(Self::MutableBuffer { chunk }) + pub fn new_mb( + chunk: Arc, + partition_key: impl Into, + open: bool, + ) -> Arc { + let partition_key = Arc::new(partition_key.into()); + Arc::new(Self::MutableBuffer { + chunk, + partition_key, + open, + }) } /// create a new read buffer chunk @@ -79,13 +94,54 @@ impl DBChunk { partition_key: impl Into, chunk_id: u32, ) -> Arc { - let partition_key = partition_key.into(); + let partition_key = Arc::new(partition_key.into()); Arc::new(Self::ReadBuffer { db, chunk_id, partition_key, }) } + + pub fn summary(&self) -> ChunkSummary { + match self { + Self::MutableBuffer { + chunk, + partition_key, + open, + } => { + let storage = if *open { + ChunkStorage::OpenMutableBuffer + } else { + ChunkStorage::ClosedMutableBuffer + }; + ChunkSummary { + partition_key: Arc::clone(partition_key), + id: chunk.id(), + storage, + estimated_bytes: chunk.size(), + } + } + Self::ReadBuffer { + db, + partition_key, + chunk_id, + } => { + let estimated_bytes = db + .chunks_size(partition_key.as_ref(), &[*chunk_id]) + .unwrap_or(0) as usize; + + ChunkSummary { + partition_key: Arc::clone(&partition_key), + id: *chunk_id, + storage: ChunkStorage::ReadBuffer, + estimated_bytes, + } + } + Self::ParquetFile => { + unimplemented!("parquet file summary not implemented") + } + } + } } #[async_trait] @@ -94,7 +150,7 @@ impl PartitionChunk for DBChunk { fn id(&self) -> u32 { match self { - Self::MutableBuffer { chunk } => chunk.id(), + Self::MutableBuffer { chunk, .. } => chunk.id(), Self::ReadBuffer { chunk_id, .. } => *chunk_id, Self::ParquetFile => unimplemented!("parquet file not implemented"), } @@ -104,7 +160,7 @@ impl PartitionChunk for DBChunk { &self, ) -> Result, Self::Error> { match self { - Self::MutableBuffer { chunk } => chunk.table_stats().context(MutableBufferChunk), + Self::MutableBuffer { chunk, .. } => chunk.table_stats().context(MutableBufferChunk), Self::ReadBuffer { .. } => unimplemented!("read buffer not implemented"), Self::ParquetFile => unimplemented!("parquet file not implemented"), } @@ -116,7 +172,7 @@ impl PartitionChunk for DBChunk { _known_tables: &StringSet, ) -> Result, Self::Error> { let names = match self { - Self::MutableBuffer { chunk } => { + Self::MutableBuffer { chunk, .. } => { if chunk.is_empty() { Some(StringSet::new()) } else { @@ -192,10 +248,10 @@ impl PartitionChunk for DBChunk { selection: Selection<'_>, ) -> Result { match self { - DBChunk::MutableBuffer { chunk } => chunk + Self::MutableBuffer { chunk, .. } => chunk .table_schema(table_name, selection) .context(MutableBufferChunk), - DBChunk::ReadBuffer { + Self::ReadBuffer { db, partition_key, chunk_id, @@ -227,7 +283,7 @@ impl PartitionChunk for DBChunk { Ok(schema) } - DBChunk::ParquetFile => { + Self::ParquetFile => { unimplemented!("parquet file not implemented for table schema") } } @@ -235,7 +291,7 @@ impl PartitionChunk for DBChunk { fn has_table(&self, table_name: &str) -> bool { match self { - Self::MutableBuffer { chunk } => chunk.has_table(table_name), + Self::MutableBuffer { chunk, .. } => chunk.has_table(table_name), Self::ReadBuffer { db, partition_key, @@ -257,7 +313,7 @@ impl PartitionChunk for DBChunk { selection: Selection<'_>, ) -> Result { match self { - Self::MutableBuffer { chunk } => { + Self::MutableBuffer { chunk, .. } => { // Note MutableBuffer doesn't support predicate // pushdown (other than pruning out the entire chunk // via `might_pass_predicate) @@ -341,7 +397,7 @@ impl PartitionChunk for DBChunk { columns: Selection<'_>, ) -> Result, Self::Error> { match self { - Self::MutableBuffer { chunk } => { + Self::MutableBuffer { chunk, .. } => { let chunk_predicate = match to_mutable_buffer_predicate(chunk, predicate) { Ok(chunk_predicate) => chunk_predicate, Err(e) => { @@ -389,7 +445,7 @@ impl PartitionChunk for DBChunk { predicate: &Predicate, ) -> Result, Self::Error> { match self { - Self::MutableBuffer { chunk } => { + Self::MutableBuffer { chunk, .. } => { use mutable_buffer::chunk::Error::UnsupportedColumnTypeForListingValues; let chunk_predicate = match to_mutable_buffer_predicate(chunk, predicate) { diff --git a/server/src/db/streams.rs b/server/src/db/streams.rs index c255bed513..5ea7d8cd17 100644 --- a/server/src/db/streams.rs +++ b/server/src/db/streams.rs @@ -8,7 +8,7 @@ use arrow_deps::{ }, datafusion::physical_plan::RecordBatchStream, }; -use data_types::selection::Selection; +use internal_types::selection::Selection; use mutable_buffer::chunk::Chunk as MBChunk; use read_buffer::ReadFilterResults; diff --git a/server/src/lib.rs b/server/src/lib.rs index 33ca685920..18d0970024 100644 --- a/server/src/lib.rs +++ b/server/src/lib.rs @@ -67,40 +67,48 @@ clippy::clone_on_ref_ptr )] -pub mod buffer; -mod config; -pub mod db; -pub mod snapshot; -mod tracker; - -#[cfg(test)] -mod query_tests; - use std::sync::{ atomic::{AtomicU32, Ordering}, Arc, }; -use crate::{ - buffer::SegmentPersistenceTask, - config::{object_store_path_for_database_config, Config, DB_RULES_FILE_NAME}, - db::Db, - tracker::TrackerRegistry, -}; -use data_types::{ - data::{lines_to_replicated_write, ReplicatedWrite}, - database_rules::{DatabaseRules, HostGroup, HostGroupId, MatchTables}, - {DatabaseName, DatabaseNameError}, -}; -use influxdb_line_protocol::ParsedLine; -use object_store::{path::ObjectStorePath, ObjectStore, ObjectStoreApi}; -use query::{exec::Executor, DatabaseStore}; - use async_trait::async_trait; use bytes::Bytes; use futures::stream::TryStreamExt; +use parking_lot::Mutex; use snafu::{OptionExt, ResultExt, Snafu}; -use tracing::error; +use tracing::{debug, error, info, warn}; + +use data_types::{ + database_rules::{DatabaseRules, WriterId}, + job::Job, + {DatabaseName, DatabaseNameError}, +}; +use influxdb_line_protocol::ParsedLine; +use internal_types::data::{lines_to_replicated_write, ReplicatedWrite}; +use object_store::{path::ObjectStorePath, ObjectStore, ObjectStoreApi}; +use query::{exec::Executor, DatabaseStore}; + +use futures::{pin_mut, FutureExt}; + +use crate::{ + config::{ + object_store_path_for_database_config, Config, GRPCConnectionString, DB_RULES_FILE_NAME, + }, + db::Db, + tracker::{ + TrackedFutureExt, Tracker, TrackerId, TrackerRegistration, TrackerRegistryWithHistory, + }, +}; + +pub mod buffer; +mod config; +pub mod db; +pub mod snapshot; +pub mod tracker; + +#[cfg(test)] +mod query_tests; type DatabaseError = Box; @@ -117,12 +125,10 @@ pub enum Error { InvalidDatabaseName { source: DatabaseNameError }, #[snafu(display("database error: {}", source))] UnknownDatabaseError { source: DatabaseError }, + #[snafu(display("getting mutable buffer chunk: {}", source))] + MutableBufferChunk { source: DatabaseError }, #[snafu(display("no local buffer for database: {}", db))] NoLocalBuffer { db: String }, - #[snafu(display("host group not found: {}", id))] - HostGroupNotFound { id: HostGroupId }, - #[snafu(display("no hosts in group: {}", id))] - NoHostInGroup { id: HostGroupId }, #[snafu(display("unable to get connection to remote server: {}", server))] UnableToGetConnection { server: String, @@ -146,6 +152,32 @@ pub enum Error { pub type Result = std::result::Result; +const JOB_HISTORY_SIZE: usize = 1000; + +/// The global job registry +#[derive(Debug)] +pub struct JobRegistry { + inner: Mutex>, +} + +impl Default for JobRegistry { + fn default() -> Self { + Self { + inner: Mutex::new(TrackerRegistryWithHistory::new(JOB_HISTORY_SIZE)), + } + } +} + +impl JobRegistry { + fn new() -> Self { + Default::default() + } + + pub fn register(&self, job: Job) -> (Tracker, TrackerRegistration) { + self.inner.lock().register(job) + } +} + const STORE_ERROR_PAUSE_SECONDS: u64 = 100; /// `Server` is the container struct for how servers store data internally, as @@ -158,18 +190,20 @@ pub struct Server { connection_manager: Arc, pub store: Arc, executor: Arc, - segment_persistence_registry: TrackerRegistry, + jobs: Arc, } impl Server { pub fn new(connection_manager: M, store: Arc) -> Self { + let jobs = Arc::new(JobRegistry::new()); + Self { id: AtomicU32::new(SERVER_ID_NOT_SET), - config: Arc::new(Config::default()), + config: Arc::new(Config::new(Arc::clone(&jobs))), store, connection_manager: Arc::new(connection_manager), executor: Arc::new(Executor::new()), - segment_persistence_registry: TrackerRegistry::new(), + jobs, } } @@ -294,18 +328,6 @@ impl Server { Ok(()) } - /// Creates a host group with a set of connection strings to hosts. These - /// host connection strings should be something that the connection - /// manager can use to return a remote server to work with. - pub async fn create_host_group(&mut self, id: HostGroupId, hosts: Vec) -> Result<()> { - // Return an error if this server hasn't yet been setup with an id - self.require_id()?; - - self.config.create_host_group(HostGroup { id, hosts }); - - Ok(()) - } - /// `write_lines` takes in raw line protocol and converts it to a /// `ReplicatedWrite`, which is then replicated to other servers based /// on the configuration of the `db`. This is step #1 from the crate @@ -361,81 +383,172 @@ impl Server { if persist { let writer_id = self.require_id()?; let store = Arc::clone(&self.store); + + let (_, tracker) = self.jobs.register(Job::PersistSegment { + writer_id, + segment_id: segment.id, + }); + segment - .persist_bytes_in_background( - &self.segment_persistence_registry, - writer_id, - db_name, - store, - ) + .persist_bytes_in_background(tracker, writer_id, db_name, store) .context(WalError)?; } } } - for host_group_id in &db.rules.replication { - self.replicate_to_host_group(host_group_id, db_name, &write) - .await?; - } - - for subscription in &db.rules.subscriptions { - match subscription.matcher.tables { - MatchTables::All => { - self.replicate_to_host_group(&subscription.host_group_id, db_name, &write) - .await? - } - MatchTables::Table(_) => unimplemented!(), - MatchTables::Regex(_) => unimplemented!(), - } - } - Ok(()) } - // replicates to a single host in the group based on hashing rules. If that host - // is unavailable an error will be returned. The request may still succeed - // if enough of the other host groups have returned a success. - async fn replicate_to_host_group( - &self, - host_group_id: &str, - db_name: &DatabaseName<'_>, - write: &ReplicatedWrite, - ) -> Result<()> { - let group = self - .config - .host_group(host_group_id) - .context(HostGroupNotFound { id: host_group_id })?; - - // TODO: handle hashing rules to determine which host in the group should get - // the write. for now, just write to the first one. - let host = group - .hosts - .get(0) - .context(NoHostInGroup { id: host_group_id })?; - - let connection = self - .connection_manager - .remote_server(host) - .await - .map_err(|e| Box::new(e) as DatabaseError) - .context(UnableToGetConnection { server: host })?; - - connection - .replicate(db_name, write) - .await - .map_err(|e| Box::new(e) as DatabaseError) - .context(ErrorReplicating {})?; - - Ok(()) - } - - pub async fn db(&self, name: &DatabaseName<'_>) -> Option> { + pub fn db(&self, name: &DatabaseName<'_>) -> Option> { self.config.db(name) } - pub async fn db_rules(&self, name: &DatabaseName<'_>) -> Option { + pub fn db_rules(&self, name: &DatabaseName<'_>) -> Option { self.config.db(name).map(|d| d.rules.clone()) } + + pub fn remotes_sorted(&self) -> Vec<(WriterId, String)> { + self.config.remotes_sorted() + } + + pub fn update_remote(&self, id: WriterId, addr: GRPCConnectionString) { + self.config.update_remote(id, addr) + } + + pub fn delete_remote(&self, id: WriterId) -> Option { + self.config.delete_remote(id) + } + + pub fn spawn_dummy_job(&self, nanos: Vec) -> Tracker { + let (tracker, registration) = self.jobs.register(Job::Dummy { + nanos: nanos.clone(), + }); + + for duration in nanos { + tokio::spawn( + tokio::time::sleep(tokio::time::Duration::from_nanos(duration)) + .track(registration.clone()), + ); + } + + tracker + } + + /// Closes a chunk and starts moving its data to the read buffer, as a + /// background job, dropping when complete. + pub fn close_chunk( + &self, + db_name: DatabaseName<'_>, + partition_key: impl Into, + chunk_id: u32, + ) -> Result> { + let db_name = db_name.to_string(); + let name = DatabaseName::new(&db_name).context(InvalidDatabaseName)?; + + let partition_key = partition_key.into(); + + let db = self + .config + .db(&name) + .context(DatabaseNotFound { db_name: &db_name })?; + + let (tracker, registration) = self.jobs.register(Job::CloseChunk { + db_name: db_name.clone(), + partition_key: partition_key.clone(), + chunk_id, + }); + + let task = async move { + // Close the chunk if it isn't already closed + if db.is_open_chunk(&partition_key, chunk_id) { + debug!(%db_name, %partition_key, %chunk_id, "Rolling over partition to close chunk"); + let result = db.rollover_partition(&partition_key).await; + + if let Err(e) = result { + info!(?e, %db_name, %partition_key, %chunk_id, "background task error during chunk closing"); + return Err(e); + } + } + + debug!(%db_name, %partition_key, %chunk_id, "background task loading chunk to read buffer"); + let result = db.load_chunk_to_read_buffer(&partition_key, chunk_id).await; + if let Err(e) = result { + info!(?e, %db_name, %partition_key, %chunk_id, "background task error loading read buffer chunk"); + return Err(e); + } + + // now, drop the chunk + debug!(%db_name, %partition_key, %chunk_id, "background task dropping mutable buffer chunk"); + let result = db.drop_mutable_buffer_chunk(&partition_key, chunk_id).await; + if let Err(e) = result { + info!(?e, %db_name, %partition_key, %chunk_id, "background task error loading read buffer chunk"); + return Err(e); + } + + debug!(%db_name, %partition_key, %chunk_id, "background task completed closing chunk"); + + Ok(()) + }; + + tokio::spawn(task.track(registration)); + + Ok(tracker) + } + + /// Returns a list of all jobs tracked by this server + pub fn tracked_jobs(&self) -> Vec> { + self.jobs.inner.lock().tracked() + } + + /// Returns a specific job tracked by this server + pub fn get_job(&self, id: TrackerId) -> Option> { + self.jobs.inner.lock().get(id) + } + + /// Background worker function for the server + pub async fn background_worker(&self, shutdown: tokio_util::sync::CancellationToken) { + info!("started background worker"); + + let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(1)); + + while !shutdown.is_cancelled() { + self.jobs.inner.lock().reclaim(); + + tokio::select! { + _ = interval.tick() => {}, + _ = shutdown.cancelled() => break + } + } + + info!("shutting down background worker"); + + let join = self.config.drain().fuse(); + pin_mut!(join); + + // Keep running reclaim whilst shutting down in case something + // is waiting on a tracker to complete + loop { + self.jobs.inner.lock().reclaim(); + + futures::select! { + _ = interval.tick().fuse() => {}, + _ = join => break + } + } + + info!("draining tracker registry"); + + // Wait for any outstanding jobs to finish - frontend shutdown should be + // sequenced before shutting down the background workers and so there + // shouldn't be any + while self.jobs.inner.lock().tracked_len() != 0 { + self.jobs.inner.lock().reclaim(); + + interval.tick().await; + } + + info!("drained tracker registry"); + } } #[async_trait] @@ -446,7 +559,7 @@ where type Database = Db; type Error = Error; - async fn db_names_sorted(&self) -> Vec { + fn db_names_sorted(&self) -> Vec { self.config .db_names_sorted() .iter() @@ -454,9 +567,9 @@ where .collect() } - async fn db(&self, name: &str) -> Option> { + fn db(&self, name: &str) -> Option> { if let Ok(name) = DatabaseName::new(name) { - return self.db(&name).await; + return self.db(&name); } None @@ -467,11 +580,11 @@ where async fn db_or_create(&self, name: &str) -> Result, Self::Error> { let db_name = DatabaseName::new(name.to_string()).context(InvalidDatabaseName)?; - let db = match self.db(&db_name).await { + let db = match self.db(&db_name) { Some(db) => db, None => { self.create_database(name, DatabaseRules::new()).await?; - self.db(&db_name).await.expect("db not inserted") + self.db(&db_name).expect("db not inserted") } }; @@ -562,21 +675,26 @@ async fn get_store_bytes( #[cfg(test)] mod tests { - use super::*; - use crate::buffer::Segment; - use arrow_deps::{assert_table_eq, datafusion::physical_plan::collect}; + use std::collections::BTreeMap; + use async_trait::async_trait; - use data_types::database_rules::{ - MatchTables, Matcher, PartitionTemplate, Subscription, TemplatePart, WalBufferConfig, - WalBufferRollover, - }; use futures::TryStreamExt; + use parking_lot::Mutex; + use snafu::Snafu; + use tokio::task::JoinHandle; + use tokio_util::sync::CancellationToken; + + use arrow_deps::{assert_table_eq, datafusion::physical_plan::collect}; + use data_types::database_rules::{ + PartitionTemplate, TemplatePart, WalBufferConfig, WalBufferRollover, + }; use influxdb_line_protocol::parse_lines; use object_store::{memory::InMemory, path::ObjectStorePath}; - use parking_lot::Mutex; - use query::frontend::sql::SQLQueryPlanner; - use snafu::Snafu; - use std::collections::BTreeMap; + use query::{frontend::sql::SQLQueryPlanner, Database}; + + use crate::buffer::Segment; + + use super::*; type TestError = Box; type Result = std::result::Result; @@ -585,7 +703,7 @@ mod tests { async fn server_api_calls_return_error_with_no_id_set() -> Result { let manager = TestConnectionManager::new(); let store = Arc::new(ObjectStore::new_in_memory(InMemory::new())); - let mut server = Server::new(manager, store); + let server = Server::new(manager, store); let rules = DatabaseRules::new(); let resp = server.create_database("foo", rules).await.unwrap_err(); @@ -595,12 +713,6 @@ mod tests { let resp = server.write_lines("foo", &lines).await.unwrap_err(); assert!(matches!(resp, Error::IdNotSet)); - let resp = server - .create_host_group("group1".to_string(), vec!["serverA".to_string()]) - .await - .unwrap_err(); - assert!(matches!(resp, Error::IdNotSet)); - Ok(()) } @@ -659,8 +771,8 @@ mod tests { server2.set_id(1); server2.load_database_configs().await.unwrap(); - let _ = server2.db(&DatabaseName::new(db2).unwrap()).await.unwrap(); - let _ = server2.db(&DatabaseName::new(name).unwrap()).await.unwrap(); + let _ = server2.db(&DatabaseName::new(db2).unwrap()).unwrap(); + let _ = server2.db(&DatabaseName::new(name).unwrap()).unwrap(); } #[tokio::test] @@ -709,7 +821,7 @@ mod tests { .expect("failed to create database"); } - let db_names_sorted = server.db_names_sorted().await; + let db_names_sorted = server.db_names_sorted(); assert_eq!(names, db_names_sorted); Ok(()) @@ -754,7 +866,7 @@ mod tests { server.write_lines("foo", &lines).await.unwrap(); let db_name = DatabaseName::new("foo").unwrap(); - let db = server.db(&db_name).await.unwrap(); + let db = server.db(&db_name).unwrap(); let planner = SQLQueryPlanner::default(); let executor = server.executor(); @@ -777,125 +889,65 @@ mod tests { } #[tokio::test] - async fn replicate_to_single_group() -> Result { - let mut manager = TestConnectionManager::new(); - let remote = Arc::new(TestRemoteServer::default()); - let remote_id = "serverA"; - manager - .remotes - .insert(remote_id.to_string(), Arc::clone(&remote)); - + async fn close_chunk() -> Result { + test_helpers::maybe_start_logging(); + let manager = TestConnectionManager::new(); let store = Arc::new(ObjectStore::new_in_memory(InMemory::new())); + let server = Arc::new(Server::new(manager, store)); + + let cancel_token = CancellationToken::new(); + let background_handle = spawn_worker(Arc::clone(&server), cancel_token.clone()); - let mut server = Server::new(manager, store); server.set_id(1); - let host_group_id = "az1".to_string(); - let rules = DatabaseRules { - replication: vec![host_group_id.clone()], - replication_count: 1, - ..Default::default() - }; + + let db_name = DatabaseName::new("foo").unwrap(); server - .create_host_group(host_group_id.clone(), vec![remote_id.to_string()]) - .await - .unwrap(); - let db_name = "foo"; - server.create_database(db_name, rules).await.unwrap(); + .create_database(db_name.as_str(), DatabaseRules::new()) + .await?; - let lines = parsed_lines("cpu bar=1 10"); - server.write_lines("foo", &lines).await.unwrap(); + let line = "cpu bar=1 10"; + let lines: Vec<_> = parse_lines(line).map(|l| l.unwrap()).collect(); + server.write_lines(&db_name, &lines).await.unwrap(); - let writes = remote.writes.lock().get(db_name).unwrap().clone(); + // start the close (note this is not an async) + let partition_key = ""; + let db_name_string = db_name.to_string(); + let tracker = server.close_chunk(db_name, partition_key, 0).unwrap(); - let write_text = r#" -writer:1, sequence:1, checksum:226387645 -partition_key: - table:cpu - bar:1 time:10 -"#; - - assert_eq!(write_text, writes[0].to_string()); - - // ensure sequence number goes up - let lines = parsed_lines("mem,server=A,region=west user=232 12"); - server.write_lines("foo", &lines).await.unwrap(); - - let writes = remote.writes.lock().get(db_name).unwrap().clone(); - assert_eq!(2, writes.len()); - - let write_text = r#" -writer:1, sequence:2, checksum:3759030699 -partition_key: - table:mem - server:A region:west user:232 time:12 -"#; - - assert_eq!(write_text, writes[1].to_string()); - - Ok(()) - } - - #[tokio::test] - async fn sends_all_to_subscriber() -> Result { - let mut manager = TestConnectionManager::new(); - let remote = Arc::new(TestRemoteServer::default()); - let remote_id = "serverA"; - manager - .remotes - .insert(remote_id.to_string(), Arc::clone(&remote)); - - let store = Arc::new(ObjectStore::new_in_memory(InMemory::new())); - - let mut server = Server::new(manager, store); - server.set_id(1); - let host_group_id = "az1".to_string(); - let rules = DatabaseRules { - subscriptions: vec![Subscription { - name: "query_server_1".to_string(), - host_group_id: host_group_id.clone(), - matcher: Matcher { - tables: MatchTables::All, - predicate: None, - }, - }], - ..Default::default() + let metadata = tracker.metadata(); + let expected_metadata = Job::CloseChunk { + db_name: db_name_string, + partition_key: partition_key.to_string(), + chunk_id: 0, }; - server - .create_host_group(host_group_id.clone(), vec![remote_id.to_string()]) - .await - .unwrap(); - let db_name = "foo"; - server.create_database(db_name, rules).await.unwrap(); + assert_eq!(metadata, &expected_metadata); - let lines = parsed_lines("cpu bar=1 10"); - server.write_lines("foo", &lines).await.unwrap(); + // wait for the job to complete + tracker.join().await; - let writes = remote.writes.lock().get(db_name).unwrap().clone(); + // Data should be in the read buffer and not in mutable buffer + let db_name = DatabaseName::new("foo").unwrap(); + let db = server.db(&db_name).unwrap(); - let write_text = r#" -writer:1, sequence:1, checksum:226387645 -partition_key: - table:cpu - bar:1 time:10 -"#; + let mut chunk_summaries = db.chunk_summaries().unwrap(); + chunk_summaries.sort_unstable(); - assert_eq!(write_text, writes[0].to_string()); + let actual = chunk_summaries + .into_iter() + .map(|s| format!("{:?} {}", s.storage, s.id)) + .collect::>(); - // ensure sequence number goes up - let lines = parsed_lines("mem,server=A,region=west user=232 12"); - server.write_lines("foo", &lines).await.unwrap(); + let expected = vec!["ReadBuffer 0", "OpenMutableBuffer 1"]; - let writes = remote.writes.lock().get(db_name).unwrap().clone(); - assert_eq!(2, writes.len()); + assert_eq!( + expected, actual, + "expected:\n{:#?}\n\nactual:{:#?}\n\n", + expected, actual + ); - let write_text = r#" -writer:1, sequence:2, checksum:3759030699 -partition_key: - table:mem - server:A region:west user:232 time:12 -"#; - - assert_eq!(write_text, writes[1].to_string()); + // ensure that we don't leave the server instance hanging around + cancel_token.cancel(); + let _ = background_handle.await; Ok(()) } @@ -950,6 +1002,30 @@ partition_key: assert_eq!(segment.writes[0].to_string(), write); } + #[tokio::test] + async fn background_task_cleans_jobs() -> Result { + let manager = TestConnectionManager::new(); + let store = Arc::new(ObjectStore::new_in_memory(InMemory::new())); + let server = Arc::new(Server::new(manager, store)); + + let cancel_token = CancellationToken::new(); + let background_handle = spawn_worker(Arc::clone(&server), cancel_token.clone()); + + let wait_nanos = 1000; + let job = server.spawn_dummy_job(vec![wait_nanos]); + + // Note: this will hang forever if the background task has not been started + job.join().await; + + assert!(job.is_complete()); + + // ensure that we don't leave the server instance hanging around + cancel_token.cancel(); + let _ = background_handle.await; + + Ok(()) + } + #[derive(Snafu, Debug, Clone)] enum TestClusterError { #[snafu(display("Test cluster error: {}", message))] @@ -1004,4 +1080,11 @@ partition_key: fn parsed_lines(lp: &str) -> Vec> { parse_lines(lp).map(|l| l.unwrap()).collect() } + + fn spawn_worker(server: Arc>, token: CancellationToken) -> JoinHandle<()> + where + M: ConnectionManager + Send + Sync + 'static, + { + tokio::task::spawn(async move { server.background_worker(token).await }) + } } diff --git a/server/src/query_tests/scenarios.rs b/server/src/query_tests/scenarios.rs index ca4bd9a868..427621606b 100644 --- a/server/src/query_tests/scenarios.rs +++ b/server/src/query_tests/scenarios.rs @@ -84,6 +84,22 @@ impl DBSetup for TwoMeasurements { } } +pub struct TwoMeasurementsUnsignedType {} +#[async_trait] +impl DBSetup for TwoMeasurementsUnsignedType { + async fn make(&self) -> Vec { + let partition_key = "1970-01-01T00"; + let lp_lines = vec![ + "restaurant,town=andover count=40000u 100", + "restaurant,town=reading count=632u 120", + "school,town=reading count=17u 150", + "school,town=andover count=25u 160", + ]; + + make_one_chunk_scenarios(partition_key, &lp_lines.join("\n")).await + } +} + /// Single measurement that has several different chunks with /// different (but compatible) schema pub struct MultiChunkSchemaMerge {} diff --git a/server/src/query_tests/sql.rs b/server/src/query_tests/sql.rs index 727853dca4..4c247a2313 100644 --- a/server/src/query_tests/sql.rs +++ b/server/src/query_tests/sql.rs @@ -143,6 +143,40 @@ async fn sql_select_with_schema_merge() { run_sql_test_case!(MultiChunkSchemaMerge {}, "SELECT * from cpu", &expected); } +#[tokio::test] +async fn sql_select_from_restaurant() { + let expected = vec![ + "+---------+-------+", + "| town | count |", + "+---------+-------+", + "| andover | 40000 |", + "| reading | 632 |", + "+---------+-------+", + ]; + run_sql_test_case!( + TwoMeasurementsUnsignedType {}, + "SELECT town, count from restaurant", + &expected + ); +} + +#[tokio::test] +async fn sql_select_from_school() { + let expected = vec![ + "+---------+-------+", + "| town | count |", + "+---------+-------+", + "| reading | 17 |", + "| andover | 25 |", + "+---------+-------+", + ]; + run_sql_test_case!( + TwoMeasurementsUnsignedType {}, + "SELECT town, count from school", + &expected + ); +} + #[tokio::test] async fn sql_select_with_schema_merge_subset() { let expected = vec![ diff --git a/server/src/query_tests/table_schema.rs b/server/src/query_tests/table_schema.rs index 9aceb80aa5..7e68ff4eb5 100644 --- a/server/src/query_tests/table_schema.rs +++ b/server/src/query_tests/table_schema.rs @@ -1,7 +1,7 @@ //! Tests for the table_names implementation use arrow_deps::arrow::datatypes::DataType; -use data_types::{schema::builder::SchemaBuilder, selection::Selection}; +use internal_types::{schema::builder::SchemaBuilder, selection::Selection}; use query::{Database, PartitionChunk}; use super::scenarios::*; @@ -113,3 +113,21 @@ async fn list_schema_disk_selection() { run_table_schema_test_case!(TwoMeasurements {}, selection, "disk", expected_schema); } + +#[tokio::test] +async fn list_schema_location_all() { + // we expect columns to come out in lexographic order by name + let expected_schema = SchemaBuilder::new() + .field("count", DataType::UInt64) + .timestamp() + .tag("town") + .build() + .unwrap(); + + run_table_schema_test_case!( + TwoMeasurementsUnsignedType {}, + Selection::All, + "restaurant", + expected_schema + ); +} diff --git a/server/src/query_tests/utils.rs b/server/src/query_tests/utils.rs index e8ff457b8c..f55cd01463 100644 --- a/server/src/query_tests/utils.rs +++ b/server/src/query_tests/utils.rs @@ -1,7 +1,8 @@ use data_types::database_rules::DatabaseRules; use mutable_buffer::MutableBufferDb; -use crate::db::Db; +use crate::{db::Db, JobRegistry}; +use std::sync::Arc; /// Used for testing: create a Database with a local store pub fn make_db() -> Db { @@ -11,5 +12,6 @@ pub fn make_db() -> Db { Some(MutableBufferDb::new(name)), read_buffer::Database::new(), None, // wal buffer + Arc::new(JobRegistry::new()), ) } diff --git a/server/src/snapshot.rs b/server/src/snapshot.rs index 69a7d1726f..76cdc88b71 100644 --- a/server/src/snapshot.rs +++ b/server/src/snapshot.rs @@ -5,10 +5,8 @@ use arrow_deps::{ datafusion::physical_plan::SendableRecordBatchStream, parquet::{self, arrow::ArrowWriter, file::writer::TryClone}, }; -use data_types::{ - partition_metadata::{PartitionSummary, TableSummary}, - selection::Selection, -}; +use data_types::partition_metadata::{PartitionSummary, TableSummary}; +use internal_types::selection::Selection; use object_store::{path::ObjectStorePath, ObjectStore, ObjectStoreApi}; use query::{predicate::EMPTY_PREDICATE, PartitionChunk}; @@ -363,7 +361,10 @@ impl TryClone for MemWriter { #[cfg(test)] mod tests { - use crate::db::{DBChunk, Db}; + use crate::{ + db::{DBChunk, Db}, + JobRegistry, + }; use read_buffer::Database as ReadBufferDb; use super::*; @@ -442,7 +443,7 @@ mem,host=A,region=west used=45 1 ]; let store = Arc::new(ObjectStore::new_in_memory(InMemory::new())); - let chunk = DBChunk::new_mb(Arc::new(ChunkWB::new(11))); + let chunk = DBChunk::new_mb(Arc::new(ChunkWB::new(11)), "key", false); let mut metadata_path = store.new_path(); metadata_path.push_dir("meta"); @@ -482,6 +483,7 @@ mem,host=A,region=west used=45 1 Some(MutableBufferDb::new(name)), ReadBufferDb::new(), None, // wal buffer + Arc::new(JobRegistry::new()), ) } } diff --git a/server/src/tracker.rs b/server/src/tracker.rs index 07e0d0d8db..007f98e511 100644 --- a/server/src/tracker.rs +++ b/server/src/tracker.rs @@ -1,241 +1,637 @@ -use std::collections::HashMap; -use std::future::Future; -use std::pin::Pin; +//! This module contains a future tracking system supporting fanout, +//! cancellation and asynchronous signalling of completion +//! +//! A Tracker is created by calling TrackerRegistry::register. TrackedFutures +//! can then be associated with this Tracker and monitored and/or cancelled. +//! +//! This is used within IOx to track futures spawned as multiple tokio tasks. +//! +//! For example, when migrating a chunk from the mutable buffer to the read +//! buffer: +//! +//! - There is a single over-arching Job being performed +//! - A single tracker is allocated from a TrackerRegistry in Server and +//! associated with the Job metadata +//! - This tracker is registered with every future that is spawned as a tokio +//! task +//! +//! This same system may in future form part of a query tracking story +//! +//! # Correctness +//! +//! The key correctness property of the Tracker system is Tracker::get_status +//! only returns Complete when all futures associated with the tracker have +//! completed and no more can be spawned. Additionally at such a point +//! all metrics - cpu_nanos, wall_nanos, created_futures should be visible +//! to the calling thread +//! +//! Note: there is no guarantee that pending_registrations or pending_futures +//! ever reaches 0, a program could call mem::forget on a TrackerRegistration, +//! leak the TrackerRegistration, spawn a future that never finishes, etc... +//! Such a program would never consider the Tracker complete and therefore this +//! doesn't violate the correctness property +//! +//! ## Proof +//! +//! 1. pending_registrations starts at 1, and is incremented on +//! TrackerRegistration::clone. A TrackerRegistration cannot be created from an +//! existing TrackerState, only another TrackerRegistration +//! +//! 2. pending_registrations is decremented with release semantics on +//! TrackerRegistration::drop +//! +//! 3. pending_futures is only incremented with a TrackerRegistration in scope +//! +//! 4. 2. + 3. -> A thread that increments pending_futures, decrements +//! pending_registrations with release semantics afterwards. By definition of +//! release semantics these writes to pending_futures cannot be reordered to +//! come after the atomic decrement of pending_registrations +//! +//! 5. 1. + 2. + drop cannot be called multiple times on the same object -> once +//! pending_registrations is decremented to 0 it can never be incremented again +//! +//! 6. 4. + 5. -> the decrement to 0 of pending_registrations must commit after +//! the last increment of pending_futures +//! +//! 7. pending_registrations is loaded with acquire semantics +//! +//! 8. By definition of acquire semantics, any thread that reads +//! pending_registrations is guaranteed to see any increments to pending_futures +//! performed before the most recent decrement of pending_registrations +//! +//! 9. 6. + 8. -> A thread that observes a pending_registrations of 0 cannot +//! subsequently observe pending_futures to increase +//! +//! 10. Tracker::get_status returns Complete if it observes +//! pending_registrations to be 0 and then pending_futures to be 0 +//! +//! 11. 9 + 10 -> A thread can only observe a tracker to be complete +//! after all futures have been dropped and no more can be created +//! +//! 12. pending_futures is decremented with Release semantics on +//! TrackedFuture::drop after any associated metrics have been incremented +//! +//! 13. pending_futures is loaded with acquire semantics +//! +//! 14. 12. + 13. -> A thread that observes a pending_futures of 0 is guaranteed +//! to see any metrics from any dropped TrackedFuture +//! +//! Note: this proof ignores the complexity of moving Trackers, TrackedFutures, +//! etc... between threads as any such functionality must perform the necessary +//! synchronisation to be well-formed. + use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; -use std::task::{Context, Poll}; +use std::time::Instant; -use futures::prelude::*; -use parking_lot::Mutex; -use pin_project::{pin_project, pinned_drop}; +use tokio_util::sync::CancellationToken; +use tracing::warn; -/// Every future registered with a `TrackerRegistry` is assigned a unique -/// `TrackerId` -#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)] -pub struct TrackerId(usize); +pub use future::{TrackedFuture, TrackedFutureExt}; +pub use history::TrackerRegistryWithHistory; +pub use registry::{TrackerId, TrackerRegistry}; +mod future; +mod history; +mod registry; + +/// The state shared between all sibling tasks #[derive(Debug)] -struct Tracker { - data: T, - abort: future::AbortHandle, +struct TrackerState { + start_instant: Instant, + cancel_token: CancellationToken, + cpu_nanos: AtomicUsize, + wall_nanos: AtomicUsize, + + created_futures: AtomicUsize, + pending_futures: AtomicUsize, + pending_registrations: AtomicUsize, + + watch: tokio::sync::watch::Receiver, } -#[derive(Debug)] -struct TrackerContextInner { - id: AtomicUsize, - trackers: Mutex>>, +/// The status of the tracker +#[derive(Debug, Clone)] +pub enum TrackerStatus { + /// More futures can be registered + Creating, + + /// No more futures can be registered + /// + /// `pending_count` and `cpu_nanos` are best-effort - + /// they may not be the absolute latest values. + /// + /// `total_count` is guaranteed to be the final value + Running { + /// The number of created futures + total_count: usize, + /// The number of pending futures + pending_count: usize, + /// The total amount of CPU time spent executing the futures + cpu_nanos: usize, + }, + + /// All futures have been dropped and no more can be registered + /// + /// All values are guaranteed to be the final values + Complete { + /// The number of created futures + total_count: usize, + /// The total amount of CPU time spent executing the futures + cpu_nanos: usize, + /// The number of nanoseconds between tracker registration and + /// the last TrackedFuture being dropped + wall_nanos: usize, + }, } -/// Allows tracking the lifecycle of futures registered by -/// `TrackedFutureExt::track` with an accompanying metadata payload of type T -/// -/// Additionally can trigger graceful termination of registered futures +/// A Tracker can be used to monitor/cancel/wait for a set of associated futures #[derive(Debug)] -pub struct TrackerRegistry { - inner: Arc>, +pub struct Tracker { + id: TrackerId, + state: Arc, + metadata: Arc, } -// Manual Clone to workaround https://github.com/rust-lang/rust/issues/26925 -impl Clone for TrackerRegistry { +impl Clone for Tracker { fn clone(&self) -> Self { Self { - inner: Arc::clone(&self.inner), + id: self.id, + state: Arc::clone(&self.state), + metadata: Arc::clone(&self.metadata), } } } -impl Default for TrackerRegistry { - fn default() -> Self { - Self { - inner: Arc::new(TrackerContextInner { - id: AtomicUsize::new(0), - trackers: Mutex::new(Default::default()), - }), - } - } -} - -impl TrackerRegistry { - pub fn new() -> Self { - Default::default() +impl Tracker { + /// Returns the ID of the Tracker - these are unique per TrackerRegistry + pub fn id(&self) -> TrackerId { + self.id } - /// Trigger graceful termination of a registered future - /// - /// Returns false if no future found with the provided ID + /// Returns a reference to the metadata stored within this Tracker + pub fn metadata(&self) -> &T { + &self.metadata + } + + /// Trigger graceful termination of any futures tracked by + /// this tracker /// /// Note: If the future is currently executing, termination /// will only occur when the future yields (returns from poll) - #[allow(dead_code)] - pub fn terminate(&self, id: TrackerId) -> bool { - if let Some(meta) = self.inner.trackers.lock().get_mut(&id) { - meta.abort.abort(); - true - } else { - false - } + /// and is then scheduled to run again + pub fn cancel(&self) { + self.state.cancel_token.cancel(); } - fn untrack(&self, id: &TrackerId) { - self.inner.trackers.lock().remove(id); + /// Returns true if all futures associated with this tracker have + /// been dropped and no more can be created + pub fn is_complete(&self) -> bool { + matches!(self.get_status(), TrackerStatus::Complete{..}) } - fn track(&self, metadata: T) -> (TrackerId, future::AbortRegistration) { - let id = TrackerId(self.inner.id.fetch_add(1, Ordering::Relaxed)); - let (abort_handle, abort_registration) = future::AbortHandle::new_pair(); + /// Gets the status of the tracker + pub fn get_status(&self) -> TrackerStatus { + // The atomic decrement in TrackerRegistration::drop has release semantics + // acquire here ensures that if a thread observes the tracker to have + // no pending_registrations it cannot subsequently observe pending_futures + // to increase. If it could, observing pending_futures==0 would be insufficient + // to conclude there are no outstanding futures + let pending_registrations = self.state.pending_registrations.load(Ordering::Acquire); - self.inner.trackers.lock().insert( - id, - Tracker { - abort: abort_handle, - data: metadata, + // The atomic decrement in TrackedFuture::drop has release semantics + // acquire therefore ensures that if a thread observes the completion of + // a TrackedFuture, it is guaranteed to see its updates (e.g. wall_nanos) + let pending_futures = self.state.pending_futures.load(Ordering::Acquire); + + match (pending_registrations == 0, pending_futures == 0) { + (false, _) => TrackerStatus::Creating, + (true, false) => TrackerStatus::Running { + total_count: self.state.created_futures.load(Ordering::Relaxed), + pending_count: self.state.pending_futures.load(Ordering::Relaxed), + cpu_nanos: self.state.cpu_nanos.load(Ordering::Relaxed), }, - ); - - (id, abort_registration) + (true, true) => TrackerStatus::Complete { + total_count: self.state.created_futures.load(Ordering::Relaxed), + cpu_nanos: self.state.cpu_nanos.load(Ordering::Relaxed), + wall_nanos: self.state.wall_nanos.load(Ordering::Relaxed), + }, + } } -} -impl TrackerRegistry { - /// Returns a list of tracked futures, with their accompanying IDs and - /// metadata - #[allow(dead_code)] - pub fn tracked(&self) -> Vec<(TrackerId, T)> { - // TODO: Improve this - (#711) - self.inner - .trackers - .lock() - .iter() - .map(|(id, value)| (*id, value.data.clone())) - .collect() + /// Returns if this tracker has been cancelled + pub fn is_cancelled(&self) -> bool { + self.state.cancel_token.is_cancelled() } -} -/// An extension trait that provides `self.track(reg, {})` allowing -/// registering this future with a `TrackerRegistry` -pub trait TrackedFutureExt: Future { - fn track(self, reg: &TrackerRegistry, metadata: T) -> TrackedFuture - where - Self: Sized, - { - let (id, registration) = reg.track(metadata); + /// Blocks until all futures associated with the tracker have been + /// dropped and no more can be created + pub async fn join(&self) { + let mut watch = self.state.watch.clone(); - TrackedFuture { - inner: future::Abortable::new(self, registration), - reg: reg.clone(), - id, + // Wait until watch is set to true or the tx side is dropped + while !*watch.borrow() { + if watch.changed().await.is_err() { + // tx side has been dropped + warn!("tracker watch dropped without signalling"); + break; + } } } } -impl TrackedFutureExt for T where T: Future {} - -/// The `Future` returned by `TrackedFutureExt::track()` -/// Unregisters the future from the registered `TrackerRegistry` on drop -/// and provides the early termination functionality used by -/// `TrackerRegistry::terminate` -#[pin_project(PinnedDrop)] -pub struct TrackedFuture { - #[pin] - inner: future::Abortable, - - reg: TrackerRegistry, - id: TrackerId, +/// A TrackerRegistration is returned by TrackerRegistry::register and can be +/// used to register new TrackedFutures +/// +/// A tracker will not be considered completed until all TrackerRegistrations +/// referencing it have been dropped. This is to prevent a race where further +/// TrackedFutures are registered with a Tracker that has already signalled +/// completion +#[derive(Debug)] +pub struct TrackerRegistration { + state: Arc, } -impl Future for TrackedFuture { - type Output = Result; +impl Clone for TrackerRegistration { + fn clone(&self) -> Self { + self.state + .pending_registrations + .fetch_add(1, Ordering::Relaxed); - fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - self.project().inner.poll(cx) + Self { + state: Arc::clone(&self.state), + } } } -#[pinned_drop] -impl PinnedDrop for TrackedFuture { - fn drop(self: Pin<&mut Self>) { - // Note: This could cause a double-panic in an extreme situation where - // the internal `TrackerRegistry` lock is poisoned and drop was - // called as part of unwinding the stack to handle another panic - let this = self.project(); - this.reg.untrack(this.id) +impl TrackerRegistration { + fn new(watch: tokio::sync::watch::Receiver) -> Self { + let state = Arc::new(TrackerState { + start_instant: Instant::now(), + cpu_nanos: AtomicUsize::new(0), + wall_nanos: AtomicUsize::new(0), + cancel_token: CancellationToken::new(), + created_futures: AtomicUsize::new(0), + pending_futures: AtomicUsize::new(0), + pending_registrations: AtomicUsize::new(1), + watch, + }); + + Self { state } + } +} + +impl Drop for TrackerRegistration { + fn drop(&mut self) { + // This synchronizes with the Acquire load in Tracker::get_status + let previous = self + .state + .pending_registrations + .fetch_sub(1, Ordering::Release); + + // This implies a TrackerRegistration has been cloned without it incrementing + // the pending_registration counter + assert_ne!(previous, 0); } } #[cfg(test)] mod tests { + use std::time::Duration; + use super::*; use tokio::sync::oneshot; #[tokio::test] async fn test_lifecycle() { let (sender, receive) = oneshot::channel(); - let reg = TrackerRegistry::new(); + let mut registry = TrackerRegistry::new(); + let (_, registration) = registry.register(()); - let task = tokio::spawn(receive.track(®, ())); + let task = tokio::spawn(receive.track(registration)); - assert_eq!(reg.tracked().len(), 1); + assert_eq!(registry.running().len(), 1); sender.send(()).unwrap(); task.await.unwrap().unwrap().unwrap(); - assert_eq!(reg.tracked().len(), 0); + assert_eq!(registry.running().len(), 0); } #[tokio::test] async fn test_interleaved() { let (sender1, receive1) = oneshot::channel(); let (sender2, receive2) = oneshot::channel(); - let reg = TrackerRegistry::new(); + let mut registry = TrackerRegistry::new(); + let (_, registration1) = registry.register(1); + let (_, registration2) = registry.register(2); - let task1 = tokio::spawn(receive1.track(®, 1)); - let task2 = tokio::spawn(receive2.track(®, 2)); + let task1 = tokio::spawn(receive1.track(registration1)); + let task2 = tokio::spawn(receive2.track(registration2)); - let mut tracked: Vec<_> = reg.tracked().iter().map(|x| x.1).collect(); - tracked.sort_unstable(); - assert_eq!(tracked, vec![1, 2]); + let tracked = sorted(registry.running()); + assert_eq!(get_metadata(&tracked), vec![1, 2]); sender2.send(()).unwrap(); task2.await.unwrap().unwrap().unwrap(); - let tracked: Vec<_> = reg.tracked().iter().map(|x| x.1).collect(); - assert_eq!(tracked, vec![1]); + let tracked: Vec<_> = sorted(registry.running()); + assert_eq!(get_metadata(&tracked), vec![1]); sender1.send(42).unwrap(); let ret = task1.await.unwrap().unwrap().unwrap(); assert_eq!(ret, 42); - assert_eq!(reg.tracked().len(), 0); + assert_eq!(registry.running().len(), 0); } #[tokio::test] async fn test_drop() { - let reg = TrackerRegistry::new(); + let mut registry = TrackerRegistry::new(); + let (_, registration) = registry.register(()); { - let f = futures::future::pending::<()>().track(®, ()); + let f = futures::future::pending::<()>().track(registration); - assert_eq!(reg.tracked().len(), 1); + assert_eq!(registry.running().len(), 1); std::mem::drop(f); } - assert_eq!(reg.tracked().len(), 0); + assert_eq!(registry.running().len(), 0); + } + + #[tokio::test] + async fn test_drop_multiple() { + let mut registry = TrackerRegistry::new(); + let (_, registration) = registry.register(()); + + { + let f = futures::future::pending::<()>().track(registration.clone()); + { + let f = futures::future::pending::<()>().track(registration); + assert_eq!(registry.running().len(), 1); + std::mem::drop(f); + } + assert_eq!(registry.running().len(), 1); + std::mem::drop(f); + } + + assert_eq!(registry.running().len(), 0); } #[tokio::test] async fn test_terminate() { - let reg = TrackerRegistry::new(); + let mut registry = TrackerRegistry::new(); + let (_, registration) = registry.register(()); - let task = tokio::spawn(futures::future::pending::<()>().track(®, ())); + let task = tokio::spawn(futures::future::pending::<()>().track(registration)); - let tracked = reg.tracked(); + let tracked = registry.running(); assert_eq!(tracked.len(), 1); - reg.terminate(tracked[0].0); + tracked[0].cancel(); let result = task.await.unwrap(); assert!(result.is_err()); - assert_eq!(reg.tracked().len(), 0); + assert_eq!(registry.running().len(), 0); + } + + #[tokio::test] + async fn test_terminate_early() { + let mut registry = TrackerRegistry::new(); + let (tracker, registration) = registry.register(()); + tracker.cancel(); + + let task1 = tokio::spawn(futures::future::pending::<()>().track(registration)); + let result1 = task1.await.unwrap(); + + assert!(result1.is_err()); + assert_eq!(registry.running().len(), 0); + } + + #[tokio::test] + async fn test_terminate_multiple() { + let mut registry = TrackerRegistry::new(); + let (_, registration) = registry.register(()); + + let task1 = tokio::spawn(futures::future::pending::<()>().track(registration.clone())); + let task2 = tokio::spawn(futures::future::pending::<()>().track(registration)); + + let tracked = registry.running(); + assert_eq!(tracked.len(), 1); + + tracked[0].cancel(); + + let result1 = task1.await.unwrap(); + let result2 = task2.await.unwrap(); + + assert!(result1.is_err()); + assert!(result2.is_err()); + assert_eq!(registry.running().len(), 0); + } + + #[tokio::test] + async fn test_reclaim() { + let mut registry = TrackerRegistry::new(); + + let (_, registration1) = registry.register(1); + let (_, registration2) = registry.register(2); + let (_, registration3) = registry.register(3); + + let task1 = tokio::spawn(futures::future::pending::<()>().track(registration1.clone())); + let task2 = tokio::spawn(futures::future::pending::<()>().track(registration1)); + let task3 = tokio::spawn(futures::future::ready(()).track(registration2.clone())); + let task4 = tokio::spawn(futures::future::pending::<()>().track(registration2)); + let task5 = tokio::spawn(futures::future::pending::<()>().track(registration3)); + + let running = sorted(registry.running()); + let tracked = sorted(registry.tracked()); + + assert_eq!(running.len(), 3); + assert_eq!(get_metadata(&running), vec![1, 2, 3]); + assert_eq!(tracked.len(), 3); + assert_eq!(get_metadata(&tracked), vec![1, 2, 3]); + + // Trigger termination of task1 and task2 + running[0].cancel(); + + let result1 = task1.await.unwrap(); + let result2 = task2.await.unwrap(); + + assert!(result1.is_err()); + assert!(result2.is_err()); + + let running = sorted(registry.running()); + let tracked = sorted(registry.tracked()); + + assert_eq!(running.len(), 2); + assert_eq!(get_metadata(&running), vec![2, 3]); + assert_eq!(tracked.len(), 3); + assert_eq!(get_metadata(&tracked), vec![1, 2, 3]); + + // Expect reclaim to find now finished registration1 + let reclaimed = sorted(registry.reclaim().collect()); + assert_eq!(reclaimed.len(), 1); + assert_eq!(get_metadata(&reclaimed), vec![1]); + + // Now expect tracked to match running + let running = sorted(registry.running()); + let tracked = sorted(registry.tracked()); + + assert_eq!(running.len(), 2); + assert_eq!(get_metadata(&running), vec![2, 3]); + assert_eq!(tracked.len(), 2); + assert_eq!(get_metadata(&tracked), vec![2, 3]); + + // Wait for task3 to finish + let result3 = task3.await.unwrap(); + assert!(result3.is_ok()); + + assert!( + matches!(tracked[0].get_status(), TrackerStatus::Running { pending_count: 1, total_count: 2, ..}) + ); + + // Trigger termination of task5 + running[1].cancel(); + + let result5 = task5.await.unwrap(); + assert!(result5.is_err()); + + let running = sorted(registry.running()); + let tracked = sorted(registry.tracked()); + + assert_eq!(running.len(), 1); + assert_eq!(get_metadata(&running), vec![2]); + assert_eq!(tracked.len(), 2); + assert_eq!(get_metadata(&tracked), vec![2, 3]); + + // Trigger termination of task4 + running[0].cancel(); + + let result4 = task4.await.unwrap(); + assert!(result4.is_err()); + assert!(matches!(running[0].get_status(), TrackerStatus::Complete { total_count: 2, ..})); + + let reclaimed = sorted(registry.reclaim().collect()); + + assert_eq!(reclaimed.len(), 2); + assert_eq!(get_metadata(&reclaimed), vec![2, 3]); + assert_eq!(registry.tracked().len(), 0); + } + + // Use n+1 threads where n is the number of "blocking" tasks + // to prevent stalling the tokio executor + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn test_timing() { + let mut registry = TrackerRegistry::new(); + let (tracker1, registration1) = registry.register(1); + let (tracker2, registration2) = registry.register(2); + let (tracker3, registration3) = registry.register(3); + + let task1 = + tokio::spawn(tokio::time::sleep(Duration::from_millis(100)).track(registration1)); + let task2 = tokio::spawn( + async move { std::thread::sleep(Duration::from_millis(100)) }.track(registration2), + ); + + let task3 = tokio::spawn( + async move { std::thread::sleep(Duration::from_millis(100)) } + .track(registration3.clone()), + ); + + let task4 = tokio::spawn( + async move { std::thread::sleep(Duration::from_millis(100)) }.track(registration3), + ); + + task1.await.unwrap().unwrap(); + task2.await.unwrap().unwrap(); + task3.await.unwrap().unwrap(); + task4.await.unwrap().unwrap(); + + let assert_fuzzy = |actual: usize, expected: std::time::Duration| { + // Number of milliseconds of toleration + let epsilon = Duration::from_millis(10).as_nanos() as usize; + let expected = expected.as_nanos() as usize; + + assert!( + actual > expected.saturating_sub(epsilon), + "Expected {} got {}", + expected, + actual + ); + assert!( + actual < expected.saturating_add(epsilon), + "Expected {} got {}", + expected, + actual + ); + }; + + let assert_complete = |status: TrackerStatus, + expected_cpu: std::time::Duration, + expected_wal: std::time::Duration| { + match status { + TrackerStatus::Complete { + cpu_nanos, + wall_nanos, + .. + } => { + assert_fuzzy(cpu_nanos, expected_cpu); + assert_fuzzy(wall_nanos, expected_wal); + } + _ => panic!("expected complete got {:?}", status), + } + }; + + assert_complete( + tracker1.get_status(), + Duration::from_millis(0), + Duration::from_millis(100), + ); + assert_complete( + tracker2.get_status(), + Duration::from_millis(100), + Duration::from_millis(100), + ); + assert_complete( + tracker3.get_status(), + Duration::from_millis(200), + Duration::from_millis(100), + ); + } + + #[tokio::test] + async fn test_register_race() { + let mut registry = TrackerRegistry::new(); + let (_, registration) = registry.register(()); + + let task1 = tokio::spawn(futures::future::ready(()).track(registration.clone())); + task1.await.unwrap().unwrap(); + + let tracked = registry.tracked(); + assert_eq!(tracked.len(), 1); + assert!(matches!(&tracked[0].get_status(), TrackerStatus::Creating)); + + // Should only consider tasks complete once cannot register more Futures + let reclaimed: Vec<_> = registry.reclaim().collect(); + assert_eq!(reclaimed.len(), 0); + + let task2 = tokio::spawn(futures::future::ready(()).track(registration)); + task2.await.unwrap().unwrap(); + + let reclaimed: Vec<_> = registry.reclaim().collect(); + assert_eq!(reclaimed.len(), 1); + } + + fn sorted(mut input: Vec>) -> Vec> { + input.sort_unstable_by_key(|x| *x.metadata()); + input + } + + fn get_metadata(input: &[Tracker]) -> Vec { + let mut ret: Vec<_> = input.iter().map(|x| *x.metadata()).collect(); + ret.sort_unstable(); + ret } } diff --git a/server/src/tracker/future.rs b/server/src/tracker/future.rs new file mode 100644 index 0000000000..1a28734bf7 --- /dev/null +++ b/server/src/tracker/future.rs @@ -0,0 +1,91 @@ +use std::pin::Pin; +use std::sync::atomic::Ordering; +use std::task::{Context, Poll}; +use std::time::Instant; + +use futures::{future::BoxFuture, prelude::*}; +use pin_project::{pin_project, pinned_drop}; + +use super::{TrackerRegistration, TrackerState}; +use std::sync::Arc; + +/// An extension trait that provides `self.track(registration)` allowing +/// associating this future with a `TrackerRegistration` +pub trait TrackedFutureExt: Future { + fn track(self, registration: TrackerRegistration) -> TrackedFuture + where + Self: Sized, + { + let tracker = Arc::clone(®istration.state); + let token = tracker.cancel_token.clone(); + + tracker.created_futures.fetch_add(1, Ordering::Relaxed); + tracker.pending_futures.fetch_add(1, Ordering::Relaxed); + + // This must occur after the increment of pending_futures + std::mem::drop(registration); + + // The future returned by CancellationToken::cancelled borrows the token + // In order to ensure we get a future with a static lifetime + // we box them up together and let async work its magic + let abort = Box::pin(async move { token.cancelled().await }); + + TrackedFuture { + inner: self, + abort, + tracker, + } + } +} + +impl TrackedFutureExt for T where T: Future {} + +/// The `Future` returned by `TrackedFutureExt::track()` +/// Unregisters the future from the registered `TrackerRegistry` on drop +/// and provides the early termination functionality used by +/// `TrackerRegistry::terminate` +#[pin_project(PinnedDrop)] +#[allow(missing_debug_implementations)] +pub struct TrackedFuture { + #[pin] + inner: F, + #[pin] + abort: BoxFuture<'static, ()>, + tracker: Arc, +} + +impl Future for TrackedFuture { + type Output = Result; + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + if self.as_mut().project().abort.poll(cx).is_ready() { + return Poll::Ready(Err(future::Aborted {})); + } + + let start = Instant::now(); + let poll = self.as_mut().project().inner.poll(cx); + let delta = start.elapsed().as_nanos() as usize; + + self.tracker.cpu_nanos.fetch_add(delta, Ordering::Relaxed); + + poll.map(Ok) + } +} + +#[pinned_drop] +impl PinnedDrop for TrackedFuture { + fn drop(self: Pin<&mut Self>) { + let state = &self.project().tracker; + + let wall_nanos = state.start_instant.elapsed().as_nanos() as usize; + + state.wall_nanos.fetch_max(wall_nanos, Ordering::Relaxed); + + // This synchronizes with the Acquire load in Tracker::get_status + let previous = state.pending_futures.fetch_sub(1, Ordering::Release); + + // Failure implies a TrackedFuture has somehow been created + // without it incrementing the pending_futures counter + assert_ne!(previous, 0); + } +} diff --git a/server/src/tracker/history.rs b/server/src/tracker/history.rs new file mode 100644 index 0000000000..7ffc462d70 --- /dev/null +++ b/server/src/tracker/history.rs @@ -0,0 +1,204 @@ +use super::{Tracker, TrackerId, TrackerRegistration, TrackerRegistry}; +use hashbrown::hash_map::Entry; +use hashbrown::HashMap; +use std::hash::Hash; +use tracing::info; + +/// A wrapper around a TrackerRegistry that automatically retains a history +#[derive(Debug)] +pub struct TrackerRegistryWithHistory { + registry: TrackerRegistry, + history: SizeLimitedHashMap>, +} + +impl TrackerRegistryWithHistory { + pub fn new(capacity: usize) -> Self { + Self { + history: SizeLimitedHashMap::new(capacity), + registry: TrackerRegistry::new(), + } + } + + /// Register a new tracker in the registry + pub fn register(&mut self, metadata: T) -> (Tracker, TrackerRegistration) { + self.registry.register(metadata) + } + + /// Get the tracker associated with a given id + pub fn get(&self, id: TrackerId) -> Option> { + match self.history.get(&id) { + Some(x) => Some(x.clone()), + None => self.registry.get(id), + } + } + + pub fn tracked_len(&self) -> usize { + self.registry.tracked_len() + } + + /// Returns a list of trackers, including those that are no longer running + pub fn tracked(&self) -> Vec> { + let mut tracked = self.registry.tracked(); + tracked.extend(self.history.values().cloned()); + tracked + } + + /// Returns a list of active trackers + pub fn running(&self) -> Vec> { + self.registry.running() + } + + /// Reclaims jobs into the historical archive + pub fn reclaim(&mut self) { + for job in self.registry.reclaim() { + info!(?job, "job finished"); + self.history.push(job.id(), job) + } + } +} + +/// A size limited hashmap that maintains a finite number +/// of key value pairs providing O(1) key lookups +/// +/// Inserts over the capacity will overwrite previous values +#[derive(Debug)] +struct SizeLimitedHashMap { + values: HashMap, + ring: Vec, + start_idx: usize, + capacity: usize, +} + +impl SizeLimitedHashMap { + pub fn new(capacity: usize) -> Self { + Self { + values: HashMap::with_capacity(capacity), + ring: Vec::with_capacity(capacity), + start_idx: 0, + capacity, + } + } + + /// Get the value associated with a specific key + pub fn get(&self, key: &K) -> Option<&V> { + self.values.get(key) + } + + /// Returns an iterator to all values stored within the ring buffer + /// + /// Note: the order is not guaranteed + pub fn values(&self) -> impl Iterator + '_ { + self.values.values() + } + + /// Push a new value into the ring buffer + /// + /// If a value with the given key already exists, it will replace the value + /// Otherwise it will add the key and value to the buffer + /// + /// If there is insufficient capacity it will drop the oldest key value pair + /// from the buffer + pub fn push(&mut self, key: K, value: V) { + if let Entry::Occupied(occupied) = self.values.entry(key) { + // If already exists - replace existing value + occupied.replace_entry(value); + + return; + } + + if self.ring.len() < self.capacity { + // Still populating the ring + assert_eq!(self.start_idx, 0); + self.ring.push(key); + self.values.insert(key, value); + + return; + } + + // Need to swap something out of the ring + let mut old = key; + std::mem::swap(&mut self.ring[self.start_idx], &mut old); + + self.start_idx += 1; + if self.start_idx == self.capacity { + self.start_idx = 0; + } + + self.values.remove(&old); + self.values.insert(key, value); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_hashmap() { + let expect = |ring: &SizeLimitedHashMap, expected: &[i32]| { + let mut values: Vec<_> = ring.values().cloned().collect(); + values.sort_unstable(); + assert_eq!(&values, expected); + }; + + let mut ring = SizeLimitedHashMap::new(5); + for i in 0..=4 { + ring.push(i, i); + } + + expect(&ring, &[0, 1, 2, 3, 4]); + + // Expect rollover + ring.push(5, 5); + expect(&ring, &[1, 2, 3, 4, 5]); + + for i in 6..=9 { + ring.push(i, i); + } + expect(&ring, &[5, 6, 7, 8, 9]); + + for i in 10..=52 { + ring.push(i + 10, i); + } + expect(&ring, &[48, 49, 50, 51, 52]); + assert_eq!(*ring.get(&60).unwrap(), 50); + } + + #[test] + fn test_tracker_archive() { + let compare = |expected_ids: &[TrackerId], archive: &TrackerRegistryWithHistory| { + let mut collected: Vec<_> = archive.history.values().map(|x| x.id()).collect(); + collected.sort(); + assert_eq!(&collected, expected_ids); + }; + + let mut archive = TrackerRegistryWithHistory::new(4); + + for i in 0..=3 { + archive.register(i); + } + + archive.reclaim(); + + compare( + &[TrackerId(0), TrackerId(1), TrackerId(2), TrackerId(3)], + &archive, + ); + + for i in 4..=7 { + archive.register(i); + } + + compare( + &[TrackerId(0), TrackerId(1), TrackerId(2), TrackerId(3)], + &archive, + ); + + archive.reclaim(); + + compare( + &[TrackerId(4), TrackerId(5), TrackerId(6), TrackerId(7)], + &archive, + ); + } +} diff --git a/server/src/tracker/registry.rs b/server/src/tracker/registry.rs new file mode 100644 index 0000000000..1c7996e2ad --- /dev/null +++ b/server/src/tracker/registry.rs @@ -0,0 +1,125 @@ +use super::{Tracker, TrackerRegistration}; +use hashbrown::HashMap; +use std::str::FromStr; +use std::sync::Arc; +use tracing::debug; + +/// Every future registered with a `TrackerRegistry` is assigned a unique +/// `TrackerId` +#[derive(Clone, Copy, Debug, Hash, PartialEq, Eq, PartialOrd, Ord)] +pub struct TrackerId(pub(super) usize); + +impl FromStr for TrackerId { + type Err = std::num::ParseIntError; + + fn from_str(s: &str) -> Result { + Ok(Self(FromStr::from_str(s)?)) + } +} + +impl ToString for TrackerId { + fn to_string(&self) -> String { + self.0.to_string() + } +} + +/// Internal data stored by TrackerRegistry +#[derive(Debug)] +struct TrackerSlot { + tracker: Tracker, + watch: tokio::sync::watch::Sender, +} + +/// Allows tracking the lifecycle of futures registered by +/// `TrackedFutureExt::track` with an accompanying metadata payload of type T +/// +/// Additionally can trigger graceful cancellation of registered futures +#[derive(Debug)] +pub struct TrackerRegistry { + next_id: usize, + trackers: HashMap>, +} + +impl Default for TrackerRegistry { + fn default() -> Self { + Self { + next_id: 0, + trackers: Default::default(), + } + } +} + +impl TrackerRegistry { + pub fn new() -> Self { + Default::default() + } + + /// Register a new tracker in the registry + pub fn register(&mut self, metadata: T) -> (Tracker, TrackerRegistration) { + let id = TrackerId(self.next_id); + self.next_id += 1; + + let (sender, receiver) = tokio::sync::watch::channel(false); + let registration = TrackerRegistration::new(receiver); + + let tracker = Tracker { + id, + metadata: Arc::new(metadata), + state: Arc::clone(®istration.state), + }; + + self.trackers.insert( + id, + TrackerSlot { + tracker: tracker.clone(), + watch: sender, + }, + ); + + (tracker, registration) + } + + /// Removes completed tasks from the registry and returns an iterator of + /// those removed + pub fn reclaim(&mut self) -> impl Iterator> + '_ { + self.trackers + .drain_filter(|_, v| v.tracker.is_complete()) + .map(|(_, v)| { + if let Err(error) = v.watch.send(true) { + // As we hold a reference to the Tracker here, this should be impossible + debug!(?error, "failed to publish tracker completion") + } + v.tracker + }) + } + + pub fn get(&self, id: TrackerId) -> Option> { + self.trackers.get(&id).map(|x| x.tracker.clone()) + } + + /// Returns the number of tracked tasks + pub fn tracked_len(&self) -> usize { + self.trackers.len() + } + + /// Returns a list of trackers, including those that are no longer running + pub fn tracked(&self) -> Vec> { + self.trackers + .iter() + .map(|(_, v)| v.tracker.clone()) + .collect() + } + + /// Returns a list of active trackers + pub fn running(&self) -> Vec> { + self.trackers + .iter() + .filter_map(|(_, v)| { + if !v.tracker.is_complete() { + return Some(v.tracker.clone()); + } + None + }) + .collect() + } +} diff --git a/src/commands/convert.rs b/src/commands/convert.rs index d04b5ff00f..0b64ae8f88 100644 --- a/src/commands/convert.rs +++ b/src/commands/convert.rs @@ -1,9 +1,9 @@ -use data_types::schema::Schema; use influxdb_line_protocol::parse_lines; use ingest::{ parquet::writer::{CompressionLevel, Error as ParquetWriterError, IOxParquetTableWriter}, ConversionSettings, Error as IngestError, LineProtocolConverter, TSMFileConverter, }; +use internal_types::schema::Schema; use packers::{Error as TableError, IOxTableWriter, IOxTableWriterSource}; use snafu::{OptionExt, ResultExt, Snafu}; use std::{ diff --git a/src/commands/database.rs b/src/commands/database.rs index d128f03a69..57a8a295e6 100644 --- a/src/commands/database.rs +++ b/src/commands/database.rs @@ -1,10 +1,21 @@ +//! This module implements the `database` CLI command +use std::{fs::File, io::Read, path::PathBuf, str::FromStr}; + use influxdb_iox_client::{ connection::Builder, - management::{generated_types::*, *}, + flight, + format::QueryOutputFormat, + management::{ + self, generated_types::*, CreateDatabaseError, GetDatabaseError, ListDatabaseError, + }, + write::{self, WriteError}, }; use structopt::StructOpt; use thiserror::Error; +mod chunk; +mod partition; + #[derive(Debug, Error)] pub enum Error { #[error("Error creating database: {0}")] @@ -18,6 +29,27 @@ pub enum Error { #[error("Error connecting to IOx: {0}")] ConnectionError(#[from] influxdb_iox_client::connection::Error), + + #[error("Error reading file {:?}: {}", file_name, source)] + ReadingFile { + file_name: PathBuf, + source: std::io::Error, + }, + + #[error("Error writing: {0}")] + WriteError(#[from] WriteError), + + #[error("Error formatting: {0}")] + FormattingError(#[from] influxdb_iox_client::format::Error), + + #[error("Error querying: {0}")] + Query(#[from] influxdb_iox_client::flight::Error), + + #[error("Error in chunk subcommand: {0}")] + Chunk(#[from] chunk::Error), + + #[error("Error in partition subcommand: {0}")] + Partition(#[from] partition::Error), } pub type Result = std::result::Result; @@ -35,53 +67,157 @@ struct Create { /// The name of the database name: String, - /// Create a mutable buffer of the specified size in bytes - #[structopt(short, long)] - mutable_buffer: Option, + /// Create a mutable buffer of the specified size in bytes. If + /// size is 0, no mutable buffer is created. + #[structopt(short, long, default_value = "104857600")] // 104857600 = 100*1024*1024 + mutable_buffer: u64, } -/// Get list of databases, or return configuration of specific database +/// Get list of databases +#[derive(Debug, StructOpt)] +struct List {} + +/// Return configuration of specific database #[derive(Debug, StructOpt)] struct Get { - /// If specified returns configuration of database - name: Option, + /// The name of the database + name: String, } +/// Write data into the specified database +#[derive(Debug, StructOpt)] +struct Write { + /// The name of the database + name: String, + + /// File with data to load. Currently supported formats are .lp + file_name: PathBuf, +} + +/// Query the data with SQL +#[derive(Debug, StructOpt)] +struct Query { + /// The name of the database + name: String, + + /// The query to run, in SQL format + query: String, + + /// Optional format ('pretty', 'json', or 'csv') + #[structopt(short, long, default_value = "pretty")] + format: String, +} + +/// All possible subcommands for database #[derive(Debug, StructOpt)] enum Command { Create(Create), + List(List), Get(Get), + Write(Write), + Query(Query), + Chunk(chunk::Config), + Partition(partition::Config), } pub async fn command(url: String, config: Config) -> Result<()> { - let connection = Builder::default().build(url).await?; - let mut client = Client::new(connection); + let connection = Builder::default().build(url.clone()).await?; match config.command { Command::Create(command) => { - client - .create_database(DatabaseRules { - name: command.name, - mutable_buffer_config: command.mutable_buffer.map(|buffer_size| { - MutableBufferConfig { - buffer_size, - ..Default::default() - } - }), + let mut client = management::Client::new(connection); + + // Configure a mutable buffer if requested + let buffer_size = command.mutable_buffer; + let mutable_buffer_config = if buffer_size > 0 { + Some(MutableBufferConfig { + buffer_size, ..Default::default() }) - .await?; + } else { + None + }; + + let rules = DatabaseRules { + name: command.name, + + mutable_buffer_config, + + // Default to hourly partitions + partition_template: Some(PartitionTemplate { + parts: vec![partition_template::Part { + part: Some(partition_template::part::Part::Time( + "%Y-%m-%d %H:00:00".into(), + )), + }], + }), + + // Note no wal buffer config + ..Default::default() + }; + + client.create_database(rules).await?; + println!("Ok"); } + Command::List(_) => { + let mut client = management::Client::new(connection); + let databases = client.list_databases().await?; + println!("{}", databases.join(", ")) + } Command::Get(get) => { - if let Some(name) = get.name { - let database = client.get_database(name).await?; - // TOOD: Do something better than this - println!("{:#?}", database); - } else { - let databases = client.list_databases().await?; - println!("{}", databases.join(", ")) + let mut client = management::Client::new(connection); + let database = client.get_database(get.name).await?; + // TOOD: Do something better than this + println!("{:#?}", database); + } + Command::Write(write) => { + let mut client = write::Client::new(connection); + + let mut file = File::open(&write.file_name).map_err(|e| Error::ReadingFile { + file_name: write.file_name.clone(), + source: e, + })?; + + let mut lp_data = String::new(); + file.read_to_string(&mut lp_data) + .map_err(|e| Error::ReadingFile { + file_name: write.file_name.clone(), + source: e, + })?; + + let lines_written = client.write(write.name, lp_data).await?; + + println!("{} Lines OK", lines_written); + } + Command::Query(query) => { + let mut client = flight::Client::new(connection); + let Query { + name, + format, + query, + } = query; + + let format = QueryOutputFormat::from_str(&format)?; + + let mut query_results = client.perform_query(&name, query).await?; + + // It might be nice to do some sort of streaming write + // rather than buffering the whole thing. + let mut batches = vec![]; + while let Some(data) = query_results.next().await? { + batches.push(data); } + + let formatted_result = format.format(&batches)?; + + println!("{}", formatted_result); + } + Command::Chunk(config) => { + chunk::command(url, config).await?; + } + Command::Partition(config) => { + partition::command(url, config).await?; } } diff --git a/src/commands/database/chunk.rs b/src/commands/database/chunk.rs new file mode 100644 index 0000000000..f5974f0385 --- /dev/null +++ b/src/commands/database/chunk.rs @@ -0,0 +1,70 @@ +//! This module implements the `chunk` CLI command +use data_types::chunk::ChunkSummary; +use generated_types::google::FieldViolation; +use influxdb_iox_client::{ + connection::Builder, + management::{self, ListChunksError}, +}; +use std::convert::TryFrom; +use structopt::StructOpt; +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum Error { + #[error("Error listing chunks: {0}")] + ListChunkError(#[from] ListChunksError), + + #[error("Error interpreting server response: {0}")] + ConvertingResponse(#[from] FieldViolation), + + #[error("Error rendering response as JSON: {0}")] + WritingJson(#[from] serde_json::Error), + + #[error("Error connecting to IOx: {0}")] + ConnectionError(#[from] influxdb_iox_client::connection::Error), +} + +pub type Result = std::result::Result; + +/// Manage IOx chunks +#[derive(Debug, StructOpt)] +pub struct Config { + #[structopt(subcommand)] + command: Command, +} + +/// List the chunks for the specified database in JSON format +#[derive(Debug, StructOpt)] +struct List { + /// The name of the database + db_name: String, +} + +/// All possible subcommands for chunk +#[derive(Debug, StructOpt)] +enum Command { + List(List), +} + +pub async fn command(url: String, config: Config) -> Result<()> { + let connection = Builder::default().build(url).await?; + + match config.command { + Command::List(get) => { + let List { db_name } = get; + + let mut client = management::Client::new(connection); + + let chunks = client.list_chunks(db_name).await?; + + let chunks = chunks + .into_iter() + .map(ChunkSummary::try_from) + .collect::, FieldViolation>>()?; + + serde_json::to_writer_pretty(std::io::stdout(), &chunks)?; + } + } + + Ok(()) +} diff --git a/src/commands/database/partition.rs b/src/commands/database/partition.rs new file mode 100644 index 0000000000..47c2cb6dcb --- /dev/null +++ b/src/commands/database/partition.rs @@ -0,0 +1,194 @@ +//! This module implements the `partition` CLI command +use data_types::chunk::ChunkSummary; +use data_types::job::Operation; +use generated_types::google::FieldViolation; +use influxdb_iox_client::{ + connection::Builder, + management::{ + self, ClosePartitionChunkError, GetPartitionError, ListPartitionChunksError, + ListPartitionsError, NewPartitionChunkError, + }, +}; +use std::convert::{TryFrom, TryInto}; +use structopt::StructOpt; +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum Error { + #[error("Error listing partitions: {0}")] + ListPartitionsError(#[from] ListPartitionsError), + + #[error("Error getting partition: {0}")] + GetPartitionsError(#[from] GetPartitionError), + + #[error("Error listing partition chunks: {0}")] + ListPartitionChunksError(#[from] ListPartitionChunksError), + + #[error("Error creating new partition chunk: {0}")] + NewPartitionChunkError(#[from] NewPartitionChunkError), + + #[error("Error closing chunk: {0}")] + ClosePartitionChunkError(#[from] ClosePartitionChunkError), + + #[error("Error rendering response as JSON: {0}")] + WritingJson(#[from] serde_json::Error), + + #[error("Received invalid response: {0}")] + InvalidResponse(#[from] FieldViolation), + + #[error("Error connecting to IOx: {0}")] + ConnectionError(#[from] influxdb_iox_client::connection::Error), +} + +pub type Result = std::result::Result; + +/// Manage IOx partitions +#[derive(Debug, StructOpt)] +pub struct Config { + #[structopt(subcommand)] + command: Command, +} + +/// List all known partition keys for a database +#[derive(Debug, StructOpt)] +struct List { + /// The name of the database + db_name: String, +} + +/// Get details of a specific partition in JSON format (TODO) +#[derive(Debug, StructOpt)] +struct Get { + /// The name of the database + db_name: String, + + /// The partition key + partition_key: String, +} + +/// lists all chunks in this partition +#[derive(Debug, StructOpt)] +struct ListChunks { + /// The name of the database + db_name: String, + + /// The partition key + partition_key: String, +} + +/// Create a new, open chunk in the partiton's Mutable Buffer which will receive +/// new writes. +#[derive(Debug, StructOpt)] +struct NewChunk { + /// The name of the database + db_name: String, + + /// The partition key + partition_key: String, +} + +/// Closes a chunk in the mutable buffer for writing and starts its migration to +/// the read buffer +#[derive(Debug, StructOpt)] +struct CloseChunk { + /// The name of the database + db_name: String, + + /// The partition key + partition_key: String, + + /// The chunk id + chunk_id: u32, +} + +/// All possible subcommands for partition +#[derive(Debug, StructOpt)] +enum Command { + // List partitions + List(List), + // Get details about a particular partition + Get(Get), + // List chunks in a partition + ListChunks(ListChunks), + // Create a new chunk in the partition + NewChunk(NewChunk), + // Close the chunk and move to read buffer + CloseChunk(CloseChunk), +} + +pub async fn command(url: String, config: Config) -> Result<()> { + let connection = Builder::default().build(url).await?; + let mut client = management::Client::new(connection); + + match config.command { + Command::List(list) => { + let List { db_name } = list; + let partitions = client.list_partitions(db_name).await?; + let partition_keys = partitions.into_iter().map(|p| p.key).collect::>(); + + serde_json::to_writer_pretty(std::io::stdout(), &partition_keys)?; + } + Command::Get(get) => { + let Get { + db_name, + partition_key, + } = get; + + let management::generated_types::Partition { key } = + client.get_partition(db_name, partition_key).await?; + + // TODO: get more details from the partition, andprint it + // out better (i.e. move to using Partition summary that + // is already in data_types) + #[derive(serde::Serialize)] + struct PartitionDetail { + key: String, + } + + let partition_detail = PartitionDetail { key }; + + serde_json::to_writer_pretty(std::io::stdout(), &partition_detail)?; + } + Command::ListChunks(list_chunks) => { + let ListChunks { + db_name, + partition_key, + } = list_chunks; + + let chunks = client.list_partition_chunks(db_name, partition_key).await?; + + let chunks = chunks + .into_iter() + .map(ChunkSummary::try_from) + .collect::, FieldViolation>>()?; + + serde_json::to_writer_pretty(std::io::stdout(), &chunks)?; + } + Command::NewChunk(new_chunk) => { + let NewChunk { + db_name, + partition_key, + } = new_chunk; + + // Ignore response for now + client.new_partition_chunk(db_name, partition_key).await?; + println!("Ok"); + } + Command::CloseChunk(close_chunk) => { + let CloseChunk { + db_name, + partition_key, + chunk_id, + } = close_chunk; + + let operation: Operation = client + .close_partition_chunk(db_name, partition_key, chunk_id) + .await? + .try_into()?; + + serde_json::to_writer_pretty(std::io::stdout(), &operation)?; + } + } + + Ok(()) +} diff --git a/src/commands/logging.rs b/src/commands/logging.rs index cb3def640a..0cd4ae0de3 100644 --- a/src/commands/logging.rs +++ b/src/commands/logging.rs @@ -2,7 +2,7 @@ use tracing_subscriber::{prelude::*, EnvFilter}; -use super::server::{Config, LogFormat}; +use super::run::{Config, LogFormat}; /// Handles setting up logging levels #[derive(Debug)] diff --git a/src/commands/operations.rs b/src/commands/operations.rs new file mode 100644 index 0000000000..037353c5dd --- /dev/null +++ b/src/commands/operations.rs @@ -0,0 +1,113 @@ +use data_types::job::Operation; +use generated_types::google::FieldViolation; +use influxdb_iox_client::{ + connection::Builder, + management, + operations::{self, Client}, +}; +use std::convert::TryInto; +use structopt::StructOpt; +use thiserror::Error; + +#[derive(Debug, Error)] +pub enum Error { + #[error("Error connecting to IOx: {0}")] + ConnectionError(#[from] influxdb_iox_client::connection::Error), + + #[error("Client error: {0}")] + ClientError(#[from] operations::Error), + + #[error("Received invalid response: {0}")] + InvalidResponse(#[from] FieldViolation), + + #[error("Failed to create dummy job: {0}")] + CreateDummyJobError(#[from] management::CreateDummyJobError), + + #[error("Output serialization error: {0}")] + SerializationError(#[from] serde_json::Error), +} + +pub type Result = std::result::Result; + +/// Manage long-running IOx operations +#[derive(Debug, StructOpt)] +pub struct Config { + #[structopt(subcommand)] + command: Command, +} + +#[derive(Debug, StructOpt)] +enum Command { + /// Get list of running operations + List, + + /// Get a specific operation + Get { + /// The id of the operation + id: usize, + }, + + /// Wait for a specific operation to complete + Wait { + /// The id of the operation + id: usize, + + /// Maximum number of nanoseconds to wait before returning current + /// status + nanos: Option, + }, + + /// Cancel a specific operation + Cancel { + /// The id of the operation + id: usize, + }, + + /// Spawns a dummy test operation + Test { nanos: Vec }, +} + +pub async fn command(url: String, config: Config) -> Result<()> { + let connection = Builder::default().build(url).await?; + + match config.command { + Command::List => { + let result: Result, _> = Client::new(connection) + .list_operations() + .await? + .into_iter() + .map(TryInto::try_into) + .collect(); + let operations = result?; + serde_json::to_writer_pretty(std::io::stdout(), &operations)?; + } + Command::Get { id } => { + let operation: Operation = Client::new(connection) + .get_operation(id) + .await? + .try_into()?; + serde_json::to_writer_pretty(std::io::stdout(), &operation)?; + } + Command::Wait { id, nanos } => { + let timeout = nanos.map(std::time::Duration::from_nanos); + let operation: Operation = Client::new(connection) + .wait_operation(id, timeout) + .await? + .try_into()?; + serde_json::to_writer_pretty(std::io::stdout(), &operation)?; + } + Command::Cancel { id } => { + Client::new(connection).cancel_operation(id).await?; + println!("Ok"); + } + Command::Test { nanos } => { + let operation: Operation = management::Client::new(connection) + .create_dummy_job(nanos) + .await? + .try_into()?; + serde_json::to_writer_pretty(std::io::stdout(), &operation)?; + } + } + + Ok(()) +} diff --git a/src/commands/run.rs b/src/commands/run.rs new file mode 100644 index 0000000000..ac33042f14 --- /dev/null +++ b/src/commands/run.rs @@ -0,0 +1,341 @@ +//! Implementation of command line option for running server + +use crate::commands::logging::LoggingLevel; +use crate::influxdb_ioxd; +use clap::arg_enum; +use std::{net::SocketAddr, net::ToSocketAddrs, path::PathBuf}; +use structopt::StructOpt; +use thiserror::Error; + +/// The default bind address for the HTTP API. +pub const DEFAULT_API_BIND_ADDR: &str = "127.0.0.1:8080"; + +/// The default bind address for the gRPC. +pub const DEFAULT_GRPC_BIND_ADDR: &str = "127.0.0.1:8082"; + +/// The AWS region to use for Amazon S3 based object storage if none is +/// specified. +pub const FALLBACK_AWS_REGION: &str = "us-east-1"; + +#[derive(Debug, Error)] +pub enum Error { + #[error("Run: {0}")] + ServerError(#[from] influxdb_ioxd::Error), +} + +pub type Result = std::result::Result; + +#[derive(Debug, StructOpt)] +#[structopt( + name = "run", + about = "Runs in server mode", + long_about = "Run the IOx server.\n\nThe configuration options below can be \ + set either with the command line flags or with the specified environment \ + variable. If there is a file named '.env' in the current working directory, \ + it is sourced before loading the configuration. + +Configuration is loaded from the following sources (highest precedence first): + - command line arguments + - user set environment variables + - .env file contents + - pre-configured default values" +)] +pub struct Config { + /// This controls the IOx server logging level, as described in + /// https://crates.io/crates/env_logger. + /// + /// Levels for different modules can be specified as well. For example + /// `debug,hyper::proto::h1=info` specifies debug logging for all modules + /// except for the `hyper::proto::h1' module which will only display info + /// level logging. + #[structopt(long = "--log", env = "RUST_LOG")] + pub rust_log: Option, + + /// Log message format. Can be one of: + /// + /// "rust" (default) + /// "logfmt" (logfmt/Heroku style - https://brandur.org/logfmt) + #[structopt(long = "--log_format", env = "INFLUXDB_IOX_LOG_FORMAT")] + pub log_format: Option, + + /// This sets logging up with a pre-configured set of convenient log levels. + /// + /// -v means 'info' log levels + /// -vv means 'verbose' log level (with the exception of some particularly + /// low level libraries) + /// + /// This option is ignored if --log / RUST_LOG are set + #[structopt( + short = "-v", + long = "--verbose", + multiple = true, + takes_value = false, + parse(from_occurrences) + )] + pub verbose_count: u64, + + /// The identifier for the server. + /// + /// Used for writing to object storage and as an identifier that is added to + /// replicated writes, WAL segments and Chunks. Must be unique in a group of + /// connected or semi-connected IOx servers. Must be a number that can be + /// represented by a 32-bit unsigned integer. + #[structopt(long = "--writer-id", env = "INFLUXDB_IOX_ID")] + pub writer_id: Option, + + /// The address on which IOx will serve HTTP API requests. + #[structopt( + long = "--api-bind", + env = "INFLUXDB_IOX_BIND_ADDR", + default_value = DEFAULT_API_BIND_ADDR, + parse(try_from_str = parse_socket_addr), + )] + pub http_bind_address: SocketAddr, + + /// The address on which IOx will serve Storage gRPC API requests. + #[structopt( + long = "--grpc-bind", + env = "INFLUXDB_IOX_GRPC_BIND_ADDR", + default_value = DEFAULT_GRPC_BIND_ADDR, + parse(try_from_str = parse_socket_addr), + )] + pub grpc_bind_address: SocketAddr, + + /// The location InfluxDB IOx will use to store files locally. + #[structopt(long = "--data-dir", env = "INFLUXDB_IOX_DB_DIR")] + pub database_directory: Option, + + #[structopt( + long = "--object-store", + env = "INFLUXDB_IOX_OBJECT_STORE", + possible_values = &ObjectStore::variants(), + case_insensitive = true, + long_help = r#"Which object storage to use. If not specified, defaults to memory. + +Possible values (case insensitive): + +* memory (default): Effectively no object persistence. +* file: Stores objects in the local filesystem. Must also set `--data-dir`. +* s3: Amazon S3. Must also set `--bucket`, `--aws-access-key-id`, `--aws-secret-access-key`, and + possibly `--aws-default-region`. +* google: Google Cloud Storage. Must also set `--bucket` and `--google-service-account`. +* azure: Microsoft Azure blob storage. Must also set `--bucket`, `--azure-storage-account`, + and `--azure-storage-access-key`. + "#, + )] + pub object_store: Option, + + /// Name of the bucket to use for the object store. Must also set + /// `--object-store` to a cloud object storage to have any effect. + /// + /// If using Google Cloud Storage for the object store, this item as well + /// as `--google-service-account` must be set. + /// + /// If using S3 for the object store, must set this item as well + /// as `--aws-access-key-id` and `--aws-secret-access-key`. Can also set + /// `--aws-default-region` if not using the fallback region. + /// + /// If using Azure for the object store, set this item to the name of a + /// container you've created in the associated storage account, under + /// Blob Service > Containers. Must also set `--azure-storage-account` and + /// `--azure-storage-access-key`. + #[structopt(long = "--bucket", env = "INFLUXDB_IOX_BUCKET")] + pub bucket: Option, + + /// When using Amazon S3 as the object store, set this to an access key that + /// has permission to read from and write to the specified S3 bucket. + /// + /// Must also set `--object-store=s3`, `--bucket`, and + /// `--aws-secret-access-key`. Can also set `--aws-default-region` if not + /// using the fallback region. + /// + /// Prefer the environment variable over the command line flag in shared + /// environments. + #[structopt(long = "--aws-access-key-id", env = "AWS_ACCESS_KEY_ID")] + pub aws_access_key_id: Option, + + /// When using Amazon S3 as the object store, set this to the secret access + /// key that goes with the specified access key ID. + /// + /// Must also set `--object-store=s3`, `--bucket`, `--aws-access-key-id`. + /// Can also set `--aws-default-region` if not using the fallback region. + /// + /// Prefer the environment variable over the command line flag in shared + /// environments. + #[structopt(long = "--aws-secret-access-key", env = "AWS_SECRET_ACCESS_KEY")] + pub aws_secret_access_key: Option, + + /// When using Amazon S3 as the object store, set this to the region + /// that goes with the specified bucket if different from the fallback + /// value. + /// + /// Must also set `--object-store=s3`, `--bucket`, `--aws-access-key-id`, + /// and `--aws-secret-access-key`. + #[structopt( + long = "--aws-default-region", + env = "AWS_DEFAULT_REGION", + default_value = FALLBACK_AWS_REGION, + )] + pub aws_default_region: String, + + /// When using Google Cloud Storage as the object store, set this to the + /// path to the JSON file that contains the Google credentials. + /// + /// Must also set `--object-store=google` and `--bucket`. + #[structopt(long = "--google-service-account", env = "GOOGLE_SERVICE_ACCOUNT")] + pub google_service_account: Option, + + /// When using Microsoft Azure as the object store, set this to the + /// name you see when going to All Services > Storage accounts > [name]. + /// + /// Must also set `--object-store=azure`, `--bucket`, and + /// `--azure-storage-access-key`. + #[structopt(long = "--azure-storage-account", env = "AZURE_STORAGE_ACCOUNT")] + pub azure_storage_account: Option, + + /// When using Microsoft Azure as the object store, set this to one of the + /// Key values in the Storage account's Settings > Access keys. + /// + /// Must also set `--object-store=azure`, `--bucket`, and + /// `--azure-storage-account`. + /// + /// Prefer the environment variable over the command line flag in shared + /// environments. + #[structopt(long = "--azure-storage-access-key", env = "AZURE_STORAGE_ACCESS_KEY")] + pub azure_storage_access_key: Option, + + /// If set, Jaeger traces are emitted to this host + /// using the OpenTelemetry tracer. + /// + /// NOTE: The OpenTelemetry agent CAN ONLY be + /// configured using environment variables. It CAN NOT be configured + /// using the command line at this time. Some useful variables: + /// + /// * OTEL_SERVICE_NAME: emitter service name (iox by default) + /// * OTEL_EXPORTER_JAEGER_AGENT_HOST: hostname/address of the collector + /// * OTEL_EXPORTER_JAEGER_AGENT_PORT: listening port of the collector. + /// + /// The entire list of variables can be found in + /// https://github.com/open-telemetry/opentelemetry-specification/blob/master/specification/sdk-environment-variables.md#jaeger-exporter + #[structopt( + long = "--oetl_exporter_jaeger_agent", + env = "OTEL_EXPORTER_JAEGER_AGENT_HOST" + )] + pub jaeger_host: Option, +} + +pub async fn command(logging_level: LoggingLevel, config: Config) -> Result<()> { + Ok(influxdb_ioxd::main(logging_level, config).await?) +} + +fn parse_socket_addr(s: &str) -> std::io::Result { + let mut addrs = s.to_socket_addrs()?; + // when name resolution fails, to_socket_address returns a validation error + // so generally there is at least one result address, unless the resolver is + // drunk. + Ok(addrs + .next() + .expect("name resolution should return at least one address")) +} + +arg_enum! { + #[derive(Debug, Copy, Clone, PartialEq)] + pub enum ObjectStore { + Memory, + File, + S3, + Google, + Azure, + } +} + +/// How to format output logging messages +#[derive(Debug, Clone, Copy)] +pub enum LogFormat { + /// Default formatted logging + /// + /// Example: + /// ``` + /// level=warn msg="NO PERSISTENCE: using memory for object storage" target="influxdb_iox::influxdb_ioxd" + /// ``` + Rust, + + /// Use the (somwhat pretentiously named) Heroku / logfmt formatted output + /// format + /// + /// Example: + /// ``` + /// Jan 31 13:19:39.059 WARN influxdb_iox::influxdb_ioxd: NO PERSISTENCE: using memory for object storage + /// ``` + LogFmt, +} + +impl Default for LogFormat { + fn default() -> Self { + Self::Rust + } +} + +impl std::str::FromStr for LogFormat { + type Err = String; + + fn from_str(s: &str) -> Result { + match s.to_ascii_lowercase().as_str() { + "rust" => Ok(Self::Rust), + "logfmt" => Ok(Self::LogFmt), + _ => Err(format!( + "Invalid log format '{}'. Valid options: rust, logfmt", + s + )), + } + } +} + +#[cfg(test)] +mod tests { + + use super::*; + + use std::net::{Ipv4Addr, Ipv6Addr, SocketAddrV4, SocketAddrV6}; + + fn to_vec(v: &[&str]) -> Vec { + v.iter().map(|s| s.to_string()).collect() + } + + #[test] + fn test_socketaddr() -> Result<(), clap::Error> { + let c = Config::from_iter_safe( + to_vec(&["server", "--api-bind", "127.0.0.1:1234"]).into_iter(), + )?; + assert_eq!( + c.http_bind_address, + SocketAddr::from(([127, 0, 0, 1], 1234)) + ); + + let c = Config::from_iter_safe( + to_vec(&["server", "--api-bind", "localhost:1234"]).into_iter(), + )?; + // depending on where the test runs, localhost will either resolve to a ipv4 or + // an ipv6 addr. + match c.http_bind_address { + SocketAddr::V4(so) => { + assert_eq!(so, SocketAddrV4::new(Ipv4Addr::new(127, 0, 0, 1), 1234)) + } + SocketAddr::V6(so) => assert_eq!( + so, + SocketAddrV6::new(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1), 1234, 0, 0) + ), + }; + + assert_eq!( + Config::from_iter_safe( + to_vec(&["server", "--api-bind", "!@INv_a1d(ad0/resp_!"]).into_iter(), + ) + .map_err(|e| e.kind) + .expect_err("must fail"), + clap::ErrorKind::ValueValidation + ); + + Ok(()) + } +} diff --git a/src/commands/server.rs b/src/commands/server.rs index 33ca6f2d0e..3bad401af3 100644 --- a/src/commands/server.rs +++ b/src/commands/server.rs @@ -1,409 +1,26 @@ //! Implementation of command line option for manipulating and showing server //! config -use clap::arg_enum; -use std::{net::SocketAddr, net::ToSocketAddrs, path::PathBuf}; +use crate::commands::server_remote; use structopt::StructOpt; +use thiserror::Error; -/// The default bind address for the HTTP API. -pub const DEFAULT_API_BIND_ADDR: &str = "127.0.0.1:8080"; +#[derive(Debug, Error)] +pub enum Error { + #[error("Remote: {0}")] + RemoteError(#[from] server_remote::Error), +} -/// The default bind address for the gRPC. -pub const DEFAULT_GRPC_BIND_ADDR: &str = "127.0.0.1:8082"; - -/// The AWS region to use for Amazon S3 based object storage if none is -/// specified. -pub const FALLBACK_AWS_REGION: &str = "us-east-1"; +pub type Result = std::result::Result; #[derive(Debug, StructOpt)] -#[structopt( - name = "server", - about = "Runs in server mode (default)", - long_about = "Run the IOx server.\n\nThe configuration options below can be \ - set either with the command line flags or with the specified environment \ - variable. If there is a file named '.env' in the current working directory, \ - it is sourced before loading the configuration. - -Configuration is loaded from the following sources (highest precedence first): - - command line arguments - - user set environment variables - - .env file contents - - pre-configured default values" -)] -pub struct Config { - /// This controls the IOx server logging level, as described in - /// https://crates.io/crates/env_logger. - /// - /// Levels for different modules can be specified as well. For example - /// `debug,hyper::proto::h1=info` specifies debug logging for all modules - /// except for the `hyper::proto::h1' module which will only display info - /// level logging. - #[structopt(long = "--log", env = "RUST_LOG")] - pub rust_log: Option, - - /// Log message format. Can be one of: - /// - /// "rust" (default) - /// "logfmt" (logfmt/Heroku style - https://brandur.org/logfmt) - #[structopt(long = "--log_format", env = "INFLUXDB_IOX_LOG_FORMAT")] - pub log_format: Option, - - /// This sets logging up with a pre-configured set of convenient log levels. - /// - /// -v means 'info' log levels - /// -vv means 'verbose' log level (with the exception of some particularly - /// low level libraries) - /// - /// This option is ignored if --log / RUST_LOG are set - #[structopt( - short = "-v", - long = "--verbose", - multiple = true, - takes_value = false, - parse(from_occurrences) - )] - pub verbose_count: u64, - - /// The identifier for the server. - /// - /// Used for writing to object storage and as an identifier that is added to - /// replicated writes, WAL segments and Chunks. Must be unique in a group of - /// connected or semi-connected IOx servers. Must be a number that can be - /// represented by a 32-bit unsigned integer. - #[structopt(long = "--writer-id", env = "INFLUXDB_IOX_ID")] - pub writer_id: Option, - - /// The address on which IOx will serve HTTP API requests. - #[structopt( - long = "--api-bind", - env = "INFLUXDB_IOX_BIND_ADDR", - default_value = DEFAULT_API_BIND_ADDR, - parse(try_from_str = parse_socket_addr), - )] - pub http_bind_address: SocketAddr, - - /// The address on which IOx will serve Storage gRPC API requests. - #[structopt( - long = "--grpc-bind", - env = "INFLUXDB_IOX_GRPC_BIND_ADDR", - default_value = DEFAULT_GRPC_BIND_ADDR, - parse(try_from_str = parse_socket_addr), - )] - pub grpc_bind_address: SocketAddr, - - /// The location InfluxDB IOx will use to store files locally. - #[structopt(long = "--data-dir", env = "INFLUXDB_IOX_DB_DIR")] - pub database_directory: Option, - - #[structopt( - long = "--object-store", - env = "INFLUXDB_IOX_OBJECT_STORE", - possible_values = &ObjectStore::variants(), - case_insensitive = true, - long_help = r#"Which object storage to use. If not specified, defaults to memory. - -Possible values (case insensitive): - -* memory (default): Effectively no object persistence. -* file: Stores objects in the local filesystem. Must also set `--data-dir`. -* s3: Amazon S3. Must also set `--bucket`, `--aws-access-key-id`, `--aws-secret-access-key`, and - possibly `--aws-default-region`. -* google: Google Cloud Storage. Must also set `--bucket` and `--google-service-account`. -* azure: Microsoft Azure blob storage. Must also set `--bucket`, `--azure-storage-account`, - and `--azure-storage-access-key`. - "#, - )] - pub object_store: Option, - - /// Name of the bucket to use for the object store. Must also set - /// `--object-store` to a cloud object storage to have any effect. - /// - /// If using Google Cloud Storage for the object store, this item as well - /// as `--google-service-account` must be set. - /// - /// If using S3 for the object store, must set this item as well - /// as `--aws-access-key-id` and `--aws-secret-access-key`. Can also set - /// `--aws-default-region` if not using the fallback region. - /// - /// If using Azure for the object store, set this item to the name of a - /// container you've created in the associated storage account, under - /// Blob Service > Containers. Must also set `--azure-storage-account` and - /// `--azure-storage-access-key`. - #[structopt(long = "--bucket", env = "INFLUXDB_IOX_BUCKET")] - pub bucket: Option, - - /// When using Amazon S3 as the object store, set this to an access key that - /// has permission to read from and write to the specified S3 bucket. - /// - /// Must also set `--object-store=s3`, `--bucket`, and - /// `--aws-secret-access-key`. Can also set `--aws-default-region` if not - /// using the fallback region. - /// - /// Prefer the environment variable over the command line flag in shared - /// environments. - #[structopt(long = "--aws-access-key-id", env = "AWS_ACCESS_KEY_ID")] - pub aws_access_key_id: Option, - - /// When using Amazon S3 as the object store, set this to the secret access - /// key that goes with the specified access key ID. - /// - /// Must also set `--object-store=s3`, `--bucket`, `--aws-access-key-id`. - /// Can also set `--aws-default-region` if not using the fallback region. - /// - /// Prefer the environment variable over the command line flag in shared - /// environments. - #[structopt(long = "--aws-secret-access-key", env = "AWS_SECRET_ACCESS_KEY")] - pub aws_secret_access_key: Option, - - /// When using Amazon S3 as the object store, set this to the region - /// that goes with the specified bucket if different from the fallback - /// value. - /// - /// Must also set `--object-store=s3`, `--bucket`, `--aws-access-key-id`, - /// and `--aws-secret-access-key`. - #[structopt( - long = "--aws-default-region", - env = "AWS_DEFAULT_REGION", - default_value = FALLBACK_AWS_REGION, - )] - pub aws_default_region: String, - - /// When using Google Cloud Storage as the object store, set this to the - /// path to the JSON file that contains the Google credentials. - /// - /// Must also set `--object-store=google` and `--bucket`. - #[structopt(long = "--google-service-account", env = "GOOGLE_SERVICE_ACCOUNT")] - pub google_service_account: Option, - - /// When using Microsoft Azure as the object store, set this to the - /// name you see when going to All Services > Storage accounts > [name]. - /// - /// Must also set `--object-store=azure`, `--bucket`, and - /// `--azure-storage-access-key`. - #[structopt(long = "--azure-storage-account", env = "AZURE_STORAGE_ACCOUNT")] - pub azure_storage_account: Option, - - /// When using Microsoft Azure as the object store, set this to one of the - /// Key values in the Storage account's Settings > Access keys. - /// - /// Must also set `--object-store=azure`, `--bucket`, and - /// `--azure-storage-account`. - /// - /// Prefer the environment variable over the command line flag in shared - /// environments. - #[structopt(long = "--azure-storage-access-key", env = "AZURE_STORAGE_ACCESS_KEY")] - pub azure_storage_access_key: Option, - - /// If set, Jaeger traces are emitted to this host - /// using the OpenTelemetry tracer. - /// - /// NOTE: The OpenTelemetry agent CAN ONLY be - /// configured using environment variables. It CAN NOT be configured - /// using the command line at this time. Some useful variables: - /// - /// * OTEL_SERVICE_NAME: emitter service name (iox by default) - /// * OTEL_EXPORTER_JAEGER_AGENT_HOST: hostname/address of the collector - /// * OTEL_EXPORTER_JAEGER_AGENT_PORT: listening port of the collector. - /// - /// The entire list of variables can be found in - /// https://github.com/open-telemetry/opentelemetry-specification/blob/master/specification/sdk-environment-variables.md#jaeger-exporter - #[structopt( - long = "--oetl_exporter_jaeger_agent", - env = "OTEL_EXPORTER_JAEGER_AGENT_HOST" - )] - pub jaeger_host: Option, +#[structopt(name = "server", about = "IOx server commands")] +pub enum Config { + Remote(crate::commands::server_remote::Config), } -/// Load the config if `server` was not specified on the command line -/// (from environment variables and default) -/// -/// This pulls in config from the following sources, in order of precedence: -/// -/// - user set environment variables -/// - .env file contents -/// - pre-configured default values -pub fn load_config() -> Box { - // Load the Config struct - this pulls in any envs set by the user or - // sourced above, and applies any defaults. - // - - //let args = std::env::args().filter(|arg| arg != "server"); - Box::new(Config::from_iter(strip_server(std::env::args()).iter())) -} - -fn parse_socket_addr(s: &str) -> std::io::Result { - let mut addrs = s.to_socket_addrs()?; - // when name resolution fails, to_socket_address returns a validation error - // so generally there is at least one result address, unless the resolver is - // drunk. - Ok(addrs - .next() - .expect("name resolution should return at least one address")) -} - -/// Strip everything prior to the "server" portion of the args so the generated -/// Clap instance plays nicely with the subcommand bits in main. -fn strip_server(args: impl Iterator) -> Vec { - let mut seen_server = false; - args.enumerate() - .filter_map(|(i, arg)| { - if i != 0 && !seen_server { - if arg == "server" { - seen_server = true; - } - None - } else { - Some(arg) - } - }) - .collect::>() -} - -arg_enum! { - #[derive(Debug, Copy, Clone, PartialEq)] - pub enum ObjectStore { - Memory, - File, - S3, - Google, - Azure, - } -} - -/// How to format output logging messages -#[derive(Debug, Clone, Copy)] -pub enum LogFormat { - /// Default formatted logging - /// - /// Example: - /// ``` - /// level=warn msg="NO PERSISTENCE: using memory for object storage" target="influxdb_iox::influxdb_ioxd" - /// ``` - Rust, - - /// Use the (somwhat pretentiously named) Heroku / logfmt formatted output - /// format - /// - /// Example: - /// ``` - /// Jan 31 13:19:39.059 WARN influxdb_iox::influxdb_ioxd: NO PERSISTENCE: using memory for object storage - /// ``` - LogFmt, -} - -impl Default for LogFormat { - fn default() -> Self { - Self::Rust - } -} - -impl std::str::FromStr for LogFormat { - type Err = String; - - fn from_str(s: &str) -> Result { - match s.to_ascii_lowercase().as_str() { - "rust" => Ok(Self::Rust), - "logfmt" => Ok(Self::LogFmt), - _ => Err(format!( - "Invalid log format '{}'. Valid options: rust, logfmt", - s - )), - } - } -} - -#[cfg(test)] -mod tests { - - use super::*; - - use std::net::{Ipv4Addr, Ipv6Addr, SocketAddrV4, SocketAddrV6}; - - #[test] - fn test_strip_server() { - assert_eq!( - strip_server(to_vec(&["cmd",]).into_iter()), - to_vec(&["cmd"]) - ); - assert_eq!( - strip_server(to_vec(&["cmd", "-v"]).into_iter()), - to_vec(&["cmd"]) - ); - assert_eq!( - strip_server(to_vec(&["cmd", "-v", "server"]).into_iter()), - to_vec(&["cmd"]) - ); - assert_eq!( - strip_server(to_vec(&["cmd", "-v", "server", "-v"]).into_iter()), - to_vec(&["cmd", "-v"]) - ); - assert_eq!( - strip_server(to_vec(&["cmd", "-v", "server", "-vv"]).into_iter()), - to_vec(&["cmd", "-vv"]) - ); - - // and it doesn't strip repeated instances of server - assert_eq!( - strip_server(to_vec(&["cmd", "-v", "server", "--gcp_path"]).into_iter()), - to_vec(&["cmd", "--gcp_path"]) - ); - assert_eq!( - strip_server(to_vec(&["cmd", "-v", "server", "--gcp_path", "server"]).into_iter()), - to_vec(&["cmd", "--gcp_path", "server"]) - ); - - assert_eq!( - strip_server(to_vec(&["cmd", "-vv"]).into_iter()), - to_vec(&["cmd"]) - ); - assert_eq!( - strip_server(to_vec(&["cmd", "-vv", "server"]).into_iter()), - to_vec(&["cmd"]) - ); - assert_eq!( - strip_server(to_vec(&["cmd", "-vv", "server", "-vv"]).into_iter()), - to_vec(&["cmd", "-vv"]) - ); - } - - fn to_vec(v: &[&str]) -> Vec { - v.iter().map(|s| s.to_string()).collect() - } - - #[test] - fn test_socketaddr() -> Result<(), clap::Error> { - let c = Config::from_iter_safe(strip_server( - to_vec(&["cmd", "server", "--api-bind", "127.0.0.1:1234"]).into_iter(), - ))?; - assert_eq!( - c.http_bind_address, - SocketAddr::from(([127, 0, 0, 1], 1234)) - ); - - let c = Config::from_iter_safe(strip_server( - to_vec(&["cmd", "server", "--api-bind", "localhost:1234"]).into_iter(), - ))?; - // depending on where the test runs, localhost will either resolve to a ipv4 or - // an ipv6 addr. - match c.http_bind_address { - SocketAddr::V4(so) => { - assert_eq!(so, SocketAddrV4::new(Ipv4Addr::new(127, 0, 0, 1), 1234)) - } - SocketAddr::V6(so) => assert_eq!( - so, - SocketAddrV6::new(Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1), 1234, 0, 0) - ), - }; - - assert_eq!( - Config::from_iter_safe(strip_server( - to_vec(&["cmd", "server", "--api-bind", "!@INv_a1d(ad0/resp_!"]).into_iter(), - )) - .map_err(|e| e.kind) - .expect_err("must fail"), - clap::ErrorKind::ValueValidation - ); - - Ok(()) +pub async fn command(url: String, config: Config) -> Result<()> { + match config { + Config::Remote(config) => Ok(server_remote::command(url, config).await?), } } diff --git a/src/commands/server_remote.rs b/src/commands/server_remote.rs new file mode 100644 index 0000000000..5a6c4fef1c --- /dev/null +++ b/src/commands/server_remote.rs @@ -0,0 +1,76 @@ +use influxdb_iox_client::{connection::Builder, management}; +use structopt::StructOpt; +use thiserror::Error; + +use prettytable::{format, Cell, Row, Table}; + +#[derive(Debug, Error)] +pub enum Error { + #[error("Error connecting to IOx: {0}")] + ConnectionError(#[from] influxdb_iox_client::connection::Error), + + #[error("Update remote error: {0}")] + UpdateError(#[from] management::UpdateRemoteError), + + #[error("List remote error: {0}")] + ListError(#[from] management::ListRemotesError), +} + +pub type Result = std::result::Result; + +#[derive(Debug, StructOpt)] +#[structopt( + name = "remote", + about = "Manage configuration about other IOx servers" +)] +pub enum Config { + /// Set connection parameters for a remote IOx server. + Set { id: u32, connection_string: String }, + /// Remove a reference to a remote IOx server. + Remove { id: u32 }, + /// List configured remote IOx server. + List, +} + +pub async fn command(url: String, config: Config) -> Result<()> { + let connection = Builder::default().build(url).await?; + + match config { + Config::Set { + id, + connection_string, + } => { + let mut client = management::Client::new(connection); + client.update_remote(id, connection_string).await?; + } + Config::Remove { id } => { + let mut client = management::Client::new(connection); + client.delete_remote(id).await?; + } + Config::List => { + let mut client = management::Client::new(connection); + + let remotes = client.list_remotes().await?; + if remotes.is_empty() { + println!("no remotes configured"); + } else { + let mut table = Table::new(); + table.set_format(*format::consts::FORMAT_NO_LINESEP_WITH_TITLE); + table.set_titles(Row::new(vec![ + Cell::new("ID"), + Cell::new("Connection string"), + ])); + + for i in remotes { + table.add_row(Row::new(vec![ + Cell::new(&format!("{}", i.id)), + Cell::new(&i.connection_string), + ])); + } + print!("{}", table); + } + } + }; + + Ok(()) +} diff --git a/src/cpu_feature_check/main.rs b/src/cpu_feature_check/main.rs deleted file mode 100644 index d4b152d7c7..0000000000 --- a/src/cpu_feature_check/main.rs +++ /dev/null @@ -1,67 +0,0 @@ -//! This program prints what available x86 features are available on this -//! processor - -macro_rules! check_feature { - ($name: tt) => { - println!(" {:10}: {}", $name, std::is_x86_feature_detected!($name)) - }; -} - -fn main() { - println!("Available CPU features on this machine:"); - - // The list of possibilities was taken from - // https://doc.rust-lang.org/reference/attributes/codegen.html#the-target_feature-attribute - // - // Features that are commented out are experimental - check_feature!("aes"); - check_feature!("pclmulqdq"); - check_feature!("rdrand"); - check_feature!("rdseed"); - check_feature!("tsc"); - check_feature!("mmx"); - check_feature!("sse"); - check_feature!("sse2"); - check_feature!("sse3"); - check_feature!("ssse3"); - check_feature!("sse4.1"); - check_feature!("sse4.2"); - check_feature!("sse4a"); - check_feature!("sha"); - check_feature!("avx"); - check_feature!("avx2"); - check_feature!("avx512f"); - check_feature!("avx512cd"); - check_feature!("avx512er"); - check_feature!("avx512pf"); - check_feature!("avx512bw"); - check_feature!("avx512dq"); - check_feature!("avx512vl"); - //check_feature!("avx512ifma"); - // check_feature!("avx512vbmi"); - // check_feature!("avx512vpopcntdq"); - // check_feature!("avx512vbmi2"); - // check_feature!("avx512gfni"); - // check_feature!("avx512vaes"); - // check_feature!("avx512vpclmulqdq"); - // check_feature!("avx512vnni"); - // check_feature!("avx512bitalg"); - // check_feature!("avx512bf16"); - // check_feature!("avx512vp2intersect"); - //check_feature!("f16c"); - check_feature!("fma"); - check_feature!("bmi1"); - check_feature!("bmi2"); - check_feature!("abm"); - check_feature!("lzcnt"); - check_feature!("tbm"); - check_feature!("popcnt"); - check_feature!("fxsr"); - check_feature!("xsave"); - check_feature!("xsaveopt"); - check_feature!("xsaves"); - check_feature!("xsavec"); - //check_feature!("cmpxchg16b"); - check_feature!("adx"); - //check_feature!("rtm"); -} diff --git a/src/influxdb_ioxd.rs b/src/influxdb_ioxd.rs index dae3b6fedc..6e21fd589d 100644 --- a/src/influxdb_ioxd.rs +++ b/src/influxdb_ioxd.rs @@ -1,8 +1,9 @@ use crate::commands::{ logging::LoggingLevel, - server::{load_config, Config, ObjectStore as ObjStoreOpt}, + run::{Config, ObjectStore as ObjStoreOpt}, }; -use hyper::Server; +use futures::{future::FusedFuture, pin_mut, FutureExt}; +use hyper::server::conn::AddrIncoming; use object_store::{ self, aws::AmazonS3, azure::MicrosoftAzure, gcp::GoogleCloudStorage, ObjectStore, }; @@ -10,7 +11,7 @@ use panic_logging::SendPanicsToTracing; use server::{ConnectionManagerImpl as ConnectionManager, Server as AppServer}; use snafu::{ResultExt, Snafu}; use std::{convert::TryFrom, fs, net::SocketAddr, path::PathBuf, sync::Arc}; -use tracing::{error, info, warn}; +use tracing::{error, info, warn, Instrument}; mod http; mod rpc; @@ -47,7 +48,7 @@ pub enum Error { ServingHttp { source: hyper::Error }, #[snafu(display("Error serving RPC: {}", source))] - ServingRPC { source: self::rpc::Error }, + ServingRPC { source: tonic::transport::Error }, #[snafu(display( "Specified {} for the object store, required configuration missing for {}", @@ -68,15 +69,33 @@ pub enum Error { pub type Result = std::result::Result; +/// On unix platforms we want to intercept SIGINT and SIGTERM +/// This method returns if either are signalled +#[cfg(unix)] +async fn wait_for_signal() { + use tokio::signal::unix::{signal, SignalKind}; + let mut term = signal(SignalKind::terminate()).expect("failed to register signal handler"); + let mut int = signal(SignalKind::interrupt()).expect("failed to register signal handler"); + + tokio::select! { + _ = term.recv() => info!("Received SIGTERM"), + _ = int.recv() => info!("Received SIGINT"), + } +} + +#[cfg(windows)] +/// ctrl_c is the cross-platform way to intercept the equivalent of SIGINT +/// This method returns if this occurs +async fn wait_for_signal() { + let _ = tokio::signal::ctrl_c().await; +} + /// This is the entry point for the IOx server. `config` represents /// command line arguments, if any /// /// The logging_level passed in is the global setting (e.g. if -v or /// -vv was passed in before 'server') -pub async fn main(logging_level: LoggingLevel, config: Option>) -> Result<()> { - // load config from environment if no command line - let config = config.unwrap_or_else(load_config); - +pub async fn main(logging_level: LoggingLevel, config: Config) -> Result<()> { // Handle the case if -v/-vv is specified both before and after the server // command let logging_level = logging_level.combine(LoggingLevel::new(config.verbose_count)); @@ -101,7 +120,7 @@ pub async fn main(logging_level: LoggingLevel, config: Option>) -> R } } - let object_store = ObjectStore::try_from(&*config)?; + let object_store = ObjectStore::try_from(&config)?; let object_storage = Arc::new(object_store); let connection_manager = ConnectionManager {}; @@ -121,39 +140,113 @@ pub async fn main(logging_level: LoggingLevel, config: Option>) -> R warn!("server ID not set. ID must be set via the INFLUXDB_IOX_ID config or API before writing or querying data."); } - // Construct and start up gRPC server + // An internal shutdown token for internal workers + let internal_shutdown = tokio_util::sync::CancellationToken::new(); + // Construct a token to trigger shutdown of API services + let frontend_shutdown = internal_shutdown.child_token(); + + // Construct and start up gRPC server let grpc_bind_addr = config.grpc_bind_address; let socket = tokio::net::TcpListener::bind(grpc_bind_addr) .await .context(StartListeningGrpc { grpc_bind_addr })?; - let grpc_server = self::rpc::make_server(socket, Arc::clone(&app_server)); + let grpc_server = rpc::serve(socket, Arc::clone(&app_server), frontend_shutdown.clone()).fuse(); info!(bind_address=?grpc_bind_addr, "gRPC server listening"); - // Construct and start up HTTP server - - let router_service = http::router_service(Arc::clone(&app_server)); - let bind_addr = config.http_bind_address; - let http_server = Server::try_bind(&bind_addr) - .context(StartListeningHttp { bind_addr })? - .serve(router_service); + let addr = AddrIncoming::bind(&bind_addr).context(StartListeningHttp { bind_addr })?; + + let http_server = http::serve(addr, Arc::clone(&app_server), frontend_shutdown.clone()).fuse(); info!(bind_address=?bind_addr, "HTTP server listening"); let git_hash = option_env!("GIT_HASH").unwrap_or("UNKNOWN"); info!(git_hash, "InfluxDB IOx server ready"); - // Wait for both the servers to complete - let (grpc_server, server) = futures::future::join(grpc_server, http_server).await; + // Get IOx background worker task + let server_worker = app_server + .background_worker(internal_shutdown.clone()) + .instrument(tracing::info_span!("server_worker")) + .fuse(); - grpc_server.context(ServingRPC)?; - server.context(ServingHttp)?; + // Shutdown signal + let signal = wait_for_signal().fuse(); - info!("InfluxDB IOx server shutting down"); + // There are two different select macros - tokio::select and futures::select + // + // tokio::select takes ownership of the passed future "moving" it into the + // select block. This works well when not running select inside a loop, or + // when using a future that can be dropped and recreated, often the case + // with tokio's futures e.g. `channel.recv()` + // + // futures::select is more flexible as it doesn't take ownership of the provided + // future. However, to safely provide this it imposes some additional + // requirements + // + // All passed futures must implement FusedFuture - it is IB to poll a future + // that has returned Poll::Ready(_). A FusedFuture has an is_terminated() + // method that indicates if it is safe to poll - e.g. false if it has + // returned Poll::Ready(_). futures::select uses this to implement its + // functionality. futures::FutureExt adds a fuse() method that + // wraps an arbitrary future and makes it a FusedFuture + // + // The additional requirement of futures::select is that if the future passed + // outlives the select block, it must be Unpin or already Pinned - Ok(()) + // pin_mut constructs a Pin<&mut T> from a T by preventing moving the T + // from the current stack frame and constructing a Pin<&mut T> to it + pin_mut!(signal); + pin_mut!(server_worker); + pin_mut!(grpc_server); + pin_mut!(http_server); + + // Return the first error encountered + let mut res = Ok(()); + + // Graceful shutdown can be triggered by sending SIGINT or SIGTERM to the + // process, or by a background task exiting - most likely with an error + // + // Graceful shutdown should then proceed in the following order + // 1. Stop accepting new HTTP and gRPC requests and drain existing connections + // 2. Trigger shutdown of internal background workers loops + // + // This is important to ensure background tasks, such as polling the tracker + // registry, don't exit before HTTP and gRPC requests dependent on them + while !grpc_server.is_terminated() && !http_server.is_terminated() { + futures::select! { + _ = signal => info!("Shutdown requested"), + _ = server_worker => { + info!("server worker shutdown prematurely"); + internal_shutdown.cancel(); + }, + result = grpc_server => match result { + Ok(_) => info!("gRPC server shutdown"), + Err(error) => { + error!(%error, "gRPC server error"); + res = res.and(Err(Error::ServingRPC{source: error})) + } + }, + result = http_server => match result { + Ok(_) => info!("HTTP server shutdown"), + Err(error) => { + error!(%error, "HTTP server error"); + res = res.and(Err(Error::ServingHttp{source: error})) + } + }, + } + + frontend_shutdown.cancel() + } + + info!("frontend shutdown completed"); + internal_shutdown.cancel(); + server_worker.await; + + info!("server completed shutting down"); + + res } impl TryFrom<&Config> for ObjectStore { @@ -198,25 +291,16 @@ impl TryFrom<&Config> for ObjectStore { config.aws_secret_access_key.as_ref(), config.aws_default_region.as_str(), ) { - (Some(bucket), Some(key_id), Some(secret_key), region) => { - Ok(Self::new_amazon_s3( - AmazonS3::new(key_id, secret_key, region, bucket) - .context(InvalidS3Config)?, - )) - } - (bucket, key_id, secret_key, _) => { + (Some(bucket), key_id, secret_key, region) => Ok(Self::new_amazon_s3( + AmazonS3::new(key_id, secret_key, region, bucket) + .context(InvalidS3Config)?, + )), + (bucket, _, _, _) => { let mut missing_args = vec![]; if bucket.is_none() { missing_args.push("bucket"); } - if key_id.is_none() { - missing_args.push("aws-access-key-id"); - } - if secret_key.is_none() { - missing_args.push("aws-secret-access-key"); - } - MissingObjectStoreConfig { object_store: ObjStoreOpt::S3, missing: missing_args.join(", "), @@ -339,8 +423,7 @@ mod tests { assert_eq!( err, - "Specified S3 for the object store, required configuration missing for \ - bucket, aws-access-key-id, aws-secret-access-key" + "Specified S3 for the object store, required configuration missing for bucket" ); } diff --git a/src/influxdb_ioxd/http.rs b/src/influxdb_ioxd/http.rs index d68594dbed..20e76d1cd3 100644 --- a/src/influxdb_ioxd/http.rs +++ b/src/influxdb_ioxd/http.rs @@ -13,11 +13,11 @@ // Influx crates use arrow_deps::datafusion::physical_plan::collect; use data_types::{ - database_rules::DatabaseRules, - http::{ListDatabasesResponse, WalMetadataQuery}, + http::WalMetadataQuery, names::{org_and_bucket_to_database, OrgBucketMappingError}, DatabaseName, }; +use influxdb_iox_client::format::QueryOutputFormat; use influxdb_line_protocol::parse_lines; use object_store::ObjectStoreApi; use query::{frontend::sql::SQLQueryPlanner, Database, DatabaseStore}; @@ -29,15 +29,18 @@ use futures::{self, StreamExt}; use http::header::{CONTENT_ENCODING, CONTENT_TYPE}; use hyper::{Body, Method, Request, Response, StatusCode}; use routerify::{prelude::*, Middleware, RequestInfo, Router, RouterError, RouterService}; -use serde::{Deserialize, Serialize}; +use serde::Deserialize; use snafu::{OptionExt, ResultExt, Snafu}; use tracing::{debug, error, info}; use data_types::http::WalMetadataResponse; -use std::{fmt::Debug, str, sync::Arc}; - -mod format; -use format::QueryOutputFormat; +use hyper::server::conn::AddrIncoming; +use std::{ + fmt::Debug, + str::{self, FromStr}, + sync::Arc, +}; +use tokio_util::sync::CancellationToken; /// Constants used in API error codes. /// @@ -187,6 +190,12 @@ pub enum ApplicationError { #[snafu(display("Internal error creating HTTP response: {}", source))] CreatingResponse { source: http::Error }, + #[snafu(display("Invalid format '{}': : {}", format, source))] + ParsingFormat { + format: String, + source: influxdb_iox_client::format::Error, + }, + #[snafu(display( "Error formatting results of SQL query '{}' using '{:?}': {}", q, @@ -196,7 +205,7 @@ pub enum ApplicationError { FormattingResult { q: String, format: QueryOutputFormat, - source: format::Error, + source: influxdb_iox_client::format::Error, }, } @@ -230,6 +239,7 @@ impl ApplicationError { Self::WALNotFound { .. } => self.not_found(), Self::CreatingResponse { .. } => self.internal_error(), Self::FormattingResult { .. } => self.internal_error(), + Self::ParsingFormat { .. } => self.bad_request(), } } @@ -305,15 +315,9 @@ where Ok(res) })) // this endpoint is for API backward compatibility with InfluxDB 2.x .post("/api/v2/write", write::) - .get("/ping", ping) .get("/health", health) - .get("/iox/api/v1/databases", list_databases::) - .put("/iox/api/v1/databases/:name", create_database::) - .get("/iox/api/v1/databases/:name", get_database::) .get("/iox/api/v1/databases/:name/query", query::) .get("/iox/api/v1/databases/:name/wal/meta", get_wal_meta::) - .put("/iox/api/v1/id", set_writer::) - .get("/iox/api/v1/id", get_writer::) .get("/api/v1/partitions", list_partitions::) .post("/api/v1/snapshot", snapshot_partition::) // Specify the error handler to handle any errors caused by @@ -462,8 +466,12 @@ where /// Parsed URI Parameters of the request to the .../query endpoint struct QueryParams { q: String, - #[serde(default)] - format: QueryOutputFormat, + #[serde(default = "default_format")] + format: String, +} + +fn default_format() -> String { + QueryOutputFormat::default().to_string() } #[tracing::instrument(level = "debug")] @@ -479,6 +487,8 @@ async fn query( query_string: uri_query, })?; + let format = QueryOutputFormat::from_str(&format).context(ParsingFormat { format })?; + let db_name_str = req .param("name") .expect("db name must have been set by routerify") @@ -489,7 +499,6 @@ async fn query( let db = server .db(&db_name) - .await .context(DatabaseNotFound { name: &db_name_str })?; let planner = SQLQueryPlanner::default(); @@ -521,69 +530,6 @@ async fn query( Ok(response) } -#[tracing::instrument(level = "debug")] -async fn list_databases(req: Request) -> Result, ApplicationError> -where - M: ConnectionManager + Send + Sync + Debug + 'static, -{ - let server = Arc::clone(&req.data::>>().expect("server state")); - - let names = server.db_names_sorted().await; - let json = serde_json::to_string(&ListDatabasesResponse { names }) - .context(InternalSerializationError)?; - Ok(Response::new(Body::from(json))) -} - -#[tracing::instrument(level = "debug")] -async fn create_database( - req: Request, -) -> Result, ApplicationError> { - let server = Arc::clone(&req.data::>>().expect("server state")); - - // with routerify, we shouldn't have gotten here without this being set - let db_name = req - .param("name") - .expect("db name must have been set") - .clone(); - let body = parse_body(req).await?; - - let rules: DatabaseRules = serde_json::from_slice(body.as_ref()).context(InvalidRequestBody)?; - - server - .create_database(db_name, rules) - .await - .context(ErrorCreatingDatabase)?; - - Ok(Response::new(Body::empty())) -} - -#[tracing::instrument(level = "debug")] -async fn get_database( - req: Request, -) -> Result, ApplicationError> { - let server = Arc::clone(&req.data::>>().expect("server state")); - - // with routerify, we shouldn't have gotten here without this being set - let db_name_str = req - .param("name") - .expect("db name must have been set") - .clone(); - let db_name = DatabaseName::new(&db_name_str).context(DatabaseNameError)?; - let db = server - .db_rules(&db_name) - .await - .context(DatabaseNotFound { name: &db_name_str })?; - - let data = serde_json::to_string(&db).context(JsonGenerationError)?; - let response = Response::builder() - .header("Content-Type", "application/json") - .status(StatusCode::OK) - .body(Body::from(data)) - .expect("builder should be successful"); - - Ok(response) -} - #[tracing::instrument(level = "debug")] async fn get_wal_meta( req: Request, @@ -610,7 +556,6 @@ async fn get_wal_meta( let db = server .db(&db_name) - .await .context(DatabaseNotFound { name: &db_name_str })?; let wal = db @@ -641,73 +586,6 @@ async fn get_wal_meta( Ok(response) } -#[tracing::instrument(level = "debug")] -async fn set_writer( - req: Request, -) -> Result, ApplicationError> { - let server = Arc::clone(&req.data::>>().expect("server state")); - - // Read the request body - let body = parse_body(req).await?; - - // Parse the JSON body into a structure - #[derive(Serialize, Deserialize)] - struct WriterIdBody { - id: u32, - } - let req: WriterIdBody = serde_json::from_slice(body.as_ref()).context(InvalidRequestBody)?; - - // Set the writer ID - server.set_id(req.id); - - // Build a HTTP 200 response - let response = Response::builder() - .status(StatusCode::OK) - .body(Body::from( - serde_json::to_string(&req).expect("json encoding should not fail"), - )) - .expect("builder should be successful"); - - Ok(response) -} - -#[tracing::instrument(level = "debug")] -async fn get_writer( - req: Request, -) -> Result, ApplicationError> { - let id = { - let server = Arc::clone(&req.data::>>().expect("server state")); - server.require_id() - }; - - // Parse the JSON body into a structure - #[derive(Serialize)] - struct WriterIdBody { - id: u32, - } - - let body = WriterIdBody { - id: id.unwrap_or(0), - }; - - // Build a HTTP 200 response - let response = Response::builder() - .status(StatusCode::OK) - .body(Body::from( - serde_json::to_string(&body).expect("json encoding should not fail"), - )) - .expect("builder should be successful"); - - Ok(response) -} - -// Route to test that the server is alive -#[tracing::instrument(level = "debug")] -async fn ping(_: Request) -> Result, ApplicationError> { - let response_body = "PONG"; - Ok(Response::new(Body::from(response_body.to_string()))) -} - #[tracing::instrument(level = "debug")] async fn health(_: Request) -> Result, ApplicationError> { let response_body = "OK"; @@ -735,7 +613,7 @@ async fn list_partitions( let db_name = org_and_bucket_to_database(&info.org, &info.bucket).context(BucketMappingError)?; - let db = server.db(&db_name).await.context(BucketNotFound { + let db = server.db(&db_name).context(BucketNotFound { org: &info.org, bucket: &info.bucket, })?; @@ -779,7 +657,7 @@ async fn snapshot_partition( +pub async fn serve( + addr: AddrIncoming, server: Arc>, -) -> RouterService { + shutdown: CancellationToken, +) -> Result<(), hyper::Error> +where + M: ConnectionManager + Send + Sync + Debug + 'static, +{ let router = router(server); - RouterService::new(router).unwrap() + let service = RouterService::new(router).unwrap(); + + hyper::Server::builder(addr) + .serve(service) + .with_graceful_shutdown(shutdown.cancelled()) + .await } #[cfg(test)] @@ -824,8 +712,6 @@ mod tests { use query::exec::Executor; use reqwest::{Client, Response}; - use hyper::Server; - use data_types::{ database_rules::{DatabaseRules, WalBufferConfig, WalBufferRollover}, wal::WriterSummary, @@ -838,22 +724,6 @@ mod tests { type Error = Box; type Result = std::result::Result; - #[tokio::test] - async fn test_ping() -> Result<()> { - let test_storage = Arc::new(AppServer::new( - ConnectionManagerImpl {}, - Arc::new(ObjectStore::new_in_memory(InMemory::new())), - )); - let server_url = test_server(Arc::clone(&test_storage)); - - let client = Client::new(); - let response = client.get(&format!("{}/ping", server_url)).send().await; - - // Print the response so if the test fails, we have a log of what went wrong - check_response("ping", response, StatusCode::OK, "PONG").await; - Ok(()) - } - #[tokio::test] async fn test_health() -> Result<()> { let test_storage = Arc::new(AppServer::new( @@ -904,7 +774,6 @@ mod tests { // Check that the data got into the right bucket let test_db = test_storage .db(&DatabaseName::new("MyOrg_MyBucket").unwrap()) - .await .expect("Database exists"); let batches = run_query(test_db.as_ref(), "select * from h2o_temperature").await; @@ -1095,7 +964,6 @@ mod tests { // Check that the data got into the right bucket let test_db = test_storage .db(&DatabaseName::new("MyOrg_MyBucket").unwrap()) - .await .expect("Database exists"); let batches = run_query(test_db.as_ref(), "select * from h2o_temperature").await; @@ -1112,38 +980,6 @@ mod tests { Ok(()) } - #[tokio::test] - async fn set_writer_id() { - let server = Arc::new(AppServer::new( - ConnectionManagerImpl {}, - Arc::new(ObjectStore::new_in_memory(InMemory::new())), - )); - server.set_id(1); - let server_url = test_server(Arc::clone(&server)); - - let data = r#"{"id":42}"#; - - let client = Client::new(); - - let response = client - .put(&format!("{}/iox/api/v1/id", server_url)) - .body(data) - .send() - .await; - - check_response("set_writer_id", response, StatusCode::OK, data).await; - - assert_eq!(server.require_id().expect("should be set"), 42); - - // Check get_writer_id - let response = client - .get(&format!("{}/iox/api/v1/id", server_url)) - .send() - .await; - - check_response("get_writer_id", response, StatusCode::OK, data).await; - } - #[tokio::test] async fn write_to_invalid_database() { let test_storage = Arc::new(AppServer::new( @@ -1178,101 +1014,6 @@ mod tests { .await; } - #[tokio::test] - async fn list_databases() { - let server = Arc::new(AppServer::new( - ConnectionManagerImpl {}, - Arc::new(ObjectStore::new_in_memory(InMemory::new())), - )); - server.set_id(1); - let server_url = test_server(Arc::clone(&server)); - - let database_names: Vec = vec!["foo_bar", "foo_baz"] - .iter() - .map(|i| i.to_string()) - .collect(); - - for database_name in &database_names { - let rules = DatabaseRules { - name: database_name.clone(), - ..Default::default() - }; - server.create_database(database_name, rules).await.unwrap(); - } - - let client = Client::new(); - let response = client - .get(&format!("{}/iox/api/v1/databases", server_url)) - .send() - .await; - - let data = serde_json::to_string(&ListDatabasesResponse { - names: database_names, - }) - .unwrap(); - check_response("list_databases", response, StatusCode::OK, &data).await; - } - - #[tokio::test] - async fn create_database() { - let server = Arc::new(AppServer::new( - ConnectionManagerImpl {}, - Arc::new(ObjectStore::new_in_memory(InMemory::new())), - )); - server.set_id(1); - let server_url = test_server(Arc::clone(&server)); - - let data = r#"{}"#; - - let database_name = DatabaseName::new("foo_bar").unwrap(); - - let client = Client::new(); - let response = client - .put(&format!( - "{}/iox/api/v1/databases/{}", - server_url, database_name - )) - .body(data) - .send() - .await; - - check_response("create_database", response, StatusCode::OK, "").await; - - server.db(&database_name).await.unwrap(); - let db_rules = server.db_rules(&database_name).await.unwrap(); - assert!(db_rules.mutable_buffer_config.is_some()); - } - - #[tokio::test] - async fn get_database() { - let server = Arc::new(AppServer::new( - ConnectionManagerImpl {}, - Arc::new(ObjectStore::new_in_memory(InMemory::new())), - )); - server.set_id(1); - let server_url = test_server(Arc::clone(&server)); - - let database_name = "foo_bar"; - let rules = DatabaseRules { - name: database_name.to_owned(), - ..Default::default() - }; - let data = serde_json::to_string(&rules).unwrap(); - - server.create_database(database_name, rules).await.unwrap(); - - let client = Client::new(); - let response = client - .get(&format!( - "{}/iox/api/v1/databases/{}", - server_url, database_name - )) - .send() - .await; - - check_response("get_database", response, StatusCode::OK, &data).await; - } - #[tokio::test] async fn get_wal_meta() { let server = Arc::new(AppServer::new( @@ -1439,13 +1180,12 @@ mod tests { /// creates an instance of the http service backed by a in-memory /// testable database. Returns the url of the server fn test_server(server: Arc>) -> String { - let make_svc = router_service(server); - // NB: specify port 0 to let the OS pick the port. let bind_addr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), 0); - let server = Server::bind(&bind_addr).serve(make_svc); - let server_url = format!("http://{}", server.local_addr()); - tokio::task::spawn(server); + let addr = AddrIncoming::bind(&bind_addr).expect("failed to bind server"); + let server_url = format!("http://{}", addr.local_addr()); + + tokio::task::spawn(serve(addr, server, CancellationToken::new())); println!("Started server at {}", server_url); server_url } @@ -1458,59 +1198,4 @@ mod tests { collect(physical_plan).await.unwrap() } - - #[test] - fn query_params_format_default() { - // default to pretty format when not otherwise specified - assert_eq!( - serde_urlencoded::from_str("q=foo"), - Ok(QueryParams { - q: "foo".to_string(), - format: QueryOutputFormat::Pretty - }) - ); - } - - #[test] - fn query_params_format_pretty() { - assert_eq!( - serde_urlencoded::from_str("q=foo&format=pretty"), - Ok(QueryParams { - q: "foo".to_string(), - format: QueryOutputFormat::Pretty - }) - ); - } - - #[test] - fn query_params_format_csv() { - assert_eq!( - serde_urlencoded::from_str("q=foo&format=csv"), - Ok(QueryParams { - q: "foo".to_string(), - format: QueryOutputFormat::CSV - }) - ); - } - - #[test] - fn query_params_format_json() { - assert_eq!( - serde_urlencoded::from_str("q=foo&format=json"), - Ok(QueryParams { - q: "foo".to_string(), - format: QueryOutputFormat::JSON - }) - ); - } - - #[test] - fn query_params_bad_format() { - assert_eq!( - serde_urlencoded::from_str::("q=foo&format=jsob") - .unwrap_err() - .to_string(), - "unknown variant `jsob`, expected one of `pretty`, `csv`, `json`" - ); - } } diff --git a/src/influxdb_ioxd/http/format.rs b/src/influxdb_ioxd/http/format.rs deleted file mode 100644 index 1f13493aae..0000000000 --- a/src/influxdb_ioxd/http/format.rs +++ /dev/null @@ -1,127 +0,0 @@ -//! Output formatting utilities for query endpoint - -use serde::Deserialize; -use snafu::{ResultExt, Snafu}; - -use arrow_deps::arrow::{ - self, csv::WriterBuilder, error::ArrowError, json::ArrayWriter, record_batch::RecordBatch, -}; - -#[derive(Debug, Snafu)] -pub enum Error { - #[snafu(display("Arrow pretty printing error: {}", source))] - PrettyArrow { source: ArrowError }, - - #[snafu(display("Arrow csv printing error: {}", source))] - CsvArrow { source: ArrowError }, - - #[snafu(display("Arrow json printing error: {}", source))] - JsonArrow { source: ArrowError }, - - #[snafu(display("Error converting CSV output to UTF-8: {}", source))] - CsvUtf8 { source: std::string::FromUtf8Error }, - - #[snafu(display("Error converting JSON output to UTF-8: {}", source))] - JsonUtf8 { source: std::string::FromUtf8Error }, -} -type Result = std::result::Result; - -#[derive(Deserialize, Debug, Copy, Clone, PartialEq)] -/// Requested output format for the query endpoint -pub enum QueryOutputFormat { - /// Arrow pretty printer format (default) - #[serde(rename = "pretty")] - Pretty, - /// Comma separated values - #[serde(rename = "csv")] - CSV, - /// Arrow JSON format - #[serde(rename = "json")] - JSON, -} - -impl Default for QueryOutputFormat { - fn default() -> Self { - Self::Pretty - } -} - -impl QueryOutputFormat { - /// Return the content type of the relevant format - pub fn content_type(&self) -> &'static str { - match self { - Self::Pretty => "text/plain", - Self::CSV => "text/csv", - Self::JSON => "application/json", - } - } -} - -impl QueryOutputFormat { - /// Format the [`RecordBatch`]es into a String in one of the - /// following formats: - /// - /// Pretty: - /// ```text - /// +----------------+--------------+-------+-----------------+------------+ - /// | bottom_degrees | location | state | surface_degrees | time | - /// +----------------+--------------+-------+-----------------+------------+ - /// | 50.4 | santa_monica | CA | 65.2 | 1568756160 | - /// +----------------+--------------+-------+-----------------+------------+ - /// ``` - /// - /// CSV: - /// ```text - /// bottom_degrees,location,state,surface_degrees,time - /// 50.4,santa_monica,CA,65.2,1568756160 - /// ``` - /// - /// JSON: - /// - /// Example (newline + whitespace added for clarity): - /// ```text - /// [ - /// {"bottom_degrees":50.4,"location":"santa_monica","state":"CA","surface_degrees":65.2,"time":1568756160}, - /// {"location":"Boston","state":"MA","surface_degrees":50.2,"time":1568756160} - /// ] - /// ``` - pub fn format(&self, batches: &[RecordBatch]) -> Result { - match self { - Self::Pretty => batches_to_pretty(&batches), - Self::CSV => batches_to_csv(&batches), - Self::JSON => batches_to_json(&batches), - } - } -} - -fn batches_to_pretty(batches: &[RecordBatch]) -> Result { - arrow::util::pretty::pretty_format_batches(batches).context(PrettyArrow) -} - -fn batches_to_csv(batches: &[RecordBatch]) -> Result { - let mut bytes = vec![]; - - { - let mut writer = WriterBuilder::new().has_headers(true).build(&mut bytes); - - for batch in batches { - writer.write(batch).context(CsvArrow)?; - } - } - let csv = String::from_utf8(bytes).context(CsvUtf8)?; - Ok(csv) -} - -fn batches_to_json(batches: &[RecordBatch]) -> Result { - let mut bytes = vec![]; - - { - let mut writer = ArrayWriter::new(&mut bytes); - writer.write_batches(batches).context(CsvArrow)?; - writer.finish().context(CsvArrow)?; - } - - let json = String::from_utf8(bytes).context(JsonUtf8)?; - - Ok(json) -} diff --git a/src/influxdb_ioxd/rpc.rs b/src/influxdb_ioxd/rpc.rs index 28e2da4c7e..f82c69e02f 100644 --- a/src/influxdb_ioxd/rpc.rs +++ b/src/influxdb_ioxd/rpc.rs @@ -1,31 +1,29 @@ use std::fmt::Debug; use std::sync::Arc; -use snafu::{ResultExt, Snafu}; use tokio::net::TcpListener; use tokio_stream::wrappers::TcpListenerStream; -use data_types::error::ErrorLogger; use server::{ConnectionManager, Server}; +use tokio_util::sync::CancellationToken; +pub mod error; mod flight; mod management; +mod operations; mod storage; mod testing; - -#[derive(Debug, Snafu)] -pub enum Error { - #[snafu(display("gRPC server error: {}", source))] - ServerError { source: tonic::transport::Error }, -} - -pub type Result = std::result::Result; +mod write; /// Instantiate a server listening on the specified address /// implementing the IOx, Storage, and Flight gRPC interfaces, the /// underlying hyper server instance. Resolves when the server has /// shutdown. -pub async fn make_server(socket: TcpListener, server: Arc>) -> Result<()> +pub async fn serve( + socket: TcpListener, + server: Arc>, + shutdown: CancellationToken, +) -> Result<(), tonic::transport::Error> where M: ConnectionManager + Send + Sync + Debug + 'static, { @@ -50,9 +48,9 @@ where .add_service(testing::make_server()) .add_service(storage::make_server(Arc::clone(&server))) .add_service(flight::make_server(Arc::clone(&server))) - .add_service(management::make_server(server)) - .serve_with_incoming(stream) + .add_service(write::make_server(Arc::clone(&server))) + .add_service(management::make_server(Arc::clone(&server))) + .add_service(operations::make_server(server)) + .serve_with_incoming_shutdown(stream, shutdown.cancelled()) .await - .context(ServerError {}) - .log_if_error("Running Tonic Server") } diff --git a/src/influxdb_ioxd/rpc/error.rs b/src/influxdb_ioxd/rpc/error.rs new file mode 100644 index 0000000000..591cf7e18a --- /dev/null +++ b/src/influxdb_ioxd/rpc/error.rs @@ -0,0 +1,54 @@ +use generated_types::google::{FieldViolation, InternalError, NotFound, PreconditionViolation}; +use tracing::error; + +/// map common `server::Error` errors to the appropriate tonic Status +pub fn default_server_error_handler(error: server::Error) -> tonic::Status { + use server::Error; + + match error { + Error::IdNotSet => PreconditionViolation { + category: "Writer ID".to_string(), + subject: "influxdata.com/iox".to_string(), + description: "Writer ID must be set".to_string(), + } + .into(), + Error::DatabaseNotFound { db_name } => NotFound { + resource_type: "database".to_string(), + resource_name: db_name, + ..Default::default() + } + .into(), + Error::InvalidDatabaseName { source } => FieldViolation { + field: "db_name".into(), + description: source.to_string(), + } + .into(), + error => { + error!(?error, "Unexpected error"); + InternalError {}.into() + } + } +} + +/// map common `server::db::Error` errors to the appropriate tonic Status +pub fn default_db_error_handler(error: server::db::Error) -> tonic::Status { + use server::db::Error; + match error { + Error::DatabaseNotReadable {} => PreconditionViolation { + category: "database".to_string(), + subject: "influxdata.com/iox".to_string(), + description: "Cannot read from database: no mutable buffer configured".to_string(), + } + .into(), + Error::DatatbaseNotWriteable {} => PreconditionViolation { + category: "database".to_string(), + subject: "influxdata.com/iox".to_string(), + description: "Cannot write to database: no mutable buffer configured".to_string(), + } + .into(), + error => { + error!(?error, "Unexpected error"); + InternalError {}.into() + } + } +} diff --git a/src/influxdb_ioxd/rpc/flight.rs b/src/influxdb_ioxd/rpc/flight.rs index fbc9c6d60a..3524191104 100644 --- a/src/influxdb_ioxd/rpc/flight.rs +++ b/src/influxdb_ioxd/rpc/flight.rs @@ -132,7 +132,6 @@ where let db = self .db_store .db(&read_info.database_name) - .await .context(DatabaseNotFound { database_name: &read_info.database_name, })?; diff --git a/src/influxdb_ioxd/rpc/management.rs b/src/influxdb_ioxd/rpc/management.rs index b45ab28ba5..02646616c1 100644 --- a/src/influxdb_ioxd/rpc/management.rs +++ b/src/influxdb_ioxd/rpc/management.rs @@ -2,37 +2,19 @@ use std::convert::TryInto; use std::fmt::Debug; use std::sync::Arc; -use tonic::{Request, Response, Status}; -use tracing::error; - use data_types::database_rules::DatabaseRules; use data_types::DatabaseName; -use generated_types::google::{ - AlreadyExists, FieldViolation, FieldViolationExt, InternalError, NotFound, - PreconditionViolation, -}; +use generated_types::google::{AlreadyExists, FieldViolation, FieldViolationExt, NotFound}; use generated_types::influxdata::iox::management::v1::*; -use query::DatabaseStore; +use query::{Database, DatabaseStore}; use server::{ConnectionManager, Error, Server}; +use tonic::{Request, Response, Status}; struct ManagementService { server: Arc>, } -fn default_error_handler(error: Error) -> tonic::Status { - match error { - Error::IdNotSet => PreconditionViolation { - category: "Writer ID".to_string(), - subject: "influxdata.com/iox".to_string(), - description: "Writer ID must be set".to_string(), - } - .into(), - error => { - error!(?error, "Unexpected error"); - InternalError {}.into() - } - } -} +use super::error::{default_db_error_handler, default_server_error_handler}; #[tonic::async_trait] impl management_service_server::ManagementService for ManagementService @@ -61,7 +43,7 @@ where &self, _: Request, ) -> Result, Status> { - let names = self.server.db_names_sorted().await; + let names = self.server.db_names_sorted(); Ok(Response::new(ListDatabasesResponse { names })) } @@ -71,7 +53,7 @@ where ) -> Result, Status> { let name = DatabaseName::new(request.into_inner().name).field("name")?; - match self.server.db_rules(&name).await { + match self.server.db_rules(&name) { Some(rules) => Ok(Response::new(GetDatabaseResponse { rules: Some(rules.into()), })), @@ -110,9 +92,217 @@ where } .into()) } - Err(e) => Err(default_error_handler(e)), + Err(e) => Err(default_server_error_handler(e)), } } + + async fn list_chunks( + &self, + request: Request, + ) -> Result, Status> { + let db_name = DatabaseName::new(request.into_inner().db_name).field("db_name")?; + + let db = match self.server.db(&db_name) { + Some(db) => db, + None => { + return Err(NotFound { + resource_type: "database".to_string(), + resource_name: db_name.to_string(), + ..Default::default() + } + .into()) + } + }; + + let chunk_summaries = match db.chunk_summaries() { + Ok(chunk_summaries) => chunk_summaries, + Err(e) => return Err(default_db_error_handler(e)), + }; + + let chunks: Vec = chunk_summaries + .into_iter() + .map(|summary| summary.into()) + .collect(); + + Ok(Response::new(ListChunksResponse { chunks })) + } + + async fn create_dummy_job( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + let tracker = self.server.spawn_dummy_job(request.nanos); + let operation = Some(super::operations::encode_tracker(tracker)?); + Ok(Response::new(CreateDummyJobResponse { operation })) + } + + async fn list_remotes( + &self, + _: Request, + ) -> Result, Status> { + let remotes = self + .server + .remotes_sorted() + .into_iter() + .map(|(id, connection_string)| Remote { + id, + connection_string, + }) + .collect(); + Ok(Response::new(ListRemotesResponse { remotes })) + } + + async fn update_remote( + &self, + request: Request, + ) -> Result, Status> { + let remote = request + .into_inner() + .remote + .ok_or_else(|| FieldViolation::required("remote"))?; + if remote.id == 0 { + return Err(FieldViolation::required("id").scope("remote").into()); + } + self.server + .update_remote(remote.id, remote.connection_string); + Ok(Response::new(UpdateRemoteResponse {})) + } + + async fn delete_remote( + &self, + request: Request, + ) -> Result, Status> { + let request = request.into_inner(); + if request.id == 0 { + return Err(FieldViolation::required("id").into()); + } + self.server + .delete_remote(request.id) + .ok_or_else(NotFound::default)?; + + Ok(Response::new(DeleteRemoteResponse {})) + } + + async fn list_partitions( + &self, + request: Request, + ) -> Result, Status> { + let ListPartitionsRequest { db_name } = request.into_inner(); + let db_name = DatabaseName::new(db_name).field("db_name")?; + + let db = self.server.db(&db_name).ok_or_else(|| NotFound { + resource_type: "database".to_string(), + resource_name: db_name.to_string(), + ..Default::default() + })?; + + let partition_keys = db.partition_keys().map_err(default_db_error_handler)?; + let partitions = partition_keys + .into_iter() + .map(|key| Partition { key }) + .collect::>(); + + Ok(Response::new(ListPartitionsResponse { partitions })) + } + + async fn get_partition( + &self, + request: Request, + ) -> Result, Status> { + let GetPartitionRequest { + db_name, + partition_key, + } = request.into_inner(); + let db_name = DatabaseName::new(db_name).field("db_name")?; + + let db = self.server.db(&db_name).ok_or_else(|| NotFound { + resource_type: "database".to_string(), + resource_name: db_name.to_string(), + ..Default::default() + })?; + + // TODO: get more actual partition details + let partition_keys = db.partition_keys().map_err(default_db_error_handler)?; + + let partition = if partition_keys.contains(&partition_key) { + Some(Partition { key: partition_key }) + } else { + None + }; + + Ok(Response::new(GetPartitionResponse { partition })) + } + + async fn list_partition_chunks( + &self, + request: Request, + ) -> Result, Status> { + let ListPartitionChunksRequest { + db_name, + partition_key, + } = request.into_inner(); + let db_name = DatabaseName::new(db_name).field("db_name")?; + + let db = self.server.db(&db_name).ok_or_else(|| NotFound { + resource_type: "database".to_string(), + resource_name: db_name.to_string(), + ..Default::default() + })?; + + let chunks: Vec = db + .partition_chunk_summaries(&partition_key) + .map(|summary| summary.into()) + .collect(); + + Ok(Response::new(ListPartitionChunksResponse { chunks })) + } + + async fn new_partition_chunk( + &self, + request: Request, + ) -> Result, Status> { + let NewPartitionChunkRequest { + db_name, + partition_key, + } = request.into_inner(); + let db_name = DatabaseName::new(db_name).field("db_name")?; + + let db = self.server.db(&db_name).ok_or_else(|| NotFound { + resource_type: "database".to_string(), + resource_name: db_name.to_string(), + ..Default::default() + })?; + + db.rollover_partition(&partition_key) + .await + .map_err(default_db_error_handler)?; + + Ok(Response::new(NewPartitionChunkResponse {})) + } + + async fn close_partition_chunk( + &self, + request: Request, + ) -> Result, Status> { + let ClosePartitionChunkRequest { + db_name, + partition_key, + chunk_id, + } = request.into_inner(); + + // Validate that the database name is legit + let db_name = DatabaseName::new(db_name).field("db_name")?; + + let tracker = self + .server + .close_chunk(db_name, partition_key, chunk_id) + .map_err(default_server_error_handler)?; + + let operation = Some(super::operations::encode_tracker(tracker)?); + + Ok(Response::new(ClosePartitionChunkResponse { operation })) + } } pub fn make_server( diff --git a/src/influxdb_ioxd/rpc/operations.rs b/src/influxdb_ioxd/rpc/operations.rs new file mode 100644 index 0000000000..298dae08a5 --- /dev/null +++ b/src/influxdb_ioxd/rpc/operations.rs @@ -0,0 +1,216 @@ +use std::fmt::Debug; +use std::sync::Arc; + +use bytes::BytesMut; +use prost::Message; +use tonic::Response; +use tracing::debug; + +use data_types::job::Job; +use generated_types::google::FieldViolationExt; +use generated_types::{ + google::{ + longrunning::*, + protobuf::{Any, Empty}, + rpc::Status, + FieldViolation, InternalError, NotFound, + }, + influxdata::iox::management::v1 as management, + protobuf_type_url, +}; +use server::{ + tracker::{Tracker, TrackerId, TrackerStatus}, + ConnectionManager, Server, +}; +use std::convert::TryInto; + +/// Implementation of the write service +struct OperationsService { + server: Arc>, +} + +pub fn encode_tracker(tracker: Tracker) -> Result { + let id = tracker.id(); + let is_cancelled = tracker.is_cancelled(); + let status = tracker.get_status(); + + let (operation_metadata, is_complete) = match status { + TrackerStatus::Creating => { + let metadata = management::OperationMetadata { + job: Some(tracker.metadata().clone().into()), + ..Default::default() + }; + + (metadata, false) + } + TrackerStatus::Running { + total_count, + pending_count, + cpu_nanos, + } => { + let metadata = management::OperationMetadata { + cpu_nanos: cpu_nanos as _, + task_count: total_count as _, + pending_count: pending_count as _, + job: Some(tracker.metadata().clone().into()), + ..Default::default() + }; + + (metadata, false) + } + TrackerStatus::Complete { + total_count, + cpu_nanos, + wall_nanos, + } => { + let metadata = management::OperationMetadata { + cpu_nanos: cpu_nanos as _, + task_count: total_count as _, + wall_nanos: wall_nanos as _, + job: Some(tracker.metadata().clone().into()), + ..Default::default() + }; + + (metadata, true) + } + }; + + let mut buffer = BytesMut::new(); + operation_metadata.encode(&mut buffer).map_err(|error| { + debug!(?error, "Unexpected error"); + InternalError {} + })?; + + let metadata = Any { + type_url: protobuf_type_url(management::OPERATION_METADATA), + value: buffer.freeze(), + }; + + let result = match (is_complete, is_cancelled) { + (true, true) => Some(operation::Result::Error(Status { + code: tonic::Code::Cancelled as _, + message: "Job cancelled".to_string(), + details: vec![], + })), + + (true, false) => Some(operation::Result::Response(Any { + type_url: "type.googleapis.com/google.protobuf.Empty".to_string(), + value: Default::default(), // TODO: Verify this is correct + })), + + _ => None, + }; + + Ok(Operation { + name: id.to_string(), + metadata: Some(metadata), + done: is_complete, + result, + }) +} + +fn get_tracker(server: &Server, tracker: String) -> Result, tonic::Status> +where + M: ConnectionManager, +{ + let tracker_id = tracker.parse::().map_err(|e| FieldViolation { + field: "name".to_string(), + description: e.to_string(), + })?; + + let tracker = server.get_job(tracker_id).ok_or(NotFound { + resource_type: "job".to_string(), + resource_name: tracker, + ..Default::default() + })?; + + Ok(tracker) +} + +#[tonic::async_trait] +impl operations_server::Operations for OperationsService +where + M: ConnectionManager + Send + Sync + Debug + 'static, +{ + async fn list_operations( + &self, + _request: tonic::Request, + ) -> Result, tonic::Status> { + // TODO: Support pagination + let operations: Result, _> = self + .server + .tracked_jobs() + .into_iter() + .map(encode_tracker) + .collect(); + + Ok(Response::new(ListOperationsResponse { + operations: operations?, + next_page_token: Default::default(), + })) + } + + async fn get_operation( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + let request = request.into_inner(); + let tracker = get_tracker(self.server.as_ref(), request.name)?; + + Ok(Response::new(encode_tracker(tracker)?)) + } + + async fn delete_operation( + &self, + _: tonic::Request, + ) -> Result, tonic::Status> { + Err(tonic::Status::unimplemented( + "IOx does not support operation deletion", + )) + } + + async fn cancel_operation( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + let request = request.into_inner(); + + let tracker = get_tracker(self.server.as_ref(), request.name)?; + tracker.cancel(); + + Ok(Response::new(Empty {})) + } + + async fn wait_operation( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + // This should take into account the context deadline timeout + // Unfortunately these are currently stripped by tonic + // - https://github.com/hyperium/tonic/issues/75 + + let request = request.into_inner(); + + let tracker = get_tracker(self.server.as_ref(), request.name)?; + if let Some(timeout) = request.timeout { + let timeout = timeout.try_into().field("timeout")?; + + // Timeout is not an error so suppress it + let _ = tokio::time::timeout(timeout, tracker.join()).await; + } else { + tracker.join().await; + } + + Ok(Response::new(encode_tracker(tracker)?)) + } +} + +/// Instantiate the write service +pub fn make_server( + server: Arc>, +) -> operations_server::OperationsServer +where + M: ConnectionManager + Send + Sync + Debug + 'static, +{ + operations_server::OperationsServer::new(OperationsService { server }) +} diff --git a/src/influxdb_ioxd/rpc/storage/expr.rs b/src/influxdb_ioxd/rpc/storage/expr.rs index eaf0a98720..397254565f 100644 --- a/src/influxdb_ioxd/rpc/storage/expr.rs +++ b/src/influxdb_ioxd/rpc/storage/expr.rs @@ -462,43 +462,30 @@ fn build_node(value: RPCValue, inputs: Vec) -> Result { /// Creates an expr from a "Logical" Node fn build_logical_node(logical: i32, inputs: Vec) -> Result { - // This ideally could be a match, but I couldn't find a safe way - // to match an i32 to RPCLogical except for ths + let logical_enum = RPCLogical::from_i32(logical); - if logical == RPCLogical::And as i32 { - build_binary_expr(Operator::And, inputs) - } else if logical == RPCLogical::Or as i32 { - build_binary_expr(Operator::Or, inputs) - } else { - UnknownLogicalNode { logical }.fail() + match logical_enum { + Some(RPCLogical::And) => build_binary_expr(Operator::And, inputs), + Some(RPCLogical::Or) => build_binary_expr(Operator::Or, inputs), + None => UnknownLogicalNode { logical }.fail(), } } /// Creates an expr from a "Comparsion" Node fn build_comparison_node(comparison: i32, inputs: Vec) -> Result { - // again, this would ideally be a match but I couldn't figure out how to - // match an i32 to the enum values + let comparison_enum = RPCComparison::from_i32(comparison); - if comparison == RPCComparison::Equal as i32 { - build_binary_expr(Operator::Eq, inputs) - } else if comparison == RPCComparison::NotEqual as i32 { - build_binary_expr(Operator::NotEq, inputs) - } else if comparison == RPCComparison::StartsWith as i32 { - StartsWithNotSupported {}.fail() - } else if comparison == RPCComparison::Regex as i32 { - RegExpNotSupported {}.fail() - } else if comparison == RPCComparison::NotRegex as i32 { - NotRegExpNotSupported {}.fail() - } else if comparison == RPCComparison::Lt as i32 { - build_binary_expr(Operator::Lt, inputs) - } else if comparison == RPCComparison::Lte as i32 { - build_binary_expr(Operator::LtEq, inputs) - } else if comparison == RPCComparison::Gt as i32 { - build_binary_expr(Operator::Gt, inputs) - } else if comparison == RPCComparison::Gte as i32 { - build_binary_expr(Operator::GtEq, inputs) - } else { - UnknownComparisonNode { comparison }.fail() + match comparison_enum { + Some(RPCComparison::Equal) => build_binary_expr(Operator::Eq, inputs), + Some(RPCComparison::NotEqual) => build_binary_expr(Operator::NotEq, inputs), + Some(RPCComparison::StartsWith) => StartsWithNotSupported {}.fail(), + Some(RPCComparison::Regex) => RegExpNotSupported {}.fail(), + Some(RPCComparison::NotRegex) => NotRegExpNotSupported {}.fail(), + Some(RPCComparison::Lt) => build_binary_expr(Operator::Lt, inputs), + Some(RPCComparison::Lte) => build_binary_expr(Operator::LtEq, inputs), + Some(RPCComparison::Gt) => build_binary_expr(Operator::Gt, inputs), + Some(RPCComparison::Gte) => build_binary_expr(Operator::GtEq, inputs), + None => UnknownComparisonNode { comparison }.fail(), } } @@ -630,36 +617,23 @@ fn convert_aggregate(aggregate: Option) -> Result }; let aggregate_type = aggregate.r#type; + let aggregate_type_enum = RPCAggregateType::from_i32(aggregate_type); - if aggregate_type == RPCAggregateType::None as i32 { - Ok(QueryAggregate::None) - } else if aggregate_type == RPCAggregateType::Sum as i32 { - Ok(QueryAggregate::Sum) - } else if aggregate_type == RPCAggregateType::Count as i32 { - Ok(QueryAggregate::Count) - } else if aggregate_type == RPCAggregateType::Min as i32 { - Ok(QueryAggregate::Min) - } else if aggregate_type == RPCAggregateType::Max as i32 { - Ok(QueryAggregate::Max) - } else if aggregate_type == RPCAggregateType::First as i32 { - Ok(QueryAggregate::First) - } else if aggregate_type == RPCAggregateType::Last as i32 { - Ok(QueryAggregate::Last) - } else if aggregate_type == RPCAggregateType::Mean as i32 { - Ok(QueryAggregate::Mean) - } else { - UnknownAggregate { aggregate_type }.fail() + match aggregate_type_enum { + Some(RPCAggregateType::None) => Ok(QueryAggregate::None), + Some(RPCAggregateType::Sum) => Ok(QueryAggregate::Sum), + Some(RPCAggregateType::Count) => Ok(QueryAggregate::Count), + Some(RPCAggregateType::Min) => Ok(QueryAggregate::Min), + Some(RPCAggregateType::Max) => Ok(QueryAggregate::Max), + Some(RPCAggregateType::First) => Ok(QueryAggregate::First), + Some(RPCAggregateType::Last) => Ok(QueryAggregate::Last), + Some(RPCAggregateType::Mean) => Ok(QueryAggregate::Mean), + None => UnknownAggregate { aggregate_type }.fail(), } } pub fn convert_group_type(group: i32) -> Result { - if group == RPCGroup::None as i32 { - Ok(RPCGroup::None) - } else if group == RPCGroup::By as i32 { - Ok(RPCGroup::By) - } else { - UnknownGroup { group_type: group }.fail() - } + RPCGroup::from_i32(group).ok_or(Error::UnknownGroup { group_type: group }) } /// Creates a representation of some struct (in another crate that we @@ -774,36 +748,25 @@ fn format_value<'a>(value: &'a RPCValue, f: &mut fmt::Formatter<'_>) -> fmt::Res } fn format_logical(v: i32, f: &mut fmt::Formatter<'_>) -> fmt::Result { - if v == RPCLogical::And as i32 { - write!(f, "AND") - } else if v == RPCLogical::Or as i32 { - write!(f, "Or") - } else { - write!(f, "UNKNOWN_LOGICAL:{}", v) + match RPCLogical::from_i32(v) { + Some(RPCLogical::And) => write!(f, "AND"), + Some(RPCLogical::Or) => write!(f, "Or"), + None => write!(f, "UNKNOWN_LOGICAL:{}", v), } } fn format_comparison(v: i32, f: &mut fmt::Formatter<'_>) -> fmt::Result { - if v == RPCComparison::Equal as i32 { - write!(f, "==") - } else if v == RPCComparison::NotEqual as i32 { - write!(f, "!=") - } else if v == RPCComparison::StartsWith as i32 { - write!(f, "StartsWith") - } else if v == RPCComparison::Regex as i32 { - write!(f, "RegEx") - } else if v == RPCComparison::NotRegex as i32 { - write!(f, "NotRegex") - } else if v == RPCComparison::Lt as i32 { - write!(f, "<") - } else if v == RPCComparison::Lte as i32 { - write!(f, "<=") - } else if v == RPCComparison::Gt as i32 { - write!(f, ">") - } else if v == RPCComparison::Gte as i32 { - write!(f, ">=") - } else { - write!(f, "UNKNOWN_COMPARISON:{}", v) + match RPCComparison::from_i32(v) { + Some(RPCComparison::Equal) => write!(f, "=="), + Some(RPCComparison::NotEqual) => write!(f, "!="), + Some(RPCComparison::StartsWith) => write!(f, "StartsWith"), + Some(RPCComparison::Regex) => write!(f, "RegEx"), + Some(RPCComparison::NotRegex) => write!(f, "NotRegex"), + Some(RPCComparison::Lt) => write!(f, "<"), + Some(RPCComparison::Lte) => write!(f, "<="), + Some(RPCComparison::Gt) => write!(f, ">"), + Some(RPCComparison::Gte) => write!(f, ">="), + None => write!(f, "UNKNOWN_COMPARISON:{}", v), } } diff --git a/src/influxdb_ioxd/rpc/storage/service.rs b/src/influxdb_ioxd/rpc/storage/service.rs index 2b026b277c..a3c1cfc24d 100644 --- a/src/influxdb_ioxd/rpc/storage/service.rs +++ b/src/influxdb_ioxd/rpc/storage/service.rs @@ -226,12 +226,7 @@ where predicate, } = read_filter_request; - info!( - "read_filter for database {}, range: {:?}, predicate: {}", - db_name, - range, - predicate.loggable() - ); + info!(%db_name, ?range, predicate=%predicate.loggable(),"read filter"); read_filter_impl( tx.clone(), @@ -268,11 +263,7 @@ where hints, } = read_group_request; - info!( - "read_group for database {}, range: {:?}, group_keys: {:?}, group: {:?}, aggregate: {:?}, predicate: {}", - db_name, range, group_keys, group, aggregate, - predicate.loggable() - ); + info!(%db_name, ?range, ?group_keys, ?group, ?aggregate,predicate=%predicate.loggable(),"read_group"); if hints != 0 { InternalHintsFieldNotSupported { hints }.fail()? @@ -326,11 +317,7 @@ where window, } = read_window_aggregate_request; - info!( - "read_window_aggregate for database {}, range: {:?}, window_every: {:?}, offset: {:?}, aggregate: {:?}, window: {:?}, predicate: {}", - db_name, range, window_every, offset, aggregate, window, - predicate.loggable() - ); + info!(%db_name, ?range, ?window_every, ?offset, ?aggregate, ?window, predicate=%predicate.loggable(),"read_window_aggregate"); let aggregate_string = format!( "aggregate: {:?}, window_every: {:?}, offset: {:?}, window: {:?}", @@ -372,12 +359,7 @@ where predicate, } = tag_keys_request; - info!( - "tag_keys for database {}, range: {:?}, predicate: {}", - db_name, - range, - predicate.loggable() - ); + info!(%db_name, ?range, predicate=%predicate.loggable(), "tag_keys"); let measurement = None; @@ -422,23 +404,18 @@ where // Special case a request for 'tag_key=_measurement" means to list all // measurements let response = if tag_key.is_measurement() { - info!( - "tag_values with tag_key=[x00] (measurement name) for database {}, range: {:?}, predicate: {} --> returning measurement_names", - db_name, range, - predicate.loggable() - ); + info!(%db_name, ?range, predicate=%predicate.loggable(), "tag_values with tag_key=[x00] (measurement name)"); if predicate.is_some() { - unimplemented!("tag_value for a measurement, with general predicate"); + return Err(Error::NotYetImplemented { + operation: "tag_value for a measurement, with general predicate".to_string(), + } + .to_status()); } measurement_name_impl(Arc::clone(&self.db_store), db_name, range).await } else if tag_key.is_field() { - info!( - "tag_values with tag_key=[xff] (field name) for database {}, range: {:?}, predicate: {} --> returning fields", - db_name, range, - predicate.loggable() - ); + info!(%db_name, ?range, predicate=%predicate.loggable(), "tag_values with tag_key=[xff] (field name)"); let fieldlist = field_names_impl(Arc::clone(&self.db_store), db_name, None, range, predicate) @@ -455,13 +432,7 @@ where } else { let tag_key = String::from_utf8(tag_key).context(ConvertingTagKeyInTagValues)?; - info!( - "tag_values for database {}, range: {:?}, tag_key: {}, predicate: {}", - db_name, - range, - tag_key, - predicate.loggable() - ); + info!(%db_name, ?range, %tag_key, predicate=%predicate.loggable(), "tag_values",); tag_values_impl( Arc::clone(&self.db_store), @@ -557,12 +528,7 @@ where .map_err(|e| e.to_status()); } - info!( - "measurement_names for database {}, range: {:?}, predicate: {}", - db_name, - range, - predicate.loggable() - ); + info!(%db_name, ?range, predicate=%predicate.loggable(), "measurement_names"); let response = measurement_name_impl(Arc::clone(&self.db_store), db_name, range) .await @@ -594,13 +560,7 @@ where predicate, } = measurement_tag_keys_request; - info!( - "measurement_tag_keys for database {}, range: {:?}, measurement: {}, predicate: {}", - db_name, - range, - measurement, - predicate.loggable() - ); + info!(%db_name, ?range, %measurement, predicate=%predicate.loggable(), "measurement_tag_keys"); let measurement = Some(measurement); @@ -641,11 +601,7 @@ where tag_key, } = measurement_tag_values_request; - info!( - "measurement_tag_values for database {}, range: {:?}, measurement: {}, tag_key: {}, predicate: {}", - db_name, range, measurement, tag_key, - predicate.loggable() - ); + info!(%db_name, ?range, %measurement, %tag_key, predicate=%predicate.loggable(), "measurement_tag_values"); let measurement = Some(measurement); @@ -686,12 +642,7 @@ where predicate, } = measurement_fields_request; - info!( - "measurement_fields for database {}, range: {:?}, predicate: {}", - db_name, - range, - predicate.loggable() - ); + info!(%db_name, ?range, predicate=%predicate.loggable(), "measurement_fields"); let measurement = Some(measurement); @@ -757,7 +708,6 @@ where let db = db_store .db(&db_name) - .await .context(DatabaseNotFound { db_name })?; let planner = InfluxRPCPlanner::new(); @@ -807,7 +757,7 @@ where })? .build(); - let db = db_store.db(&db_name).await.context(DatabaseNotFound { + let db = db_store.db(&db_name).context(DatabaseNotFound { db_name: db_name.as_str(), })?; @@ -867,10 +817,7 @@ where let db_name = db_name.as_str(); let tag_name = &tag_name; - let db = db_store - .db(db_name) - .await - .context(DatabaseNotFound { db_name })?; + let db = db_store.db(db_name).context(DatabaseNotFound { db_name })?; let planner = InfluxRPCPlanner::new(); @@ -927,10 +874,7 @@ where let owned_db_name = db_name; let db_name = owned_db_name.as_str(); - let db = db_store - .db(db_name) - .await - .context(DatabaseNotFound { db_name })?; + let db = db_store.db(db_name).context(DatabaseNotFound { db_name })?; let executor = db_store.executor(); @@ -1018,7 +962,6 @@ where let db = db_store .db(&db_name) - .await .context(DatabaseNotFound { db_name })?; let planner = InfluxRPCPlanner::new(); @@ -1090,10 +1033,7 @@ where .build(); let db_name = db_name.as_str(); - let db = db_store - .db(db_name) - .await - .context(DatabaseNotFound { db_name })?; + let db = db_store.db(db_name).context(DatabaseNotFound { db_name })?; let planner = InfluxRPCPlanner::new(); diff --git a/src/influxdb_ioxd/rpc/write.rs b/src/influxdb_ioxd/rpc/write.rs new file mode 100644 index 0000000000..67e48fb438 --- /dev/null +++ b/src/influxdb_ioxd/rpc/write.rs @@ -0,0 +1,60 @@ +use std::sync::Arc; + +use generated_types::{google::FieldViolation, influxdata::iox::write::v1::*}; +use influxdb_line_protocol::parse_lines; +use server::{ConnectionManager, Server}; +use std::fmt::Debug; +use tonic::Response; +use tracing::debug; + +use super::error::default_server_error_handler; + +/// Implementation of the write service +struct WriteService { + server: Arc>, +} + +#[tonic::async_trait] +impl write_service_server::WriteService for WriteService +where + M: ConnectionManager + Send + Sync + Debug + 'static, +{ + async fn write( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + let request = request.into_inner(); + + let db_name = request.db_name; + let lp_data = request.lp_data; + let lp_chars = lp_data.len(); + + let lines = parse_lines(&lp_data) + .collect::, influxdb_line_protocol::Error>>() + .map_err(|e| FieldViolation { + field: "lp_data".into(), + description: format!("Invalid Line Protocol: {}", e), + })?; + + let lp_line_count = lines.len(); + debug!(%db_name, %lp_chars, lp_line_count, "Writing lines into database"); + + self.server + .write_lines(&db_name, &lines) + .await + .map_err(default_server_error_handler)?; + + let lines_written = lp_line_count as u64; + Ok(Response::new(WriteResponse { lines_written })) + } +} + +/// Instantiate the write service +pub fn make_server( + server: Arc>, +) -> write_service_server::WriteServiceServer +where + M: ConnectionManager + Send + Sync + Debug + 'static, +{ + write_service_server::WriteServiceServer::new(WriteService { server }) +} diff --git a/src/main.rs b/src/main.rs index d2064e3ce1..cd2146b27a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -12,7 +12,7 @@ use std::str::FromStr; use dotenv::dotenv; use structopt::StructOpt; use tokio::runtime::Runtime; -use tracing::{debug, error, warn}; +use tracing::{debug, warn}; use commands::logging::LoggingLevel; use ingest::parquet::writer::CompressionLevel; @@ -23,7 +23,10 @@ mod commands { mod input; pub mod logging; pub mod meta; + pub mod operations; + pub mod run; pub mod server; + pub mod server_remote; pub mod stats; pub mod writer; } @@ -45,13 +48,13 @@ Examples: influxdb_iox # Display all server settings - influxdb_iox server --help + influxdb_iox run --help # Run the InfluxDB IOx server with extra verbose logging - influxdb_iox -v + influxdb_iox run -v # Run InfluxDB IOx with full debug logging specified with RUST_LOG - RUST_LOG=debug influxdb_iox + RUST_LOG=debug influxdb_iox run # converts line protocol formatted data in temperature.lp to out.parquet influxdb_iox convert temperature.lp out.parquet @@ -61,6 +64,14 @@ Examples: # Dumps storage statistics about out.parquet to stdout influxdb_iox stats out.parquet + +Command are generally structured in the form: + + +For example, a command such as the following shows all actions + available for database chunks, including get and list. + + influxdb_iox database chunk --help "# )] struct Config { @@ -83,7 +94,7 @@ struct Config { num_threads: Option, #[structopt(subcommand)] - command: Option, + command: Command, } #[derive(Debug, StructOpt)] @@ -109,10 +120,12 @@ enum Command { input: String, }, Database(commands::database::Config), - Stats(commands::stats::Config), // Clippy recommended boxing this variant because it's much larger than the others - Server(Box), + Run(Box), + Stats(commands::stats::Config), + Server(commands::server::Config), Writer(commands::writer::Config), + Operation(commands::operations::Config), } fn main() -> Result<(), std::io::Error> { @@ -131,13 +144,12 @@ fn main() -> Result<(), std::io::Error> { let tokio_runtime = get_runtime(config.num_threads)?; tokio_runtime.block_on(async move { let host = config.host; - match config.command { - Some(Command::Convert { + Command::Convert { input, output, compression_level, - }) => { + } => { logging_level.setup_basic_logging(); let compression_level = CompressionLevel::from_str(&compression_level).unwrap(); @@ -149,7 +161,7 @@ fn main() -> Result<(), std::io::Error> { } } } - Some(Command::Meta { input }) => { + Command::Meta { input } => { logging_level.setup_basic_logging(); match commands::meta::dump_meta(&input) { Ok(()) => debug!("Metadata dump completed successfully"), @@ -159,7 +171,7 @@ fn main() -> Result<(), std::io::Error> { } } } - Some(Command::Stats(config)) => { + Command::Stats(config) => { logging_level.setup_basic_logging(); match commands::stats::stats(&config).await { Ok(()) => debug!("Storage statistics dump completed successfully"), @@ -169,37 +181,40 @@ fn main() -> Result<(), std::io::Error> { } } } - Some(Command::Database(config)) => { + Command::Database(config) => { logging_level.setup_basic_logging(); if let Err(e) = commands::database::command(host, config).await { eprintln!("{}", e); std::process::exit(ReturnCode::Failure as _) } } - Some(Command::Writer(config)) => { + Command::Writer(config) => { logging_level.setup_basic_logging(); if let Err(e) = commands::writer::command(host, config).await { eprintln!("{}", e); std::process::exit(ReturnCode::Failure as _) } } - Some(Command::Server(config)) => { - // Note don't set up basic logging here, different logging rules apply in server - // mode - let res = influxdb_ioxd::main(logging_level, Some(config)).await; - - if let Err(e) = res { - error!("Server shutdown with error: {}", e); - std::process::exit(ReturnCode::Failure as _); + Command::Operation(config) => { + logging_level.setup_basic_logging(); + if let Err(e) = commands::operations::command(host, config).await { + eprintln!("{}", e); + std::process::exit(ReturnCode::Failure as _) } } - None => { + Command::Server(config) => { + logging_level.setup_basic_logging(); + if let Err(e) = commands::server::command(host, config).await { + eprintln!("Server command failed: {}", e); + std::process::exit(ReturnCode::Failure as _) + } + } + Command::Run(config) => { // Note don't set up basic logging here, different logging rules apply in server // mode - let res = influxdb_ioxd::main(logging_level, None).await; - if let Err(e) = res { - error!("Server shutdown with error: {}", e); - std::process::exit(ReturnCode::Failure as _); + if let Err(e) = commands::run::command(logging_level, *config).await { + eprintln!("Server command failed: {}", e); + std::process::exit(ReturnCode::Failure as _) } } } diff --git a/tests/common/mod.rs b/tests/common/mod.rs new file mode 100644 index 0000000000..a057eb4940 --- /dev/null +++ b/tests/common/mod.rs @@ -0,0 +1 @@ +pub mod server_fixture; diff --git a/tests/common/server_fixture.rs b/tests/common/server_fixture.rs new file mode 100644 index 0000000000..2f3c07257d --- /dev/null +++ b/tests/common/server_fixture.rs @@ -0,0 +1,422 @@ +use assert_cmd::prelude::*; +use std::{ + fs::File, + num::NonZeroU32, + process::{Child, Command}, + str, + sync::{ + atomic::{AtomicUsize, Ordering::SeqCst}, + Weak, + }, +}; + +use futures::prelude::*; +use std::time::Duration; +use tempfile::TempDir; + +type Error = Box; +type Result = std::result::Result; + +// These port numbers are chosen to not collide with a development ioxd server +// running locally. +// TODO(786): allocate random free ports instead of hardcoding. +// TODO(785): we cannot use localhost here. +static NEXT_PORT: AtomicUsize = AtomicUsize::new(8090); + +/// This structure contains all the addresses a test server should use +struct BindAddresses { + http_port: usize, + grpc_port: usize, + + http_bind_addr: String, + grpc_bind_addr: String, + + http_base: String, + iox_api_v1_base: String, + grpc_base: String, +} + +impl BindAddresses { + /// return a new port assignment suitable for this test's use + fn new() -> Self { + let http_port = NEXT_PORT.fetch_add(1, SeqCst); + let grpc_port = NEXT_PORT.fetch_add(1, SeqCst); + + let http_bind_addr = format!("127.0.0.1:{}", http_port); + let grpc_bind_addr = format!("127.0.0.1:{}", grpc_port); + + let http_base = format!("http://{}", http_bind_addr); + let iox_api_v1_base = format!("http://{}/iox/api/v1", http_bind_addr); + let grpc_base = format!("http://{}", grpc_bind_addr); + + Self { + http_port, + grpc_port, + http_bind_addr, + grpc_bind_addr, + http_base, + iox_api_v1_base, + grpc_base, + } + } +} + +const TOKEN: &str = "InfluxDB IOx doesn't have authentication yet"; + +use std::sync::Arc; + +use once_cell::sync::OnceCell; +use tokio::sync::Mutex; + +/// Represents a server that has been started and is available for +/// testing. +pub struct ServerFixture { + server: Arc, + grpc_channel: tonic::transport::Channel, +} + +/// Specifieds should we configure a server initially +enum InitialConfig { + /// Set the writer id to something so it can accept writes + SetWriterId, + + /// leave the writer id empty so the test can set it + None, +} + +impl ServerFixture { + /// Create a new server fixture and wait for it to be ready. This + /// is called "create" rather than new because it is async and + /// waits. The shared database is configured with a writer id and + /// can be used immediately + /// + /// This is currently implemented as a singleton so all tests *must* + /// use a new database and not interfere with the existing database. + pub async fn create_shared() -> Self { + // Try and reuse the same shared server, if there is already + // one present + static SHARED_SERVER: OnceCell>> = OnceCell::new(); + + let shared_server = SHARED_SERVER.get_or_init(|| parking_lot::Mutex::new(Weak::new())); + + let mut shared_server = shared_server.lock(); + + // is a shared server already present? + let server = match shared_server.upgrade() { + Some(server) => server, + None => { + // if not, create one + let server = TestServer::new().expect("Could start test server"); + let server = Arc::new(server); + + // ensure the server is ready + server.wait_until_ready(InitialConfig::SetWriterId).await; + // save a reference for other threads that may want to + // use this server, but don't prevent it from being + // destroyed when going out of scope + *shared_server = Arc::downgrade(&server); + server + } + }; + std::mem::drop(shared_server); + + Self::create_common(server).await + } + + /// Create a new server fixture and wait for it to be ready. This + /// is called "create" rather than new because it is async and + /// waits. The database is left unconfigured (no writer id) and + /// is not shared with any other tests. + pub async fn create_single_use() -> Self { + let server = TestServer::new().expect("Could start test server"); + let server = Arc::new(server); + + // ensure the server is ready + server.wait_until_ready(InitialConfig::None).await; + Self::create_common(server).await + } + + async fn create_common(server: Arc) -> Self { + let grpc_channel = server + .grpc_channel() + .await + .expect("The server should have been up"); + + ServerFixture { + server, + grpc_channel, + } + } + + /// Return a channel connected to the gRPC API. Panics if the + /// server is not yet up + pub fn grpc_channel(&self) -> tonic::transport::Channel { + self.grpc_channel.clone() + } + + /// Return the url base of the grpc management API + pub fn grpc_base(&self) -> &str { + &self.server.addrs().grpc_base + } + + /// Return the http base URL for the HTTP API + pub fn http_base(&self) -> &str { + &self.server.addrs().http_base + } + + /// Return the base URL for the IOx V1 API + pub fn iox_api_v1_base(&self) -> &str { + &self.server.addrs().iox_api_v1_base + } + + /// Return an a http client suitable suitable for communicating with this + /// server + pub fn influxdb2_client(&self) -> influxdb2_client::Client { + influxdb2_client::Client::new(self.http_base(), TOKEN) + } + + /// Return a management client suitable for communicating with this + /// server + pub fn management_client(&self) -> influxdb_iox_client::management::Client { + influxdb_iox_client::management::Client::new(self.grpc_channel()) + } + + /// Return a operations client suitable for communicating with this + /// server + pub fn operations_client(&self) -> influxdb_iox_client::operations::Client { + influxdb_iox_client::operations::Client::new(self.grpc_channel()) + } + + /// Return a write client suitable for communicating with this + /// server + pub fn write_client(&self) -> influxdb_iox_client::write::Client { + influxdb_iox_client::write::Client::new(self.grpc_channel()) + } + + /// Return a flight client suitable for communicating with this + /// server + pub fn flight_client(&self) -> influxdb_iox_client::flight::Client { + influxdb_iox_client::flight::Client::new(self.grpc_channel()) + } +} + +#[derive(Debug)] +/// Represents the current known state of a TestServer +enum ServerState { + Started, + Ready, + Error, +} + +struct TestServer { + /// Is the server ready to accept connections? + ready: Mutex, + + /// Handle to the server process being controlled + server_process: Child, + + /// Which ports this server should use + addrs: BindAddresses, + + // The temporary directory **must** be last so that it is + // dropped after the database closes. + dir: TempDir, +} + +impl TestServer { + fn new() -> Result { + let addrs = BindAddresses::new(); + let ready = Mutex::new(ServerState::Started); + + let dir = test_helpers::tmp_dir().unwrap(); + // Make a log file in the temporary dir (don't auto delete it to help debugging + // efforts) + let mut log_path = std::env::temp_dir(); + log_path.push(format!( + "server_fixture_{}_{}.log", + addrs.http_port, addrs.grpc_port + )); + + println!("****************"); + println!("Server Logging to {:?}", log_path); + println!("****************"); + let log_file = File::create(log_path).expect("Opening log file"); + + let stdout_log_file = log_file + .try_clone() + .expect("cloning file handle for stdout"); + let stderr_log_file = log_file; + + let server_process = Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("run") + // Can enable for debugging + //.arg("-vv") + .env("INFLUXDB_IOX_BIND_ADDR", &addrs.http_bind_addr) + .env("INFLUXDB_IOX_GRPC_BIND_ADDR", &addrs.grpc_bind_addr) + // redirect output to log file + .stdout(stdout_log_file) + .stderr(stderr_log_file) + .spawn() + .unwrap(); + + Ok(Self { + ready, + dir, + addrs, + server_process, + }) + } + + #[allow(dead_code)] + fn restart(&mut self) -> Result<()> { + self.server_process.kill().unwrap(); + self.server_process.wait().unwrap(); + self.server_process = Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("run") + // Can enable for debugging + //.arg("-vv") + .env("INFLUXDB_IOX_DB_DIR", self.dir.path()) + .spawn() + .unwrap(); + Ok(()) + } + + async fn wait_until_ready(&self, initial_config: InitialConfig) { + let mut ready = self.ready.lock().await; + match *ready { + ServerState::Started => {} // first time, need to try and start it + ServerState::Ready => { + return; + } + ServerState::Error => { + panic!("Server was previously found to be in Error, aborting"); + } + } + + // Poll the RPC and HTTP servers separately as they listen on + // different ports but both need to be up for the test to run + let try_grpc_connect = async { + let mut interval = tokio::time::interval(Duration::from_millis(500)); + + loop { + match self.grpc_channel().await { + Ok(channel) => { + println!("Successfully connected to server"); + + let mut health = influxdb_iox_client::health::Client::new(channel); + + match health.check_storage().await { + Ok(_) => { + println!("Storage service is running"); + return; + } + Err(e) => { + println!("Error checking storage service status: {}", e); + } + } + } + Err(e) => { + println!("Waiting for gRPC API to be up: {}", e); + } + } + interval.tick().await; + } + }; + + let try_http_connect = async { + let client = reqwest::Client::new(); + let url = format!("{}/health", self.addrs().http_base); + let mut interval = tokio::time::interval(Duration::from_millis(500)); + loop { + match client.get(&url).send().await { + Ok(resp) => { + println!("Successfully got a response from HTTP: {:?}", resp); + return; + } + Err(e) => { + println!("Waiting for HTTP server to be up: {}", e); + } + } + interval.tick().await; + } + }; + + let pair = future::join(try_http_connect, try_grpc_connect); + + let capped_check = tokio::time::timeout(Duration::from_secs(3), pair); + + match capped_check.await { + Ok(_) => { + println!("Successfully started {}", self); + *ready = ServerState::Ready; + } + Err(e) => { + // tell others that this server had some problem + *ready = ServerState::Error; + std::mem::drop(ready); + panic!("Server was not ready in required time: {}", e); + } + } + + // Set the writer id, if requested; otherwise default to the default writer id: + // 1. + let id = match initial_config { + InitialConfig::SetWriterId => { + NonZeroU32::new(42).expect("42 is non zero, among its other properties") + } + InitialConfig::None => { + NonZeroU32::new(1).expect("1 is non zero; the first one to be so, moreover") + } + }; + + let channel = self.grpc_channel().await.expect("gRPC should be running"); + let mut management_client = influxdb_iox_client::management::Client::new(channel); + + if let Ok(id) = management_client.get_writer_id().await { + // tell others that this server had some problem + *ready = ServerState::Error; + std::mem::drop(ready); + panic!("Server already has an ID ({}); possibly a stray/orphan server from another test run.", id); + } + + management_client + .update_writer_id(id) + .await + .expect("set ID failed"); + println!("Set writer_id to {:?}", id); + } + + /// Create a connection channel for the gRPR endpoing + async fn grpc_channel( + &self, + ) -> influxdb_iox_client::connection::Result { + influxdb_iox_client::connection::Builder::default() + .build(&self.addrs().grpc_base) + .await + } + + fn addrs(&self) -> &BindAddresses { + &self.addrs + } +} + +impl std::fmt::Display for TestServer { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::result::Result<(), std::fmt::Error> { + write!( + f, + "TestServer (grpc {}, http {})", + self.addrs().grpc_base, + self.addrs().http_base + ) + } +} + +impl Drop for TestServer { + fn drop(&mut self) { + self.server_process + .kill() + .expect("Should have been able to kill the test server"); + } +} diff --git a/tests/end-to-end.rs b/tests/end-to-end.rs index f31a3c521a..1f27dbcb09 100644 --- a/tests/end-to-end.rs +++ b/tests/end-to-end.rs @@ -1,460 +1,9 @@ // The test in this file runs the server in a separate thread and makes HTTP // requests as a smoke test for the integration of the whole system. // -// As written, only one test of this style can run at a time. Add more data to -// the existing test to test more scenarios rather than adding more tests in the -// same style. +// The servers under test are managed using [`ServerFixture`] // -// Or, change the way this test behaves to create isolated instances by: -// -// - Finding an unused port for the server to run on and using that port in the -// URL -// - Creating a temporary directory for an isolated database path -// -// Or, change the tests to use one server and isolate through `org_id` by: -// -// - Starting one server before all the relevant tests are run -// - Creating a unique org_id per test -// - Stopping the server after all relevant tests are run - -use std::convert::TryInto; -use std::process::{Child, Command}; -use std::str; -use std::time::{Duration, SystemTime}; -use std::u32; - -use assert_cmd::prelude::*; -use futures::prelude::*; -use prost::Message; -use tempfile::TempDir; - -use data_types::{names::org_and_bucket_to_database, DatabaseName}; -use end_to_end_cases::*; -use generated_types::{ - influxdata::iox::management::v1::DatabaseRules, storage_client::StorageClient, ReadSource, - TimestampRange, -}; - -// These port numbers are chosen to not collide with a development ioxd server -// running locally. -// TODO(786): allocate random free ports instead of hardcoding. -// TODO(785): we cannot use localhost here. -macro_rules! http_bind_addr { - () => { - "127.0.0.1:8090" - }; -} -macro_rules! grpc_bind_addr { - () => { - "127.0.0.1:8092" - }; -} - -const HTTP_BIND_ADDR: &str = http_bind_addr!(); -const GRPC_BIND_ADDR: &str = grpc_bind_addr!(); - -const HTTP_BASE: &str = concat!("http://", http_bind_addr!()); -const IOX_API_V1_BASE: &str = concat!("http://", http_bind_addr!(), "/iox/api/v1"); -const GRPC_URL_BASE: &str = concat!("http://", grpc_bind_addr!(), "/"); - -const TOKEN: &str = "InfluxDB IOx doesn't have authentication yet"; - -type Error = Box; -type Result = std::result::Result; +// The tests are defined in the submodules of [`end_to_end_cases`] +pub mod common; mod end_to_end_cases; - -#[tokio::test] -async fn read_and_write_data() { - let server = TestServer::new().unwrap(); - server.wait_until_ready().await; - - let http_client = reqwest::Client::new(); - let influxdb2 = influxdb2_client::Client::new(HTTP_BASE, TOKEN); - let grpc = influxdb_iox_client::connection::Builder::default() - .build(GRPC_URL_BASE) - .await - .unwrap(); - let mut storage_client = StorageClient::new(grpc.clone()); - let mut management_client = influxdb_iox_client::management::Client::new(grpc); - - // These tests share data; TODO: a better way to indicate this - { - let scenario = Scenario::default() - .set_org_id("0000111100001111") - .set_bucket_id("1111000011110000"); - - create_database(&mut management_client, &scenario.database_name()).await; - - let expected_read_data = load_data(&influxdb2, &scenario).await; - let sql_query = "select * from cpu_load_short"; - - read_api::test(&http_client, &scenario, sql_query, &expected_read_data).await; - storage_api::test(&mut storage_client, &scenario).await; - flight_api::test(&scenario, sql_query, &expected_read_data).await; - } - - // These tests manage their own data - storage_api::read_group_test(&mut management_client, &influxdb2, &mut storage_client).await; - storage_api::read_window_aggregate_test( - &mut management_client, - &influxdb2, - &mut storage_client, - ) - .await; - management_api::test(&mut management_client).await; - management_cli::test(GRPC_URL_BASE).await; - test_http_error_messages(&influxdb2).await.unwrap(); -} - -// TODO: Randomly generate org and bucket ids to ensure test data independence -// where desired - -#[derive(Debug)] -pub struct Scenario { - org_id_str: String, - bucket_id_str: String, - ns_since_epoch: i64, -} - -impl Default for Scenario { - fn default() -> Self { - let ns_since_epoch = SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH) - .expect("System time should have been after the epoch") - .as_nanos() - .try_into() - .expect("Unable to represent system time"); - - Self { - ns_since_epoch, - org_id_str: Default::default(), - bucket_id_str: Default::default(), - } - } -} - -impl Scenario { - fn set_org_id(mut self, org_id: impl Into) -> Self { - self.org_id_str = org_id.into(); - self - } - - fn set_bucket_id(mut self, bucket_id: impl Into) -> Self { - self.bucket_id_str = bucket_id.into(); - self - } - - fn org_id_str(&self) -> &str { - &self.org_id_str - } - - fn bucket_id_str(&self) -> &str { - &self.bucket_id_str - } - - fn org_id(&self) -> u64 { - u64::from_str_radix(&self.org_id_str, 16).unwrap() - } - - fn bucket_id(&self) -> u64 { - u64::from_str_radix(&self.bucket_id_str, 16).unwrap() - } - - fn database_name(&self) -> DatabaseName<'_> { - org_and_bucket_to_database(&self.org_id_str, &self.bucket_id_str).unwrap() - } - - fn ns_since_epoch(&self) -> i64 { - self.ns_since_epoch - } - - fn read_source(&self) -> Option { - let partition_id = u64::from(u32::MAX); - let read_source = ReadSource { - org_id: self.org_id(), - bucket_id: self.bucket_id(), - partition_id, - }; - - let mut d = bytes::BytesMut::new(); - read_source.encode(&mut d).unwrap(); - let read_source = generated_types::google::protobuf::Any { - type_url: "/TODO".to_string(), - value: d.freeze(), - }; - - Some(read_source) - } - - fn timestamp_range(&self) -> Option { - Some(TimestampRange { - start: self.ns_since_epoch(), - end: self.ns_since_epoch() + 10, - }) - } -} - -async fn create_database( - client: &mut influxdb_iox_client::management::Client, - database_name: &str, -) { - client - .create_database(DatabaseRules { - name: database_name.to_string(), - mutable_buffer_config: Some(Default::default()), - ..Default::default() - }) - .await - .unwrap(); -} - -async fn load_data(influxdb2: &influxdb2_client::Client, scenario: &Scenario) -> Vec { - // TODO: make a more extensible way to manage data for tests, such as in - // external fixture files or with factories. - let points = vec![ - influxdb2_client::DataPoint::builder("cpu_load_short") - .tag("host", "server01") - .tag("region", "us-west") - .field("value", 0.64) - .timestamp(scenario.ns_since_epoch()) - .build() - .unwrap(), - influxdb2_client::DataPoint::builder("cpu_load_short") - .tag("host", "server01") - .field("value", 27.99) - .timestamp(scenario.ns_since_epoch() + 1) - .build() - .unwrap(), - influxdb2_client::DataPoint::builder("cpu_load_short") - .tag("host", "server02") - .tag("region", "us-west") - .field("value", 3.89) - .timestamp(scenario.ns_since_epoch() + 2) - .build() - .unwrap(), - influxdb2_client::DataPoint::builder("cpu_load_short") - .tag("host", "server01") - .tag("region", "us-east") - .field("value", 1234567.891011) - .timestamp(scenario.ns_since_epoch() + 3) - .build() - .unwrap(), - influxdb2_client::DataPoint::builder("cpu_load_short") - .tag("host", "server01") - .tag("region", "us-west") - .field("value", 0.000003) - .timestamp(scenario.ns_since_epoch() + 4) - .build() - .unwrap(), - influxdb2_client::DataPoint::builder("system") - .tag("host", "server03") - .field("uptime", 1303385) - .timestamp(scenario.ns_since_epoch() + 5) - .build() - .unwrap(), - influxdb2_client::DataPoint::builder("swap") - .tag("host", "server01") - .tag("name", "disk0") - .field("in", 3) - .field("out", 4) - .timestamp(scenario.ns_since_epoch() + 6) - .build() - .unwrap(), - influxdb2_client::DataPoint::builder("status") - .field("active", true) - .timestamp(scenario.ns_since_epoch() + 7) - .build() - .unwrap(), - influxdb2_client::DataPoint::builder("attributes") - .field("color", "blue") - .timestamp(scenario.ns_since_epoch() + 8) - .build() - .unwrap(), - ]; - write_data(&influxdb2, scenario, points).await.unwrap(); - - substitute_nanos( - scenario.ns_since_epoch(), - &[ - "+----------+---------+---------------------+----------------+", - "| host | region | time | value |", - "+----------+---------+---------------------+----------------+", - "| server01 | us-west | ns0 | 0.64 |", - "| server01 | | ns1 | 27.99 |", - "| server02 | us-west | ns2 | 3.89 |", - "| server01 | us-east | ns3 | 1234567.891011 |", - "| server01 | us-west | ns4 | 0.000003 |", - "+----------+---------+---------------------+----------------+", - ], - ) -} - -async fn write_data( - client: &influxdb2_client::Client, - scenario: &Scenario, - points: Vec, -) -> Result<()> { - client - .write( - scenario.org_id_str(), - scenario.bucket_id_str(), - stream::iter(points), - ) - .await?; - Ok(()) -} - -// Don't make a separate #test function so that we can reuse the same -// server process -async fn test_http_error_messages(client: &influxdb2_client::Client) -> Result<()> { - // send malformed request (bucket id is invalid) - let result = client - .write_line_protocol("Bar", "Foo", "arbitrary") - .await - .expect_err("Should have errored"); - - let expected_error = "HTTP request returned an error: 400 Bad Request, `{\"error\":\"Error parsing line protocol: A generic parsing error occurred: TakeWhile1\",\"error_code\":100}`"; - assert_eq!(result.to_string(), expected_error); - - Ok(()) -} - -/// substitutes "ns" --> ns_since_epoch, ns1-->ns_since_epoch+1, etc -fn substitute_nanos(ns_since_epoch: i64, lines: &[&str]) -> Vec { - let substitutions = vec![ - ("ns0", format!("{}", ns_since_epoch)), - ("ns1", format!("{}", ns_since_epoch + 1)), - ("ns2", format!("{}", ns_since_epoch + 2)), - ("ns3", format!("{}", ns_since_epoch + 3)), - ("ns4", format!("{}", ns_since_epoch + 4)), - ("ns5", format!("{}", ns_since_epoch + 5)), - ("ns6", format!("{}", ns_since_epoch + 6)), - ]; - - lines - .iter() - .map(|line| { - let mut line = line.to_string(); - for (from, to) in &substitutions { - line = line.replace(from, to); - } - line - }) - .collect() -} - -struct TestServer { - server_process: Child, - - // The temporary directory **must** be last so that it is - // dropped after the database closes. - #[allow(dead_code)] - dir: TempDir, -} - -impl TestServer { - fn new() -> Result { - let dir = test_helpers::tmp_dir().unwrap(); - - let server_process = Command::cargo_bin("influxdb_iox") - .unwrap() - // Can enable for debbugging - //.arg("-vv") - .env("INFLUXDB_IOX_ID", "1") - .env("INFLUXDB_IOX_BIND_ADDR", HTTP_BIND_ADDR) - .env("INFLUXDB_IOX_GRPC_BIND_ADDR", GRPC_BIND_ADDR) - .spawn() - .unwrap(); - - Ok(Self { - dir, - server_process, - }) - } - - #[allow(dead_code)] - fn restart(&mut self) -> Result<()> { - self.server_process.kill().unwrap(); - self.server_process.wait().unwrap(); - self.server_process = Command::cargo_bin("influxdb_iox") - .unwrap() - // Can enable for debbugging - //.arg("-vv") - .env("INFLUXDB_IOX_DB_DIR", self.dir.path()) - .env("INFLUXDB_IOX_ID", "1") - .spawn() - .unwrap(); - Ok(()) - } - - async fn wait_until_ready(&self) { - // Poll the RPC and HTTP servers separately as they listen on - // different ports but both need to be up for the test to run - let try_grpc_connect = async { - let mut interval = tokio::time::interval(Duration::from_millis(500)); - - loop { - match influxdb_iox_client::connection::Builder::default() - .build(GRPC_URL_BASE) - .await - { - Ok(connection) => { - println!("Successfully connected to server"); - - let mut health = influxdb_iox_client::health::Client::new(connection); - - match health.check_storage().await { - Ok(_) => { - println!("Storage service is running"); - return; - } - Err(e) => { - println!("Error checking storage service status: {}", e); - } - } - } - Err(e) => { - println!("Waiting for gRPC API to be up: {}", e); - } - } - interval.tick().await; - } - }; - - let try_http_connect = async { - let client = reqwest::Client::new(); - let url = format!("{}/health", HTTP_BASE); - let mut interval = tokio::time::interval(Duration::from_millis(500)); - loop { - match client.get(&url).send().await { - Ok(resp) => { - println!("Successfully got a response from HTTP: {:?}", resp); - return; - } - Err(e) => { - println!("Waiting for HTTP server to be up: {}", e); - } - } - interval.tick().await; - } - }; - - let pair = future::join(try_http_connect, try_grpc_connect); - - let capped_check = tokio::time::timeout(Duration::from_secs(3), pair); - - match capped_check.await { - Ok(_) => println!("Server is up correctly"), - Err(e) => println!("WARNING: server was not ready: {}", e), - } - } -} - -impl Drop for TestServer { - fn drop(&mut self) { - self.server_process - .kill() - .expect("Should have been able to kill the test server"); - } -} diff --git a/tests/end_to_end_cases/flight_api.rs b/tests/end_to_end_cases/flight_api.rs index 7e8872ac0d..31ffa9574d 100644 --- a/tests/end_to_end_cases/flight_api.rs +++ b/tests/end_to_end_cases/flight_api.rs @@ -1,11 +1,21 @@ -use crate::{Scenario, GRPC_URL_BASE}; +use super::scenario::Scenario; +use crate::common::server_fixture::ServerFixture; use arrow_deps::assert_table_eq; -use influxdb_iox_client::{connection::Builder, flight::Client}; -pub async fn test(scenario: &Scenario, sql_query: &str, expected_read_data: &[String]) { - let connection = Builder::default().build(GRPC_URL_BASE).await.unwrap(); +#[tokio::test] +pub async fn test() { + let server_fixture = ServerFixture::create_shared().await; - let mut client = Client::new(connection); + let influxdb2 = server_fixture.influxdb2_client(); + let mut management_client = server_fixture.management_client(); + + let scenario = Scenario::new(); + scenario.create_database(&mut management_client).await; + + let expected_read_data = scenario.load_data(&influxdb2).await; + let sql_query = "select * from cpu_load_short"; + + let mut client = server_fixture.flight_client(); let mut query_results = client .perform_query(scenario.database_name(), sql_query) diff --git a/tests/end_to_end_cases/http.rs b/tests/end_to_end_cases/http.rs new file mode 100644 index 0000000000..3f076dfc0d --- /dev/null +++ b/tests/end_to_end_cases/http.rs @@ -0,0 +1,16 @@ +use crate::common::server_fixture::ServerFixture; + +#[tokio::test] +async fn test_http_error_messages() { + let server_fixture = ServerFixture::create_shared().await; + let client = server_fixture.influxdb2_client(); + + // send malformed request (bucket id is invalid) + let result = client + .write_line_protocol("Bar", "Foo", "arbitrary") + .await + .expect_err("Should have errored"); + + let expected_error = "HTTP request returned an error: 400 Bad Request, `{\"error\":\"Error parsing line protocol: A generic parsing error occurred: TakeWhile1\",\"error_code\":100}`"; + assert_eq!(result.to_string(), expected_error); +} diff --git a/tests/end_to_end_cases/management_api.rs b/tests/end_to_end_cases/management_api.rs index 7992f25c64..4a8577873f 100644 --- a/tests/end_to_end_cases/management_api.rs +++ b/tests/end_to_end_cases/management_api.rs @@ -1,20 +1,86 @@ use std::num::NonZeroU32; -use rand::{distributions::Alphanumeric, thread_rng, Rng}; +use generated_types::{ + google::protobuf::{Duration, Empty}, + influxdata::iox::management::v1::*, +}; +use influxdb_iox_client::management::CreateDatabaseError; +use test_helpers::assert_contains; -use generated_types::google::protobuf::Empty; -use generated_types::{google::protobuf::Duration, influxdata::iox::management::v1::*}; -use influxdb_iox_client::management::{Client, CreateDatabaseError}; +use super::{ + operations_api::get_operation_metadata, + scenario::{ + create_readable_database, create_two_partition_database, create_unreadable_database, + rand_name, + }, +}; +use crate::common::server_fixture::ServerFixture; -pub async fn test(client: &mut Client) { - test_set_get_writer_id(client).await; - test_create_database_duplicate_name(client).await; - test_create_database_invalid_name(client).await; - test_list_databases(client).await; - test_create_get_database(client).await; +#[tokio::test] +async fn test_list_update_remotes() { + let server_fixture = ServerFixture::create_single_use().await; + let mut client = server_fixture.management_client(); + + const TEST_REMOTE_ID_1: u32 = 42; + const TEST_REMOTE_ADDR_1: &str = "1.2.3.4:1234"; + const TEST_REMOTE_ID_2: u32 = 84; + const TEST_REMOTE_ADDR_2: &str = "4.3.2.1:4321"; + const TEST_REMOTE_ADDR_2_UPDATED: &str = "40.30.20.10:4321"; + + let res = client.list_remotes().await.expect("list remotes failed"); + assert_eq!(res.len(), 0); + + client + .update_remote(TEST_REMOTE_ID_1, TEST_REMOTE_ADDR_1) + .await + .expect("update failed"); + + let res = client.list_remotes().await.expect("list remotes failed"); + assert_eq!(res.len(), 1); + + client + .update_remote(TEST_REMOTE_ID_2, TEST_REMOTE_ADDR_2) + .await + .expect("update failed"); + + let res = client.list_remotes().await.expect("list remotes failed"); + assert_eq!(res.len(), 2); + assert_eq!(res[0].id, TEST_REMOTE_ID_1); + assert_eq!(res[0].connection_string, TEST_REMOTE_ADDR_1); + assert_eq!(res[1].id, TEST_REMOTE_ID_2); + assert_eq!(res[1].connection_string, TEST_REMOTE_ADDR_2); + + client + .delete_remote(TEST_REMOTE_ID_1) + .await + .expect("delete failed"); + + client + .delete_remote(TEST_REMOTE_ID_1) + .await + .expect_err("expected delete to fail"); + + let res = client.list_remotes().await.expect("list remotes failed"); + assert_eq!(res.len(), 1); + assert_eq!(res[0].id, TEST_REMOTE_ID_2); + assert_eq!(res[0].connection_string, TEST_REMOTE_ADDR_2); + + client + .update_remote(TEST_REMOTE_ID_2, TEST_REMOTE_ADDR_2_UPDATED) + .await + .expect("update failed"); + + let res = client.list_remotes().await.expect("list remotes failed"); + assert_eq!(res.len(), 1); + assert_eq!(res[0].id, TEST_REMOTE_ID_2); + assert_eq!(res[0].connection_string, TEST_REMOTE_ADDR_2_UPDATED); } -async fn test_set_get_writer_id(client: &mut Client) { +#[tokio::test] +async fn test_set_get_writer_id() { + let server_fixture = ServerFixture::create_single_use().await; + let mut client = server_fixture.management_client(); + const TEST_ID: u32 = 42; client @@ -27,7 +93,11 @@ async fn test_set_get_writer_id(client: &mut Client) { assert_eq!(got.get(), TEST_ID); } -async fn test_create_database_duplicate_name(client: &mut Client) { +#[tokio::test] +async fn test_create_database_duplicate_name() { + let server_fixture = ServerFixture::create_shared().await; + let mut client = server_fixture.management_client(); + let db_name = rand_name(); client @@ -52,7 +122,11 @@ async fn test_create_database_duplicate_name(client: &mut Client) { )) } -async fn test_create_database_invalid_name(client: &mut Client) { +#[tokio::test] +async fn test_create_database_invalid_name() { + let server_fixture = ServerFixture::create_shared().await; + let mut client = server_fixture.management_client(); + let err = client .create_database(DatabaseRules { name: "my_example\ndb".to_string(), @@ -64,7 +138,11 @@ async fn test_create_database_invalid_name(client: &mut Client) { assert!(matches!(dbg!(err), CreateDatabaseError::InvalidArgument(_))); } -async fn test_list_databases(client: &mut Client) { +#[tokio::test] +async fn test_list_databases() { + let server_fixture = ServerFixture::create_shared().await; + let mut client = server_fixture.management_client(); + let name = rand_name(); client .create_database(DatabaseRules { @@ -81,7 +159,11 @@ async fn test_list_databases(client: &mut Client) { assert!(names.contains(&name)); } -async fn test_create_get_database(client: &mut Client) { +#[tokio::test] +async fn test_create_get_database() { + let server_fixture = ServerFixture::create_shared().await; + let mut client = server_fixture.management_client(); + let db_name = rand_name(); // Specify everything to allow direct comparison between request and response @@ -93,27 +175,6 @@ async fn test_create_get_database(client: &mut Client) { part: Some(partition_template::part::Part::Table(Empty {})), }], }), - replication_config: Some(ReplicationConfig { - replications: vec!["cupcakes".to_string()], - replication_count: 3, - replication_queue_max_size: 20, - }), - subscription_config: Some(SubscriptionConfig { - subscriptions: vec![subscription_config::Subscription { - name: "subscription".to_string(), - host_group_id: "hostgroup".to_string(), - matcher: Some(Matcher { - predicate: "pred".to_string(), - table_matcher: Some(matcher::TableMatcher::All(Empty {})), - }), - }], - }), - query_config: Some(QueryConfig { - query_local: true, - primary: Default::default(), - secondaries: vec![], - read_only_partitions: vec![], - }), wal_buffer_config: Some(WalBufferConfig { buffer_size: 24, segment_size: 2, @@ -150,10 +211,419 @@ async fn test_create_get_database(client: &mut Client) { assert_eq!(response, rules); } -fn rand_name() -> String { - thread_rng() - .sample_iter(&Alphanumeric) - .take(10) - .map(char::from) - .collect() +#[tokio::test] +async fn test_chunk_get() { + use generated_types::influxdata::iox::management::v1::{Chunk, ChunkStorage}; + + let fixture = ServerFixture::create_shared().await; + let mut management_client = fixture.management_client(); + let mut write_client = fixture.write_client(); + + let db_name = rand_name(); + create_readable_database(&db_name, fixture.grpc_channel()).await; + + let lp_lines = vec![ + "cpu,region=west user=23.2 100", + "cpu,region=west user=21.0 150", + "disk,region=east bytes=99i 200", + ]; + + write_client + .write(&db_name, lp_lines.join("\n")) + .await + .expect("write succeded"); + + let mut chunks = management_client + .list_chunks(&db_name) + .await + .expect("listing chunks"); + + // ensure the output order is consistent + chunks.sort_by(|c1, c2| c1.partition_key.cmp(&c2.partition_key)); + + let expected: Vec = vec![ + Chunk { + partition_key: "cpu".into(), + id: 0, + storage: ChunkStorage::OpenMutableBuffer as i32, + estimated_bytes: 145, + }, + Chunk { + partition_key: "disk".into(), + id: 0, + storage: ChunkStorage::OpenMutableBuffer as i32, + estimated_bytes: 107, + }, + ]; + assert_eq!( + expected, chunks, + "expected:\n\n{:#?}\n\nactual:{:#?}", + expected, chunks + ); +} + +#[tokio::test] +async fn test_chunk_get_errors() { + let fixture = ServerFixture::create_shared().await; + let mut management_client = fixture.management_client(); + let db_name = rand_name(); + + let err = management_client + .list_chunks(&db_name) + .await + .expect_err("no db had been created"); + + assert_contains!( + err.to_string(), + "Some requested entity was not found: Resource database" + ); + + create_unreadable_database(&db_name, fixture.grpc_channel()).await; + + let err = management_client + .list_chunks(&db_name) + .await + .expect_err("db can't be read"); + + assert_contains!( + err.to_string(), + "Precondition violation influxdata.com/iox - database" + ); + assert_contains!( + err.to_string(), + "Cannot read from database: no mutable buffer configured" + ); +} + +#[tokio::test] +async fn test_partition_list() { + let fixture = ServerFixture::create_shared().await; + let mut management_client = fixture.management_client(); + + let db_name = rand_name(); + create_two_partition_database(&db_name, fixture.grpc_channel()).await; + + let mut partitions = management_client + .list_partitions(&db_name) + .await + .expect("listing partition"); + + // ensure the output order is consistent + partitions.sort_by(|p1, p2| p1.key.cmp(&p2.key)); + + let expected = vec![ + Partition { + key: "cpu".to_string(), + }, + Partition { + key: "mem".to_string(), + }, + ]; + + assert_eq!( + expected, partitions, + "expected:\n\n{:#?}\n\nactual:{:#?}", + expected, partitions + ); +} + +#[tokio::test] +async fn test_partition_list_error() { + let fixture = ServerFixture::create_shared().await; + let mut management_client = fixture.management_client(); + + let err = management_client + .list_partitions("this database does not exist") + .await + .expect_err("expected error"); + + assert_contains!(err.to_string(), "Database not found"); +} + +#[tokio::test] +async fn test_partition_get() { + use generated_types::influxdata::iox::management::v1::Partition; + + let fixture = ServerFixture::create_shared().await; + let mut management_client = fixture.management_client(); + + let db_name = rand_name(); + create_two_partition_database(&db_name, fixture.grpc_channel()).await; + + let partition_key = "cpu"; + let partition = management_client + .get_partition(&db_name, partition_key) + .await + .expect("getting partition"); + + let expected = Partition { key: "cpu".into() }; + + assert_eq!( + expected, partition, + "expected:\n\n{:#?}\n\nactual:{:#?}", + expected, partition + ); +} + +#[tokio::test] +async fn test_partition_get_error() { + let fixture = ServerFixture::create_shared().await; + let mut management_client = fixture.management_client(); + let mut write_client = fixture.write_client(); + + let err = management_client + .list_partitions("this database does not exist") + .await + .expect_err("expected error"); + + assert_contains!(err.to_string(), "Database not found"); + + let db_name = rand_name(); + create_readable_database(&db_name, fixture.grpc_channel()).await; + + let lp_lines = + vec!["processes,host=foo running=4i,sleeping=514i,total=519i 1591894310000000000"]; + + write_client + .write(&db_name, lp_lines.join("\n")) + .await + .expect("write succeded"); + + let err = management_client + .get_partition(&db_name, "non existent partition") + .await + .expect_err("exepcted error getting partition"); + + assert_contains!(err.to_string(), "Partition not found"); +} + +#[tokio::test] +async fn test_list_partition_chunks() { + let fixture = ServerFixture::create_shared().await; + let mut management_client = fixture.management_client(); + let mut write_client = fixture.write_client(); + + let db_name = rand_name(); + create_readable_database(&db_name, fixture.grpc_channel()).await; + + let lp_lines = vec![ + "cpu,region=west user=23.2 100", + "cpu,region=west user=21.0 150", + "disk,region=east bytes=99i 200", + ]; + + write_client + .write(&db_name, lp_lines.join("\n")) + .await + .expect("write succeded"); + + let partition_key = "cpu"; + let chunks = management_client + .list_partition_chunks(&db_name, partition_key) + .await + .expect("getting partition chunks"); + + let expected: Vec = vec![Chunk { + partition_key: "cpu".into(), + id: 0, + storage: ChunkStorage::OpenMutableBuffer as i32, + estimated_bytes: 145, + }]; + + assert_eq!( + expected, chunks, + "expected:\n\n{:#?}\n\nactual:{:#?}", + expected, chunks + ); +} + +#[tokio::test] +async fn test_list_partition_chunk_errors() { + let fixture = ServerFixture::create_shared().await; + let mut management_client = fixture.management_client(); + let db_name = rand_name(); + + let err = management_client + .list_partition_chunks(&db_name, "cpu") + .await + .expect_err("no db had been created"); + + assert_contains!( + err.to_string(), + "Some requested entity was not found: Resource database" + ); +} + +#[tokio::test] +async fn test_new_partition_chunk() { + let fixture = ServerFixture::create_shared().await; + let mut management_client = fixture.management_client(); + let mut write_client = fixture.write_client(); + + let db_name = rand_name(); + create_readable_database(&db_name, fixture.grpc_channel()).await; + + let lp_lines = vec!["cpu,region=west user=23.2 100"]; + + write_client + .write(&db_name, lp_lines.join("\n")) + .await + .expect("write succeded"); + + let chunks = management_client + .list_chunks(&db_name) + .await + .expect("listing chunks"); + + assert_eq!(chunks.len(), 1, "Chunks: {:#?}", chunks); + let partition_key = "cpu"; + + // Rollover the a second chunk + management_client + .new_partition_chunk(&db_name, partition_key) + .await + .expect("new partition chunk"); + + // Load some more data and now expect that we have a second chunk + + let lp_lines = vec!["cpu,region=west user=21.0 150"]; + + write_client + .write(&db_name, lp_lines.join("\n")) + .await + .expect("write succeded"); + + let chunks = management_client + .list_chunks(&db_name) + .await + .expect("listing chunks"); + + assert_eq!(chunks.len(), 2, "Chunks: {:#?}", chunks); + + // Made all chunks in the same partition + assert_eq!( + chunks.iter().filter(|c| c.partition_key == "cpu").count(), + 2, + "Chunks: {:#?}", + chunks + ); + + // Rollover a (currently non existent) partition which is OK + management_client + .new_partition_chunk(&db_name, "non_existent_partition") + .await + .expect("new partition chunk"); + + assert_eq!(chunks.len(), 2, "Chunks: {:#?}", chunks); + assert_eq!( + chunks.iter().filter(|c| c.partition_key == "cpu").count(), + 2, + "Chunks: {:#?}", + chunks + ); + assert_eq!( + chunks + .iter() + .filter(|c| c.partition_key == "non_existent_partition") + .count(), + 0, + "Chunks: {:#?}", + chunks + ); +} + +#[tokio::test] +async fn test_new_partition_chunk_error() { + let fixture = ServerFixture::create_shared().await; + let mut management_client = fixture.management_client(); + + let err = management_client + .new_partition_chunk("this database does not exist", "nor_does_this_partition") + .await + .expect_err("expected error"); + + assert_contains!(err.to_string(), "Database not found"); +} + +#[tokio::test] +async fn test_close_partition_chunk() { + use influxdb_iox_client::management::generated_types::operation_metadata::Job; + use influxdb_iox_client::management::generated_types::ChunkStorage; + + let fixture = ServerFixture::create_shared().await; + let mut management_client = fixture.management_client(); + let mut write_client = fixture.write_client(); + let mut operations_client = fixture.operations_client(); + + let db_name = rand_name(); + create_readable_database(&db_name, fixture.grpc_channel()).await; + + let partition_key = "cpu"; + let lp_lines = vec!["cpu,region=west user=23.2 100"]; + + write_client + .write(&db_name, lp_lines.join("\n")) + .await + .expect("write succeded"); + + let chunks = management_client + .list_chunks(&db_name) + .await + .expect("listing chunks"); + + assert_eq!(chunks.len(), 1, "Chunks: {:#?}", chunks); + assert_eq!(chunks[0].id, 0); + assert_eq!(chunks[0].storage, ChunkStorage::OpenMutableBuffer as i32); + + // Move the chunk to read buffer + let operation = management_client + .close_partition_chunk(&db_name, partition_key, 0) + .await + .expect("new partition chunk"); + + println!("Operation response is {:?}", operation); + let operation_id = operation.name.parse().expect("not an integer"); + + let meta = get_operation_metadata(operation.metadata); + + // ensure we got a legit job description back + if let Some(Job::CloseChunk(close_chunk)) = meta.job { + assert_eq!(close_chunk.db_name, db_name); + assert_eq!(close_chunk.partition_key, partition_key); + assert_eq!(close_chunk.chunk_id, 0); + } else { + panic!("unexpected job returned") + }; + + // wait for the job to be done + operations_client + .wait_operation(operation_id, Some(std::time::Duration::from_secs(1))) + .await + .expect("failed to wait operation"); + + // And now the chunk should be good + let mut chunks = management_client + .list_chunks(&db_name) + .await + .expect("listing chunks"); + chunks.sort_by(|c1, c2| c1.id.cmp(&c2.id)); + + assert_eq!(chunks.len(), 2, "Chunks: {:#?}", chunks); + assert_eq!(chunks[0].id, 0); + assert_eq!(chunks[0].storage, ChunkStorage::ReadBuffer as i32); + assert_eq!(chunks[1].id, 1); + assert_eq!(chunks[1].storage, ChunkStorage::OpenMutableBuffer as i32); +} + +#[tokio::test] +async fn test_close_partition_chunk_error() { + let fixture = ServerFixture::create_shared().await; + let mut management_client = fixture.management_client(); + + let err = management_client + .close_partition_chunk("this database does not exist", "nor_does_this_partition", 0) + .await + .expect_err("expected error"); + + assert_contains!(err.to_string(), "Database not found"); } diff --git a/tests/end_to_end_cases/management_cli.rs b/tests/end_to_end_cases/management_cli.rs index 56b9b4424a..5e21722513 100644 --- a/tests/end_to_end_cases/management_cli.rs +++ b/tests/end_to_end_cases/management_cli.rs @@ -1,12 +1,16 @@ use assert_cmd::Command; +use data_types::job::{Job, Operation}; use predicates::prelude::*; +use test_helpers::make_temp_file; -pub async fn test(addr: impl AsRef) { - test_writer_id(addr.as_ref()).await; - test_create_database(addr.as_ref()).await; -} +use crate::common::server_fixture::ServerFixture; -async fn test_writer_id(addr: &str) { +use super::scenario::{create_readable_database, rand_name}; + +#[tokio::test] +async fn test_writer_id() { + let server_fixture = ServerFixture::create_shared().await; + let addr = server_fixture.grpc_base(); Command::cargo_bin("influxdb_iox") .unwrap() .arg("writer") @@ -29,8 +33,12 @@ async fn test_writer_id(addr: &str) { .stdout(predicate::str::contains("32")); } -async fn test_create_database(addr: &str) { - let db = "management-cli-test"; +#[tokio::test] +async fn test_create_database() { + let server_fixture = ServerFixture::create_shared().await; + let addr = server_fixture.grpc_base(); + let db_name = rand_name(); + let db = &db_name; Command::cargo_bin("influxdb_iox") .unwrap() @@ -54,16 +62,57 @@ async fn test_create_database(addr: &str) { .success() .stdout(predicate::str::contains("Ok")); + // Listing the databases includes the newly created database Command::cargo_bin("influxdb_iox") .unwrap() .arg("database") - .arg("get") + .arg("list") .arg("--host") .arg(addr) .assert() .success() .stdout(predicate::str::contains(db)); + // Retrieving the database includes the name and a mutable buffer configuration + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("get") + .arg(db) + .arg("--host") + .arg(addr) + .assert() + .success() + .stdout( + predicate::str::contains(db) + .and(predicate::str::contains(format!("name: \"{}\"", db))) + // validate the defaults have been set reasonably + .and(predicate::str::contains("%Y-%m-%d %H:00:00")) + .and(predicate::str::contains("buffer_size: 104857600")) + .and(predicate::str::contains("MutableBufferConfig")), + ); +} + +#[tokio::test] +async fn test_create_database_size() { + let server_fixture = ServerFixture::create_shared().await; + let addr = server_fixture.grpc_base(); + let db_name = rand_name(); + let db = &db_name; + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("create") + .arg(db) + .arg("-m") + .arg("1000") + .arg("--host") + .arg(addr) + .assert() + .success() + .stdout(predicate::str::contains("Ok")); + Command::cargo_bin("influxdb_iox") .unwrap() .arg("database") @@ -73,5 +122,460 @@ async fn test_create_database(addr: &str) { .arg(addr) .assert() .success() - .stdout(predicate::str::contains(format!("name: \"{}\"", db))); + .stdout( + predicate::str::contains("buffer_size: 1000") + .and(predicate::str::contains("MutableBufferConfig")), + ); +} + +#[tokio::test] +async fn test_create_database_zero_size() { + let server_fixture = ServerFixture::create_shared().await; + let addr = server_fixture.grpc_base(); + let db_name = rand_name(); + let db = &db_name; + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("create") + .arg(db) + .arg("-m") + .arg("0") + .arg("--host") + .arg(addr) + .assert() + .success() + .stdout(predicate::str::contains("Ok")); + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("get") + .arg(db) + .arg("--host") + .arg(addr) + .assert() + .success() + // Should not have a mutable buffer + .stdout(predicate::str::contains("MutableBufferConfig").not()); +} + +#[tokio::test] +async fn test_get_chunks() { + let server_fixture = ServerFixture::create_shared().await; + let addr = server_fixture.grpc_base(); + let db_name = rand_name(); + + create_readable_database(&db_name, server_fixture.grpc_channel()).await; + + let lp_data = vec![ + "cpu,region=west user=23.2 100", + "cpu,region=west user=21.0 150", + ]; + + load_lp(addr, &db_name, lp_data); + + let expected = r#"[ + { + "partition_key": "cpu", + "id": 0, + "storage": "OpenMutableBuffer", + "estimated_bytes": 145 + } +]"#; + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("chunk") + .arg("list") + .arg(&db_name) + .arg("--host") + .arg(addr) + .assert() + .success() + .stdout(predicate::str::contains(expected)); +} + +#[tokio::test] +async fn test_list_chunks_error() { + let server_fixture = ServerFixture::create_shared().await; + let addr = server_fixture.grpc_base(); + let db_name = rand_name(); + + // note don't make the database, expect error + + // list the chunks + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("chunk") + .arg("list") + .arg(&db_name) + .arg("--host") + .arg(addr) + .assert() + .failure() + .stderr( + predicate::str::contains("Some requested entity was not found: Resource database") + .and(predicate::str::contains(&db_name)), + ); +} + +#[tokio::test] +async fn test_remotes() { + let server_fixture = ServerFixture::create_single_use().await; + let addr = server_fixture.grpc_base(); + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("server") + .arg("remote") + .arg("list") + .arg("--host") + .arg(addr) + .assert() + .success() + .stdout(predicate::str::contains("no remotes configured")); + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("server") + .arg("remote") + .arg("set") + .arg("1") + .arg("http://1.2.3.4:1234") + .arg("--host") + .arg(addr) + .assert() + .success(); + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("server") + .arg("remote") + .arg("list") + .arg("--host") + .arg(addr) + .assert() + .success() + .stdout(predicate::str::contains("http://1.2.3.4:1234")); + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("server") + .arg("remote") + .arg("remove") + .arg("1") + .arg("--host") + .arg(addr) + .assert() + .success(); + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("server") + .arg("remote") + .arg("list") + .arg("--host") + .arg(addr) + .assert() + .success() + .stdout(predicate::str::contains("no remotes configured")); +} + +#[tokio::test] +async fn test_list_partitions() { + let server_fixture = ServerFixture::create_shared().await; + let addr = server_fixture.grpc_base(); + let db_name = rand_name(); + + create_readable_database(&db_name, server_fixture.grpc_channel()).await; + + let lp_data = vec![ + "cpu,region=west user=23.2 100", + "mem,region=west free=100000 150", + ]; + load_lp(addr, &db_name, lp_data); + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("partition") + .arg("list") + .arg(&db_name) + .arg("--host") + .arg(addr) + .assert() + .success() + .stdout(predicate::str::contains("cpu").and(predicate::str::contains("mem"))); +} + +#[tokio::test] +async fn test_list_partitions_error() { + let server_fixture = ServerFixture::create_shared().await; + let addr = server_fixture.grpc_base(); + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("partition") + .arg("list") + .arg("non_existent_database") + .arg("--host") + .arg(addr) + .assert() + .failure() + .stderr(predicate::str::contains("Database not found")); +} + +#[tokio::test] +async fn test_get_partition() { + let server_fixture = ServerFixture::create_shared().await; + let addr = server_fixture.grpc_base(); + let db_name = rand_name(); + + create_readable_database(&db_name, server_fixture.grpc_channel()).await; + + let lp_data = vec![ + "cpu,region=west user=23.2 100", + "mem,region=west free=100000 150", + ]; + load_lp(addr, &db_name, lp_data); + + let expected = r#""key": "cpu""#; + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("partition") + .arg("get") + .arg(&db_name) + .arg("cpu") + .arg("--host") + .arg(addr) + .assert() + .success() + .stdout(predicate::str::contains(expected)); +} + +#[tokio::test] +async fn test_get_partition_error() { + let server_fixture = ServerFixture::create_shared().await; + let addr = server_fixture.grpc_base(); + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("partition") + .arg("get") + .arg("cpu") + .arg("non_existent_database") + .arg("--host") + .arg(addr) + .assert() + .failure() + .stderr(predicate::str::contains("Database not found")); +} + +#[tokio::test] +async fn test_list_partition_chunks() { + let server_fixture = ServerFixture::create_shared().await; + let addr = server_fixture.grpc_base(); + let db_name = rand_name(); + + create_readable_database(&db_name, server_fixture.grpc_channel()).await; + + let lp_data = vec![ + "cpu,region=west user=23.2 100", + "cpu2,region=west user=21.0 150", + ]; + + load_lp(addr, &db_name, lp_data); + + let expected = r#" + "partition_key": "cpu", + "id": 0, + "storage": "OpenMutableBuffer", +"#; + + let partition_key = "cpu"; + // should not contain anything related to cpu2 partition + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("partition") + .arg("list-chunks") + .arg(&db_name) + .arg(&partition_key) + .arg("--host") + .arg(addr) + .assert() + .success() + .stdout(predicate::str::contains(expected).and(predicate::str::contains("cpu2").not())); +} + +#[tokio::test] +async fn test_list_partition_chunks_error() { + let server_fixture = ServerFixture::create_shared().await; + let addr = server_fixture.grpc_base(); + let db_name = rand_name(); + + // note don't make the database, expect error + + // list the chunks + let partition_key = "cpu"; + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("partition") + .arg("list-chunks") + .arg(&db_name) + .arg(&partition_key) + .arg("--host") + .arg(addr) + .assert() + .failure() + .stderr( + predicate::str::contains("Some requested entity was not found: Resource database") + .and(predicate::str::contains(&db_name)), + ); +} + +#[tokio::test] +async fn test_new_partition_chunk() { + let server_fixture = ServerFixture::create_shared().await; + let addr = server_fixture.grpc_base(); + let db_name = rand_name(); + + create_readable_database(&db_name, server_fixture.grpc_channel()).await; + + let lp_data = vec!["cpu,region=west user=23.2 100"]; + load_lp(addr, &db_name, lp_data); + + let expected = "Ok"; + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("partition") + .arg("new-chunk") + .arg(&db_name) + .arg("cpu") + .arg("--host") + .arg(addr) + .assert() + .success() + .stdout(predicate::str::contains(expected)); + + let expected = "ClosedMutableBuffer"; + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("chunk") + .arg("list") + .arg(&db_name) + .arg("--host") + .arg(addr) + .assert() + .success() + .stdout(predicate::str::contains(expected)); +} + +#[tokio::test] +async fn test_new_partition_chunk_error() { + let server_fixture = ServerFixture::create_shared().await; + let addr = server_fixture.grpc_base(); + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("partition") + .arg("new-chunk") + .arg("non_existent_database") + .arg("non_existent_partition") + .arg("--host") + .arg(addr) + .assert() + .failure() + .stderr(predicate::str::contains("Database not found")); +} + +#[tokio::test] +async fn test_close_partition_chunk() { + let server_fixture = ServerFixture::create_shared().await; + let addr = server_fixture.grpc_base(); + let db_name = rand_name(); + + create_readable_database(&db_name, server_fixture.grpc_channel()).await; + + let lp_data = vec!["cpu,region=west user=23.2 100"]; + load_lp(addr, &db_name, lp_data); + + let stdout: Operation = serde_json::from_slice( + &Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("partition") + .arg("close-chunk") + .arg(&db_name) + .arg("cpu") + .arg("0") + .arg("--host") + .arg(addr) + .assert() + .success() + .get_output() + .stdout, + ) + .expect("Expected JSON output"); + + let expected_job = Job::CloseChunk { + db_name, + partition_key: "cpu".into(), + chunk_id: 0, + }; + + assert_eq!( + Some(expected_job), + stdout.job, + "operation was {:#?}", + stdout + ); +} + +#[tokio::test] +async fn test_close_partition_chunk_error() { + let server_fixture = ServerFixture::create_shared().await; + let addr = server_fixture.grpc_base(); + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("partition") + .arg("close-chunk") + .arg("non_existent_database") + .arg("non_existent_partition") + .arg("0") + .arg("--host") + .arg(addr) + .assert() + .failure() + .stderr(predicate::str::contains("Database not found")); +} + +/// Loads the specified lines into the named database +fn load_lp(addr: &str, db_name: &str, lp_data: Vec<&str>) { + let lp_data_file = make_temp_file(lp_data.join("\n")); + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("write") + .arg(&db_name) + .arg(lp_data_file.as_ref()) + .arg("--host") + .arg(addr) + .assert() + .success() + .stdout(predicate::str::contains("Lines OK")); } diff --git a/tests/end_to_end_cases/mod.rs b/tests/end_to_end_cases/mod.rs index 99fb1b6459..776420167a 100644 --- a/tests/end_to_end_cases/mod.rs +++ b/tests/end_to_end_cases/mod.rs @@ -1,5 +1,12 @@ pub mod flight_api; +pub mod http; pub mod management_api; pub mod management_cli; +pub mod operations_api; +pub mod operations_cli; pub mod read_api; +pub mod read_cli; +pub mod scenario; pub mod storage_api; +pub mod write_api; +pub mod write_cli; diff --git a/tests/end_to_end_cases/operations_api.rs b/tests/end_to_end_cases/operations_api.rs new file mode 100644 index 0000000000..0fde9ea716 --- /dev/null +++ b/tests/end_to_end_cases/operations_api.rs @@ -0,0 +1,96 @@ +use crate::common::server_fixture::ServerFixture; +use generated_types::google::protobuf::Any; +use influxdb_iox_client::{management::generated_types::*, operations, protobuf_type_url_eq}; +use std::time::Duration; + +// TODO remove after #1001 and use something directly in the influxdb_iox_client +// crate +pub fn get_operation_metadata(metadata: Option) -> OperationMetadata { + assert!(metadata.is_some()); + let metadata = metadata.unwrap(); + assert!(protobuf_type_url_eq(&metadata.type_url, OPERATION_METADATA)); + prost::Message::decode(metadata.value).expect("failed to decode metadata") +} + +#[tokio::test] +async fn test_operations() { + let server_fixture = ServerFixture::create_single_use().await; + let mut management_client = server_fixture.management_client(); + let mut operations_client = server_fixture.operations_client(); + + let running_ops = operations_client + .list_operations() + .await + .expect("list operations failed"); + + assert_eq!(running_ops.len(), 0); + + let nanos = vec![Duration::from_secs(20).as_nanos() as _, 1]; + + let operation = management_client + .create_dummy_job(nanos.clone()) + .await + .expect("create dummy job failed"); + + let running_ops = operations_client + .list_operations() + .await + .expect("list operations failed"); + + assert_eq!(running_ops.len(), 1); + assert_eq!(running_ops[0].name, operation.name); + + let id = operation.name.parse().expect("not an integer"); + + // Check we can fetch metadata for Operation + let fetched = operations_client + .get_operation(id) + .await + .expect("get operation failed"); + let meta = get_operation_metadata(fetched.metadata); + let job = meta.job.expect("expected a job"); + + assert_eq!(meta.task_count, 2); + assert_eq!(meta.pending_count, 1); + assert_eq!(job, operation_metadata::Job::Dummy(Dummy { nanos })); + assert!(!fetched.done); + + // Check wait times out correctly + let fetched = operations_client + .wait_operation(id, Some(Duration::from_micros(10))) + .await + .expect("failed to wait operation"); + + assert!(!fetched.done); + // Shouldn't specify wall_nanos as not complete + assert_eq!(meta.wall_nanos, 0); + + let wait = tokio::spawn(async move { + let mut operations_client = server_fixture.operations_client(); + operations_client + .wait_operation(id, None) + .await + .expect("failed to wait operation") + }); + + operations_client + .cancel_operation(id) + .await + .expect("failed to cancel operation"); + + let waited = wait.await.unwrap(); + let meta = get_operation_metadata(waited.metadata); + + assert!(waited.done); + assert!(meta.wall_nanos > 0); + assert!(meta.cpu_nanos > 0); + assert_eq!(meta.pending_count, 0); + assert_eq!(meta.task_count, 2); + + match waited.result { + Some(operations::generated_types::operation::Result::Error(status)) => { + assert_eq!(status.code, tonic::Code::Cancelled as i32) + } + _ => panic!("expected error"), + } +} diff --git a/tests/end_to_end_cases/operations_cli.rs b/tests/end_to_end_cases/operations_cli.rs new file mode 100644 index 0000000000..ac802b3889 --- /dev/null +++ b/tests/end_to_end_cases/operations_cli.rs @@ -0,0 +1,88 @@ +use crate::common::server_fixture::ServerFixture; +use assert_cmd::Command; +use data_types::job::{Job, Operation, OperationStatus}; +use predicates::prelude::*; + +#[tokio::test] +async fn test_start_stop() { + let server_fixture = ServerFixture::create_single_use().await; + let addr = server_fixture.grpc_base(); + let duration = std::time::Duration::from_secs(10).as_nanos() as u64; + + let stdout: Operation = serde_json::from_slice( + &Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("operation") + .arg("test") + .arg(duration.to_string()) + .arg("--host") + .arg(addr) + .assert() + .success() + .get_output() + .stdout, + ) + .expect("expected JSON output"); + + assert_eq!(stdout.task_count, 1); + match stdout.job { + Some(Job::Dummy { nanos }) => assert_eq!(nanos, vec![duration]), + _ => panic!("expected dummy job got {:?}", stdout.job), + } + + let operations: Vec = serde_json::from_slice( + &Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("operation") + .arg("list") + .arg("--host") + .arg(addr) + .assert() + .success() + .get_output() + .stdout, + ) + .expect("expected JSON output"); + + assert_eq!(operations.len(), 1); + match &operations[0].job { + Some(Job::Dummy { nanos }) => { + assert_eq!(nanos.len(), 1); + assert_eq!(nanos[0], duration); + } + _ => panic!("expected dummy job got {:?}", &operations[0].job), + } + + let id = operations[0].id; + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("operation") + .arg("cancel") + .arg(id.to_string()) + .arg("--host") + .arg(addr) + .assert() + .success() + .stdout(predicate::str::contains("Ok")); + + let completed: Operation = serde_json::from_slice( + &Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("operation") + .arg("wait") + .arg(id.to_string()) + .arg("--host") + .arg(addr) + .assert() + .success() + .get_output() + .stdout, + ) + .expect("expected JSON output"); + + assert_eq!(completed.pending_count, 0); + assert_eq!(completed.task_count, 1); + assert_eq!(completed.status, OperationStatus::Cancelled); + assert_eq!(&completed.job, &operations[0].job) +} diff --git a/tests/end_to_end_cases/read_api.rs b/tests/end_to_end_cases/read_api.rs index 8a05c1f062..2128782f97 100644 --- a/tests/end_to_end_cases/read_api.rs +++ b/tests/end_to_end_cases/read_api.rs @@ -1,29 +1,23 @@ -use crate::{Scenario, IOX_API_V1_BASE}; +use super::scenario::Scenario; +use crate::common::server_fixture::ServerFixture; -pub async fn test( - client: &reqwest::Client, - scenario: &Scenario, - sql_query: &str, - expected_read_data: &[String], -) { - let text = read_data_as_sql(&client, scenario, sql_query).await; +#[tokio::test] +pub async fn test() { + let server_fixture = ServerFixture::create_shared().await; + let mut management_client = server_fixture.management_client(); + let influxdb2 = server_fixture.influxdb2_client(); - assert_eq!( - text, expected_read_data, - "Actual:\n{:#?}\nExpected:\n{:#?}", - text, expected_read_data - ); -} + let scenario = Scenario::new(); + scenario.create_database(&mut management_client).await; -async fn read_data_as_sql( - client: &reqwest::Client, - scenario: &Scenario, - sql_query: &str, -) -> Vec { + let expected_read_data = scenario.load_data(&influxdb2).await; + let sql_query = "select * from cpu_load_short"; + + let client = reqwest::Client::new(); let db_name = format!("{}_{}", scenario.org_id_str(), scenario.bucket_id_str()); let path = format!("/databases/{}/query", db_name); - let url = format!("{}{}", IOX_API_V1_BASE, path); - let lines = client + let url = format!("{}{}", server_fixture.iox_api_v1_base(), path); + let lines: Vec<_> = client .get(&url) .query(&[("q", sql_query)]) .send() @@ -36,5 +30,10 @@ async fn read_data_as_sql( .split('\n') .map(str::to_string) .collect(); - lines + + assert_eq!( + lines, expected_read_data, + "Actual:\n{:#?}\nExpected:\n{:#?}", + lines, expected_read_data + ); } diff --git a/tests/end_to_end_cases/read_cli.rs b/tests/end_to_end_cases/read_cli.rs new file mode 100644 index 0000000000..739f8ac2de --- /dev/null +++ b/tests/end_to_end_cases/read_cli.rs @@ -0,0 +1,169 @@ +use assert_cmd::Command; +use predicates::prelude::*; +use test_helpers::make_temp_file; + +use crate::common::server_fixture::ServerFixture; + +use super::scenario::rand_name; + +#[tokio::test] +pub async fn test() { + let server_fixture = ServerFixture::create_single_use().await; + let db_name = rand_name(); + let addr = server_fixture.grpc_base(); + + create_database(&db_name, addr).await; + test_read_default(&db_name, addr).await; + test_read_format_pretty(&db_name, addr).await; + test_read_format_csv(&db_name, addr).await; + test_read_format_json(&db_name, addr).await; + test_read_error(&db_name, addr).await; +} + +async fn create_database(db_name: &str, addr: &str) { + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("create") + .arg(db_name) + .arg("-m") + .arg("100") // give it a mutable buffer + .arg("--host") + .arg(addr) + .assert() + .success() + .stdout(predicate::str::contains("Ok")); + + let lp_data = vec![ + "cpu,region=west user=23.2 100", + "cpu,region=west user=21.0 150", + ]; + + let lp_data_file = make_temp_file(lp_data.join("\n")); + + // read from temp file + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("write") + .arg(db_name) + .arg(lp_data_file.as_ref()) + .arg("--host") + .arg(addr) + .assert() + .success() + .stdout(predicate::str::contains("2 Lines OK")); +} + +async fn test_read_default(db_name: &str, addr: &str) { + let expected = "+--------+------+------+\n\ + | region | time | user |\n\ + +--------+------+------+\n\ + | west | 100 | 23.2 |\n\ + | west | 150 | 21 |\n\ + +--------+------+------+"; + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("query") + .arg(db_name) + .arg("select * from cpu") + .arg("--host") + .arg(addr) + .assert() + .success() + .stdout(predicate::str::contains(expected)); +} + +async fn test_read_format_pretty(db_name: &str, addr: &str) { + let expected = "+--------+------+------+\n\ + | region | time | user |\n\ + +--------+------+------+\n\ + | west | 100 | 23.2 |\n\ + | west | 150 | 21 |\n\ + +--------+------+------+"; + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("query") + .arg(db_name) + .arg("select * from cpu") + .arg("--host") + .arg(addr) + .arg("--format") + .arg("pretty") + .assert() + .success() + .stdout(predicate::str::contains(expected)); +} + +async fn test_read_format_csv(db_name: &str, addr: &str) { + let expected = "region,time,user\nwest,100,23.2\nwest,150,21.0"; + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("query") + .arg(db_name) + .arg("select * from cpu") + .arg("--host") + .arg(addr) + .arg("--format") + .arg("csv") + .assert() + .success() + .stdout(predicate::str::contains(expected)); +} + +async fn test_read_format_json(db_name: &str, addr: &str) { + let expected = + r#"[{"region":"west","time":100,"user":23.2},{"region":"west","time":150,"user":21.0}]"#; + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("query") + .arg(db_name) + .arg("select * from cpu") + .arg("--host") + .arg(addr) + .arg("--format") + .arg("json") + .assert() + .success() + .stdout(predicate::str::contains(expected)); +} + +async fn test_read_error(db_name: &str, addr: &str) { + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("query") + .arg(db_name) + .arg("select * from unknown_table") + .arg("--host") + .arg(addr) + .assert() + .failure() + .stderr(predicate::str::contains( + "no chunks found in builder for table", + )); + + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("query") + .arg(db_name) + .arg("select * from cpu") + .arg("--host") + .arg(addr) + .arg("--format") + .arg("not_a_valid_format") + .assert() + .failure() + .stderr(predicate::str::contains( + "Unknown format type: not_a_valid_format. Expected one of 'pretty', 'csv' or 'json'", + )); +} diff --git a/tests/end_to_end_cases/scenario.rs b/tests/end_to_end_cases/scenario.rs new file mode 100644 index 0000000000..72c83954eb --- /dev/null +++ b/tests/end_to_end_cases/scenario.rs @@ -0,0 +1,327 @@ +use std::time::SystemTime; + +use generated_types::google::protobuf::Empty; +use generated_types::influxdata::iox::management::v1::*; +use rand::{ + distributions::{Alphanumeric, Standard}, + thread_rng, Rng, +}; + +use std::{convert::TryInto, str, u32}; + +use futures::prelude::*; +use prost::Message; + +use data_types::{names::org_and_bucket_to_database, DatabaseName}; +use generated_types::{influxdata::iox::management::v1::DatabaseRules, ReadSource, TimestampRange}; + +type Error = Box; +type Result = std::result::Result; + +/// A test fixture used for working with the influxdb v2 data model +/// (storage gRPC api and v2 write api). +/// +/// Each scenario is assigned a a random org and bucket id to ensure +/// tests do not interfere with one another +#[derive(Debug)] +pub struct Scenario { + org_id: String, + bucket_id: String, + ns_since_epoch: i64, +} + +impl Scenario { + /// Create a new `Scenario` with a random org_id and bucket_id + pub fn new() -> Self { + let ns_since_epoch = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .expect("System time should have been after the epoch") + .as_nanos() + .try_into() + .expect("Unable to represent system time"); + + Self { + ns_since_epoch, + org_id: rand_id(), + bucket_id: rand_id(), + } + } + + pub fn org_id_str(&self) -> &str { + &self.org_id + } + + pub fn bucket_id_str(&self) -> &str { + &self.bucket_id + } + + pub fn org_id(&self) -> u64 { + u64::from_str_radix(&self.org_id, 16).unwrap() + } + + pub fn bucket_id(&self) -> u64 { + u64::from_str_radix(&self.bucket_id, 16).unwrap() + } + + pub fn database_name(&self) -> DatabaseName<'_> { + org_and_bucket_to_database(&self.org_id, &self.bucket_id).unwrap() + } + + pub fn ns_since_epoch(&self) -> i64 { + self.ns_since_epoch + } + + pub fn read_source(&self) -> Option { + let partition_id = u64::from(u32::MAX); + let read_source = ReadSource { + org_id: self.org_id(), + bucket_id: self.bucket_id(), + partition_id, + }; + + let mut d = bytes::BytesMut::new(); + read_source.encode(&mut d).unwrap(); + let read_source = generated_types::google::protobuf::Any { + type_url: "/TODO".to_string(), + value: d.freeze(), + }; + + Some(read_source) + } + + pub fn timestamp_range(&self) -> Option { + Some(TimestampRange { + start: self.ns_since_epoch(), + end: self.ns_since_epoch() + 10, + }) + } + + /// Create's the database on the server for this scenario + pub async fn create_database(&self, client: &mut influxdb_iox_client::management::Client) { + client + .create_database(DatabaseRules { + name: self.database_name().to_string(), + mutable_buffer_config: Some(Default::default()), + ..Default::default() + }) + .await + .unwrap(); + } + + pub async fn load_data(&self, influxdb2: &influxdb2_client::Client) -> Vec { + // TODO: make a more extensible way to manage data for tests, such as in + // external fixture files or with factories. + let points = vec![ + influxdb2_client::DataPoint::builder("cpu_load_short") + .tag("host", "server01") + .tag("region", "us-west") + .field("value", 0.64) + .timestamp(self.ns_since_epoch()) + .build() + .unwrap(), + influxdb2_client::DataPoint::builder("cpu_load_short") + .tag("host", "server01") + .field("value", 27.99) + .timestamp(self.ns_since_epoch() + 1) + .build() + .unwrap(), + influxdb2_client::DataPoint::builder("cpu_load_short") + .tag("host", "server02") + .tag("region", "us-west") + .field("value", 3.89) + .timestamp(self.ns_since_epoch() + 2) + .build() + .unwrap(), + influxdb2_client::DataPoint::builder("cpu_load_short") + .tag("host", "server01") + .tag("region", "us-east") + .field("value", 1234567.891011) + .timestamp(self.ns_since_epoch() + 3) + .build() + .unwrap(), + influxdb2_client::DataPoint::builder("cpu_load_short") + .tag("host", "server01") + .tag("region", "us-west") + .field("value", 0.000003) + .timestamp(self.ns_since_epoch() + 4) + .build() + .unwrap(), + influxdb2_client::DataPoint::builder("system") + .tag("host", "server03") + .field("uptime", 1303385) + .timestamp(self.ns_since_epoch() + 5) + .build() + .unwrap(), + influxdb2_client::DataPoint::builder("swap") + .tag("host", "server01") + .tag("name", "disk0") + .field("in", 3) + .field("out", 4) + .timestamp(self.ns_since_epoch() + 6) + .build() + .unwrap(), + influxdb2_client::DataPoint::builder("status") + .field("active", true) + .timestamp(self.ns_since_epoch() + 7) + .build() + .unwrap(), + influxdb2_client::DataPoint::builder("attributes") + .field("color", "blue") + .timestamp(self.ns_since_epoch() + 8) + .build() + .unwrap(), + ]; + self.write_data(&influxdb2, points).await.unwrap(); + + substitute_nanos( + self.ns_since_epoch(), + &[ + "+----------+---------+---------------------+----------------+", + "| host | region | time | value |", + "+----------+---------+---------------------+----------------+", + "| server01 | us-west | ns0 | 0.64 |", + "| server01 | | ns1 | 27.99 |", + "| server02 | us-west | ns2 | 3.89 |", + "| server01 | us-east | ns3 | 1234567.891011 |", + "| server01 | us-west | ns4 | 0.000003 |", + "+----------+---------+---------------------+----------------+", + ], + ) + } + + async fn write_data( + &self, + client: &influxdb2_client::Client, + points: Vec, + ) -> Result<()> { + client + .write( + self.org_id_str(), + self.bucket_id_str(), + stream::iter(points), + ) + .await?; + Ok(()) + } +} + +/// substitutes "ns" --> ns_since_epoch, ns1-->ns_since_epoch+1, etc +pub fn substitute_nanos(ns_since_epoch: i64, lines: &[&str]) -> Vec { + let substitutions = vec![ + ("ns0", format!("{}", ns_since_epoch)), + ("ns1", format!("{}", ns_since_epoch + 1)), + ("ns2", format!("{}", ns_since_epoch + 2)), + ("ns3", format!("{}", ns_since_epoch + 3)), + ("ns4", format!("{}", ns_since_epoch + 4)), + ("ns5", format!("{}", ns_since_epoch + 5)), + ("ns6", format!("{}", ns_since_epoch + 6)), + ]; + + lines + .iter() + .map(|line| { + let mut line = line.to_string(); + for (from, to) in &substitutions { + line = line.replace(from, to); + } + line + }) + .collect() +} + +/// Return a random string suitable for use as a database name +pub fn rand_name() -> String { + thread_rng() + .sample_iter(&Alphanumeric) + .take(10) + .map(char::from) + .collect() +} + +// return a random 16 digit string comprised of numbers suitable for +// use as a influxdb2 org_id or bucket_id +pub fn rand_id() -> String { + thread_rng() + .sample_iter(&Standard) + .filter_map(|c: u8| { + if c.is_ascii_digit() { + Some(char::from(c)) + } else { + // discard if out of range + None + } + }) + .take(16) + .collect() +} + +/// given a channel to talk with the managment api, create a new +/// database with the specified name configured with a 10MB mutable +/// buffer, partitioned on table +pub async fn create_readable_database( + db_name: impl Into, + channel: tonic::transport::Channel, +) { + let mut management_client = influxdb_iox_client::management::Client::new(channel); + + let rules = DatabaseRules { + name: db_name.into(), + partition_template: Some(PartitionTemplate { + parts: vec![partition_template::Part { + part: Some(partition_template::part::Part::Table(Empty {})), + }], + }), + mutable_buffer_config: Some(MutableBufferConfig { + buffer_size: 10 * 1024 * 1024, + ..Default::default() + }), + ..Default::default() + }; + + management_client + .create_database(rules.clone()) + .await + .expect("create database failed"); +} + +/// given a channel to talk with the managment api, create a new +/// database with no mutable buffer configured, no partitioning rules +pub async fn create_unreadable_database( + db_name: impl Into, + channel: tonic::transport::Channel, +) { + let mut management_client = influxdb_iox_client::management::Client::new(channel); + + let rules = DatabaseRules { + name: db_name.into(), + ..Default::default() + }; + + management_client + .create_database(rules.clone()) + .await + .expect("create database failed"); +} + +/// given a channel to talk with the managment api, create a new +/// database with the specified name configured with a 10MB mutable +/// buffer, partitioned on table, with some data written into two partitions +pub async fn create_two_partition_database( + db_name: impl Into, + channel: tonic::transport::Channel, +) { + let mut write_client = influxdb_iox_client::write::Client::new(channel.clone()); + + let db_name = db_name.into(); + create_readable_database(&db_name, channel).await; + + let lp_lines = vec![ + "mem,host=foo free=27875999744i,cached=0i,available_percent=62.2 1591894320000000000", + "cpu,host=foo running=4i,sleeping=514i,total=519i 1592894310000000000", + ]; + + write_client + .write(&db_name, lp_lines.join("\n")) + .await + .expect("write succeded"); +} diff --git a/tests/end_to_end_cases/storage_api.rs b/tests/end_to_end_cases/storage_api.rs index dbd74f6441..cd516e73de 100644 --- a/tests/end_to_end_cases/storage_api.rs +++ b/tests/end_to_end_cases/storage_api.rs @@ -1,8 +1,11 @@ -use crate::{create_database, substitute_nanos, Scenario}; +use super::scenario::{substitute_nanos, Scenario}; +use crate::common::server_fixture::ServerFixture; + use futures::prelude::*; use generated_types::{ aggregate::AggregateType, google::protobuf::{Any, Empty}, + measurement_fields_response::FieldType, node::{Comparison, Type as NodeType, Value}, read_group_request::Group, read_response::{frame::Data, *}, @@ -11,20 +14,30 @@ use generated_types::{ MeasurementTagValuesRequest, Node, Predicate, ReadFilterRequest, ReadGroupRequest, ReadWindowAggregateRequest, Tag, TagKeysRequest, TagValuesRequest, TimestampRange, }; -use influxdb_iox_client::management; use std::str; use test_helpers::tag_key_bytes_to_strings; use tonic::transport::Channel; -pub async fn test(storage_client: &mut StorageClient, scenario: &Scenario) { - capabilities_endpoint(storage_client).await; - read_filter_endpoint(storage_client, scenario).await; - tag_keys_endpoint(storage_client, scenario).await; - tag_values_endpoint(storage_client, scenario).await; - measurement_names_endpoint(storage_client, scenario).await; - measurement_tag_keys_endpoint(storage_client, scenario).await; - measurement_tag_values_endpoint(storage_client, scenario).await; - measurement_fields_endpoint(storage_client, scenario).await; +#[tokio::test] +pub async fn test() { + let storage_fixture = ServerFixture::create_shared().await; + + let influxdb2 = storage_fixture.influxdb2_client(); + let mut storage_client = StorageClient::new(storage_fixture.grpc_channel()); + let mut management_client = storage_fixture.management_client(); + + let scenario = Scenario::new(); + scenario.create_database(&mut management_client).await; + scenario.load_data(&influxdb2).await; + + capabilities_endpoint(&mut storage_client).await; + read_filter_endpoint(&mut storage_client, &scenario).await; + tag_keys_endpoint(&mut storage_client, &scenario).await; + tag_values_endpoint(&mut storage_client, &scenario).await; + measurement_names_endpoint(&mut storage_client, &scenario).await; + measurement_tag_keys_endpoint(&mut storage_client, &scenario).await; + measurement_tag_values_endpoint(&mut storage_client, &scenario).await; + measurement_fields_endpoint(&mut storage_client, &scenario).await; } /// Validate that capabilities storage endpoint is hooked up @@ -277,29 +290,28 @@ async fn measurement_fields_endpoint( let field = &fields[0]; assert_eq!(field.key, "value"); - assert_eq!(field.r#type, DataType::Float as i32); + assert_eq!(field.r#type(), FieldType::Float); assert_eq!(field.timestamp, scenario.ns_since_epoch() + 4); } -pub async fn read_group_test( - management: &mut management::Client, - influxdb2: &influxdb2_client::Client, - storage_client: &mut StorageClient, -) { - let scenario = Scenario::default() - .set_org_id("0000111100001110") - .set_bucket_id("1111000011110001"); +#[tokio::test] +pub async fn read_group_test() { + let fixture = ServerFixture::create_shared().await; + let mut management = fixture.management_client(); + let mut storage_client = StorageClient::new(fixture.grpc_channel()); + let influxdb2 = fixture.influxdb2_client(); - create_database(management, &scenario.database_name()).await; + let scenario = Scenario::new(); + scenario.create_database(&mut management).await; load_read_group_data(&influxdb2, &scenario).await; let read_source = scenario.read_source(); - test_read_group_none_agg(storage_client, &read_source).await; - test_read_group_none_agg_with_predicate(storage_client, &read_source).await; - test_read_group_sum_agg(storage_client, &read_source).await; - test_read_group_last_agg(storage_client, &read_source).await; + test_read_group_none_agg(&mut storage_client, &read_source).await; + test_read_group_none_agg_with_predicate(&mut storage_client, &read_source).await; + test_read_group_sum_agg(&mut storage_client, &read_source).await; + test_read_group_last_agg(&mut storage_client, &read_source).await; } async fn load_read_group_data(client: &influxdb2_client::Client, scenario: &Scenario) { @@ -534,17 +546,17 @@ async fn test_read_group_last_agg( } // Standalone test that all the pipes are hooked up for read window aggregate -pub async fn read_window_aggregate_test( - management: &mut management::Client, - influxdb2: &influxdb2_client::Client, - storage_client: &mut StorageClient, -) { - let scenario = Scenario::default() - .set_org_id("0000111100001100") - .set_bucket_id("1111000011110011"); +#[tokio::test] +pub async fn read_window_aggregate_test() { + let fixture = ServerFixture::create_shared().await; + let mut management = fixture.management_client(); + let mut storage_client = StorageClient::new(fixture.grpc_channel()); + let influxdb2 = fixture.influxdb2_client(); + + let scenario = Scenario::new(); let read_source = scenario.read_source(); - create_database(management, &scenario.database_name()).await; + scenario.create_database(&mut management).await; let line_protocol = vec![ "h2o,state=MA,city=Boston temp=70.0 100", diff --git a/tests/end_to_end_cases/write_api.rs b/tests/end_to_end_cases/write_api.rs new file mode 100644 index 0000000000..3ab14ef312 --- /dev/null +++ b/tests/end_to_end_cases/write_api.rs @@ -0,0 +1,52 @@ +use influxdb_iox_client::write::WriteError; +use test_helpers::assert_contains; + +use crate::common::server_fixture::ServerFixture; + +use super::scenario::{create_readable_database, rand_name}; + +#[tokio::test] +async fn test_write() { + let fixture = ServerFixture::create_shared().await; + let mut write_client = fixture.write_client(); + + let db_name = rand_name(); + create_readable_database(&db_name, fixture.grpc_channel()).await; + + let lp_lines = vec![ + "cpu,region=west user=23.2 100", + "cpu,region=west user=21.0 150", + "disk,region=east bytes=99i 200", + ]; + + let num_lines_written = write_client + .write(&db_name, lp_lines.join("\n")) + .await + .expect("write succeded"); + + assert_eq!(num_lines_written, 3); + + // ---- test bad data ---- + let err = write_client + .write(&db_name, "XXX") + .await + .expect_err("expected write to fail"); + + assert_contains!( + err.to_string(), + r#"Client specified an invalid argument: Violation for field "lp_data": Invalid Line Protocol: A generic parsing error occurred"# + ); + assert!(matches!(dbg!(err), WriteError::ServerError(_))); + + // ---- test non existent database ---- + let err = write_client + .write("Non_existent_database", lp_lines.join("\n")) + .await + .expect_err("expected write to fail"); + + assert_contains!( + err.to_string(), + r#"Unexpected server error: Some requested entity was not found: Resource database/Non_existent_database not found"# + ); + assert!(matches!(dbg!(err), WriteError::ServerError(_))); +} diff --git a/tests/end_to_end_cases/write_cli.rs b/tests/end_to_end_cases/write_cli.rs new file mode 100644 index 0000000000..e4adf7e04b --- /dev/null +++ b/tests/end_to_end_cases/write_cli.rs @@ -0,0 +1,67 @@ +use assert_cmd::Command; +use predicates::prelude::*; +use test_helpers::make_temp_file; + +use crate::common::server_fixture::ServerFixture; + +use super::scenario::rand_name; + +#[tokio::test] +async fn test() { + let server_fixture = ServerFixture::create_shared().await; + let db_name = rand_name(); + let addr = server_fixture.grpc_base(); + create_database(&db_name, addr).await; + test_write(&db_name, addr).await; +} + +async fn create_database(db_name: &str, addr: &str) { + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("create") + .arg(db_name) + .arg("--host") + .arg(addr) + .assert() + .success() + .stdout(predicate::str::contains("Ok")); +} + +async fn test_write(db_name: &str, addr: &str) { + let lp_data = vec![ + "cpu,region=west user=23.2 100", + "cpu,region=west user=21.0 150", + ]; + + let lp_data_file = make_temp_file(lp_data.join("\n")); + + // read from temp file + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("write") + .arg(db_name) + .arg(lp_data_file.as_ref()) + .arg("--host") + .arg(addr) + .assert() + .success() + .stdout(predicate::str::contains("2 Lines OK")); + + // try reading a non existent file + Command::cargo_bin("influxdb_iox") + .unwrap() + .arg("database") + .arg("write") + .arg(db_name) + .arg("this_file_does_not_exist") + .arg("--host") + .arg(addr) + .assert() + .failure() + .stderr( + predicate::str::contains(r#"Error reading file "this_file_does_not_exist":"#) + .and(predicate::str::contains("No such file or directory")), + ); +}