diff --git a/Cargo.lock b/Cargo.lock index 42f01ab82c..18f2148c99 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -99,9 +99,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.42" +version = "1.0.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "595d3cfa7a60d4555cb5067b99f07142a08ea778de5cf993f7b75c7d8fabc486" +checksum = "28ae2b3dec75a406790005a200b1bd89785afc02517a00ca99ecfe093ee9e6cf" [[package]] name = "arrayref" @@ -126,9 +126,9 @@ checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" [[package]] name = "arrow" -version = "5.1.0" +version = "5.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddf189dff0c7e0f40588fc25adbe5bb6837b82fc61bb7cadf5d76de030f710bb" +checksum = "31ef7f4383e52f2da72da147040148a385b7cbf9ec0fc45cd3e2f369a9d442fe" dependencies = [ "bitflags", "chrono", @@ -150,9 +150,9 @@ dependencies = [ [[package]] name = "arrow-flight" -version = "5.1.0" +version = "5.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "947ab99b9dc270706b1a4bd8109628416dc52348f1645075fcc19fe6b9819055" +checksum = "254bb81cc3809fc2b796bab1d0ebac6b6201b214076e728143382c07815285c3" dependencies = [ "arrow", "base64 0.13.0", @@ -161,8 +161,8 @@ dependencies = [ "prost 0.8.0", "prost-derive 0.8.0", "tokio", - "tonic 0.5.0", - "tonic-build 0.5.1", + "tonic 0.5.2", + "tonic-build 0.5.2", ] [[package]] @@ -197,7 +197,7 @@ checksum = "c98233c6673d8601ab23e77eb38f999c51100d46c5703b17288c57fddf3a1ffe" dependencies = [ "bstr", "doc-comment", - "predicates 2.0.1", + "predicates 2.0.2", "predicates-core", "predicates-tree", "wait-timeout", @@ -270,7 +270,7 @@ dependencies = [ "log", "md5", "oauth2", - "paste 1.0.5", + "paste", "quick-error", "reqwest", "serde", @@ -868,7 +868,7 @@ dependencies = [ "num_cpus", "ordered-float 2.7.0", "parquet", - "paste 1.0.5", + "paste", "pin-project-lite", "rand 0.8.4", "smallvec", @@ -1303,8 +1303,8 @@ dependencies = [ "serde", "serde_json", "thiserror", - "tonic 0.5.0", - "tonic-build 0.5.1", + "tonic 0.5.2", + "tonic-build 0.5.2", ] [[package]] @@ -1371,7 +1371,7 @@ dependencies = [ "futures", "grpc-router-test-gen", "observability_deps", - "paste 1.0.5", + "paste", "prost 0.8.0", "prost-build 0.8.0", "prost-types 0.8.0", @@ -1379,8 +1379,8 @@ dependencies = [ "tokio", "tokio-stream", "tokio-util", - "tonic 0.5.0", - "tonic-build 0.5.1", + "tonic 0.5.2", + "tonic-build 0.5.2", "tonic-reflection", ] @@ -1391,8 +1391,8 @@ dependencies = [ "prost 0.8.0", "prost-build 0.8.0", "prost-types 0.8.0", - "tonic 0.5.0", - "tonic-build 0.5.1", + "tonic 0.5.2", + "tonic-build 0.5.2", ] [[package]] @@ -1491,9 +1491,9 @@ dependencies = [ [[package]] name = "http-body" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60daa14be0e0786db0f03a9e57cb404c9d756eed2b6c62b9ea98ec5743ec75a9" +checksum = "399c583b2979440c60be0821a6199eca73bc3c8dcd9d070d75ac726e2c6186e5" dependencies = [ "bytes", "http", @@ -1715,7 +1715,7 @@ dependencies = [ "tokio", "tokio-stream", "tokio-util", - "tonic 0.5.0", + "tonic 0.5.2", "tonic-health", "tonic-reflection", "trace", @@ -1742,7 +1742,7 @@ dependencies = [ "serde_json", "thiserror", "tokio", - "tonic 0.5.0", + "tonic 0.5.2", "tower", ] @@ -1902,9 +1902,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.98" +version = "0.2.99" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "320cfe77175da3a483efed4bc0adc1968ca050b098ce4f2f1c13a56626128790" +checksum = "a7f823d141fe0a24df1e23b4af4e3c7ba9e5966ec514ea068c93024aa7deb765" [[package]] name = "libloading" @@ -2007,9 +2007,9 @@ dependencies = [ [[package]] name = "matches" -version = "0.1.8" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" +checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" [[package]] name = "md5" @@ -2177,9 +2177,9 @@ dependencies = [ [[package]] name = "native-tls" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8d96b2e1c8da3957d58100b09f102c6d9cfdfced01b7ec5a8974044bb09dbd4" +checksum = "48ba9f7719b5a0f42f338907614285fb5fd70e53858141f69898a1fb7203b24d" dependencies = [ "lazy_static", "libc", @@ -2204,14 +2204,15 @@ dependencies = [ [[package]] name = "nix" -version = "0.20.0" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa9b4819da1bc61c0ea48b63b7bc8604064dd43013e7cc325df098d49cd7c18a" +checksum = "df8e5e343312e7fbeb2a52139114e9e702991ef9c2aea6817ff2440b35647d56" dependencies = [ "bitflags", "cc", "cfg-if", "libc", + "memoffset", ] [[package]] @@ -2640,16 +2641,16 @@ dependencies = [ "cfg-if", "instant", "libc", - "redox_syscall 0.2.9", + "redox_syscall 0.2.10", "smallvec", "winapi", ] [[package]] name = "parquet" -version = "5.1.0" +version = "5.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db54d10313f64ea22e1fe0332864abefb19aa1c322bfbc67999ff743aebf868d" +checksum = "e29eb4b97c8b1d573437e71a9b6f4e8c7c8fab7da62fed3acc6361e3f1fa574c" dependencies = [ "arrow", "base64 0.13.0", @@ -2710,31 +2711,12 @@ dependencies = [ "uuid", ] -[[package]] -name = "paste" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45ca20c77d80be666aef2b45486da86238fabe33e38306bd3118fe4af33fa880" -dependencies = [ - "paste-impl", - "proc-macro-hack", -] - [[package]] name = "paste" version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf547ad0c65e31259204bd90935776d1c693cec2f4ff7abb7a1bbbd40dfe58" -[[package]] -name = "paste-impl" -version = "0.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d95a7db200b97ef370c8e6de0088252f7e0dfff7d047a28528e47456c0fc98b6" -dependencies = [ - "proc-macro-hack", -] - [[package]] name = "peeking_take_while" version = "0.1.2" @@ -2915,9 +2897,9 @@ dependencies = [ [[package]] name = "predicates" -version = "2.0.1" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc3d91237f5de3bcd9d927e24d03b495adb6135097b001cea7403e2d573d00a9" +checksum = "c143348f141cc87aab5b950021bac6145d0e5ae754b0591de23244cee42c9308" dependencies = [ "difflib", "itertools 0.10.1", @@ -2932,9 +2914,9 @@ checksum = "57e35a3326b75e49aa85f5dc6ec15b41108cf5aee58eabb1f274dd18b73c2451" [[package]] name = "predicates-tree" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15f553275e5721409451eb85e15fd9a860a6e5ab4496eb215987502b5f5391f2" +checksum = "d7dd0fd014130206c9352efbdc92be592751b2b9274dff685348341082c6ea3d" dependencies = [ "predicates-core", "treeline", @@ -3128,9 +3110,9 @@ dependencies = [ [[package]] name = "protobuf" -version = "2.24.1" +version = "2.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db50e77ae196458ccd3dc58a31ea1a90b0698ab1b7928d89f644c25d72070267" +checksum = "020f86b07722c5c4291f7c723eac4676b3892d47d9a7708dc2779696407f039b" [[package]] name = "query" @@ -3400,9 +3382,9 @@ checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" [[package]] name = "redox_syscall" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ab49abadf3f9e1c4bc499e8845e152ad87d2ad2d30371841171169e9d75feee" +checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff" dependencies = [ "bitflags", ] @@ -3425,7 +3407,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64" dependencies = [ "getrandom 0.2.3", - "redox_syscall 0.2.9", + "redox_syscall 0.2.10", ] [[package]] @@ -3819,9 +3801,9 @@ dependencies = [ [[package]] name = "serde_cbor" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e18acfa2f90e8b735b2836ab8d538de304cbb6729a7360729ea5a895d15a622" +checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5" dependencies = [ "half", "serde", @@ -4010,9 +3992,9 @@ dependencies = [ [[package]] name = "slab" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f173ac3d1a7e3b28003f40de0b5ce7fe2710f9b9dc3fc38664cebee46b3b6527" +checksum = "c307a32c1c5c437f38c7fd45d753050587732ba8628319fbdf12a7e289ccc590" [[package]] name = "smallvec" @@ -4271,7 +4253,7 @@ dependencies = [ "cfg-if", "libc", "rand 0.8.4", - "redox_syscall 0.2.9", + "redox_syscall 0.2.10", "remove_dir_all", "winapi", ] @@ -4368,20 +4350,20 @@ dependencies = [ [[package]] name = "tikv-jemalloc-ctl" -version = "0.4.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f28c80e4338857639f443169a601fafe49866aed8d7a8d565c2f5bfb1a021adf" +checksum = "eb833c46ecbf8b6daeccb347cefcabf9c1beb5c9b0f853e1cec45632d9963e69" dependencies = [ "libc", - "paste 0.1.18", + "paste", "tikv-jemalloc-sys", ] [[package]] name = "tikv-jemalloc-sys" -version = "0.4.1+5.2.1-patched" +version = "0.4.2+5.2.1-patched.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a26331b05179d4cb505c8d6814a7e18d298972f0a551b0e3cefccff927f86d3" +checksum = "5844e429d797c62945a566f8da4e24c7fe3fbd5d6617fd8bf7a0b7dc1ee0f22e" dependencies = [ "cc", "fs_extra", @@ -4473,9 +4455,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b7b349f11a7047e6d1276853e612d152f5e8a352c61917887cc2169e2366b4c" +checksum = "01cf844b23c6131f624accf65ce0e4e9956a8bb329400ea5bcc26ae3a5c20b0b" dependencies = [ "autocfg", "bytes", @@ -4598,9 +4580,9 @@ dependencies = [ [[package]] name = "tonic" -version = "0.5.0" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b584f064fdfc50017ec39162d5aebce49912f1eb16fd128e04b7f4ce4907c7e5" +checksum = "796c5e1cd49905e65dd8e700d4cb1dffcbfdb4fc9d017de08c1a537afd83627c" dependencies = [ "async-stream", "async-trait", @@ -4641,9 +4623,9 @@ dependencies = [ [[package]] name = "tonic-build" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d12faebbe071b06f486be82cc9318350814fdd07fcb28f3690840cd770599283" +checksum = "12b52d07035516c2b74337d2ac7746075e7dcae7643816c1b12c5ff8a7484c08" dependencies = [ "proc-macro2", "prost-build 0.8.0", @@ -4653,17 +4635,17 @@ dependencies = [ [[package]] name = "tonic-health" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14e6de0a7a1b27d9899031b01b83eb09fdc36f3fe8e6254a81840006a463c6d5" +checksum = "493fcae35818dffa28437b210a615119d791116c1cac80716f571f35dd55b1b9" dependencies = [ "async-stream", "bytes", "prost 0.8.0", "tokio", "tokio-stream", - "tonic 0.5.0", - "tonic-build 0.5.1", + "tonic 0.5.2", + "tonic-build 0.5.2", ] [[package]] @@ -4677,8 +4659,8 @@ dependencies = [ "prost-types 0.8.0", "tokio", "tokio-stream", - "tonic 0.5.0", - "tonic-build 0.5.1", + "tonic 0.5.2", + "tonic-build 0.5.2", ] [[package]] @@ -4869,12 +4851,9 @@ checksum = "879f6906492a7cd215bfa4cf595b600146ccfac0c79bcbd1f3000162af5e8b06" [[package]] name = "unicode-bidi" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eeb8be209bb1c96b7c177c7420d26e04eccacb0eeae6b980e35fcb74678107e0" -dependencies = [ - "matches", -] +checksum = "246f4c42e67e7a4e3c6106ff716a5d067d4132a642840b242e357e468a2a0085" [[package]] name = "unicode-normalization" diff --git a/server/src/db.rs b/server/src/db.rs index f88da1f5e0..f7a8e633d8 100644 --- a/server/src/db.rs +++ b/server/src/db.rs @@ -4218,6 +4218,39 @@ mod tests { ); } + #[tokio::test] + async fn drop_unpersisted_chunk_on_persisted_db() { + // We don't support dropping unpersisted chunks from a persisted DB because we would forget the write buffer + // progress (partition checkpoints are only created when new parquet files are stored). + // See https://github.com/influxdata/influxdb_iox/issues/2291 + let test_db = TestDb::builder() + .lifecycle_rules(LifecycleRules { + persist: true, + ..Default::default() + }) + .build() + .await; + let db = Arc::new(test_db.db); + + write_lp(db.as_ref(), "cpu bar=1 10").await; + + let partition_key = "1970-01-01T00"; + let chunks = db.partition_chunk_summaries(partition_key); + assert_eq!(chunks.len(), 1); + let chunk_id = chunks[0].id; + + let err = db + .drop_chunk("cpu", partition_key, chunk_id) + .await + .unwrap_err(); + assert!(matches!( + err, + Error::LifecycleError { + source: super::lifecycle::Error::CannotDropUnpersistedChunk { .. } + } + )); + } + async fn create_parquet_chunk(db: &Arc) -> (String, String, u32) { write_lp(db, "cpu bar=1 10").await; let partition_key = "1970-01-01T00"; diff --git a/server/src/db/lifecycle/drop.rs b/server/src/db/lifecycle/drop.rs index ad737ec911..d2710cb34b 100644 --- a/server/src/db/lifecycle/drop.rs +++ b/server/src/db/lifecycle/drop.rs @@ -9,7 +9,7 @@ use snafu::ResultExt; use tracker::{TaskTracker, TrackedFuture, TrackedFutureExt}; use super::{ - error::{CommitError, Result}, + error::{CannotDropUnpersistedChunk, CommitError, Result}, LockableCatalogChunk, LockableCatalogPartition, }; use crate::db::catalog::{ @@ -34,6 +34,16 @@ pub fn drop_chunk( chunk: guard.addr().clone(), }); + // check if we're dropping an unpersisted chunk in a persisted DB + // See https://github.com/influxdata/influxdb_iox/issues/2291 + if db.rules().lifecycle_rules.persist && !matches!(guard.stage(), ChunkStage::Persisted { .. }) + { + return CannotDropUnpersistedChunk { + addr: guard.addr().clone(), + } + .fail(); + } + guard.set_dropping(®istration)?; // Drop locks diff --git a/server/src/db/lifecycle/error.rs b/server/src/db/lifecycle/error.rs index 6fb416f2ac..24e8227789 100644 --- a/server/src/db/lifecycle/error.rs +++ b/server/src/db/lifecycle/error.rs @@ -59,6 +59,9 @@ pub enum Error { #[snafu(display("Cannot write chunk: {}", addr))] CannotWriteChunk { addr: ChunkAddr }, + + #[snafu(display("Cannot drop unpersisted chunk: {}", addr))] + CannotDropUnpersistedChunk { addr: ChunkAddr }, } pub type Result = std::result::Result; diff --git a/trace/src/tower.rs b/trace/src/tower.rs index 313f173735..bc9e2ac84b 100644 --- a/trace/src/tower.rs +++ b/trace/src/tower.rs @@ -16,7 +16,7 @@ use crate::{ctx::SpanContext, span::EnteredSpan, TraceCollector}; use futures::ready; use http::{Request, Response}; use http_body::SizeHint; -use observability_deps::tracing::error; +use observability_deps::tracing::{error, info}; use pin_project::pin_project; use std::future::Future; use std::pin::Pin; @@ -69,6 +69,9 @@ where } fn call(&mut self, mut request: Request) -> Self::Future { + use itertools::Itertools; + // TEMPORARY + info!(headers=%request.headers().keys().join(","), "gRPC request with headers"); let span = match SpanContext::from_headers(&self.collector, request.headers()) { Ok(Some(ctx)) => { let span = ctx.child("IOx");