Influxdata
/
influxdb
mirror of https://github.com/influxdata/influxdb.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge branch 'main' into crepererum/simplify_skip_replay_scenario

pull/24376/head
kodiakhq[bot] 2021-08-16 13:53:43 +00:00 committed by GitHub
parent 6b907f94da 0b1c48dae3
commit c911b8ec5d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 118 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

155
Cargo.lock generated
View File

@ -99,9 +99,9 @@ dependencies = [
[[package]]
name = "anyhow"
version = "1.0.42"
version = "1.0.43"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "595d3cfa7a60d4555cb5067b99f07142a08ea778de5cf993f7b75c7d8fabc486"
checksum = "28ae2b3dec75a406790005a200b1bd89785afc02517a00ca99ecfe093ee9e6cf"
[[package]]
name = "arrayref"
@ -126,9 +126,9 @@ checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b"
[[package]]
name = "arrow"
version = "5.1.0"
version = "5.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ddf189dff0c7e0f40588fc25adbe5bb6837b82fc61bb7cadf5d76de030f710bb"
checksum = "31ef7f4383e52f2da72da147040148a385b7cbf9ec0fc45cd3e2f369a9d442fe"
dependencies = [
"bitflags",
"chrono",
@ -150,9 +150,9 @@ dependencies = [
[[package]]
name = "arrow-flight"
version = "5.1.0"
version = "5.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "947ab99b9dc270706b1a4bd8109628416dc52348f1645075fcc19fe6b9819055"
checksum = "254bb81cc3809fc2b796bab1d0ebac6b6201b214076e728143382c07815285c3"
dependencies = [
"arrow",
"base64 0.13.0",
@ -161,8 +161,8 @@ dependencies = [
"prost 0.8.0",
"prost-derive 0.8.0",
"tokio",
"tonic 0.5.0",
"tonic-build 0.5.1",
"tonic 0.5.2",
"tonic-build 0.5.2",
]
[[package]]
@ -197,7 +197,7 @@ checksum = "c98233c6673d8601ab23e77eb38f999c51100d46c5703b17288c57fddf3a1ffe"
dependencies = [
"bstr",
"doc-comment",
"predicates 2.0.1",
"predicates 2.0.2",
"predicates-core",
"predicates-tree",
"wait-timeout",
@ -270,7 +270,7 @@ dependencies = [
"log",
"md5",
"oauth2",
"paste 1.0.5",
"paste",
"quick-error",
"reqwest",
"serde",
@ -868,7 +868,7 @@ dependencies = [
"num_cpus",
"ordered-float 2.7.0",
"parquet",
"paste 1.0.5",
"paste",
"pin-project-lite",
"rand 0.8.4",
"smallvec",
@ -1303,8 +1303,8 @@ dependencies = [
"serde",
"serde_json",
"thiserror",
"tonic 0.5.0",
"tonic-build 0.5.1",
"tonic 0.5.2",
"tonic-build 0.5.2",
]
[[package]]
@ -1371,7 +1371,7 @@ dependencies = [
"futures",
"grpc-router-test-gen",
"observability_deps",
"paste 1.0.5",
"paste",
"prost 0.8.0",
"prost-build 0.8.0",
"prost-types 0.8.0",
@ -1379,8 +1379,8 @@ dependencies = [
"tokio",
"tokio-stream",
"tokio-util",
"tonic 0.5.0",
"tonic-build 0.5.1",
"tonic 0.5.2",
"tonic-build 0.5.2",
"tonic-reflection",
]
@ -1391,8 +1391,8 @@ dependencies = [
"prost 0.8.0",
"prost-build 0.8.0",
"prost-types 0.8.0",
"tonic 0.5.0",
"tonic-build 0.5.1",
"tonic 0.5.2",
"tonic-build 0.5.2",
]
[[package]]
@ -1491,9 +1491,9 @@ dependencies = [
[[package]]
name = "http-body"
version = "0.4.2"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60daa14be0e0786db0f03a9e57cb404c9d756eed2b6c62b9ea98ec5743ec75a9"
checksum = "399c583b2979440c60be0821a6199eca73bc3c8dcd9d070d75ac726e2c6186e5"
dependencies = [
"bytes",
"http",
@ -1715,7 +1715,7 @@ dependencies = [
"tokio",
"tokio-stream",
"tokio-util",
"tonic 0.5.0",
"tonic 0.5.2",
"tonic-health",
"tonic-reflection",
"trace",
@ -1742,7 +1742,7 @@ dependencies = [
"serde_json",
"thiserror",
"tokio",
"tonic 0.5.0",
"tonic 0.5.2",
"tower",
]
@ -1902,9 +1902,9 @@ dependencies = [
[[package]]
name = "libc"
version = "0.2.98"
version = "0.2.99"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "320cfe77175da3a483efed4bc0adc1968ca050b098ce4f2f1c13a56626128790"
checksum = "a7f823d141fe0a24df1e23b4af4e3c7ba9e5966ec514ea068c93024aa7deb765"
[[package]]
name = "libloading"
@ -2007,9 +2007,9 @@ dependencies = [
[[package]]
name = "matches"
version = "0.1.8"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08"
checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f"
[[package]]
name = "md5"
@ -2177,9 +2177,9 @@ dependencies = [
[[package]]
name = "native-tls"
version = "0.2.7"
version = "0.2.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8d96b2e1c8da3957d58100b09f102c6d9cfdfced01b7ec5a8974044bb09dbd4"
checksum = "48ba9f7719b5a0f42f338907614285fb5fd70e53858141f69898a1fb7203b24d"
dependencies = [
"lazy_static",
"libc",
@ -2204,14 +2204,15 @@ dependencies = [
[[package]]
name = "nix"
version = "0.20.0"
version = "0.20.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fa9b4819da1bc61c0ea48b63b7bc8604064dd43013e7cc325df098d49cd7c18a"
checksum = "df8e5e343312e7fbeb2a52139114e9e702991ef9c2aea6817ff2440b35647d56"
dependencies = [
"bitflags",
"cc",
"cfg-if",
"libc",
"memoffset",
]
[[package]]
@ -2640,16 +2641,16 @@ dependencies = [
"cfg-if",
"instant",
"libc",
"redox_syscall 0.2.9",
"redox_syscall 0.2.10",
"smallvec",
"winapi",
]
[[package]]
name = "parquet"
version = "5.1.0"
version = "5.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db54d10313f64ea22e1fe0332864abefb19aa1c322bfbc67999ff743aebf868d"
checksum = "e29eb4b97c8b1d573437e71a9b6f4e8c7c8fab7da62fed3acc6361e3f1fa574c"
dependencies = [
"arrow",
"base64 0.13.0",
@ -2710,31 +2711,12 @@ dependencies = [
"uuid",
]
[[package]]
name = "paste"
version = "0.1.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "45ca20c77d80be666aef2b45486da86238fabe33e38306bd3118fe4af33fa880"
dependencies = [
"paste-impl",
"proc-macro-hack",
]
[[package]]
name = "paste"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "acbf547ad0c65e31259204bd90935776d1c693cec2f4ff7abb7a1bbbd40dfe58"
[[package]]
name = "paste-impl"
version = "0.1.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d95a7db200b97ef370c8e6de0088252f7e0dfff7d047a28528e47456c0fc98b6"
dependencies = [
"proc-macro-hack",
]
[[package]]
name = "peeking_take_while"
version = "0.1.2"
@ -2915,9 +2897,9 @@ dependencies = [
[[package]]
name = "predicates"
version = "2.0.1"
version = "2.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc3d91237f5de3bcd9d927e24d03b495adb6135097b001cea7403e2d573d00a9"
checksum = "c143348f141cc87aab5b950021bac6145d0e5ae754b0591de23244cee42c9308"
dependencies = [
"difflib",
"itertools 0.10.1",
@ -2932,9 +2914,9 @@ checksum = "57e35a3326b75e49aa85f5dc6ec15b41108cf5aee58eabb1f274dd18b73c2451"
[[package]]
name = "predicates-tree"
version = "1.0.2"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "15f553275e5721409451eb85e15fd9a860a6e5ab4496eb215987502b5f5391f2"
checksum = "d7dd0fd014130206c9352efbdc92be592751b2b9274dff685348341082c6ea3d"
dependencies = [
"predicates-core",
"treeline",
@ -3128,9 +3110,9 @@ dependencies = [
[[package]]
name = "protobuf"
version = "2.24.1"
version = "2.25.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db50e77ae196458ccd3dc58a31ea1a90b0698ab1b7928d89f644c25d72070267"
checksum = "020f86b07722c5c4291f7c723eac4676b3892d47d9a7708dc2779696407f039b"
[[package]]
name = "query"
@ -3400,9 +3382,9 @@ checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce"
[[package]]
name = "redox_syscall"
version = "0.2.9"
version = "0.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5ab49abadf3f9e1c4bc499e8845e152ad87d2ad2d30371841171169e9d75feee"
checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff"
dependencies = [
"bitflags",
]
@ -3425,7 +3407,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64"
dependencies = [
"getrandom 0.2.3",
"redox_syscall 0.2.9",
"redox_syscall 0.2.10",
]
[[package]]
@ -3819,9 +3801,9 @@ dependencies = [
[[package]]
name = "serde_cbor"
version = "0.11.1"
version = "0.11.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e18acfa2f90e8b735b2836ab8d538de304cbb6729a7360729ea5a895d15a622"
checksum = "2bef2ebfde456fb76bbcf9f59315333decc4fda0b2b44b420243c11e0f5ec1f5"
dependencies = [
"half",
"serde",
@ -4010,9 +3992,9 @@ dependencies = [
[[package]]
name = "slab"
version = "0.4.3"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f173ac3d1a7e3b28003f40de0b5ce7fe2710f9b9dc3fc38664cebee46b3b6527"
checksum = "c307a32c1c5c437f38c7fd45d753050587732ba8628319fbdf12a7e289ccc590"
[[package]]
name = "smallvec"
@ -4271,7 +4253,7 @@ dependencies = [
"cfg-if",
"libc",
"rand 0.8.4",
"redox_syscall 0.2.9",
"redox_syscall 0.2.10",
"remove_dir_all",
"winapi",
]
@ -4368,20 +4350,20 @@ dependencies = [
[[package]]
name = "tikv-jemalloc-ctl"
version = "0.4.1"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f28c80e4338857639f443169a601fafe49866aed8d7a8d565c2f5bfb1a021adf"
checksum = "eb833c46ecbf8b6daeccb347cefcabf9c1beb5c9b0f853e1cec45632d9963e69"
dependencies = [
"libc",
"paste 0.1.18",
"paste",
"tikv-jemalloc-sys",
]
[[package]]
name = "tikv-jemalloc-sys"
version = "0.4.1+5.2.1-patched"
version = "0.4.2+5.2.1-patched.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8a26331b05179d4cb505c8d6814a7e18d298972f0a551b0e3cefccff927f86d3"
checksum = "5844e429d797c62945a566f8da4e24c7fe3fbd5d6617fd8bf7a0b7dc1ee0f22e"
dependencies = [
"cc",
"fs_extra",
@ -4473,9 +4455,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"
[[package]]
name = "tokio"
version = "1.9.0"
version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b7b349f11a7047e6d1276853e612d152f5e8a352c61917887cc2169e2366b4c"
checksum = "01cf844b23c6131f624accf65ce0e4e9956a8bb329400ea5bcc26ae3a5c20b0b"
dependencies = [
"autocfg",
"bytes",
@ -4598,9 +4580,9 @@ dependencies = [
[[package]]
name = "tonic"
version = "0.5.0"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b584f064fdfc50017ec39162d5aebce49912f1eb16fd128e04b7f4ce4907c7e5"
checksum = "796c5e1cd49905e65dd8e700d4cb1dffcbfdb4fc9d017de08c1a537afd83627c"
dependencies = [
"async-stream",
"async-trait",
@ -4641,9 +4623,9 @@ dependencies = [
[[package]]
name = "tonic-build"
version = "0.5.1"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d12faebbe071b06f486be82cc9318350814fdd07fcb28f3690840cd770599283"
checksum = "12b52d07035516c2b74337d2ac7746075e7dcae7643816c1b12c5ff8a7484c08"
dependencies = [
"proc-macro2",
"prost-build 0.8.0",
@ -4653,17 +4635,17 @@ dependencies = [
[[package]]
name = "tonic-health"
version = "0.4.0"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "14e6de0a7a1b27d9899031b01b83eb09fdc36f3fe8e6254a81840006a463c6d5"
checksum = "493fcae35818dffa28437b210a615119d791116c1cac80716f571f35dd55b1b9"
dependencies = [
"async-stream",
"bytes",
"prost 0.8.0",
"tokio",
"tokio-stream",
"tonic 0.5.0",
"tonic-build 0.5.1",
"tonic 0.5.2",
"tonic-build 0.5.2",
]
[[package]]
@ -4677,8 +4659,8 @@ dependencies = [
"prost-types 0.8.0",
"tokio",
"tokio-stream",
"tonic 0.5.0",
"tonic-build 0.5.1",
"tonic 0.5.2",
"tonic-build 0.5.2",
]
[[package]]
@ -4869,12 +4851,9 @@ checksum = "879f6906492a7cd215bfa4cf595b600146ccfac0c79bcbd1f3000162af5e8b06"
[[package]]
name = "unicode-bidi"
version = "0.3.5"
version = "0.3.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eeb8be209bb1c96b7c177c7420d26e04eccacb0eeae6b980e35fcb74678107e0"
dependencies = [
"matches",
]
checksum = "246f4c42e67e7a4e3c6106ff716a5d067d4132a642840b242e357e468a2a0085"
[[package]]
name = "unicode-normalization"

33
server/src/db.rs
View File

@ -4218,6 +4218,39 @@ mod tests {
);
}
#[tokio::test]
async fn drop_unpersisted_chunk_on_persisted_db() {
// We don't support dropping unpersisted chunks from a persisted DB because we would forget the write buffer
// progress (partition checkpoints are only created when new parquet files are stored).
// See https://github.com/influxdata/influxdb_iox/issues/2291
let test_db = TestDb::builder()
.lifecycle_rules(LifecycleRules {
persist: true,
..Default::default()
})
.build()
.await;
let db = Arc::new(test_db.db);
write_lp(db.as_ref(), "cpu bar=1 10").await;
let partition_key = "1970-01-01T00";
let chunks = db.partition_chunk_summaries(partition_key);
assert_eq!(chunks.len(), 1);
let chunk_id = chunks[0].id;
let err = db
.drop_chunk("cpu", partition_key, chunk_id)
.await
.unwrap_err();
assert!(matches!(
err,
Error::LifecycleError {
source: super::lifecycle::Error::CannotDropUnpersistedChunk { .. }
}
));
}
async fn create_parquet_chunk(db: &Arc<Db>) -> (String, String, u32) {
write_lp(db, "cpu bar=1 10").await;
let partition_key = "1970-01-01T00";

12
server/src/db/lifecycle/drop.rs
View File

@ -9,7 +9,7 @@ use snafu::ResultExt;
use tracker::{TaskTracker, TrackedFuture, TrackedFutureExt};
use super::{
error::{CommitError, Result},
error::{CannotDropUnpersistedChunk, CommitError, Result},
LockableCatalogChunk, LockableCatalogPartition,
};
use crate::db::catalog::{
@ -34,6 +34,16 @@ pub fn drop_chunk(
chunk: guard.addr().clone(),
});
// check if we're dropping an unpersisted chunk in a persisted DB
// See https://github.com/influxdata/influxdb_iox/issues/2291
if db.rules().lifecycle_rules.persist && !matches!(guard.stage(), ChunkStage::Persisted { .. })
{
return CannotDropUnpersistedChunk {
addr: guard.addr().clone(),
}
.fail();
}
guard.set_dropping(&registration)?;
// Drop locks

3
server/src/db/lifecycle/error.rs
View File

@ -59,6 +59,9 @@ pub enum Error {
#[snafu(display("Cannot write chunk: {}", addr))]
CannotWriteChunk { addr: ChunkAddr },
#[snafu(display("Cannot drop unpersisted chunk: {}", addr))]
CannotDropUnpersistedChunk { addr: ChunkAddr },
}
pub type Result<T, E = Error> = std::result::Result<T, E>;

5
trace/src/tower.rs
View File

@ -16,7 +16,7 @@ use crate::{ctx::SpanContext, span::EnteredSpan, TraceCollector};
use futures::ready;
use http::{Request, Response};
use http_body::SizeHint;
use observability_deps::tracing::error;
use observability_deps::tracing::{error, info};
use pin_project::pin_project;
use std::future::Future;
use std::pin::Pin;
@ -69,6 +69,9 @@ where
}
fn call(&mut self, mut request: Request<ReqBody>) -> Self::Future {
use itertools::Itertools;
// TEMPORARY
info!(headers=%request.headers().keys().join(","), "gRPC request with headers");
let span = match SpanContext::from_headers(&self.collector, request.headers()) {
Ok(Some(ctx)) => {
let span = ctx.child("IOx");