feat: Add jemalloc stats
Marko Mikulicic
parent
a2fd01e388
commit
b579ef8646
|
|
@ -251,7 +251,7 @@ dependencies = [
|
|||
"log",
|
||||
"md5",
|
||||
"oauth2",
|
||||
"paste",
|
||||
"paste 1.0.5",
|
||||
"quick-error",
|
||||
"reqwest",
|
||||
"serde",
|
||||
|
|
@ -799,7 +799,7 @@ dependencies = [
|
|||
"num_cpus",
|
||||
"ordered-float 2.1.1",
|
||||
"parquet",
|
||||
"paste",
|
||||
"paste 1.0.5",
|
||||
"pin-project-lite",
|
||||
"smallvec",
|
||||
"sqlparser 0.9.0",
|
||||
|
|
@ -1511,6 +1511,7 @@ dependencies = [
|
|||
"tempfile",
|
||||
"test_helpers",
|
||||
"thiserror",
|
||||
"tikv-jemalloc-ctl",
|
||||
"tikv-jemallocator",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
|
|
@ -2412,12 +2413,31 @@ dependencies = [
|
|||
"tracker",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "paste"
|
||||
version = "0.1.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "45ca20c77d80be666aef2b45486da86238fabe33e38306bd3118fe4af33fa880"
|
||||
dependencies = [
|
||||
"paste-impl",
|
||||
"proc-macro-hack",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "paste"
|
||||
version = "1.0.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "acbf547ad0c65e31259204bd90935776d1c693cec2f4ff7abb7a1bbbd40dfe58"
|
||||
|
||||
[[package]]
|
||||
name = "paste-impl"
|
||||
version = "0.1.18"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d95a7db200b97ef370c8e6de0088252f7e0dfff7d047a28528e47456c0fc98b6"
|
||||
dependencies = [
|
||||
"proc-macro-hack",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "peeking_take_while"
|
||||
version = "0.1.2"
|
||||
|
|
@ -3400,6 +3420,7 @@ dependencies = [
|
|||
"snap",
|
||||
"tempfile",
|
||||
"test_helpers",
|
||||
"tikv-jemalloc-ctl",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"tracker",
|
||||
|
|
@ -3771,6 +3792,17 @@ dependencies = [
|
|||
"threadpool",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tikv-jemalloc-ctl"
|
||||
version = "0.4.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f28c80e4338857639f443169a601fafe49866aed8d7a8d565c2f5bfb1a021adf"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"paste 0.1.18",
|
||||
"tikv-jemalloc-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tikv-jemalloc-sys"
|
||||
version = "0.4.1+5.2.1-patched"
|
||||
|
|
|
|||
|
|
@ -91,6 +91,7 @@ snafu = "0.6.9"
|
|||
structopt = "0.3.21"
|
||||
thiserror = "1.0.23"
|
||||
tikv-jemallocator = "0.4.0"
|
||||
tikv-jemalloc-ctl = "0.4.0"
|
||||
tokio = { version = "1.0", features = ["macros", "rt-multi-thread", "parking_lot", "signal"] }
|
||||
tokio-stream = { version = "0.1.2", features = ["net"] }
|
||||
tokio-util = { version = "0.6.3" }
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ serde_json = "1.0"
|
|||
snafu = "0.6"
|
||||
snap = "1.0.0"
|
||||
tempfile = "3.1.0"
|
||||
tikv-jemalloc-ctl = "0.4.0"
|
||||
tokio = { version = "1.0", features = ["macros", "time"] }
|
||||
tokio-util = { version = "0.6.3" }
|
||||
tracker = { path = "../tracker" }
|
||||
|
|
|
|||
|
|
@ -88,7 +88,7 @@ use internal_types::{
|
|||
entry::{self, lines_to_sharded_entries, Entry, ShardedEntry},
|
||||
once::OnceNonZeroU32,
|
||||
};
|
||||
use metrics::MetricRegistry;
|
||||
use metrics::{KeyValue, MetricRegistry};
|
||||
use object_store::{path::ObjectStorePath, ObjectStore, ObjectStoreApi};
|
||||
use query::{exec::Executor, DatabaseStore};
|
||||
use tracker::{TaskId, TaskRegistration, TaskRegistryWithHistory, TaskTracker, TrackedFutureExt};
|
||||
|
|
@ -261,6 +261,9 @@ pub struct ServerMetrics {
|
|||
|
||||
/// The number of Entry bytes ingested
|
||||
pub ingest_entries_bytes_total: metrics::Counter,
|
||||
|
||||
/// Internal memory allocator stats
|
||||
pub jemalloc_memstats: metrics::Gauge,
|
||||
}
|
||||
|
||||
impl ServerMetrics {
|
||||
|
|
@ -268,6 +271,7 @@ impl ServerMetrics {
|
|||
// Server manages multiple domains.
|
||||
let http_domain = registry.register_domain("http");
|
||||
let ingest_domain = registry.register_domain("ingest");
|
||||
let jemalloc_domain = registry.register_domain("jemalloc");
|
||||
|
||||
Self {
|
||||
http_requests: http_domain.register_red_metric(None),
|
||||
|
|
@ -286,6 +290,34 @@ impl ServerMetrics {
|
|||
Some("bytes"),
|
||||
"total Entry bytes ingested",
|
||||
),
|
||||
jemalloc_memstats: jemalloc_domain.register_gauge_metric_with_labels_and_callback(
|
||||
"memstats",
|
||||
Some("bytes"),
|
||||
"jemalloc memstats",
|
||||
vec![],
|
||||
|observer| {
|
||||
use tikv_jemalloc_ctl::{epoch, stats};
|
||||
epoch::advance().unwrap();
|
||||
|
||||
let active = stats::allocated::read().unwrap();
|
||||
observer.observe(active as f64, &[KeyValue::new("stat", "active")]);
|
||||
|
||||
let allocated = stats::allocated::read().unwrap();
|
||||
observer.observe(allocated as f64, &[KeyValue::new("stat", "alloc")]);
|
||||
|
||||
let metadata = stats::metadata::read().unwrap();
|
||||
observer.observe(metadata as f64, &[KeyValue::new("stat", "metadata")]);
|
||||
|
||||
let mapped = stats::mapped::read().unwrap();
|
||||
observer.observe(mapped as f64, &[KeyValue::new("stat", "mapped")]);
|
||||
|
||||
let resident = stats::resident::read().unwrap();
|
||||
observer.observe(resident as f64, &[KeyValue::new("stat", "resident")]);
|
||||
|
||||
let retained = stats::retained::read().unwrap();
|
||||
observer.observe(retained as f64, &[KeyValue::new("stat", "retained")]);
|
||||
},
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue