diff --git a/Cargo.lock b/Cargo.lock index 92758047da..0bb3115454 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -756,6 +756,16 @@ dependencies = [ "sct", ] +[[package]] +name = "dashmap" +version = "4.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c" +dependencies = [ + "cfg-if 1.0.0", + "num_cpus", +] + [[package]] name = "data_types" version = "0.1.0" @@ -2121,6 +2131,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "514d24875c140ed269eecc2d1b56d7b71b573716922a763c317fb1b1b4b58f15" dependencies = [ "async-trait", + "dashmap", + "fnv", "futures", "js-sys", "lazy_static", @@ -2146,6 +2158,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "opentelemetry-prometheus" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5407eac459699e03e83f55a557920c612d09f202b58f44212b5cdc4e8a7666e" +dependencies = [ + "opentelemetry", + "prometheus", + "protobuf", +] + [[package]] name = "ordered-float" version = "1.1.1" @@ -2508,6 +2531,21 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "prometheus" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8425533e7122f0c3cc7a37e6244b16ad3a2cc32ae7ac6276e2a75da0d9c200d" +dependencies = [ + "cfg-if 1.0.0", + "fnv", + "lazy_static", + "parking_lot", + "protobuf", + "regex", + "thiserror", +] + [[package]] name = "prost" version = "0.7.0" @@ -2559,6 +2597,12 @@ dependencies = [ "prost", ] +[[package]] +name = "protobuf" +version = "2.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b7f4a129bb3754c25a4e04032a90173c68f85168f77118ac4cb4936e7f06f92" + [[package]] name = "query" version = "0.1.0" @@ -3921,6 +3965,8 @@ dependencies = [ "env_logger", "opentelemetry", "opentelemetry-jaeger", + "opentelemetry-prometheus", + "prometheus", "tracing", "tracing-futures", "tracing-opentelemetry", diff --git a/Cargo.toml b/Cargo.toml index dcda8e496c..46d5747ac9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,6 +72,7 @@ flate2 = "1.0" futures = "0.3.1" http = "0.2.0" hyper = "0.14" +parking_lot = "0.11.1" # used by arrow/datafusion anyway prettytable-rs = "0.8" prost = "0.7" diff --git a/src/commands/metrics.rs b/src/commands/metrics.rs new file mode 100644 index 0000000000..870857b280 --- /dev/null +++ b/src/commands/metrics.rs @@ -0,0 +1,34 @@ +use parking_lot::{const_rwlock, RwLock}; +use tracing_deps::{ + opentelemetry::{sdk::Resource, KeyValue}, + opentelemetry_prometheus, + prometheus::{Encoder, TextEncoder}, +}; + +static PROMETHEUS_EXPORTER: RwLock> = + const_rwlock(None); + +pub fn init_metrics() { + let exporter = opentelemetry_prometheus::exporter() + .with_resource(Resource::new(vec![KeyValue::new( + "service.name", + "influxdb-iox", + )])) + .init(); + let mut guard = PROMETHEUS_EXPORTER.write(); + *guard = Some(exporter); +} + +pub fn metrics_as_text() -> Vec { + let metric_families = PROMETHEUS_EXPORTER + .read() + .as_ref() + .unwrap() + .registry() + .gather(); + let mut result = Vec::new(); + TextEncoder::new() + .encode(&metric_families, &mut result) + .unwrap(); + result +} diff --git a/src/influxdb_ioxd.rs b/src/influxdb_ioxd.rs index c161809ad8..24f086f6ba 100644 --- a/src/influxdb_ioxd.rs +++ b/src/influxdb_ioxd.rs @@ -1,5 +1,6 @@ use crate::commands::{ logging::LoggingLevel, + metrics, run::{Config, ObjectStore as ObjStoreOpt}, }; use futures::{future::FusedFuture, pin_mut, FutureExt}; @@ -101,6 +102,7 @@ pub async fn main(logging_level: LoggingLevel, config: Config) -> Result<()> { let logging_level = logging_level.combine(LoggingLevel::new(config.verbose_count)); let _drop_handle = logging_level.setup_logging(&config); + let _metrics_guard = metrics::init_metrics(); // Install custom panic handler and forget about it. // diff --git a/src/influxdb_ioxd/http.rs b/src/influxdb_ioxd/http.rs index 9fcd043b63..a82b054e9e 100644 --- a/src/influxdb_ioxd/http.rs +++ b/src/influxdb_ioxd/http.rs @@ -11,6 +11,7 @@ //! database names and may remove this quasi /v2 API. // Influx crates +use super::super::commands::metrics; use arrow_deps::datafusion::physical_plan::collect; use data_types::{ http::WalMetadataQuery, @@ -316,6 +317,7 @@ where })) // this endpoint is for API backward compatibility with InfluxDB 2.x .post("/api/v2/write", write::) .get("/health", health) + .get("/metrics", handle_metrics) .get("/iox/api/v1/databases/:name/query", query::) .get("/iox/api/v1/databases/:name/wal/meta", get_wal_meta::) .get("/api/v1/partitions", list_partitions::) @@ -592,6 +594,11 @@ async fn health(_: Request) -> Result, ApplicationError> { Ok(Response::new(Body::from(response_body.to_string()))) } +#[tracing::instrument(level = "debug")] +async fn handle_metrics(_: Request) -> Result, ApplicationError> { + Ok(Response::new(Body::from(metrics::metrics_as_text()))) +} + #[derive(Deserialize, Debug)] /// Arguments in the query string of the request to /partitions struct DatabaseInfo { diff --git a/src/main.rs b/src/main.rs index 5d193a8582..bfb7eafecd 100644 --- a/src/main.rs +++ b/src/main.rs @@ -23,6 +23,7 @@ mod commands { mod input; pub mod logging; pub mod meta; + pub mod metrics; pub mod operations; pub mod run; pub mod server; diff --git a/tracing_deps/Cargo.toml b/tracing_deps/Cargo.toml index e1738482d2..d12940d8f2 100644 --- a/tracing_deps/Cargo.toml +++ b/tracing_deps/Cargo.toml @@ -1,4 +1,5 @@ [package] +# TODO(jacobmarble): rename to something like "observability_deps" name = "tracing_deps" version = "0.1.0" authors = ["Paul Dix "] @@ -7,8 +8,10 @@ description = "Tracing ecosystem dependencies for InfluxDB IOx, to ensure consis [dependencies] # In alphabetical order env_logger = "0.8.3" -opentelemetry = { version = "0.12", default-features = false, features = ["trace", "tokio-support"] } +opentelemetry = { version = "0.12", default-features = false, features = ["trace", "metrics", "tokio-support"] } opentelemetry-jaeger = { version = "0.11", features = ["tokio"] } +opentelemetry-prometheus = "0.5.0" +prometheus = "0.11" tracing = { version = "0.1", features = ["release_max_level_debug"] } tracing-futures = "0.2.4" tracing-opentelemetry = "0.11.0" diff --git a/tracing_deps/src/lib.rs b/tracing_deps/src/lib.rs index 687dc439fd..79df3ecdf9 100644 --- a/tracing_deps/src/lib.rs +++ b/tracing_deps/src/lib.rs @@ -5,6 +5,8 @@ pub use env_logger; pub use opentelemetry; pub use opentelemetry_jaeger; +pub use opentelemetry_prometheus; +pub use prometheus; pub use tracing; pub use tracing::instrument; pub use tracing_futures;