feat: add Prometheus metrics endpoint

This change adds HTTP endpoint /metrics. Now, I need to add a metric.

Cargo.lock

@@ -756,6 +756,16 @@ dependencies = [
  "sct",
 ]
 
+[[package]]
+name = "dashmap"
+version = "4.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c"
+dependencies = [
+ "cfg-if 1.0.0",
+ "num_cpus",
+]
+
 [[package]]
 name = "data_types"
 version = "0.1.0"
@@ -2121,6 +2131,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "514d24875c140ed269eecc2d1b56d7b71b573716922a763c317fb1b1b4b58f15"
 dependencies = [
  "async-trait",
+ "dashmap",
+ "fnv",
  "futures",
  "js-sys",
  "lazy_static",
@@ -2146,6 +2158,17 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "opentelemetry-prometheus"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c5407eac459699e03e83f55a557920c612d09f202b58f44212b5cdc4e8a7666e"
+dependencies = [
+ "opentelemetry",
+ "prometheus",
+ "protobuf",
+]
+
 [[package]]
 name = "ordered-float"
 version = "1.1.1"
@@ -2508,6 +2531,21 @@ dependencies = [
  "unicode-xid",
 ]
 
+[[package]]
+name = "prometheus"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c8425533e7122f0c3cc7a37e6244b16ad3a2cc32ae7ac6276e2a75da0d9c200d"
+dependencies = [
+ "cfg-if 1.0.0",
+ "fnv",
+ "lazy_static",
+ "parking_lot",
+ "protobuf",
+ "regex",
+ "thiserror",
+]
+
 [[package]]
 name = "prost"
 version = "0.7.0"
@@ -2559,6 +2597,12 @@ dependencies = [
  "prost",
 ]
 
+[[package]]
+name = "protobuf"
+version = "2.22.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1b7f4a129bb3754c25a4e04032a90173c68f85168f77118ac4cb4936e7f06f92"
+
 [[package]]
 name = "query"
 version = "0.1.0"
@@ -3921,6 +3965,8 @@ dependencies = [
  "env_logger",
  "opentelemetry",
  "opentelemetry-jaeger",
+ "opentelemetry-prometheus",
+ "prometheus",
  "tracing",
  "tracing-futures",
  "tracing-opentelemetry",

Cargo.toml

@@ -72,6 +72,7 @@ flate2 = "1.0"
 futures = "0.3.1"
 http = "0.2.0"
 hyper = "0.14"
+parking_lot = "0.11.1"
 # used by arrow/datafusion anyway
 prettytable-rs = "0.8"
 prost = "0.7"

src/commands/metrics.rs

@@ -0,0 +1,34 @@
+use parking_lot::{const_rwlock, RwLock};
+use tracing_deps::{
+    opentelemetry::{sdk::Resource, KeyValue},
+    opentelemetry_prometheus,
+    prometheus::{Encoder, TextEncoder},
+};
+
+static PROMETHEUS_EXPORTER: RwLock<Option<opentelemetry_prometheus::PrometheusExporter>> =
+    const_rwlock(None);
+
+pub fn init_metrics() {
+    let exporter = opentelemetry_prometheus::exporter()
+        .with_resource(Resource::new(vec![KeyValue::new(
+            "service.name",
+            "influxdb-iox",
+        )]))
+        .init();
+    let mut guard = PROMETHEUS_EXPORTER.write();
+    *guard = Some(exporter);
+}
+
+pub fn metrics_as_text() -> Vec<u8> {
+    let metric_families = PROMETHEUS_EXPORTER
+        .read()
+        .as_ref()
+        .unwrap()
+        .registry()
+        .gather();
+    let mut result = Vec::new();
+    TextEncoder::new()
+        .encode(&metric_families, &mut result)
+        .unwrap();
+    result
+}

src/influxdb_ioxd.rs

@@ -1,5 +1,6 @@
 use crate::commands::{
     logging::LoggingLevel,
+    metrics,
     run::{Config, ObjectStore as ObjStoreOpt},
 };
 use futures::{future::FusedFuture, pin_mut, FutureExt};
@@ -101,6 +102,7 @@ pub async fn main(logging_level: LoggingLevel, config: Config) -> Result<()> {
     let logging_level = logging_level.combine(LoggingLevel::new(config.verbose_count));
 
     let _drop_handle = logging_level.setup_logging(&config);
+    let _metrics_guard = metrics::init_metrics();
 
     // Install custom panic handler and forget about it.
     //

src/influxdb_ioxd/http.rs

@@ -11,6 +11,7 @@
 //! database names and may remove this quasi /v2 API.
 
 // Influx crates
+use super::super::commands::metrics;
 use arrow_deps::datafusion::physical_plan::collect;
 use data_types::{
     http::WalMetadataQuery,
@@ -316,6 +317,7 @@ where
         })) // this endpoint is for API backward compatibility with InfluxDB 2.x
         .post("/api/v2/write", write::<M>)
         .get("/health", health)
+        .get("/metrics", handle_metrics)
         .get("/iox/api/v1/databases/:name/query", query::<M>)
         .get("/iox/api/v1/databases/:name/wal/meta", get_wal_meta::<M>)
         .get("/api/v1/partitions", list_partitions::<M>)
@@ -592,6 +594,11 @@ async fn health(_: Request<Body>) -> Result<Response<Body>, ApplicationError> {
     Ok(Response::new(Body::from(response_body.to_string())))
 }
 
+#[tracing::instrument(level = "debug")]
+async fn handle_metrics(_: Request<Body>) -> Result<Response<Body>, ApplicationError> {
+    Ok(Response::new(Body::from(metrics::metrics_as_text())))
+}
+
 #[derive(Deserialize, Debug)]
 /// Arguments in the query string of the request to /partitions
 struct DatabaseInfo {

src/main.rs

@@ -23,6 +23,7 @@ mod commands {
     mod input;
     pub mod logging;
     pub mod meta;
+    pub mod metrics;
     pub mod operations;
     pub mod run;
     pub mod server;

tracing_deps/Cargo.toml

@@ -1,4 +1,5 @@
 [package]
+# TODO(jacobmarble): rename to something like "observability_deps"
 name = "tracing_deps"
 version = "0.1.0"
 authors = ["Paul Dix <paul@pauldix.net>"]
@@ -7,8 +8,10 @@ description = "Tracing ecosystem dependencies for InfluxDB IOx, to ensure consis
 
 [dependencies] # In alphabetical order
 env_logger = "0.8.3"
-opentelemetry = { version = "0.12", default-features = false, features = ["trace", "tokio-support"] }
+opentelemetry = { version = "0.12", default-features = false, features = ["trace", "metrics", "tokio-support"] }
 opentelemetry-jaeger = { version = "0.11", features = ["tokio"] }
+opentelemetry-prometheus = "0.5.0"
+prometheus = "0.11"
 tracing = { version = "0.1", features = ["release_max_level_debug"] }
 tracing-futures = "0.2.4"
 tracing-opentelemetry = "0.11.0"

tracing_deps/src/lib.rs

@@ -5,6 +5,8 @@
 pub use env_logger;
 pub use opentelemetry;
 pub use opentelemetry_jaeger;
+pub use opentelemetry_prometheus;
+pub use prometheus;
 pub use tracing;
 pub use tracing::instrument;
 pub use tracing_futures;