feat: Attempt to dump a stacktrace to stderr prior to process abort on SIGSEGV/SIGILL/SIGBUS (#2155)

* feat: Attempt to dump a stacktrace to stdout prior to process abort

* refactor: rewrite signal handling in terms of libc, remove sig dep

* refactor: print to stderr

* fix: Update src/main.rs

* docs: note provenance

Cargo.lock

@@ -1619,6 +1619,7 @@ dependencies = [
  "arrow-flight",
  "arrow_util",
  "assert_cmd",
+ "backtrace",
  "byteorder",
  "bytes",
  "chrono",
@@ -1640,6 +1641,7 @@ dependencies = [
  "influxdb_line_protocol",
  "internal_types",
  "itertools 0.10.1",
+ "libc",
  "logfmt",
  "metrics",
  "mutable_buffer",

Cargo.toml

@@ -77,6 +77,7 @@ trogging = { path = "trogging", features = ["structopt"] }
 # Crates.io dependencies, in alphabetical order
 arrow = { version = "5.0", features = ["prettyprint"] }
 arrow-flight = "5.0"
+backtrace = "0.3"
 byteorder = "1.3.4"
 bytes = "1.0"
 chrono = "0.4"
@@ -88,6 +89,7 @@ flate2 = "1.0"
 futures = "0.3"
 http = "0.2.0"
 hyper = "0.14"
+libc = { version = "0.2" }
 once_cell = { version = "1.4.0", features = ["parking_lot"] }
 opentelemetry-jaeger = { version = "0.12", features = ["tokio"] }
 opentelemetry-otlp = "0.6"

src/main.rs

@@ -137,6 +137,8 @@ enum Command {
 }
 
 fn main() -> Result<(), std::io::Error> {
+    install_crash_handler(); // attempt to render a useful stacktrace to stderr
+
     // load all environment variables from .env before doing anything
     load_dotenv();
 
@@ -256,3 +258,50 @@ fn load_dotenv() {
         }
     };
 }
+
+// Based on ideas from
+// https://github.com/servo/servo/blob/f03ddf6c6c6e94e799ab2a3a89660aea4a01da6f/ports/servo/main.rs#L58-L79
+fn install_crash_handler() {
+    unsafe {
+        set_signal_handler(libc::SIGSEGV, signal_handler); // handle segfaults
+        set_signal_handler(libc::SIGILL, signal_handler); // handle stack overflow and unsupported CPUs
+        set_signal_handler(libc::SIGBUS, signal_handler); // handle invalid memory access
+    }
+}
+
+unsafe extern "C" fn signal_handler(sig: i32) {
+    use backtrace::Backtrace;
+    use std::process::abort;
+    let name = std::thread::current()
+        .name()
+        .map(|n| format!(" for thread \"{}\"", n))
+        .unwrap_or_else(|| "".to_owned());
+    eprintln!(
+        "Signal {}, Stack trace{}\n{:?}",
+        sig,
+        name,
+        Backtrace::new()
+    );
+    abort();
+}
+
+// based on https://github.com/adjivas/sig/blob/master/src/lib.rs#L34-L52
+unsafe fn set_signal_handler(signal: libc::c_int, handler: unsafe extern "C" fn(libc::c_int)) {
+    use libc::{sigaction, sigfillset, sighandler_t};
+    let mut sigset = std::mem::zeroed();
+
+    // Block all signals during the handler. This is the expected behavior, but
+    // it's not guaranteed by `signal()`.
+    if sigfillset(&mut sigset) != -1 {
+        // Done because sigaction has private members.
+        // This is safe because sa_restorer and sa_handlers are pointers that
+        // might be null (that is, zero).
+        let mut action: sigaction = std::mem::zeroed();
+
+        // action.sa_flags = 0;
+        action.sa_mask = sigset;
+        action.sa_sigaction = handler as sighandler_t;
+
+        sigaction(signal, &action, std::ptr::null_mut());
+    }
+}