Merge branch 'main' into jpg/for-whom-the-while-tolls

.gitignore

@@ -2,7 +2,9 @@
 **/*.rs.bk
 .idea/
 .env
+.gdb_history
 *.tsm
 **/.DS_Store
 **/.vscode
-query_tests/cases/**/*.out
+query_tests/cases/**/*.out
+valgrind-out.txt

Cargo.lock

@@ -191,9 +191,9 @@ dependencies = [
 
 [[package]]
 name = "assert_cmd"
-version = "1.0.8"
+version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c98233c6673d8601ab23e77eb38f999c51100d46c5703b17288c57fddf3a1ffe"
+checksum = "54f002ce7d0c5e809ebb02be78fd503aeed4a511fd0fcaff6e6914cbdabbfa33"
 dependencies = [
  "bstr",
  "doc-comment",
@@ -602,9 +602,9 @@ dependencies = [
 
 [[package]]
 name = "cloud-storage"
-version = "0.9.0"
+version = "0.10.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e781cbdd8c1c39dc43698f7a0f736abe624b9fa5f2aa952fb1fa2478291349b0"
+checksum = "6882d1c3788951dcfa4afe71da40a2a3913b3a81e2485b130f64714ce76b39c2"
 dependencies = [
  "base64 0.13.0",
  "bytes",
@@ -3983,9 +3983,9 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
 
 [[package]]
 name = "serde"
-version = "1.0.127"
+version = "1.0.128"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f03b9878abf6d14e6779d3f24f07b2cfa90352cfec4acc5aab8f1ac7f146fae8"
+checksum = "1056a0db1978e9dbf0f6e4fca677f6f9143dc1c19de346f22cac23e422196834"
 dependencies = [
  "serde_derive",
 ]
@@ -4014,9 +4014,9 @@ dependencies = [
 
 [[package]]
 name = "serde_derive"
-version = "1.0.127"
+version = "1.0.128"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a024926d3432516606328597e0f224a51355a493b49fdd67e9209187cbe55ecc"
+checksum = "13af2fbb8b60a8950d6c72a56d2095c28870367cc8e10c55e9745bac4995a2c4"
 dependencies = [
  "proc-macro2",
  "quote",

Cargo.toml

@@ -142,7 +142,7 @@ parking_lot = "0.11.1"
 write_buffer = { path = "write_buffer" }
 
 # Crates.io dependencies, in alphabetical order
-assert_cmd = "1.0.0"
+assert_cmd = "2.0.0"
 flate2 = "1.0"
 hex = "0.4.2"
 predicates = "2.0.2"

docs/sql.md

@@ -1,6 +1,6 @@
 # InfluxDB SQL Repl Tips and Tricks
 
-InfluxDB IOx contains a built in SQL client you can use to interact with the server using SQL
+InfluxDB IOx contains a built in SQL client you can use to interact with the server using SQL.
 
 ## Quick Start
 
@@ -198,8 +198,12 @@ LIMIT 20;
 ### Total row count by table
 
 ```sql
-SELECT database_name, table_name, max(count) as total_rows
+SELECT database_name, table_name, sum(total_rows) as total_rows
-FROM columns
+FROM (
+  SELECT database_name, table_name, max(row_count) as total_rows
+  FROM chunk_columns
+  GROUP BY database_name, partition_key, table_name
+)
 GROUP BY database_name, table_name
 ORDER BY total_rows DESC
 LIMIT 20;
@@ -208,8 +212,8 @@ LIMIT 20;
 ### Total row count by partition and table
 
 ```sql
-SELECT database_name, partition_key, table_name, max(count) as total_rows
+SELECT database_name, partition_key, table_name, max(row_count) as total_rows
-FROM columns
+FROM chunk_columns
 GROUP BY database_name, partition_key, table_name
 ORDER BY total_rows DESC
 LIMIT 20;
@@ -229,9 +233,9 @@ total_rows
 from
 (select table_name,
         column_name,
-        sum(count) as total_rows,
+        sum(row_count) as total_rows,
         max(cast(max_value as double)) - min(cast(min_value as double)) as range
- from system.chunk_columns
+ from chunk_columns
  where column_name = 'time'
  group by table_name, column_name
 )

docs/valgrind.md

@@ -0,0 +1,42 @@
+# Valgrind
+This document explains how to use [Valgrind] to perform certain debug tasks.
+
+## Build
+Create a debug build that uses the system memory allocator (i.e. neither [heappy] nor [jemalloc]):
+
+```console
+$ cargo build --no-default-features
+```
+
+## Memory Leaks
+There is a script that does most of the config setting. Just start the server with:
+
+```console
+$ ./scripts/valgrind_leak ./target/debug/influxdb_iox run ...
+```
+
+You can kill the server w/ `CTRL-C` when you're ready. The [Valgrind] output will be written to `valgrind-out.txt`.
+
+## Suppression Rules
+[Valgrind] allows you to suppress certain outputs. This can be used to ignore known "issues" like that [lazycell] leaks.
+For IOx we provide a file that is used by the scripts (under `scripts/valgrind.supp`). If you plan to write your own
+rules, here are some useful links:
+
+- <https://valgrind.org/docs/manual/mc-manual.html#mc-manual.suppfiles>
+- <https://www.valgrind.org/docs/manual/manual-core.html#manual-core.suppress>
+- <https://wiki.wxwidgets.org/Valgrind_Suppression_File_Howto>
+
+You may also use the `--gen-suppressions=all` to auto-generate supppression rules:
+
+```console
+$ ./scripts/valgrind_leak --gen-suppressions=all ./target/debug/influxdb_iox run ...
+```
+
+Note that Rust symbols like `influxdb_iox::main` are mangled in a way that [Valgrind] cannot parse them (e.g. to
+`_ZN12influxdb_iox4main17h940b8bf02831a9d8E`). The easiest way is to replace `::` w/ `*` and prepand and append an
+additional wildcard `*`, so `influxdb_iox::main` gets `*influxdb_iox*main*`.
+
+[heappy]: https://github.com/mkmik/heappy
+[jemalloc]: ttps://github.com/jemalloc/jemalloc
+[lazycell]: https://crates.io/crates/lazycell
+[Valgrind]: https://valgrind.org/

influxdb_iox_client/Cargo.toml

@@ -21,7 +21,7 @@ http = "0.2.3"
 hyper = "0.14"
 prost = "0.8"
 rand = "0.8.3"
-serde = "1.0.118"
+serde = "1.0.128"
 serde_json = { version = "1.0.44", optional = true }
 thiserror = "1.0.23"
 tokio = { version = "1.0", features = ["macros"] }

object_store/Cargo.toml

@@ -13,7 +13,7 @@ azure_storage = { git = "https://github.com/Azure/azure-sdk-for-rust.git", rev =
 bytes = "1.0"
 chrono = "0.4"
 # Google Cloud Storage integration
-cloud-storage = {version = "0.9.0", optional = true}
+cloud-storage = {version = "0.10.2", optional = true}
 futures = "0.3"
 itertools = "0.10.1"
 percent-encoding = "2.1"

scripts/valgrind.supp

@@ -0,0 +1,31 @@
+{
+    dlopen
+    Memcheck:Leak
+    ...
+    fun:do_dlopen
+    ...
+}
+
+{
+    lazystatic
+    Memcheck:Leak
+    ...
+    fun:__static_ref_initialize
+    ...
+}
+
+{
+    oncecell
+    Memcheck:Leak
+    ...
+    fun:*once_cell*imp*initialize_inner*
+    ...
+}
+
+{
+    init_logs_and_tracing
+    Memcheck:Leak
+    ...
+    fun:*init_logs_and_tracing*
+    ...
+}

scripts/valigrind_leak

@@ -0,0 +1,14 @@
+#!/bin/bash
+set -eu -o pipefail
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+exec valgrind \
+    --leak-check=full \
+    --log-file=valgrind-out.txt \
+    --num-callers=50 \
+    --show-leak-kinds=all \
+    --suppressions="$SCRIPT_DIR/valgrind.supp" \
+    --track-origins=yes \
+    --verbose \
+    $@

src/influxdb_ioxd.rs

@@ -130,18 +130,41 @@ fn make_server(
     app_server
 }
 
+#[cfg(all(not(feature = "heappy"), not(feature = "jemalloc_replacing_malloc")))]
+fn build_malloc_conf() -> String {
+    "system".to_string()
+}
+
+#[cfg(all(feature = "heappy", not(feature = "jemalloc_replacing_malloc")))]
+fn build_malloc_conf() -> String {
+    "heappy".to_string()
+}
+
+#[cfg(all(not(feature = "heappy"), feature = "jemalloc_replacing_malloc"))]
+fn build_malloc_conf() -> String {
+    tikv_jemalloc_ctl::config::malloc_conf::mib()
+        .unwrap()
+        .read()
+        .unwrap()
+        .to_string()
+}
+
+#[cfg(all(feature = "heappy", feature = "jemalloc_replacing_malloc"))]
+fn build_malloc_conf() -> String {
+    compile_error!("must use exactly one memory allocator")
+}
+
 /// This is the entry point for the IOx server. `config` represents
 /// command line arguments, if any.
 pub async fn main(config: Config) -> Result<()> {
     let git_hash = option_env!("GIT_HASH").unwrap_or("UNKNOWN");
     let num_cpus = num_cpus::get();
-    let build_malloc_conf = tikv_jemalloc_ctl::config::malloc_conf::mib()
+    let build_malloc_conf = build_malloc_conf();
-        .unwrap()
-        .read()
-        .unwrap();
     info!(
         git_hash,
-        num_cpus, build_malloc_conf, "InfluxDB IOx server starting"
+        num_cpus,
+        %build_malloc_conf,
+        "InfluxDB IOx server starting",
     );
 
     // Install custom panic handler and forget about it.

src/main.rs

@@ -46,9 +46,6 @@ static VERSION_STRING: Lazy<String> = Lazy::new(|| {
     )
 });
 
-#[cfg(not(any(feature = "heappy", feature = "jemalloc_replacing_malloc")))]
-compile_error!("you need to pick either heappy or jemalloc_replacing_malloc feature, cargo can't pick a default out of a mutually exclusive set");
-
 #[cfg(all(feature = "heappy", feature = "jemalloc_replacing_malloc"))]
 compile_error!("heappy and jemalloc_replacing_malloc features are mutually exclusive");