From 4a2f0b0a5205ff86d35a7499e2f9a8586830a652 Mon Sep 17 00:00:00 2001 From: Marko Mikulicic Date: Thu, 19 Aug 2021 18:36:14 +0200 Subject: [PATCH] fix: Make heappy and nonheappy mutually exclusive Closes #2288 --- .circleci/config.yml | 8 ++++---- Cargo.lock | 12 +----------- Cargo.toml | 22 +++++++++++++++------- src/main.rs | 6 ++++++ 4 files changed, 26 insertions(+), 22 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index decf27aada..752bce57b8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -149,7 +149,7 @@ jobs: - cache_restore - run: name: End to end tests (with heappy) - command: cargo test --features=heappy end_to_end + command: cargo test --no-default-features --features=heappy end_to_end - cache_save test_kafka_integration: @@ -214,7 +214,7 @@ jobs: command: cargo test --workspace --benches --no-run - run: name: Build with object store + exporter support + HEAP profiling - command: cargo build --features="aws,gcp,azure,jaeger,otlp,heappy,pprof" + command: cargo build --no-default-features --features="aws,gcp,azure,jaeger,otlp,heappy,pprof" - cache_save # Lint protobufs. @@ -273,10 +273,10 @@ jobs: - cache_restore - run: name: Print rustc target CPU options - command: cargo run --release --features="aws,gcp,azure,jaeger,otlp,heappy" --bin print_cpu + command: cargo run --release --no-default-features --features="aws,gcp,azure,jaeger,otlp,heappy" --bin print_cpu - run: name: Cargo release build with target arch set for CRoaring - command: cargo build --release --features="aws,gcp,azure,jaeger,otlp,heappy" + command: cargo build --release --no-default-features --features="aws,gcp,azure,jaeger,otlp,heappy" - run: | echo sha256sum after build is sha256sum target/release/influxdb_iox diff --git a/Cargo.lock b/Cargo.lock index 0bc7bd2c64..d7f695d944 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1801,7 +1801,7 @@ dependencies = [ "test_helpers", "thiserror", "tikv-jemalloc-ctl", - "tikv-jemallocator", + "tikv-jemalloc-sys", "tokio", "tokio-stream", "tokio-util", @@ -4579,16 +4579,6 @@ dependencies = [ "libc", ] -[[package]] -name = "tikv-jemallocator" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c14a5a604eb8715bc5785018a37d00739b180bcf609916ddf4393d33d49ccdf" -dependencies = [ - "libc", - "tikv-jemalloc-sys", -] - [[package]] name = "time" version = "0.1.43" diff --git a/Cargo.toml b/Cargo.toml index b793257a74..05d03643ee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -63,7 +63,6 @@ datafusion = { path = "datafusion" } data_types = { path = "data_types" } entry = { path = "entry" } generated_types = { path = "generated_types" } -heappy = { git = "https://github.com/mkmik/heappy", rev = "82a383128e484039cc2f31476e6289bed48a6701", features = ["enable_heap_profiler", "jemalloc_shim", "measure_free"], optional = true } influxdb_iox_client = { path = "influxdb_iox_client", features = ["format"] } influxdb_line_protocol = { path = "influxdb_line_protocol" } @@ -117,9 +116,7 @@ serde_urlencoded = "0.7.0" snafu = "0.6.9" structopt = "0.3.21" thiserror = "1.0.23" -# remember to put "unprefixed_malloc_on_supported_platforms" if you disable heappy -tikv-jemallocator = { version = "0.4.0" } -tikv-jemalloc-ctl = "0.4.0" +tikv-jemalloc-ctl = { version = "0.4.0" } tokio = { version = "1.0", features = ["macros", "rt-multi-thread", "parking_lot", "signal"] } tokio-stream = { version = "0.1.2", features = ["net"] } tokio-util = { version = "0.6.3" } @@ -128,6 +125,11 @@ tonic-health = "0.4.0" tonic-reflection = "0.2.0" uuid = { version = "0.8", features = ["v4"] } +# jemalloc-sys with unprefixed_malloc_on_supported_platforms feature and heappy are mutually exclusive +tikv-jemalloc-sys = { version = "0.4.0", optional = true, features = ["unprefixed_malloc_on_supported_platforms"]} +heappy = { git = "https://github.com/mkmik/heappy", rev = "82a383128e484039cc2f31476e6289bed48a6701", features = ["enable_heap_profiler", "jemalloc_shim", "measure_free"], optional = true } + + [dev-dependencies] # Workspace dependencies, in alphabetical order arrow_util = { path = "arrow_util" } @@ -150,11 +152,17 @@ reqwest = "0.11" tempfile = "3.1.0" [features] +default = [ "jemalloc_replacing_malloc" ] + azure = ["object_store/azure"] # Optional Azure Object store support gcp = ["object_store/gcp"] # Optional GCP object store support aws = ["object_store/aws"] # Optional AWS / S3 object store support jaeger = ["trogging/jaeger", "trace_exporters/jaeger"] # Enable optional jaeger tracing support otlp = ["trogging/otlp"] # Enable optional open telemetry collector -# heappy is also an optional feature; Not on by default as it -# runtime overhead on all allocations (calls to malloc) -# pprof is also an optional feature for pprof support \ No newline at end of file +# pprof is an optional feature for pprof support + +# heappy is an optional feature; Not on by default as it +# runtime overhead on all allocations (calls to malloc). +# Cargo cannot currently implement mutually exclusive features so let's force every build +# to pick either heappy or jemalloc_replacing_malloc feature at least until we figure out something better. +jemalloc_replacing_malloc = ["tikv-jemalloc-sys"] diff --git a/src/main.rs b/src/main.rs index 5ae2f5410d..ff6355d108 100644 --- a/src/main.rs +++ b/src/main.rs @@ -46,6 +46,12 @@ static VERSION_STRING: Lazy = Lazy::new(|| { ) }); +#[cfg(not(any(feature = "heappy", feature = "jemalloc_replacing_malloc")))] +compile_error!("you need to pick either heappy or jemalloc_replacing_malloc feature, cargo can't pick a default out of a mutually exclusive set"); + +#[cfg(all(feature = "heappy", feature = "jemalloc_replacing_malloc"))] +compile_error!("heappy and jemalloc_replacing_malloc features are mutually exclusive"); + #[derive(Debug, StructOpt)] #[structopt( name = "influxdb_iox",