From 991bf462ca19796765ee98e9be7dec70b04f08a4 Mon Sep 17 00:00:00 2001 From: Scott Anderson Date: Wed, 5 Feb 2025 12:46:57 -0700 Subject: [PATCH] Add missing influxdb3 serve configuration options (#5822) * add missing influxdb3 serve configuration options * Apply suggestions from code review Co-authored-by: Jason Stirnaman --------- Co-authored-by: Jason Stirnaman --- .../core/reference/cli/influxdb3/serve.md | 2 + .../core/reference/config-options.md | 30 ++++++- .../reference/cli/influxdb3/serve.md | 7 ++ .../enterprise/reference/config-options.md | 89 +++++++++++++++++-- 4 files changed, 121 insertions(+), 7 deletions(-) diff --git a/content/influxdb3/core/reference/cli/influxdb3/serve.md b/content/influxdb3/core/reference/cli/influxdb3/serve.md index 1c426eaaa..2e8cc59d2 100644 --- a/content/influxdb3/core/reference/cli/influxdb3/serve.md +++ b/content/influxdb3/core/reference/cli/influxdb3/serve.md @@ -90,6 +90,8 @@ influxdb3 serve [OPTIONS] --node-id | | `--distinct-cache-eviction-interval` | _See [configuration options](/influxdb3/core/reference/config-options/#distinct-cache-eviction-interval)_ | | | `--plugin-dir` | _See [configuration options](/influxdb3/core/reference/config-options/#plugin-dir)_ | | | `--force-snapshot-mem-threshold` | _See [configuration options](/influxdb3/core/reference/config-options/#force-snapshot-mem-threshold)_ | +| | `--virtual-env-location` | _See [configuration options](/influxdb3/core/reference/config-options/#virtual-env-location)_ | +| | `--package-manager` | _See [configuration options](/influxdb3/core/reference/config-options/#package-manager)_ | | | `--query-file-limit` | _See [configuration options](/influxdb3/core/reference/config-options/#query-file-limit)_ | {{< caption >}} diff --git a/content/influxdb3/core/reference/config-options.md b/content/influxdb3/core/reference/config-options.md index 53b0f9726..7c88929c8 100644 --- a/content/influxdb3/core/reference/config-options.md +++ b/content/influxdb3/core/reference/config-options.md @@ -127,8 +127,10 @@ influxdb3 serve - [disable-parquet-mem-cache](#disable-parquet-mem-cache) - [last-cache-eviction-interval](#last-cache-eviction-interval) - [distinct-cache-eviction-interval](#distinct-cache-eviction-interval) -- [Plugins](#plugins) +- [Processing engine](#processing-engine) - [plugin-dir](#plugin-dir) + - [virtual-env-location](#virtual-env-location) + - [package-manager](#package-manager) --- @@ -1017,9 +1019,11 @@ expressed as a human-readable time--for example: `20s`, `1m`, `1h`. --- -### Plugins +### Processing engine - [plugin-dir](#plugin-dir) +- [virtual-env-location](#virtual-env-location) +- [package-manager](#package-manager) #### plugin-dir @@ -1029,3 +1033,25 @@ Specifies the local directory that contains Python plugins and their test files. | :--------------------- | :--------------------- | | `--plugin-dir` | `INFLUXDB3_PLUGIN_DIR` | +--- + +#### virtual-env-location + +Specifies the location of the Python virtual environment that the processing +engine uses. + +| influxdb3 serve option | Environment variable | +| :----------------------- | :--------------------- | +| `--virtual-env-location` | `VIRTUAL_ENV_LOCATION` | + +--- + +#### package-manager + +Specifies the Python package manager that the processing engine uses. + +**Default:** `10s` + +| influxdb3 serve option | Environment variable | +| :--------------------- | :------------------- | +| `--package-manager` | `PACKAGE_MANAGER` | diff --git a/content/influxdb3/enterprise/reference/cli/influxdb3/serve.md b/content/influxdb3/enterprise/reference/cli/influxdb3/serve.md index a61bf6e0e..f3aa34165 100644 --- a/content/influxdb3/enterprise/reference/cli/influxdb3/serve.md +++ b/content/influxdb3/enterprise/reference/cli/influxdb3/serve.md @@ -76,6 +76,7 @@ influxdb3 serve [OPTIONS] --node-id | | `--wal-flush-interval` | _See [configuration options](/influxdb3/enterprise/reference/config-options/#wal-flush-interval)_ | | | `--wal-snapshot-size` | _See [configuration options](/influxdb3/enterprise/reference/config-options/#wal-snapshot-size)_ | | | `--wal-max-write-buffer-size` | _See [configuration options](/influxdb3/enterprise/reference/config-options/#wal-max-write-buffer-size)_ | +| | `--snapshotted-wal-files-to-keep` | _See [configuration options](/influxdb3/enterprise/reference/config-options/#snapshotted-wal-files-to-keep)_ | | | `--query-log-size` | _See [configuration options](/influxdb3/enterprise/reference/config-options/#query-log-size)_ | | | `--buffer-mem-limit-mb` | _See [configuration options](/influxdb3/enterprise/reference/config-options/#buffer-mem-limit-mb)_ | | {{< req "\*" >}} | `--node-id` | _See [configuration options](/influxdb3/enterprise/reference/config-options/#node-id)_ | @@ -89,14 +90,20 @@ influxdb3 serve [OPTIONS] --node-id | | `--compaction-max-num-files-per-plan` | _See [configuration options](/influxdb3/enterprise/reference/config-options/#compaction-max-num-files-per-plan)_ | | | `--compaction-gen2-duration` | _See [configuration options](/influxdb3/enterprise/reference/config-options/#compaction-gen2-duration)_ | | | `--compaction-multipliers` | _See [configuration options](/influxdb3/enterprise/reference/config-options/#compaction-multipliers)_ | +| | `--license-email` | _See [configuration options](/influxdb3/enterprise/reference/config-options/#license-email)_ | | | `--preemptive-cache-age` | _See [configuration options](/influxdb3/enterprise/reference/config-options/#preemptive-cache-age)_ | | | `--parquet-mem-cache-size-mb` | _See [configuration options](/influxdb3/enterprise/reference/config-options/#parquet-mem-cache-size-mb)_ | | | `--parquet-mem-cache-prune-percentage` | _See [configuration options](/influxdb3/enterprise/reference/config-options/#parquet-mem-cache-prune-percentage)_ | | | `--parquet-mem-cache-prune-interval` | _See [configuration options](/influxdb3/enterprise/reference/config-options/#parquet-mem-cache-prune-interval)_ | | | `--disable-parquet-mem-cache` | _See [configuration options](/influxdb3/enterprise/reference/config-options/#disable-parquet-mem-cache)_ | +| | `--parquet-mem-cache-query-path-duration` | _See [configuration options](/influxdb3/enterprise/reference/config-options/#parquet-mem-cache-query-path-duration)_ | | | `--last-cache-eviction-interval` | _See [configuration options](/influxdb3/enterprise/reference/config-options/#last-cache-eviction-interval)_ | | | `--distinct-cache-eviction-interval` | _See [configuration options](/influxdb3/enterprise/reference/config-options/#distinct-cache-eviction-interval)_ | | | `--plugin-dir` | _See [configuration options](/influxdb3/enterprise/reference/config-options/#plugin-dir)_ | +| | `--force-snapshot-mem-threshold` | _See [configuration options](/influxdb3/enterprise/reference/config-options/#force-snapshot-mem-threshold)_ | +| | `--virtual-env-location` | _See [configuration options](/influxdb3/enterprise/reference/config-options/#virtual-env-location)_ | +| | `--package-manager` | _See [configuration options](/influxdb3/enterprise/reference/config-options/#package-manager)_ | +| | `--query-file-limit` | _See [configuration options](/influxdb3/enterprise/reference/config-options/#query-file-limit)_ | {{< caption >}} {{< req text="\* Required options" >}} diff --git a/content/influxdb3/enterprise/reference/config-options.md b/content/influxdb3/enterprise/reference/config-options.md index c062e5eb6..ed3b92a63 100644 --- a/content/influxdb3/enterprise/reference/config-options.md +++ b/content/influxdb3/enterprise/reference/config-options.md @@ -55,6 +55,8 @@ influxdb3 serve - [data-dir](#data-dir) - [node-id](#node-id) - [mode](#mode) + - [license-email](#license-email) + - [query-file-limit](#query-file-limit) - [AWS](#aws) - [aws-access-key-id](#aws-access-key-id) - [aws-secret-access-key](#aws-secret-access-key) @@ -135,11 +137,14 @@ influxdb3 serve - [parquet-mem-cache-prune-percentage](#parquet-mem-cache-prune-percentage) - [parquet-mem-cache-prune-interval](#parquet-mem-cache-prune-interval) - [disable-parquet-mem-cache](#disable-parquet-mem-cache) + - [parquet-mem-cache-query-path-duration](#parquet-mem-cache-query-path-duration) - [last-cache-eviction-interval](#last-cache-eviction-interval) - [distinct-cache-eviction-interval](#distinct-cache-eviction-interval) -- [Plugins](#plugins) +- [Processing engine](#processing-engine) - [plugin-dir](#plugin-dir) - + - [virtual-env-location](#virtual-env-location) + - [package-manager](#package-manager) + --- ### General @@ -149,6 +154,8 @@ influxdb3 serve - [data-dir](#data-dir) - [node-id](#node-id) - [mode](#mode) +- [license-email](#license-email) +- [query-file-limit](#query-file-limit) #### object-store @@ -208,6 +215,40 @@ This option supports the following values: --- +#### license-email + +Specifies the email address to associate with your {{< product-name >}} license +and automatically responds to the interactive email prompt when the server starts. + +| influxdb3 serve option | Environment variable | +| :--------------------- | :----------------------------------- | +| `--license-email` | `INFLUXDB3_ENTERPRISE_LICENSE_EMAIL` | + +--- + +#### query-file-limit + +Limits the number of Parquet files a query can access. +If a query attempts to read more than this limit, InfluxDB returns an error. + +**Default:** `432` + +You can increase this limit to allow more files to be queried, but be aware of +the following side-effects: + +- Degraded query performance for queries that read more Parquet files +- Increased memory usage +- Your system potentially killing the `influxdb3` process due to Out-of-Memory + (OOM) errors +- If using object storage to store data, many GET requests to access the data + (as many as 2 requests per file) + +| influxdb3 serve option | Environment variable | +| :--------------------- | :--------------------------- | +| `--query-file-limit` | `INFLUXDB3_QUERY_FILE_LIMIT` | + +--- + ### AWS - [aws-access-key-id](#aws-access-key-id) @@ -1051,6 +1092,7 @@ compactor can merge into larger generations. - [parquet-mem-cache-prune-percentage](#parquet-mem-cache-prune-percentage) - [parquet-mem-cache-prune-interval](#parquet-mem-cache-prune-interval) - [disable-parquet-mem-cache](#disable-parquet-mem-cache) +- [parquet-mem-cache-query-path-duration](#parquet-mem-cache-query-path-duration) - [last-cache-eviction-interval](#last-cache-eviction-interval) - [distinct-cache-eviction-interval](#distinct-cache-eviction-interval) @@ -1113,10 +1155,23 @@ Disables the in-memory Parquet cache. By default, the cache is enabled. --- +#### parquet-mem-cache-query-path-duration + +Specifies the duration to check if Parquet files pulled in query path +require caching, expressed as a human-readable duration (starting from _now_)--for example: `5h`, `3d`. + +**Default:** `5h` + +| influxdb3 serve option | Environment variable | +| :---------------------------- | :------------------------------------ | +| `--parquet-mem-cache-query-path-duration` | `INFLUXDB3_PARQUET_MEM_CACHE_QUERY_PATH_DURATION` | + +--- + #### last-cache-eviction-interval Specifies the interval to evict expired entries from the Last-N-Value cache, -expressed as a human-readable time--for example: `20s`, `1m`, `1h`. +expressed as a human-readable duration--for example: `20s`, `1m`, `1h`. **Default:** `10s` @@ -1129,7 +1184,7 @@ expressed as a human-readable time--for example: `20s`, `1m`, `1h`. #### distinct-cache-eviction-interval Specifies the interval to evict expired entries from the distinct value cache, -expressed as a human-readable time--for example: `20s`, `1m`, `1h`. +expressed as a human-readable duration--for example: `20s`, `1m`, `1h`. **Default:** `10s` @@ -1139,9 +1194,11 @@ expressed as a human-readable time--for example: `20s`, `1m`, `1h`. --- -### Plugins +### Processing engine - [plugin-dir](#plugin-dir) +- [virtual-env-location](#virtual-env-location) +- [package-manager](#package-manager) #### plugin-dir @@ -1151,3 +1208,25 @@ Specifies the local directory that contains Python plugins and their test files. | :--------------------- | :--------------------- | | `--plugin-dir` | `INFLUXDB3_PLUGIN_DIR` | +--- + +#### virtual-env-location + +Specifies the location of the Python virtual environment that the processing +engine uses. + +| influxdb3 serve option | Environment variable | +| :----------------------- | :--------------------- | +| `--virtual-env-location` | `VIRTUAL_ENV_LOCATION` | + +--- + +#### package-manager + +Specifies the Python package manager that the processing engine uses. + +**Default:** `10s` + +| influxdb3 serve option | Environment variable | +| :--------------------- | :------------------- | +| `--package-manager` | `PACKAGE_MANAGER` |