Influxdata
/
influxdb
mirror of https://github.com/influxdata/influxdb.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

feat: build and use python-build-standalone with official builds (#25969) 

* feat(ci): fetch and configure for python-build-standalone binaries

* fix: make the process engine usable on windows

* feat(ci): build with python-build-standalone (and drop musl)

* fix(ci): set rpath on Linux and libpath on OSX in ci

* fix: set PYTHONHOME everywhere and PYTHONPATH on Windows

* chore(ci): update to use more recent ci-packager-next

* fix(ci): adjust validate to allow certain dynamically linked libraries

* chore: remove install_influxdb.sh (using install_influxdb3.sh instead)

* chore(install_influxdb3.sh): update for processing engine and release builds

* fix: temporarily use rpm --nodeps until compile with old GLIBC

* feat(ci): build docker with python-build-standalone

* chore: add README_processing_engine.md

* chore: add a few more details to README_processing_engine.md

* fix(ci): use patchelf --set-rpath

Not all patchelf versions support --add-rpath for appending to the
RPATH, but --set-path can be used with a colon-separated list. Use
--set-rpath first for maximum compatibility.

* chore: update README_processing_engine.md for standalone local builds

* fix(Dockerfile): also use patchelf --set-rpath

* chore: update code comment for accuracy

* chore: typos, grammar and formatting change in README_processing_engine.md

* chore: update README_processing_engine.md for Docker arm64 (thanks Jackson)
pull/26021/head
Jamie Strandboge 2025-02-13 15:16:05 -07:00 committed by GitHub
parent 962e31df47
commit ccd5d22aab
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 899 additions and 443 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

117
.circleci/config.yml
View File

@ -213,6 +213,35 @@ jobs:
name: cargo nextest name: cargo nextest
command: TEST_LOG= RUST_LOG=info RUST_LOG_SPAN_EVENTS=full RUST_BACKTRACE=1 cargo nextest run --workspace --failure-output immediate-final --no-fail-fast command: TEST_LOG= RUST_LOG=info RUST_LOG_SPAN_EVENTS=full RUST_BACKTRACE=1 cargo nextest run --workspace --failure-output immediate-final --no-fail-fast
# Fetch python-build-standalone for official builds
fetch-python:
machine:
image: ubuntu-2204:current
resource_class: medium
# environment variables for python-build-standalone. Should correspond to
# https://github.com/astral-sh/python-build-standalone/releases. See
# scripts/fetch-python-standalone.bash for details. This should match the
# 'build-docker' job, below.
environment:
PBS_DATE: "20250106"
PBS_VERSION: "3.11.11"
steps:
- checkout
- run:
name: pull Python Build Standalone
command: |
echo "PBS_DATE=$PBS_DATE"
.circleci/scripts/fetch-python-standalone.bash \
"python-artifacts" \
"$PBS_DATE" \
"$PBS_VERSION"
- store_artifacts:
path: python-artifacts
- persist_to_workspace:
root: .
paths:
- python-artifacts
# Build a dev binary. # Build a dev binary.
# #
# Compiles a binary with the default ("dev") cargo profile from the influxdb3 source # Compiles a binary with the default ("dev") cargo profile from the influxdb3 source
@ -220,7 +249,7 @@ jobs:
# Build a dev binary. # Build a dev binary.
build-dev: build-dev:
docker: docker:
- image: us-east1-docker.pkg.dev/influxdata-team-edge/ci-support/ci-cross-influxdb3:latest - image: us-east1-docker.pkg.dev/influxdata-team-edge/ci-support/ci-cross-influxdb3@sha256:63726f571865bfb13232006bbca7aac42d2178f4c19a3526a7e5ee02ada836f8
auth: auth:
username: _json_key username: _json_key
password: $CISUPPORT_GCS_AUTHORIZATION password: $CISUPPORT_GCS_AUTHORIZATION
@ -242,12 +271,23 @@ jobs:
type: string type: string
steps: steps:
- checkout - checkout
- attach_workspace:
at: /tmp/workspace
- run:
name: Extract python for this target
command: |
tar -C /tmp/workspace/python-artifacts -zxvf /tmp/workspace/python-artifacts/all.tar.gz ./<< parameters.target >>
- run:
name: Show PYO3_CONFIG_FILE
command: cat /tmp/workspace/python-artifacts/<< parameters.target >>/pyo3_config_file.txt
- run: - run:
name: Install Target name: Install Target
command: rustup target add << parameters.target >> command: rustup target add << parameters.target >>
- run: - run:
name: Cargo build name: Cargo build
command: target-env cargo build --target=<< parameters.target >> --workspace command: |
export PYO3_CONFIG_FILE=/tmp/workspace/python-artifacts/<< parameters.target >>/pyo3_config_file.txt
target-env cargo build --target=<< parameters.target >> --features="system-py" --workspace
- when: - when:
condition: condition:
not: not:
@ -270,7 +310,7 @@ jobs:
# Compile cargo "release" profile binaries for influxdb3 edge releases # Compile cargo "release" profile binaries for influxdb3 edge releases
build-release: build-release:
docker: docker:
- image: us-east1-docker.pkg.dev/influxdata-team-edge/ci-support/ci-cross-influxdb3:latest - image: us-east1-docker.pkg.dev/influxdata-team-edge/ci-support/ci-cross-influxdb3@sha256:63726f571865bfb13232006bbca7aac42d2178f4c19a3526a7e5ee02ada836f8
auth: auth:
username: _json_key username: _json_key
password: $CISUPPORT_GCS_AUTHORIZATION password: $CISUPPORT_GCS_AUTHORIZATION
@ -295,19 +335,59 @@ jobs:
default: release default: release
steps: steps:
- checkout - checkout
- attach_workspace:
at: /tmp/workspace
- run:
name: Extract python for this target
command: |
tar -C /tmp/workspace/python-artifacts -zxvf /tmp/workspace/python-artifacts/all.tar.gz ./<< parameters.target >>
- run:
name: Show PYO3_CONFIG_FILE
command: cat /tmp/workspace/python-artifacts/<< parameters.target >>/pyo3_config_file.txt
- run: - run:
name: Install Target name: Install Target
command: rustup target add << parameters.target >> command: rustup target add << parameters.target >>
- run: - run:
name: Cargo release build name: Cargo release build
command: target-env cargo build --target=<< parameters.target >> --profile=<< parameters.profile >> --workspace command: |
export PYO3_CONFIG_FILE=/tmp/workspace/python-artifacts/<< parameters.target >>/pyo3_config_file.txt
target-env cargo build --target=<< parameters.target >> --features="system-py" --profile=<< parameters.profile >> --workspace
# linking might take a while and doesn't produce CLI output # linking might take a while and doesn't produce CLI output
no_output_timeout: 30m no_output_timeout: 30m
- when:
condition:
or:
- equal: [ << parameters.target >>, aarch64-unknown-linux-gnu ]
- equal: [ << parameters.target >>, x86_64-unknown-linux-gnu ]
steps:
- run:
# XXX: better to use 'cargo:rustc-link-arg=-Wl,-rpath,$ORIGIN/python/lib'
name: adjust RPATH for linux
command: |
# tarballs need $ORIGIN/python/lib, deb/rpm need $ORIGIN/../lib/influxdb3/python/lib
echo "Running: patchelf --set-rpath '$ORIGIN/python/lib:$ORIGIN/../lib/influxdb3/python/lib' '${PWD}/target/<< parameters.target >>/<< parameters.profile >>/influxdb3'"
patchelf --set-rpath '$ORIGIN/python/lib:$ORIGIN/../lib/influxdb3/python/lib' "${PWD}/target/<< parameters.target >>/<< parameters.profile >>/influxdb3"
- when:
condition:
equal: [ << parameters.target >>, aarch64-apple-darwin ]
steps:
- run:
# XXX: better to use 'cargo:rustc-link-arg=-Wl,-rpath,@executable_path/python/lib'
name: adjust LC_LOAD_DYLIB path for darwin
command: |
export PBS_LIBPYTHON=$(grep '^lib_name=' /tmp/workspace/python-artifacts/<< parameters.target >>/pyo3_config_file.txt | cut -d = -f 2)
echo "Running: /osxcross/bin/aarch64-apple-darwin22.2-install_name_tool -change '/install/lib/lib${PBS_LIBPYTHON}.dylib' '@executable_path/python/lib/lib${PBS_LIBPYTHON}.dylib' '${PWD}/target/<< parameters.target >>/<< parameters.profile >>/influxdb3'"
/osxcross/bin/aarch64-apple-darwin22.2-install_name_tool -change "/install/lib/lib${PBS_LIBPYTHON}.dylib" "@executable_path/python/lib/lib${PBS_LIBPYTHON}.dylib" "${PWD}/target/<< parameters.target >>/<< parameters.profile >>/influxdb3"
# re-sign after install_name_tool since osxcross won't do it
echo "Running: /usr/local/bin/rcodesign sign '${PWD}/target/<< parameters.target >>/<< parameters.profile >>/influxdb3'"
/usr/local/bin/rcodesign sign "${PWD}/target/<< parameters.target >>/<< parameters.profile >>/influxdb3"
- run: - run:
name: tar and gzip build artifacts name: tar and gzip build artifacts
command: | command: |
mkdir -p artifacts mkdir -p artifacts
tar --ignore-failed-read -czvf "${PWD}/artifacts/influxdb3-core_<< parameters.target >>.tar.gz" -C "${PWD}/target/<< parameters.target >>/<< parameters.profile >>" influxdb3{,.exe} tar --ignore-failed-read -cvf "${PWD}/artifacts/influxdb3-core_<< parameters.target >>.tar" -C "${PWD}/target/<< parameters.target >>/<< parameters.profile >>" influxdb3{,.exe}
tar --ignore-failed-read -rvf "${PWD}/artifacts/influxdb3-core_<< parameters.target >>.tar" -C "/tmp/workspace/python-artifacts/<< parameters.target >>" python
gzip "${PWD}/artifacts/influxdb3-core_<< parameters.target >>.tar"
- store_artifacts: - store_artifacts:
path: artifacts path: artifacts
- persist_to_workspace: - persist_to_workspace:
@ -316,7 +396,7 @@ jobs:
- artifacts - artifacts
build-packages: build-packages:
docker: docker:
- image: us-east1-docker.pkg.dev/influxdata-team-edge/ci-support/ci-packager-next:latest - image: us-east1-docker.pkg.dev/influxdata-team-edge/ci-support/ci-packager-next@sha256:db0cd91a5445c4287154cea1d4d5566735cb0d3b7b9e2a95724a83f9d979d497
auth: auth:
username: _json_key username: _json_key
password: $CISUPPORT_GCS_AUTHORIZATION password: $CISUPPORT_GCS_AUTHORIZATION
@ -457,6 +537,13 @@ jobs:
image_name: image_name:
type: string type: string
default: influxdb3-core default: influxdb3-core
# environment variables for python-build-standalone. Should correspond to
# https://github.com/astral-sh/python-build-standalone/releases. See
# scripts/fetch-python-standalone.bash for details. This should match
# the 'fetch-python' job, above.
environment:
PBS_DATE: "20250106"
PBS_VERSION: "3.11.11"
machine: machine:
image: default image: default
resource_class: << parameters.resource_class >> resource_class: << parameters.resource_class >>
@ -477,6 +564,8 @@ jobs:
"influxdb3" \ "influxdb3" \
"aws,gcp,azure,jemalloc_replacing_malloc,tokio_console,system-py" \ "aws,gcp,azure,jemalloc_replacing_malloc,tokio_console,system-py" \
"<< parameters.image_name >>:latest-<< parameters.platform >>" \ "<< parameters.image_name >>:latest-<< parameters.platform >>" \
"$PBS_DATE" \
"$PBS_VERSION" \
"<< parameters.platform >>" \ "<< parameters.platform >>" \
"$DOCKER_PROFILE" "$DOCKER_PROFILE"
@ -519,6 +608,8 @@ workflows:
version: 2 version: 2
snapshot: snapshot:
jobs: jobs:
- fetch-python:
<<: *main_filter
- build-release: - build-release:
<<: *main_filter <<: *main_filter
name: build-snapshot-<< matrix.target >> name: build-snapshot-<< matrix.target >>
@ -529,10 +620,10 @@ workflows:
target: target:
- aarch64-apple-darwin - aarch64-apple-darwin
- aarch64-unknown-linux-gnu - aarch64-unknown-linux-gnu
- aarch64-unknown-linux-musl
- x86_64-pc-windows-gnu - x86_64-pc-windows-gnu
- x86_64-unknown-linux-gnu - x86_64-unknown-linux-gnu
- x86_64-unknown-linux-musl requires:
- fetch-python
- build-packages: - build-packages:
<<: *main_filter <<: *main_filter
requires: requires:
@ -559,6 +650,8 @@ workflows:
<<: *any_filter <<: *any_filter
- cargo-audit: - cargo-audit:
<<: *any_filter <<: *any_filter
- fetch-python:
<<: *any_filter
- test: - test:
<<: *any_filter <<: *any_filter
- build-dev: - build-dev:
@ -573,10 +666,10 @@ workflows:
target: target:
- aarch64-apple-darwin - aarch64-apple-darwin
- aarch64-unknown-linux-gnu - aarch64-unknown-linux-gnu
- aarch64-unknown-linux-musl
- x86_64-pc-windows-gnu - x86_64-pc-windows-gnu
- x86_64-unknown-linux-gnu - x86_64-unknown-linux-gnu
- x86_64-unknown-linux-musl requires:
- fetch-python
- doc: - doc:
<<: *any_filter <<: *any_filter
- build-release: - build-release:
@ -587,10 +680,10 @@ workflows:
target: target:
- aarch64-apple-darwin - aarch64-apple-darwin
- aarch64-unknown-linux-gnu - aarch64-unknown-linux-gnu
- aarch64-unknown-linux-musl
- x86_64-pc-windows-gnu - x86_64-pc-windows-gnu
- x86_64-unknown-linux-gnu - x86_64-unknown-linux-gnu
- x86_64-unknown-linux-musl requires:
- fetch-python
- build-packages: - build-packages:
<<: *release_filter <<: *release_filter
requires: requires:

9
.circleci/packages/config.yaml
View File

@ -5,19 +5,19 @@ version:
value: '3.0.0+snapshot-{{env.CIRCLE_SHA1[:8]}}' value: '3.0.0+snapshot-{{env.CIRCLE_SHA1[:8]}}'
sources: sources:
- binary: /tmp/workspace/artifacts/influxdb3-core_x86_64-unknown-linux-musl.tar.gz - binary: /tmp/workspace/artifacts/influxdb3-core_x86_64-unknown-linux-gnu.tar.gz
target: artifacts/ target: artifacts/
arch: amd64 arch: amd64
plat: linux plat: linux
- binary: /tmp/workspace/artifacts/influxdb3-core_aarch64-unknown-linux-musl.tar.gz - binary: /tmp/workspace/artifacts/influxdb3-core_aarch64-unknown-linux-gnu.tar.gz
target: artifacts/ target: artifacts/
arch: arm64 arch: arm64
plat: linux plat: linux
- binary: /tmp/workspace/artifacts/influxdb3-core_aarch64-apple-darwin.tar.gz - binary: /tmp/workspace/artifacts/influxdb3-core_aarch64-apple-darwin.tar.gz
target: artifacts/ target: artifacts/
arch: amd64 arch: arm64
plat: darwin plat: darwin
- binary: /tmp/workspace/artifacts/influxdb3-core_x86_64-pc-windows-gnu.tar.gz - binary: /tmp/workspace/artifacts/influxdb3-core_x86_64-pc-windows-gnu.tar.gz
@ -37,6 +37,9 @@ packages:
binaries: binaries:
- influxdb3 - influxdb3
- influxdb3.exe - influxdb3.exe
python-runtimes:
- source: python
target: usr/lib/influxdb3
extras: extras:
- source: LICENSE-APACHE - source: LICENSE-APACHE
target: usr/share/influxdb3/LICENSE-APACHE target: usr/share/influxdb3/LICENSE-APACHE

0
.circleci/packages/influxdb3/fs/usr/lib/influxdb3/.keepdir Normal file
View File

10
.circleci/scripts/docker_build_release.bash
View File

@ -5,8 +5,10 @@ set -euo pipefail
readonly PACKAGE="$1" readonly PACKAGE="$1"
readonly FEATURES="$2" readonly FEATURES="$2"
readonly TAG="$3" readonly TAG="$3"
readonly ARCH="${4:-amd64}" # Default to amd64 if not specified readonly PBS_DATE="$4"
readonly PROFILE="${5:-release}" # Default to release if not specified readonly PBS_VERSION="$5"
readonly ARCH="${6:-amd64}" # Default to amd64 if not specified
readonly PROFILE="${7:-release}" # Default to release if not specified
RUST_VERSION="$(sed -E -ne 's/channel = "(.*)"/\1/p' rust-toolchain.toml)" RUST_VERSION="$(sed -E -ne 's/channel = "(.*)"/\1/p' rust-toolchain.toml)"
COMMIT_SHA="$(git rev-parse HEAD)" COMMIT_SHA="$(git rev-parse HEAD)"
@ -24,6 +26,8 @@ exec docker buildx build \
--build-arg RUST_VERSION="$RUST_VERSION" \ --build-arg RUST_VERSION="$RUST_VERSION" \
--build-arg PACKAGE="$PACKAGE" \ --build-arg PACKAGE="$PACKAGE" \
--build-arg PROFILE="$PROFILE" \ --build-arg PROFILE="$PROFILE" \
--build-arg PBS_DATE="$PBS_DATE" \
--build-arg PBS_VERSION="$PBS_VERSION" \
--platform "$PLATFORM" \ --platform "$PLATFORM" \
--label org.opencontainers.image.created="$NOW" \ --label org.opencontainers.image.created="$NOW" \
--label org.opencontainers.image.url="$REPO_URL" \ --label org.opencontainers.image.url="$REPO_URL" \
@ -35,4 +39,4 @@ exec docker buildx build \
--label com.influxdata.image.package="$PACKAGE" \ --label com.influxdata.image.package="$PACKAGE" \
--progress plain \ --progress plain \
--tag "$TAG" \ --tag "$TAG" \
. .

170
.circleci/scripts/fetch-python-standalone.bash Executable file
View File

@ -0,0 +1,170 @@
#!/bin/bash
set -euo pipefail
# See https://github.com/astral-sh/python-build-standalone/releases
# USAGE:
# fetch-python-standalone.bash <dir> <RELEASE DATE> <RELEASE VERSION>
#
# Eg:
# $ fetch-python-standalone.bash ./python-artifacts 20250106 3.11.11
#
# This script is meant to be called by CircleCI such that the specified <dir>
# is persisted to a workspace that is later attached at /tmp/workspace/<dir>.
# In this manner, build script can do something like:
# PYO3_CONFIG_FILE=/tmp/workspace/<dir>/pyo3_config_file.txt cargo build...
readonly DOWNLOAD_DIR="$1"
# URLs are constructed from this. Eg:
# https://github.com/astral-sh/.../<PBS_DATE>/cpython-<PBS_VERSION>+<PBS_DATE>-<ARCH>...
readonly PBS_DATE="$2"
readonly PBS_VERSION="$3"
readonly PBS_MAJ_MIN=${PBS_VERSION%.*}
readonly PBS_TOP_DIR="/tmp/workspace"
# Official influxdb3 builds use python-build-standalone since it:
# - is built to run well as an embedded interpreter
# - has a good upstream maintenance story (https://github.com/astral-sh) with
# lots of users and corporate sponsor
# - should deliver a consistent experience across OSes and architectures
#
# python-build-standalone provides many different builds. Official influxdb3
# build targets:
# - aarch64-apple-darwin
# - aarch64-unknown-linux-gnu
# - x86_64-unknown-linux-gnu
# - x86_64-pc-windows-msvc-shared
#
# Note: musl builds of python-build-standablone currently (as of 2025-02-04)
# have limitations:
# - don't support importing bre-built python wheels (must compile and link 3rd
# party extensions into the binary (influxdb3)
# - historical performance issues with python and musl
# - availability limited to x86_64 (no aarch64)
#
# References
# - https://github.com/astral-sh/python-build-standalone/blob/main/docs/distributions.rst
# - https://github.com/astral-sh/python-build-standalone/blob/main/docs/running.rst
# - https://edu.chainguard.dev/chainguard/chainguard-images/about/images-compiled-programs/glibc-vs-musl/#python-builds
# - https://pythonspeed.com/articles/alpine-docker-python/
readonly TARGETS="aarch64-apple-darwin aarch64-unknown-linux-gnu x86_64-unknown-linux-gnu x86_64-pc-windows-msvc-shared"
fetch() {
target="$1"
suffix="${2}"
if [ "${suffix}" = "full.tar.zst" ]; then
if [ "${target}" = "x86_64-pc-windows-msvc-shared" ]; then
suffix="pgo-${2}"
else
suffix="debug-${2}"
fi
fi
binary="cpython-${PBS_VERSION}+${PBS_DATE}-${target}-${suffix}"
url="https://github.com/astral-sh/python-build-standalone/releases/download/${PBS_DATE}/${binary}"
echo "Downloading ${binary}"
curl --proto '=https' --tlsv1.2 -sS -L "$url" -o "${DOWNLOAD_DIR}/${binary}"
echo "Downloading ${binary}.sha256"
curl --proto '=https' --tlsv1.2 -sS -L "${url}.sha256" -o "${DOWNLOAD_DIR}/${binary}.sha256"
dl_sha=$(cut -d ' ' -f 1 "${DOWNLOAD_DIR}/${binary}.sha256")
if [ -z "$dl_sha" ]; then
echo "Could not find properly formatted SHA256 in '${DOWNLOAD_DIR}/${binary}.sha256'"
exit 1
fi
printf "Verifying %s: " "${binary}"
ch_sha=$(sha256sum "${DOWNLOAD_DIR}/${binary}" | cut -d ' ' -f 1)
if [ "$ch_sha" = "$dl_sha" ]; then
echo "OK"
else
echo "ERROR (${ch_sha} != ${dl_sha})"
exit 1
fi
echo "Unpacking ${binary} to '${DOWNLOAD_DIR}'"
UNPACK_DIR="${DOWNLOAD_DIR}/${target}"
if [ "${target}" = "x86_64-pc-windows-msvc-shared" ]; then
UNPACK_DIR="${DOWNLOAD_DIR}/x86_64-pc-windows-gnu"
fi
mkdir "${UNPACK_DIR}" 2>/dev/null || true
if [[ "${suffix}" = *full.tar.zst ]]; then
# we only need the licensing from the full distribution
tar -C "${UNPACK_DIR}" --zstd -xf "${DOWNLOAD_DIR}/${binary}" python/PYTHON.json python/licenses
mv "${UNPACK_DIR}/python/PYTHON.json" "${UNPACK_DIR}/python/licenses"
else
tar -C "${UNPACK_DIR}" -zxf "${DOWNLOAD_DIR}/${binary}"
fi
echo "Removing ${binary}"
rm -f "${DOWNLOAD_DIR}/${binary}" "${DOWNLOAD_DIR}/${binary}.sha256"
if [[ "${suffix}" = *install_only_stripped.tar.gz ]]; then
echo "Creating ${UNPACK_DIR}/pyo3_config_file.txt"
PYO3_CONFIG_FILE="${UNPACK_DIR}/pyo3_config_file.txt"
PBS_DIR="${PBS_TOP_DIR}"/$(basename "${DOWNLOAD_DIR}")/$(basename "${UNPACK_DIR}")
if [ "${target}" = "x86_64-pc-windows-msvc-shared" ]; then
cat > "${PYO3_CONFIG_FILE}" <<EOM
implementation=CPython
version=${PBS_MAJ_MIN}
shared=true
abi3=false
lib_name=python${PBS_MAJ_MIN//./}
lib_dir=${PBS_DIR}/python/libs
executable=${PBS_DIR}/python/python.exe
pointer_width=64
build_flags=
suppress_build_script_link_lines=false
EOM
else
cat > "${PYO3_CONFIG_FILE}" <<EOM
implementation=CPython
version=${PBS_MAJ_MIN}
shared=true
abi3=false
lib_name=python${PBS_MAJ_MIN}
lib_dir=${PBS_DIR}/python/lib
executable=${PBS_DIR}/python/bin/python3.11
pointer_width=64
build_flags=
suppress_build_script_link_lines=false
EOM
fi
echo "Creating ${UNPACK_DIR}/python/LICENSE.md"
cat > "${UNPACK_DIR}/python/LICENSE.md" <<EOM
This version of InfluxDB is built against and uses the redistributable build of
'python-build-standalone' downloaded from:
${url}
EOM
else
echo "Adding licenses/ provenance to ${UNPACK_DIR}/python/LICENSE.md"
cat >> "${UNPACK_DIR}/python/LICENSE.md" <<EOM
License information for this 'python-build-standalone' build can be found in
the licenses/ sub-folder. The contents of this folder was extracted from:
${url}
References:
- https://github.com/astral-sh/python-build-standalone/releases/tag/${PBS_DATE}
- https://github.com/astral-sh/python-build-standalone/blob/${PBS_DATE}/docs/running.rst#licensing
EOM
fi
echo
}
mkdir "${DOWNLOAD_DIR}"
for t in $TARGETS ; do
fetch "$t" "install_only_stripped.tar.gz" # for runtime
fetch "$t" "full.tar.zst" # for licenses
done
# This speeds up CircleCI
echo "Creating '${DOWNLOAD_DIR}/all.tar.gz'"
cd "${DOWNLOAD_DIR}"
tar -zcf ./.all.tar.gz ./[a-z]*
rm -rf ./[a-z]*
mv ./.all.tar.gz ./all.tar.gz

23
.circleci/scripts/package-validation/validate
View File

@ -45,7 +45,11 @@ install_rpm() {
# see "install_deb" for "update" # see "install_deb" for "update"
yum update -y yum update -y
yum install -y binutils yum install -y binutils
yum localinstall -y "$(realpath "${PACKAGE_PATH}")" # temporary install with rpm --nodeps until we compile with older glibc
#yum localinstall -y "$(realpath "${PACKAGE_PATH}")"
yum install -y shadow-utils # for useradd
yum install -y libxcrypt-compat # for libcrypt.so.1
rpm -ivh --nodeps "$(realpath "${PACKAGE_PATH}")"
} }
case ${PACKAGE_TYPE} case ${PACKAGE_TYPE}
@ -79,12 +83,25 @@ fi
if [[ "${NEEDED:-}" ]] if [[ "${NEEDED:-}" ]]
then then
cat <<'EOF' if echo "$NEEDED" | grep -Eq "Shared library: \[libpython" ; then
# if have libpython, ensure we are only linking things we expect
if echo "$NEEDED" | grep -Ev "Shared library: \[(ld-linux.*|libc|libdl|libgcc_s|libm|libpthread|libpython3.[0-9]{2})\.so" ; then
cat <<'EOF'
ERROR: found unexpected dynamically linked libraries! This may
prevent all platforms from running influxdb3 without
installing these dependencies.
EOF
exit 2
fi
else
# if no libpython, then complain if any are NEEDED
cat <<'EOF'
ERROR: influxdb3 not statically linked! This may prevent all ERROR: influxdb3 not statically linked! This may prevent all
platforms from running influxdb3 without installing platforms from running influxdb3 without installing
separate dependencies. separate dependencies.
EOF EOF
exit 2 exit 2
fi
fi fi
printf 'Finished validating influxdb3!\n' printf 'Finished validating influxdb3!\n'

29
Dockerfile
View File

@ -6,7 +6,7 @@ FROM rust:${RUST_VERSION}-slim-bookworm as build
USER root USER root
RUN apt update \ RUN apt update \
&& apt install --yes binutils build-essential pkg-config libssl-dev clang lld git protobuf-compiler python3 python3-dev python3-pip \ && apt install --yes binutils build-essential curl pkg-config libssl-dev clang lld git patchelf protobuf-compiler zstd \
&& rm -rf /var/lib/{apt,dpkg,cache,log} && rm -rf /var/lib/{apt,dpkg,cache,log}
# Build influxdb3 # Build influxdb3
@ -18,11 +18,24 @@ ARG CARGO_NET_GIT_FETCH_WITH_CLI=false
ARG PROFILE=release ARG PROFILE=release
ARG FEATURES=aws,gcp,azure,jemalloc_replacing_malloc,system-py ARG FEATURES=aws,gcp,azure,jemalloc_replacing_malloc,system-py
ARG PACKAGE=influxdb3 ARG PACKAGE=influxdb3
ARG PBS_DATE=unset
ARG PBS_VERSION=unset
ENV CARGO_INCREMENTAL=$CARGO_INCREMENTAL \ ENV CARGO_INCREMENTAL=$CARGO_INCREMENTAL \
CARGO_NET_GIT_FETCH_WITH_CLI=$CARGO_NET_GIT_FETCH_WITH_CLI \ CARGO_NET_GIT_FETCH_WITH_CLI=$CARGO_NET_GIT_FETCH_WITH_CLI \
PROFILE=$PROFILE \ PROFILE=$PROFILE \
FEATURES=$FEATURES \ FEATURES=$FEATURES \
PACKAGE=$PACKAGE PACKAGE=$PACKAGE \
PBS_TARGET=x86_64-unknown-linux-gnu \
PBS_DATE=$PBS_DATE \
PBS_VERSION=$PBS_VERSION
# obtain python-build-standalone and configure PYO3_CONFIG_FILE
RUN \
sed -i "s/^readonly TARGETS=.*/readonly TARGETS=${PBS_TARGET}/" ./.circleci/scripts/fetch-python-standalone.bash && \
./.circleci/scripts/fetch-python-standalone.bash /influxdb3/python-artifacts "${PBS_DATE}" "${PBS_VERSION}" && \
tar -C /influxdb3/python-artifacts -zxf /influxdb3/python-artifacts/all.tar.gz "./${PBS_TARGET}" && \
sed -i 's#tmp/workspace#influxdb3#' "/influxdb3/python-artifacts/${PBS_TARGET}/pyo3_config_file.txt" && \
cat "/influxdb3/python-artifacts/${PBS_TARGET}/pyo3_config_file.txt"
RUN \ RUN \
--mount=type=cache,id=influxdb3_rustup,sharing=locked,target=/usr/local/rustup \ --mount=type=cache,id=influxdb3_rustup,sharing=locked,target=/usr/local/rustup \
@ -30,16 +43,18 @@ RUN \
--mount=type=cache,id=influxdb3_git,sharing=locked,target=/usr/local/cargo/git \ --mount=type=cache,id=influxdb3_git,sharing=locked,target=/usr/local/cargo/git \
--mount=type=cache,id=influxdb3_target,sharing=locked,target=/influxdb3/target \ --mount=type=cache,id=influxdb3_target,sharing=locked,target=/influxdb3/target \
du -cshx /usr/local/rustup /usr/local/cargo/registry /usr/local/cargo/git /influxdb3/target && \ du -cshx /usr/local/rustup /usr/local/cargo/registry /usr/local/cargo/git /influxdb3/target && \
cargo build --target-dir /influxdb3/target --package="$PACKAGE" --profile="$PROFILE" --no-default-features --features="$FEATURES" && \ PYO3_CONFIG_FILE="/influxdb3/python-artifacts/$PBS_TARGET/pyo3_config_file.txt" cargo build --target-dir /influxdb3/target --package="$PACKAGE" --profile="$PROFILE" --no-default-features --features="$FEATURES" && \
objcopy --compress-debug-sections "target/$PROFILE/$PACKAGE" && \ objcopy --compress-debug-sections "target/$PROFILE/$PACKAGE" && \
cp "/influxdb3/target/$PROFILE/$PACKAGE" /root/$PACKAGE && \ cp "/influxdb3/target/$PROFILE/$PACKAGE" "/root/$PACKAGE" && \
patchelf --set-rpath '$ORIGIN/python/lib:$ORIGIN/../lib/influxdb3/python/lib' "/root/$PACKAGE" && \
cp -a "/influxdb3/python-artifacts/$PBS_TARGET/python" /root/python && \
du -cshx /usr/local/rustup /usr/local/cargo/registry /usr/local/cargo/git /influxdb3/target du -cshx /usr/local/rustup /usr/local/cargo/registry /usr/local/cargo/git /influxdb3/target
FROM debian:bookworm-slim FROM debian:bookworm-slim
RUN apt update \ RUN apt update \
&& apt install --yes ca-certificates gettext-base libssl3 python3 python3-dev python3-pip python3-venv wget curl --no-install-recommends \ && apt install --yes ca-certificates gettext-base libssl3 wget curl --no-install-recommends \
&& rm -rf /var/lib/{apt,dpkg,cache,log} \ && rm -rf /var/lib/{apt,dpkg,cache,log} \
&& groupadd --gid 1500 influxdb3 \ && groupadd --gid 1500 influxdb3 \
&& useradd --uid 1500 --gid influxdb3 --shell /bin/bash --create-home influxdb3 && useradd --uid 1500 --gid influxdb3 --shell /bin/bash --create-home influxdb3
@ -47,6 +62,10 @@ RUN apt update \
RUN mkdir /var/lib/influxdb3 && \ RUN mkdir /var/lib/influxdb3 && \
chown influxdb3:influxdb3 /var/lib/influxdb3 chown influxdb3:influxdb3 /var/lib/influxdb3
RUN mkdir -p /usr/lib/influxdb3
COPY --from=build /root/python /usr/lib/influxdb3/python
RUN chown -R root:root /usr/lib/influxdb3
USER influxdb3 USER influxdb3
RUN mkdir ~/.influxdb3 RUN mkdir ~/.influxdb3

1
Dockerfile.dockerignore
View File

@ -3,5 +3,6 @@ target/
tests/ tests/
docker/ docker/
!.cargo/ !.cargo/
!.circleci/scripts/fetch-python-standalone.bash
!.git/ !.git/
!docker/entrypoint.sh !docker/entrypoint.sh

452
README_processing_engine.md Normal file
View File

@ -0,0 +1,452 @@
The InfluxDB 3 Processing engine is an embedded Python VM for running code
inside the database to process and transform data. This document discusses how
the processing engine is built within InfluxDB. For usage instructions, see:
https://docs.influxdata.com/influxdb3/core/
See the 'Discussion' section for more information on why the processing engine
is implemented the way it is.
## Implementation
InfluxDB uses the [PYO3 crate](https://crates.io/crates/pyo3) to build InfluxDB
with an embedded python and the processing engine is enabled during the build
by specifying the `system-py` feature to `cargo build`. Eg:
```
$ cargo build --features system-py
```
PYO3 will then inspect the system to find a python runtime to build and link
against. The resulting `influxdb3` binary will be dynamically linked to the
`libpython` that PYO3 found during the build. Eg, on a typical Debian or Ubuntu
system, if you install the following, then InfluxDB can be built against the
system's python:
```sh
# build dependencies
$ sudo apt-get install build-essential pkg-config libssl-dev clang lld \
git protobuf-compiler python3 python3-dev python3-pip
# runtime dependencies
$ sudo apt-get install python3 python3-pip python3-venv
# build
$ cargo build --features system-py
```
The choice of python can be influenced by setting the `PYTHONHOME` environment
variable for `cargo build` or creating a `PYO3_CONFIG_FILE` file for more
specialized setups (such as 'Official builds', below). For details, see
https://pyo3.rs/main/building-and-distribution.html
In order for InfluxDB to successfully use the python it was built against, the
same `libpython` version as well as the full runtime environment of the python
install (ie, its standard library) must be available to InfluxDB in a location
that it can find it. Building against the system python can be a reasonable
choice for users who target their builds to a specific release of an OS as
InfluxDB will simply use the installed python from the system.
## Official builds
To provide a consistent, robust and maintained python environment for InfluxDB
that is portable across a range of operating systems, InfluxData's official
InfluxDB is built against a pre-built release of
[python-build-standalone](https://astral.sh/blog/python-build-standalone) (a
CPython standalone python distribution). For a given release of InfluxDB,
official builds will use the same version of python for all install methods and
operating systems.
The following operating systems and architectures are currently supported:
* Linux amd64/arm64 (`tar.gz`, `deb` and `rpm`)
* Darwin arm64 (`tar.gz`)
* Windows amd64 (`zip`)
* Docker (Linux amd64/arm64)
Due to constraints with `python-build-standalone` and statically linking, all
builds are dynamically linked to `python-build-standalone`'s `libpython` as
well as a few OS-specific libraries. Specifically:
* Linux (seen with `ldd` and `strings` on the binary):
* `python-build-standalone` is linked against `glibc` and is compatible with
`glibc` [2.17+](https://github.com/astral-sh/python-build-standalone/blob/main/docs/running.rst#linux)
* `influxdb3` is linked against `libpython` from `python-build-standalone` as
well as `glibc` (currently compatible with `glibc` 2.36+ (though 2.35 is
known to work; future releases will be built against an earlier `glibc`
release to improve compatibility))
* Darwin (seen with `otool -L`; cross-compiled with [osxcross](https://github.com/tpoechtrager/osxcross)):
* `python-build-standalone` is linked against:
* `CoreFoundation.framework/Versions/A/CoreFoundation` compatibility
version 150.0.0
* `libSystem.B.dylib` compatibility version 1.0.0
* `influxdb3` is linked against:
* `CoreFoundation.framework/Versions/A/CoreFoundation` compatibility
version 150.0.0
* `IOKit.framework/Versions/A/IOKit` compatibility version 1.0.0
* `libiconv.2.dylib` compatibility version 7.0.0
* `libobjc.A.dylib` compatibility version 1.0.0
* `libSystem.B.dylib` compatibility version 1.0.0
* `Security.framework/Versions/A/Security` compatibility version 1.0.0
* `SystemConfiguration.framework/Versions/A/SystemConfiguration`
compatibility version 1.0.0
* Windows (seen with `dumpbin /HEADERS ...` and `dumpbin /DEPENDENTS ...`):
* `python-build-standalone` claims [Windows 8/Windows Server 2012](https://github.com/astral-sh/python-build-standalone/blob/main/docs/running.rst#windows) or newer. Specifically, it has:
* 14.42 linker version
* 6.00 operating system version
* 6.00 subsystem version
* `influxdb3` has:
* 2.40 linker version
* 4.00 operating system version
* 5.02 subsystem version
At a high level, the build process for Official builds consists of:
1. downloading an appropriate build of `python-build-standalone` for the
target OS and architecture from https://github.com/astral-sh/python-build-standalone/releases
2. unpacking the `python-build-standalone` build on disk
3. creating a `pyo3` build configuration file to point to the unpacked
directory and setting the `PYO3_CONFIG_FILE` environment variable to point
to it. Eg (on Linux):
```
implementation=CPython
version=3.11
shared=true
abi3=false
lib_name=python3.11
lib_dir=/path/to/python-standalone/python/lib
executable=/path/to/python-standalone/python/bin/python3.11
pointer_width=64
build_flags=
suppress_build_script_link_lines=false
```
PYO3 will try to auto-detect the location which can work well with a system
python, but not with an unpacked `python-build-standalone`. While the
`PYO3_PYTHON` environment variable can be used to point to the unpacked
directory (eg,
`PYO3_PYTHON=/path/to/python-standalone/python/bin/python3`), this was not
sufficient. Defining the build configuration in the `PYO3_CONFIG_FILE`
correctly worked for all supported environments with our current build
process
4. run `PYO3_CONFIG_FILE=/path/to/pyo3_config_file.txt cargo build --features=system-py` to build InfluxDB
5. adjust the library search paths for Linux and Darwin so `libpython` can
found (see 'Discussion', below)
6. create official build artifacts:
* Linux/Darwin `tar.gz` contain `influxdb3` and `python/...`
* Linux `deb` and `rpm` contain `/usr/bin/influxdb3` and
`/usr/lib/influxdb3/python`
* Windows `zip` contains `influxdb3`, `*.dll` files from `python/...` and
`python/...` (see 'Discussion', below)
Licensing information for `python-build-standalone` as distributed by official
builds of InfluxDB can found in the `python/LICENSE.md`.
With the above, `influxdb3` can be run in the normal way. Eg, on Linux:
```
# unpack tarball to /here
$ tar -C /here --strip-components=1 -zxvf /path/to/build/influxdb3-<VERSION>_linux_amd64.tar.gz
# without processing engine
$ /here/influxdb3 serve ...
$ /here/influxdb3 query ...
# with the processing engine without an activated venv
$ mkdir /path/to/plugins
$ /here/influxdb3 serve --plugin-dir /path/to/plugins ... # server
$ /here/influxdb3 create database foo # client
$ /here/influxdb3 test schedule_plugin -d foo testme.py # client
# create a venv
$ /here/python/bin/python3 -m venv /path/to/venv
$ source /path/to/venv/bin/activate
(venv)$ pip install requests
...
(venv)$ deactivate
# start server in the venv
$ source /path/to/venv/bin/activate # server
(venv)$ /here/influxdb3 serve --plugin-dir /path/to/plugins ... # server
... <plugins can now 'import requests'> ...
$ /here/influxdb3 test schedule_plugin -d foo test-requests.py # client
```
### Local development with python-build-standalone
Local development with python-build-standalone currently consists of:
1. download python-build-standalone and unpack it somewhere
* get from https://github.com/astral-sh/python-build-standalone/releases
* based on your host OS, choose one of `aarch64-apple-darwin-install_only_stripped.tar.gz`, `aarch64-unknown-linux-gnu-install_only_stripped.tar.gz`, `x86_64-pc-windows-msvc-shared-install_only_stripped.tar.gz`, `x86_64-unknown-linux-gnu-install_only_stripped.tar.gz`
2. create `pyo3_config_file.txt` to match the unpacked dir and downloaded python version. Eg, if downloaded and unpacked a 3.11.x version to `/tmp/python`:
```
$ cat ./pyo3_config_file.txt
implementation=CPython
version=3.11
shared=true
abi3=false
lib_name=python3.11
lib_dir=/tmp/python/lib
executable=/tmp/python/bin/python3.11
pointer_width=64
build_flags=
suppress_build_script_link_lines=false
```
3. build with:
```
# note: PYO3_CONFIG_FILE must be an absolute path
$ PYO3_CONFIG_FILE=${PWD}/pyo3_config_file.txt cargo build --features "aws,gcp,azure,jemalloc_replacing_malloc,system-py"
```
4. Linux/OSX: patch up the binary to find libpython:
```
# linux
$ patchelf --set-rpath '$ORIGIN/python/lib' ./target/<profile>/influxdb3
# osx (be sure to match the libpython version with what you downloaded)
$ install_name_tool -change '/install/lib/libpython3.11.dylib' '@executable_path/python/lib/libpython3.11.dylib' ./target/<profile>/influxdb3
```
5. Linux/OSX: put the python runtime in the expected location (XXX: may be
possible at run time to see where the libpython we are using is and adjust
the code to base the location of the runtime on that). Eg, if unpacked to
`/tmp/python`:
```
$ test -e ./target/<profile>/python || ln -s /tmp/python ./target/<profile>/python
```
6. run with:
```
$ mkdir -p /path/to/plugin/dir
# linux and osx (if can't find libpython or the runtime, check previous steps)
$ ./target/<profile>/influxdb3 ... --plugin-dir /path/to/plugin/dir
# windows requires moving the binary into the python-build-standalone unpack directory
$ cp ./target/<profile>/influxdb3 \path\to\python-standalone\python
# run influxdb with
$ \path\to\python-standalone\python\influxdb3.exe ... --plugin-dir \path\to\plugin\dir
```
## Discussion
### Why python-build-standalone?
`python-build-standalone` is designed to be
[portable](https://astral.sh/blog/python-build-standalone#whats-a-standalone-python-distribution),
[maintained](https://astral.sh/blog/python-build-standalone#the-future-of-standalone-python-distributions)
and [permissively licensed](https://github.com/astral-sh/python-build-standalone/blob/main/docs/running.rst#licensing).
It is purpose-built for embedding and being redistributable and has a good
upstream maintenance story (https://github.com/astral-sh) with lots of users
and a corporate sponsor.
An alternative to using a standalone python distribution is to use the system
python. While this can be a reasonable choice on systems where the python
version and installation locations can be relied upon, it is not a good choice
for official builds since users would have to ensure they had a python
installation that met InfluxDB's requirements and because the myriad of
operating systems, architectures and installed python versions would be a
problem to support.
By choosing `python-build-standalone`, InfluxDB should deliver a consistent
experience across OSes and architectures for all users as well as providing a
reasonable maintenance story.
### Which builds of python-build-standalone are used?
`python-build-standalone` provides [many different builds](https://github.com/astral-sh/python-build-standalone/blob/main/docs/distributions.rst).
Official InfluxDB builds use the following `python-build-standalone`
[recommended](https://github.com/astral-sh/python-build-standalone/blob/main/docs/running.rst)
builds:
* `aarch64-apple-darwin-install_only_stripped.tar.gz`
* `aarch64-unknown-linux-gnu-install_only_stripped.tar.gz`
* `x86_64-unknown-linux-gnu-install_only_stripped.tar.gz`
* `x86_64-pc-windows-msvc-shared-install_only_stripped.tar.gz`
### How will InfluxData maintain the embedded interpreter?
The https://github.com/astral-sh project performs timely builds of CPython
micro-releases for `python-build-standalone` based on the release cadence of
upstream Python. InfluxData need only update the build to pull in the new
micro-release for security and maintenance releases. This is done by updating
the `PBS_DATE` and `PBS_VERSION` environment variables in
`.circleci/config.yaml`. See that file and
`.circleci/scripts/fetch-python-standalone.bash` for details.
astral-sh creates new builds for CPython minor releases as they become
available from upstream Python. Updating the official builds to pull in a new
minor release is straightforward, but processes for verifying builds of
InfluxDB with the new `python-build-standalone` minor release are TBD.
### How is python-build-standalone licensed?
Release builds of `python-build-standalone` are
[permissively licensed](https://github.com/astral-sh/python-build-standalone/blob/main/docs/running.rst#licensing)
and contain no copyleft code.
The licensing information from release builds of `python-build-standalone` are
obtained by extracting the `python/PYTHON.json` and `python/licenses/*` files
from the `<arch>-debug-full.tar.zst` (Linux/Darwin) and
`<arch>-pgo-full.tar.zst` release tarballs, placing them in the
`python/licenses` directory of the InfluxDB build and generating a
`python/LICENSE.md` file with provenance information.
Linux builds are dynamically linked against [`glibc`](https://www.gnu.org/software/libc/)
(which is permitted by the LGPL without copyleft attachment). InfluxDB does not
statically link against `glibc` nor does it redistribute `libc` (et al) in
official builds.
### Why not just statically link with, eg, MUSL?
In an ideal world, InfluxDB would build against a version of
`python-build-standalone` and statically link against it and not have to worry
about dynamic library compatibility. Unfortunately, this is not possible for
many reasons:
* static `python-build-standalone` builds for Darwin are [not available](https://github.com/astral-sh/python-build-standalone/blob/main/docs/running.rst) and doing so may have [license implications](https://github.com/astral-sh/python-build-standalone/blob/main/docs/quirks.rst#linking-static-library-on-macos)
* static `python-build-standalone` builds for Windows are [not stable](https://github.com/astral-sh/python-build-standalone/blob/main/docs/running.rst) and considered [brittle](https://github.com/astral-sh/python-build-standalone/blob/main/docs/quirks.rst#windows-static-distributions-are-extremely-brittle)
* static `python-build-standalone` builds for Linux/arm64 (aarch64) are [not available](https://github.com/astral-sh/python-build-standalone/blob/main/docs/running.rst)
* static `python-build-standalone` builds for Linux/amd64 (x86_64) are
available using MUSL libc, but:
* because they are static, they [cannot load compiled Python extensions](https://github.com/astral-sh/python-build-standalone/blob/main/docs/running.rst)
(aka, 'wheels' that have compiled C, Rust, etc code instead of pure python)
outside of the Python standard library, greatly diminishing the utility of
the processing engine. This is a limitation of [ELF](https://github.com/astral-sh/python-build-standalone/blob/main/docs/quirks.rst#static-linking-of-musl-libc-prevents-extension-module-library-loading)
* there are historical [performance issues](https://edu.chainguard.dev/chainguard/chainguard-images/about/images-compiled-programs/glibc-vs-musl/#python-builds) with python and MUSL
It is theoretically possible to statically link `glibc`, but in practice this
is technically very problematic and statically linking `glibc` has copyleft
attachment.
### What about alpine?
Because MUSL can't be used with `python-build-standalone` without crippling the
InfluxDB processing engine, MUSL builds that are compatible with Alpine are not
available at this time. Alpine users can choose one of:
* build InfluxDB locally on Alpine against Alpine's system python
* run official InfluxDB within a chroot that contains `glibc`
* run official InfluxDB with [gcompat](https://git.adelielinux.org/adelie/gcompat) (untested)
See https://wiki.alpinelinux.org/wiki/Running_glibc_programs for details.
InfluxData may provide Alpine builds at a future date.
### GLIBC portability is a problem. How will you address that?
`glibc` is designed with portability and uses 'compat symbols' to achieve
[backward compatibility](https://developers.redhat.com/blog/2019/08/01/how-the-gnu-c-library-handles-backward-compatibility).
Most 3rd party applications for Linux use the system's `glibc` in some fashion
and this is possible because of 'compat symbols' and this has worked very well
for many, many years.
In essence, 'compat symbols' let `glibc` and the linker choose a particular
implementation of the function. All symbols in `glibc` are versioned and when a
library function changes in an incompatible way, `glibc` keeps the old
implementation in place (with the old symbol version) while adding the new
implementation with a new symbol version. In this manner, if an application is
compiled and linked against `glibc` 2.27, it will only ever lookup symbols that
are 2.27 or earlier. When 2.28 comes out, it updates any symbols it needs to
2.28, leaving the rest as they are. When the application linked against 2.27
runs on a system with 2.28, everything is ok since 2.28 will resolve all the
2.27 symbols in the expected way the application needs.
Where portability becomes a problem is when the application is linked against a
newer version of `glibc` than is on the system. If the aforementioned
application compiled and linked against 2.27 was run on a system with 2.19, it
would fail to run because the symbol versions it is looking up (ie, anything
from 2.20 and later) are not available.
Unfortunately for developers seeking portability, compiling and linking against
the system's `glibc` means the application will reference the latest available
symbols in that `glibc`. There is no facility for telling the linker to only
use symbols from a particular `glibc` version and earlier. It's also difficult
to tell the linker to use an alternate `glibc` separate from the system's. As a
result, `glibc`-using software seeking wide Linux portability typically needs
to be compiled on an older system with a `glibc` with the desired version.
`python-build-standalone` and `rust` both support systems with `glibc` 2.17+,
which is covers distributions going back to 2014 (CentOS/RHEL 7 (EOL), Debian 8
(Jessie; EOL), Ubuntu 14.04 LTS (EOL), Fedora 21, etc.
Certain InfluxDB alpha releases are compiled against a too new `glibc` (2.36).
This will be addressed before release.
### How does InfluxDB find the correct libpython and the python runtime?
For the best user experience, users should not have to perform any extra setup
to use the InfluxDB processing engine. This is achieved by:
* Using an appropriate `PYO3_CONFIG_FILE` file during the build (see 'Official
builds', above)
* Build artifacts putting the runtime in an expected location (see 'Official
builds, above)
* At runtime, ensuring that Linux and Darwin binaries look for the runtime in
the expected location. Ideally this would be done with linker arguments at
builds time, but current (alpha) builds adjust the library search paths like
so:
```sh
# linux
$ patchelf --set-rpath '$ORIGIN/python/lib:$ORIGIN/../lib/influxdb3/python/lib' target/.../influxdb3
# osx
$ install_name_tool -change '/install/lib/libpython3.NN.dylib' \
'@executable_path/python/lib/libpythonNN.dylib' target/.../influxdb3
$ rcodesign sign target/.../influxdb3 # only with osxcross' install_name_tool
```
This is required, in part, due to how `python-build-standalone` is
[built](https://github.com/astral-sh/python-build-standalone/blob/main/docs/quirks.rst#references-to-build-time-paths).
When using `osxcross`'s version of `install_name_tool`, must also use
`rcodesign` from [apple-codesign](https://crates.io/crates/apple-codesign)
to re-sign the binaries (Apple's `install_name_tool` does this
automatically). Rust may gain [support](https://github.com/rust-lang/cargo/issues/5077)
for setting arbitrary rpaths at some point.
* The Windows `zip` file for the current (alpha) builds has copies of the
top-level DLL files from the 'python/' directory alongside `influxdb3`.
Windows requires that the dynamically linked DLLs needed by the application
are either in the same directory as the binary or found somewhere in `PATH`
(and open source tooling doesn't seem to support modifying this). For user
convenience, the `*.dll` files are shipped alongside the binary on Windows
to avoid having to setup the `PATH`. Rust believes this shouldn't be handled
by [rustc](https://github.com/rust-lang/cargo/issues/1500). This may be
addressed in a future release
### There is no `pip.exe` on Windows. Why?
From [upstream](https://github.com/astral-sh/python-build-standalone/blob/main/docs/quirks.rst#no-pipexe-on-windows):
"The Windows distributions have pip installed however no `Scripts/pip.exe`,
`Scripts/pip3.exe`, and `Scripts/pipX.Y.exe` files are provided because the way
these executables are built isn't portable. (It might be possible to change how
these are built to make them portable.)
To use pip, run `python.exe -m pip`. (It is generally a best practice to invoke
pip via `python -m pip` on all platforms so you can be explicit about the
python executable that pip uses.)"
### What limitations are there?
See https://github.com/influxdata/influxdb/issues?q=is%3Aissue%20state%3Aopen%20label%3Av3

67
influxdb3/src/main.rs
View File

@ -14,6 +14,8 @@ use dotenvy::dotenv;
use influxdb3_clap_blocks::tokio::TokioIoConfig; use influxdb3_clap_blocks::tokio::TokioIoConfig;
use influxdb3_process::VERSION_STRING; use influxdb3_process::VERSION_STRING;
use observability_deps::tracing::warn; use observability_deps::tracing::warn;
use std::env;
use std::path::{Path, PathBuf};
use trogging::{ use trogging::{
cli::LoggingConfigBuilderExt, cli::LoggingConfigBuilderExt,
tracing_subscriber::{prelude::*, Registry}, tracing_subscriber::{prelude::*, Registry},
@ -120,6 +122,12 @@ fn main() -> Result<(), std::io::Error> {
#[cfg(unix)] #[cfg(unix)]
install_crash_handler(); // attempt to render a useful stacktrace to stderr install_crash_handler(); // attempt to render a useful stacktrace to stderr
#[cfg(feature = "system-py")]
set_pythonhome();
#[cfg(all(target_os = "windows", feature = "system-py"))]
set_pythonpath();
// load all environment variables from .env before doing anything // load all environment variables from .env before doing anything
load_dotenv(); load_dotenv();
@ -301,3 +309,62 @@ fn init_logs_and_tracing(
let subscriber = Registry::default().with(layers); let subscriber = Registry::default().with(layers);
trogging::install_global(subscriber) trogging::install_global(subscriber)
} }
// XXX: this should be somewhere more appropriate
#[cfg(feature = "system-py")]
fn set_pythonhome() {
// This would ideally be detected by pyo3, but it isn't
match env::var("PYTHONHOME") {
Ok(_) => {}
Err(env::VarError::NotPresent) => {
let exe_path = env::current_exe().unwrap();
let exe_dir = exe_path.parent().unwrap();
let pythonhome: PathBuf = if cfg!(target_os = "linux")
&& (exe_dir == Path::new("/usr/bin") || exe_dir == Path::new("/usr/local/bin"))
{
// Official Linux builds may be in /usr or /usr/local
// XXX: handle this for local build and install (eg DESTDIR)
let parent_dir = exe_dir.parent().unwrap();
parent_dir.join("lib/influxdb3/python")
} else {
exe_dir.join("python")
};
if pythonhome.is_dir() {
unsafe { env::set_var("PYTHONHOME", pythonhome.to_str().unwrap()) };
//println!("Set PYTHONHOME to '{}'", env::var("PYTHONHOME").unwrap());
} else {
// TODO: use logger
eprintln!("Could not find python installation. May need to set PYTHONHOME");
}
}
Err(e) => {
eprintln!("Failed to retrieve PYTHONHOME: {e}");
}
};
}
// XXX: this should be somewhere more appropriate
#[cfg(target_os = "windows")]
fn set_pythonpath() {
let exe_path = env::current_exe().unwrap();
let exe_dir = exe_path.parent().unwrap();
let pythonpath = exe_dir.join("python/Lib");
// This shouldn't be needed, but it is on Windows
match env::var("PYTHONPATH") {
Ok(v) => {
let new_path = format!("{};{}", pythonpath.display(), v);
unsafe { env::set_var("PYTHONPATH", &new_path) };
//println!("Updated PYTHONPATH to: {}", env::var("PYTHONPATH").unwrap());
}
Err(env::VarError::NotPresent) => {
unsafe { env::set_var("PYTHONPATH", &pythonpath) };
//println!("Updated PYTHONPATH to: {}", env::var("PYTHONPATH").unwrap());
}
Err(e) => {
eprintln!("Failed to retrieve PYTHONPATH: {e}");
}
}
}

39
influxdb3_processing_engine/src/virtualenv.rs
View File

@ -17,7 +17,14 @@ pub enum VenvError {
} }
fn get_python_version() -> Result<(u8, u8), std::io::Error> { fn get_python_version() -> Result<(u8, u8), std::io::Error> {
let output = Command::new("python3") // linux/osx have python3, but windows only has python
let python_exe = if cfg!(target_os = "windows") {
"python"
} else {
"python3"
};
let output = Command::new(python_exe)
.args([ .args([
"-c", "-c",
"import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')", "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')",
@ -63,11 +70,17 @@ pub fn init_pyo3() {
}); });
} }
#[cfg(unix)] // FIXME: this still doesn't work right on windows (sys.path isn't adding the
// venv's site-packages). Perhaps look at /path/to/venv/pyvenv.cfg?
pub(crate) fn initialize_venv(venv_path: &Path) -> Result<(), VenvError> { pub(crate) fn initialize_venv(venv_path: &Path) -> Result<(), VenvError> {
use std::process::Command; use std::process::Command;
let activate_script = venv_path.join("bin").join("activate"); let activate_script = if cfg!(target_os = "windows") {
venv_path.join("Scripts").join("activate")
} else {
venv_path.join("bin").join("activate")
};
if !activate_script.exists() { if !activate_script.exists() {
return Err(VenvError::InitError(format!( return Err(VenvError::InitError(format!(
"Activation script not found at {:?}", "Activation script not found at {:?}",
@ -75,13 +88,19 @@ pub(crate) fn initialize_venv(venv_path: &Path) -> Result<(), VenvError> {
))); )));
} }
let output = Command::new("bash") let output = if cfg!(target_os = "windows") {
.arg("-c") Command::new("cmd")
.arg(format!( .args(["/C", activate_script.to_str().unwrap()])
"source {} && env", .output()?
activate_script.to_str().unwrap() } else {
)) Command::new("bash")
.output()?; .arg("-c")
.arg(format!(
"source {} && env",
activate_script.to_str().unwrap()
))
.output()?
};
if !output.status.success() { if !output.status.success() {
return Err(VenvError::InitError( return Err(VenvError::InitError(

394
install_influxdb.sh
View File

@ -1,394 +0,0 @@
#!/bin/sh -e
readonly GREEN='\033[0;32m'
readonly BOLD='\033[1m'
readonly BOLDGREEN='\033[1;32m'
readonly DIM='\033[2m'
readonly NC='\033[0m' # No Color
ARCHITECTURE=$(uname -m)
ARTIFACT=""
IS_MUSL=""
OS=""
INSTALL_LOC=~/.influxdb
BINARY_NAME="influxdb3"
PORT=8181
EDITION="Core"
EDITION_TAG="core"
if [ "$1" = "enterprise" ]; then
EDITION="Enterprise"
EDITION_TAG="enterprise"
shift 1
fi
### OS AND ARCHITECTURE DETECTION ###
case "$(uname -s)" in
Linux*) OS="Linux";;
Darwin*) OS="Darwin";;
*) OS="UNKNOWN";;
esac
if [ "${OS}" = "Linux" ]; then
# ldd is a shell script but on some systems (eg Ubuntu) security hardening
# prevents it from running when invoked directly. Since we only want to
# use '--verbose', find the path to ldd, then invoke under sh to bypass ldd
# hardening.
# XXX: use 'uname -o | grep GNU' instead?
ldd_exec=$(command -v ldd)
if [ "${ARCHITECTURE}" = "x86_64" ] || [ "${ARCHITECTURE}" = "amd64" ]; then
# Check if we're on a GNU/Linux system, otherwise default to musl
if [ -n "$ldd_exec" ] && sh -c "$ldd_exec --version" 2>&1 | grep -Eq "(GNU|GLIBC)"; then
ARTIFACT="x86_64-unknown-linux-gnu"
else
ARTIFACT="x86_64-unknown-linux-musl"
IS_MUSL="yes"
fi
elif [ "${ARCHITECTURE}" = "aarch64" ] || [ "${ARCHITECTURE}" = "arm64" ]; then
if [ -n "$ldd_exec" ] && sh -c "$ldd_exec --version" 2>&1 | grep -Eq "(GNU|GLIBC)"; then
ARTIFACT="aarch64-unknown-linux-gnu"
else
ARTIFACT="aarch64-unknown-linux-musl"
IS_MUSL="yes"
fi
fi
elif [ "${OS}" = "Darwin" ]; then
if [ "${ARCHITECTURE}" = "x86_64" ]; then
printf "Intel Mac support is coming soon!\n"
printf "Visit our public Discord at \033[4;94mhttps://discord.gg/az4jPm8x${NC} for additional guidance.\n"
printf "View alternative binaries on our Getting Started guide at \033[4;94mhttps://docs.influxdata.com/influxdb3/${EDITION_TAG}/${NC}.\n"
exit 1
else
ARTIFACT="aarch64-apple-darwin"
fi
fi
# Exit if unsupported system
[ -n "${ARTIFACT}" ] || {
printf "Unfortunately this script doesn't support your '${OS}' | '${ARCHITECTURE}' setup, or was unable to identify it correctly.\n"
printf "Visit our public Discord at \033[4;94mhttps://discord.gg/az4jPm8x${NC} for additional guidance.\n"
printf "View alternative binaries on our Getting Started guide at \033[4;94mhttps://docs.influxdata.com/influxdb3/${EDITION_TAG}/${NC}.\n"
exit 1
}
URL="https://dl.influxdata.com/influxdb/snapshots/influxdb3-${EDITION_TAG}_${ARTIFACT}.tar.gz"
START_TIME=$(date +%s)
# Attempt to clear screen and show welcome message
clear 2>/dev/null || true # clear isn't available everywhere
printf "┌───────────────────────────────────────────────────┐\n"
printf "│ ${BOLD}Welcome to InfluxDB!${NC} We'll make this quick. │\n"
printf "└───────────────────────────────────────────────────┘\n"
echo
printf "${BOLD}Select Installation Type${NC}\n"
echo
printf "1) ${GREEN}Docker Image${NC} ${DIM}(More Powerful, More Complex)${NC}\n"
printf " ├─ Requires knowledge of Docker and Docker management\n"
printf " └─ Includes the Processing Engine for real-time data transformation,\n"
printf " enrichment, and general custom Python code execution.\n\n"
printf "2) ${GREEN}Simple Download${NC} ${DIM}(Automated Install, Quick Setup)${NC}\n"
printf " ├─ No external dependencies required\n"
printf " └─ The Processing Engine will be available soon for binary installations,\n"
printf " bringing the same powerful processing capabilities to local deployments.\n"
echo
printf "Enter your choice (1-2): "
read -r INSTALL_TYPE
case "$INSTALL_TYPE" in
1)
printf "\n\n${BOLD}Download and Tag Docker Image${NC}\n"
printf "├─ ${DIM}docker pull quay.io/influxdb/influxdb3-${EDITION_TAG}:latest${NC}\n"
printf "└─ ${DIM}docker tag quay.io/influxdb/influxdb3-${EDITION_TAG}:latest influxdb3-${EDITION_TAG}${NC}\n\n"
if ! docker pull "quay.io/influxdb/influxdb3-${EDITION_TAG}:latest"; then
printf "└─ Error: Failed to download Docker image.\n"
exit 1
fi
docker tag quay.io/influxdb/influxdb3-${EDITION_TAG}:latest influxdb3-${EDITION_TAG}
# Exit script after Docker installation
echo
printf "${BOLD}NEXT STEPS${NC}\n"
printf "1) Run the Docker image:\n"
printf " ├─ ${BOLD}mkdir plugins${NC} ${DIM}(To store and access plugins)${NC}\n"
printf " └─ ${BOLD}docker run -it -p ${PORT}:${PORT} -v ./plugins:/plugins influxdb3-${EDITION_TAG} serve --object-store memory --node-id node0 --plugin-dir /plugins${NC} ${DIM}(To start)${NC}\n"
printf "2) View documentation at \033[4;94mhttps://docs.influxdata.com/influxdb3/${EDITION_TAG}/${NC}\n\n"
END_TIME=$(date +%s)
DURATION=$((END_TIME - START_TIME))
out=" Time is everything. This process took $DURATION seconds. "
mid=""
for _ in $(seq 1 ${#out}); do
mid="${mid}─"
done
printf "┌%s┐\n" "$mid"
printf "│%s│\n" "$out"
printf "└%s┘\n" "$mid"
exit 0
;;
2)
printf "\n\n"
;;
*)
printf "Invalid choice. Defaulting to binary installation.\n\n"
;;
esac
# attempt to find the user's shell config
shellrc=
if [ -n "$SHELL" ]; then
tmp=~/.$(basename "$SHELL")rc
if [ -e "$tmp" ]; then
shellrc="$tmp"
fi
fi
printf "${BOLD}Downloading InfluxDB 3 %s to %s${NC}\n" "$EDITION" "$INSTALL_LOC"
printf "├─${DIM} mkdir -p '%s'${NC}\n" "$INSTALL_LOC"
mkdir -p "$INSTALL_LOC"
printf "└─${DIM} curl -sS '%s' -o '%s/influxdb3.tar.gz'${NC}\n" "${URL}" "$INSTALL_LOC"
curl -sS "${URL}" -o "$INSTALL_LOC/influxdb3.tar.gz"
echo
printf "${BOLD}Verifying '%s/influxdb3.tar.gz'${NC}\n" "$INSTALL_LOC"
printf "└─${DIM} curl -sS '%s.sha256' -o '%s/influxdb3.tar.gz.sha256'${NC}\n" "${URL}" "$INSTALL_LOC"
curl -sS "${URL}.sha256" -o "$INSTALL_LOC/influxdb3.tar.gz.sha256"
dl_sha=$(cut -d ' ' -f 1 "$INSTALL_LOC/influxdb3.tar.gz.sha256" | grep -E '^[0-9a-f]{64}$')
if [ -z "$dl_sha" ]; then
printf "Could not find properly formatted SHA256 in '%s/influxdb3.tar.gz.sha256'. Aborting.\n" "$INSTALL_LOC"
exit 1
fi
printf "└─${DIM} sha256sum '%s/influxdb3.tar.gz'" "$INSTALL_LOC"
ch_sha=$(sha256sum "$INSTALL_LOC/influxdb3.tar.gz" | cut -d ' ' -f 1)
if [ "$ch_sha" = "$dl_sha" ]; then
printf " (OK: %s = %s)${NC}\n" "$ch_sha" "$dl_sha"
else
printf " (ERROR: %s != %s). Aborting.${NC}\n" "$ch_sha" "$dl_sha"
exit 1
fi
printf "└─${DIM} rm '%s/influxdb3.tar.gz.sha256'${NC}\n" "$INSTALL_LOC"
rm "$INSTALL_LOC/influxdb3.tar.gz.sha256"
echo
printf "${BOLD}Extracting and Processing${NC}\n"
printf "├─${DIM} tar -xf '%s/influxdb3.tar.gz' -C '%s'${NC}\n" "$INSTALL_LOC" "$INSTALL_LOC"
tar -xf "$INSTALL_LOC/influxdb3.tar.gz" -C "$INSTALL_LOC"
printf "└─${DIM} rm '%s/influxdb3.tar.gz'${NC}\n" "$INSTALL_LOC"
rm "$INSTALL_LOC/influxdb3.tar.gz"
if [ -n "$shellrc" ] && ! grep -q "export PATH=.*$INSTALL_LOC" "$shellrc"; then
echo
printf "${BOLD}Adding InfluxDB to '%s'${NC}\n" "$shellrc"
printf "└─${DIM} export PATH=\"\$PATH:%s/\" >> '%s'${NC}\n" "$INSTALL_LOC" "$shellrc"
echo "export PATH=\"\$PATH:$INSTALL_LOC/\"" >> "$shellrc"
fi
if [ "${EDITION}" = "Core" ]; then
# Prompt user to start the service
echo
printf "${BOLD}Configuration Options${NC}\n"
printf "└─ Start InfluxDB Now? (y/n): "
read -r START_SERVICE
if echo "$START_SERVICE" | grep -q "^[Yy]$" ; then
# Prompt for Node ID
echo
printf "${BOLD}Enter Your Node ID${NC}\n"
printf "├─ A Node ID is a unique, uneditable identifier for a service.\n"
printf "└─ Enter a Node ID (default: node0): "
read -r NODE_ID
NODE_ID=${NODE_ID:-node0}
# Prompt for storage solution
echo
printf "${BOLD}Select Your Storage Solution${NC}\n"
printf "├─ 1) In-memory storage (Fastest, data cleared on restart)\n"
printf "├─ 2) File storage (Persistent local storage)\n"
printf "├─ 3) Object storage (Cloud-compatible storage)\n"
printf "└─ Enter your choice (1-3): "
read -r STORAGE_CHOICE
case "$STORAGE_CHOICE" in
1)
STORAGE_TYPE="memory"
STORAGE_FLAGS="--object-store=memory"
STORAGE_FLAGS_ECHO="$STORAGE_FLAGS"
;;
2)
STORAGE_TYPE="File Storage"
echo
printf "Enter storage path (default: %s/data): " "${INSTALL_LOC}"
read -r STORAGE_PATH
STORAGE_PATH=${STORAGE_PATH:-"${INSTALL_LOC}/data"}
STORAGE_FLAGS="--object-store=file --data-dir ${STORAGE_PATH}"
STORAGE_FLAGS_ECHO="$STORAGE_FLAGS"
;;
3)
STORAGE_TYPE="Object Storage"
echo
printf "${BOLD}Select Cloud Provider${NC}\n"
printf "├─ 1) Amazon S3\n"
printf "├─ 2) Azure Storage\n"
printf "├─ 3) Google Cloud Storage\n"
printf "└─ Enter your choice (1-3): "
read -r CLOUD_CHOICE
case $CLOUD_CHOICE in
1) # AWS S3
echo
printf "${BOLD}AWS S3 Configuration${NC}\n"
printf "├─ Enter AWS Access Key ID: "
read -r AWS_KEY
printf "├─ Enter AWS Secret Access Key: "
stty -echo
read -r AWS_SECRET
stty echo
echo
printf "├─ Enter S3 Bucket: "
read -r AWS_BUCKET
printf "└─ Enter AWS Region (default: us-east-1): "
read -r AWS_REGION
AWS_REGION=${AWS_REGION:-"us-east-1"}
STORAGE_FLAGS="--object-store=s3 --bucket=${AWS_BUCKET}"
if [ -n "$AWS_REGION" ]; then
STORAGE_FLAGS="$STORAGE_FLAGS --aws-default-region=${AWS_REGION}"
fi
STORAGE_FLAGS="$STORAGE_FLAGS --aws-access-key-id=${AWS_KEY}"
STORAGE_FLAGS_ECHO="$STORAGE_FLAGS --aws-secret-access-key=..."
STORAGE_FLAGS="$STORAGE_FLAGS --aws-secret-access-key=${AWS_SECRET}"
;;
2) # Azure Storage
echo
printf "${BOLD}Azure Storage Configuration${NC}\n"
printf "├─ Enter Storage Account Name: "
read -r AZURE_ACCOUNT
printf "└─ Enter Storage Access Key: "
stty -echo
read -r AZURE_KEY
stty echo
echo
STORAGE_FLAGS="--object-store=azure --azure-storage-account=${AZURE_ACCOUNT}"
STORAGE_FLAGS_ECHO="$STORAGE_FLAGS --azure-storage-access-key=..."
STORAGE_FLAGS="$STORAGE_FLAGS --azure-storage-access-key=${AZURE_KEY}"
;;
3) # Google Cloud Storage
echo
printf "${BOLD}Google Cloud Storage Configuration${NC}\n"
printf "└─ Enter path to service account JSON file: "
read -r GOOGLE_SA
STORAGE_FLAGS="--object-store=google --google-service-account=${GOOGLE_SA}"
STORAGE_FLAGS_ECHO="$STORAGE_FLAGS"
;;
*)
printf "Invalid cloud provider choice. Defaulting to file storage.\n"
STORAGE_TYPE="File Storage"
STORAGE_FLAGS="--object-store=file --data-dir ${INSTALL_LOC}/data"
STORAGE_FLAGS_ECHO="$STORAGE_FLAGS"
;;
esac
;;
*)
printf "Invalid choice. Defaulting to in-memory.\n"
STORAGE_TYPE="Memory"
STORAGE_FLAGS="--object-store=memory"
STORAGE_FLAGS_ECHO="$STORAGE_FLAGS"
;;
esac
# Ensure port is available; if not, find a new one. If IS_MUSL is set,
# assume we are on a busybox-like system whose lsof doesn't support the
# args we need
lsof_exec=$(command -v lsof) && {
while [ -n "$lsof_exec" ] && [ "$IS_MUSL" != "yes" ] && lsof -i:"$PORT" -t >/dev/null 2>&1; do
printf "├─${DIM} Port %s is in use. Finding new port.${NC}\n" "$PORT"
PORT=$((PORT + 1))
if [ "$PORT" -gt 32767 ]; then
printf "└─${DIM} Could not find an available port. Aborting.${NC}\n"
exit 1
fi
if ! "$lsof_exec" -i:"$PORT" -t >/dev/null 2>&1; then
printf "└─${DIM} Found an available port: %s${NC}\n" "$PORT"
break
fi
done
}
# Start and give up to 30 seconds to respond
echo
printf "${BOLD}Starting InfluxDB${NC}\n"
printf "├─${DIM} Node ID: %s${NC}\n" "$NODE_ID"
printf "├─${DIM} Storage: %s${NC}\n" "$STORAGE_TYPE"
printf "├─${DIM} '%s' serve --node-id='%s' --http-bind='0.0.0.0:%s' %s${NC}\n" "$INSTALL_LOC/$BINARY_NAME" "$NODE_ID" "$PORT" "$STORAGE_FLAGS_ECHO"
"$INSTALL_LOC/$BINARY_NAME" serve --node-id="$NODE_ID" --http-bind="0.0.0.0:$PORT" $STORAGE_FLAGS > /dev/null &
PID="$!"
SUCCESS=0
for _ in $(seq 1 30); do
# on systems without a usable lsof, sleep a second to see if the pid is
# still there to give influxdb a chance to error out in case an already
# running influxdb is running on this port
if [ -z "$lsof_exec" ] || [ "$IS_MUSL" = "yes" ]; then
sleep 1
fi
if ! kill -0 "$PID" 2>/dev/null ; then
break
fi
if curl --max-time 3 -s "http://localhost:$PORT/health" >/dev/null 2>&1; then
printf "└─${BOLDGREEN} ✓ InfluxDB 3 ${EDITION} is now installed and running on port %s. Nice!${NC}\n" "$PORT"
SUCCESS=1
break
fi
sleep 1
done
if [ $SUCCESS -eq 0 ]; then
printf "└─${BOLD} ERROR: InfluxDB failed to start; check permissions or other potential issues.${NC}\n" "$PORT"
exit 1
fi
else
echo
printf "${BOLDGREEN}✓ InfluxDB 3 ${EDITION} is now installed. Nice!${NC}\n"
fi
else
echo
printf "${BOLDGREEN}✓ InfluxDB 3 ${EDITION} is now installed. Nice!${NC}\n"
fi
### SUCCESS INFORMATION ###
echo
printf "${BOLD}Further Info${NC}\n"
if [ -n "$shellrc" ]; then
printf "├─ Run ${BOLD}source '%s'${NC}, then access InfluxDB with ${BOLD}influxdb3${NC} command.\n" "$shellrc"
else
printf "├─ Access InfluxDB with the ${BOLD}%s${NC} command.\n" "$INSTALL_LOC/$BINARY_NAME"
fi
printf "├─ View the Getting Started guide at \033[4;94mhttps://docs.influxdata.com/influxdb3/${EDITION_TAG}/${NC}.\n"
printf "└─ Visit our public Discord at \033[4;94mhttps://discord.gg/az4jPm8x${NC} for additional guidance.\n"
echo
END_TIME=$(date +%s)
DURATION=$((END_TIME - START_TIME))
out=" Time is everything. This process took $DURATION seconds. "
mid=""
for _ in $(seq 1 ${#out}); do
mid="${mid}─"
done
printf "┌%s┐\n" "$mid"
printf "│%s│\n" "$out"
printf "└%s┘\n" "$mid"

31
install_influxdb3.sh
View File

@ -85,13 +85,9 @@ echo
printf "${BOLD}Select Installation Type${NC}\n" printf "${BOLD}Select Installation Type${NC}\n"
echo echo
printf "1) ${GREEN}Docker Image${NC} ${DIM}(More Powerful, More Complex)${NC}\n" printf "1) ${GREEN}Docker Image${NC} ${DIM}(More Powerful, More Complex)${NC}\n"
printf " ├─ Requires knowledge of Docker and Docker management\n" printf " └─ Requires knowledge of Docker and Docker management\n"
printf " └─ Includes the Processing Engine for real-time data transformation,\n"
printf " enrichment, and general custom Python code execution.\n\n"
printf "2) ${GREEN}Simple Download${NC} ${DIM}(Automated Install, Quick Setup)${NC}\n" printf "2) ${GREEN}Simple Download${NC} ${DIM}(Automated Install, Quick Setup)${NC}\n"
printf " ├─ No external dependencies required\n" printf " └─ No external dependencies required\n"
printf " └─ The Processing Engine will be available soon for binary installations,\n"
printf " bringing the same powerful processing capabilities to local deployments.\n"
echo echo
printf "Enter your choice (1-2): " printf "Enter your choice (1-2): "
read -r INSTALL_TYPE read -r INSTALL_TYPE
@ -147,13 +143,13 @@ fi
printf "${BOLD}Downloading InfluxDB 3 %s to %s${NC}\n" "$EDITION" "$INSTALL_LOC" printf "${BOLD}Downloading InfluxDB 3 %s to %s${NC}\n" "$EDITION" "$INSTALL_LOC"
printf "├─${DIM} mkdir -p '%s'${NC}\n" "$INSTALL_LOC" printf "├─${DIM} mkdir -p '%s'${NC}\n" "$INSTALL_LOC"
mkdir -p "$INSTALL_LOC" mkdir -p "$INSTALL_LOC"
printf "└─${DIM} curl -sS '%s' -o '%s/influxdb3.tar.gz'${NC}\n" "${URL}" "$INSTALL_LOC" printf "└─${DIM} curl -sSL '%s' -o '%s/influxdb3.tar.gz'${NC}\n" "${URL}" "$INSTALL_LOC"
curl -sS "${URL}" -o "$INSTALL_LOC/influxdb3.tar.gz" curl -sSL "${URL}" -o "$INSTALL_LOC/influxdb3.tar.gz"
echo echo
printf "${BOLD}Verifying '%s/influxdb3.tar.gz'${NC}\n" "$INSTALL_LOC" printf "${BOLD}Verifying '%s/influxdb3.tar.gz'${NC}\n" "$INSTALL_LOC"
printf "└─${DIM} curl -sS '%s.sha256' -o '%s/influxdb3.tar.gz.sha256'${NC}\n" "${URL}" "$INSTALL_LOC" printf "└─${DIM} curl -sSL '%s.sha256' -o '%s/influxdb3.tar.gz.sha256'${NC}\n" "${URL}" "$INSTALL_LOC"
curl -sS "${URL}.sha256" -o "$INSTALL_LOC/influxdb3.tar.gz.sha256" curl -sSL "${URL}.sha256" -o "$INSTALL_LOC/influxdb3.tar.gz.sha256"
dl_sha=$(cut -d ' ' -f 1 "$INSTALL_LOC/influxdb3.tar.gz.sha256" | grep -E '^[0-9a-f]{64}$') dl_sha=$(cut -d ' ' -f 1 "$INSTALL_LOC/influxdb3.tar.gz.sha256" | grep -E '^[0-9a-f]{64}$')
if [ -z "$dl_sha" ]; then if [ -z "$dl_sha" ]; then
printf "Could not find properly formatted SHA256 in '%s/influxdb3.tar.gz.sha256'. Aborting.\n" "$INSTALL_LOC" printf "Could not find properly formatted SHA256 in '%s/influxdb3.tar.gz.sha256'. Aborting.\n" "$INSTALL_LOC"
@ -172,8 +168,15 @@ rm "$INSTALL_LOC/influxdb3.tar.gz.sha256"
echo echo
printf "${BOLD}Extracting and Processing${NC}\n" printf "${BOLD}Extracting and Processing${NC}\n"
printf "├─${DIM} tar -xf '%s/influxdb3.tar.gz' -C '%s'${NC}\n" "$INSTALL_LOC" "$INSTALL_LOC"
tar -xf "$INSTALL_LOC/influxdb3.tar.gz" -C "$INSTALL_LOC" # some tarballs have a leading component, check for that
TAR_LEVEL=0
if tar -tf "$INSTALL_LOC/influxdb3.tar.gz" | grep -q '[a-zA-Z0-9]/influxdb3$' ; then
TAR_LEVEL=1
fi
printf "├─${DIM} tar -xf '%s/influxdb3.tar.gz' --strip-components=${TAR_LEVEL} -C '%s'${NC}\n" "$INSTALL_LOC" "$INSTALL_LOC"
tar -xf "$INSTALL_LOC/influxdb3.tar.gz" --strip-components="${TAR_LEVEL}" -C "$INSTALL_LOC"
printf "└─${DIM} rm '%s/influxdb3.tar.gz'${NC}\n" "$INSTALL_LOC" printf "└─${DIM} rm '%s/influxdb3.tar.gz'${NC}\n" "$INSTALL_LOC"
rm "$INSTALL_LOC/influxdb3.tar.gz" rm "$INSTALL_LOC/influxdb3.tar.gz"
@ -378,7 +381,9 @@ else
printf "├─ Access InfluxDB with the ${BOLD}%s${NC} command.\n" "$INSTALL_LOC/$BINARY_NAME" printf "├─ Access InfluxDB with the ${BOLD}%s${NC} command.\n" "$INSTALL_LOC/$BINARY_NAME"
fi fi
printf "├─ View the Getting Started guide at \033[4;94mhttps://docs.influxdata.com/influxdb3/${EDITION_TAG}/${NC}.\n" printf "├─ View the Getting Started guide at \033[4;94mhttps://docs.influxdata.com/influxdb3/${EDITION_TAG}/${NC}.\n"
printf "└─ Visit our public Discord at \033[4;94mhttps://discord.gg/az4jPm8x${NC} for additional guidance.\n" printf "├─ Visit our public Discord at \033[4;94mhttps://discord.gg/az4jPm8x${NC} for additional guidance.\n"
printf "└─ The Processing Engine is now included for real-time data transformation,\n"
printf " enrichment, and general custom Python code execution.\n\n"
echo echo
END_TIME=$(date +%s) END_TIME=$(date +%s)