milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

enhance: enable memory prof based on jemalloc (#40731) 

issue: #40730

also see: https://github.com/milvus-io/cgosymbolizer/pull/2

After these PR, at linux:

- the milvus will always enable jemalloc by default.
- jemalloc will always compiled with --enable-prof options.
- all image will always enable the jemalloc prof by default.
- a pprof http service for jemalloc at `/debug/jemalloc/` will be
registered into restful.
- `jeprof` can remote profile the memory of milvus.

Signed-off-by: chyezh <chyezh@outlook.com>
pull/40766/head
Zhen Ye 2025-03-19 14:46:18 +08:00 committed by GitHub
parent cf223bae7b
commit 8db708f67d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
14 changed files with 23 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -14,7 +14,7 @@ PWD := $(shell pwd)
GOPATH := $(shell $(GO) env GOPATH)
SHELL := /bin/bash
OBJPREFIX := "github.com/milvus-io/milvus/cmd/milvus"
MILVUS_GO_BUILD_TAGS := "dynamic,sonic"
MILVUS_GO_BUILD_TAGS := "dynamic,sonic,with_jemalloc"
INSTALL_PATH := $(PWD)/bin
LIBRARY_PATH := $(PWD)/lib

2
build/docker/milvus/amazonlinux2023/Dockerfile
View File

@ -32,7 +32,7 @@ COPY --chown=root:root --chmod=774 ./lib/ /milvus/lib/
ENV PATH=/milvus/bin:$PATH
ENV LD_LIBRARY_PATH=/milvus/lib:$LD_LIBRARY_PATH:/usr/lib
ENV LD_PRELOAD=${MILVUS_ASAN_LIB}:/milvus/lib/libjemalloc.so
ENV MALLOC_CONF=background_thread:true
ENV MALLOC_CONF=background_thread:true,prof:true
ENTRYPOINT ["/tini", "--"]

2
build/docker/milvus/gpu/ubuntu20.04/Dockerfile
View File

@ -23,4 +23,4 @@ COPY --chown=root:root --chmod=774 ./lib/ /milvus/lib/
ENV PATH=/milvus/bin:$PATH
ENV LD_LIBRARY_PATH=/milvus/lib:$LD_LIBRARY_PATH:/usr/lib
ENV LD_PRELOAD=/milvus/lib/libjemalloc.so
ENV MALLOC_CONF=background_thread:true
ENV MALLOC_CONF=background_thread:true,prof:true

2
build/docker/milvus/gpu/ubuntu22.04/Dockerfile
View File

@ -19,7 +19,7 @@ COPY --chown=root:root --chmod=774 ./lib/ /milvus/lib/
ENV PATH=/milvus/bin:$PATH
ENV LD_LIBRARY_PATH=/milvus/lib:$LD_LIBRARY_PATH:/usr/lib
ENV LD_PRELOAD=/milvus/lib/libjemalloc.so
ENV MALLOC_CONF=background_thread:true
ENV MALLOC_CONF=background_thread:true,prof:true
ENTRYPOINT ["/tini", "--"]

2
build/docker/milvus/rockylinux8/Dockerfile
View File

@ -36,7 +36,7 @@ COPY ./lib/ /milvus/lib/
ENV PATH=/milvus/bin:$PATH
ENV LD_LIBRARY_PATH=/milvus/lib:$LD_LIBRARY_PATH:/usr/lib
ENV LD_PRELOAD=${MILVUS_ASAN_LIB}:/milvus/lib/libjemalloc.so
ENV MALLOC_CONF=background_thread:true
ENV MALLOC_CONF=background_thread:true,prof:true
ENTRYPOINT ["/tini", "--"]

2
build/docker/milvus/ubuntu20.04/Dockerfile
View File

@ -33,7 +33,7 @@ COPY --chown=root:root --chmod=774 ./lib/ /milvus/lib/
ENV PATH=/milvus/bin:$PATH
ENV LD_LIBRARY_PATH=/milvus/lib:$LD_LIBRARY_PATH:/usr/lib
ENV LD_PRELOAD=${MILVUS_ASAN_LIB}:/milvus/lib/libjemalloc.so
ENV MALLOC_CONF=background_thread:true
ENV MALLOC_CONF=background_thread:true,prof:true
ENTRYPOINT ["/tini", "--"]

2
build/docker/milvus/ubuntu22.04/Dockerfile
View File

@ -33,7 +33,7 @@ COPY --chown=root:root --chmod=774 ./lib/ /milvus/lib/
ENV PATH=/milvus/bin:$PATH
ENV LD_LIBRARY_PATH=/milvus/lib:$LD_LIBRARY_PATH:/usr/lib
ENV LD_PRELOAD=${MILVUS_ASAN_LIB}:/milvus/lib/libjemalloc.so
ENV MALLOC_CONF=background_thread:true
ENV MALLOC_CONF=background_thread:true,prof:true
ENTRYPOINT ["/tini", "--"]

2
go.mod
View File

@ -292,7 +292,7 @@ replace (
github.com/expr-lang/expr => github.com/SimFG/expr v0.0.0-20241226082220-a9a764953bf8
github.com/go-kit/kit => github.com/go-kit/kit v0.1.0
github.com/greatroar/blobloom => github.com/milvus-io/blobloom v0.0.0-20240603110411-471ae49f3b93
github.com/ianlancetaylor/cgosymbolizer => github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119
github.com/ianlancetaylor/cgosymbolizer => github.com/milvus-io/cgosymbolizer v0.0.0-20250318084424-114f4050c3a6
github.com/milvus-io/milvus/pkg/v2 => ./pkg
github.com/streamnative/pulsarctl => github.com/xiaofan-luan/pulsarctl v0.5.1
github.com/tecbot/gorocksdb => github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b // indirect

4
go.sum
View File

@ -730,8 +730,8 @@ github.com/milvus-io/arrow/go/v17 v17.0.0 h1:/3B2KLEzJYLJ5hxwTvBlXAn0uF663tzvbtt
github.com/milvus-io/arrow/go/v17 v17.0.0/go.mod h1:jR7QHkODl15PfYyjM2nU+yTLScZ/qfj7OSUZmJ8putc=
github.com/milvus-io/blobloom v0.0.0-20240603110411-471ae49f3b93 h1:xnIeuG1nuTEHKbbv51OwNGO82U+d6ut08ppTmZVm+VY=
github.com/milvus-io/blobloom v0.0.0-20240603110411-471ae49f3b93/go.mod h1:mjMJ1hh1wjGVfr93QIHJ6FfDNVrA0IELv8OvMHJxHKs=
github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119 h1:9VXijWuf+oW/9m+sirIDL4wQb2BoZNXORbcJbkPOChY=
github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg=
github.com/milvus-io/cgosymbolizer v0.0.0-20250318084424-114f4050c3a6 h1:YHMFI6LHBzFJSudBOjtX8zD0+0NSzS44HHKjR8HspLA=
github.com/milvus-io/cgosymbolizer v0.0.0-20250318084424-114f4050c3a6/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg=
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZzUfIfYe5qYDBzt4ZYRqzUjTR6CvUzjat8=
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4=
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.0-beta.0.20250225103150-0a1988183e53 h1:HoaZPKnE/LhkubU7f8qN8J4LfDIroiqoufWS0kPumM4=

3
internal/core/thirdparty/jemalloc/CMakeLists.txt vendored
View File

@ -52,7 +52,8 @@ endif ()
list(APPEND
JEMALLOC_CONFIGURE_COMMAND
"--prefix=${JEMALLOC_PREFIX}"
"--libdir=${JEMALLOC_LIB_DIR}")
"--libdir=${JEMALLOC_LIB_DIR}"
"--enable-prof")
if (CMAKE_BUILD_TYPE EQUAL "DEBUG")
# Enable jemalloc debug checks when Milvus itself has debugging enabled
list(APPEND JEMALLOC_CONFIGURE_COMMAND "--enable-debug")

2
pkg/go.mod
View File

@ -213,7 +213,7 @@ replace (
github.com/bketelsen/crypt => github.com/bketelsen/crypt v0.0.4 // Fix security alert for core-os/etcd
github.com/expr-lang/expr => github.com/SimFG/expr v0.0.0-20241226082220-a9a764953bf8
github.com/go-kit/kit => github.com/go-kit/kit v0.1.0
github.com/ianlancetaylor/cgosymbolizer => github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119
github.com/ianlancetaylor/cgosymbolizer => github.com/milvus-io/cgosymbolizer v0.0.0-20250318084424-114f4050c3a6
github.com/streamnative/pulsarctl => github.com/xiaofan-luan/pulsarctl v0.5.1
github.com/tecbot/gorocksdb => github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b // indirect
)

4
pkg/go.sum
View File

@ -547,8 +547,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfr
github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8=
github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119 h1:9VXijWuf+oW/9m+sirIDL4wQb2BoZNXORbcJbkPOChY=
github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg=
github.com/milvus-io/cgosymbolizer v0.0.0-20250318084424-114f4050c3a6 h1:YHMFI6LHBzFJSudBOjtX8zD0+0NSzS44HHKjR8HspLA=
github.com/milvus-io/cgosymbolizer v0.0.0-20250318084424-114f4050c3a6/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg=
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZzUfIfYe5qYDBzt4ZYRqzUjTR6CvUzjat8=
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4=
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.0-beta.0.20250225103150-0a1988183e53 h1:HoaZPKnE/LhkubU7f8qN8J4LfDIroiqoufWS0kPumM4=

11
scripts/start_cluster.sh
View File

@ -21,6 +21,7 @@ if [[ "$OSTYPE" == "linux-gnu"* ]]; then
if test -f "$LIBJEMALLOC"; then
#echo "Found $LIBJEMALLOC"
export LD_PRELOAD="$LIBJEMALLOC"
export MALLOC_CONF=background_thread:true,prof:true
else
echo "WARN: Cannot find $LIBJEMALLOC"
fi
@ -28,16 +29,16 @@ if [[ "$OSTYPE" == "linux-gnu"* ]]; then
fi
echo "Starting mixcoord..."
nohup ./bin/milvus run mixture -rootcoord -querycoord -datacoord --run-with-subprocess > /tmp/mixcoord.log 2>&1 &
nohup ./bin/milvus run mixture -rootcoord -querycoord -datacoord --run-with-subprocess >/tmp/mixcoord.log 2>&1 &
echo "Starting datanode..."
nohup ./bin/milvus run datanode --run-with-subprocess > /tmp/datanode.log 2>&1 &
nohup ./bin/milvus run datanode --run-with-subprocess >/tmp/datanode.log 2>&1 &
echo "Starting proxy..."
nohup ./bin/milvus run proxy --run-with-subprocess > /tmp/proxy.log 2>&1 &
nohup ./bin/milvus run proxy --run-with-subprocess >/tmp/proxy.log 2>&1 &
echo "Starting querynode..."
nohup ./bin/milvus run querynode --run-with-subprocess > /tmp/querynode.log 2>&1 &
nohup ./bin/milvus run querynode --run-with-subprocess >/tmp/querynode.log 2>&1 &
echo "Starting streamingnode..."
nohup ./bin/milvus run streamingnode --run-with-subprocess > /tmp/streamingnode.log 2>&1 &
nohup ./bin/milvus run streamingnode --run-with-subprocess >/tmp/streamingnode.log 2>&1 &

3
scripts/start_standalone.sh
View File

@ -21,6 +21,7 @@ if [[ "$OSTYPE" == "linux-gnu"* ]]; then
if test -f "$LIBJEMALLOC"; then
#echo "Found $LIBJEMALLOC"
export LD_PRELOAD="$LIBJEMALLOC"
export MALLOC_CONF=background_thread:true,prof:true
else
echo "WARN: Cannot find $LIBJEMALLOC"
fi
@ -28,4 +29,4 @@ if [[ "$OSTYPE" == "linux-gnu"* ]]; then
fi
echo "Starting standalone..."
nohup ./bin/milvus run standalone --run-with-subprocess > /tmp/standalone.log 2>&1 &
nohup ./bin/milvus run standalone --run-with-subprocess >/tmp/standalone.log 2>&1 &