Skip to content

Commit 8db708f

Browse files
chyezhchyezh
authored
enhance: enable memory prof based on jemalloc (milvus-io#40731)
issue: milvus-io#40730 also see: milvus-io/cgosymbolizer#2 After these PR, at linux: - the milvus will always enable jemalloc by default. - jemalloc will always compiled with --enable-prof options. - all image will always enable the jemalloc prof by default. - a pprof http service for jemalloc at `/debug/jemalloc/` will be registered into restful. - `jeprof` can remote profile the memory of milvus. Signed-off-by: chyezh <[email protected]>
1 parent cf223ba commit 8db708f

File tree

14 files changed

+23
-20
lines changed

14 files changed

+23
-20
lines changed

‎Makefile

+1-1
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ PWD := $(shell pwd)
1414
GOPATH := $(shell $(GO) env GOPATH)
1515
SHELL := /bin/bash
1616
OBJPREFIX := "github.com/milvus-io/milvus/cmd/milvus"
17-
MILVUS_GO_BUILD_TAGS := "dynamic,sonic"
17+
MILVUS_GO_BUILD_TAGS := "dynamic,sonic,with_jemalloc"
1818

1919
INSTALL_PATH := $(PWD)/bin
2020
LIBRARY_PATH := $(PWD)/lib

‎build/docker/milvus/amazonlinux2023/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ COPY --chown=root:root --chmod=774 ./lib/ /milvus/lib/
3232
ENV PATH=/milvus/bin:$PATH
3333
ENV LD_LIBRARY_PATH=/milvus/lib:$LD_LIBRARY_PATH:/usr/lib
3434
ENV LD_PRELOAD=${MILVUS_ASAN_LIB}:/milvus/lib/libjemalloc.so
35-
ENV MALLOC_CONF=background_thread:true
35+
ENV MALLOC_CONF=background_thread:true,prof:true
3636

3737
ENTRYPOINT ["/tini", "--"]
3838

‎build/docker/milvus/gpu/ubuntu20.04/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -23,4 +23,4 @@ COPY --chown=root:root --chmod=774 ./lib/ /milvus/lib/
2323
ENV PATH=/milvus/bin:$PATH
2424
ENV LD_LIBRARY_PATH=/milvus/lib:$LD_LIBRARY_PATH:/usr/lib
2525
ENV LD_PRELOAD=/milvus/lib/libjemalloc.so
26-
ENV MALLOC_CONF=background_thread:true
26+
ENV MALLOC_CONF=background_thread:true,prof:true

‎build/docker/milvus/gpu/ubuntu22.04/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ COPY --chown=root:root --chmod=774 ./lib/ /milvus/lib/
1919
ENV PATH=/milvus/bin:$PATH
2020
ENV LD_LIBRARY_PATH=/milvus/lib:$LD_LIBRARY_PATH:/usr/lib
2121
ENV LD_PRELOAD=/milvus/lib/libjemalloc.so
22-
ENV MALLOC_CONF=background_thread:true
22+
ENV MALLOC_CONF=background_thread:true,prof:true
2323

2424
ENTRYPOINT ["/tini", "--"]
2525

‎build/docker/milvus/rockylinux8/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ COPY ./lib/ /milvus/lib/
3636
ENV PATH=/milvus/bin:$PATH
3737
ENV LD_LIBRARY_PATH=/milvus/lib:$LD_LIBRARY_PATH:/usr/lib
3838
ENV LD_PRELOAD=${MILVUS_ASAN_LIB}:/milvus/lib/libjemalloc.so
39-
ENV MALLOC_CONF=background_thread:true
39+
ENV MALLOC_CONF=background_thread:true,prof:true
4040

4141
ENTRYPOINT ["/tini", "--"]
4242

‎build/docker/milvus/ubuntu20.04/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ COPY --chown=root:root --chmod=774 ./lib/ /milvus/lib/
3333
ENV PATH=/milvus/bin:$PATH
3434
ENV LD_LIBRARY_PATH=/milvus/lib:$LD_LIBRARY_PATH:/usr/lib
3535
ENV LD_PRELOAD=${MILVUS_ASAN_LIB}:/milvus/lib/libjemalloc.so
36-
ENV MALLOC_CONF=background_thread:true
36+
ENV MALLOC_CONF=background_thread:true,prof:true
3737

3838
ENTRYPOINT ["/tini", "--"]
3939

‎build/docker/milvus/ubuntu22.04/Dockerfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ COPY --chown=root:root --chmod=774 ./lib/ /milvus/lib/
3333
ENV PATH=/milvus/bin:$PATH
3434
ENV LD_LIBRARY_PATH=/milvus/lib:$LD_LIBRARY_PATH:/usr/lib
3535
ENV LD_PRELOAD=${MILVUS_ASAN_LIB}:/milvus/lib/libjemalloc.so
36-
ENV MALLOC_CONF=background_thread:true
36+
ENV MALLOC_CONF=background_thread:true,prof:true
3737

3838
ENTRYPOINT ["/tini", "--"]
3939

‎go.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -292,7 +292,7 @@ replace (
292292
github.com/expr-lang/expr => github.com/SimFG/expr v0.0.0-20241226082220-a9a764953bf8
293293
github.com/go-kit/kit => github.com/go-kit/kit v0.1.0
294294
github.com/greatroar/blobloom => github.com/milvus-io/blobloom v0.0.0-20240603110411-471ae49f3b93
295-
github.com/ianlancetaylor/cgosymbolizer => github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119
295+
github.com/ianlancetaylor/cgosymbolizer => github.com/milvus-io/cgosymbolizer v0.0.0-20250318084424-114f4050c3a6
296296
github.com/milvus-io/milvus/pkg/v2 => ./pkg
297297
github.com/streamnative/pulsarctl => github.com/xiaofan-luan/pulsarctl v0.5.1
298298
github.com/tecbot/gorocksdb => github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b // indirect

‎go.sum

+2-2
Original file line numberDiff line numberDiff line change
@@ -730,8 +730,8 @@ github.com/milvus-io/arrow/go/v17 v17.0.0 h1:/3B2KLEzJYLJ5hxwTvBlXAn0uF663tzvbtt
730730
github.com/milvus-io/arrow/go/v17 v17.0.0/go.mod h1:jR7QHkODl15PfYyjM2nU+yTLScZ/qfj7OSUZmJ8putc=
731731
github.com/milvus-io/blobloom v0.0.0-20240603110411-471ae49f3b93 h1:xnIeuG1nuTEHKbbv51OwNGO82U+d6ut08ppTmZVm+VY=
732732
github.com/milvus-io/blobloom v0.0.0-20240603110411-471ae49f3b93/go.mod h1:mjMJ1hh1wjGVfr93QIHJ6FfDNVrA0IELv8OvMHJxHKs=
733-
github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119 h1:9VXijWuf+oW/9m+sirIDL4wQb2BoZNXORbcJbkPOChY=
734-
github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg=
733+
github.com/milvus-io/cgosymbolizer v0.0.0-20250318084424-114f4050c3a6 h1:YHMFI6LHBzFJSudBOjtX8zD0+0NSzS44HHKjR8HspLA=
734+
github.com/milvus-io/cgosymbolizer v0.0.0-20250318084424-114f4050c3a6/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg=
735735
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZzUfIfYe5qYDBzt4ZYRqzUjTR6CvUzjat8=
736736
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4=
737737
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.0-beta.0.20250225103150-0a1988183e53 h1:HoaZPKnE/LhkubU7f8qN8J4LfDIroiqoufWS0kPumM4=

‎internal/core/thirdparty/jemalloc/CMakeLists.txt

+2-1
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,8 @@ endif ()
5252
list(APPEND
5353
JEMALLOC_CONFIGURE_COMMAND
5454
"--prefix=${JEMALLOC_PREFIX}"
55-
"--libdir=${JEMALLOC_LIB_DIR}")
55+
"--libdir=${JEMALLOC_LIB_DIR}"
56+
"--enable-prof")
5657
if (CMAKE_BUILD_TYPE EQUAL "DEBUG")
5758
# Enable jemalloc debug checks when Milvus itself has debugging enabled
5859
list(APPEND JEMALLOC_CONFIGURE_COMMAND "--enable-debug")

‎pkg/go.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -213,7 +213,7 @@ replace (
213213
github.com/bketelsen/crypt => github.com/bketelsen/crypt v0.0.4 // Fix security alert for core-os/etcd
214214
github.com/expr-lang/expr => github.com/SimFG/expr v0.0.0-20241226082220-a9a764953bf8
215215
github.com/go-kit/kit => github.com/go-kit/kit v0.1.0
216-
github.com/ianlancetaylor/cgosymbolizer => github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119
216+
github.com/ianlancetaylor/cgosymbolizer => github.com/milvus-io/cgosymbolizer v0.0.0-20250318084424-114f4050c3a6
217217
github.com/streamnative/pulsarctl => github.com/xiaofan-luan/pulsarctl v0.5.1
218218
github.com/tecbot/gorocksdb => github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b // indirect
219219
)

‎pkg/go.sum

+2-2
Original file line numberDiff line numberDiff line change
@@ -547,8 +547,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfr
547547
github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8=
548548
github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
549549
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
550-
github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119 h1:9VXijWuf+oW/9m+sirIDL4wQb2BoZNXORbcJbkPOChY=
551-
github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg=
550+
github.com/milvus-io/cgosymbolizer v0.0.0-20250318084424-114f4050c3a6 h1:YHMFI6LHBzFJSudBOjtX8zD0+0NSzS44HHKjR8HspLA=
551+
github.com/milvus-io/cgosymbolizer v0.0.0-20250318084424-114f4050c3a6/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg=
552552
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZzUfIfYe5qYDBzt4ZYRqzUjTR6CvUzjat8=
553553
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4=
554554
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.0-beta.0.20250225103150-0a1988183e53 h1:HoaZPKnE/LhkubU7f8qN8J4LfDIroiqoufWS0kPumM4=

‎scripts/start_cluster.sh

+6-5
Original file line numberDiff line numberDiff line change
@@ -21,23 +21,24 @@ if [[ "$OSTYPE" == "linux-gnu"* ]]; then
2121
if test -f "$LIBJEMALLOC"; then
2222
#echo "Found $LIBJEMALLOC"
2323
export LD_PRELOAD="$LIBJEMALLOC"
24+
export MALLOC_CONF=background_thread:true,prof:true
2425
else
2526
echo "WARN: Cannot find $LIBJEMALLOC"
2627
fi
2728
export LD_LIBRARY_PATH=$PWD/internal/core/output/lib/:$LD_LIBRARY_PATH
2829
fi
2930

3031
echo "Starting mixcoord..."
31-
nohup ./bin/milvus run mixture -rootcoord -querycoord -datacoord --run-with-subprocess > /tmp/mixcoord.log 2>&1 &
32+
nohup ./bin/milvus run mixture -rootcoord -querycoord -datacoord --run-with-subprocess >/tmp/mixcoord.log 2>&1 &
3233

3334
echo "Starting datanode..."
34-
nohup ./bin/milvus run datanode --run-with-subprocess > /tmp/datanode.log 2>&1 &
35+
nohup ./bin/milvus run datanode --run-with-subprocess >/tmp/datanode.log 2>&1 &
3536

3637
echo "Starting proxy..."
37-
nohup ./bin/milvus run proxy --run-with-subprocess > /tmp/proxy.log 2>&1 &
38+
nohup ./bin/milvus run proxy --run-with-subprocess >/tmp/proxy.log 2>&1 &
3839

3940
echo "Starting querynode..."
40-
nohup ./bin/milvus run querynode --run-with-subprocess > /tmp/querynode.log 2>&1 &
41+
nohup ./bin/milvus run querynode --run-with-subprocess >/tmp/querynode.log 2>&1 &
4142

4243
echo "Starting streamingnode..."
43-
nohup ./bin/milvus run streamingnode --run-with-subprocess > /tmp/streamingnode.log 2>&1 &
44+
nohup ./bin/milvus run streamingnode --run-with-subprocess >/tmp/streamingnode.log 2>&1 &

‎scripts/start_standalone.sh

+2-1
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,12 @@ if [[ "$OSTYPE" == "linux-gnu"* ]]; then
2121
if test -f "$LIBJEMALLOC"; then
2222
#echo "Found $LIBJEMALLOC"
2323
export LD_PRELOAD="$LIBJEMALLOC"
24+
export MALLOC_CONF=background_thread:true,prof:true
2425
else
2526
echo "WARN: Cannot find $LIBJEMALLOC"
2627
fi
2728
export LD_LIBRARY_PATH=$PWD/internal/core/output/lib/:$LD_LIBRARY_PATH
2829
fi
2930

3031
echo "Starting standalone..."
31-
nohup ./bin/milvus run standalone --run-with-subprocess > /tmp/standalone.log 2>&1 &
32+
nohup ./bin/milvus run standalone --run-with-subprocess >/tmp/standalone.log 2>&1 &

0 commit comments

Comments
 (0)