From 9000270270f9e6c187a6888bf16ca1f089a6e21a Mon Sep 17 00:00:00 2001 From: Ezra Chung Date: Tue, 15 Apr 2025 14:54:31 -0500 Subject: [PATCH 1/3] Download and build-from-source valgrind 3.24.0 when not present --- .../components/funcs/install_valgrind.py | 46 ++++++++++++++ .../config_generator/components/funcs/test.py | 1 + .../config_generator/components/valgrind.py | 2 + .evergreen/generated_configs/functions.yml | 63 +++++++++++++++++++ .evergreen/generated_configs/tasks.yml | 9 +++ .evergreen/scripts/test.sh | 12 +--- .evergreen/scripts/valgrind-installer.sh | 18 ++++++ 7 files changed, 141 insertions(+), 10 deletions(-) create mode 100644 .evergreen/config_generator/components/funcs/install_valgrind.py create mode 100755 .evergreen/scripts/valgrind-installer.sh diff --git a/.evergreen/config_generator/components/funcs/install_valgrind.py b/.evergreen/config_generator/components/funcs/install_valgrind.py new file mode 100644 index 0000000000..58a4a75151 --- /dev/null +++ b/.evergreen/config_generator/components/funcs/install_valgrind.py @@ -0,0 +1,46 @@ +from config_generator.components.funcs.set_cache_dir import SetCacheDir + +from config_generator.etc.function import Function +from config_generator.etc.utils import bash_exec + +from shrub.v3.evg_command import EvgCommandType, expansions_update + + +class InstallValgrind(Function): + name = 'install-valgrind' + commands = SetCacheDir.commands + [ + bash_exec( + command_type=EvgCommandType.SETUP, + script='''\ + set -o errexit + set -o pipefail + + if [[ ! -n "${MONGO_CXX_DRIVER_CACHE_DIR}" ]]; then + echo "MONGO_CXX_DRIVER_CACHE_DIR is not defined!" 1>&2 + exit 1 + fi + + valgrind_install_dir="${MONGO_CXX_DRIVER_CACHE_DIR}/valgrind-3.24.0" + mkdir -p "$valgrind_install_dir" + + if ! command -v "$valgrind_install_dir/bin/valgrind" 2>/dev/null; then + env \ + install_prefix="${MONGO_CXX_DRIVER_CACHE_DIR}/valgrind-3.24.0" \ + mongo-cxx-driver/.evergreen/scripts/valgrind-installer.sh + fi + + PATH="$valgrind_install_dir/bin:$PATH" command -V valgrind + PATH="$valgrind_install_dir/bin:$PATH" valgrind --version + + printf "VALGRIND_INSTALL_DIR: %s\\n" "$valgrind_install_dir/bin" >|expansions.valgrind.yml + ''', + ), + expansions_update( + command_type=EvgCommandType.SETUP, + file='expansions.valgrind.yml', + ), + ] + + +def functions(): + return InstallValgrind.defn() diff --git a/.evergreen/config_generator/components/funcs/test.py b/.evergreen/config_generator/components/funcs/test.py index 0734e97875..8c65210db5 100644 --- a/.evergreen/config_generator/components/funcs/test.py +++ b/.evergreen/config_generator/components/funcs/test.py @@ -37,6 +37,7 @@ class Test(Function): 'TEST_WITH_VALGRIND', 'use_mongocryptd', 'USE_STATIC_LIBS', + 'VALGRIND_INSTALL_DIR', ], working_dir='mongo-cxx-driver', script='.evergreen/scripts/test.sh', diff --git a/.evergreen/config_generator/components/valgrind.py b/.evergreen/config_generator/components/valgrind.py index 48140b8a0b..a2609c8fef 100644 --- a/.evergreen/config_generator/components/valgrind.py +++ b/.evergreen/config_generator/components/valgrind.py @@ -1,6 +1,7 @@ from config_generator.components.funcs.compile import Compile from config_generator.components.funcs.fetch_det import FetchDET from config_generator.components.funcs.install_c_driver import InstallCDriver +from config_generator.components.funcs.install_valgrind import InstallValgrind from config_generator.components.funcs.install_uv import InstallUV from config_generator.components.funcs.run_kms_servers import RunKMSServers from config_generator.components.funcs.setup import Setup @@ -67,6 +68,7 @@ def tasks(): commands += [ Setup.call(), + InstallValgrind.call(), StartMongod.call(mongodb_version=mongodb_version, topology=topology), InstallCDriver.call(vars=icd_vars), InstallUV.call(), diff --git a/.evergreen/generated_configs/functions.yml b/.evergreen/generated_configs/functions.yml index dec36b12e2..50fa9321e6 100644 --- a/.evergreen/generated_configs/functions.yml +++ b/.evergreen/generated_configs/functions.yml @@ -392,6 +392,68 @@ functions: type: setup params: file: expansions.uv.yml + install-valgrind: + - command: subprocess.exec + type: setup + params: + binary: bash + args: + - -c + - | + if [[ -n "$XDG_CACHE_DIR" ]]; then + cache_dir="$XDG_CACHE_DIR" # XDG Base Directory specification. + elif [[ -n "$LOCALAPPDATA" ]]; then + cache_dir="$LOCALAPPDATA" # Windows. + elif [[ -n "$USERPROFILE" ]]; then + cache_dir="$USERPROFILE/.cache" # Windows (fallback). + elif [[ -d "$HOME/Library/Caches" ]]; then + cache_dir="$HOME/Library/Caches" # MacOS. + elif [[ -n "$HOME" ]]; then + cache_dir="$HOME/.cache" # Linux-like. + elif [[ -d ~/.cache ]]; then + cache_dir="~/.cache" # Linux-like (fallback). + else + cache_dir="$(pwd)/.cache" # EVG task directory (fallback). + fi + + mkdir -p "$cache_dir/mongo-cxx-driver" || exit + cache_dir="$(cd "$cache_dir/mongo-cxx-driver" && pwd)" || exit + + printf "MONGO_CXX_DRIVER_CACHE_DIR: %s\n" "$cache_dir" >|expansions.set-cache-dir.yml + - command: expansions.update + type: setup + params: + file: expansions.set-cache-dir.yml + - command: subprocess.exec + type: setup + params: + binary: bash + args: + - -c + - | + set -o errexit + set -o pipefail + + if [[ ! -n "${MONGO_CXX_DRIVER_CACHE_DIR}" ]]; then + echo "MONGO_CXX_DRIVER_CACHE_DIR is not defined!" 1>&2 + exit 1 + fi + + valgrind_install_dir="${MONGO_CXX_DRIVER_CACHE_DIR}/valgrind-3.24.0" + mkdir -p "$valgrind_install_dir" + + if ! command -v "$valgrind_install_dir/bin/valgrind" 2>/dev/null; then + env install_prefix="${MONGO_CXX_DRIVER_CACHE_DIR}/valgrind-3.24.0" mongo-cxx-driver/.evergreen/scripts/valgrind-installer.sh + fi + + PATH="$valgrind_install_dir/bin:$PATH" command -V valgrind + PATH="$valgrind_install_dir/bin:$PATH" valgrind --version + + printf "VALGRIND_INSTALL_DIR: %s\n" "$valgrind_install_dir/bin" >|expansions.valgrind.yml + - command: expansions.update + type: setup + params: + file: expansions.valgrind.yml install_c_driver: - command: expansions.update type: setup @@ -607,6 +669,7 @@ functions: - TEST_WITH_VALGRIND - use_mongocryptd - USE_STATIC_LIBS + - VALGRIND_INSTALL_DIR args: - -c - .evergreen/scripts/test.sh diff --git a/.evergreen/generated_configs/tasks.yml b/.evergreen/generated_configs/tasks.yml index fcb7f06fca..3230c99b0a 100644 --- a/.evergreen/generated_configs/tasks.yml +++ b/.evergreen/generated_configs/tasks.yml @@ -15889,6 +15889,7 @@ tasks: updates: - { key: build_type, value: Debug } - func: setup + - func: install-valgrind - func: start_mongod vars: TOPOLOGY: replica_set @@ -15918,6 +15919,7 @@ tasks: updates: - { key: build_type, value: Debug } - func: setup + - func: install-valgrind - func: start_mongod vars: TOPOLOGY: sharded_cluster @@ -15947,6 +15949,7 @@ tasks: updates: - { key: build_type, value: Debug } - func: setup + - func: install-valgrind - func: start_mongod vars: mongodb_version: "4.0" @@ -15975,6 +15978,7 @@ tasks: updates: - { key: build_type, value: Debug } - func: setup + - func: install-valgrind - func: start_mongod vars: TOPOLOGY: replica_set @@ -16004,6 +16008,7 @@ tasks: updates: - { key: build_type, value: Debug } - func: setup + - func: install-valgrind - func: start_mongod vars: TOPOLOGY: sharded_cluster @@ -16033,6 +16038,7 @@ tasks: updates: - { key: build_type, value: Debug } - func: setup + - func: install-valgrind - func: start_mongod vars: mongodb_version: "8.0" @@ -16061,6 +16067,7 @@ tasks: updates: - { key: build_type, value: Debug } - func: setup + - func: install-valgrind - func: start_mongod vars: TOPOLOGY: replica_set @@ -16090,6 +16097,7 @@ tasks: updates: - { key: build_type, value: Debug } - func: setup + - func: install-valgrind - func: start_mongod vars: TOPOLOGY: sharded_cluster @@ -16119,6 +16127,7 @@ tasks: updates: - { key: build_type, value: Debug } - func: setup + - func: install-valgrind - func: start_mongod vars: mongodb_version: latest diff --git a/.evergreen/scripts/test.sh b/.evergreen/scripts/test.sh index 3a7fc2f56d..ccd2c622cf 100755 --- a/.evergreen/scripts/test.sh +++ b/.evergreen/scripts/test.sh @@ -30,6 +30,7 @@ set -o pipefail : "${TEST_WITH_VALGRIND:-}" : "${use_mongocryptd:-}" : "${USE_STATIC_LIBS:-}" +: "${VALGRIND_INSTALL_DIR:-}" # Only when `TEST_WITH_VALGRIND` is set to "ON". working_dir="$(pwd)" @@ -282,16 +283,7 @@ else export UBSAN_OPTIONS="print_stacktrace=1" export PATH="/opt/mongodbtoolchain/v4/bin:${PATH:-}" # llvm-symbolizer elif [[ "${TEST_WITH_VALGRIND:-}" == "ON" ]]; then - if ! command -v valgrind >/dev/null; then - if command -v yum >/dev/null; then - sudo yum install -q -y valgrind - elif command -v apt-get >/dev/null; then - sudo apt-get install -q -y valgrind - else - echo "Unknown how to install valgrind on this distro: ${distro_id:?}" 1>&2 - exit 1 - fi - fi + PATH="${VALGRIND_INSTALL_DIR:?}:${PATH:-}" valgrind --version run_test() { valgrind --leak-check=full --track-origins=yes --num-callers=50 --error-exitcode=1 --error-limit=no --read-var-info=yes --suppressions=../etc/memcheck.suppressions "$@" "${test_args[@]:?}" diff --git a/.evergreen/scripts/valgrind-installer.sh b/.evergreen/scripts/valgrind-installer.sh new file mode 100755 index 0000000000..66bef883a5 --- /dev/null +++ b/.evergreen/scripts/valgrind-installer.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +: "${install_prefix:?}" + +cd "$(mktemp -d)" + +# https://valgrind.org/downloads/current.html +curl -sSL -m 60 --retry 5 -o valgrind-3.24.0.tar.bz2 https://sourceware.org/pub/valgrind/valgrind-3.24.0.tar.bz2 +cat >checksum.txt <<<'6fc0470fedc0d85dae3e042297cabd13c6100749 *valgrind-3.24.0.tar.bz2' +sha1sum -c checksum.txt >/dev/null + +tar -xjf valgrind-3.24.0.tar.bz2 +cd valgrind-3.24.0 + +# https://valgrind.org/docs/manual/manual-core.html#manual-core.install +./configure --prefix "${install_prefix:?}" >/dev/null +make --no-print-directory -j "$(nproc)" >/dev/null +make --no-print-directory install >/dev/null From 12a9ac408705fbb851a70881cc72e38b9b4dafa2 Mon Sep 17 00:00:00 2001 From: Ezra Chung Date: Thu, 17 Apr 2025 11:16:44 -0500 Subject: [PATCH 2/3] Set errexit and pipefail --- .evergreen/scripts/valgrind-installer.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.evergreen/scripts/valgrind-installer.sh b/.evergreen/scripts/valgrind-installer.sh index 66bef883a5..29e5e9f3fe 100755 --- a/.evergreen/scripts/valgrind-installer.sh +++ b/.evergreen/scripts/valgrind-installer.sh @@ -1,5 +1,8 @@ #!/usr/bin/env bash +set -o errexit +set -o pipefail + : "${install_prefix:?}" cd "$(mktemp -d)" From 6985ddb8ee437bf393726c472ed90210ca3ddf25 Mon Sep 17 00:00:00 2001 From: Ezra Chung Date: Thu, 17 Apr 2025 11:19:55 -0500 Subject: [PATCH 3/3] Fix formatting of Bash commands in EVG config --- .../config_generator/components/funcs/install_valgrind.py | 4 ++-- .evergreen/generated_configs/functions.yml | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.evergreen/config_generator/components/funcs/install_valgrind.py b/.evergreen/config_generator/components/funcs/install_valgrind.py index 58a4a75151..14baf123ad 100644 --- a/.evergreen/config_generator/components/funcs/install_valgrind.py +++ b/.evergreen/config_generator/components/funcs/install_valgrind.py @@ -24,8 +24,8 @@ class InstallValgrind(Function): mkdir -p "$valgrind_install_dir" if ! command -v "$valgrind_install_dir/bin/valgrind" 2>/dev/null; then - env \ - install_prefix="${MONGO_CXX_DRIVER_CACHE_DIR}/valgrind-3.24.0" \ + env \\ + install_prefix="${MONGO_CXX_DRIVER_CACHE_DIR}/valgrind-3.24.0" \\ mongo-cxx-driver/.evergreen/scripts/valgrind-installer.sh fi diff --git a/.evergreen/generated_configs/functions.yml b/.evergreen/generated_configs/functions.yml index 50fa9321e6..c23ad4baea 100644 --- a/.evergreen/generated_configs/functions.yml +++ b/.evergreen/generated_configs/functions.yml @@ -443,7 +443,9 @@ functions: mkdir -p "$valgrind_install_dir" if ! command -v "$valgrind_install_dir/bin/valgrind" 2>/dev/null; then - env install_prefix="${MONGO_CXX_DRIVER_CACHE_DIR}/valgrind-3.24.0" mongo-cxx-driver/.evergreen/scripts/valgrind-installer.sh + env \ + install_prefix="${MONGO_CXX_DRIVER_CACHE_DIR}/valgrind-3.24.0" \ + mongo-cxx-driver/.evergreen/scripts/valgrind-installer.sh fi PATH="$valgrind_install_dir/bin:$PATH" command -V valgrind