From de96d1d3448fa6f5ecd9c3de636f2f881e1d4483 Mon Sep 17 00:00:00 2001 From: Fathi Boudra Date: Wed, 12 Aug 2026 01:06:57 +0200 Subject: [PATCH] common: migrate the deprecated --mmap/--no-mmap to --load-mode Replace the deprecated --mmap, --no-mmap, --mlock, and --direct-io flags with the unified --load-mode argument across scripts, examples, and documentation. Internal warning message and env var docs updated accordingly. Signed-off-by: Fathi Boudra --- docs/backend/SYCL.md | 8 ++++---- docs/backend/snapdragon/developer.md | 2 +- examples/sycl/run-llama2.sh | 2 +- examples/sycl/start-svr.sh | 4 ++-- examples/sycl/test.sh | 4 ++-- examples/sycl/win-run-llama2.bat | 2 +- examples/sycl/win-start-svr.bat | 4 ++-- examples/sycl/win-test.bat | 4 ++-- scripts/bench-models.sh | 4 ++-- scripts/snapdragon/adb/run-bench.sh | 2 +- scripts/snapdragon/adb/run-cli.sh | 2 +- scripts/snapdragon/adb/run-completion.sh | 2 +- scripts/snapdragon/adb/run-mtmd.sh | 2 +- scripts/snapdragon/windows/run-bench.ps1 | 2 +- scripts/snapdragon/windows/run-cli.ps1 | 2 +- scripts/snapdragon/windows/run-completion.ps1 | 2 +- scripts/snapdragon/windows/run-mtmd.ps1 | 2 +- src/llama-model-loader.cpp | 2 +- tools/completion/README.md | 16 +++++++++------- tools/llama-bench/README.md | 4 ++-- tools/server/README.md | 17 ++++++++++++----- 21 files changed, 49 insertions(+), 40 deletions(-) diff --git a/docs/backend/SYCL.md b/docs/backend/SYCL.md index 73f89a70632e..01374227aaa3 100644 --- a/docs/backend/SYCL.md +++ b/docs/backend/SYCL.md @@ -428,13 +428,13 @@ Examples: - Use device 0: ```sh -ZES_ENABLE_SYSMAN=1 ./build/bin/llama-completion -no-cnv -m models/llama-2-7b.Q4_0.gguf -p "Building a website can be done in 10 simple steps:" -n 400 -e -ngl 99 -sm none -mg 0 --mmap +ZES_ENABLE_SYSMAN=1 ./build/bin/llama-completion -no-cnv -m models/llama-2-7b.Q4_0.gguf -p "Building a website can be done in 10 simple steps:" -n 400 -e -ngl 99 -sm none -mg 0 --load-mode auto ``` - Use multiple devices: ```sh -ZES_ENABLE_SYSMAN=1 ./build/bin/llama-completion -no-cnv -m models/llama-2-7b.Q4_0.gguf -p "Building a website can be done in 10 simple steps:" -n 400 -e -ngl 99 -sm layer --mmap +ZES_ENABLE_SYSMAN=1 ./build/bin/llama-completion -no-cnv -m models/llama-2-7b.Q4_0.gguf -p "Building a website can be done in 10 simple steps:" -n 400 -e -ngl 99 -sm layer --load-mode auto ``` *Notes:* @@ -741,13 +741,13 @@ Examples: - Use device 0: ``` -build\bin\llama-completion.exe -no-cnv -m models\llama-2-7b.Q4_0.gguf -p "Building a website can be done in 10 simple steps:\nStep 1:" -n 400 -e -ngl 99 -sm none -mg 0 --mmap +build\bin\llama-completion.exe -no-cnv -m models\llama-2-7b.Q4_0.gguf -p "Building a website can be done in 10 simple steps:\nStep 1:" -n 400 -e -ngl 99 -sm none -mg 0 --load-mode auto ``` - Use multiple devices: ``` -build\bin\llama-completion.exe -no-cnv -m models\llama-2-7b.Q4_0.gguf -p "Building a website can be done in 10 simple steps:\nStep 1:" -n 400 -e -ngl 99 -sm layer --mmap +build\bin\llama-completion.exe -no-cnv -m models\llama-2-7b.Q4_0.gguf -p "Building a website can be done in 10 simple steps:\nStep 1:" -n 400 -e -ngl 99 -sm layer --load-mode auto ``` diff --git a/docs/backend/snapdragon/developer.md b/docs/backend/snapdragon/developer.md index fc4d160e9392..9d56638e3d59 100644 --- a/docs/backend/snapdragon/developer.md +++ b/docs/backend/snapdragon/developer.md @@ -53,7 +53,7 @@ M=gpt-oss-20b-Q4_0.gguf NDEV=4 D=HTP0,HTP1,HTP2,HTP3 P=surfing.txt scripts/snapd ... LD_LIBRARY_PATH=/data/local/tmp/llama.cpp/lib ADSP_LIBRARY_PATH=/data/local/tmp/llama.cpp/lib -GGML_HEXAGON_NDEV=4 ./bin/llama-cli --no-mmap -m /data/local/tmp/llama.cpp/../gguf/gpt-oss-20b-Q4_0.gguf +GGML_HEXAGON_NDEV=4 ./bin/llama-cli --load-mode none -m /data/local/tmp/llama.cpp/../gguf/gpt-oss-20b-Q4_0.gguf -t 4 --ctx-size 8192 --batch-size 128 -ctk q8_0 -ctv q8_0 -fa on -ngl 99 --device HTP0,HTP1,HTP2,HTP3 -no-cnv -f surfing.txt ... llama_model_loader: - type f32: 289 tensors diff --git a/examples/sycl/run-llama2.sh b/examples/sycl/run-llama2.sh index 6ed2535bbb83..c5490a51505d 100755 --- a/examples/sycl/run-llama2.sh +++ b/examples/sycl/run-llama2.sh @@ -18,7 +18,7 @@ CONTEXT=4096 #support malloc device memory more than 4GB. export UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=1 -LOAD_MODE='--mmap' +LOAD_MODE='--load-mode auto' if [ $# -gt 0 ]; then GGML_SYCL_DEVICE=$1 echo "use $GGML_SYCL_DEVICE as main GPU" diff --git a/examples/sycl/start-svr.sh b/examples/sycl/start-svr.sh index 49177ba2dc5d..c3e1b6b998f3 100755 --- a/examples/sycl/start-svr.sh +++ b/examples/sycl/start-svr.sh @@ -124,7 +124,7 @@ else GPUS_SETTING="-sm ${SPLIT_MODE}" fi -echo "run cmd: ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE} --device ${SYCL_DEVICES} --mmap --host 0.0.0.0 --port 8000" -ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE} --device ${SYCL_DEVICES} --mmap --host 0.0.0.0 --port 8000 +echo "run cmd: ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE} --device ${SYCL_DEVICES} --load-mode auto --host 0.0.0.0 --port 8000" +ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE} --device ${SYCL_DEVICES} --load-mode auto --host 0.0.0.0 --port 8000 diff --git a/examples/sycl/test.sh b/examples/sycl/test.sh index b9498f49b78f..28c2dcb20a6e 100755 --- a/examples/sycl/test.sh +++ b/examples/sycl/test.sh @@ -133,6 +133,6 @@ else GPUS_SETTING="-sm ${SPLIT_MODE}" fi -echo "run cmd: ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -no-cnv -p "${INPUT_PROMPT}" -n 200 -e -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE} --device ${SYCL_DEVICES} --mmap " -ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -no-cnv -p "${INPUT_PROMPT}" -n 200 -e -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE} --device ${SYCL_DEVICES} --mmap +echo "run cmd: ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -no-cnv -p "${INPUT_PROMPT}" -n 200 -e -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE} --device ${SYCL_DEVICES} --load-mode auto " +ZES_ENABLE_SYSMAN=1 ${BIN_FILE} -m ${MODEL_FILE} -no-cnv -p "${INPUT_PROMPT}" -n 200 -e -ngl ${NGL} -s ${SEED} -c ${CONTEXT} ${GPUS_SETTING} -lv ${LOG_VERBOSE} --device ${SYCL_DEVICES} --load-mode auto diff --git a/examples/sycl/win-run-llama2.bat b/examples/sycl/win-run-llama2.bat index 1f2dab8d0a84..8bc47887d261 100644 --- a/examples/sycl/win-run-llama2.bat +++ b/examples/sycl/win-run-llama2.bat @@ -7,5 +7,5 @@ set INPUT2="Building a website can be done in 10 simple steps:\nStep 1:" :: support malloc device memory more than 4GB. set UR_L0_ENABLE_RELAXED_ALLOCATION_LIMITS=1 -set LOAD_MODE="--mmap" +set LOAD_MODE="--load-mode auto" .\build\bin\llama-completion.exe -m models\llama-2-7b.Q4_0.gguf -no-cnv -p %INPUT2% -n 400 -e -ngl 99 -s 0 %LOAD_MODE% diff --git a/examples/sycl/win-start-svr.bat b/examples/sycl/win-start-svr.bat index 807710589305..474212c992e0 100644 --- a/examples/sycl/win-start-svr.bat +++ b/examples/sycl/win-start-svr.bat @@ -188,9 +188,9 @@ if not "%GGML_SYCL_DEVICE%"=="-1" ( set "GPUS_SETTING=-sm %SPLIT_MODE%" ) -echo run cmd: ZES_ENABLE_SYSMAN=1 %BIN_FILE% -m "%MODEL_FILE%" -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --device %SYCL_DEVICES% --mmap --host 0.0.0.0 --port 8000 +echo run cmd: ZES_ENABLE_SYSMAN=1 %BIN_FILE% -m "%MODEL_FILE%" -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --device %SYCL_DEVICES% --load-mode auto --host 0.0.0.0 --port 8000 set "ZES_ENABLE_SYSMAN=1" -%BIN_FILE% -m "%MODEL_FILE%" -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --device "%SYCL_DEVICES%" --mmap --host 0.0.0.0 --port 8000 +%BIN_FILE% -m "%MODEL_FILE%" -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --device "%SYCL_DEVICES%" --load-mode auto --host 0.0.0.0 --port 8000 endlocal diff --git a/examples/sycl/win-test.bat b/examples/sycl/win-test.bat index cc6da4413889..a7c3dbb79a78 100644 --- a/examples/sycl/win-test.bat +++ b/examples/sycl/win-test.bat @@ -211,9 +211,9 @@ else ( set "GPUS_SETTING=-sm %SPLIT_MODE%" ) -echo run cmd: ZES_ENABLE_SYSMAN=1 %BIN_FILE% -m %MODEL_FILE% -no-cnv -p "%INPUT_PROMPT%" -n 200 -e -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --device %SYCL_DEVICES% --mmap +echo run cmd: ZES_ENABLE_SYSMAN=1 %BIN_FILE% -m %MODEL_FILE% -no-cnv -p "%INPUT_PROMPT%" -n 200 -e -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --device %SYCL_DEVICES% --load-mode auto set "ZES_ENABLE_SYSMAN=1" -%BIN_FILE% -m "%MODEL_FILE%" -no-cnv -p "%INPUT_PROMPT%" -n 200 -e -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --device "%SYCL_DEVICES%" --mmap +%BIN_FILE% -m "%MODEL_FILE%" -no-cnv -p "%INPUT_PROMPT%" -n 200 -e -ngl %NGL% -s %SEED% -c %CONTEXT% %GPUS_SETTING% -lv %LOG_VERBOSE% --device "%SYCL_DEVICES%" --load-mode auto endlocal diff --git a/scripts/bench-models.sh b/scripts/bench-models.sh index c241013040f6..205f2d6b42d7 100755 --- a/scripts/bench-models.sh +++ b/scripts/bench-models.sh @@ -22,8 +22,8 @@ if (( QUICK )); then fi if (( DIO )); then - ARGS_BB="${ARGS_BB} --no-mmap --direct-io" - ARGS_B="${ARGS_B} -mmp 0 -dio 1" + ARGS_BB="${ARGS_BB} --load-mode dio" + ARGS_B="${ARGS_B} --load-mode dio" fi run_model() { diff --git a/scripts/snapdragon/adb/run-bench.sh b/scripts/snapdragon/adb/run-bench.sh index bbe7146b4445..eaae80a77d6f 100755 --- a/scripts/snapdragon/adb/run-bench.sh +++ b/scripts/snapdragon/adb/run-bench.sh @@ -43,7 +43,7 @@ adb $adbserial $adbhost shell " \ cd $basedir; \ LD_LIBRARY_PATH=$basedir/$branch/lib \ ADSP_LIBRARY_PATH=$basedir/$branch/lib \ - $ndev $nhvx $opmask $verbose $profile $hb ./$branch/bin/llama-bench --device $device --mmap 0 -m $basedir/../gguf/$model \ + $ndev $nhvx $opmask $verbose $profile $hb ./$branch/bin/llama-bench --device $device --load-mode none -m $basedir/../gguf/$model \ --poll 1000 -t 6 --cpu-mask 0xfc --cpu-strict 1 \ --ubatch-size 1024 -fa 1 -ngl 99 $cli_opts $@ \ " diff --git a/scripts/snapdragon/adb/run-cli.sh b/scripts/snapdragon/adb/run-cli.sh index 48127dfa2522..27a4a14195f0 100755 --- a/scripts/snapdragon/adb/run-cli.sh +++ b/scripts/snapdragon/adb/run-cli.sh @@ -71,7 +71,7 @@ adb $adbserial $adbhost shell " \ LD_LIBRARY_PATH=$basedir/$branch/lib \ ADSP_LIBRARY_PATH=$basedir/$branch/lib \ $verbose $sched $opmask $profile $nhvx $hmx $ndev $hb $opbatch $opqueue $opflt $vmem $mbuf \ - ./$branch/bin/llama-cli --no-mmap -m $basedir/../gguf/$model \ + ./$branch/bin/llama-cli --load-mode none -m $basedir/../gguf/$model \ --poll 1000 -t 6 --cpu-mask 0xfc --cpu-strict 1 \ --ctx-size 8192 --ubatch-size 1024 -fa on \ -ngl 99 --device $device $cli_opts $@ \ diff --git a/scripts/snapdragon/adb/run-completion.sh b/scripts/snapdragon/adb/run-completion.sh index 2130b9a74f6c..30893ed293a2 100755 --- a/scripts/snapdragon/adb/run-completion.sh +++ b/scripts/snapdragon/adb/run-completion.sh @@ -79,7 +79,7 @@ adb $adbserial $adbhost shell " \ LD_LIBRARY_PATH=$basedir/$branch/lib \ ADSP_LIBRARY_PATH=$basedir/$branch/lib \ $verbose $sched $opmask $profile $nhvx $hmx $ndev $hb $opbatch $opqueue $oppoll $opflt $opfuse $vmem $mbuf $mmsel $fasel \ - ./$branch/bin/llama-completion --no-mmap -m $basedir/../gguf/$model \ + ./$branch/bin/llama-completion --load-mode none -m $basedir/../gguf/$model \ --poll 1000 -t 6 --cpu-mask 0xfc --cpu-strict 1 \ --ctx-size 8192 --ubatch-size 1024 -fa on \ -ngl 99 --device $device $cli_opts $@ \ diff --git a/scripts/snapdragon/adb/run-mtmd.sh b/scripts/snapdragon/adb/run-mtmd.sh index 992045cb9b3e..65dd6ec59e51 100755 --- a/scripts/snapdragon/adb/run-mtmd.sh +++ b/scripts/snapdragon/adb/run-mtmd.sh @@ -62,7 +62,7 @@ adb $adbserial $adbhost shell " \ LD_LIBRARY_PATH=$basedir/$branch/lib \ ADSP_LIBRARY_PATH=$basedir/$branch/lib \ $verbose $experimental $sched $opmask $profile $hmx $nhvx $ndev $mtmd_backend \ - ./$branch/bin/llama-mtmd-cli --no-mmap -m $basedir/../gguf/$model \ + ./$branch/bin/llama-mtmd-cli --load-mode none -m $basedir/../gguf/$model \ --mmproj $basedir/../gguf/$mmproj \ --image $basedir/../gguf/$image \ --poll 1000 -t 6 --cpu-mask 0xfc --cpu-strict 1 \ diff --git a/scripts/snapdragon/windows/run-bench.ps1 b/scripts/snapdragon/windows/run-bench.ps1 index 5ee81df6889d..6eb656e66d32 100644 --- a/scripts/snapdragon/windows/run-bench.ps1 +++ b/scripts/snapdragon/windows/run-bench.ps1 @@ -43,6 +43,6 @@ if ($null -ne $env:HB) { $env:ADSP_LIBRARY_PATH="$basedir\lib" & "$basedir\bin\llama-bench.exe" ` - --mmap 0 -m $basedir\..\..\gguf\$model ` + --load-mode none -m $basedir\..\..\gguf\$model ` --poll 1000 -t 6 --cpu-mask 0xfc --cpu-strict 1 ` --ubatch-size 1024 -ngl 99 --device $device $cli_opts diff --git a/scripts/snapdragon/windows/run-cli.ps1 b/scripts/snapdragon/windows/run-cli.ps1 index b51149bec25b..5da8bff33e34 100644 --- a/scripts/snapdragon/windows/run-cli.ps1 +++ b/scripts/snapdragon/windows/run-cli.ps1 @@ -47,7 +47,7 @@ if ($null -ne $env:HB) { $env:ADSP_LIBRARY_PATH="$basedir\lib" & "$basedir\bin\llama-cli.exe" ` - --no-mmap -m $basedir\..\..\gguf\$model ` + --load-mode none -m $basedir\..\..\gguf\$model ` --poll 1000 -t 6 --cpu-mask 0xfc --cpu-strict 1 ` --ctx-size 8192 --ubatch-size 1024 -fa on ` -ngl 99 --device $device $cli_opts diff --git a/scripts/snapdragon/windows/run-completion.ps1 b/scripts/snapdragon/windows/run-completion.ps1 index ffce8184dc07..08ef139b7e2a 100644 --- a/scripts/snapdragon/windows/run-completion.ps1 +++ b/scripts/snapdragon/windows/run-completion.ps1 @@ -47,7 +47,7 @@ if ($null -ne $env:HB) { $env:ADSP_LIBRARY_PATH="$basedir\lib" & "$basedir\bin\llama-completion.exe" ` - --no-mmap -m $basedir\..\..\gguf\$model ` + --load-mode none -m $basedir\..\..\gguf\$model ` --poll 1000 -t 6 --cpu-mask 0xfc --cpu-strict 1 ` --ctx-size 8192 --ubatch-size 1024 -fa on ` -ngl 99 -no-cnv --device $device $cli_opts diff --git a/scripts/snapdragon/windows/run-mtmd.ps1 b/scripts/snapdragon/windows/run-mtmd.ps1 index b38fae35fe45..6e270ec90b57 100644 --- a/scripts/snapdragon/windows/run-mtmd.ps1 +++ b/scripts/snapdragon/windows/run-mtmd.ps1 @@ -60,7 +60,7 @@ if ($null -ne $env:MTMD_DEVICE) { $env:ADSP_LIBRARY_PATH="$basedir\lib" & "$basedir\bin\llama-mtmd-cli.exe" ` - --no-mmap -m $basedir\..\..\gguf\$model ` + --load-mode none -m $basedir\..\..\gguf\$model ` --mmproj $basedir\..\..\gguf\$mmproj ` --image $basedir\..\..\gguf\$image ` --poll 1000 -t 6 --cpu-mask 0xfc --cpu-strict 1 ` diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index 51ba05439682..2ffb1c255f5d 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -1178,7 +1178,7 @@ struct ggml_tensor * llama_model_loader::create_tensor( if (use_mmap) { static std::once_flag once; std::call_once(once, [] { - LLAMA_LOG_WARN("llama_model_loader: tensor overrides to CPU are used with mmap enabled - consider using --no-mmap for better performance\n"); + LLAMA_LOG_WARN("llama_model_loader: tensor overrides to CPU are used with mmap enabled - consider using --load-mode none for better performance\n"); }); } } else { diff --git a/tools/completion/README.md b/tools/completion/README.md index c2e52ac066e7..9c5d03fb1f75 100644 --- a/tools/completion/README.md +++ b/tools/completion/README.md @@ -523,13 +523,15 @@ These options help improve the performance and memory usage of the LLaMA models. - `-t N, --threads N`: Set the number of threads to use during generation. For optimal performance, it is recommended to set this value to the number of physical CPU cores your system has (as opposed to the logical number of cores). Using the correct number of threads can greatly improve performance. - `-tb N, --threads-batch N`: Set the number of threads to use during batch and prompt processing. In some systems, it is beneficial to use a higher number of threads during batch processing than during generation. If not specified, the number of threads used for batch processing will be the same as the number of threads used for generation. -### Mlock - -- `--mlock`: Lock the model in memory, preventing it from being swapped out when memory-mapped. This can improve performance but trades away some of the advantages of memory-mapping by requiring more RAM to run and potentially slowing down load times as the model loads into RAM. - -### No Memory Mapping - -- `--no-mmap`: Do not memory-map the model. By default, models are mapped into memory, which allows the system to load only the necessary parts of the model as needed. However, if the model is larger than your total amount of RAM or if your system is low on available memory, using mmap might increase the risk of pageouts, negatively impacting performance. Disabling mmap results in slower load times but may reduce pageouts if you're not using `--mlock`. Note that if the model is larger than the total amount of RAM, turning off mmap would prevent the model from loading at all. +### Model Loading Mode + +- `-lm MODE, --load-mode MODE`: Specify the model loading mode (default: `auto`). + - `auto`: Memory-map the model, unless the device does not support it. + - `none`: No special loading mode. Disabling mmap results in slower load times but may reduce pageouts if you're not using `mlock`. Note that if the model is larger than the total amount of RAM, turning off mmap would prevent the model from loading at all. + - `mmap`: Memory-map the model. + - `mlock`: Lock the model in memory, preventing it from being swapped out when memory-mapped. This can improve performance but trades away some of the advantages of memory-mapping by requiring more RAM to run and potentially slowing down load times as the model loads into RAM. + - `mmap+mlock`: Memory-map the model and lock it in memory. + - `dio`: Use DirectIO if available. ### NUMA support diff --git a/tools/llama-bench/README.md b/tools/llama-bench/README.md index d53978548a16..42cb14859f07 100644 --- a/tools/llama-bench/README.md +++ b/tools/llama-bench/README.md @@ -67,8 +67,8 @@ test parameters: -nkvo, --no-kv-offload <0|1> (default: 0) -fa, --flash-attn (default: auto) -dev, --device (default: auto) - -mmp, --mmap <0|1> (default: 1) - -dio, --direct-io <0|1> (default: 0) + -mmp, --mmap <0|1> (DEPRECATED IN FAVOUR OF --load-mode) + -dio, --direct-io <0|1> (DEPRECATED IN FAVOUR OF --load-mode) -embd, --embeddings <0|1> (default: 0) -ts, --tensor-split (default: 0) -ot --override-tensor =;... diff --git a/tools/server/README.md b/tools/server/README.md index a2ab872b4c1a..ec1be130bc5a 100644 --- a/tools/server/README.md +++ b/tools/server/README.md @@ -295,10 +295,17 @@ For the full list of features, please refer to [server's changelog](https://gith Note: If both command line argument and environment variable are both set for the same param, the argument will take precedence over env var. -For boolean options like `--mmap` or `--kv-offload`, the environment variable is handled as shown in this example: -- `LLAMA_ARG_MMAP=true` means enabled, other accepted values are: `1`, `on`, `enabled` -- `LLAMA_ARG_MMAP=false` means disabled, other accepted values are: `0`, `off`, `disabled` -- If `LLAMA_ARG_NO_MMAP` is present (no matter the value), it means disabling mmap +For string options like `--load-mode`, the environment variable is handled as shown in this example: +- `LLAMA_ARG_LOAD_MODE=auto` sets the loading mode to auto (default) +- `LLAMA_ARG_LOAD_MODE=none` disables special loading +- `LLAMA_ARG_LOAD_MODE=mmap` enables memory-mapping +- `LLAMA_ARG_LOAD_MODE=mlock` locks the model in RAM +- `LLAMA_ARG_LOAD_MODE=mmap+mlock` enables memory-mapping and locks in RAM +- `LLAMA_ARG_LOAD_MODE=dio` uses DirectIO if available + +For boolean options like `--kv-offload`: +- `LLAMA_ARG_KV_OFFLOAD=true` means enabled, other accepted values are: `1`, `on`, `enabled` +- `LLAMA_ARG_KV_OFFLOAD=false` means disabled, other accepted values are: `0`, `off`, `disabled` Example usage of docker compose with environment variables: @@ -1892,7 +1899,7 @@ Example events: } // note for "loading" status: // - subsequent events will follow the same order of "stages" list -// - mmap is may report incorrect progress on some platforms; if you need exact progress, use --no-mmap +// - mmap may report incorrect progress on some platforms; if you need exact progress, use --load-mode none { "model": "...",