Better error handling

Signed-off-by: Kaiyu Xie <[email protected]>

Author: kaiyux

examples/disaggregated/slurm/benchmark/disaggr_torch.slurm

@@ -7,6 +7,10 @@
 #SBATCH --job-name=${job_name} # add your job name here or specify in the sbatch command
 #SBATCH --time=02:00:00
 
+set -u
+set -e
+set -x
+
 # Context servers arguments
 num_ctx_servers=${1}
 ctx_tp_size=${2}
@@ -227,9 +231,10 @@ srun -l --container-name=${container_name} \
 
 # start benchmarking
 srun -l --container-name=${container_name} \
-        --container-mounts=${mounts} \
-        --mpi=pmix --overlap -N 1 -n 1 \
-        bash ${workdir}/run_benchmark.sh ${isl} ${osl} ${multi_round} ${model_dir} "${concurrency}" ${streaming} ${full_logdir} > ${full_logdir}/benchmark.log 2>&1
+    --container-mounts=${mounts} \
+    --mpi=pmix --overlap -N 1 -n 1 \
+    bash ${workdir}/run_benchmark.sh ${isl} ${osl} ${multi_round} ${model_dir} "${concurrency}" ${streaming} ${full_logdir} \
+        &> ${full_logdir}/benchmark.log 2>&1
 
 # try to kill the server and workers
 srun -l --container-name=${container_name} \

examples/disaggregated/slurm/benchmark/run_benchmark.sh

@@ -1,8 +1,7 @@
 #!/bin/bash
-
-# Add error handling
-set -e
 set -u
+set -e
+set -x
 trap 'echo "Error occurred at line $LINENO"; exit 1' ERR
 
 # Add parameter validation
@@ -26,7 +25,6 @@ if [[ ${SLURM_PROCID} != "0" ]]; then
     exit 0
 fi
 
-set -x
 config_file=${log_path}/server_config.yaml
 
 # check if the config file exists every 10 seconds timeout 1800 seconds

examples/disaggregated/slurm/benchmark/start_server.sh

@@ -1,4 +1,7 @@
 #!/bin/bash
+set -u
+set -e
+set -x
 
 num_ctx_servers=$1
 num_gen_servers=$2

examples/disaggregated/slurm/benchmark/start_worker.sh

@@ -1,4 +1,6 @@
 #! /bin/bash
+set -u
+set -e
 set -x
 
 role=$1