diff --git a/tests/performance_tests/shell_test_utils/run_perf_test.sh b/tests/performance_tests/shell_test_utils/run_perf_test.sh index 8e649c90f9d..8fd68be8099 100755 --- a/tests/performance_tests/shell_test_utils/run_perf_test.sh +++ b/tests/performance_tests/shell_test_utils/run_perf_test.sh @@ -40,7 +40,10 @@ YQ=/usr/local/bin/yq mkdir -p "$RESULTS_ROOT" RESULTS_JSON="$RESULTS_ROOT/results.json" SERVER_LOG_DIR="$RESULTS_ROOT/server_logs" -mkdir -p "$SERVER_LOG_DIR" +# launch_jet_workload.py retries unless torchrun's per-rank std*.log assets exist. +ASSETS_ROOT="$(dirname "$RESULTS_ROOT")" +TORCHRUN_LOG_DIR="$ASSETS_ROOT/logs/1" +mkdir -p "$SERVER_LOG_DIR" "$TORCHRUN_LOG_DIR" # Clean any pre-existing results.json so this run starts fresh. rm -f "$RESULTS_JSON" @@ -150,6 +153,9 @@ SERVER_COMMON_ARGS=( --nproc-per-node "$WORLD_SIZE" \ --master_addr "$MASTER_ADDR" \ --master_port "$MASTER_PORT" \ + --log-dir "$TORCHRUN_LOG_DIR" \ + --tee "0:3" \ + --redirects "3" \ -m tools.run_dynamic_text_generation_server \ "${SERVER_COMMON_ARGS[@]}" \ "${MODEL_ARGS[@]}" \