diff --git a/tests/integration/defs/perf/utils.py b/tests/integration/defs/perf/utils.py index 24222bb83f3b..d5b3cab6e4fc 100644 --- a/tests/integration/defs/perf/utils.py +++ b/tests/integration/defs/perf/utils.py @@ -22,6 +22,7 @@ import signal import socket import subprocess +import tempfile import threading import time from datetime import datetime @@ -391,10 +392,17 @@ def start_server(self) -> None: "--host", self._host, "--port", str(self._port) ] - self._server_log_path = os.path.join(os.getcwd(), - "trtllm-serve-perf.log") print_info(f"Starting trtllm-serve: {' '.join(cmd)}") - self._server_log_file = open(self._server_log_path, "w") + # Cluster jobs can share a working directory. Create a unique log for + # each launch so another server cannot truncate its startup metrics. + self._server_log_file = tempfile.NamedTemporaryFile( + mode="w", + prefix="trtllm-serve-perf-", + suffix=".log", + dir=os.getcwd(), + delete=False) + self._server_log_path = self._server_log_file.name + print_info(f"trtllm-serve log: {self._server_log_path}") self._server_proc = subprocess.Popen(cmd, env=self.server_env, stdout=self._server_log_file,