From b74eee6954d7444ccfb12b2800e73030c7a2088c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?oliver=20k=C3=B6nig?= Date: Fri, 6 Feb 2026 13:51:21 +0000 Subject: [PATCH 1/4] fix: Use nargs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: oliver könig --- scripts/performance/argument_parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/performance/argument_parser.py b/scripts/performance/argument_parser.py index 0b6fdca692..5d6a350419 100644 --- a/scripts/performance/argument_parser.py +++ b/scripts/performance/argument_parser.py @@ -422,8 +422,8 @@ def parse_cli_args(): slurm_args.add_argument( "-cb", "--custom_bash_cmds", - type=list_of_strings, - help="Comma separated string of bash commands", + nargs="*", + help="List of bash commands to execute before the main command", default=[], ) slurm_args.add_argument( From 744f2ea028c79bb67d0d111bbfd81124259dbbc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?oliver=20k=C3=B6nig?= Date: Fri, 6 Feb 2026 14:29:33 +0000 Subject: [PATCH 2/4] fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: oliver könig --- scripts/performance/utils/executors.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/performance/utils/executors.py b/scripts/performance/utils/executors.py index bd8eaffc20..f44102942d 100644 --- a/scripts/performance/utils/executors.py +++ b/scripts/performance/utils/executors.py @@ -127,11 +127,11 @@ def slurm_executor( numa_cmd = f"numactl --cpunodebind=$((SLURM_LOCALID/{numa_divisor})) --membind=$((SLURM_LOCALID/{numa_divisor}))" if gpu.lower() in ["b300"]: numa_cmd += " -C $((SLURM_LOCALID * 16)),$((SLURM_LOCALID * 16 + 1))" - custom_bash_cmds.append(numa_cmd) + custom_bash_cmds.append(f" ; {numa_cmd}") launcher = SlurmTemplate( template_inline=INLINE_TEMPLATE, - template_vars={"pre_cmds": " ; ".join(custom_bash_cmds)}, + template_vars={"pre_cmds": " ".join(custom_bash_cmds)}, ) executor = run.SlurmExecutor( From a8ea6248a75618ce9aceeb3abefd710eb23cc7f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?oliver=20k=C3=B6nig?= Date: Fri, 6 Feb 2026 15:09:05 +0000 Subject: [PATCH 3/4] generalize MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: oliver könig --- scripts/performance/argument_parser.py | 1 + scripts/performance/setup_experiment.py | 2 +- scripts/performance/utils/executors.py | 8 ++++---- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/scripts/performance/argument_parser.py b/scripts/performance/argument_parser.py index 5d6a350419..04d1eccd76 100644 --- a/scripts/performance/argument_parser.py +++ b/scripts/performance/argument_parser.py @@ -423,6 +423,7 @@ def parse_cli_args(): "-cb", "--custom_bash_cmds", nargs="*", + action="append", help="List of bash commands to execute before the main command", default=[], ) diff --git a/scripts/performance/setup_experiment.py b/scripts/performance/setup_experiment.py index 8089b25639..2cb1f5acf1 100755 --- a/scripts/performance/setup_experiment.py +++ b/scripts/performance/setup_experiment.py @@ -206,7 +206,7 @@ def main( custom_mounts: List[str], custom_env_vars: Dict[str, str], custom_srun_args: List[str], - custom_bash_cmds: List[str], + custom_bash_cmds: List[List[str]], nccl_ub: bool, pretrained_checkpoint: Optional[str], num_gpus: int, diff --git a/scripts/performance/utils/executors.py b/scripts/performance/utils/executors.py index f44102942d..4592b355f0 100644 --- a/scripts/performance/utils/executors.py +++ b/scripts/performance/utils/executors.py @@ -64,7 +64,7 @@ def slurm_executor( nemo_home: str = DEFAULT_NEMO_HOME, wandb_key: str = None, network: str = None, - custom_bash_cmds: List[str] = None, + custom_bash_cmds: List[List[str]] = None, additional_slurm_params: Dict[str, Any] = None, gres: Optional[str] = None, ) -> run.SlurmExecutor: @@ -79,7 +79,7 @@ def slurm_executor( #SBATCH --nodelist=node001,node002 #SBATCH --constraint=gpu """ - custom_bash_cmds = [] if custom_bash_cmds is None else custom_bash_cmds + custom_bash_cmds = [] if custom_bash_cmds is None else [" ".join(cmd) for cmd in custom_bash_cmds] mounts = [] # Explicitly request GPU resources to ensure proper allocation # Without --gres=gpu:N, some clusters only allocate 1 GPU regardless of ntasks_per_node @@ -127,11 +127,11 @@ def slurm_executor( numa_cmd = f"numactl --cpunodebind=$((SLURM_LOCALID/{numa_divisor})) --membind=$((SLURM_LOCALID/{numa_divisor}))" if gpu.lower() in ["b300"]: numa_cmd += " -C $((SLURM_LOCALID * 16)),$((SLURM_LOCALID * 16 + 1))" - custom_bash_cmds.append(f" ; {numa_cmd}") + custom_bash_cmds.append(numa_cmd) launcher = SlurmTemplate( template_inline=INLINE_TEMPLATE, - template_vars={"pre_cmds": " ".join(custom_bash_cmds)}, + template_vars={"pre_cmds": " ; ".join(custom_bash_cmds)}, ) executor = run.SlurmExecutor( From a095c544569dde5a62b54f5eca06d31c00d53068 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?oliver=20k=C3=B6nig?= Date: Fri, 6 Feb 2026 15:26:22 +0000 Subject: [PATCH 4/4] default MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: oliver könig --- scripts/performance/argument_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/performance/argument_parser.py b/scripts/performance/argument_parser.py index 04d1eccd76..97b550bdfa 100644 --- a/scripts/performance/argument_parser.py +++ b/scripts/performance/argument_parser.py @@ -425,7 +425,7 @@ def parse_cli_args(): nargs="*", action="append", help="List of bash commands to execute before the main command", - default=[], + default=None, ) slurm_args.add_argument( "--gres",