From 8d4c5a6bf9512bdad2a0196387bbb0c02b34ff07 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?oliver=20k=C3=B6nig?= <okoenig@nvidia.com>
Date: Mon, 1 Jun 2026 07:25:30 +0000
Subject: [PATCH 01/15] ci: prune cluster artifacts older than 14 days during
 weekly/release runs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Adds a JET workload (`_cleanup.yaml`, scope `cleanup`, `gpus: 0`) that runs
`find -mtime +14 | xargs rm -rf` against the parents of `{assets_dir}` and
`{artifacts_dir}` on each cluster.

Wires per-cluster `functional:cleanup_dgx_{a100,h100,gb200}` jobs in
`.gitlab/stages/04.functional-tests.yml`, gated on
`FUNCTIONAL_TEST_SCOPE in [release, weekly]`. Each `functional:run_*` job
adds the matching cleanup as `optional: true` so MR/nightly pipelines are
unaffected.

Signed-off-by: oliver könig <okoenig@nvidia.com>
---
 .gitlab/stages/04.functional-tests.yml | 108 +++++++++++++++++++++++++
 tests/test_utils/recipes/_cleanup.yaml |  45 +++++++++++
 2 files changed, 153 insertions(+)
 create mode 100644 tests/test_utils/recipes/_cleanup.yaml

diff --git a/.gitlab/stages/04.functional-tests.yml b/.gitlab/stages/04.functional-tests.yml
index a12b1c87213..88322c43d2b 100644
--- a/.gitlab/stages/04.functional-tests.yml
+++ b/.gitlab/stages/04.functional-tests.yml
@@ -176,9 +176,92 @@ functional:configure:
   inherit:
     variables: true
 
+# Prune artifacts older than 14 days from the JET assets/artifacts roots on
+# each cluster. Gated on release/weekly scope so it does not run on every MR
+# pipeline. Functional `run_*` jobs depend on the matching cleanup job via
+# `optional: true`, so pipelines without a cleanup job (e.g. `mr` scope) still
+# start their workloads immediately.
+.functional_cleanup_rules:
+  stage: functional_tests
+  rules:
+    - if: $BUILD == "no"
+      when: never
+    - if: $FUNCTIONAL_TEST == "yes" && ($FUNCTIONAL_TEST_SCOPE == "release" || $FUNCTIONAL_TEST_SCOPE == "weekly")
+      when: on_success
+    - when: never
+
+.functional_cleanup:
+  needs:
+    - functional:configure
+    - test:build_image
+  extends: [.functional_cleanup_rules]
+  image: ${UTILITY_IMAGE}:${CI_PIPELINE_ID}
+  tags:
+    - arch/amd64
+    - env/prod
+    - origin/jet-fleet
+    - owner/jet-core
+    - purpose/utility
+    - team/megatron
+  timeout: 30m
+  allow_failure: true
+  script:
+    - |
+      case "$CLEANUP_PLATFORM" in
+        dgx_a100)
+          CLEANUP_CLUSTER=$([[ "$CLUSTER_A100" != "" ]] && echo "$CLUSTER_A100" || echo "$DEFAULT_A100_CLUSTER")
+          ;;
+        dgx_h100)
+          CLEANUP_CLUSTER=$([[ "$CLUSTER_H100" != "" ]] && echo "$CLUSTER_H100" || echo "$DEFAULT_H100_CLUSTER")
+          ;;
+        dgx_gb200)
+          CLEANUP_CLUSTER=$([[ "$CLUSTER_GB200" != "" ]] && echo "$CLUSTER_GB200" || echo "$DEFAULT_GB200_CLUSTER")
+          ;;
+        *)
+          echo "Unknown CLEANUP_PLATFORM: $CLEANUP_PLATFORM"
+          exit 1
+          ;;
+      esac
+    - export PYTHONPATH=$(pwd)
+    - export RO_API_TOKEN=${PAT}
+    - |
+      python tests/test_utils/python_scripts/launch_jet_workload.py \
+        --model cleanup \
+        --test-case cleanup_old_artifacts \
+        --environment dev \
+        --n-repeat 1 \
+        --time-limit 1800 \
+        --scope cleanup \
+        --account ${CI_SLURM_ACCOUNT} \
+        --cluster ${CLEANUP_CLUSTER} \
+        --platform ${CLEANUP_PLATFORM} \
+        --container-tag ${CI_PIPELINE_ID} \
+        --container-image ${UTILITY_IMAGE} \
+        --record-checkpoints false
+
+functional:cleanup_dgx_a100:
+  extends: [.functional_cleanup]
+  variables:
+    CLEANUP_PLATFORM: dgx_a100
+
+functional:cleanup_dgx_h100:
+  extends: [.functional_cleanup]
+  variables:
+    CLEANUP_PLATFORM: dgx_h100
+
+functional:cleanup_dgx_gb200:
+  extends: [.functional_cleanup]
+  variables:
+    CLEANUP_PLATFORM: dgx_gb200
+
 functional:run_lts_dgx_a100:
   extends: [.functional_run]
   allow_failure: true
+  needs:
+    - functional:configure
+    - test:build_image
+    - job: functional:cleanup_dgx_a100
+      optional: true
   variables:
     ENVIRONMENT: lts
     CLUSTER: A100
@@ -186,6 +269,11 @@ functional:run_lts_dgx_a100:
 functional:run_lts_dgx_h100:
   extends: [.functional_run]
   allow_failure: true
+  needs:
+    - functional:configure
+    - test:build_image
+    - job: functional:cleanup_dgx_h100
+      optional: true
   variables:
     ENVIRONMENT: lts
     CLUSTER: H100
@@ -193,24 +281,44 @@ functional:run_lts_dgx_h100:
 functional:run_lts_dgx_gb200:
   extends: [.functional_run]
   allow_failure: true
+  needs:
+    - functional:configure
+    - test:build_image
+    - job: functional:cleanup_dgx_gb200
+      optional: true
   variables:
     ENVIRONMENT: lts
     CLUSTER: GB200
 
 functional:run_dev_dgx_a100:
   extends: [.functional_run]
+  needs:
+    - functional:configure
+    - test:build_image
+    - job: functional:cleanup_dgx_a100
+      optional: true
   variables:
     ENVIRONMENT: dev
     CLUSTER: A100
 
 functional:run_dev_dgx_h100:
   extends: [.functional_run]
+  needs:
+    - functional:configure
+    - test:build_image
+    - job: functional:cleanup_dgx_h100
+      optional: true
   variables:
     ENVIRONMENT: dev
     CLUSTER: H100
 
 functional:run_dev_dgx_gb200:
   extends: [.functional_run]
+  needs:
+    - functional:configure
+    - test:build_image
+    - job: functional:cleanup_dgx_gb200
+      optional: true
   variables:
     ENVIRONMENT: dev
     CLUSTER: GB200
diff --git a/tests/test_utils/recipes/_cleanup.yaml b/tests/test_utils/recipes/_cleanup.yaml
new file mode 100644
index 00000000000..06da350218f
--- /dev/null
+++ b/tests/test_utils/recipes/_cleanup.yaml
@@ -0,0 +1,45 @@
+type: basic
+format_version: 1
+maintainers: [mcore]
+loggers: [stdout]
+spec:
+  name: '{test_case}_{platforms}'
+  model: cleanup
+  build: mcore-pyt-{environment}
+  nodes: 1
+  gpus: 0
+  n_repeat: 1
+  platforms: dgx_h100
+  script_setup: |
+    echo "Cleanup workload — no setup required."
+  script: |-
+    set -euo pipefail
+
+    THRESHOLD_DAYS=14
+    ASSETS_PARENT=$(dirname "{assets_dir}")
+    ARTIFACTS_PARENT=$(dirname "{artifacts_dir}")
+
+    echo "Removing entries older than $THRESHOLD_DAYS days under:"
+    echo "  - $ASSETS_PARENT"
+    echo "  - $ARTIFACTS_PARENT"
+
+    for path in "$ASSETS_PARENT" "$ARTIFACTS_PARENT"; do
+        if [[ -z "$path" || "$path" == "/" || ! -d "$path" ]]; then
+            echo "Skipping invalid path: '$path'"
+            continue
+        fi
+        echo "--- Candidates in $path ---"
+        find "$path" -mindepth 1 -maxdepth 1 -mtime +$THRESHOLD_DAYS -print 2>/dev/null || true
+        echo "--- Deleting ---"
+        find "$path" -mindepth 1 -maxdepth 1 -mtime +$THRESHOLD_DAYS -print0 2>/dev/null \
+            | xargs -0 -r rm -rf 2>/dev/null || true
+    done
+
+    echo "Cleanup complete."
+
+products:
+  - test_case: [cleanup_old_artifacts]
+    products:
+      - environment: [dev]
+        scope: [cleanup]
+        platforms: [dgx_a100, dgx_h100, dgx_gb200]

From 161202565cc79d9bff9cf0e1f5088777c51fe22b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?oliver=20k=C3=B6nig?= <okoenig@nvidia.com>
Date: Mon, 1 Jun 2026 09:04:07 +0000
Subject: [PATCH 02/15] ci: route cleanup workloads to CPU partitions
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Per-cluster CPU launchers now live in dl/jet/ci:
`mcore/{coreweave,oci-hsg,draco-oci-iad,eos}-cpu`. They mirror the GPU
sibling but use partition `cpu` (Eos: `interactive`, since Eos has no
dedicated CPU partition), drop `gpus_per_node`, `exclusive`, and the
`srun ntasks/cpus_per_task` block.

- `resolve_cluster_config` maps `cpu_dgx*` cluster ids to the matching
  `*-cpu` branches.
- `.functional_cleanup` derives `CLEANUP_CLUSTER=cpu_${GPU_CLUSTER}` from
  the active per-platform GPU cluster, so cleanup follows whichever
  cluster the H100/A100/GB200 functional run resolved to.

The recipe still declares `gpus: 0` defensively; the partition switch is
what actually prevents GPU reservation.

Signed-off-by: oliver könig <okoenig@nvidia.com>
---
 .gitlab/stages/04.functional-tests.yml           | 8 +++++---
 tests/test_utils/python_scripts/recipe_parser.py | 8 ++++++++
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/.gitlab/stages/04.functional-tests.yml b/.gitlab/stages/04.functional-tests.yml
index 88322c43d2b..c41cc7480c9 100644
--- a/.gitlab/stages/04.functional-tests.yml
+++ b/.gitlab/stages/04.functional-tests.yml
@@ -209,19 +209,21 @@ functional:configure:
     - |
       case "$CLEANUP_PLATFORM" in
         dgx_a100)
-          CLEANUP_CLUSTER=$([[ "$CLUSTER_A100" != "" ]] && echo "$CLUSTER_A100" || echo "$DEFAULT_A100_CLUSTER")
+          GPU_CLUSTER=$([[ "$CLUSTER_A100" != "" ]] && echo "$CLUSTER_A100" || echo "$DEFAULT_A100_CLUSTER")
           ;;
         dgx_h100)
-          CLEANUP_CLUSTER=$([[ "$CLUSTER_H100" != "" ]] && echo "$CLUSTER_H100" || echo "$DEFAULT_H100_CLUSTER")
+          GPU_CLUSTER=$([[ "$CLUSTER_H100" != "" ]] && echo "$CLUSTER_H100" || echo "$DEFAULT_H100_CLUSTER")
           ;;
         dgx_gb200)
-          CLEANUP_CLUSTER=$([[ "$CLUSTER_GB200" != "" ]] && echo "$CLUSTER_GB200" || echo "$DEFAULT_GB200_CLUSTER")
+          GPU_CLUSTER=$([[ "$CLUSTER_GB200" != "" ]] && echo "$CLUSTER_GB200" || echo "$DEFAULT_GB200_CLUSTER")
           ;;
         *)
           echo "Unknown CLEANUP_PLATFORM: $CLEANUP_PLATFORM"
           exit 1
           ;;
       esac
+    - CLEANUP_CLUSTER="cpu_${GPU_CLUSTER}"
+    - echo "Cleanup target = $CLEANUP_CLUSTER (CPU partition of $GPU_CLUSTER)"
     - export PYTHONPATH=$(pwd)
     - export RO_API_TOKEN=${PAT}
     - |
diff --git a/tests/test_utils/python_scripts/recipe_parser.py b/tests/test_utils/python_scripts/recipe_parser.py
index 018cc043a51..3893a5b9ec7 100644
--- a/tests/test_utils/python_scripts/recipe_parser.py
+++ b/tests/test_utils/python_scripts/recipe_parser.py
@@ -97,6 +97,14 @@ def resolve_cluster_config(cluster: str) -> str:
         return "draco-oci-ord"
     if cluster == "dgxh100_coreweave":
         return "coreweave"
+    if cluster == "cpu_dgxh100_eos":
+        return "eos-cpu"
+    if cluster == "cpu_dgxh100_coreweave":
+        return "coreweave-cpu"
+    if cluster == "cpu_dgxa100_dracooci":
+        return "draco-oci-iad-cpu"
+    if cluster == "cpu_dgxgb200_oci-hsg":
+        return "oci-hsg-cpu"
     if cluster == "ghci":
         return "ghci"
     raise ValueError(f"Unknown cluster {cluster} provided.")

From 2d7bd9298086f662218ea693cb75bba01f5f2bb7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?oliver=20k=C3=B6nig?= <okoenig@nvidia.com>
Date: Mon, 1 Jun 2026 09:05:19 +0000
Subject: [PATCH 03/15] ci: switch cleanup workload to dry-run
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Prints the entries that would be removed under the JET assets/artifacts
roots without actually deleting them. To re-enable deletion, swap the
final `find ... -print` back to a `find ... -print0 | xargs -0 -r rm -rf`.

Signed-off-by: oliver könig <okoenig@nvidia.com>
---
 tests/test_utils/recipes/_cleanup.yaml | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/tests/test_utils/recipes/_cleanup.yaml b/tests/test_utils/recipes/_cleanup.yaml
index 06da350218f..467fb61b880 100644
--- a/tests/test_utils/recipes/_cleanup.yaml
+++ b/tests/test_utils/recipes/_cleanup.yaml
@@ -19,7 +19,7 @@ spec:
     ASSETS_PARENT=$(dirname "{assets_dir}")
     ARTIFACTS_PARENT=$(dirname "{artifacts_dir}")
 
-    echo "Removing entries older than $THRESHOLD_DAYS days under:"
+    echo "[DRY-RUN] Entries older than $THRESHOLD_DAYS days under:"
     echo "  - $ASSETS_PARENT"
     echo "  - $ARTIFACTS_PARENT"
 
@@ -28,14 +28,12 @@ spec:
             echo "Skipping invalid path: '$path'"
             continue
         fi
-        echo "--- Candidates in $path ---"
+        echo "--- Would delete from $path ---"
         find "$path" -mindepth 1 -maxdepth 1 -mtime +$THRESHOLD_DAYS -print 2>/dev/null || true
-        echo "--- Deleting ---"
-        find "$path" -mindepth 1 -maxdepth 1 -mtime +$THRESHOLD_DAYS -print0 2>/dev/null \
-            | xargs -0 -r rm -rf 2>/dev/null || true
+        echo "--- (dry-run: no deletion) ---"
     done
 
-    echo "Cleanup complete."
+    echo "Dry-run complete; no files removed."
 
 products:
   - test_case: [cleanup_old_artifacts]

From e89f6df7b0dae6b70e57c4394b517b6f35bfacae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?oliver=20k=C3=B6nig?= <okoenig@nvidia.com>
Date: Mon, 1 Jun 2026 09:16:11 +0000
Subject: [PATCH 04/15] ci: shorten cpu_dgx* cluster ids to fit JET 20-char
 limit
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

JET caps launcher keys at 20 chars (pydantic ValidationError on the long
form `cpu_dgxh100_coreweave`). Renamed all four cleanup clusters to drop
the redundant `dgx<arch>_` prefix, matching the existing
`dlalgo-ci/draco-oci-ord-cpu` precedent. `.functional_cleanup` now derives
the cleanup cluster by stripping the `dgx*_` prefix from the active GPU
cluster name.

Signed-off-by: oliver könig <okoenig@nvidia.com>
---
 .gitlab/stages/04.functional-tests.yml           | 2 +-
 tests/test_utils/python_scripts/recipe_parser.py | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.gitlab/stages/04.functional-tests.yml b/.gitlab/stages/04.functional-tests.yml
index c41cc7480c9..5a76fe66b05 100644
--- a/.gitlab/stages/04.functional-tests.yml
+++ b/.gitlab/stages/04.functional-tests.yml
@@ -222,7 +222,7 @@ functional:configure:
           exit 1
           ;;
       esac
-    - CLEANUP_CLUSTER="cpu_${GPU_CLUSTER}"
+    - CLEANUP_CLUSTER="cpu_${GPU_CLUSTER#dgx*_}"
     - echo "Cleanup target = $CLEANUP_CLUSTER (CPU partition of $GPU_CLUSTER)"
     - export PYTHONPATH=$(pwd)
     - export RO_API_TOKEN=${PAT}
diff --git a/tests/test_utils/python_scripts/recipe_parser.py b/tests/test_utils/python_scripts/recipe_parser.py
index 3893a5b9ec7..a67b131b91d 100644
--- a/tests/test_utils/python_scripts/recipe_parser.py
+++ b/tests/test_utils/python_scripts/recipe_parser.py
@@ -97,13 +97,13 @@ def resolve_cluster_config(cluster: str) -> str:
         return "draco-oci-ord"
     if cluster == "dgxh100_coreweave":
         return "coreweave"
-    if cluster == "cpu_dgxh100_eos":
+    if cluster == "cpu_eos":
         return "eos-cpu"
-    if cluster == "cpu_dgxh100_coreweave":
+    if cluster == "cpu_coreweave":
         return "coreweave-cpu"
-    if cluster == "cpu_dgxa100_dracooci":
+    if cluster == "cpu_dracooci":
         return "draco-oci-iad-cpu"
-    if cluster == "cpu_dgxgb200_oci-hsg":
+    if cluster == "cpu_oci-hsg":
         return "oci-hsg-cpu"
     if cluster == "ghci":
         return "ghci"

From e4c9af07af6ab9f2d73befb2c7ed2583accb7374 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?oliver=20k=C3=B6nig?= <okoenig@nvidia.com>
Date: Mon, 1 Jun 2026 09:18:18 +0000
Subject: [PATCH 05/15] ci: target explicit lustre roots in cleanup workload
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The chmod-g+w paths in each GPU mcore sibling's before_script (e.g.
mcore/coreweave: `/lustre/fsw/.../release-testing` and `.../mcore_ci`) are
the canonical artifact roots. Every CPU mcore branch mounts the cluster's
lustre at `/lustre/fsw/coreai_dlalgo_mcore` inside the container, so the
container-side paths are uniform across clusters. The `if [[ ! -d ]]`
guard makes the recipe safely degrade on clusters where a root happens
not to exist.

Signed-off-by: oliver könig <okoenig@nvidia.com>
---
 tests/test_utils/recipes/_cleanup.yaml | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/tests/test_utils/recipes/_cleanup.yaml b/tests/test_utils/recipes/_cleanup.yaml
index 467fb61b880..51691f1baa8 100644
--- a/tests/test_utils/recipes/_cleanup.yaml
+++ b/tests/test_utils/recipes/_cleanup.yaml
@@ -16,16 +16,19 @@ spec:
     set -euo pipefail
 
     THRESHOLD_DAYS=14
-    ASSETS_PARENT=$(dirname "{assets_dir}")
-    ARTIFACTS_PARENT=$(dirname "{artifacts_dir}")
+    CLEANUP_PATHS=(
+        /lustre/fsw/coreai_dlalgo_mcore/release-testing
+        /lustre/fsw/coreai_dlalgo_mcore/mcore_ci
+    )
 
     echo "[DRY-RUN] Entries older than $THRESHOLD_DAYS days under:"
-    echo "  - $ASSETS_PARENT"
-    echo "  - $ARTIFACTS_PARENT"
+    for path in "${{CLEANUP_PATHS[@]}}"; do
+        echo "  - $path"
+    done
 
-    for path in "$ASSETS_PARENT" "$ARTIFACTS_PARENT"; do
-        if [[ -z "$path" || "$path" == "/" || ! -d "$path" ]]; then
-            echo "Skipping invalid path: '$path'"
+    for path in "${{CLEANUP_PATHS[@]}}"; do
+        if [[ ! -d "$path" ]]; then
+            echo "Skipping (not a directory on this cluster): $path"
             continue
         fi
         echo "--- Would delete from $path ---"

From f01e5887dfb262cc8ffb80f5bb79f9a656e85f35 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?oliver=20k=C3=B6nig?= <okoenig@nvidia.com>
Date: Mon, 1 Jun 2026 09:22:15 +0000
Subject: [PATCH 06/15] ci: source cleanup paths from CLEANUP_PATHS env var
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Recipe no longer hardcodes lustre roots. Per-cluster CI/CD variables
(`CLEANUP_PATHS_DGX_A100` / `_H100` / `_GB200`, set in GitLab project
Settings) hold the newline-separated path list; `.functional_cleanup`
forwards the active platform's variable as `CLEANUP_PATHS` and
`launch_jet_workload.py` propagates it into the JET workload env.

The recipe exits cleanly with "nothing to do" when the variable is
unset, so a pipeline can run before the variables are configured
without failing.

Signed-off-by: oliver könig <okoenig@nvidia.com>
---
 .gitlab/stages/04.functional-tests.yml         |  3 +++
 .../python_scripts/launch_jet_workload.py      |  1 +
 tests/test_utils/recipes/_cleanup.yaml         | 18 +++++++++---------
 3 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/.gitlab/stages/04.functional-tests.yml b/.gitlab/stages/04.functional-tests.yml
index 5a76fe66b05..a71a6896ef2 100644
--- a/.gitlab/stages/04.functional-tests.yml
+++ b/.gitlab/stages/04.functional-tests.yml
@@ -245,16 +245,19 @@ functional:cleanup_dgx_a100:
   extends: [.functional_cleanup]
   variables:
     CLEANUP_PLATFORM: dgx_a100
+    CLEANUP_PATHS: $CLEANUP_PATHS_DGX_A100
 
 functional:cleanup_dgx_h100:
   extends: [.functional_cleanup]
   variables:
     CLEANUP_PLATFORM: dgx_h100
+    CLEANUP_PATHS: $CLEANUP_PATHS_DGX_H100
 
 functional:cleanup_dgx_gb200:
   extends: [.functional_cleanup]
   variables:
     CLEANUP_PLATFORM: dgx_gb200
+    CLEANUP_PATHS: $CLEANUP_PATHS_DGX_GB200
 
 functional:run_lts_dgx_a100:
   extends: [.functional_run]
diff --git a/tests/test_utils/python_scripts/launch_jet_workload.py b/tests/test_utils/python_scripts/launch_jet_workload.py
index d5629bd432d..198c071af9c 100644
--- a/tests/test_utils/python_scripts/launch_jet_workload.py
+++ b/tests/test_utils/python_scripts/launch_jet_workload.py
@@ -153,6 +153,7 @@ def launch_and_wait_for_completion(
                                         "TRANSFORMERS_OFFLINE": "1",
                                         "CLUSTER": cluster,
                                         "RUN_ID": str(uuid.uuid4()),
+                                        "CLEANUP_PATHS": os.getenv("CLEANUP_PATHS") or "",
                                     }
                                 }
                             }
diff --git a/tests/test_utils/recipes/_cleanup.yaml b/tests/test_utils/recipes/_cleanup.yaml
index 51691f1baa8..1f564473c34 100644
--- a/tests/test_utils/recipes/_cleanup.yaml
+++ b/tests/test_utils/recipes/_cleanup.yaml
@@ -16,17 +16,17 @@ spec:
     set -euo pipefail
 
     THRESHOLD_DAYS=14
-    CLEANUP_PATHS=(
-        /lustre/fsw/coreai_dlalgo_mcore/release-testing
-        /lustre/fsw/coreai_dlalgo_mcore/mcore_ci
-    )
+
+    if [[ -z "${{CLEANUP_PATHS:-}}" ]]; then
+        echo "CLEANUP_PATHS env var not set; nothing to do."
+        exit 0
+    fi
 
     echo "[DRY-RUN] Entries older than $THRESHOLD_DAYS days under:"
-    for path in "${{CLEANUP_PATHS[@]}}"; do
-        echo "  - $path"
-    done
+    printf '%s\n' "$CLEANUP_PATHS" | sed 's/^/  - /'
 
-    for path in "${{CLEANUP_PATHS[@]}}"; do
+    while IFS= read -r path; do
+        [[ -z "$path" ]] && continue
         if [[ ! -d "$path" ]]; then
             echo "Skipping (not a directory on this cluster): $path"
             continue
@@ -34,7 +34,7 @@ spec:
         echo "--- Would delete from $path ---"
         find "$path" -mindepth 1 -maxdepth 1 -mtime +$THRESHOLD_DAYS -print 2>/dev/null || true
         echo "--- (dry-run: no deletion) ---"
-    done
+    done <<<"$CLEANUP_PATHS"
 
     echo "Dry-run complete; no files removed."
 

From adea29a328a47d7a27e16e97c49b4644f7b7ce01 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?oliver=20k=C3=B6nig?= <okoenig@nvidia.com>
Date: Mon, 1 Jun 2026 09:52:14 +0000
Subject: [PATCH 07/15] =?UTF-8?q?ci:=20invert=20cleanup=20semantics=20?=
 =?UTF-8?q?=E2=80=94=20CLEANUP=5FPATHS=20now=20protects?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Previous behavior: scan the listed paths and delete their contents.
That's wrong for the configured roots — those directories contain
long-lived assets (datasets, caches) that must not be pruned, only the
*siblings* around them in the parent should age out.

New behavior: CLEANUP_PATHS is the protect list. The recipe walks each
unique parent directory, lists entries older than 14 days, and
explicitly excludes the protected basenames via `find ! -name`. Parents
matching `/` are refused as a safety guard.

Signed-off-by: oliver könig <okoenig@nvidia.com>
---
 tests/test_utils/recipes/_cleanup.yaml | 46 +++++++++++++++++++++-----
 1 file changed, 37 insertions(+), 9 deletions(-)

diff --git a/tests/test_utils/recipes/_cleanup.yaml b/tests/test_utils/recipes/_cleanup.yaml
index 1f564473c34..b7861a65a90 100644
--- a/tests/test_utils/recipes/_cleanup.yaml
+++ b/tests/test_utils/recipes/_cleanup.yaml
@@ -17,24 +17,52 @@ spec:
 
     THRESHOLD_DAYS=14
 
+    # CLEANUP_PATHS holds the paths to PROTECT (never delete). The recipe
+    # scans the parent directory of each protected path and lists entries
+    # older than THRESHOLD_DAYS that are *not* protected.
     if [[ -z "${{CLEANUP_PATHS:-}}" ]]; then
         echo "CLEANUP_PATHS env var not set; nothing to do."
         exit 0
     fi
 
-    echo "[DRY-RUN] Entries older than $THRESHOLD_DAYS days under:"
-    printf '%s\n' "$CLEANUP_PATHS" | sed 's/^/  - /'
+    PROTECT_PATHS=()
+    while IFS= read -r p; do
+        [[ -z "$p" ]] && continue
+        PROTECT_PATHS+=("$p")
+    done <<<"$CLEANUP_PATHS"
+
+    declare -A SEEN_PARENTS
+    PARENTS=()
+    for p in "${{PROTECT_PATHS[@]}}"; do
+        parent=$(dirname "$p")
+        if [[ -z "${{SEEN_PARENTS[$parent]:-}}" ]]; then
+            SEEN_PARENTS[$parent]=1
+            PARENTS+=("$parent")
+        fi
+    done
+
+    echo "[DRY-RUN] Protect (never delete): ${{PROTECT_PATHS[*]}}"
+    echo "[DRY-RUN] Scanning parents: ${{PARENTS[*]}}"
+    echo "[DRY-RUN] Threshold: $THRESHOLD_DAYS days"
 
-    while IFS= read -r path; do
-        [[ -z "$path" ]] && continue
-        if [[ ! -d "$path" ]]; then
-            echo "Skipping (not a directory on this cluster): $path"
+    for parent in "${{PARENTS[@]}}"; do
+        if [[ "$parent" == "/" || ! -d "$parent" ]]; then
+            echo "Skipping unsafe or missing parent: $parent"
             continue
         fi
-        echo "--- Would delete from $path ---"
-        find "$path" -mindepth 1 -maxdepth 1 -mtime +$THRESHOLD_DAYS -print 2>/dev/null || true
+
+        find_args=( "$parent" -mindepth 1 -maxdepth 1 -mtime +$THRESHOLD_DAYS )
+        for p in "${{PROTECT_PATHS[@]}}"; do
+            if [[ "$(dirname "$p")" == "$parent" ]]; then
+                find_args+=( ! -name "$(basename "$p")" )
+            fi
+        done
+        find_args+=( -print )
+
+        echo "--- Would delete from $parent (excluding protected entries) ---"
+        find "${{find_args[@]}}" 2>/dev/null || true
         echo "--- (dry-run: no deletion) ---"
-    done <<<"$CLEANUP_PATHS"
+    done
 
     echo "Dry-run complete; no files removed."
 

From 1d2d9c44090515bf6bc7058a809d3eeb8cced657 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?oliver=20k=C3=B6nig?= <okoenig@nvidia.com>
Date: Mon, 1 Jun 2026 09:54:57 +0000
Subject: [PATCH 08/15] ci: revert cleanup to scan-inside semantics
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

CLEANUP_PATHS is again the GC root list — the recipe lists direct
children older than THRESHOLD_DAYS inside each listed root. Protect-by-
exclude logic is gone; protection is implicit (anything not listed is
untouched). The GitLab CI variables now hold only the GC roots
(release-testing); mcore_ci is no longer in the list.

Signed-off-by: oliver könig <okoenig@nvidia.com>
---
 tests/test_utils/recipes/_cleanup.yaml | 48 ++++++--------------------
 1 file changed, 11 insertions(+), 37 deletions(-)

diff --git a/tests/test_utils/recipes/_cleanup.yaml b/tests/test_utils/recipes/_cleanup.yaml
index b7861a65a90..95579fda04c 100644
--- a/tests/test_utils/recipes/_cleanup.yaml
+++ b/tests/test_utils/recipes/_cleanup.yaml
@@ -17,52 +17,26 @@ spec:
 
     THRESHOLD_DAYS=14
 
-    # CLEANUP_PATHS holds the paths to PROTECT (never delete). The recipe
-    # scans the parent directory of each protected path and lists entries
-    # older than THRESHOLD_DAYS that are *not* protected.
+    # CLEANUP_PATHS is the newline-separated list of GC roots. For each
+    # path, the recipe lists direct children older than THRESHOLD_DAYS.
     if [[ -z "${{CLEANUP_PATHS:-}}" ]]; then
         echo "CLEANUP_PATHS env var not set; nothing to do."
         exit 0
     fi
 
-    PROTECT_PATHS=()
-    while IFS= read -r p; do
-        [[ -z "$p" ]] && continue
-        PROTECT_PATHS+=("$p")
-    done <<<"$CLEANUP_PATHS"
-
-    declare -A SEEN_PARENTS
-    PARENTS=()
-    for p in "${{PROTECT_PATHS[@]}}"; do
-        parent=$(dirname "$p")
-        if [[ -z "${{SEEN_PARENTS[$parent]:-}}" ]]; then
-            SEEN_PARENTS[$parent]=1
-            PARENTS+=("$parent")
-        fi
-    done
-
-    echo "[DRY-RUN] Protect (never delete): ${{PROTECT_PATHS[*]}}"
-    echo "[DRY-RUN] Scanning parents: ${{PARENTS[*]}}"
-    echo "[DRY-RUN] Threshold: $THRESHOLD_DAYS days"
+    echo "[DRY-RUN] Pruning entries older than $THRESHOLD_DAYS days under:"
+    printf '%s\n' "$CLEANUP_PATHS" | sed 's/^/  - /'
 
-    for parent in "${{PARENTS[@]}}"; do
-        if [[ "$parent" == "/" || ! -d "$parent" ]]; then
-            echo "Skipping unsafe or missing parent: $parent"
+    while IFS= read -r path; do
+        [[ -z "$path" ]] && continue
+        if [[ "$path" == "/" || ! -d "$path" ]]; then
+            echo "Skipping unsafe or missing path: $path"
             continue
         fi
-
-        find_args=( "$parent" -mindepth 1 -maxdepth 1 -mtime +$THRESHOLD_DAYS )
-        for p in "${{PROTECT_PATHS[@]}}"; do
-            if [[ "$(dirname "$p")" == "$parent" ]]; then
-                find_args+=( ! -name "$(basename "$p")" )
-            fi
-        done
-        find_args+=( -print )
-
-        echo "--- Would delete from $parent (excluding protected entries) ---"
-        find "${{find_args[@]}}" 2>/dev/null || true
+        echo "--- Would delete from $path ---"
+        find "$path" -mindepth 1 -maxdepth 1 -mtime +$THRESHOLD_DAYS -print 2>/dev/null || true
         echo "--- (dry-run: no deletion) ---"
-    done
+    done <<<"$CLEANUP_PATHS"
 
     echo "Dry-run complete; no files removed."
 

From 6fc4bf5f47621277e8653a7f30aa89d2062efaca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?oliver=20k=C3=B6nig?= <okoenig@nvidia.com>
Date: Mon, 1 Jun 2026 10:03:12 +0000
Subject: [PATCH 09/15] =?UTF-8?q?ci:=20drop=20-maxdepth=201=20=E2=80=94=20?=
 =?UTF-8?q?recurse=20to=20leaves=20in=20cleanup=20find?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Without the depth cap, `find -mindepth 1 -mtime +14` traverses the full
tree under each CLEANUP_PATHS root, listing every entry whose mtime is
older than the threshold. When this flips from dry-run to actual delete,
leaves are pruned without leaving recent-parent directories tagged.

Signed-off-by: oliver könig <okoenig@nvidia.com>
---
 tests/test_utils/recipes/_cleanup.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_utils/recipes/_cleanup.yaml b/tests/test_utils/recipes/_cleanup.yaml
index 95579fda04c..09fe29d5433 100644
--- a/tests/test_utils/recipes/_cleanup.yaml
+++ b/tests/test_utils/recipes/_cleanup.yaml
@@ -34,7 +34,7 @@ spec:
             continue
         fi
         echo "--- Would delete from $path ---"
-        find "$path" -mindepth 1 -maxdepth 1 -mtime +$THRESHOLD_DAYS -print 2>/dev/null || true
+        find "$path" -mindepth 1 -mtime +$THRESHOLD_DAYS -print 2>/dev/null || true
         echo "--- (dry-run: no deletion) ---"
     done <<<"$CLEANUP_PATHS"
 

From a47500f34b6be4b66a5bd9d0182a9f15449610d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?oliver=20k=C3=B6nig?= <okoenig@nvidia.com>
Date: Mon, 1 Jun 2026 10:17:39 +0000
Subject: [PATCH 10/15] ci: restrict cleanup find to regular files only
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Adds `-type f` so the recursive traversal lists files only — directories
(whose mtime updates on any child change) won't show up as candidates,
and an actual delete pass would touch leaves without removing the
directory skeleton.

Signed-off-by: oliver könig <okoenig@nvidia.com>
---
 tests/test_utils/recipes/_cleanup.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_utils/recipes/_cleanup.yaml b/tests/test_utils/recipes/_cleanup.yaml
index 09fe29d5433..86b401c65cf 100644
--- a/tests/test_utils/recipes/_cleanup.yaml
+++ b/tests/test_utils/recipes/_cleanup.yaml
@@ -34,7 +34,7 @@ spec:
             continue
         fi
         echo "--- Would delete from $path ---"
-        find "$path" -mindepth 1 -mtime +$THRESHOLD_DAYS -print 2>/dev/null || true
+        find "$path" -mindepth 1 -type f -mtime +$THRESHOLD_DAYS -print 2>/dev/null || true
         echo "--- (dry-run: no deletion) ---"
     done <<<"$CLEANUP_PATHS"
 

From a72b2efa8bfb925f901d53257149f120deea206a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?oliver=20k=C3=B6nig?= <okoenig@nvidia.com>
Date: Mon, 1 Jun 2026 10:18:30 +0000
Subject: [PATCH 11/15] ci: add empty-directory sweep after file find
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Second pass per cleanup root: `find -mindepth 1 -depth -type d -empty
-print`. Depth-first traversal so deeper empty dirs surface before their
parents — when this pairs with real deletion, parents that become empty
through the recursion are also pruned in the same sweep.

In dry-run, this lists only the directories that are already empty
(since the file pass hasn't actually removed anything yet).

Signed-off-by: oliver könig <okoenig@nvidia.com>
---
 tests/test_utils/recipes/_cleanup.yaml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/test_utils/recipes/_cleanup.yaml b/tests/test_utils/recipes/_cleanup.yaml
index 86b401c65cf..baa65658479 100644
--- a/tests/test_utils/recipes/_cleanup.yaml
+++ b/tests/test_utils/recipes/_cleanup.yaml
@@ -33,8 +33,10 @@ spec:
             echo "Skipping unsafe or missing path: $path"
             continue
         fi
-        echo "--- Would delete from $path ---"
+        echo "--- Would delete (files older than $THRESHOLD_DAYS days) from $path ---"
         find "$path" -mindepth 1 -type f -mtime +$THRESHOLD_DAYS -print 2>/dev/null || true
+        echo "--- Would also prune (empty directories, post-file-sweep) from $path ---"
+        find "$path" -mindepth 1 -depth -type d -empty -print 2>/dev/null || true
         echo "--- (dry-run: no deletion) ---"
     done <<<"$CLEANUP_PATHS"
 

From 8bef9212edafd3c025ae4c3aecf67aabac91a174 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?oliver=20k=C3=B6nig?= <okoenig@nvidia.com>
Date: Mon, 1 Jun 2026 10:37:55 +0000
Subject: [PATCH 12/15] ci: add CLEANUP_PROTECT env var to skip subtrees during
 cleanup find
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

CLEANUP_PATHS becomes the scan-root list, descended recursively, and a
new CLEANUP_PROTECT env var holds newline-separated subtrees that
`find -path X -prune` skips during traversal. Both file-find and
empty-dir sweep apply the prune.

Per-cluster CI variables: `CLEANUP_PATHS_DGX_<X>` (scan roots) and
`CLEANUP_PROTECT_DGX_<X>` (skip list); both forwarded to JET via
`launch_jet_workload.py` and the cleanup gitlab job.

Signed-off-by: oliver könig <okoenig@nvidia.com>
---
 .gitlab/stages/04.functional-tests.yml        |  3 ++
 .../python_scripts/launch_jet_workload.py     |  1 +
 tests/test_utils/recipes/_cleanup.yaml        | 41 +++++++++++++++----
 3 files changed, 38 insertions(+), 7 deletions(-)

diff --git a/.gitlab/stages/04.functional-tests.yml b/.gitlab/stages/04.functional-tests.yml
index a71a6896ef2..17130514cd2 100644
--- a/.gitlab/stages/04.functional-tests.yml
+++ b/.gitlab/stages/04.functional-tests.yml
@@ -246,18 +246,21 @@ functional:cleanup_dgx_a100:
   variables:
     CLEANUP_PLATFORM: dgx_a100
     CLEANUP_PATHS: $CLEANUP_PATHS_DGX_A100
+    CLEANUP_PROTECT: $CLEANUP_PROTECT_DGX_A100
 
 functional:cleanup_dgx_h100:
   extends: [.functional_cleanup]
   variables:
     CLEANUP_PLATFORM: dgx_h100
     CLEANUP_PATHS: $CLEANUP_PATHS_DGX_H100
+    CLEANUP_PROTECT: $CLEANUP_PROTECT_DGX_H100
 
 functional:cleanup_dgx_gb200:
   extends: [.functional_cleanup]
   variables:
     CLEANUP_PLATFORM: dgx_gb200
     CLEANUP_PATHS: $CLEANUP_PATHS_DGX_GB200
+    CLEANUP_PROTECT: $CLEANUP_PROTECT_DGX_GB200
 
 functional:run_lts_dgx_a100:
   extends: [.functional_run]
diff --git a/tests/test_utils/python_scripts/launch_jet_workload.py b/tests/test_utils/python_scripts/launch_jet_workload.py
index 198c071af9c..0434d083ebf 100644
--- a/tests/test_utils/python_scripts/launch_jet_workload.py
+++ b/tests/test_utils/python_scripts/launch_jet_workload.py
@@ -154,6 +154,7 @@ def launch_and_wait_for_completion(
                                         "CLUSTER": cluster,
                                         "RUN_ID": str(uuid.uuid4()),
                                         "CLEANUP_PATHS": os.getenv("CLEANUP_PATHS") or "",
+                                        "CLEANUP_PROTECT": os.getenv("CLEANUP_PROTECT") or "",
                                     }
                                 }
                             }
diff --git a/tests/test_utils/recipes/_cleanup.yaml b/tests/test_utils/recipes/_cleanup.yaml
index baa65658479..c77b2ca98a5 100644
--- a/tests/test_utils/recipes/_cleanup.yaml
+++ b/tests/test_utils/recipes/_cleanup.yaml
@@ -17,15 +17,34 @@ spec:
 
     THRESHOLD_DAYS=14
 
-    # CLEANUP_PATHS is the newline-separated list of GC roots. For each
-    # path, the recipe lists direct children older than THRESHOLD_DAYS.
+    # CLEANUP_PATHS: newline-separated scan roots. The recipe descends each
+    # root and lists files older than THRESHOLD_DAYS, then a second pass
+    # for empty directories.
+    # CLEANUP_PROTECT: newline-separated subtrees to prune (skip entirely)
+    # during traversal.
     if [[ -z "${{CLEANUP_PATHS:-}}" ]]; then
         echo "CLEANUP_PATHS env var not set; nothing to do."
         exit 0
     fi
 
-    echo "[DRY-RUN] Pruning entries older than $THRESHOLD_DAYS days under:"
+    PRUNE_ARGS=()
+    if [[ -n "${{CLEANUP_PROTECT:-}}" ]]; then
+        while IFS= read -r protect; do
+            [[ -z "$protect" ]] && continue
+            if [[ ${{#PRUNE_ARGS[@]}} -gt 0 ]]; then
+                PRUNE_ARGS+=( -o )
+            fi
+            PRUNE_ARGS+=( -path "$protect" )
+        done <<<"$CLEANUP_PROTECT"
+    fi
+
+    echo "[DRY-RUN] Scan roots:"
     printf '%s\n' "$CLEANUP_PATHS" | sed 's/^/  - /'
+    if [[ ${{#PRUNE_ARGS[@]}} -gt 0 ]]; then
+        echo "[DRY-RUN] Pruned subtrees:"
+        printf '%s\n' "$CLEANUP_PROTECT" | sed 's/^/  - /'
+    fi
+    echo "[DRY-RUN] Threshold: $THRESHOLD_DAYS days"
 
     while IFS= read -r path; do
         [[ -z "$path" ]] && continue
@@ -33,10 +52,18 @@ spec:
             echo "Skipping unsafe or missing path: $path"
             continue
         fi
-        echo "--- Would delete (files older than $THRESHOLD_DAYS days) from $path ---"
-        find "$path" -mindepth 1 -type f -mtime +$THRESHOLD_DAYS -print 2>/dev/null || true
-        echo "--- Would also prune (empty directories, post-file-sweep) from $path ---"
-        find "$path" -mindepth 1 -depth -type d -empty -print 2>/dev/null || true
+        echo "--- Would delete (files older than $THRESHOLD_DAYS days) under $path ---"
+        if [[ ${{#PRUNE_ARGS[@]}} -gt 0 ]]; then
+            find "$path" -mindepth 1 \( "${{PRUNE_ARGS[@]}}" \) -prune -o -type f -mtime +$THRESHOLD_DAYS -print 2>/dev/null || true
+        else
+            find "$path" -mindepth 1 -type f -mtime +$THRESHOLD_DAYS -print 2>/dev/null || true
+        fi
+        echo "--- Would also prune (empty directories, post-file-sweep) under $path ---"
+        if [[ ${{#PRUNE_ARGS[@]}} -gt 0 ]]; then
+            find "$path" -mindepth 1 \( "${{PRUNE_ARGS[@]}}" \) -prune -o -depth -type d -empty -print 2>/dev/null || true
+        else
+            find "$path" -mindepth 1 -depth -type d -empty -print 2>/dev/null || true
+        fi
         echo "--- (dry-run: no deletion) ---"
     done <<<"$CLEANUP_PATHS"
 

From fd106d7a3ae3e669013ae2556786adbb6175d6f2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?oliver=20k=C3=B6nig?= <okoenig@nvidia.com>
Date: Mon, 1 Jun 2026 10:57:25 +0000
Subject: [PATCH 13/15] ci: enable live deletion in cleanup workload, raise
 threshold to 28d
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Switch from `-print` to `-print -delete` for both file and empty-dir
passes. Because `-delete` implies `-depth` (which makes `-prune` a
silent no-op), the protected subtrees are now filtered via
`! -path X ! -path X/*` predicates instead of `-prune`. find still
descends into the protected subtree but never matches anything inside
it — verified by local bash test against a synthetic tree.

THRESHOLD_DAYS bumped from 14 to 28.

Signed-off-by: oliver könig <okoenig@nvidia.com>
---
 tests/test_utils/recipes/_cleanup.yaml | 44 +++++++++++---------------
 1 file changed, 18 insertions(+), 26 deletions(-)

diff --git a/tests/test_utils/recipes/_cleanup.yaml b/tests/test_utils/recipes/_cleanup.yaml
index c77b2ca98a5..f0f5ccf3099 100644
--- a/tests/test_utils/recipes/_cleanup.yaml
+++ b/tests/test_utils/recipes/_cleanup.yaml
@@ -15,11 +15,11 @@ spec:
   script: |-
     set -euo pipefail
 
-    THRESHOLD_DAYS=14
+    THRESHOLD_DAYS=28
 
     # CLEANUP_PATHS: newline-separated scan roots. The recipe descends each
-    # root and lists files older than THRESHOLD_DAYS, then a second pass
-    # for empty directories.
+    # root, deletes files older than THRESHOLD_DAYS, then sweeps empty
+    # directories.
     # CLEANUP_PROTECT: newline-separated subtrees to prune (skip entirely)
     # during traversal.
     if [[ -z "${{CLEANUP_PATHS:-}}" ]]; then
@@ -27,24 +27,25 @@ spec:
         exit 0
     fi
 
-    PRUNE_ARGS=()
+    # Use `! -path X ! -path X/*` filters rather than `-prune`: `-delete`
+    # implies `-depth`, and `-depth` makes `-prune` a silent no-op. With
+    # negative-path matching, find still descends into the protected
+    # subtree but never matches any entry inside it.
+    SKIP_ARGS=()
     if [[ -n "${{CLEANUP_PROTECT:-}}" ]]; then
         while IFS= read -r protect; do
             [[ -z "$protect" ]] && continue
-            if [[ ${{#PRUNE_ARGS[@]}} -gt 0 ]]; then
-                PRUNE_ARGS+=( -o )
-            fi
-            PRUNE_ARGS+=( -path "$protect" )
+            SKIP_ARGS+=( ! -path "$protect" ! -path "$protect/*" )
         done <<<"$CLEANUP_PROTECT"
     fi
 
-    echo "[DRY-RUN] Scan roots:"
+    echo "Scan roots:"
     printf '%s\n' "$CLEANUP_PATHS" | sed 's/^/  - /'
-    if [[ ${{#PRUNE_ARGS[@]}} -gt 0 ]]; then
-        echo "[DRY-RUN] Pruned subtrees:"
+    if [[ -n "${{CLEANUP_PROTECT:-}}" ]]; then
+        echo "Protected subtrees (skipped):"
         printf '%s\n' "$CLEANUP_PROTECT" | sed 's/^/  - /'
     fi
-    echo "[DRY-RUN] Threshold: $THRESHOLD_DAYS days"
+    echo "Threshold: $THRESHOLD_DAYS days"
 
     while IFS= read -r path; do
         [[ -z "$path" ]] && continue
@@ -52,22 +53,13 @@ spec:
             echo "Skipping unsafe or missing path: $path"
             continue
         fi
-        echo "--- Would delete (files older than $THRESHOLD_DAYS days) under $path ---"
-        if [[ ${{#PRUNE_ARGS[@]}} -gt 0 ]]; then
-            find "$path" -mindepth 1 \( "${{PRUNE_ARGS[@]}}" \) -prune -o -type f -mtime +$THRESHOLD_DAYS -print 2>/dev/null || true
-        else
-            find "$path" -mindepth 1 -type f -mtime +$THRESHOLD_DAYS -print 2>/dev/null || true
-        fi
-        echo "--- Would also prune (empty directories, post-file-sweep) under $path ---"
-        if [[ ${{#PRUNE_ARGS[@]}} -gt 0 ]]; then
-            find "$path" -mindepth 1 \( "${{PRUNE_ARGS[@]}}" \) -prune -o -depth -type d -empty -print 2>/dev/null || true
-        else
-            find "$path" -mindepth 1 -depth -type d -empty -print 2>/dev/null || true
-        fi
-        echo "--- (dry-run: no deletion) ---"
+        echo "--- Deleting files older than $THRESHOLD_DAYS days under $path ---"
+        find "$path" -mindepth 1 "${{SKIP_ARGS[@]}}" -type f -mtime +$THRESHOLD_DAYS -print -delete 2>/dev/null || true
+        echo "--- Pruning empty directories under $path ---"
+        find "$path" -mindepth 1 "${{SKIP_ARGS[@]}}" -type d -empty -print -delete 2>/dev/null || true
     done <<<"$CLEANUP_PATHS"
 
-    echo "Dry-run complete; no files removed."
+    echo "Cleanup complete."
 
 products:
   - test_case: [cleanup_old_artifacts]

From ffedb815c72ab0fe84b6030b8165cd6ab7f05d19 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?oliver=20k=C3=B6nig?= <okoenig@nvidia.com>
Date: Mon, 1 Jun 2026 14:38:25 +0000
Subject: [PATCH 14/15] ci: scope cleanup to weekly-* and release-* top-level
 dirs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- Recipe now targets only `weekly-*` and `release-*` directories at
  depth 1 of each `CLEANUP_PATHS` base dir; everything else is left alone.
- Pattern match replaces the `CLEANUP_PROTECT` traversal filter as the
  safety boundary. Drop `CLEANUP_PROTECT` plumbing from
  `launch_jet_workload.py` and `04.functional-tests.yml`.
- Switch from per-file `find -delete` + empty-dir sweep to a single
  `find ... -print0 | xargs -0 -r rm -rf` over matched directory trees.

Signed-off-by: oliver könig <okoenig@nvidia.com>
---
 .gitlab/stages/04.functional-tests.yml        |  3 --
 .../python_scripts/launch_jet_workload.py     |  1 -
 tests/test_utils/recipes/_cleanup.yaml        | 38 ++++++-------------
 3 files changed, 12 insertions(+), 30 deletions(-)

diff --git a/.gitlab/stages/04.functional-tests.yml b/.gitlab/stages/04.functional-tests.yml
index 17130514cd2..a71a6896ef2 100644
--- a/.gitlab/stages/04.functional-tests.yml
+++ b/.gitlab/stages/04.functional-tests.yml
@@ -246,21 +246,18 @@ functional:cleanup_dgx_a100:
   variables:
     CLEANUP_PLATFORM: dgx_a100
     CLEANUP_PATHS: $CLEANUP_PATHS_DGX_A100
-    CLEANUP_PROTECT: $CLEANUP_PROTECT_DGX_A100
 
 functional:cleanup_dgx_h100:
   extends: [.functional_cleanup]
   variables:
     CLEANUP_PLATFORM: dgx_h100
     CLEANUP_PATHS: $CLEANUP_PATHS_DGX_H100
-    CLEANUP_PROTECT: $CLEANUP_PROTECT_DGX_H100
 
 functional:cleanup_dgx_gb200:
   extends: [.functional_cleanup]
   variables:
     CLEANUP_PLATFORM: dgx_gb200
     CLEANUP_PATHS: $CLEANUP_PATHS_DGX_GB200
-    CLEANUP_PROTECT: $CLEANUP_PROTECT_DGX_GB200
 
 functional:run_lts_dgx_a100:
   extends: [.functional_run]
diff --git a/tests/test_utils/python_scripts/launch_jet_workload.py b/tests/test_utils/python_scripts/launch_jet_workload.py
index 0434d083ebf..198c071af9c 100644
--- a/tests/test_utils/python_scripts/launch_jet_workload.py
+++ b/tests/test_utils/python_scripts/launch_jet_workload.py
@@ -154,7 +154,6 @@ def launch_and_wait_for_completion(
                                         "CLUSTER": cluster,
                                         "RUN_ID": str(uuid.uuid4()),
                                         "CLEANUP_PATHS": os.getenv("CLEANUP_PATHS") or "",
-                                        "CLEANUP_PROTECT": os.getenv("CLEANUP_PROTECT") or "",
                                     }
                                 }
                             }
diff --git a/tests/test_utils/recipes/_cleanup.yaml b/tests/test_utils/recipes/_cleanup.yaml
index f0f5ccf3099..ae4d6974d49 100644
--- a/tests/test_utils/recipes/_cleanup.yaml
+++ b/tests/test_utils/recipes/_cleanup.yaml
@@ -17,34 +17,19 @@ spec:
 
     THRESHOLD_DAYS=28
 
-    # CLEANUP_PATHS: newline-separated scan roots. The recipe descends each
-    # root, deletes files older than THRESHOLD_DAYS, then sweeps empty
-    # directories.
-    # CLEANUP_PROTECT: newline-separated subtrees to prune (skip entirely)
-    # during traversal.
+    # CLEANUP_PATHS: newline-separated base directories. For each base dir,
+    # the recipe deletes immediate child directories whose names match
+    # `weekly-*` or `release-*` and whose mtime is older than
+    # THRESHOLD_DAYS. The pattern match is the safety boundary — nothing
+    # outside those two name prefixes is touched.
     if [[ -z "${{CLEANUP_PATHS:-}}" ]]; then
         echo "CLEANUP_PATHS env var not set; nothing to do."
         exit 0
     fi
 
-    # Use `! -path X ! -path X/*` filters rather than `-prune`: `-delete`
-    # implies `-depth`, and `-depth` makes `-prune` a silent no-op. With
-    # negative-path matching, find still descends into the protected
-    # subtree but never matches any entry inside it.
-    SKIP_ARGS=()
-    if [[ -n "${{CLEANUP_PROTECT:-}}" ]]; then
-        while IFS= read -r protect; do
-            [[ -z "$protect" ]] && continue
-            SKIP_ARGS+=( ! -path "$protect" ! -path "$protect/*" )
-        done <<<"$CLEANUP_PROTECT"
-    fi
-
-    echo "Scan roots:"
+    echo "Base dirs (top-level only):"
     printf '%s\n' "$CLEANUP_PATHS" | sed 's/^/  - /'
-    if [[ -n "${{CLEANUP_PROTECT:-}}" ]]; then
-        echo "Protected subtrees (skipped):"
-        printf '%s\n' "$CLEANUP_PROTECT" | sed 's/^/  - /'
-    fi
+    echo "Match patterns: weekly-* release-*"
     echo "Threshold: $THRESHOLD_DAYS days"
 
     while IFS= read -r path; do
@@ -53,10 +38,11 @@ spec:
             echo "Skipping unsafe or missing path: $path"
             continue
         fi
-        echo "--- Deleting files older than $THRESHOLD_DAYS days under $path ---"
-        find "$path" -mindepth 1 "${{SKIP_ARGS[@]}}" -type f -mtime +$THRESHOLD_DAYS -print -delete 2>/dev/null || true
-        echo "--- Pruning empty directories under $path ---"
-        find "$path" -mindepth 1 "${{SKIP_ARGS[@]}}" -type d -empty -print -delete 2>/dev/null || true
+        echo "--- Deleting weekly-*/release-* dirs older than $THRESHOLD_DAYS days under $path ---"
+        find "$path" -mindepth 1 -maxdepth 1 -type d \
+            \( -name 'weekly-*' -o -name 'release-*' \) \
+            -mtime +$THRESHOLD_DAYS -print0 \
+          | xargs -0 -r rm -rf
     done <<<"$CLEANUP_PATHS"
 
     echo "Cleanup complete."

From 80c4ed3b66df977863b9fb236cb38083b81ecd56 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?oliver=20k=C3=B6nig?= <okoenig@nvidia.com>
Date: Mon, 1 Jun 2026 15:37:16 +0000
Subject: [PATCH 15/15] ci: stop launcher loop when JET workload emits only
 mainrank logs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The cleanup recipe runs with gpus: 0 and no torchrun, so
extract_torchrunlogs_to_string returns empty even when the JET pipeline
succeeds. The previous "No logs found, retry" guard tripped purely on
the absent per-rank logs and looped until n_attempts hit 9. Treat the
run as having logs when either allranks OR mainrank output is non-empty;
the cleanup workload still produces output_script-0.log, so the launcher
now exits on JET pipeline status.

Signed-off-by: oliver könig <okoenig@nvidia.com>
---
 tests/test_utils/python_scripts/launch_jet_workload.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/test_utils/python_scripts/launch_jet_workload.py b/tests/test_utils/python_scripts/launch_jet_workload.py
index 198c071af9c..2ffa5c93bfb 100644
--- a/tests/test_utils/python_scripts/launch_jet_workload.py
+++ b/tests/test_utils/python_scripts/launch_jet_workload.py
@@ -488,7 +488,7 @@ def main(
             ["\n".join(log_lines) for log_lines in allranks_logs.values()]
         )
         concat_mainrank_log = "\n".join(mainrank_log)
-        if concat_allranks_logs.strip() == "":
+        if concat_allranks_logs.strip() == "" and concat_mainrank_log.strip() == "":
             logger.error("No logs found. Try again.")
             n_attempts += 1
             continue