ci: prune old artifacts on cluster lustre during weekly/release runs

.gitlab/stages/04.functional-tests.yml

@@ -176,41 +176,154 @@ functional:configure:
   inherit:
     variables: true
 
+# Prune artifacts older than 14 days from the JET assets/artifacts roots on
+# each cluster. Gated on release/weekly scope so it does not run on every MR
+# pipeline. Functional `run_*` jobs depend on the matching cleanup job via
+# `optional: true`, so pipelines without a cleanup job (e.g. `mr` scope) still
+# start their workloads immediately.
+.functional_cleanup_rules:
+  stage: functional_tests
+  rules:
+    - if: $BUILD == "no"
+      when: never
+    - if: $FUNCTIONAL_TEST == "yes" && ($FUNCTIONAL_TEST_SCOPE == "release" || $FUNCTIONAL_TEST_SCOPE == "weekly")
+      when: on_success
+    - when: never
+
+.functional_cleanup:
+  needs:
+    - functional:configure
+    - test:build_image
+  extends: [.functional_cleanup_rules]
+  image: ${UTILITY_IMAGE}:${CI_PIPELINE_ID}
+  tags:
+    - arch/amd64
+    - env/prod
+    - origin/jet-fleet
+    - owner/jet-core
+    - purpose/utility
+    - team/megatron
+  timeout: 30m
+  allow_failure: true
+  script:
+    - |
+      case "$CLEANUP_PLATFORM" in
+        dgx_a100)
+          GPU_CLUSTER=$([[ "$CLUSTER_A100" != "" ]] && echo "$CLUSTER_A100" || echo "$DEFAULT_A100_CLUSTER")
+          ;;
+        dgx_h100)
+          GPU_CLUSTER=$([[ "$CLUSTER_H100" != "" ]] && echo "$CLUSTER_H100" || echo "$DEFAULT_H100_CLUSTER")
+          ;;
+        dgx_gb200)
+          GPU_CLUSTER=$([[ "$CLUSTER_GB200" != "" ]] && echo "$CLUSTER_GB200" || echo "$DEFAULT_GB200_CLUSTER")
+          ;;
+        *)
+          echo "Unknown CLEANUP_PLATFORM: $CLEANUP_PLATFORM"
+          exit 1
+          ;;
+      esac
+    - CLEANUP_CLUSTER="cpu_${GPU_CLUSTER#dgx*_}"
+    - echo "Cleanup target = $CLEANUP_CLUSTER (CPU partition of $GPU_CLUSTER)"
+    - export PYTHONPATH=$(pwd)
+    - export RO_API_TOKEN=${PAT}
+    - |
+      python tests/test_utils/python_scripts/launch_jet_workload.py \
+        --model cleanup \
+        --test-case cleanup_old_artifacts \
+        --environment dev \
+        --n-repeat 1 \
+        --time-limit 1800 \
+        --scope cleanup \
+        --account ${CI_SLURM_ACCOUNT} \
+        --cluster ${CLEANUP_CLUSTER} \
+        --platform ${CLEANUP_PLATFORM} \
+        --container-tag ${CI_PIPELINE_ID} \
+        --container-image ${UTILITY_IMAGE} \
+        --record-checkpoints false
+
+functional:cleanup_dgx_a100:
+  extends: [.functional_cleanup]
+  variables:
+    CLEANUP_PLATFORM: dgx_a100
+    CLEANUP_PATHS: $CLEANUP_PATHS_DGX_A100
+
+functional:cleanup_dgx_h100:
+  extends: [.functional_cleanup]
+  variables:
+    CLEANUP_PLATFORM: dgx_h100
+    CLEANUP_PATHS: $CLEANUP_PATHS_DGX_H100
+
+functional:cleanup_dgx_gb200:
+  extends: [.functional_cleanup]
+  variables:
+    CLEANUP_PLATFORM: dgx_gb200
+    CLEANUP_PATHS: $CLEANUP_PATHS_DGX_GB200
+
 functional:run_lts_dgx_a100:
   extends: [.functional_run]
   allow_failure: true
+  needs:
+    - functional:configure
+    - test:build_image
+    - job: functional:cleanup_dgx_a100
+      optional: true
   variables:
     ENVIRONMENT: lts
     CLUSTER: A100
 
 functional:run_lts_dgx_h100:
   extends: [.functional_run]
   allow_failure: true
+  needs:
+    - functional:configure
+    - test:build_image
+    - job: functional:cleanup_dgx_h100
+      optional: true
   variables:
     ENVIRONMENT: lts
     CLUSTER: H100
 
 functional:run_lts_dgx_gb200:
   extends: [.functional_run]
   allow_failure: true
+  needs:
+    - functional:configure
+    - test:build_image
+    - job: functional:cleanup_dgx_gb200
+      optional: true
   variables:
     ENVIRONMENT: lts
     CLUSTER: GB200
 
 functional:run_dev_dgx_a100:
   extends: [.functional_run]
+  needs:
+    - functional:configure
+    - test:build_image
+    - job: functional:cleanup_dgx_a100
+      optional: true
   variables:
     ENVIRONMENT: dev
     CLUSTER: A100
 
 functional:run_dev_dgx_h100:
   extends: [.functional_run]
+  needs:
+    - functional:configure
+    - test:build_image
+    - job: functional:cleanup_dgx_h100
+      optional: true
   variables:
     ENVIRONMENT: dev
     CLUSTER: H100
 
 functional:run_dev_dgx_gb200:
   extends: [.functional_run]
+  needs:
+    - functional:configure
+    - test:build_image
+    - job: functional:cleanup_dgx_gb200
+      optional: true
   variables:
     ENVIRONMENT: dev
     CLUSTER: GB200

tests/test_utils/python_scripts/launch_jet_workload.py

@@ -153,6 +153,7 @@ def launch_and_wait_for_completion(
                                         "TRANSFORMERS_OFFLINE": "1",
                                         "CLUSTER": cluster,
                                         "RUN_ID": str(uuid.uuid4()),
+                                        "CLEANUP_PATHS": os.getenv("CLEANUP_PATHS") or "",
                                     }
                                 }
                             }
@@ -487,7 +488,7 @@ def main(
             ["\n".join(log_lines) for log_lines in allranks_logs.values()]
         )
         concat_mainrank_log = "\n".join(mainrank_log)
-        if concat_allranks_logs.strip() == "":
+        if concat_allranks_logs.strip() == "" and concat_mainrank_log.strip() == "":
             logger.error("No logs found. Try again.")
             n_attempts += 1
             continue

tests/test_utils/python_scripts/recipe_parser.py

@@ -97,6 +97,14 @@ def resolve_cluster_config(cluster: str) -> str:
         return "draco-oci-ord"
     if cluster == "dgxh100_coreweave":
         return "coreweave"
+    if cluster == "cpu_eos":
+        return "eos-cpu"
+    if cluster == "cpu_coreweave":
+        return "coreweave-cpu"
+    if cluster == "cpu_dracooci":
+        return "draco-oci-iad-cpu"
+    if cluster == "cpu_oci-hsg":
+        return "oci-hsg-cpu"
     if cluster == "ghci":
         return "ghci"
     raise ValueError(f"Unknown cluster {cluster} provided.")

tests/test_utils/recipes/_cleanup.yaml

@@ -0,0 +1,55 @@
+type: basic
+format_version: 1
+maintainers: [mcore]
+loggers: [stdout]
+spec:
+  name: '{test_case}_{platforms}'
+  model: cleanup
+  build: mcore-pyt-{environment}
+  nodes: 1
+  gpus: 0
+  n_repeat: 1
+  platforms: dgx_h100
+  script_setup: |
+    echo "Cleanup workload — no setup required."
+  script: |-
+    set -euo pipefail
+
+    THRESHOLD_DAYS=28
+
+    # CLEANUP_PATHS: newline-separated base directories. For each base dir,
+    # the recipe deletes immediate child directories whose names match
+    # `weekly-*` or `release-*` and whose mtime is older than
+    # THRESHOLD_DAYS. The pattern match is the safety boundary — nothing
+    # outside those two name prefixes is touched.
+    if [[ -z "${{CLEANUP_PATHS:-}}" ]]; then
+        echo "CLEANUP_PATHS env var not set; nothing to do."
+        exit 0
+    fi
+
+    echo "Base dirs (top-level only):"
+    printf '%s\n' "$CLEANUP_PATHS" | sed 's/^/  - /'
+    echo "Match patterns: weekly-* release-*"
+    echo "Threshold: $THRESHOLD_DAYS days"
+
+    while IFS= read -r path; do
+        [[ -z "$path" ]] && continue
+        if [[ "$path" == "/" || ! -d "$path" ]]; then
+            echo "Skipping unsafe or missing path: $path"
+            continue
+        fi
+        echo "--- Deleting weekly-*/release-* dirs older than $THRESHOLD_DAYS days under $path ---"
+        find "$path" -mindepth 1 -maxdepth 1 -type d \
+            \( -name 'weekly-*' -o -name 'release-*' \) \
+            -mtime +$THRESHOLD_DAYS -print0 \
+          | xargs -0 -r rm -rf
+    done <<<"$CLEANUP_PATHS"
+
+    echo "Cleanup complete."
+
+products:
+  - test_case: [cleanup_old_artifacts]
+    products:
+      - environment: [dev]
+        scope: [cleanup]
+        platforms: [dgx_a100, dgx_h100, dgx_gb200]