diff --git a/.gitlab/stages/04.functional-tests.yml b/.gitlab/stages/04.functional-tests.yml index a12b1c87213..a71a6896ef2 100644 --- a/.gitlab/stages/04.functional-tests.yml +++ b/.gitlab/stages/04.functional-tests.yml @@ -176,9 +176,97 @@ functional:configure: inherit: variables: true +# Prune artifacts older than 14 days from the JET assets/artifacts roots on +# each cluster. Gated on release/weekly scope so it does not run on every MR +# pipeline. Functional `run_*` jobs depend on the matching cleanup job via +# `optional: true`, so pipelines without a cleanup job (e.g. `mr` scope) still +# start their workloads immediately. +.functional_cleanup_rules: + stage: functional_tests + rules: + - if: $BUILD == "no" + when: never + - if: $FUNCTIONAL_TEST == "yes" && ($FUNCTIONAL_TEST_SCOPE == "release" || $FUNCTIONAL_TEST_SCOPE == "weekly") + when: on_success + - when: never + +.functional_cleanup: + needs: + - functional:configure + - test:build_image + extends: [.functional_cleanup_rules] + image: ${UTILITY_IMAGE}:${CI_PIPELINE_ID} + tags: + - arch/amd64 + - env/prod + - origin/jet-fleet + - owner/jet-core + - purpose/utility + - team/megatron + timeout: 30m + allow_failure: true + script: + - | + case "$CLEANUP_PLATFORM" in + dgx_a100) + GPU_CLUSTER=$([[ "$CLUSTER_A100" != "" ]] && echo "$CLUSTER_A100" || echo "$DEFAULT_A100_CLUSTER") + ;; + dgx_h100) + GPU_CLUSTER=$([[ "$CLUSTER_H100" != "" ]] && echo "$CLUSTER_H100" || echo "$DEFAULT_H100_CLUSTER") + ;; + dgx_gb200) + GPU_CLUSTER=$([[ "$CLUSTER_GB200" != "" ]] && echo "$CLUSTER_GB200" || echo "$DEFAULT_GB200_CLUSTER") + ;; + *) + echo "Unknown CLEANUP_PLATFORM: $CLEANUP_PLATFORM" + exit 1 + ;; + esac + - CLEANUP_CLUSTER="cpu_${GPU_CLUSTER#dgx*_}" + - echo "Cleanup target = $CLEANUP_CLUSTER (CPU partition of $GPU_CLUSTER)" + - export PYTHONPATH=$(pwd) + - export RO_API_TOKEN=${PAT} + - | + python tests/test_utils/python_scripts/launch_jet_workload.py \ + --model cleanup \ + --test-case cleanup_old_artifacts \ + --environment dev \ + --n-repeat 1 \ + --time-limit 1800 \ + --scope cleanup \ + --account ${CI_SLURM_ACCOUNT} \ + --cluster ${CLEANUP_CLUSTER} \ + --platform ${CLEANUP_PLATFORM} \ + --container-tag ${CI_PIPELINE_ID} \ + --container-image ${UTILITY_IMAGE} \ + --record-checkpoints false + +functional:cleanup_dgx_a100: + extends: [.functional_cleanup] + variables: + CLEANUP_PLATFORM: dgx_a100 + CLEANUP_PATHS: $CLEANUP_PATHS_DGX_A100 + +functional:cleanup_dgx_h100: + extends: [.functional_cleanup] + variables: + CLEANUP_PLATFORM: dgx_h100 + CLEANUP_PATHS: $CLEANUP_PATHS_DGX_H100 + +functional:cleanup_dgx_gb200: + extends: [.functional_cleanup] + variables: + CLEANUP_PLATFORM: dgx_gb200 + CLEANUP_PATHS: $CLEANUP_PATHS_DGX_GB200 + functional:run_lts_dgx_a100: extends: [.functional_run] allow_failure: true + needs: + - functional:configure + - test:build_image + - job: functional:cleanup_dgx_a100 + optional: true variables: ENVIRONMENT: lts CLUSTER: A100 @@ -186,6 +274,11 @@ functional:run_lts_dgx_a100: functional:run_lts_dgx_h100: extends: [.functional_run] allow_failure: true + needs: + - functional:configure + - test:build_image + - job: functional:cleanup_dgx_h100 + optional: true variables: ENVIRONMENT: lts CLUSTER: H100 @@ -193,24 +286,44 @@ functional:run_lts_dgx_h100: functional:run_lts_dgx_gb200: extends: [.functional_run] allow_failure: true + needs: + - functional:configure + - test:build_image + - job: functional:cleanup_dgx_gb200 + optional: true variables: ENVIRONMENT: lts CLUSTER: GB200 functional:run_dev_dgx_a100: extends: [.functional_run] + needs: + - functional:configure + - test:build_image + - job: functional:cleanup_dgx_a100 + optional: true variables: ENVIRONMENT: dev CLUSTER: A100 functional:run_dev_dgx_h100: extends: [.functional_run] + needs: + - functional:configure + - test:build_image + - job: functional:cleanup_dgx_h100 + optional: true variables: ENVIRONMENT: dev CLUSTER: H100 functional:run_dev_dgx_gb200: extends: [.functional_run] + needs: + - functional:configure + - test:build_image + - job: functional:cleanup_dgx_gb200 + optional: true variables: ENVIRONMENT: dev CLUSTER: GB200 diff --git a/tests/test_utils/python_scripts/launch_jet_workload.py b/tests/test_utils/python_scripts/launch_jet_workload.py index d5629bd432d..2ffa5c93bfb 100644 --- a/tests/test_utils/python_scripts/launch_jet_workload.py +++ b/tests/test_utils/python_scripts/launch_jet_workload.py @@ -153,6 +153,7 @@ def launch_and_wait_for_completion( "TRANSFORMERS_OFFLINE": "1", "CLUSTER": cluster, "RUN_ID": str(uuid.uuid4()), + "CLEANUP_PATHS": os.getenv("CLEANUP_PATHS") or "", } } } @@ -487,7 +488,7 @@ def main( ["\n".join(log_lines) for log_lines in allranks_logs.values()] ) concat_mainrank_log = "\n".join(mainrank_log) - if concat_allranks_logs.strip() == "": + if concat_allranks_logs.strip() == "" and concat_mainrank_log.strip() == "": logger.error("No logs found. Try again.") n_attempts += 1 continue diff --git a/tests/test_utils/python_scripts/recipe_parser.py b/tests/test_utils/python_scripts/recipe_parser.py index 018cc043a51..a67b131b91d 100644 --- a/tests/test_utils/python_scripts/recipe_parser.py +++ b/tests/test_utils/python_scripts/recipe_parser.py @@ -97,6 +97,14 @@ def resolve_cluster_config(cluster: str) -> str: return "draco-oci-ord" if cluster == "dgxh100_coreweave": return "coreweave" + if cluster == "cpu_eos": + return "eos-cpu" + if cluster == "cpu_coreweave": + return "coreweave-cpu" + if cluster == "cpu_dracooci": + return "draco-oci-iad-cpu" + if cluster == "cpu_oci-hsg": + return "oci-hsg-cpu" if cluster == "ghci": return "ghci" raise ValueError(f"Unknown cluster {cluster} provided.") diff --git a/tests/test_utils/recipes/_cleanup.yaml b/tests/test_utils/recipes/_cleanup.yaml new file mode 100644 index 00000000000..ae4d6974d49 --- /dev/null +++ b/tests/test_utils/recipes/_cleanup.yaml @@ -0,0 +1,55 @@ +type: basic +format_version: 1 +maintainers: [mcore] +loggers: [stdout] +spec: + name: '{test_case}_{platforms}' + model: cleanup + build: mcore-pyt-{environment} + nodes: 1 + gpus: 0 + n_repeat: 1 + platforms: dgx_h100 + script_setup: | + echo "Cleanup workload — no setup required." + script: |- + set -euo pipefail + + THRESHOLD_DAYS=28 + + # CLEANUP_PATHS: newline-separated base directories. For each base dir, + # the recipe deletes immediate child directories whose names match + # `weekly-*` or `release-*` and whose mtime is older than + # THRESHOLD_DAYS. The pattern match is the safety boundary — nothing + # outside those two name prefixes is touched. + if [[ -z "${{CLEANUP_PATHS:-}}" ]]; then + echo "CLEANUP_PATHS env var not set; nothing to do." + exit 0 + fi + + echo "Base dirs (top-level only):" + printf '%s\n' "$CLEANUP_PATHS" | sed 's/^/ - /' + echo "Match patterns: weekly-* release-*" + echo "Threshold: $THRESHOLD_DAYS days" + + while IFS= read -r path; do + [[ -z "$path" ]] && continue + if [[ "$path" == "/" || ! -d "$path" ]]; then + echo "Skipping unsafe or missing path: $path" + continue + fi + echo "--- Deleting weekly-*/release-* dirs older than $THRESHOLD_DAYS days under $path ---" + find "$path" -mindepth 1 -maxdepth 1 -type d \ + \( -name 'weekly-*' -o -name 'release-*' \) \ + -mtime +$THRESHOLD_DAYS -print0 \ + | xargs -0 -r rm -rf + done <<<"$CLEANUP_PATHS" + + echo "Cleanup complete." + +products: + - test_case: [cleanup_old_artifacts] + products: + - environment: [dev] + scope: [cleanup] + platforms: [dgx_a100, dgx_h100, dgx_gb200]