ci: default functional test time limit to 4h for release/weekly scopes

tools/trigger_internal_ci.md

@@ -40,6 +40,7 @@ python tools/trigger_internal_ci.py \
   [--functional-test-scope mr] \
   [--functional-test-repeat 5] \
   [--functional-test-cases all] \
+  [--functional-test-time-limit 14400] \
   [--dry-run]
 ```
 
@@ -50,6 +51,7 @@ python tools/trigger_internal_ci.py \
 | `--functional-test-scope` | `mr` | `FUNCTIONAL_TEST_SCOPE` pipeline variable |
 | `--functional-test-repeat` | `5` | `FUNCTIONAL_TEST_REPEAT` pipeline variable |
 | `--functional-test-cases` | `all` | `FUNCTIONAL_TEST_CASES` pipeline variable |
+| `--functional-test-time-limit` | *(scope-dependent)* | `FUNCTIONAL_TEST_TIME_LIMIT` pipeline variable, in seconds. Defaults to `14400` (4h) for the long-running `release` and `weekly` scopes; left unset otherwise. |
 | `--dry-run` | off | Print what would happen without pushing or triggering |
 
 ## Example

tools/trigger_internal_ci.py

@@ -40,9 +40,32 @@
     "INTEGRATION_TEST": "no",
 }
 
+# Scopes whose recipes run full convergence/checkpointing workloads and need a
+# long wall-clock budget. The default short-scope time limit is left untouched.
+LONG_RUNNING_SCOPES = ("release", "weekly")
+LONG_RUNNING_TIME_LIMIT_SECONDS = 4 * 60 * 60
+
 logger = logging.getLogger(__name__)
 
 
+def resolve_time_limit(scope, override):
+    """Resolve the FUNCTIONAL_TEST_TIME_LIMIT value for a functional test scope.
+
+    Args:
+        scope: The functional test scope (e.g. ``mr``, ``release``, ``weekly``).
+        override: Explicit time limit in seconds, or ``None`` to auto-resolve.
+
+    Returns:
+        The time limit in seconds when one applies, otherwise ``None`` so the
+        variable is left unset and short-running scopes keep their default.
+    """
+    if override is not None:
+        return override
+    if scope in LONG_RUNNING_SCOPES:
+        return LONG_RUNNING_TIME_LIMIT_SECONDS
+    return None
+
+
 def get_remote_url(origin):
     """Return the fetch URL configured for the given git remote name."""
     result = subprocess.run(
@@ -77,7 +100,9 @@ def get_current_branch():
 def git_push(origin, target_branch, dry_run=False):
     """Force-push HEAD to the given branch on the named git remote."""
     if dry_run:
-        logger.info("[DRY RUN] Would push HEAD to remote '%s' as %s", origin, target_branch)
+        logger.info(
+            "[DRY RUN] Would push HEAD to remote '%s' as %s", origin, target_branch
+        )
         return
     subprocess.run(
         ["git", "push", origin, f"HEAD:{target_branch}", "--force"],
@@ -96,7 +121,10 @@ def trigger_pipeline(gitlab_url, access_token, ref, pipeline_vars, dry_run=False
         )
         return
     logger.info(
-        "Triggering pipeline on https://%s project %s @ %s", gitlab_url, GITLAB_PROJECT_ID, ref
+        "Triggering pipeline on https://%s project %s @ %s",
+        gitlab_url,
+        GITLAB_PROJECT_ID,
+        ref,
     )
     gl = gitlab.Gitlab(f"https://{gitlab_url}", private_token=access_token)
     project = gl.projects.get(GITLAB_PROJECT_ID, lazy=True)
@@ -136,6 +164,16 @@ def main():
         default="all",
         help="FUNCTIONAL_TEST_CASES pipeline variable (default: all)",
     )
+    parser.add_argument(
+        "--functional-test-time-limit",
+        type=int,
+        default=None,
+        help=(
+            "FUNCTIONAL_TEST_TIME_LIMIT pipeline variable in seconds. Defaults to "
+            "14400 (4h) for the long-running 'release' and 'weekly' scopes and is "
+            "left unset for other scopes."
+        ),
+    )
     parser.add_argument(
         "--cluster-a100",
         default=None,
@@ -180,6 +218,12 @@ def main():
         "FUNCTIONAL_TEST_CASES": args.functional_test_cases,
     }
 
+    time_limit = resolve_time_limit(
+        args.functional_test_scope, args.functional_test_time_limit
+    )
+    if time_limit is not None:
+        pipeline_vars["FUNCTIONAL_TEST_TIME_LIMIT"] = str(time_limit)
+
     for var, val in [
         ("CLUSTER_A100", args.cluster_a100),
         ("CLUSTER_H100", args.cluster_h100),
@@ -189,9 +233,13 @@ def main():
             pipeline_vars[var] = val
 
     trigger_pipeline(
-        gitlab_hostname, args.access_token, target_branch, pipeline_vars, dry_run=args.dry_run
+        gitlab_hostname,
+        args.access_token,
+        target_branch,
+        pipeline_vars,
+        dry_run=args.dry_run,
     )
 
 
 if __name__ == "__main__":
-    main()
+    main()