diff --git a/tools/trigger_internal_ci.md b/tools/trigger_internal_ci.md index 77239bcbace..5a6e949b523 100644 --- a/tools/trigger_internal_ci.md +++ b/tools/trigger_internal_ci.md @@ -40,6 +40,7 @@ python tools/trigger_internal_ci.py \ [--functional-test-scope mr] \ [--functional-test-repeat 5] \ [--functional-test-cases all] \ + [--functional-test-time-limit 14400] \ [--dry-run] ``` @@ -50,6 +51,7 @@ python tools/trigger_internal_ci.py \ | `--functional-test-scope` | `mr` | `FUNCTIONAL_TEST_SCOPE` pipeline variable | | `--functional-test-repeat` | `5` | `FUNCTIONAL_TEST_REPEAT` pipeline variable | | `--functional-test-cases` | `all` | `FUNCTIONAL_TEST_CASES` pipeline variable | +| `--functional-test-time-limit` | *(scope-dependent)* | `FUNCTIONAL_TEST_TIME_LIMIT` pipeline variable, in seconds. Defaults to `14400` (4h) for the long-running `release` and `weekly` scopes; left unset otherwise. | | `--dry-run` | off | Print what would happen without pushing or triggering | ## Example diff --git a/tools/trigger_internal_ci.py b/tools/trigger_internal_ci.py index 6b462309c4a..9afc9515cf4 100644 --- a/tools/trigger_internal_ci.py +++ b/tools/trigger_internal_ci.py @@ -40,9 +40,32 @@ "INTEGRATION_TEST": "no", } +# Scopes whose recipes run full convergence/checkpointing workloads and need a +# long wall-clock budget. The default short-scope time limit is left untouched. +LONG_RUNNING_SCOPES = ("release", "weekly") +LONG_RUNNING_TIME_LIMIT_SECONDS = 4 * 60 * 60 + logger = logging.getLogger(__name__) +def resolve_time_limit(scope, override): + """Resolve the FUNCTIONAL_TEST_TIME_LIMIT value for a functional test scope. + + Args: + scope: The functional test scope (e.g. ``mr``, ``release``, ``weekly``). + override: Explicit time limit in seconds, or ``None`` to auto-resolve. + + Returns: + The time limit in seconds when one applies, otherwise ``None`` so the + variable is left unset and short-running scopes keep their default. + """ + if override is not None: + return override + if scope in LONG_RUNNING_SCOPES: + return LONG_RUNNING_TIME_LIMIT_SECONDS + return None + + def get_remote_url(origin): """Return the fetch URL configured for the given git remote name.""" result = subprocess.run( @@ -77,7 +100,9 @@ def get_current_branch(): def git_push(origin, target_branch, dry_run=False): """Force-push HEAD to the given branch on the named git remote.""" if dry_run: - logger.info("[DRY RUN] Would push HEAD to remote '%s' as %s", origin, target_branch) + logger.info( + "[DRY RUN] Would push HEAD to remote '%s' as %s", origin, target_branch + ) return subprocess.run( ["git", "push", origin, f"HEAD:{target_branch}", "--force"], @@ -96,7 +121,10 @@ def trigger_pipeline(gitlab_url, access_token, ref, pipeline_vars, dry_run=False ) return logger.info( - "Triggering pipeline on https://%s project %s @ %s", gitlab_url, GITLAB_PROJECT_ID, ref + "Triggering pipeline on https://%s project %s @ %s", + gitlab_url, + GITLAB_PROJECT_ID, + ref, ) gl = gitlab.Gitlab(f"https://{gitlab_url}", private_token=access_token) project = gl.projects.get(GITLAB_PROJECT_ID, lazy=True) @@ -136,6 +164,16 @@ def main(): default="all", help="FUNCTIONAL_TEST_CASES pipeline variable (default: all)", ) + parser.add_argument( + "--functional-test-time-limit", + type=int, + default=None, + help=( + "FUNCTIONAL_TEST_TIME_LIMIT pipeline variable in seconds. Defaults to " + "14400 (4h) for the long-running 'release' and 'weekly' scopes and is " + "left unset for other scopes." + ), + ) parser.add_argument( "--cluster-a100", default=None, @@ -180,6 +218,12 @@ def main(): "FUNCTIONAL_TEST_CASES": args.functional_test_cases, } + time_limit = resolve_time_limit( + args.functional_test_scope, args.functional_test_time_limit + ) + if time_limit is not None: + pipeline_vars["FUNCTIONAL_TEST_TIME_LIMIT"] = str(time_limit) + for var, val in [ ("CLUSTER_A100", args.cluster_a100), ("CLUSTER_H100", args.cluster_h100), @@ -189,9 +233,13 @@ def main(): pipeline_vars[var] = val trigger_pipeline( - gitlab_hostname, args.access_token, target_branch, pipeline_vars, dry_run=args.dry_run + gitlab_hostname, + args.access_token, + target_branch, + pipeline_vars, + dry_run=args.dry_run, ) if __name__ == "__main__": - main() \ No newline at end of file + main()