No block slurm

plugins/hydra_submitit_launcher/hydra_plugins/hydra_submitit_launcher/config.py

@@ -28,6 +28,11 @@ class BaseQueueConf:
     # redirect stderr to stdout
     stderr_to_stdout: bool = False
 
+    # If True, the launcher will not wait for the job to finish (useful for very long runs)
+    # This value is not passed to submitit, it is used by the launcher itself. When enabled,
+    # a set of sentinel values are returned to the sweeper to indicate that the job is running.
+    no_block: bool = False
+
 
 @dataclass
 class SlurmQueueConf(BaseQueueConf):

plugins/hydra_submitit_launcher/hydra_plugins/hydra_submitit_launcher/submitit_launcher.py

@@ -5,7 +5,13 @@
 from typing import Any, Dict, List, Optional, Sequence
 
 from hydra.core.singleton import Singleton
-from hydra.core.utils import JobReturn, filter_overrides, run_job, setup_globals
+from hydra.core.utils import (
+    JobReturn,
+    JobStatus,
+    filter_overrides,
+    run_job,
+    setup_globals,
+)
 from hydra.plugins.launcher import Launcher
 from hydra.types import HydraContext, TaskFunction
 from omegaconf import DictConfig, OmegaConf, open_dict
@@ -19,6 +25,7 @@ class BaseSubmititLauncher(Launcher):
     _EXECUTOR = "abstract"
 
     def __init__(self, **params: Any) -> None:
+        self.no_block = params.pop("no_block", False)
         self.params = {}
         for k, v in params.items():
             if OmegaConf.is_config(v):
@@ -142,7 +149,12 @@ def launch(
             )
 
         jobs = executor.map_array(self, *zip(*job_params))
-        return [j.results()[0] for j in jobs]
+
+        if self.no_block:
+            sentinal = JobReturn(status=JobStatus.COMPLETED, _return_value=None)
+            return [sentinal] * num_jobs
+        else:
+            return [j.results()[0] for j in jobs]
 
 
 class LocalLauncher(BaseSubmititLauncher):

plugins/hydra_submitit_launcher/tests/test_submitit_launcher.py

@@ -61,3 +61,16 @@ def test_example(tmpdir: Path) -> None:
         ],
         allow_warnings=True,
     )
+
+
+def test_example_no_block(tmpdir: Path) -> None:
+    run_python_script(
+        [
+            "example/my_app.py",
+            "-m",
+            f"hydra.sweep.dir={tmpdir}",
+            "hydra/launcher=submitit_local",
+            "hydra.launcher.no_block=True",
+        ],
+        allow_warnings=True,
+    )