Add preview CLI option (#2914)

This commit adds a new CLI run option `-preview`.

When running a workflow script with preview all process executions are skipped.
This is useful to evaluate the pipeline execution without actually running.

Also when used along with `-with-dag` option it allows the rendering of the
execution DAG without running the pipeline.

Signed-off-by: Paolo Di Tommaso <[email protected]>

Author: pditommaso

docs/cli.rst

@@ -783,6 +783,8 @@ the :ref:`k8s-page` section.
 +---------------------------+-------------+--------------------------------------------------------------------------------+
 | -pod-image                |             | Specify the container image for the Nextflow pod.                              |
 +---------------------------+-------------+--------------------------------------------------------------------------------+
+| -preview                  |             | Run the workflow script skipping the execution of all processes                |
++---------------------------+-------------+--------------------------------------------------------------------------------+
 | -process.                 | {}          | Set process options. Syntax ``-process.key=value``                             |
 +---------------------------+-------------+--------------------------------------------------------------------------------+
 | -profile                  |             | Choose a configuration profile.                                                |

modules/nextflow/src/main/groovy/nextflow/Session.groovy

@@ -447,16 +447,26 @@ class Session implements ISession {
         igniters.add(action)
     }
 
-    void fireDataflowNetwork() {
+    void fireDataflowNetwork(boolean preview=false) {
         checkConfig()
         notifyFlowBegin()
 
-        if( !NextflowMeta.instance.isDsl2() )
+        if( !NextflowMeta.instance.isDsl2() ) {
             return
+        }
 
         // bridge any dataflow queue into a broadcast channel
         CH.broadcast()
 
+        if( preview ) {
+            terminated = true
+        }
+        else {
+            callIgniters()
+        }
+    }
+
+    private void callIgniters() {
         log.debug "Ignite dataflow network (${igniters.size()})"
         for( Closure action : igniters ) {
             try {
@@ -611,15 +621,15 @@ class Session implements ISession {
         terminated = true
         monitorsBarrier.awaitCompletion()
         log.debug "Session await > all barriers passed"
+        if( !aborted ) {
+            joinAllOperators()
+            log.trace "Session > after processors join"
+        }
     }
 
     void destroy() {
         try {
             log.trace "Session > destroying"
-            if( !aborted ) {
-                joinAllOperators()
-                log.trace "Session > after processors join"
-            }
 
             // invoke shutdown callbacks
             shutdown0()

modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy

@@ -237,6 +237,9 @@ class CmdRun extends CmdBase implements HubOptions {
     @Parameter(names=['-stub-run','-stub'], description = 'Execute the workflow replacing process scripts with command stubs')
     boolean stubRun
 
+    @Parameter(names=['-preview'], description = "Run the workflow script skipping the execution of all processes")
+    boolean preview
+
     @Parameter(names=['-plugins'], description = 'Specify the plugins to be applied for this run e.g. nf-amazon,nf-tower')
     String plugins
 
@@ -309,6 +312,7 @@ class CmdRun extends CmdBase implements HubOptions {
         // -- create a new runner instance
         final runner = new ScriptRunner(config)
         runner.setScript(scriptFile)
+        runner.setPreview(this.preview)
         runner.session.profile = profile
         runner.session.commandLine = launcher.cliString
         runner.session.ansiLog = launcher.options.ansiLog
@@ -350,11 +354,11 @@ class CmdRun extends CmdBase implements HubOptions {
         NextflowMeta.instance.enableDsl(dsl)
         // -- show launch info
         final ver = NF.dsl2 ? DSL2 : DSL1
+        final head = preview ? "* PREVIEW * $scriptFile.repository" : "Launching `$scriptFile.repository`"
         if( scriptFile.repository )
-            log.info "Launching `$scriptFile.repository` [$runName] DSL${ver} - revision: ${scriptFile.revisionInfo}"
+            log.info "${head} [$runName] DSL${ver} - revision: ${scriptFile.revisionInfo}"
         else
-            log.info "Launching `$scriptFile.source` [$runName] DSL${ver} - revision: ${scriptFile.getScriptId()?.substring(0,10)}"
-
+            log.info "${head} [$runName] DSL${ver} - revision: ${scriptFile.getScriptId()?.substring(0,10)}"
     }
 
     static String detectDslMode(ConfigMap config, String scriptText, Map sysEnv) {

modules/nextflow/src/main/groovy/nextflow/script/ScriptRunner.groovy

@@ -57,6 +57,11 @@ class ScriptRunner {
      */
     private def result
 
+    /**
+     * Simulate execution and exit
+     */
+    private boolean preview
+
     /**
      * Instantiate the runner object creating a new session
      */
@@ -81,6 +86,11 @@ class ScriptRunner {
         return this
     }
 
+    ScriptRunner setPreview(boolean  value ) {
+        this.preview = value
+        return this
+    }
+
     Session getSession() { session }
 
     /**
@@ -118,8 +128,10 @@ class ScriptRunner {
             parseScript(scriptFile, entryName)
             // run the code
             run()
-            // await termination
-            terminate()
+            // await completion
+            await()
+            // shutdown session
+            shutdown()
         }
         catch (Throwable e) {
             session.abort(e)
@@ -213,12 +225,17 @@ class ScriptRunner {
         // -- normalise output
         result = normalizeOutput(scriptParser.getResult())
         // -- ignite dataflow network
-        session.fireDataflowNetwork()
+        session.fireDataflowNetwork(preview)
     }
 
-    protected terminate() {
+    protected await() {
+        if( preview )
+            return
         log.debug "> Await termination "
         session.await()
+    }
+
+    protected shutdown() {
         session.destroy()
         session.cleanup()
         log.debug "> Execution complete -- Goodbye"