Parameter for api_url and get run_name and workDir from Seqera Platform API

.github/workflows/ci.yml

@@ -21,10 +21,12 @@ jobs:
     # Only run on push if this is the nf-aggregate dev branch (merged PRs)
     if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'seqeralabs/nf-aggregate') }}"
     runs-on: ubuntu-latest
+    env:
+      TOWER_ACCESS_TOKEN: ${{ secrets.TOWER_ACCESS_TOKEN }}
     strategy:
       matrix:
         NXF_VER:
-          - "23.04.0"
+          - "23.10.0"
           - "latest-everything"
     steps:
       - name: Check out pipeline code

.prettierignore

@@ -10,3 +10,4 @@ testing/
 testing*
 *.pyc
 bin/
+*.html

assets/schema_input.json

@@ -16,8 +16,8 @@
                 "type": "string",
                 "pattern": "^[a-z]+/[a-z]+$",
                 "errorMessage": "Please provide a valid Seqera Platform Workspace name"
-            },
+            }
         },
         "required": ["id", "workspace"]
     }
-}
+}

bin/plot_run_gantt.py

@@ -73,6 +73,7 @@ def build_gantt(title: str, input_dir: str, output_file: str):
             "instance": f"{d.get('instanceId', 'HEAD')} ({d.get('machineType', 'unknown')})",
         }
         for d in data
+        if d.get("complete")
     )
 
     fig = px.timeline(

conf/container.config

@@ -1,3 +1,3 @@
-process { withName: 'NF_AGGREGATE:SEQERA_RUNS_DUMP' { container = 'public.ecr.aws/seqera-labs/wave/containers:tower-cli-0.9.0--acdf39d9a634af2c' } }
-process { withName: 'NF_AGGREGATE:MULTIQC' { container = 'public.ecr.aws/seqera-labs/wave/containers:multiqc--1e9e68bfb966352f' } }
-process { withName: 'NF_AGGREGATE:PLOT_RUN_GANTT' { container = 'public.ecr.aws/seqera-labs/wave/containers:click-8.0.1_pandas-1.1.5_plotly_express-0.4.1_typing-3.10.0.0--ce5937852dee34bb' } }
+process { withName: 'NF_AGGREGATE:SEQERA_RUNS_DUMP' { container = 'public.ecr.aws/seqera-labs/wave/containers:tower-cli-0.9.0--2cb0f2e9d85d026b' } }
+process { withName: 'NF_AGGREGATE:MULTIQC' { container = 'public.ecr.aws/seqera-labs/wave/containers:multiqc--545f544898344f7d' } }
+process { withName: 'NF_AGGREGATE:PLOT_RUN_GANTT' { container = 'public.ecr.aws/seqera-labs/wave/containers:click-8.0.1_pandas-1.1.5_plotly_express-0.4.1_typing-3.10.0.0--342dabfe6548a051' } }

modules/local/seqera_runs_dump/main.nf

@@ -1,3 +1,47 @@
+@Grab('com.github.groovy-wslite:groovy-wslite:1.1.2')
+import wslite.rest.RESTClient
+
+Long getWorkspaceId(orgName, workspaceName, client, authHeader) {
+    def orgResponse = client.get(path: '/orgs', headers: authHeader)
+    if (orgResponse.statusCode == 200) {
+        def orgMap = orgResponse.json?.organizations.collectEntries { org -> [org.name, org.orgId]}
+        def orgId = orgMap.get(orgName)
+        if(!orgId) log.warn "Could not find organization '${orgName}'"
+
+        // GET the workspaces in this org
+        def workspaceReponse = client.get(path: "/orgs/${orgId}/workspaces", headers: authHeader)
+        if (workspaceReponse.statusCode == 200) {
+            def workspaceMap = workspaceReponse.json?.workspaces.collectEntries { ws -> [ws.name, ws.id]}
+            return workspaceMap?.get(workspaceName)
+        }
+    }
+    return null
+}
+
+Map getRunMetadata(meta, log) {
+    def runId = meta.id
+    (orgName, workspaceName) = meta.workspace.split("/")
+
+    def client = new RESTClient("https://api.tower.nf")
+    token = System.getenv("TOWER_ACCESS_TOKEN")
+    authHeader = ["Authorization": "Bearer ${token}"]
+
+    try {
+        def workspaceId = getWorkspaceId(orgName, workspaceName, client, authHeader)
+        if (workspaceId) {
+            def workflowResponse = client.get(path: "/workflow/${runId}", query: ["workspaceId":workspaceId], headers: authHeader)
+            if (workflowResponse.statusCode == 200) {
+                metaMap = workflowResponse?.json?.workflow?.subMap("runName", "workDir")
+                return metaMap ?: [:]
+            }
+        }
+    } catch(Exception ex) {
+        log.warn "Could not get workflow details for workflow ${runId} in workspace ${meta.workspace}"
+    }
+
+    return [:]
+}
+
 process SEQERA_RUNS_DUMP {
     tag "$meta.id"
     conda 'tower-cli=0.9.0'
@@ -6,18 +50,20 @@ process SEQERA_RUNS_DUMP {
     val meta
 
     output:
-    tuple val(meta), path("${prefix}")    , emit: run_dump
-    tuple val(meta), path("workflow.json"), emit: workflow_json
+    tuple val(newMeta), path("${prefix}")    , emit: run_dump
+    tuple val(newMeta), path("workflow.json"), emit: workflow_json
     path "versions.yml"                   , emit: versions
 
     script:
     def args = task.ext.args ?: ''
     def args2 = task.ext.args2 ?: ''
     prefix = task.ext.prefix ?: "${meta.id}"
+    newMeta = meta + getRunMetadata(meta, log)
     """
     tw \\
-        --access-token=\$TOWER_ACCESS_TOKEN \\
         $args \\
+        --url=${params.api_endpoint_url} \\
+        --access-token=\$TOWER_ACCESS_TOKEN \\
         runs \\
         dump \\
         -id=${meta.id} \\

nextflow.config

@@ -13,6 +13,9 @@ params {
     input                      = null
     skip_run_gantt             = false
 
+    // Endpoint options
+    api_endpoint_url           = "https://api.tower.nf"
+
     // MultiQC options
     multiqc_config             = null
     multiqc_title              = null
@@ -50,7 +53,6 @@ profiles {
         params.input = "${projectDir}/workflows/nf_aggregate/assets/test_run_ids.csv"
     }
     wave {
-        fusion.enabled = true
         wave.enabled = true
         wave.strategy = ['conda']
         wave.build.repository = 'public.ecr.aws/seqera-labs/wave/containers'

nextflow_schema.json

@@ -40,6 +40,19 @@
                 }
             }
         },
+        "endpoint_options": {
+            "title": "Seqera Platform options",
+            "type": "object",
+            "fa_icon": "fas fa-terminal",
+            "description": "Define interactions with the Seqera Platform.",
+            "required": ["api_endpoint_url"],
+            "properties": {
+                "api_endpoint_url": {
+                    "type": "string",
+                    "default": "https://api.tower.nf"
+                }
+            }
+        },
         "generic_options": {
             "title": "Generic options",
             "type": "object",