add links for skipped tasks in voxelytics workflow reports (#8006)

* add links for skipped tasks in voxelytics workflow reports

* changelog

* comments

* styling

* add evolution and comment

---------

Co-authored-by: Florian M <[email protected]>

Author: 2 people

CHANGELOG.unreleased.md

@@ -32,6 +32,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released
 - The alignment job is in a separate tab of the "AI Tools" now. The "Align Sections" AI job now supports including manually created matches between adjacent section given as skeletons. [#7967](https://github.com/scalableminds/webknossos/pull/7967)  
 - Added `api.tracing.createNode(position, options)`` to the front-end API. [#7998](https://github.com/scalableminds/webknossos/pull/7998)
 - Added a feature to register all segments for a given bounding box at once via the context menu of the bounding box. [#7979](https://github.com/scalableminds/webknossos/pull/7979)
+- Added links in the workflow report for skipped tasks to the corresponding workflow view. [#8006](https://github.com/scalableminds/webknossos/pull/8006)
 
 ### Changed
 - Replaced skeleton tab component with antd's `<Tree />`component. Added support for selecting tree ranges with SHIFT. [#7819](https://github.com/scalableminds/webknossos/pull/7819) 

app/controllers/VoxelyticsController.scala

@@ -1,13 +1,13 @@
 package controllers
 
-import play.silhouette.api.Silhouette
-import play.silhouette.api.actions.SecuredRequest
 import com.scalableminds.util.time.Instant
 import com.scalableminds.util.tools.{Fox, FoxImplicits}
 import models.organization.OrganizationDAO
 import models.voxelytics._
 import play.api.libs.json._
 import play.api.mvc._
+import play.silhouette.api.Silhouette
+import play.silhouette.api.actions.SecuredRequest
 import security.WkEnv
 import utils.{ObjectId, WkConf}
 
@@ -138,7 +138,7 @@ class VoxelyticsController @Inject()(
         combinedTaskRuns <- voxelyticsDAO.findCombinedTaskRuns(sortedRuns.map(_.id), conf.staleTimeout)
 
         // Fetch artifact data for task runs
-        artifacts <- voxelyticsDAO.findArtifacts(sortedRuns.map(_.id), conf.staleTimeout)
+        artifacts <- voxelyticsDAO.findArtifacts(request.identity, sortedRuns.map(_.id), conf.staleTimeout)
 
         // Fetch task configs
         tasks <- voxelyticsDAO.findTasks(mostRecentRun.id)

app/models/voxelytics/VoxelyticsDAO.scala

@@ -108,23 +108,26 @@ class VoxelyticsDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContex
         ) running_chunks ON running_chunks.name = tc.name AND running_chunks.executionId = tc.executionId AND running_chunks.chunkName = tc.chunkName
         WHERE TRUE ${taskName.map(t => q"AND tc.name = $t").getOrElse(q"")}"""
 
-  private def latestCompleteTaskQ(runIds: List[ObjectId], taskName: Option[String], staleTimeout: FiniteDuration) =
+  private def latestCompleteOrSkippedTaskQ(runIds: List[ObjectId],
+                                           taskName: Option[String],
+                                           staleTimeout: FiniteDuration) =
     q"""SELECT
           DISTINCT ON (t.name)
           t.name,
-          t._id
+          t._id,
+          ts.state
         FROM (${tasksWithStateQ(staleTimeout)}) ts
         JOIN webknossos.voxelytics_tasks t ON t._id = ts._id
         WHERE
-          ts.state = ${VoxelyticsRunState.COMPLETE}
+          ts.state IN ${SqlToken.tupleFromValues(VoxelyticsRunState.COMPLETE, VoxelyticsRunState.SKIPPED)}
           AND t._run IN ${SqlToken.tupleFromList(runIds)}
           ${taskName.map(t => q" AND t.name = $t").getOrElse(q"")}
         ORDER BY t.name, ts.beginTime DESC"""
 
-  def findArtifacts(runIds: List[ObjectId], staleTimeout: FiniteDuration): Fox[List[ArtifactEntry]] =
+  def findArtifacts(currentUser: User, runIds: List[ObjectId], staleTimeout: FiniteDuration): Fox[List[ArtifactEntry]] =
     for {
       r <- run(q"""
-        WITH latest_complete_tasks AS (${latestCompleteTaskQ(runIds, None, staleTimeout)})
+        WITH latest_complete_tasks AS (${latestCompleteOrSkippedTaskQ(runIds, None, staleTimeout)})
         SELECT
           a._id,
           a._task,
@@ -134,10 +137,25 @@ class VoxelyticsDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContex
           a.inodeCount,
           a.version,
           a.metadata,
-          t.name AS taskName
+          t.name AS taskName,
+          o.workflow_hash AS other_workflow_hash,
+          o._run AS other_run
         FROM webknossos.voxelytics_artifacts a
         JOIN latest_complete_tasks t ON t._id = a._task
-        """.as[(String, String, String, String, Long, Long, String, String, String)])
+        LEFT JOIN ( -- when the task is skipped, the artifact from the producing task run is joined for linking
+          -- Fan out is not possible because of distinct path selection in combination with unique (_task, path) constraint
+          SELECT
+            DISTINCT ON (a.path)
+            a.path path,
+            r.workflow_hash workflow_hash,
+            r._id _run
+          FROM webknossos.voxelytics_artifacts a
+          JOIN webknossos.voxelytics_tasks t ON a._task = t._id
+          JOIN (${tasksWithStateQ(staleTimeout)}) ts ON ts._id = t._id AND ts.state = ${VoxelyticsRunState.COMPLETE}
+          JOIN (${visibleRunsQ(currentUser, allowUnlisted = true)}) r ON r._id = t._run
+          ORDER BY a.path, ts.beginTime DESC
+        ) o ON o.path = a.path AND t.state = ${VoxelyticsRunState.SKIPPED}
+        """.as[(String, String, String, String, Long, Long, String, String, String, Option[String], Option[String])])
     } yield
       r.toList.map(
         row =>
@@ -150,7 +168,11 @@ class VoxelyticsDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContex
             inodeCount = row._6,
             version = row._7,
             metadata = Json.parse(row._8).as[JsObject],
-            taskName = row._9
+            taskName = row._9,
+            foreignWorkflow = (row._10, row._11) match {
+              case (Some(workflow_hash), Some(run)) => Some((workflow_hash, run))
+              case _                                => None
+            }
         ))
 
   def findTasks(runId: ObjectId): Fox[List[TaskEntry]] =
@@ -725,7 +747,7 @@ class VoxelyticsDAO @Inject()(sqlClient: SqlClient)(implicit ec: ExecutionContex
                            staleTimeout: FiniteDuration): Fox[List[ArtifactChecksumEntry]] =
     for {
       r <- run(q"""
-        WITH latest_complete_tasks AS (${latestCompleteTaskQ(runIds, Some(taskName), staleTimeout)})
+        WITH latest_complete_tasks AS (${latestCompleteOrSkippedTaskQ(runIds, Some(taskName), staleTimeout)})
         SELECT
           t.name,
           a.name,

app/models/voxelytics/VoxelyticsService.scala

@@ -100,7 +100,8 @@ case class ArtifactEntry(artifactId: ObjectId,
                          inodeCount: Long,
                          version: String,
                          metadata: JsObject,
-                         taskName: String)
+                         taskName: String,
+                         foreignWorkflow: Option[(String, String)])
 
 object ArtifactEntry {
   implicit val jsonFormat: OFormat[ArtifactEntry] = Json.format[ArtifactEntry]

conf/evolutions/118-voxelytics-artifacts-index.sql

@@ -0,0 +1,9 @@
+START TRANSACTION;
+
+do $$ begin ASSERT (select schemaVersion from webknossos.releaseInformation) = 117, 'Previous schema version mismatch'; end; $$ LANGUAGE plpgsql;
+
+ALTER TABLE "webknossos"."voxelytics_artifacts" ADD CONSTRAINT "voxelytics_artifacts__task_path_key" UNIQUE ("_task","path");
+
+UPDATE webknossos.releaseInformation SET schemaVersion = 118;
+
+COMMIT TRANSACTION;

conf/evolutions/reversions/118-voxelytics-artifacts-index.sql

@@ -0,0 +1,9 @@
+START TRANSACTION;
+
+do $$ begin ASSERT (select schemaVersion from webknossos.releaseInformation) = 118, 'Previous schema version mismatch'; end; $$ LANGUAGE plpgsql;
+
+ALTER TABLE "webknossos"."voxelytics_artifacts" DROP CONSTRAINT "voxelytics_artifacts__task_path_key";
+
+UPDATE webknossos.releaseInformation SET schemaVersion = 117;
+
+COMMIT TRANSACTION;

frontend/javascripts/admin/voxelytics/artifacts_view.tsx

@@ -1,11 +1,12 @@
 import React from "react";
 import { JSONTree } from "react-json-tree";
 import { Button, Card, message } from "antd";
-import { CopyOutlined } from "@ant-design/icons";
+import { CopyOutlined, ExportOutlined } from "@ant-design/icons";
 import { VoxelyticsArtifactConfig } from "types/api_flow_types";
 import { getVoxelyticsArtifactChecksums } from "admin/admin_rest_api";
 import { formatCountToDataAmountUnit } from "libs/format_utils";
 import { copyToClipboad, isObjectEmpty, useTheme } from "./utils";
+import { Link } from "react-router-dom";
 
 export function renderArtifactPath(artifact: VoxelyticsArtifactConfig) {
   return (
@@ -126,7 +127,19 @@ function ArtifactsView({
   function renderArtifact(artifactName: string, artifact: VoxelyticsArtifactConfig) {
     const title = (
       <>
-        <span>{artifactName}</span>
+        <span>
+          {artifact.foreignWorkflow != null ? (
+            <>
+              <Link
+                to={`/workflows/${artifact.foreignWorkflow[0]}?runId=${artifact.foreignWorkflow[1]}`}
+              >
+                {artifactName} <ExportOutlined />
+              </Link>
+            </>
+          ) : (
+            artifactName
+          )}
+        </span>
         <span style={{ fontSize: "10px", marginLeft: 10 }}>
           \\ version {artifact.version}, {formatCountToDataAmountUnit(artifact.fileSize)},{" "}
           {artifact.inodeCount.toLocaleString()} inodes

frontend/javascripts/admin/voxelytics/task_list_view.tsx

@@ -24,6 +24,7 @@ import {
   ExclamationCircleOutlined,
   LeftOutlined,
   FieldTimeOutlined,
+  ExportOutlined,
 } from "@ant-design/icons";
 import MiniSearch from "minisearch";
 
@@ -512,6 +513,12 @@ export default function TaskListView({
       return undefined;
     }
 
+    const taskArtifacts = report.artifacts[task.taskName] || {};
+    let foreignWorkflow: null | [string, string] = null;
+    if (taskInfo.state === VoxelyticsRunState.SKIPPED) {
+      foreignWorkflow = Object.values(taskArtifacts)[0]?.foreignWorkflow ?? null;
+    }
+
     return {
       key: task.taskName,
       id: `task-panel-${task.taskName}`,
@@ -527,7 +534,17 @@ export default function TaskListView({
               backgroundColor: colorHasher.hex(task.task),
             }}
           />
-          {task.taskName}
+          {foreignWorkflow != null ? (
+            <>
+              <Link to={`/workflows/${foreignWorkflow[0]}?runId=${foreignWorkflow[1]}`}>
+                {task.taskName}
+                &nbsp;
+                <ExportOutlined />
+              </Link>
+            </>
+          ) : (
+            task.taskName
+          )}
           <wbr />
           {task.config.name != null && <span className="task-panel-name">{task.config.name}</span>}
           <wbr />
@@ -542,7 +559,7 @@ export default function TaskListView({
           workflowHash={report.workflow.hash}
           runId={singleRunId}
           task={task}
-          artifacts={report.artifacts[task.taskName] || []}
+          artifacts={taskArtifacts}
           dag={report.dag}
           taskInfo={taskInfo}
           onSelectTask={handleSelectTask}

frontend/javascripts/types/api_flow_types.ts

@@ -902,6 +902,7 @@ export type VoxelyticsArtifactConfig = {
     iframes: Record<string, string>;
     links: Record<string, string>;
   };
+  foreignWorkflow: [string, string] | null;
 };
 
 export type VoxelyticsRunInfo = {

frontend/stylesheets/main.less

@@ -427,10 +427,6 @@ button.narrow {
 .voxelytics-view {
   min-width: 1300px;
 
-  .tasks-header {
-    border-bottom: 1px solid var(--ant-color-border);
-  }
-
   .task-panel {
     height: calc(100vh - 100px);
     position: relative;

tools/postgres/schema.sql

@@ -20,7 +20,7 @@ CREATE TABLE webknossos.releaseInformation (
   schemaVersion BIGINT NOT NULL
 );
 
-INSERT INTO webknossos.releaseInformation(schemaVersion) values(117);
+INSERT INTO webknossos.releaseInformation(schemaVersion) values(118);
 COMMIT TRANSACTION;
 
 
@@ -594,6 +594,7 @@ CREATE TABLE webknossos.voxelytics_artifacts(
     metadata JSONB,
     PRIMARY KEY (_id),
     UNIQUE (_task, name),
+    UNIQUE (_task, path),
     CONSTRAINT metadataIsJsonObject CHECK(jsonb_typeof(metadata) = 'object')
 );