From a662f87884d35686b891fefffcc299cc9bca1ec1 Mon Sep 17 00:00:00 2001 From: Norman Rzepka Date: Mon, 6 Feb 2023 15:05:59 +0100 Subject: [PATCH 01/12] adds OME-TIFF export support --- app/controllers/JobsController.scala | 25 +- conf/webknossos.latest.routes | 2 +- frontend/javascripts/admin/admin_rest_api.ts | 55 ++- frontend/javascripts/admin/job/job_hooks.ts | 93 +++++ frontend/javascripts/libs/utils.ts | 4 + .../view/action-bar/download_modal_view.tsx | 13 +- .../left-border-tabs/layer_settings_tab.tsx | 7 +- .../export_bounding_box_modal.tsx | 359 +++++++++++------- frontend/javascripts/types/api_flow_types.ts | 2 +- 9 files changed, 367 insertions(+), 193 deletions(-) diff --git a/app/controllers/JobsController.scala b/app/controllers/JobsController.scala index cdb5b0c468f..5c04964c132 100644 --- a/app/controllers/JobsController.scala +++ b/app/controllers/JobsController.scala @@ -1,13 +1,10 @@ package controllers -import java.util.Date - import com.mohiva.play.silhouette.api.Silhouette import com.scalableminds.util.accesscontext.GlobalAccessContext import com.scalableminds.util.tools.Fox -import javax.inject.Inject import models.binary.DataSetDAO -import models.job.{JobDAO, JobService, JobState, WorkerDAO, WorkerService} +import models.job._ import models.organization.OrganizationDAO import oxalis.security.{WkEnv, WkSilhouetteEnvironment} import oxalis.telemetry.SlackNotificationService @@ -16,6 +13,8 @@ import play.api.libs.json._ import play.api.mvc.{Action, AnyContent} import utils.{ObjectId, WkConf} +import java.util.Date +import javax.inject.Inject import scala.concurrent.ExecutionContext class JobsController @Inject()(jobDAO: JobDAO, @@ -226,12 +225,10 @@ class JobsController @Inject()(jobDAO: JobDAO, dataSetName: String, bbox: String, layerName: Option[String], + mag: Option[String], annotationLayerName: Option[String], annotationId: Option[String], - annotationType: Option[String], - hideUnmappedIds: Option[Boolean], - mappingName: Option[String], - mappingType: Option[String]): Action[AnyContent] = + asOmeTiff: Boolean): Action[AnyContent] = sil.SecuredAction.async { implicit request => log(Some(slackNotificationService.noticeFailedJobRequest)) { for { @@ -242,20 +239,20 @@ class JobsController @Inject()(jobDAO: JobDAO, userAuthToken <- wkSilhouetteEnvironment.combinedAuthenticatorService.findOrCreateToken( request.identity.loginInfo) command = "export_tiff" - exportFileName = s"${formatDateForFilename(new Date())}__${dataSetName}__${annotationLayerName.map(_ => "volume").getOrElse(layerName.getOrElse(""))}.zip" + exportFileName = if (asOmeTiff) + s"${formatDateForFilename(new Date())}__${dataSetName}__${annotationLayerName.map(_ => "volume").getOrElse(layerName.getOrElse(""))}.ome.tif" + else + s"${formatDateForFilename(new Date())}__${dataSetName}__${annotationLayerName.map(_ => "volume").getOrElse(layerName.getOrElse(""))}.zip" commandArgs = Json.obj( "organization_name" -> organizationName, "dataset_name" -> dataSetName, "bbox" -> bbox, "export_file_name" -> exportFileName, "layer_name" -> layerName, + "mag" -> mag, "user_auth_token" -> userAuthToken.id, "annotation_layer_name" -> annotationLayerName, - "annotation_id" -> annotationId, - "annotation_type" -> annotationType, - "mapping_name" -> mappingName, - "mapping_type" -> mappingType, - "hide_unmapped_ids" -> hideUnmappedIds + "annotation_id" -> annotationId ) job <- jobService.submitJob(command, commandArgs, request.identity, dataSet._dataStore) ?~> "job.couldNotRunTiffExport" js <- jobService.publicWrites(job) diff --git a/conf/webknossos.latest.routes b/conf/webknossos.latest.routes index 9588dee3fbf..ce5fcecbcd6 100644 --- a/conf/webknossos.latest.routes +++ b/conf/webknossos.latest.routes @@ -251,7 +251,7 @@ GET /jobs GET /jobs/status controllers.JobsController.status POST /jobs/run/convertToWkw/:organizationName/:dataSetName controllers.JobsController.runConvertToWkwJob(organizationName: String, dataSetName: String, scale: String) POST /jobs/run/computeMeshFile/:organizationName/:dataSetName controllers.JobsController.runComputeMeshFileJob(organizationName: String, dataSetName: String, layerName: String, mag: String, agglomerateView: Option[String]) -POST /jobs/run/exportTiff/:organizationName/:dataSetName controllers.JobsController.runExportTiffJob(organizationName: String, dataSetName: String, bbox: String, layerName: Option[String], annotationLayerName: Option[String], annotationId: Option[String], annotationType: Option[String], hideUnmappedIds: Option[Boolean], mappingName: Option[String], mappingType: Option[String]) +POST /jobs/run/exportTiff/:organizationName/:dataSetName controllers.JobsController.runExportTiffJob(organizationName: String, dataSetName: String, bbox: String, layerName: Option[String], mag: Option[String], annotationLayerName: Option[String], annotationId: Option[String], asOmeTiff: Boolean) POST /jobs/run/inferNuclei/:organizationName/:dataSetName controllers.JobsController.runInferNucleiJob(organizationName: String, dataSetName: String, layerName: String, newDatasetName: String) POST /jobs/run/inferNeurons/:organizationName/:dataSetName controllers.JobsController.runInferNeuronsJob(organizationName: String, dataSetName: String, layerName: String, bbox: String, newDatasetName: String) POST /jobs/run/globalizeFloodfills/:organizationName/:dataSetName controllers.JobsController.runGlobalizeFloodfills(organizationName: String, dataSetName: String, fallbackLayerName: String, annotationId: String, annotationType: String, newDatasetName: String, volumeLayerName: Option[String]) diff --git a/frontend/javascripts/admin/admin_rest_api.ts b/frontend/javascripts/admin/admin_rest_api.ts index 4f53d3866ea..0aefd98632a 100644 --- a/frontend/javascripts/admin/admin_rest_api.ts +++ b/frontend/javascripts/admin/admin_rest_api.ts @@ -1083,6 +1083,29 @@ export async function getJobs(): Promise { ); } +export async function getJob(jobId: string): Promise { + const job = await Request.receiveJSON(`/api/jobs/${jobId}`); + return { + id: job.id, + type: job.command, + datasetName: job.commandArgs.dataset_name, + organizationName: job.commandArgs.organization_name, + layerName: job.commandArgs.layer_name || job.commandArgs.volume_layer_name, + annotationLayerName: job.commandArgs.annotation_layer_name, + boundingBox: job.commandArgs.bbox, + exportFileName: job.commandArgs.export_file_name, + tracingId: job.commandArgs.volume_tracing_id, + annotationId: job.commandArgs.annotation_id, + annotationType: job.commandArgs.annotation_type, + mergeSegments: job.commandArgs.merge_segments, + state: adaptJobState(job.command, job.state, job.manualState), + manualState: job.manualState, + result: job.returnValue, + resultLink: job.resultLink, + createdAt: job.created, + }; +} + function adaptJobState( command: string, celeryState: APIJobCeleryState, @@ -1138,26 +1161,26 @@ export async function startExportTiffJob( organizationName: string, bbox: Vector6, layerName: string | null | undefined, + mag: string | null | undefined, annotationId: string | null | undefined, - annotationType: APIAnnotationType | null | undefined, annotationLayerName: string | null | undefined, - mappingName: string | null | undefined, - mappingType: string | null | undefined, - hideUnmappedIds: boolean | null | undefined, + asOmeTiff: boolean, ): Promise { - const layerNameSuffix = layerName != null ? `&layerName=${layerName}` : ""; - const annotationIdSuffix = annotationId != null ? `&annotationId=${annotationId}` : ""; - const annotationTypeSuffix = annotationType != null ? `&annotationType=${annotationType}` : ""; - const annotationLayerNameSuffix = - annotationLayerName != null ? `&annotationLayerName=${annotationLayerName}` : ""; - const mappingNameSuffix = mappingName != null ? `&mappingName=${mappingName}` : ""; - const mappingTypeSuffix = mappingType != null ? `&mappingType=${mappingType}` : ""; - const hideUnmappedIdsSuffix = - hideUnmappedIds != null ? `&hideUnmappedIds=${hideUnmappedIds.toString()}` : ""; + const params = new URLSearchParams({ bbox: bbox.join(","), asOmeTiff: asOmeTiff.toString() }); + if (layerName != null) { + params.append("layerName", layerName); + } + if (mag != null) { + params.append("mag", mag); + } + if (annotationId != null) { + params.append("annotationId", annotationId); + } + if (annotationLayerName != null) { + params.append("annotationLayerName", annotationLayerName); + } return Request.receiveJSON( - `/api/jobs/run/exportTiff/${organizationName}/${datasetName}?bbox=${bbox.join( - ",", - )}${layerNameSuffix}${annotationIdSuffix}${annotationTypeSuffix}${annotationLayerNameSuffix}${mappingNameSuffix}${mappingTypeSuffix}${hideUnmappedIdsSuffix}`, + `/api/jobs/run/exportTiff/${organizationName}/${datasetName}?${params}`, { method: "POST", }, diff --git a/frontend/javascripts/admin/job/job_hooks.ts b/frontend/javascripts/admin/job/job_hooks.ts index 296e9aad238..c450001fc70 100644 --- a/frontend/javascripts/admin/job/job_hooks.ts +++ b/frontend/javascripts/admin/job/job_hooks.ts @@ -103,3 +103,96 @@ export function useStartAndPollJob({ mostRecentSuccessfulJob, }; } + +export function useStartAndPollJob2({ + startJobFn, + onSuccess, + onError, + pollingInterval = 2000, +}: { + startJobFn: (() => Promise) | null; + findJobPred: (job: APIJob) => boolean; + onSuccess: (job: APIJob) => void; + onError: (job: APIJob) => void; + pollingInterval?: number; +}): { + startJob: (() => Promise) | null; + activeJob: APIJob | null; +} { + const activeUser = useSelector((state: OxalisState) => state.activeUser); + const [jobs, setJobs] = useState(null); + const [activeJob, setActiveJob] = useState(null); + const areJobsEnabled = features().jobsEnabled; + const potentialJobs = (jobs || []).filter(findJobPred); + const mostRecentSuccessfulJob = potentialJobs.find((job) => job.state === "SUCCESS"); + + const wrappedStartJobFn = + startJobFn != null && activeUser != null && areJobsEnabled + ? async () => { + const job = await startJobFn(); + setActiveJob(job); + Toast.info(jobStartedMessage); + } + : null; + + useInterval(async () => { + if (activeUser == null || !areJobsEnabled) { + return; + } + setJobs(await getJobs()); + }, interval); + + useEffect(() => { + const newActiveJob = activeJob != null ? jobs?.find((job) => job.id === activeJob.id) : null; + + if (newActiveJob != null) { + // We are aware of a running job. Check whether the job is finished now. + switch (newActiveJob.state) { + case "SUCCESS": { + Toast.success(successMessage); + setActiveJob(null); + break; + } + + case "STARTED": + case "UNKNOWN": + case "PENDING": { + break; + } + + case "FAILURE": { + Toast.info(failureMessage); + setActiveJob(null); + break; + } + + case "MANUAL": { + Toast.info( + "The job didn't finish properly. The job will be handled by an admin shortly. Please check back here soon.", + ); + setActiveJob(null); + + break; + } + + default: { + break; + } + } + } else { + // Check whether there is an active job (e.g., the user + // started the job earlier and reopened WEBKNOSSOS in the meantime). + const pendingJobs = potentialJobs.filter( + (job) => job.state === "STARTED" || job.state === "PENDING", + ); + const newestActiveJob = pendingJobs.length > 0 ? pendingJobs[0] : null; + setActiveJob(newestActiveJob); + } + }, [jobs]); + + return { + startJob: wrappedStartJobFn, + activeJob, + mostRecentSuccessfulJob, + }; +} diff --git a/frontend/javascripts/libs/utils.ts b/frontend/javascripts/libs/utils.ts index 6d0b21f6875..b758cdd6747 100644 --- a/frontend/javascripts/libs/utils.ts +++ b/frontend/javascripts/libs/utils.ts @@ -299,6 +299,10 @@ export function computeArrayFromBoundingBox(bb: BoundingBoxType): Vector6 { ]; } +export function computeShapeFromBoundingBox(bb: BoundingBoxType): Vector3 { + return [bb.max[0] - bb.min[0], bb.max[1] - bb.min[1], bb.max[2] - bb.min[2]]; +} + export function aggregateBoundingBox(boundingBoxes: Array): BoundingBoxType { if (boundingBoxes.length === 0) { return { diff --git a/frontend/javascripts/oxalis/view/action-bar/download_modal_view.tsx b/frontend/javascripts/oxalis/view/action-bar/download_modal_view.tsx index 4bebba538c8..dd20e9f1ae3 100644 --- a/frontend/javascripts/oxalis/view/action-bar/download_modal_view.tsx +++ b/frontend/javascripts/oxalis/view/action-bar/download_modal_view.tsx @@ -139,12 +139,6 @@ function _DownloadModalView(props: Props): JSX.Element { const userBoundingBoxes = useSelector((state: OxalisState) => getUserBoundingBoxesFromState(state), ); - const isMergerModeEnabled = useSelector( - (state: OxalisState) => state.temporaryConfiguration.isMergerModeEnabled, - ); - const activeMappingInfos = useSelector( - (state: OxalisState) => state.temporaryConfiguration.activeMappingByLayer, - ); const layers = getDataLayers(dataset); @@ -169,12 +163,7 @@ function _DownloadModalView(props: Props): JSX.Element { } else { const selectedLayer = getLayerByName(dataset, selectedLayerName); if (selectedLayer != null && selectedBoundingBox != null) { - const layerInfos = getLayerInfos( - selectedLayer, - tracing, - activeMappingInfos, - isMergerModeEnabled, - ); + const layerInfos = getLayerInfos(selectedLayer, tracing); await handleStartExport( dataset, layerInfos, diff --git a/frontend/javascripts/oxalis/view/left-border-tabs/layer_settings_tab.tsx b/frontend/javascripts/oxalis/view/left-border-tabs/layer_settings_tab.tsx index 5e6939a077d..201f5a6e973 100644 --- a/frontend/javascripts/oxalis/view/left-border-tabs/layer_settings_tab.tsx +++ b/frontend/javascripts/oxalis/view/left-border-tabs/layer_settings_tab.tsx @@ -1058,12 +1058,7 @@ class DatasetSettings extends React.PureComponent { <> - diff --git a/frontend/javascripts/oxalis/view/right-border-tabs/export_bounding_box_modal.tsx b/frontend/javascripts/oxalis/view/right-border-tabs/export_bounding_box_modal.tsx index 9a525809241..765cb562ee9 100644 --- a/frontend/javascripts/oxalis/view/right-border-tabs/export_bounding_box_modal.tsx +++ b/frontend/javascripts/oxalis/view/right-border-tabs/export_bounding_box_modal.tsx @@ -1,69 +1,68 @@ -import { Button, Modal, Alert } from "antd"; +import { Button, Modal, Alert, Form, Radio, Slider, Divider, Row, Col } from "antd"; import { useSelector } from "react-redux"; import React, { useState } from "react"; -import type { APIDataset, APIDataLayer, APIAnnotationType } from "types/api_flow_types"; -import type { BoundingBoxType } from "oxalis/constants"; -import { MappingStatusEnum } from "oxalis/constants"; +import type { APIDataset, APIDataLayer, APIJob } from "types/api_flow_types"; +import type { BoundingBoxType, Vector3 } from "oxalis/constants"; import type { OxalisState, Tracing, HybridTracing } from "oxalis/store"; -import { getResolutionInfo, getMappingInfo } from "oxalis/model/accessors/dataset_accessor"; +import { + getResolutionInfo, + getDatasetResolutionInfo, + ResolutionInfo, + getByteCountFromLayer, +} from "oxalis/model/accessors/dataset_accessor"; import { getVolumeTracingById } from "oxalis/model/accessors/volumetracing_accessor"; -import { startExportTiffJob } from "admin/admin_rest_api"; +import { doWithToken, getJob, getJobs, startExportTiffJob } from "admin/admin_rest_api"; import { Model } from "oxalis/singletons"; import * as Utils from "libs/utils"; import features from "features"; import _ from "lodash"; -import { getReadableNameOfVolumeLayer } from "./starting_job_modals"; +import { getReadableNameOfVolumeLayer, LayerSelection } from "./starting_job_modals"; +import { formatBytes } from "libs/format_utils"; +import { usePolling } from "libs/react_hooks"; +import Toast from "libs/toast"; +import { SyncOutlined } from "@ant-design/icons"; + type Props = { handleClose: () => void; - tracing: Tracing | null | undefined; + tracing: Tracing; dataset: APIDataset; boundingBox: BoundingBoxType; }; type LayerInfos = { displayName: string; - layerName: string | null | undefined; - tracingId: string | null | undefined; - annotationId: string | null | undefined; - annotationType: APIAnnotationType | null | undefined; - hasMag1: boolean; - mappingName: string | null | undefined; - mappingType: string | null | undefined; - hideUnmappedIds: boolean | null | undefined; - isColorLayer: boolean | null | undefined; + layerName: string | null; + mags: ResolutionInfo; + byteCount: number; + tracingId: string | null; + annotationId: string | null; + isColorLayer: boolean; }; -const exportKey = (layerInfos: LayerInfos) => - (layerInfos.layerName || "") + (layerInfos.tracingId || ""); +enum ExportFormat { + OME_TIFF = "OME_TIFF", + TIFF_STACK = "TIFF_STACK", +} + +const EXPECTED_DOWNSAMPLING_FILE_SIZE_FACTOR = 1.33; + +const exportKey = (layerInfos: LayerInfos, resolutionIndex: number) => + `${layerInfos.layerName || ""}__${layerInfos.tracingId || ""}__${resolutionIndex}`; export function getLayerInfos( layer: APIDataLayer, tracing: HybridTracing | null | undefined, - activeMappingInfos: any, - isMergerModeEnabled: boolean, ): LayerInfos { const annotationId = tracing != null ? tracing.annotationId : null; - const annotationType = tracing != null ? tracing.annotationType : null; - - const hasMag1 = (dataLayer: APIDataLayer) => getResolutionInfo(dataLayer.resolutions).hasIndex(0); - const { mappingStatus, hideUnmappedIds, mappingName, mappingType } = getMappingInfo( - activeMappingInfos, - layer.name, - ); - const existsActivePersistentMapping = - mappingStatus === MappingStatusEnum.ENABLED && !isMergerModeEnabled; const isColorLayer = layer.category === "color"; if (layer.category === "color" || !layer.tracingId) { return { displayName: layer.name, layerName: layer.name, + mags: getResolutionInfo(layer.resolutions), + byteCount: getByteCountFromLayer(layer), tracingId: null, annotationId: null, - annotationType: null, - hasMag1: hasMag1(layer), - hideUnmappedIds: !isColorLayer && existsActivePersistentMapping ? hideUnmappedIds : null, - mappingName: !isColorLayer && existsActivePersistentMapping ? mappingName : null, - mappingType: !isColorLayer && existsActivePersistentMapping ? mappingType : null, isColorLayer, }; } @@ -81,13 +80,10 @@ export function getLayerInfos( return { displayName: readableVolumeLayerName, layerName: layer.fallbackLayerInfo.name, + mags: getResolutionInfo(layer.resolutions), + byteCount: getByteCountFromLayer(layer), tracingId: volumeTracing.tracingId, annotationId, - annotationType, - hasMag1: hasMag1(layer), - hideUnmappedIds: existsActivePersistentMapping ? hideUnmappedIds : null, - mappingName: existsActivePersistentMapping ? mappingName : null, - mappingType: existsActivePersistentMapping ? mappingType : null, isColorLayer: false, }; } @@ -95,68 +91,37 @@ export function getLayerInfos( return { displayName: readableVolumeLayerName, layerName: null, + mags: getResolutionInfo(layer.resolutions), + byteCount: getByteCountFromLayer(layer), tracingId: volumeTracing.tracingId, annotationId, - annotationType, - hasMag1: hasMag1(layer), - hideUnmappedIds: null, - mappingName: null, - mappingType: null, isColorLayer: false, }; } -export async function handleStartExport( - dataset: APIDataset, - layerInfos: LayerInfos, - boundingBox: BoundingBoxType, - startedExports: string[], - setStartedExports?: React.Dispatch>, -) { - if (setStartedExports) { - setStartedExports(startedExports.concat(exportKey(layerInfos))); - } - - if (layerInfos.tracingId) { - await Model.ensureSavedState(); - } - - await startExportTiffJob( - dataset.name, - dataset.owningOrganization, - Utils.computeArrayFromBoundingBox(boundingBox), - layerInfos.layerName, - layerInfos.annotationId, - layerInfos.annotationType, - layerInfos.displayName, - layerInfos.mappingName, - layerInfos.mappingType, - layerInfos.hideUnmappedIds, - ); -} - -export function isBoundingBoxExportable(boundingBox: BoundingBoxType) { - const dimensions = boundingBox.max.map((maxItem, index) => maxItem - boundingBox.min[index]); - const volume = dimensions[0] * dimensions[1] * dimensions[2]; +export function isBoundingBoxExportable(boundingBox: BoundingBoxType, mag: Vector3) { + const shape = Utils.computeShapeFromBoundingBox(boundingBox); + const volume = + Math.ceil(shape[0] / mag[0]) * Math.ceil(shape[1] / mag[1]) * Math.ceil(shape[2] / mag[2]); const volumeExceeded = volume > features().exportTiffMaxVolumeMVx * 1024 * 1024; - const edgeLengthExceeded = dimensions.some( - (length) => length > features().exportTiffMaxEdgeLengthVx, + const edgeLengthExceeded = shape.some( + (length, index) => length / mag[index] > features().exportTiffMaxEdgeLengthVx, ); - const dimensionString = dimensions.join(", "); - - const volumeExceededMessage = volumeExceeded - ? `The volume of the selected bounding box (${volume} vx) is too large. Tiff export is only supported for up to ${ - features().exportTiffMaxVolumeMVx - } Megavoxels.` - : null; - const edgeLengthExceededMessage = edgeLengthExceeded - ? `An edge length of the selected bounding box (${dimensionString}) is too large. Tiff export is only supported for boxes with no edge length over ${ - features().exportTiffMaxEdgeLengthVx - } vx.` - : null; - - const alertMessage = _.compact([volumeExceededMessage, edgeLengthExceededMessage]).join("\n"); + const alertMessage = _.compact([ + volumeExceeded + ? `The volume of the selected bounding box (${volume} vx) is too large. Tiff export is only supported for up to ${ + features().exportTiffMaxVolumeMVx + } Megavoxels.` + : null, + edgeLengthExceeded + ? `An edge length of the selected bounding box (${shape.join( + ", ", + )}) is too large. Tiff export is only supported for boxes with no edge length over ${ + features().exportTiffMaxEdgeLengthVx + } vx.` + : null, + ]).join("\n"); const alerts = alertMessage.length > 0 ? : null; return { @@ -166,49 +131,67 @@ export function isBoundingBoxExportable(boundingBox: BoundingBoxType) { } function ExportBoundingBoxModal({ handleClose, dataset, boundingBox, tracing }: Props) { - const [startedExports, setStartedExports] = useState([]); + const [runningJobs, setRunningJobs] = useState>([]); const isMergerModeEnabled = useSelector( (state: OxalisState) => state.temporaryConfiguration.isMergerModeEnabled, ); - const activeMappingInfos = useSelector( - (state: OxalisState) => state.temporaryConfiguration.activeMappingByLayer, + const [exportFormat, setExportFormat] = useState(ExportFormat.OME_TIFF); + const [selectedLayerName, setSelectedLayerName] = useState( + dataset.dataSource.dataLayers[0].name, ); - const allLayerInfos = dataset.dataSource.dataLayers.map((layer: APIDataLayer) => - getLayerInfos(layer, tracing, activeMappingInfos, isMergerModeEnabled), + const selectedLayer = dataset.dataSource.dataLayers.find( + (l) => l.name === selectedLayerName, + ) as APIDataLayer; + const selectedLayerInfos = getLayerInfos(selectedLayer, tracing); + + const highestResolutionIndex = getResolutionInfo( + selectedLayer.resolutions, + ).getHighestResolutionIndex(); + const lowestResolutionIndex = getResolutionInfo( + selectedLayer.resolutions, + ).getClosestExistingIndex(0); + const datasetResolutionInfo = getDatasetResolutionInfo(dataset); + + const [rawResolutionIndex, setResolutionIndex] = useState(lowestResolutionIndex); + const resolutionIndex = Utils.clamp( + lowestResolutionIndex, + rawResolutionIndex, + highestResolutionIndex, ); - const exportButtonsList = allLayerInfos.map((layerInfos) => { - const parenthesesInfos = [ - startedExports.includes(exportKey(layerInfos)) ? "started" : null, - layerInfos.mappingName != null ? `using mapping "${layerInfos.mappingName}"` : null, - !layerInfos.hasMag1 ? "resolution 1 missing" : null, - ].filter((el) => el); - const parenthesesInfosString = - parenthesesInfos.length > 0 ? ` (${parenthesesInfos.join(", ")})` : ""; - return layerInfos ? ( -

- -

- ) : null; - }); - const { isExportable, alerts } = isBoundingBoxExportable(boundingBox); + const { isExportable, alerts } = isBoundingBoxExportable( + boundingBox, + datasetResolutionInfo.getResolutionByIndexOrThrow(resolutionIndex), + ); + + async function checkForJobs() { + for (const [, jobId] of runningJobs) { + const job = await getJob(jobId); + if (job.state === "SUCCESS" && job.resultLink != null) { + const token = await doWithToken(async (t) => t); + window.open(`${job.resultLink}?token=${token}`, "_blank"); + setRunningJobs((previous) => previous.filter(([, j]) => j !== jobId)); + } else if (job.state === "FAILURE") { + Toast.error("Error when exporting data. Please contact us for support."); + setRunningJobs((previous) => previous.filter(([, j]) => j !== jobId)); + } else if (job.state === "MANUAL") { + Toast.error( + "The data could not be exported automatically. The job will be handled by an admin shortly.", + ); + setRunningJobs((previous) => previous.filter(([, j]) => j !== jobId)); + } + } + } + + usePolling( + checkForJobs, + runningJobs.length > 0 ? 1000 : null, + runningJobs.map(([key]) => key), + ); const downloadHint = - startedExports.length > 0 ? ( + runningJobs.length > 0 ? (

Go to{" "} @@ -217,35 +200,125 @@ function ExportBoundingBoxModal({ handleClose, dataset, boundingBox, tracing }: to see running exports and to download the results.

) : null; - const bboxText = Utils.computeArrayFromBoundingBox(boundingBox).join(", "); - let activeMappingMessage = null; - if (isMergerModeEnabled) { - activeMappingMessage = - "Exporting a volume layer does not export merger mode currently. Please disable merger mode before exporting data of the volume layer."; - } + const estimatedFileSize = (() => { + const mag = getResolutionInfo(selectedLayer.resolutions).getResolutionByIndexOrThrow( + resolutionIndex, + ); + const shape = Utils.computeShapeFromBoundingBox(boundingBox); + const volume = + Math.ceil(shape[0] / mag[0]) * Math.ceil(shape[1] / mag[1]) * Math.ceil(shape[2] / mag[2]); + return ( + volume * + getByteCountFromLayer(selectedLayer) * + (exportFormat === ExportFormat.OME_TIFF ? EXPECTED_DOWNSAMPLING_FILE_SIZE_FACTOR : 1) + ); + })(); return ( key === exportKey(selectedLayerInfos, resolutionIndex)) || // The export is already running or... + isMergerModeEnabled // Merger mode is enabled + } + onClick={async () => { + if (selectedLayerInfos.tracingId != null) { + await Model.ensureSavedState(); + } + const job = await startExportTiffJob( + dataset.name, + dataset.owningOrganization, + Utils.computeArrayFromBoundingBox(boundingBox), + selectedLayerInfos.layerName, + datasetResolutionInfo.getResolutionByIndexOrThrow(resolutionIndex).join("-"), + selectedLayerInfos.annotationId, + selectedLayerInfos.displayName, + exportFormat === ExportFormat.OME_TIFF, + ); + setRunningJobs((previous) => [ + ...previous, + [exportKey(selectedLayerInfos, resolutionIndex), job.id], + ]); + }} + > + {runningJobs.some(([key]) => key === exportKey(selectedLayerInfos, resolutionIndex)) && ( + <> + {" "} + + )} + Export + + } >

- Data from the selected bounding box at {bboxText} will be exported as a tiff stack zip - archive. {activeMappingMessage} + Data from the selected bounding box at{" "} + {Utils.computeArrayFromBoundingBox(boundingBox).join(", ")} will be exported.

{alerts} - {!isExportable ? null : ( -
- {" "} -

Please select a layer to export:

{exportButtonsList} -
- )} + + Export format + + setExportFormat(ev.target.value)}> + OME-TIFF + TIFF stack (as .zip) + + + + Layer + + + + + Mag + + + + + datasetResolutionInfo.getResolutionByIndexOrThrow(resolutionIndex).join("-"), + }} + min={lowestResolutionIndex} + max={highestResolutionIndex} + step={1} + value={resolutionIndex} + onChange={(value) => setResolutionIndex(value)} + /> + + + {datasetResolutionInfo.getResolutionByIndexOrThrow(resolutionIndex).join("-")} + + +

Estimated file size: {formatBytes(estimatedFileSize)}

{downloadHint} diff --git a/frontend/javascripts/types/api_flow_types.ts b/frontend/javascripts/types/api_flow_types.ts index c70cf8247b9..90bd6063780 100644 --- a/frontend/javascripts/types/api_flow_types.ts +++ b/frontend/javascripts/types/api_flow_types.ts @@ -600,7 +600,7 @@ export type APIJob = { readonly boundingBox: string | null | undefined; readonly mergeSegments: boolean | null | undefined; readonly type: APIJobType; - readonly state: string; + readonly state: APIJobState; readonly manualState: string; readonly result: string | null | undefined; readonly resultLink: string | null | undefined; From df586a1700b49ec8ba9bf8a9b22c68f9de1e1c6e Mon Sep 17 00:00:00 2001 From: Norman Rzepka Date: Tue, 7 Feb 2023 15:13:13 +0100 Subject: [PATCH 02/12] stuff --- frontend/javascripts/admin/job/job_hooks.ts | 93 ---------- .../view/action-bar/download_modal_view.tsx | 150 +++++++++++++--- .../export_bounding_box_modal.tsx | 169 +++++++++++------- .../right-border-tabs/starting_job_modals.tsx | 4 +- 4 files changed, 229 insertions(+), 187 deletions(-) diff --git a/frontend/javascripts/admin/job/job_hooks.ts b/frontend/javascripts/admin/job/job_hooks.ts index c450001fc70..296e9aad238 100644 --- a/frontend/javascripts/admin/job/job_hooks.ts +++ b/frontend/javascripts/admin/job/job_hooks.ts @@ -103,96 +103,3 @@ export function useStartAndPollJob({ mostRecentSuccessfulJob, }; } - -export function useStartAndPollJob2({ - startJobFn, - onSuccess, - onError, - pollingInterval = 2000, -}: { - startJobFn: (() => Promise) | null; - findJobPred: (job: APIJob) => boolean; - onSuccess: (job: APIJob) => void; - onError: (job: APIJob) => void; - pollingInterval?: number; -}): { - startJob: (() => Promise) | null; - activeJob: APIJob | null; -} { - const activeUser = useSelector((state: OxalisState) => state.activeUser); - const [jobs, setJobs] = useState(null); - const [activeJob, setActiveJob] = useState(null); - const areJobsEnabled = features().jobsEnabled; - const potentialJobs = (jobs || []).filter(findJobPred); - const mostRecentSuccessfulJob = potentialJobs.find((job) => job.state === "SUCCESS"); - - const wrappedStartJobFn = - startJobFn != null && activeUser != null && areJobsEnabled - ? async () => { - const job = await startJobFn(); - setActiveJob(job); - Toast.info(jobStartedMessage); - } - : null; - - useInterval(async () => { - if (activeUser == null || !areJobsEnabled) { - return; - } - setJobs(await getJobs()); - }, interval); - - useEffect(() => { - const newActiveJob = activeJob != null ? jobs?.find((job) => job.id === activeJob.id) : null; - - if (newActiveJob != null) { - // We are aware of a running job. Check whether the job is finished now. - switch (newActiveJob.state) { - case "SUCCESS": { - Toast.success(successMessage); - setActiveJob(null); - break; - } - - case "STARTED": - case "UNKNOWN": - case "PENDING": { - break; - } - - case "FAILURE": { - Toast.info(failureMessage); - setActiveJob(null); - break; - } - - case "MANUAL": { - Toast.info( - "The job didn't finish properly. The job will be handled by an admin shortly. Please check back here soon.", - ); - setActiveJob(null); - - break; - } - - default: { - break; - } - } - } else { - // Check whether there is an active job (e.g., the user - // started the job earlier and reopened WEBKNOSSOS in the meantime). - const pendingJobs = potentialJobs.filter( - (job) => job.state === "STARTED" || job.state === "PENDING", - ); - const newestActiveJob = pendingJobs.length > 0 ? pendingJobs[0] : null; - setActiveJob(newestActiveJob); - } - }, [jobs]); - - return { - startJob: wrappedStartJobFn, - activeJob, - mostRecentSuccessfulJob, - }; -} diff --git a/frontend/javascripts/oxalis/view/action-bar/download_modal_view.tsx b/frontend/javascripts/oxalis/view/action-bar/download_modal_view.tsx index dd20e9f1ae3..5aca22c7317 100644 --- a/frontend/javascripts/oxalis/view/action-bar/download_modal_view.tsx +++ b/frontend/javascripts/oxalis/view/action-bar/download_modal_view.tsx @@ -1,8 +1,8 @@ -import { Divider, Modal, Checkbox, Row, Col, Tabs, Typography, Button } from "antd"; +import { Divider, Modal, Checkbox, Row, Col, Tabs, Typography, Button, Radio, Slider } from "antd"; import { CopyOutlined } from "@ant-design/icons"; import React, { useState } from "react"; import { makeComponentLazy, useFetch } from "libs/react_helpers"; -import type { APIAnnotationType } from "types/api_flow_types"; +import type { APIAnnotationType, APIDataLayer } from "types/api_flow_types"; import Toast from "libs/toast"; import messages from "messages"; import { Model } from "oxalis/singletons"; @@ -15,14 +15,20 @@ import { } from "oxalis/view/right-border-tabs/starting_job_modals"; import { getUserBoundingBoxesFromState } from "oxalis/model/accessors/tracing_accessor"; import { hasVolumeTracings } from "oxalis/model/accessors/volumetracing_accessor"; -import { getDataLayers, getLayerByName } from "oxalis/model/accessors/dataset_accessor"; +import { + getDataLayers, + getDatasetResolutionInfo, + getLayerByName, +} from "oxalis/model/accessors/dataset_accessor"; import { useSelector } from "react-redux"; -import type { OxalisState } from "oxalis/store"; +import type { OxalisState, UserBoundingBox } from "oxalis/store"; import { - handleStartExport, getLayerInfos, isBoundingBoxExportable, + ExportFormat, + estimateFileSize, } from "../right-border-tabs/export_bounding_box_modal"; +import { clamp, computeBoundingBoxFromBoundingBoxObject } from "libs/utils"; const CheckboxGroup = Checkbox.Group; const { TabPane } = Tabs; const { Paragraph, Text } = Typography; @@ -123,30 +129,60 @@ function Footer({ ) : null; } -function _DownloadModalView(props: Props): JSX.Element { - const { isOpen, onClose, annotationType, annotationId, hasVolumeFallback } = props; +function _DownloadModalView({ + isOpen, + onClose, + annotationType, + annotationId, + hasVolumeFallback, +}: Props): JSX.Element { + const activeUser = useSelector((state: OxalisState) => state.activeUser); + const tracing = useSelector((state: OxalisState) => state.tracing); + const dataset = useSelector((state: OxalisState) => state.dataset); + const rawUserBoundingBoxes = useSelector((state: OxalisState) => + getUserBoundingBoxesFromState(state), + ); const [activeTabKey, setActiveTabKey] = useState("download"); const [includeVolumeData, setIncludeVolumeData] = useState(true); const [keepWindowOpen, setKeepWindowOpen] = useState(false); const [startedExports, setStartedExports] = useState([]); - const [selectedLayerName, setSelectedLayerName] = useState(null); - const [selectedBoundingBoxID, setSelectedBoundingBoxId] = useState(-1); - - const activeUser = useSelector((state: OxalisState) => state.activeUser); - const tracing = useSelector((state: OxalisState) => state.tracing); - const dataset = useSelector((state: OxalisState) => state.dataset); - const userBoundingBoxes = useSelector((state: OxalisState) => - getUserBoundingBoxesFromState(state), + const [selectedLayerName, setSelectedLayerName] = useState( + dataset.dataSource.dataLayers[0].name, ); const layers = getDataLayers(dataset); - const selectedBoundingBox = userBoundingBoxes.find((bbox) => bbox.id === selectedBoundingBoxID); - let boundingBoxCompatibleInfo = null; - if (selectedBoundingBox != null) { - boundingBoxCompatibleInfo = isBoundingBoxExportable(selectedBoundingBox.boundingBox); - } + const selectedLayer = dataset.dataSource.dataLayers.find( + (l) => l.name === selectedLayerName, + ) as APIDataLayer; + const selectedLayerInfos = getLayerInfos(selectedLayer, tracing); + + const userBoundingBoxes = [ + ...rawUserBoundingBoxes, + { + id: -1, + name: "Full dataset", + boundingBox: computeBoundingBoxFromBoundingBoxObject(selectedLayer.boundingBox), + color: [255, 255, 255], + isVisible: true, + } as UserBoundingBox, + ]; + const [selectedBoundingBoxId, setSelectedBoundingBoxId] = useState(userBoundingBoxes[0].id); + + const datasetResolutionInfo = getDatasetResolutionInfo(dataset); + const { lowestResolutionIndex, highestResolutionIndex } = selectedLayerInfos; + const [rawResolutionIndex, setResolutionIndex] = useState(lowestResolutionIndex); + const resolutionIndex = clamp(lowestResolutionIndex, rawResolutionIndex, highestResolutionIndex); + const [exportFormat, setExportFormat] = useState(ExportFormat.OME_TIFF); + + const selectedBoundingBox = userBoundingBoxes.find( + (bbox) => bbox.id === selectedBoundingBoxId, + ) as UserBoundingBox; + const boundingBoxCompatibleInfo = isBoundingBoxExportable( + selectedBoundingBox.boundingBox, + datasetResolutionInfo.getResolutionByIndexOrThrow(resolutionIndex), + ); const handleOk = async () => { if (activeTabKey === "download") { @@ -154,11 +190,11 @@ function _DownloadModalView(props: Props): JSX.Element { downloadAnnotation(annotationId, annotationType, hasVolumeFallback, {}, includeVolumeData); onClose(); } else if (activeTabKey === "export") { - const missingSelection = selectedLayerName == null || selectedBoundingBoxID === -1; + const missingSelection = selectedLayerName == null || selectedBoundingBoxId === -1; const basicWarning = "Starting an export job with the chosen parameters was not possible."; const missingSelectionWarning = " Please choose a layer and a bounding box for export."; - if (selectedLayerName == null || selectedBoundingBoxID === -1) { + if (selectedLayerName == null || selectedBoundingBoxId === -1) { Toast.warning(basicWarning + missingSelectionWarning); } else { const selectedLayer = getLayerByName(dataset, selectedLayerName); @@ -289,14 +325,13 @@ with wk.webknossos_context( title="Download this annotation" open={isOpen} width={600} - footer={[ + footer={