From a6424f596ee96acd91643ea2a6dc8785b7e89866 Mon Sep 17 00:00:00 2001 From: jorgee Date: Fri, 4 Apr 2025 13:06:26 +0200 Subject: [PATCH 1/5] Add cid quick wins Signed-off-by: jorgee --- .../groovy/nextflow/cli/CmdCidTest.groovy | 41 +++++---- .../nextflow/data/cid/CidHistoryFile.groovy | 41 +-------- .../nextflow/data/cid/CidHistoryLog.groovy | 10 +-- .../nextflow/data/cid/CidHistoryRecord.groovy | 9 +- .../main/nextflow/data/cid/CidObserver.groovy | 87 ++++++++++++------- .../main/nextflow/data/cid/CidUtils.groovy | 80 +++++++++++++---- .../data/cid/cli/CidCommandImpl.groovy | 18 +--- .../data/cid/fs/CidFileSystemProvider.groovy | 14 +-- ...ultsPath.groovy => CidMetadataPath.groovy} | 8 +- .../main/nextflow/data/cid/fs/CidPath.groovy | 66 ++++++++++---- .../{Output.groovy => DataOutput.groovy} | 9 +- .../nextflow/data/cid/model/TaskOutput.groovy | 32 ------- ...{TaskResults.groovy => TaskOutputs.groovy} | 10 ++- .../nextflow/data/cid/model/TaskRun.groovy | 3 +- .../data/cid/model/WorkflowOutput.groovy | 33 ------- ...wResults.groovy => WorkflowOutputs.groovy} | 9 +- .../data/cid/model/WorkflowRun.groovy | 1 + .../nextflow/data/cid/serde/CidEncoder.groovy | 26 +++--- .../data/cid/CidHistoryFileTest.groovy | 23 ++--- .../data/cid/CidHistoryRecordTest.groovy | 7 +- .../nextflow/data/cid/CidObserverTest.groovy | 39 +++++---- .../nextflow/data/cid/CidUtilsTest.groovy | 2 +- .../data/cid/DefaultCidStoreTest.groovy | 17 ++-- .../cid/fs/CidFileSystemProviderTest.groovy | 10 +-- .../nextflow/data/cid/fs/CidPathTest.groovy | 24 ++--- .../data/cid/serde/CidEncoderTest.groovy | 52 ++++++----- .../nextflow/serde/gson/InstantAdapter.groovy | 6 ++ .../data/cid/h2/H2CidHistoryLog.groovy | 29 +------ .../nextflow/data/cid/h2/H2CidStore.groovy | 1 - .../data/cid/h2/H2CidHistoryLogTest.groovy | 38 ++------ .../data/cid/h2/H2CidStoreTest.groovy | 15 ++-- 31 files changed, 351 insertions(+), 409 deletions(-) rename modules/nf-cid/src/main/nextflow/data/cid/fs/{CidResultsPath.groovy => CidMetadataPath.groovy} (90%) rename modules/nf-cid/src/main/nextflow/data/cid/model/{Output.groovy => DataOutput.groovy} (88%) delete mode 100644 modules/nf-cid/src/main/nextflow/data/cid/model/TaskOutput.groovy rename modules/nf-cid/src/main/nextflow/data/cid/model/{TaskResults.groovy => TaskOutputs.groovy} (86%) delete mode 100644 modules/nf-cid/src/main/nextflow/data/cid/model/WorkflowOutput.groovy rename modules/nf-cid/src/main/nextflow/data/cid/model/{WorkflowResults.groovy => WorkflowOutputs.groovy} (86%) diff --git a/modules/nextflow/src/test/groovy/nextflow/cli/CmdCidTest.groovy b/modules/nextflow/src/test/groovy/nextflow/cli/CmdCidTest.groovy index f94f7376e3..0f65e0f053 100644 --- a/modules/nextflow/src/test/groovy/nextflow/cli/CmdCidTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/cli/CmdCidTest.groovy @@ -27,9 +27,8 @@ import nextflow.data.cid.CidHistoryRecord import nextflow.data.cid.CidStoreFactory import nextflow.data.cid.model.Checksum import nextflow.data.cid.model.Parameter -import nextflow.data.cid.model.TaskOutput +import nextflow.data.cid.model.DataOutput import nextflow.data.cid.model.TaskRun -import nextflow.data.cid.model.WorkflowOutput import nextflow.plugin.Plugins import org.junit.Rule import spock.lang.Specification @@ -77,7 +76,7 @@ class CmdCidTest extends Specification { def launcher = Mock(Launcher){ getOptions() >> new CliOptions(config: [configFile.toString()]) } - def recordEntry = "${CidHistoryRecord.TIMESTAMP_FMT.format(date)}\trun_name\t${uniqueId}\tcid://123456\tcid://456789".toString() + def recordEntry = "${CidHistoryRecord.TIMESTAMP_FMT.format(date)}\trun_name\t${uniqueId}\tcid://123456".toString() historyFile.text = recordEntry when: def cidCmd = new CmdCid(launcher: launcher, args: ["log"]) @@ -136,10 +135,10 @@ class CmdCidTest extends Specification { def launcher = Mock(Launcher){ getOptions() >> new CliOptions(config: [configFile.toString()]) } - def time = Instant.ofEpochMilli(123456789).toString() + def time = Instant.ofEpochMilli(123456789) def encoder = new CidEncoder().withPrettyPrint(true) - def entry = new WorkflowOutput("path/to/file",new Checksum("45372qe","nextflow","standard"), - "cid://123987/file.bam", 1234, time, time, null) + def entry = new DataOutput("path/to/file",new Checksum("45372qe","nextflow","standard"), + "cid://123987/file.bam","cid://123987/", 1234, time, time, null) def jsonSer = encoder.encode(entry) def expectedOutput = jsonSer cidFile.text = jsonSer @@ -208,22 +207,26 @@ class CmdCidTest extends Specification { Files.createDirectories(cidFile4.parent) Files.createDirectories(cidFile5.parent) def encoder = new CidEncoder() - def time = Instant.ofEpochMilli(123456789).toString() - def entry = new WorkflowOutput("path/to/file",new Checksum("45372qe","nextflow","standard"), - "cid://123987/file.bam", 1234, time, time, null) + def time = Instant.ofEpochMilli(123456789) + def entry = new DataOutput("path/to/file",new Checksum("45372qe","nextflow","standard"), + "cid://123987/file.bam", "cid://45678", 1234, time, time, null) cidFile.text = encoder.encode(entry) - entry = new TaskOutput("path/to/file",new Checksum("45372qe","nextflow","standard"), - "cid://123987", 1234, time, time, null) + entry = new DataOutput("path/to/file",new Checksum("45372qe","nextflow","standard"), + "cid://123987", "cid://123987", 1234, time, time, null) cidFile2.text = encoder.encode(entry) - entry = new TaskRun("u345-2346-1stw2", "foo", new Checksum("abcde2345","nextflow","standard"), + entry = new TaskRun("u345-2346-1stw2", "foo", + new Checksum("abcde2345","nextflow","standard"), + new Checksum("abfsc2375","nextflow","standard"), [new Parameter( "ValueInParam", "sample_id","ggal_gut"), new Parameter("FileInParam","reads",["cid://45678/output.txt"])], null, null, null, null, [:],[], null) cidFile3.text = encoder.encode(entry) - entry = new TaskOutput("path/to/file",new Checksum("45372qe","nextflow","standard"), - "cid://45678", 1234, time, time, null) + entry = new DataOutput("path/to/file",new Checksum("45372qe","nextflow","standard"), + "cid://45678", "cid://45678", 1234, time, time, null) cidFile4.text = encoder.encode(entry) - entry = new TaskRun("u345-2346-1stw2", "bar", new Checksum("abfs2556","nextflow","standard"), + entry = new TaskRun("u345-2346-1stw2", "bar", + new Checksum("abfs2556","nextflow","standard"), + new Checksum("abfsc2375","nextflow","standard"), null,null, null, null, null, [:],[], null) cidFile5.text = encoder.encode(entry) final network = """flowchart BT @@ -275,14 +278,14 @@ class CmdCidTest extends Specification { getOptions() >> new CliOptions(config: [configFile.toString()]) } def encoder = new CidEncoder().withPrettyPrint(true) - def time = Instant.ofEpochMilli(123456789).toString() - def entry = new WorkflowOutput("path/to/file",new Checksum("45372qe","nextflow","standard"), - "cid://123987/file.bam", 1234, time, time, null) + def time = Instant.ofEpochMilli(123456789) + def entry = new DataOutput("path/to/file",new Checksum("45372qe","nextflow","standard"), + "cid://123987/file.bam", "cid://123987/", 1234, time, time, null) def jsonSer = encoder.encode(entry) def expectedOutput = jsonSer cidFile.text = jsonSer when: - def cidCmd = new CmdCid(launcher: launcher, args: ["show", "cid:///?type=WorkflowOutput"]) + def cidCmd = new CmdCid(launcher: launcher, args: ["show", "cid:///?type=DataOutput"]) cidCmd.run() def stdout = capture .toString() diff --git a/modules/nf-cid/src/main/nextflow/data/cid/CidHistoryFile.groovy b/modules/nf-cid/src/main/nextflow/data/cid/CidHistoryFile.groovy index d8217f6186..43594bcb38 100644 --- a/modules/nf-cid/src/main/nextflow/data/cid/CidHistoryFile.groovy +++ b/modules/nf-cid/src/main/nextflow/data/cid/CidHistoryFile.groovy @@ -38,13 +38,13 @@ class CidHistoryFile implements CidHistoryLog { this.path = file } - void write(String name, UUID key, String runCid, String resultsCid, Date date = null) { + void write(String name, UUID key, String runCid, Date date = null) { assert key withFileLock { def timestamp = date ?: new Date() log.trace("Writting record for $key in CID history file $this") - path << new CidHistoryRecord(timestamp, name, key, runCid, resultsCid).toString() << '\n' + path << new CidHistoryRecord(timestamp, name, key, runCid).toString() << '\n' } } @@ -59,17 +59,6 @@ class CidHistoryFile implements CidHistoryLog { } } - void updateResultsCid(UUID sessionId, String resultsCid) { - assert sessionId - - try { - withFileLock { updateResultsCid0(sessionId, resultsCid) } - } - catch (Throwable e) { - log.warn "Can't update CID history file: $this", e.message - } - } - List getRecords(){ List list = new LinkedList() try { @@ -105,31 +94,7 @@ class CidHistoryFile implements CidHistoryLog { def current = line ? CidHistoryRecord.parse(line) : null if (current.sessionId == id) { log.trace("Updating record for $id in CID history file $this") - final newRecord = new CidHistoryRecord(current.timestamp, current.runName, current.sessionId, runCid, current.resultsCid) - newHistory << newRecord.toString() << '\n' - } else { - newHistory << line << '\n' - } - } - catch (IllegalArgumentException e) { - log.warn("Can't read CID history file: $this", e.message) - } - } - - // rewrite the history content - this.path.setText(newHistory.toString()) - } - - private void updateResultsCid0(UUID id, String resultsCid) { - assert id - def newHistory = new StringBuilder() - - this.path.readLines().each { line -> - try { - def current = line ? CidHistoryRecord.parse(line) : null - if (current.sessionId == id) { - log.trace("Updating record for $id in CID history file $this") - final newRecord = new CidHistoryRecord(current.timestamp, current.runName, current.sessionId, current.runCid, resultsCid) + final newRecord = new CidHistoryRecord(current.timestamp, current.runName, current.sessionId, runCid) newHistory << newRecord.toString() << '\n' } else { newHistory << line << '\n' diff --git a/modules/nf-cid/src/main/nextflow/data/cid/CidHistoryLog.groovy b/modules/nf-cid/src/main/nextflow/data/cid/CidHistoryLog.groovy index bc541b7760..3b71e911e8 100644 --- a/modules/nf-cid/src/main/nextflow/data/cid/CidHistoryLog.groovy +++ b/modules/nf-cid/src/main/nextflow/data/cid/CidHistoryLog.groovy @@ -30,7 +30,7 @@ interface CidHistoryLog { * @param runCid Workflow run CID. * @param resultsCid Workflow results CID. */ - void write(String name, UUID sessionId, String runCid, String resultsCid) + void write(String name, UUID sessionId, String runCid) /** * Updates the run CID for a given session ID. @@ -40,14 +40,6 @@ interface CidHistoryLog { */ void updateRunCid(UUID sessionId, String runCid) - /** - * Updates the results CID for a given session ID. - * - * @param sessionId Workflow session ID. - * @param resultsCid Workflow results CID. - */ - void updateResultsCid(UUID sessionId, String resultsCid) - /** * Get the store records in the CidHistoryLog. * diff --git a/modules/nf-cid/src/main/nextflow/data/cid/CidHistoryRecord.groovy b/modules/nf-cid/src/main/nextflow/data/cid/CidHistoryRecord.groovy index 744b114e22..544ee26ac5 100644 --- a/modules/nf-cid/src/main/nextflow/data/cid/CidHistoryRecord.groovy +++ b/modules/nf-cid/src/main/nextflow/data/cid/CidHistoryRecord.groovy @@ -35,14 +35,12 @@ class CidHistoryRecord { final String runName final UUID sessionId final String runCid - final String resultsCid - CidHistoryRecord(Date timestamp, String name, UUID sessionId, String runCid, String resultsCid = null) { + CidHistoryRecord(Date timestamp, String name, UUID sessionId, String runCid) { this.timestamp = timestamp this.runName = name this.sessionId = sessionId this.runCid = runCid - this.resultsCid = resultsCid } CidHistoryRecord(UUID sessionId, String name = null) { @@ -58,7 +56,6 @@ class CidHistoryRecord { line << (runName ?: '-') line << (sessionId.toString()) line << (runCid ?: '-') - line << (resultsCid ?: '-') } @Override @@ -71,8 +68,8 @@ class CidHistoryRecord { if (cols.size() == 2) return new CidHistoryRecord(UUID.fromString(cols[0])) - if (cols.size() == 5) { - return new CidHistoryRecord(TIMESTAMP_FMT.parse(cols[0]), cols[1], UUID.fromString(cols[2]), cols[3], cols[4]) + if (cols.size() == 4) { + return new CidHistoryRecord(TIMESTAMP_FMT.parse(cols[0]), cols[1], UUID.fromString(cols[2]), cols[3]) } throw new IllegalArgumentException("Not a valid history entry: `$line`") diff --git a/modules/nf-cid/src/main/nextflow/data/cid/CidObserver.groovy b/modules/nf-cid/src/main/nextflow/data/cid/CidObserver.groovy index 8ffcb5960a..cd1a70dcbe 100644 --- a/modules/nf-cid/src/main/nextflow/data/cid/CidObserver.groovy +++ b/modules/nf-cid/src/main/nextflow/data/cid/CidObserver.groovy @@ -30,11 +30,10 @@ import nextflow.Session import nextflow.data.cid.model.Checksum import nextflow.data.cid.model.DataPath import nextflow.data.cid.model.Parameter -import nextflow.data.cid.model.TaskOutput -import nextflow.data.cid.model.TaskResults +import nextflow.data.cid.model.DataOutput +import nextflow.data.cid.model.TaskOutputs import nextflow.data.cid.model.Workflow -import nextflow.data.cid.model.WorkflowOutput -import nextflow.data.cid.model.WorkflowResults +import nextflow.data.cid.model.WorkflowOutputs import nextflow.data.cid.model.WorkflowRun import nextflow.data.cid.serde.CidEncoder import nextflow.file.FileHelper @@ -64,7 +63,7 @@ class CidObserver implements TraceObserver { private String executionHash private CidStore store private Session session - private WorkflowResults workflowResults + private WorkflowOutputs workflowResults private Map outputsStoreDirCid = new HashMap(10) private CidEncoder encoder = new CidEncoder() @@ -75,7 +74,7 @@ class CidObserver implements TraceObserver { @Override void onFlowCreate(Session session) { - this.store.getHistoryLog().write(session.runName, session.uniqueId, '-', '-') + this.store.getHistoryLog().write(session.runName, session.uniqueId, '-') } @TestOnly @@ -85,8 +84,8 @@ class CidObserver implements TraceObserver { void onFlowBegin() { executionHash = storeWorkflowRun() final executionUri = asUriString(executionHash) - workflowResults = new WorkflowResults( - Instant.now().toString(), + workflowResults = new WorkflowOutputs( + Instant.now(), executionUri, new HashMap() ) @@ -96,10 +95,9 @@ class CidObserver implements TraceObserver { @Override void onFlowComplete(){ if (this.workflowResults){ - workflowResults.creationTime = System.currentTimeMillis() - final key = CacheHelper.hasher(workflowResults).hash().toString() - this.store.save("${key}", workflowResults) - this.store.getHistoryLog().updateResultsCid(session.uniqueId, asUriString(key)) + workflowResults.createdAt = Instant.now() + final key = executionHash + SEPARATOR + 'outputs' + this.store.save(key, workflowResults) } } @@ -165,23 +163,33 @@ class CidObserver implements TraceObserver { // store the task run entry storeTaskRun(task, pathNormalizer) // store all task results - storeTaskResults(task) + storeTaskResults(task, pathNormalizer) } - protected String storeTaskResults(TaskRun task ){ + protected String storeTaskResults(TaskRun task, PathNormalizer normalizer){ + final outputParams = getNormalizedTaskOutputs(task, normalizer) + final value = new TaskOutputs( asUriString(task.hash.toString()), asUriString(executionHash), Instant.now(), outputParams ) + final key = CacheHelper.hasher(value).hash().toString() + store.save(key,value) + return key + } + + private List getNormalizedTaskOutputs( TaskRun task, PathNormalizer normalizer){ final outputs = task.getOutputs() final outputParams = new LinkedList() outputs.forEach { OutParam key, Object value -> if (key instanceof FileOutParam) { - outputParams.add(new Parameter(key.class.simpleName, key.name, manageFileOutParams(value, task))) + outputParams.add( new Parameter( key.class.simpleName, key.name, manageFileOutParams(value, task) ) ) } else { - outputParams.add(new Parameter(key.class.simpleName, key.name, value) ) + if( value instanceof Path ) + outputParams.add( new Parameter( key.class.simpleName, key.name, normalizer.normalizePath( value as Path ) ) ) + else if ( value instanceof CharSequence ) + outputParams.add( new Parameter( key.class.simpleName, key.name, normalizer.normalizePath( value.toString() ) ) ) + else + outputParams.add( new Parameter( key.class.simpleName, key.name, value) ) } } - final value = new TaskResults(asUriString(task.hash.toString()), asUriString(executionHash), Instant.now().toString(), outputParams) - final key = CacheHelper.hasher(value).hash().toString() - store.save(key,value) - return key + return outputParams } private Object manageFileOutParams( Object value, TaskRun task) { @@ -200,10 +208,13 @@ class CidObserver implements TraceObserver { protected String storeTaskRun(TaskRun task, PathNormalizer normalizer) { final codeChecksum = new Checksum(CacheHelper.hasher(session.stubRun ? task.stubSource: task.source).hash().toString(), "nextflow", CacheHelper.HashMode.DEFAULT().toString().toLowerCase()) + final scriptChecksum = new Checksum(CacheHelper.hasher(task.script).hash().toString(), + "nextflow", CacheHelper.HashMode.DEFAULT().toString().toLowerCase()) final value = new nextflow.data.cid.model.TaskRun( session.uniqueId.toString(), task.getName(), codeChecksum, + scriptChecksum, task.inputs ? manageInputs(task.inputs, normalizer): null, task.isContainerEnabled() ? task.getContainerFingerprint(): null, normalizer.normalizePath(task.getCondaEnv()), @@ -226,15 +237,14 @@ class CidObserver implements TraceObserver { protected String storeTaskOutput(TaskRun task, Path path) { try { final attrs = readAttributes(path) - final rel = getTaskRelative(task, path) - final cid = "${task.hash}/${rel}" - final key = cid.toString() + final key = getTaskOutputKey(task, path) final checksum = new Checksum( CacheHelper.hasher(path).hash().toString(), "nextflow", CacheHelper.HashMode.DEFAULT().toString().toLowerCase() ) - final value = new TaskOutput( + final value = new DataOutput( path.toUriString(), checksum, asUriString(task.hash.toString()), + asUriString(task.hash.toString()), attrs.size(), CidUtils.toDate(attrs?.creationTime()), CidUtils.toDate(attrs?.lastModifiedTime())) @@ -242,9 +252,21 @@ class CidObserver implements TraceObserver { return key } catch (Throwable e) { log.warn("Exception storing CID output $path for task ${task.name}. ${e.getLocalizedMessage()}") + return path.toUriString() } } + protected String getTaskOutputKey(TaskRun task, Path path) { + final rel = getTaskRelative(task, path) + return task.hash.toString() + SEPARATOR + 'outputs' + SEPARATOR + rel + } + + protected String getWorkflowOutputKey(Path destination) { + final rel = getWorkflowRelative(destination) + return executionHash + SEPARATOR + 'outputs' + SEPARATOR + rel + + } + protected String getTaskRelative(TaskRun task, Path path){ if (path.isAbsolute()) { final rel = getTaskRelative0(task, path) @@ -292,20 +314,18 @@ class CidObserver implements TraceObserver { "nextflow", CacheHelper.HashMode.DEFAULT().toString().toLowerCase() ) - final rel = getWorkflowRelative(destination) - final key = "$executionHash/${rel}" - + final key = getWorkflowOutputKey(destination) final sourceReference = source ? getSourceReference(source) : asUriString(executionHash) final attrs = readAttributes(destination) - final value = new WorkflowOutput( + final value = new DataOutput( destination.toUriString(), checksum, sourceReference, + asUriString(executionHash), attrs.size(), CidUtils.toDate(attrs?.creationTime()), CidUtils.toDate(attrs?.lastModifiedTime()), annotations) - value.publishedBy = asUriString(executionHash) store.save(key, value) } catch (Throwable e) { log.warn("Exception storing published file $destination for workflow ${executionHash}.", e) @@ -334,8 +354,13 @@ class CidObserver implements TraceObserver { private Object convertPathsToCidReferences(Object value){ if( value instanceof Path ) { - final rel = getWorkflowRelative(value) - return rel ? asUriString(executionHash, rel) : value + try { + final key = getWorkflowOutputKey(value) + return asUriString(key) + } catch (Throwable e){ + //Workflow output key not found + return value + } } if( value instanceof Collection ) { diff --git a/modules/nf-cid/src/main/nextflow/data/cid/CidUtils.groovy b/modules/nf-cid/src/main/nextflow/data/cid/CidUtils.groovy index 96f37858a9..4a448c26b8 100644 --- a/modules/nf-cid/src/main/nextflow/data/cid/CidUtils.groovy +++ b/modules/nf-cid/src/main/nextflow/data/cid/CidUtils.groovy @@ -20,9 +20,12 @@ package nextflow.data.cid import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import nextflow.data.cid.fs.CidPath +import nextflow.data.cid.model.WorkflowRun +import nextflow.data.cid.model.TaskRun +import nextflow.data.cid.serde.CidEncoder import nextflow.data.cid.serde.CidSerializable +import nextflow.serde.gson.GsonEncoder -import java.nio.file.Path import java.nio.file.attribute.FileTime import java.time.Instant @@ -102,31 +105,53 @@ class CidUtils { final object = store.load(key) if (object) { if (children && children.size() > 0) { - final output = navigate(object, children.join('.')) + final output = getSubObject(store, key, object, children) if (output) { treatObject(output, params, results) } else { - throw new FileNotFoundException("Cid object $key/${children ? children.join('/') : ''} not found.") + throw new FileNotFoundException("Cid object $key#${children.join('.')} not found.") } } else { treatObject(object, params, results) } } else { - // If there isn't metadata check the parent to check if it is a subfolder of a task/workflow output - final currentPath = Path.of(key) - final parent = currentPath.getParent() - if (parent) { - ArrayList newChildren = new ArrayList() - newChildren.add(currentPath.getFileName().toString()) - newChildren.addAll(children) - return searchPath(store, parent.toString(), params, newChildren as String[]) - } else { - throw new FileNotFoundException("Cid object $key/${children ? children.join('/') : ''} not found.") - } + throw new FileNotFoundException("Cid object $key not found.") } return results } + /** + * Get a metadata sub-object. + * If the requested sub-object is the workflow or task outputs, retrieves the outputs from the outputs description. + * + * @param store CidStore to retrieve metadata objects. + * @param key Parent metadata key. + * @param object Parent object. + * @param children Array of string in indicating the properties to navigate to get the sub-object. + * @return Sub-object or null in it does not exist. + */ + static Object getSubObject(CidStore store, String key, CidSerializable object, String[] children) { + if( isSearchingOutputs(object, children) ) { + // When asking for a Workflow or task output retrieve the outputs description + final outputs = store.load("${key}/outputs") + if (outputs) + return navigate(outputs, children.join('.')) + else + return null + } + return navigate(object, children.join('.')) + } + /** + * Check if the Cid pseudo path or query is for Task or Workflow outputs. + * + * @param object Parent Cid metadata object + * @param children Array of string in indicating the properties to navigate to get the sub-object. + * @return return 'true' if the parent is a Task/Workflow run and the first element in children is 'outputs'. Otherwise 'false' + */ + public static boolean isSearchingOutputs(CidSerializable object, String[] children) { + return (object instanceof WorkflowRun || object instanceof TaskRun) && children && children[0] == 'outputs' + } + /** * Evaluates object or the objects in a collection matches a set of parameter-value pairs. It includes in the results collection in case of match. * @param object Object or collection of objects to evaluate @@ -210,11 +235,11 @@ class CidUtils { * Helper function to convert from FileTime to ISO 8601. * * @param time File time to convert - * @return ISO Date format or 'N/A' in case of not available (null) + * @return Instant or null in case of not available (null) */ - static String toDate(FileTime time){ + static Instant toDate(FileTime time){ if (time) - return Instant.ofEpochMilli(time.toMillis()).toString() + return Instant.ofEpochMilli(time.toMillis()) else return null } @@ -230,4 +255,25 @@ class CidUtils { return null return FileTime.from(Instant.parse(date)) } + + /** + * Helper function to unify the encoding of outputs when querying and navigating the CID pseudoFS. + * Outputs can include CidSerializable objects, collections or parts of these objects. + * CidSerializable objects can be encoded with the CidEncoder, but collections or parts of + * these objects require to extend the GsonEncoder. + * + * @param output Output to encode + * @return Output encoded as a JSON string + */ + static String encodeSearchOutputs(Object output, boolean prettyPrint) { + if (output instanceof CidSerializable){ + return new CidEncoder().withPrettyPrint(prettyPrint).encode(output) + } else { + return new GsonEncoder() {} + .withPrettyPrint(prettyPrint) + .withSerializeNulls(true) + .withTypeAdapterFactory(CidEncoder.newCidTypeAdapterFactory()) + .encode(output) + } + } } diff --git a/modules/nf-cid/src/main/nextflow/data/cid/cli/CidCommandImpl.groovy b/modules/nf-cid/src/main/nextflow/data/cid/cli/CidCommandImpl.groovy index ebe19eebfa..78b95e6d9d 100644 --- a/modules/nf-cid/src/main/nextflow/data/cid/cli/CidCommandImpl.groovy +++ b/modules/nf-cid/src/main/nextflow/data/cid/cli/CidCommandImpl.groovy @@ -32,16 +32,11 @@ import nextflow.data.cid.CidHistoryRecord import nextflow.data.cid.CidStore import nextflow.data.cid.CidStoreFactory import nextflow.data.cid.CidUtils -import nextflow.data.cid.model.Output +import nextflow.data.cid.model.DataOutput import nextflow.data.cid.model.Parameter -import nextflow.data.cid.model.TaskOutput import nextflow.data.cid.model.TaskRun -import nextflow.data.cid.model.WorkflowOutput import nextflow.data.cid.model.WorkflowRun -import nextflow.data.cid.serde.CidEncoder -import nextflow.data.cid.serde.CidSerializable import nextflow.script.params.FileInParam -import nextflow.serde.gson.GsonEncoder import nextflow.ui.TableBuilder import org.eclipse.jgit.diff.DiffAlgorithm import org.eclipse.jgit.diff.DiffFormatter @@ -81,7 +76,6 @@ class CidCommandImpl implements CmdCid.CidCommand { .head('RUN NAME') .head('SESSION ID') .head('RUN CID') - .head('RESULT CID') for( CidHistoryRecord record: records ){ table.append(record.toList()) } @@ -101,10 +95,7 @@ class CidCommandImpl implements CmdCid.CidCommand { def entries = CidUtils.query(store, new URI(args[0])) if( entries ) { entries = entries.size() == 1 ? entries[0] : entries - if (entries instanceof CidSerializable) - println new CidEncoder().withPrettyPrint(true).encode(entries as CidSerializable) - else - println new GsonEncoder(){}.withPrettyPrint(true).encode(entries) + println CidUtils.encodeSearchOutputs(entries, true) } else { println "No entries found for ${args[0]}." } @@ -153,10 +144,9 @@ class CidCommandImpl implements CmdCid.CidCommand { final key = nodeToRender.substring(CID_PROT.size()) final cidObject = store.load(key) switch (cidObject.getClass()) { - case TaskOutput: - case WorkflowOutput: + case DataOutput: lines << " ${nodeToRender}@{shape: document, label: \"${nodeToRender}\"}".toString(); - final source = (cidObject as Output).source + final source = (cidObject as DataOutput).source if (source) { if (isCidUri(source)) { nodes.add(source) diff --git a/modules/nf-cid/src/main/nextflow/data/cid/fs/CidFileSystemProvider.groovy b/modules/nf-cid/src/main/nextflow/data/cid/fs/CidFileSystemProvider.groovy index 378b9da42b..7761a3bb7d 100644 --- a/modules/nf-cid/src/main/nextflow/data/cid/fs/CidFileSystemProvider.groovy +++ b/modules/nf-cid/src/main/nextflow/data/cid/fs/CidFileSystemProvider.groovy @@ -113,8 +113,8 @@ class CidFileSystemProvider extends FileSystemProvider { InputStream newInputStream(Path path, OpenOption... options) throws IOException { final cid = toCidPath(path) final realPath = cid.getTargetPath(true) - if (realPath instanceof CidResultsPath) - return (realPath as CidResultsPath).newInputStream() + if (realPath instanceof CidMetadataPath) + return (realPath as CidMetadataPath).newInputStream() else return realPath.fileSystem.provider().newInputStream(realPath, options) } @@ -131,8 +131,8 @@ class CidFileSystemProvider extends FileSystemProvider { } final realPath = cid.getTargetPath(true) SeekableByteChannel channel - if (realPath instanceof CidResultsPath){ - channel = (realPath as CidResultsPath).newSeekableByteChannel() + if (realPath instanceof CidMetadataPath){ + channel = (realPath as CidMetadataPath).newSeekableByteChannel() } else { channel = realPath.fileSystem.provider().newByteChannel(realPath, options, attrs) } @@ -300,7 +300,7 @@ class CidFileSystemProvider extends FileSystemProvider { throw new AccessDeniedException("Execute mode not supported") } final real = cid.getTargetPath(true) - if (real instanceof CidResultsPath) + if (real instanceof CidMetadataPath) return real.fileSystem.provider().checkAccess(real, modes) } @@ -314,8 +314,8 @@ class CidFileSystemProvider extends FileSystemProvider { A readAttributes(Path path, Class type, LinkOption... options) throws IOException { final cid = toCidPath(path) final real = cid.getTargetPath(true) - if (real instanceof CidResultsPath) - return (real as CidResultsPath).readAttributes(type) + if (real instanceof CidMetadataPath) + return (real as CidMetadataPath).readAttributes(type) else return real.fileSystem.provider().readAttributes(real,type,options) } diff --git a/modules/nf-cid/src/main/nextflow/data/cid/fs/CidResultsPath.groovy b/modules/nf-cid/src/main/nextflow/data/cid/fs/CidMetadataPath.groovy similarity index 90% rename from modules/nf-cid/src/main/nextflow/data/cid/fs/CidResultsPath.groovy rename to modules/nf-cid/src/main/nextflow/data/cid/fs/CidMetadataPath.groovy index bb3d791fdc..1c98ca3f84 100644 --- a/modules/nf-cid/src/main/nextflow/data/cid/fs/CidResultsPath.groovy +++ b/modules/nf-cid/src/main/nextflow/data/cid/fs/CidMetadataPath.groovy @@ -24,17 +24,17 @@ import java.nio.file.attribute.BasicFileAttributes import java.nio.file.attribute.FileTime /** - * Class to model the metadata results description as a file. + * Class to model the metadata descriptions as a file. * * @author Jorge Ejarque */ @CompileStatic -class CidResultsPath extends CidPath { +class CidMetadataPath extends CidPath { private byte[] results private FileTime creationTime - CidResultsPath (String resultsObject, FileTime creationTime, CidFileSystem fs, String path, String[] childs) { - super(fs, path, childs) + CidMetadataPath(String resultsObject, FileTime creationTime, CidFileSystem fs, String path, String[] childs) { + super(fs, "${path}${childs ? '#'+ childs.join('.') : ''}") this.results = resultsObject.getBytes("UTF-8") this.creationTime = creationTime } diff --git a/modules/nf-cid/src/main/nextflow/data/cid/fs/CidPath.groovy b/modules/nf-cid/src/main/nextflow/data/cid/fs/CidPath.groovy index c287fe46d4..4364eb97ca 100644 --- a/modules/nf-cid/src/main/nextflow/data/cid/fs/CidPath.groovy +++ b/modules/nf-cid/src/main/nextflow/data/cid/fs/CidPath.groovy @@ -18,12 +18,9 @@ package nextflow.data.cid.fs import groovy.util.logging.Slf4j -import nextflow.data.cid.CidUtils -import nextflow.data.cid.model.Output -import nextflow.data.cid.serde.CidEncoder +import nextflow.data.cid.model.DataOutput import nextflow.data.cid.serde.CidSerializable import nextflow.file.RealPathAware -import nextflow.serde.gson.GsonEncoder import nextflow.util.CacheHelper import nextflow.util.TestOnly @@ -31,6 +28,7 @@ import java.nio.file.attribute.FileTime import java.time.Instant import static nextflow.data.cid.fs.CidFileSystemProvider.* +import static nextflow.data.cid.CidUtils.* import java.nio.file.FileSystem import java.nio.file.LinkOption @@ -115,7 +113,7 @@ class CidPath implements Path, RealPathAware { return first } - private static void validateHash(Output cidObject) { + private static void validateHash(DataOutput cidObject) { final hashedPath = FileHelper.toCanonicalPath(cidObject.path as String) if( !hashedPath.exists() ) throw new FileNotFoundException("Target path $cidObject.path does not exists.") @@ -151,10 +149,9 @@ class CidPath implements Path, RealPathAware { throw new Exception("CID store not found. Check Nextflow configuration.") final object = store.load(filePath) if ( object ){ - if( object instanceof Output ) { + if( object instanceof DataOutput ) { return getTargetPathFromOutput(object, children) } - if( resultsAsPath ){ return getMetadataAsTargetPath(object, fs, filePath, children) } @@ -166,7 +163,8 @@ class CidPath implements Path, RealPathAware { ArrayList newChildren = new ArrayList() newChildren.add(currentPath.getFileName().toString()) newChildren.addAll(children) - return findTarget(fs, parent.toString(), resultsAsPath, newChildren as String[]) + //resultsAsPath set to false because parent paths are only inspected for DataOutputs + return findTarget(fs, parent.toString(), false, newChildren as String[]) } } throw new FileNotFoundException("Target path $filePath does not exists.") @@ -174,20 +172,49 @@ class CidPath implements Path, RealPathAware { protected static Path getMetadataAsTargetPath(CidSerializable results, CidFileSystem fs, String filePath, String[] children){ if( results ) { - def creationTime = CidUtils.toFileTime(CidUtils.navigate(results, 'creationTime') as String) ?: FileTime.from(Instant.now()) if( children && children.size() > 0 ) { - final output = CidUtils.navigate(results, children.join('.')) - if( output ){ - return new CidResultsPath(new GsonEncoder(){}.withPrettyPrint(true).encode(output), creationTime, fs, filePath, children) - } + return getSubObjectAsPath(fs, filePath, results, children) + }else { + return generateCidMetadataPath(fs, filePath, results, children) } - return new CidResultsPath(new CidEncoder().withPrettyPrint(true).encode(results), creationTime, fs, filePath, children) } throw new FileNotFoundException("Target path $filePath does not exists.") } - private static Path getTargetPathFromOutput(Output object, String[] children) { - final cidObject = object as Output + /** + * Get a metadata sub-object as CidMetadataPath. + * If the requested sub-object is the workflow or task outputs, retrieves the outputs from the outputs description. + * + * @param fs CidFilesystem for the te. + * @param key Parent metadata key. + * @param object Parent object. + * @param children Array of string in indicating the properties to navigate to get the sub-object. + * @return CidMetadataPath or null in it does not exist. + */ + static CidMetadataPath getSubObjectAsPath(CidFileSystem fs, String key, CidSerializable object, String[] children) { + if( isSearchingOutputs(object, children) ) { + // When asking for a Workflow or task output retrieve the outputs description + final outputs = fs.cidStore.load("${key}/outputs") + if( outputs ) { + return generateCidMetadataPath(fs, key, outputs, children) + } else + throw new FileNotFoundException("Target path $key#outputs does not exists.") + } else { + return generateCidMetadataPath(fs, key, object, children) + } + } + + private static CidMetadataPath generateCidMetadataPath(CidFileSystem fs, String key, Object object, String[] children){ + def creationTime = FileTime.from(navigate(object, 'createdAt') as Instant ?: Instant.now()) + final output = children ? navigate(object, children.join('.')) : object + if( output ){ + return new CidMetadataPath(encodeSearchOutputs(output, true), creationTime, fs, key, children) + } + throw new FileNotFoundException("Target path $key#${children.join('.')} does not exists.") + } + + private static Path getTargetPathFromOutput(DataOutput object, String[] children) { + final cidObject = object as DataOutput // return the real path stored in the metadata validateHash(cidObject) def realPath = FileHelper.toCanonicalPath(cidObject.path as String) @@ -264,7 +291,7 @@ class CidPath implements Path, RealPathAware { @Override Path getFileName() { final result = Path.of(filePath).getFileName()?.toString() - return result ? new CidPath(null, result) : null + return result ? new CidPath( fragment, query, result, null) : null } @Override @@ -287,6 +314,9 @@ class CidPath implements Path, RealPathAware { if( index<0 ) throw new IllegalArgumentException("Path name index cannot be less than zero - offending value: $index") final path = Path.of(filePath) + if (index == path.nameCount - 1){ + return new CidPath( fragment, query, path.getName(index).toString(), null) + } return new CidPath(index==0 ? fileSystem : null, path.getName(index).toString()) } @@ -395,7 +425,7 @@ class CidPath implements Path, RealPathAware { } protected Path getTargetPath(boolean resultsAsPath=false){ - return findTarget(fileSystem, filePath, resultsAsPath, CidUtils.parseChildrenFormFragment(fragment)) + return findTarget(fileSystem, filePath, resultsAsPath, parseChildrenFormFragment(fragment)) } @Override diff --git a/modules/nf-cid/src/main/nextflow/data/cid/model/Output.groovy b/modules/nf-cid/src/main/nextflow/data/cid/model/DataOutput.groovy similarity index 88% rename from modules/nf-cid/src/main/nextflow/data/cid/model/Output.groovy rename to modules/nf-cid/src/main/nextflow/data/cid/model/DataOutput.groovy index c5d0d94a1c..a2b6d24554 100644 --- a/modules/nf-cid/src/main/nextflow/data/cid/model/Output.groovy +++ b/modules/nf-cid/src/main/nextflow/data/cid/model/DataOutput.groovy @@ -21,6 +21,8 @@ import groovy.transform.Canonical import groovy.transform.CompileStatic import nextflow.data.cid.serde.CidSerializable +import java.time.Instant + /** * Model a base class for workflow and task outputs * @@ -28,12 +30,13 @@ import nextflow.data.cid.serde.CidSerializable */ @Canonical @CompileStatic -abstract class Output implements CidSerializable { +class DataOutput implements CidSerializable { String path Checksum checksum String source + String run long size - String createdAt - String modifiedAt + Instant createdAt + Instant modifiedAt Map annotations } diff --git a/modules/nf-cid/src/main/nextflow/data/cid/model/TaskOutput.groovy b/modules/nf-cid/src/main/nextflow/data/cid/model/TaskOutput.groovy deleted file mode 100644 index f0f2828dff..0000000000 --- a/modules/nf-cid/src/main/nextflow/data/cid/model/TaskOutput.groovy +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2013-2025, Seqera Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package nextflow.data.cid.model - -import groovy.transform.Canonical -import groovy.transform.CompileStatic -import groovy.transform.InheritConstructors - -/** - * Model a task output object - * - * @author Paolo Di Tommaso - */ -@Canonical -@CompileStatic -@InheritConstructors -class TaskOutput extends Output { -} diff --git a/modules/nf-cid/src/main/nextflow/data/cid/model/TaskResults.groovy b/modules/nf-cid/src/main/nextflow/data/cid/model/TaskOutputs.groovy similarity index 86% rename from modules/nf-cid/src/main/nextflow/data/cid/model/TaskResults.groovy rename to modules/nf-cid/src/main/nextflow/data/cid/model/TaskOutputs.groovy index b51b0e07f1..a80cf11b64 100644 --- a/modules/nf-cid/src/main/nextflow/data/cid/model/TaskResults.groovy +++ b/modules/nf-cid/src/main/nextflow/data/cid/model/TaskOutputs.groovy @@ -21,6 +21,8 @@ import groovy.transform.Canonical import groovy.transform.CompileStatic import nextflow.data.cid.serde.CidSerializable +import java.time.Instant + /** * Models task results. * @@ -28,10 +30,10 @@ import nextflow.data.cid.serde.CidSerializable */ @Canonical @CompileStatic -class TaskResults implements CidSerializable { +class TaskOutputs implements CidSerializable { String taskRun - String runBy - String creationTime + String workflowRun + Instant createdAt List outputs - List annotations + Map annotations } diff --git a/modules/nf-cid/src/main/nextflow/data/cid/model/TaskRun.groovy b/modules/nf-cid/src/main/nextflow/data/cid/model/TaskRun.groovy index e6b9fbdadc..e45356461b 100644 --- a/modules/nf-cid/src/main/nextflow/data/cid/model/TaskRun.groovy +++ b/modules/nf-cid/src/main/nextflow/data/cid/model/TaskRun.groovy @@ -32,6 +32,7 @@ class TaskRun implements CidSerializable { String sessionId String name Checksum codeChecksum + Checksum scriptChecksum List inputs String container String conda @@ -39,5 +40,5 @@ class TaskRun implements CidSerializable { String architecture Map globalVars List binEntries - List annotations + Map annotations } diff --git a/modules/nf-cid/src/main/nextflow/data/cid/model/WorkflowOutput.groovy b/modules/nf-cid/src/main/nextflow/data/cid/model/WorkflowOutput.groovy deleted file mode 100644 index 8a65951a8d..0000000000 --- a/modules/nf-cid/src/main/nextflow/data/cid/model/WorkflowOutput.groovy +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2013-2025, Seqera Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package nextflow.data.cid.model - -import groovy.transform.Canonical -import groovy.transform.CompileStatic -import groovy.transform.InheritConstructors - -/** - * Model a workflow output object - * - * @author Paolo Di Tommaso - */ -@Canonical -@CompileStatic -@InheritConstructors -class WorkflowOutput extends Output { - String publishedBy -} diff --git a/modules/nf-cid/src/main/nextflow/data/cid/model/WorkflowResults.groovy b/modules/nf-cid/src/main/nextflow/data/cid/model/WorkflowOutputs.groovy similarity index 86% rename from modules/nf-cid/src/main/nextflow/data/cid/model/WorkflowResults.groovy rename to modules/nf-cid/src/main/nextflow/data/cid/model/WorkflowOutputs.groovy index 0d308967e5..fa9799d084 100644 --- a/modules/nf-cid/src/main/nextflow/data/cid/model/WorkflowResults.groovy +++ b/modules/nf-cid/src/main/nextflow/data/cid/model/WorkflowOutputs.groovy @@ -21,6 +21,8 @@ import groovy.transform.Canonical import groovy.transform.CompileStatic import nextflow.data.cid.serde.CidSerializable +import java.time.Instant + /** * Models the results of a workflow execution. * @@ -28,8 +30,9 @@ import nextflow.data.cid.serde.CidSerializable */ @Canonical @CompileStatic -class WorkflowResults implements CidSerializable { - String creationTime - String runId +class WorkflowOutputs implements CidSerializable { + Instant createdAt + String workflowRun Map outputs + Map annotations } diff --git a/modules/nf-cid/src/main/nextflow/data/cid/model/WorkflowRun.groovy b/modules/nf-cid/src/main/nextflow/data/cid/model/WorkflowRun.groovy index d0f76871f1..87f401d7e9 100644 --- a/modules/nf-cid/src/main/nextflow/data/cid/model/WorkflowRun.groovy +++ b/modules/nf-cid/src/main/nextflow/data/cid/model/WorkflowRun.groovy @@ -33,4 +33,5 @@ class WorkflowRun implements CidSerializable { String sessionId String name List params + Map annotations } diff --git a/modules/nf-cid/src/main/nextflow/data/cid/serde/CidEncoder.groovy b/modules/nf-cid/src/main/nextflow/data/cid/serde/CidEncoder.groovy index aa65f56d61..6f23cdbc69 100644 --- a/modules/nf-cid/src/main/nextflow/data/cid/serde/CidEncoder.groovy +++ b/modules/nf-cid/src/main/nextflow/data/cid/serde/CidEncoder.groovy @@ -16,17 +16,16 @@ package nextflow.data.cid.serde - import groovy.transform.CompileStatic -import nextflow.data.cid.model.TaskOutput -import nextflow.data.cid.model.TaskResults +import nextflow.data.cid.model.DataOutput +import nextflow.data.cid.model.TaskOutputs import nextflow.data.cid.model.TaskRun import nextflow.data.cid.model.Workflow -import nextflow.data.cid.model.WorkflowOutput -import nextflow.data.cid.model.WorkflowResults +import nextflow.data.cid.model.WorkflowOutputs import nextflow.data.cid.model.WorkflowRun import nextflow.serde.gson.GsonEncoder import nextflow.serde.gson.RuntimeTypeAdapterFactory + /** * Implements a JSON encoder for CID model objects * @@ -36,16 +35,19 @@ import nextflow.serde.gson.RuntimeTypeAdapterFactory class CidEncoder extends GsonEncoder { CidEncoder() { - withTypeAdapterFactory(RuntimeTypeAdapterFactory.of(CidSerializable.class, "type") + withTypeAdapterFactory(newCidTypeAdapterFactory()) + // enable rendering of null values + withSerializeNulls(true) + } + + static RuntimeTypeAdapterFactory newCidTypeAdapterFactory(){ + RuntimeTypeAdapterFactory.of(CidSerializable.class, "type") .registerSubtype(WorkflowRun, WorkflowRun.simpleName) - .registerSubtype(WorkflowResults, WorkflowResults.simpleName) + .registerSubtype(WorkflowOutputs, WorkflowOutputs.simpleName) .registerSubtype(Workflow, Workflow.simpleName) - .registerSubtype(WorkflowOutput, WorkflowOutput.simpleName) .registerSubtype(TaskRun, TaskRun.simpleName) - .registerSubtype(TaskOutput, TaskOutput.simpleName) - .registerSubtype(TaskResults, TaskResults.simpleName) ) - // enable rendering of null values - withSerializeNulls(true) + .registerSubtype(TaskOutputs, TaskOutputs.simpleName) + .registerSubtype(DataOutput, DataOutput.simpleName) } } diff --git a/modules/nf-cid/src/test/nextflow/data/cid/CidHistoryFileTest.groovy b/modules/nf-cid/src/test/nextflow/data/cid/CidHistoryFileTest.groovy index ac9d3a627a..c8385bf9c4 100644 --- a/modules/nf-cid/src/test/nextflow/data/cid/CidHistoryFileTest.groovy +++ b/modules/nf-cid/src/test/nextflow/data/cid/CidHistoryFileTest.groovy @@ -48,10 +48,9 @@ class CidHistoryFileTest extends Specification { UUID sessionId = UUID.randomUUID() String runName = "TestRun" String runCid = "cid://123" - String resultsCid = "cid://456" when: - cidHistoryFile.write(runName, sessionId, runCid, resultsCid) + cidHistoryFile.write(runName, sessionId, runCid) then: def lines = Files.readAllLines(historyFile) @@ -60,7 +59,6 @@ class CidHistoryFileTest extends Specification { parsedRecord.sessionId == sessionId parsedRecord.runName == runName parsedRecord.runCid == runCid - parsedRecord.resultsCid == resultsCid } def "should return correct record for existing session"() { @@ -68,10 +66,9 @@ class CidHistoryFileTest extends Specification { UUID sessionId = UUID.randomUUID() String runName = "Run1" String runCid = "cid://123" - String resultsCid = "cid://456" and: - cidHistoryFile.write(runName, sessionId, runCid, resultsCid) + cidHistoryFile.write(runName, sessionId, runCid) when: def record = cidHistoryFile.getRecord(sessionId) @@ -79,7 +76,6 @@ class CidHistoryFileTest extends Specification { record.sessionId == sessionId record.runName == runName record.runCid == runCid - record.resultsCid == resultsCid } def "should return null if session does not exist"() { @@ -87,7 +83,7 @@ class CidHistoryFileTest extends Specification { cidHistoryFile.getRecord(UUID.randomUUID()) == null } - def "update should modify existing Cids for given session"() { + def "update should modify existing Cid for given session"() { given: UUID sessionId = UUID.randomUUID() String runName = "Run1" @@ -95,18 +91,16 @@ class CidHistoryFileTest extends Specification { String resultsCidUpdated = "results-cid-updated" and: - cidHistoryFile.write(runName, sessionId, 'run-cid-initial', 'results-cid-inital') + cidHistoryFile.write(runName, sessionId, 'run-cid-initial') when: cidHistoryFile.updateRunCid(sessionId, runCidUpdated) - cidHistoryFile.updateResultsCid(sessionId, resultsCidUpdated) then: def lines = Files.readAllLines(historyFile) lines.size() == 1 def parsedRecord = CidHistoryRecord.parse(lines[0]) parsedRecord.runCid == runCidUpdated - parsedRecord.resultsCid == resultsCidUpdated } def "update should do nothing if session does not exist"() { @@ -115,19 +109,16 @@ class CidHistoryFileTest extends Specification { UUID nonExistingSessionId = UUID.randomUUID() String runName = "Run1" String runCid = "cid://123" - String resultsCid = "cid://456" and: - cidHistoryFile.write(runName, existingSessionId, runCid, resultsCid) + cidHistoryFile.write(runName, existingSessionId, runCid) when: cidHistoryFile.updateRunCid(nonExistingSessionId, "new-cid") - cidHistoryFile.updateRunCid(nonExistingSessionId, "new-res-cid") then: def lines = Files.readAllLines(historyFile) lines.size() == 1 def parsedRecord = CidHistoryRecord.parse(lines[0]) parsedRecord.runCid == runCid - parsedRecord.resultsCid == resultsCid } def 'should get records' () { @@ -135,9 +126,8 @@ class CidHistoryFileTest extends Specification { UUID sessionId = UUID.randomUUID() String runName = "Run1" String runCid = "cid://123" - String resultsCid = "cid://456" and: - cidHistoryFile.write(runName, sessionId, runCid, resultsCid) + cidHistoryFile.write(runName, sessionId, runCid) when: def records = cidHistoryFile.getRecords() @@ -146,7 +136,6 @@ class CidHistoryFileTest extends Specification { records[0].sessionId == sessionId records[0].runName == runName records[0].runCid == runCid - records[0].resultsCid == resultsCid } } diff --git a/modules/nf-cid/src/test/nextflow/data/cid/CidHistoryRecordTest.groovy b/modules/nf-cid/src/test/nextflow/data/cid/CidHistoryRecordTest.groovy index 150e6c8bee..6a104dcafc 100644 --- a/modules/nf-cid/src/test/nextflow/data/cid/CidHistoryRecordTest.groovy +++ b/modules/nf-cid/src/test/nextflow/data/cid/CidHistoryRecordTest.groovy @@ -36,7 +36,7 @@ class CidHistoryRecordTest extends Specification { given: def timestamp = new Date() def formattedTimestamp = CidHistoryRecord.TIMESTAMP_FMT.format(timestamp) - def line = "${formattedTimestamp}\trun-1\t${UUID.randomUUID()}\tcid://123\tcid://456" + def line = "${formattedTimestamp}\trun-1\t${UUID.randomUUID()}\tcid://123" when: def record = CidHistoryRecord.parse(line) @@ -45,19 +45,18 @@ class CidHistoryRecordTest extends Specification { record.timestamp != null record.runName == "run-1" record.runCid == "cid://123" - record.resultsCid == "cid://456" } def "CidRecord toString should produce tab-separated format"() { given: UUID sessionId = UUID.randomUUID() - def record = new CidHistoryRecord(new Date(), "TestRun", sessionId, "cid://123", "cid://456") + def record = new CidHistoryRecord(new Date(), "TestRun", sessionId, "cid://123") when: def line = record.toString() then: line.contains("\t") - line.split("\t").size() == 5 + line.split("\t").size() == 4 } } diff --git a/modules/nf-cid/src/test/nextflow/data/cid/CidObserverTest.groovy b/modules/nf-cid/src/test/nextflow/data/cid/CidObserverTest.groovy index 7003b0439b..54556be919 100644 --- a/modules/nf-cid/src/test/nextflow/data/cid/CidObserverTest.groovy +++ b/modules/nf-cid/src/test/nextflow/data/cid/CidObserverTest.groovy @@ -27,10 +27,9 @@ import com.google.common.hash.HashCode import nextflow.Session import nextflow.data.cid.model.Checksum import nextflow.data.cid.model.DataPath -import nextflow.data.cid.model.TaskOutput +import nextflow.data.cid.model.DataOutput import nextflow.data.cid.model.Workflow -import nextflow.data.cid.model.WorkflowOutput -import nextflow.data.cid.model.WorkflowResults +import nextflow.data.cid.model.WorkflowOutputs import nextflow.data.cid.model.WorkflowRun import nextflow.data.cid.serde.CidEncoder import nextflow.data.config.DataConfig @@ -112,14 +111,17 @@ class CidObserverTest extends Specification { getHash() >> hash getProcessor() >> processor getSource() >> 'echo task source' + getScript() >> 'this is the script' } - def sourceHash =CacheHelper.hasher('echo task source').hash().toString() + def sourceHash = CacheHelper.hasher('echo task source').hash().toString() + def scriptHash = CacheHelper.hasher('this is the script').hash().toString() def normalizer = Mock(PathNormalizer.class) { normalizePath( _ as Path) >> {Path p -> p?.toString()} normalizePath( _ as String) >> {String p -> p} } def taskDescription = new nextflow.data.cid.model.TaskRun(uniqueId.toString(), "foo", new Checksum(sourceHash, "nextflow", "standard"), + new Checksum(scriptHash, "nextflow", "standard"), null, null, null, null, null, [:], [], null ) when: observer.storeTaskRun(task, normalizer) @@ -159,15 +161,15 @@ class CidObserverTest extends Specification { } and: def attrs = Files.readAttributes(outFile, BasicFileAttributes) - def output = new TaskOutput(outFile.toString(), new Checksum(fileHash, "nextflow", "standard"), - "cid://15cd5b07", attrs.size(), CidUtils.toDate(attrs.creationTime()), CidUtils.toDate(attrs.lastModifiedTime()) ) + def output = new DataOutput(outFile.toString(), new Checksum(fileHash, "nextflow", "standard"), + "cid://15cd5b07", "cid://15cd5b07", attrs.size(), CidUtils.toDate(attrs.creationTime()), CidUtils.toDate(attrs.lastModifiedTime()) ) and: observer.readAttributes(outFile) >> attrs when: observer.storeTaskOutput(task, outFile) then: - folder.resolve(".meta/${hash}/foo/bar/file.bam/.data.json").text == new CidEncoder().encode(output) + folder.resolve(".meta/${hash}/outputs/foo/bar/file.bam/.data.json").text == new CidEncoder().encode(output) cleanup: folder?.deleteDir() @@ -325,10 +327,10 @@ class CidObserverTest extends Specification { then: 'check file 1 output metadata in cid store' def attrs1 = Files.readAttributes(outFile1, BasicFileAttributes) def fileHash1 = CacheHelper.hasher(outFile1).hash().toString() - def output1 = new WorkflowOutput(outFile1.toString(), new Checksum(fileHash1, "nextflow", "standard"), "cid://123987/file.bam", - attrs1.size(), CidUtils.toDate(attrs1.creationTime()), CidUtils.toDate(attrs1.lastModifiedTime()) ) - output1.setPublishedBy("$CID_PROT${observer.executionHash}".toString()) - folder.resolve(".meta/${observer.executionHash}/foo/file.bam/.data.json").text == encoder.encode(output1) + def output1 = new DataOutput(outFile1.toString(), new Checksum(fileHash1, "nextflow", "standard"), + "cid://123987/file.bam", "$CID_PROT${observer.executionHash}", + attrs1.size(), CidUtils.toDate(attrs1.creationTime()), CidUtils.toDate(attrs1.lastModifiedTime()) ) + folder.resolve(".meta/${observer.executionHash}/outputs/foo/file.bam/.data.json").text == encoder.encode(output1) when: 'publish without source path' def outFile2 = outputDir.resolve('foo/file2.bam') @@ -339,18 +341,17 @@ class CidObserverTest extends Specification { observer.onFilePublish(outFile2) observer.onWorkflowPublish("b", outFile2) then: 'Check outFile2 metadata in cid store' - def output2 = new WorkflowOutput(outFile2.toString(), new Checksum(fileHash2, "nextflow", "standard"), "cid://${observer.executionHash}" , - attrs2.size(), CidUtils.toDate(attrs2.creationTime()), CidUtils.toDate(attrs2.lastModifiedTime()) ) - output2.setPublishedBy("$CID_PROT${observer.executionHash}".toString()) - folder.resolve(".meta/${observer.executionHash}/foo/file2.bam/.data.json").text == encoder.encode(output2) + def output2 = new DataOutput(outFile2.toString(), new Checksum(fileHash2, "nextflow", "standard"), + "cid://${observer.executionHash}" , "cid://${observer.executionHash}", + attrs2.size(), CidUtils.toDate(attrs2.creationTime()), CidUtils.toDate(attrs2.lastModifiedTime()) ) + folder.resolve(".meta/${observer.executionHash}/outputs/foo/file2.bam/.data.json").text == encoder.encode(output2) when: 'Workflow complete' observer.onFlowComplete() then: 'Check history file is updated and Workflow Result is written in the cid store' - def finalCid = store.getHistoryLog().getRecord(uniqueId).resultsCid.substring(CID_PROT.size()) - finalCid != observer.executionHash - def resultsRetrieved = store.load(finalCid) as WorkflowResults - resultsRetrieved.outputs == [a: "cid://${observer.executionHash}/foo/file.bam", b: "cid://${observer.executionHash}/foo/file2.bam"] + def finalCid = store.getHistoryLog().getRecord(uniqueId).runCid.substring(CID_PROT.size()) + def resultsRetrieved = store.load("${finalCid}/outputs") as WorkflowOutputs + resultsRetrieved.outputs == [a: "cid://${observer.executionHash}/outputs/foo/file.bam", b: "cid://${observer.executionHash}/outputs/foo/file2.bam"] cleanup: folder?.deleteDir() diff --git a/modules/nf-cid/src/test/nextflow/data/cid/CidUtilsTest.groovy b/modules/nf-cid/src/test/nextflow/data/cid/CidUtilsTest.groovy index 88565b01cf..d473297b25 100644 --- a/modules/nf-cid/src/test/nextflow/data/cid/CidUtilsTest.groovy +++ b/modules/nf-cid/src/test/nextflow/data/cid/CidUtilsTest.groovy @@ -33,7 +33,7 @@ class CidUtilsTest extends Specification{ where: FILE_TIME | DATE null | null - FileTime.fromMillis(1234) | Instant.ofEpochMilli(1234).toString() + FileTime.fromMillis(1234) | Instant.ofEpochMilli(1234) } def 'should convert to FileTime'(){ diff --git a/modules/nf-cid/src/test/nextflow/data/cid/DefaultCidStoreTest.groovy b/modules/nf-cid/src/test/nextflow/data/cid/DefaultCidStoreTest.groovy index 5756dcadad..9a7259b908 100644 --- a/modules/nf-cid/src/test/nextflow/data/cid/DefaultCidStoreTest.groovy +++ b/modules/nf-cid/src/test/nextflow/data/cid/DefaultCidStoreTest.groovy @@ -18,9 +18,9 @@ package nextflow.data.cid import nextflow.data.cid.model.DataPath +import nextflow.data.cid.model.DataOutput import nextflow.data.cid.model.Parameter import nextflow.data.cid.model.Workflow -import nextflow.data.cid.model.WorkflowOutput import nextflow.data.cid.model.WorkflowRun import java.nio.file.Files @@ -28,7 +28,6 @@ import java.nio.file.Path import java.time.Instant import nextflow.data.cid.model.Checksum -import nextflow.data.cid.model.TaskOutput import nextflow.data.cid.serde.CidEncoder import nextflow.data.config.DataConfig import spock.lang.Specification @@ -68,7 +67,7 @@ class DefaultCidStoreTest extends Specification { def "save should store value in the correct file location"() { given: def key = "testKey" - def value = new TaskOutput("/path/to/file", new Checksum("hash_value", "hash_algorithm", "standard"), "cid://source", 1234) + def value = new DataOutput("/path/to/file", new Checksum("hash_value", "hash_algorithm", "standard"), "cid://source", "cid://run", 1234) def cidStore = new DefaultCidStore() cidStore.open(config) @@ -84,7 +83,7 @@ class DefaultCidStoreTest extends Specification { def "load should retrieve stored value correctly"() { given: def key = "testKey" - def value = new TaskOutput("/path/to/file", new Checksum("hash_value", "hash_algorithm", "standard"), "cid://source", 1234) + def value = new DataOutput("/path/to/file", new Checksum("hash_value", "hash_algorithm", "standard"), "cid://source", "cid://run", 1234) def cidStore = new DefaultCidStore() cidStore.open(config) cidStore.save(key, value) @@ -105,17 +104,17 @@ class DefaultCidStoreTest extends Specification { def 'should query' () { given: def uniqueId = UUID.randomUUID() - def time = Instant.ofEpochMilli(1234567).toString() + def time = Instant.ofEpochMilli(1234567) def mainScript = new DataPath("file://path/to/main.nf", new Checksum("78910", "nextflow", "standard")) def workflow = new Workflow(mainScript, [],"https://nextflow.io/nf-test/", "123456" ) def key = "testKey" def value1 = new WorkflowRun(workflow, uniqueId.toString(), "test_run", [ new Parameter("String", "param1", "value1"), new Parameter("String", "param2", "value2")] ) def key2 = "testKey2" - def value2 = new WorkflowOutput("/path/tp/file1", new Checksum("78910", "nextflow", "standard"), "testkey", 1234, time, time, [key1:"value1", key2:"value2"]) + def value2 = new DataOutput("/path/tp/file1", new Checksum("78910", "nextflow", "standard"), "testkey", "testkey", 1234, time, time, [key1:"value1", key2:"value2"]) def key3 = "testKey3" - def value3 = new WorkflowOutput("/path/tp/file2", new Checksum("78910", "nextflow", "standard"), "testkey", 1234, time, time, [key2:"value2", key3:"value3"]) + def value3 = new DataOutput("/path/tp/file2", new Checksum("78910", "nextflow", "standard"), "testkey", "testkey", 1234, time, time, [key2:"value2", key3:"value3"]) def key4 = "testKey4" - def value4 = new WorkflowOutput("/path/tp/file", new Checksum("78910", "nextflow", "standard"), "testkey", 1234, time, time, [key3:"value3", key4:"value4"]) + def value4 = new DataOutput("/path/tp/file", new Checksum("78910", "nextflow", "standard"), "testkey", "testkey", 1234, time, time, [key3:"value3", key4:"value4"]) def cidStore = new DefaultCidStore() cidStore.open(config) @@ -125,7 +124,7 @@ class DefaultCidStoreTest extends Specification { cidStore.save(key4, value4) when: - def results3 = cidStore.search("type=WorkflowOutput&annotations.key2=value2") + def results3 = cidStore.search("type=DataOutput&annotations.key2=value2") then: results3.size() == 2 } diff --git a/modules/nf-cid/src/test/nextflow/data/cid/fs/CidFileSystemProviderTest.groovy b/modules/nf-cid/src/test/nextflow/data/cid/fs/CidFileSystemProviderTest.groovy index d03264133d..f7899528b4 100644 --- a/modules/nf-cid/src/test/nextflow/data/cid/fs/CidFileSystemProviderTest.groovy +++ b/modules/nf-cid/src/test/nextflow/data/cid/fs/CidFileSystemProviderTest.groovy @@ -127,7 +127,7 @@ class CidFileSystemProviderTest extends Specification { def output = data.resolve("output.txt") output.text = "Hello, World!" outputMeta.mkdirs() - outputMeta.resolve(".data.json").text = '{"type":"WorkflowOutput","path":"'+output.toString()+'"}' + outputMeta.resolve(".data.json").text = '{"type":"DataOutput","path":"'+output.toString()+'"}' Global.session = Mock(Session) { getConfig()>>config } and: @@ -157,7 +157,7 @@ class CidFileSystemProviderTest extends Specification { def output = data.resolve("output.txt") output.text = "Hello, World!" outputMeta.mkdirs() - outputMeta.resolve(".data.json").text = '{"type":"WorkflowOutput","path":"'+output.toString()+'"}' + outputMeta.resolve(".data.json").text = '{"type":"DataOutput","path":"'+output.toString()+'"}' Global.session = Mock(Session) { getConfig()>>config } and: @@ -198,7 +198,7 @@ class CidFileSystemProviderTest extends Specification { meta.resolve('12345/output1').mkdirs() meta.resolve('12345/output2').mkdirs() meta.resolve('12345/.data.json').text = '{"type":"TaskRun"}' - meta.resolve('12345/output1/.data.json').text = '{"type":"TaskOutput", "path": "' + output1.toString() + '"}' + meta.resolve('12345/output1/.data.json').text = '{"type":"DataOutput", "path": "' + output1.toString() + '"}' and: def config = [workflow:[data:[store:[location:wdir.toString()]]]] @@ -313,7 +313,7 @@ class CidFileSystemProviderTest extends Specification { output.resolve('abc').text = 'file1' output.resolve('.foo').text = 'file2' meta.resolve('12345/output').mkdirs() - meta.resolve('12345/output/.data.json').text = '{"type":"TaskOutput", "path": "' + output.toString() + '"}' + meta.resolve('12345/output/.data.json').text = '{"type":"DataOutput", "path": "' + output.toString() + '"}' and: def provider = new CidFileSystemProvider() def cid1 = provider.getPath(CidPath.asUri('cid://12345/output/abc')) @@ -333,7 +333,7 @@ class CidFileSystemProviderTest extends Specification { def file = data.resolve('abc') file.text = 'Hello' meta.resolve('12345/abc').mkdirs() - meta.resolve('12345/abc/.data.json').text = '{"type":"TaskOutput", "path": "' + file.toString() + '"}' + meta.resolve('12345/abc/.data.json').text = '{"type":"DataOutput", "path": "' + file.toString() + '"}' Global.session = Mock(Session) { getConfig()>>config } and: def provider = new CidFileSystemProvider() diff --git a/modules/nf-cid/src/test/nextflow/data/cid/fs/CidPathTest.groovy b/modules/nf-cid/src/test/nextflow/data/cid/fs/CidPathTest.groovy index 984247e6f1..06bc689e07 100644 --- a/modules/nf-cid/src/test/nextflow/data/cid/fs/CidPathTest.groovy +++ b/modules/nf-cid/src/test/nextflow/data/cid/fs/CidPathTest.groovy @@ -17,10 +17,10 @@ package nextflow.data.cid.fs -import nextflow.data.cid.model.WorkflowResults +import nextflow.data.cid.CidUtils +import nextflow.data.cid.model.WorkflowOutputs import nextflow.data.cid.serde.CidEncoder import nextflow.file.FileHelper -import nextflow.serde.gson.GsonEncoder import java.nio.file.Files import java.time.Instant @@ -116,21 +116,21 @@ class CidPathTest extends Specification { def outputFile = data.resolve('file2.txt') outputFile.text = "this is file2" - def cidFs = new FileHelper().getOrCreateFileSystemFor('cid', [enabled: true, store: [location: cid.parent.toString()]] ) + def cidFs = new FileHelper().getOrCreateFileSystemFor('cid', [enabled: true, store: [location: cid.parent.toString()]] ) as CidFileSystem cid.resolve('12345/output1').mkdirs() cid.resolve('12345/path/to/file2.txt').mkdirs() cid.resolve('12345/.data.json').text = '{"type":"TaskRun"}' - cid.resolve('12345/output1/.data.json').text = '{"type":"TaskOutput", "path": "' + outputFolder.toString() + '"}' - cid.resolve('12345/path/to/file2.txt/.data.json').text = '{"type":"TaskOutput", "path": "' + outputFile.toString() + '"}' - def time = Instant.now().toString() - def wfResultsMetadata = new CidEncoder().withPrettyPrint(true).encode(new WorkflowResults(time, "cid://1234", [a: "cid://1234/a.txt"])) + cid.resolve('12345/output1/.data.json').text = '{"type":"DataOutput", "path": "' + outputFolder.toString() + '"}' + cid.resolve('12345/path/to/file2.txt/.data.json').text = '{"type":"DataOutput", "path": "' + outputFile.toString() + '"}' + def time = Instant.now() + def wfResultsMetadata = new CidEncoder().withPrettyPrint(true).encode(new WorkflowOutputs(time, "cid://1234", [a: "cid://1234/a.txt"])) cid.resolve('5678/').mkdirs() cid.resolve('5678/.data.json').text = wfResultsMetadata expect: 'Get real path when CidPath is the output data or a subfolder' - new CidPath(cidFs,'12345/output1' ).getTargetPath() == outputFolder - new CidPath(cidFs,'12345/output1/some/path' ).getTargetPath() == outputSubFolder + new CidPath(cidFs, '12345/output1').getTargetPath() == outputFolder + new CidPath(cidFs,'12345/output1/some/path').getTargetPath() == outputSubFolder new CidPath(cidFs,'12345/output1/some/path/file1.txt').getTargetPath().text == outputSubFolderFile.text new CidPath(cidFs, '12345/path/to/file2.txt').getTargetPath().text == outputFile.text @@ -162,14 +162,14 @@ class CidPathTest extends Specification { when: 'Cid description' def result = new CidPath(cidFs, '5678').getTargetPath(true) then: - result instanceof CidResultsPath + result instanceof CidMetadataPath result.text == wfResultsMetadata when: 'Cid description subobject' def result2 = new CidPath(cidFs, '5678#outputs').getTargetPath(true) then: - result2 instanceof CidResultsPath - result2.text == new GsonEncoder(){}.withPrettyPrint(true).encode([a: "cid://1234/a.txt"]) + result2 instanceof CidMetadataPath + result2.text == CidUtils.encodeSearchOutputs([a: "cid://1234/a.txt"], true) when: 'Cid subobject does not exist' new CidPath(cidFs, '23456#notexists').getTargetPath(true) diff --git a/modules/nf-cid/src/test/nextflow/data/cid/serde/CidEncoderTest.groovy b/modules/nf-cid/src/test/nextflow/data/cid/serde/CidEncoderTest.groovy index 5dd46575b5..e450090754 100644 --- a/modules/nf-cid/src/test/nextflow/data/cid/serde/CidEncoderTest.groovy +++ b/modules/nf-cid/src/test/nextflow/data/cid/serde/CidEncoderTest.groovy @@ -2,12 +2,12 @@ package nextflow.data.cid.serde import nextflow.data.cid.model.Checksum import nextflow.data.cid.model.DataPath -import nextflow.data.cid.model.Output import nextflow.data.cid.model.Parameter -import nextflow.data.cid.model.TaskOutput +import nextflow.data.cid.model.DataOutput +import nextflow.data.cid.model.TaskOutputs import nextflow.data.cid.model.TaskRun import nextflow.data.cid.model.Workflow -import nextflow.data.cid.model.WorkflowResults +import nextflow.data.cid.model.WorkflowOutputs import nextflow.data.cid.model.WorkflowRun import spock.lang.Specification @@ -19,15 +19,15 @@ class CidEncoderTest extends Specification{ given: def encoder = new CidEncoder() and: - def output = new TaskOutput("/path/to/file", new Checksum("hash_value", "hash_algorithm", "standard"), "cid://source", 1234) + def output = new DataOutput("/path/to/file", new Checksum("hash_value", "hash_algorithm", "standard"), "cid://source", "cid://run", 1234) when: def encoded = encoder.encode(output) def object = encoder.decode(encoded) then: - object instanceof Output - def result = object as Output + object instanceof DataOutput + def result = object as DataOutput result.path == "/path/to/file" result.checksum instanceof Checksum result.checksum.value == "hash_value" @@ -70,17 +70,17 @@ class CidEncoderTest extends Specification{ given: def encoder = new CidEncoder() and: - def time = Instant.now().toString() - def wfResults = new WorkflowResults(time, "cid://1234", [a: "A", b: "B"]) + def time = Instant.now() + def wfResults = new WorkflowOutputs(time, "cid://1234", [a: "A", b: "B"]) when: def encoded = encoder.encode(wfResults) def object = encoder.decode(encoded) then: - object instanceof WorkflowResults - def result = object as WorkflowResults - result.creationTime == time - result.runId == "cid://1234" + object instanceof WorkflowOutputs + def result = object as WorkflowOutputs + result.createdAt == time + result.workflowRun == "cid://1234" result.outputs == [a: "A", b: "B"] } @@ -90,7 +90,7 @@ class CidEncoderTest extends Specification{ and: def uniqueId = UUID.randomUUID() def taskRun = new TaskRun( - uniqueId.toString(),"name", new Checksum("78910", "nextflow", "standard"), + uniqueId.toString(),"name", new Checksum("78910", "nextflow", "standard"), new Checksum("74517", "nextflow", "standard"), [new Parameter("String", "param1", "value1")], "container:version", "conda", "spack", "amd64", [a: "A", b: "B"], [new DataPath("path/to/file", new Checksum("78910", "nextflow", "standard"))] ) @@ -103,6 +103,7 @@ class CidEncoderTest extends Specification{ result.sessionId == uniqueId.toString() result.name == "name" result.codeChecksum.value == "78910" + result.scriptChecksum.value == "74517" result.inputs.size() == 1 result.inputs.get(0).name == "param1" result.container == "container:version" @@ -119,32 +120,35 @@ class CidEncoderTest extends Specification{ given: def encoder = new CidEncoder() and: - def time = Instant.now().toString() - def wfResults = new WorkflowResults(time, "cid://1234", [a: "A", b: "B"]) + def time = Instant.now() + def parameter = new Parameter("a","b", "c") + def wfResults = new TaskOutputs("cid://1234", "cid://5678", time, [parameter], null) when: def encoded = encoder.encode(wfResults) def object = encoder.decode(encoded) then: - object instanceof WorkflowResults - def result = object as WorkflowResults - result.creationTime == time - result.runId == "cid://1234" - result.outputs == [a: "A", b: "B"] + object instanceof TaskOutputs + def result = object as TaskOutputs + result.createdAt == time + result.taskRun == "cid://1234" + result.workflowRun == "cid://5678" + result.outputs.size() == 1 + result.outputs[0] == parameter } def 'object with null date attributes' () { given: def encoder = new CidEncoder() and: - def wfResults = new WorkflowResults(null, "cid://1234") + def wfResults = new WorkflowOutputs(null, "cid://1234") when: def encoded = encoder.encode(wfResults) def object = encoder.decode(encoded) then: - encoded == '{"type":"WorkflowResults","creationTime":null,"runId":"cid://1234","outputs":null}' - def result = object as WorkflowResults - result.creationTime == null + encoded == '{"type":"WorkflowOutputs","createdAt":null,"workflowRun":"cid://1234","outputs":null,"annotations":null}' + def result = object as WorkflowOutputs + result.createdAt == null } } diff --git a/modules/nf-commons/src/main/nextflow/serde/gson/InstantAdapter.groovy b/modules/nf-commons/src/main/nextflow/serde/gson/InstantAdapter.groovy index 5a09bec52a..cfed8e5c69 100644 --- a/modules/nf-commons/src/main/nextflow/serde/gson/InstantAdapter.groovy +++ b/modules/nf-commons/src/main/nextflow/serde/gson/InstantAdapter.groovy @@ -16,6 +16,8 @@ package nextflow.serde.gson +import com.google.gson.stream.JsonToken + import java.time.Instant import com.google.gson.TypeAdapter @@ -37,6 +39,10 @@ class InstantAdapter extends TypeAdapter { @Override Instant read(JsonReader reader) throws IOException { + if (reader.peek() == JsonToken.NULL) { + reader.nextNull(); + return null; + } return Instant.parse(reader.nextString()) } } diff --git a/plugins/nf-cid-h2/src/main/nextflow/data/cid/h2/H2CidHistoryLog.groovy b/plugins/nf-cid-h2/src/main/nextflow/data/cid/h2/H2CidHistoryLog.groovy index e112f0daea..56aad878ea 100644 --- a/plugins/nf-cid-h2/src/main/nextflow/data/cid/h2/H2CidHistoryLog.groovy +++ b/plugins/nf-cid-h2/src/main/nextflow/data/cid/h2/H2CidHistoryLog.groovy @@ -43,14 +43,14 @@ class H2CidHistoryLog implements CidHistoryLog { } @Override - void write(String name, UUID sessionId, String runCid, String resultsCid) { + void write(String name, UUID sessionId, String runCid) { try(final sql=new Sql(dataSource)) { def query = """ - INSERT INTO cid_history_record (timestamp, run_name, session_id, run_cid, results_cid) - VALUES (?, ?, ?, ?, ?) + INSERT INTO cid_history_record (timestamp, run_name, session_id, run_cid) + VALUES (?, ?, ?, ?) """ def timestamp = new Timestamp(System.currentTimeMillis()) // Current timestamp - sql.executeInsert(query, List.of(timestamp, name, sessionId.toString(), runCid, resultsCid)) + sql.executeInsert(query, List.of(timestamp, name, sessionId.toString(), runCid)) } } @@ -73,25 +73,6 @@ class H2CidHistoryLog implements CidHistoryLog { } } - @Override - void updateResultsCid(UUID sessionId, String resultsCid) { - try(final sql=new Sql(dataSource)) { - def query = """ - UPDATE cid_history_record - SET results_cid = ? - WHERE session_id = ? - """ - - final count = sql.executeUpdate(query, List.of(resultsCid, sessionId.toString())) - if (count > 0) { - log.debug "Successfully updated run_cid for session_id: $sessionId" - } - else { - log.warn "No record found with session_id: $sessionId" - } - } - } - @Override List getRecords() { try(final sql=new Sql(dataSource)) { @@ -105,7 +86,6 @@ class H2CidHistoryLog implements CidHistoryLog { row.run_name as String, UUID.fromString(row.session_id as String), row.run_cid as String, - row.results_cid as String ) ) } @@ -125,7 +105,6 @@ class H2CidHistoryLog implements CidHistoryLog { row.run_name as String, UUID.fromString(row.session_id as String), row.run_cid as String, - row.results_cid as String ) } } diff --git a/plugins/nf-cid-h2/src/main/nextflow/data/cid/h2/H2CidStore.groovy b/plugins/nf-cid-h2/src/main/nextflow/data/cid/h2/H2CidStore.groovy index d29f1dcd48..5856a10839 100644 --- a/plugins/nf-cid-h2/src/main/nextflow/data/cid/h2/H2CidStore.groovy +++ b/plugins/nf-cid-h2/src/main/nextflow/data/cid/h2/H2CidStore.groovy @@ -89,7 +89,6 @@ class H2CidStore implements CidStore { run_name VARCHAR(255) NOT NULL, session_id UUID NOT NULL, run_cid VARCHAR(255) NOT NULL, - results_cid VARCHAR(255) NOT NULL, UNIQUE (run_name, session_id) -- Enforce uniqueness constraint ); ''') diff --git a/plugins/nf-cid-h2/src/test/nextflow/data/cid/h2/H2CidHistoryLogTest.groovy b/plugins/nf-cid-h2/src/test/nextflow/data/cid/h2/H2CidHistoryLogTest.groovy index 43bd6f527e..e8878bf2fc 100644 --- a/plugins/nf-cid-h2/src/test/nextflow/data/cid/h2/H2CidHistoryLogTest.groovy +++ b/plugins/nf-cid-h2/src/test/nextflow/data/cid/h2/H2CidHistoryLogTest.groovy @@ -49,7 +49,7 @@ class H2CidHistoryLogTest extends Specification { def log = store.getHistoryLog() def uuid = UUID.randomUUID() when: - log.write('foo', uuid, '1234', '4321') + log.write('foo', uuid, '1234') then: noExceptionThrown() @@ -59,7 +59,6 @@ class H2CidHistoryLogTest extends Specification { rec.runName == 'foo' rec.sessionId == uuid rec.runCid == '1234' - rec.resultsCid == '4321' } def 'should update run cid' () { @@ -67,7 +66,7 @@ class H2CidHistoryLogTest extends Specification { def log = store.getHistoryLog() def uuid = UUID.randomUUID() when: - log.write('foo', uuid, '1234', '4321') + log.write('foo', uuid, '1234') then: noExceptionThrown() @@ -82,30 +81,6 @@ class H2CidHistoryLogTest extends Specification { rec.runName == 'foo' rec.sessionId == uuid rec.runCid == '4444' - rec.resultsCid == '4321' - } - - def 'should update results cid' () { - given: - def log = store.getHistoryLog() - def uuid = UUID.randomUUID() - when: - log.write('foo', uuid, '1234', '4321') - then: - noExceptionThrown() - - when: - log.updateResultsCid(uuid, '5555') - then: - noExceptionThrown() - - when: - def rec = log.getRecord(uuid) - then: - rec.runName == 'foo' - rec.sessionId == uuid - rec.runCid == '1234' - rec.resultsCid == '5555' } def 'should update get records' () { @@ -115,9 +90,9 @@ class H2CidHistoryLogTest extends Specification { def uuid2 = UUID.randomUUID() def uuid3 = UUID.randomUUID() when: - log.write('foo1', uuid1, '1', '11') - log.write('foo2', uuid2, '2', '22') - log.write('foo3', uuid3, '3', '33') + log.write('foo1', uuid1, '1') + log.write('foo2', uuid2, '2') + log.write('foo3', uuid3, '3') then: noExceptionThrown() @@ -129,17 +104,14 @@ class H2CidHistoryLogTest extends Specification { all[0].runName == 'foo1' all[0].sessionId == uuid1 all[0].runCid == '1' - all[0].resultsCid == '11' and: all[1].runName == 'foo2' all[1].sessionId == uuid2 all[1].runCid == '2' - all[1].resultsCid == '22' and: all[2].runName == 'foo3' all[2].sessionId == uuid3 all[2].runCid == '3' - all[2].resultsCid == '33' } } diff --git a/plugins/nf-cid-h2/src/test/nextflow/data/cid/h2/H2CidStoreTest.groovy b/plugins/nf-cid-h2/src/test/nextflow/data/cid/h2/H2CidStoreTest.groovy index a6c67b91ae..e2b6ee3dda 100644 --- a/plugins/nf-cid-h2/src/test/nextflow/data/cid/h2/H2CidStoreTest.groovy +++ b/plugins/nf-cid-h2/src/test/nextflow/data/cid/h2/H2CidStoreTest.groovy @@ -19,10 +19,9 @@ package nextflow.data.cid.h2 import nextflow.data.cid.model.Checksum import nextflow.data.cid.model.DataPath +import nextflow.data.cid.model.DataOutput import nextflow.data.cid.model.Parameter -import nextflow.data.cid.model.TaskOutput import nextflow.data.cid.model.Workflow -import nextflow.data.cid.model.WorkflowOutput import nextflow.data.cid.model.WorkflowRun import nextflow.data.config.DataConfig import spock.lang.Shared @@ -51,7 +50,7 @@ class H2CidStoreTest extends Specification { def 'should store and get a value' () { given: - def value = new TaskOutput("/path/to/file", new Checksum("hash_value", "hash_algorithm", "standard"), "cid://source", 1234) + def value = new DataOutput("/path/to/file", new Checksum("hash_value", "hash_algorithm", "standard"), "cid://source", "cid://run", 1234) when: store.save('/some/key', value) then: @@ -63,22 +62,22 @@ class H2CidStoreTest extends Specification { def uniqueId = UUID.randomUUID() def mainScript = new DataPath("file://path/to/main.nf", new Checksum("78910", "nextflow", "standard")) def workflow = new Workflow(mainScript, [], "https://nextflow.io/nf-test/", "123456") - def time = Instant.ofEpochMilli(1234567).toString() + def time = Instant.ofEpochMilli(1234567) def key = "testKey" def value1 = new WorkflowRun(workflow, uniqueId.toString(), "test_run", [new Parameter("String", "param1", "value1"), new Parameter("String", "param2", "value2")]) def key2 = "testKey2" - def value2 = new WorkflowOutput("/path/tp/file1", new Checksum("78910", "nextflow", "standard"), "testkey", 1234, time, time, [key1: "value1", key2: "value2"]) + def value2 = new DataOutput("/path/tp/file1", new Checksum("78910", "nextflow", "standard"), "testkey", "cid://run", 1234, time, time, [key1: "value1", key2: "value2"]) def key3 = "testKey3" - def value3 = new WorkflowOutput("/path/tp/file2", new Checksum("78910", "nextflow", "standard"), "testkey", 1234, time, time, [key2: "value2", key3: "value3"]) + def value3 = new DataOutput("/path/tp/file2", new Checksum("78910", "nextflow", "standard"), "testkey", "cid://run", 1234, time, time, [key2: "value2", key3: "value3"]) def key4 = "testKey4" - def value4 = new WorkflowOutput("/path/tp/file", new Checksum("78910", "nextflow", "standard"), "testkey", 1234, time, time, [key3: "value3", key4: "value4"]) + def value4 = new DataOutput("/path/tp/file", new Checksum("78910", "nextflow", "standard"), "testkey", "cid://run", 1234, time, time, [key3: "value3", key4: "value4"]) store.save(key, value1) store.save(key2, value2) store.save(key3, value3) store.save(key4, value4) when: - def results = store.search("type=WorkflowOutput&annotations.key2=value2") + def results = store.search("type=DataOutput&annotations.key2=value2") then: results.size() == 2 } From ee7506f4d32ae828506f9d82b246851cd7c184ec Mon Sep 17 00:00:00 2001 From: jorgee Date: Fri, 4 Apr 2025 17:52:42 +0200 Subject: [PATCH 2/5] fix outputs bug and add workflowRun Signed-off-by: jorgee --- modules/nf-cid/src/main/nextflow/data/cid/CidObserver.groovy | 5 +++-- .../nf-cid/src/main/nextflow/data/cid/model/TaskRun.groovy | 1 + .../nf-cid/src/test/nextflow/data/cid/CidObserverTest.groovy | 5 +++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/modules/nf-cid/src/main/nextflow/data/cid/CidObserver.groovy b/modules/nf-cid/src/main/nextflow/data/cid/CidObserver.groovy index cd1a70dcbe..a93c4cc19a 100644 --- a/modules/nf-cid/src/main/nextflow/data/cid/CidObserver.groovy +++ b/modules/nf-cid/src/main/nextflow/data/cid/CidObserver.groovy @@ -225,7 +225,8 @@ class CidObserver implements TraceObserver { normalizer.normalizePath(p.normalize()), new Checksum(CacheHelper.hasher(p).hash().toString(), "nextflow", CacheHelper.HashMode.DEFAULT().toString().toLowerCase()) ) - } + }, + asUriString(executionHash) ) // store in the underlying persistence @@ -336,7 +337,7 @@ class CidObserver implements TraceObserver { final hash = FileHelper.getTaskHashFromPath(source, session.workDir) if (hash) { final target = FileHelper.getWorkFolder(session.workDir, hash).relativize(source).toString() - return asUriString(hash.toString(), target) + return asUriString(hash.toString(), 'outputs', target) } final storeDirReference = outputsStoreDirCid.get(source.toString()) return storeDirReference ? asUriString(storeDirReference) : null diff --git a/modules/nf-cid/src/main/nextflow/data/cid/model/TaskRun.groovy b/modules/nf-cid/src/main/nextflow/data/cid/model/TaskRun.groovy index e45356461b..6711d9616c 100644 --- a/modules/nf-cid/src/main/nextflow/data/cid/model/TaskRun.groovy +++ b/modules/nf-cid/src/main/nextflow/data/cid/model/TaskRun.groovy @@ -40,5 +40,6 @@ class TaskRun implements CidSerializable { String architecture Map globalVars List binEntries + String workflowRun Map annotations } diff --git a/modules/nf-cid/src/test/nextflow/data/cid/CidObserverTest.groovy b/modules/nf-cid/src/test/nextflow/data/cid/CidObserverTest.groovy index 54556be919..699700a6bb 100644 --- a/modules/nf-cid/src/test/nextflow/data/cid/CidObserverTest.groovy +++ b/modules/nf-cid/src/test/nextflow/data/cid/CidObserverTest.groovy @@ -98,6 +98,7 @@ class CidObserverTest extends Specification { } store.open(DataConfig.create(session)) def observer = new CidObserver(session, store) + observer.executionHash = "hash" and: def hash = HashCode.fromInt(123456789) and: @@ -122,7 +123,7 @@ class CidObserverTest extends Specification { def taskDescription = new nextflow.data.cid.model.TaskRun(uniqueId.toString(), "foo", new Checksum(sourceHash, "nextflow", "standard"), new Checksum(scriptHash, "nextflow", "standard"), - null, null, null, null, null, [:], [], null ) + null, null, null, null, null, [:], [], "cid://hash", null) when: observer.storeTaskRun(task, normalizer) then: @@ -328,7 +329,7 @@ class CidObserverTest extends Specification { def attrs1 = Files.readAttributes(outFile1, BasicFileAttributes) def fileHash1 = CacheHelper.hasher(outFile1).hash().toString() def output1 = new DataOutput(outFile1.toString(), new Checksum(fileHash1, "nextflow", "standard"), - "cid://123987/file.bam", "$CID_PROT${observer.executionHash}", + "cid://123987/outputs/file.bam", "$CID_PROT${observer.executionHash}", attrs1.size(), CidUtils.toDate(attrs1.creationTime()), CidUtils.toDate(attrs1.lastModifiedTime()) ) folder.resolve(".meta/${observer.executionHash}/outputs/foo/file.bam/.data.json").text == encoder.encode(output1) From c3ced0db74475f1ab93a882775af144742ec5852 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sat, 5 Apr 2025 16:48:08 +0200 Subject: [PATCH 3/5] Render cid usage on missing sub-command [ci fast] Signed-off-by: Paolo Di Tommaso --- modules/nextflow/src/main/groovy/nextflow/cli/CmdCid.groovy | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdCid.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CmdCid.groovy index dfa47d4347..48c384bd23 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdCid.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/CmdCid.groovy @@ -77,6 +77,7 @@ class CmdCid extends CmdBase implements UsageAware { @Override void run() { if( !args ) { + usage(List.of()) return } // setup the plugins system and load the secrets provider From e8538a7739d642763c3195284ed28dd6fbbcd332 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sat, 5 Apr 2025 16:51:26 +0200 Subject: [PATCH 4/5] Fix warning message [ci fast] Signed-off-by: Paolo Di Tommaso --- .../src/main/nextflow/data/cid/CidHistoryFile.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/nf-cid/src/main/nextflow/data/cid/CidHistoryFile.groovy b/modules/nf-cid/src/main/nextflow/data/cid/CidHistoryFile.groovy index 43594bcb38..01de195323 100644 --- a/modules/nf-cid/src/main/nextflow/data/cid/CidHistoryFile.groovy +++ b/modules/nf-cid/src/main/nextflow/data/cid/CidHistoryFile.groovy @@ -16,14 +16,14 @@ */ package nextflow.data.cid -import groovy.util.logging.Slf4j - import java.nio.channels.FileChannel import java.nio.channels.FileLock import java.nio.file.Files import java.nio.file.Path import java.nio.file.StandardOpenOption +import groovy.util.logging.Slf4j +import nextflow.extension.FilesEx /** * File to store a history of the workflow executions and their corresponding CIDs * @@ -101,7 +101,7 @@ class CidHistoryFile implements CidHistoryLog { } } catch (IllegalArgumentException e) { - log.warn("Can't read CID history file: $this", e.message) + log.warn("Can't read CID history file: ${FilesEx.toUriString(this.path)}", e.message) } } @@ -165,4 +165,4 @@ class CidHistoryFile implements CidHistoryLog { file.delete() } } -} \ No newline at end of file +} From 1cc9d7d2cc954b3dbc4755edb23195310f760b54 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sat, 5 Apr 2025 16:55:17 +0200 Subject: [PATCH 5/5] Improve cid history logs [ci fast] Signed-off-by: Paolo Di Tommaso --- .../nextflow/data/cid/CidHistoryFile.groovy | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/modules/nf-cid/src/main/nextflow/data/cid/CidHistoryFile.groovy b/modules/nf-cid/src/main/nextflow/data/cid/CidHistoryFile.groovy index 01de195323..cfac30f820 100644 --- a/modules/nf-cid/src/main/nextflow/data/cid/CidHistoryFile.groovy +++ b/modules/nf-cid/src/main/nextflow/data/cid/CidHistoryFile.groovy @@ -22,6 +22,7 @@ import java.nio.file.Files import java.nio.file.Path import java.nio.file.StandardOpenOption +import groovy.transform.CompileStatic import groovy.util.logging.Slf4j import nextflow.extension.FilesEx /** @@ -30,6 +31,7 @@ import nextflow.extension.FilesEx * @author Jorge Ejarque */ @Slf4j +@CompileStatic class CidHistoryFile implements CidHistoryLog { Path path @@ -43,7 +45,7 @@ class CidHistoryFile implements CidHistoryLog { withFileLock { def timestamp = date ?: new Date() - log.trace("Writting record for $key in CID history file $this") + log.trace("Writting record for $key in CID history file ${FilesEx.toUriString(this.path)}") path << new CidHistoryRecord(timestamp, name, key, runCid).toString() << '\n' } } @@ -55,7 +57,7 @@ class CidHistoryFile implements CidHistoryLog { withFileLock { updateRunCid0(sessionId, runCid) } } catch (Throwable e) { - log.warn "Can't update CID history file: $this", e.message + log.warn "Can't update CID history file: ${FilesEx.toUriString(this.path)}", e.message } } @@ -65,7 +67,7 @@ class CidHistoryFile implements CidHistoryLog { withFileLock { this.path.eachLine {list.add(CidHistoryRecord.parse(it)) } } } catch (Throwable e) { - log.warn "Can't read records from CID history file: $this", e.message + log.warn "Can't read records from CID history file: ${FilesEx.toUriString(this.path)}", e.message } return list } @@ -80,7 +82,7 @@ class CidHistoryFile implements CidHistoryLog { return current } } - log.warn("Can't find session $id in CID history file $this") + log.warn("Can't find session $id in CID history file ${FilesEx.toUriString(this.path)}") return null } @@ -89,11 +91,11 @@ class CidHistoryFile implements CidHistoryLog { assert id def newHistory = new StringBuilder() - this.path.readLines().each { line -> + for( String line : this.path.readLines()) { try { def current = line ? CidHistoryRecord.parse(line) : null if (current.sessionId == id) { - log.trace("Updating record for $id in CID history file $this") + log.trace("Updating record for $id in CID history file ${FilesEx.toUriString(this.path)}") final newRecord = new CidHistoryRecord(current.timestamp, current.runName, current.sessionId, runCid) newHistory << newRecord.toString() << '\n' } else { @@ -126,11 +128,11 @@ class CidHistoryFile implements CidHistoryLog { try { fos = FileChannel.open(file, StandardOpenOption.WRITE, StandardOpenOption.CREATE) } catch (UnsupportedOperationException e){ - log.warn("File System Provider for ${this.path} do not support file locking. Continuing without lock...") + log.warn("File System Provider for ${this.path} do not support file locking - Attemting without locking", e) return action.call() } if (!fos){ - throw new IllegalStateException("Can't create a file channel for ${this.path.toAbsolutePath()}") + throw new IllegalStateException("Can't create a file channel for ${FilesEx.toUriString(this.path)}") } try { Throwable error @@ -143,7 +145,7 @@ class CidHistoryFile implements CidHistoryLog { if (System.currentTimeMillis() - ts < 1_000) sleep rnd.nextInt(75) else { - error = new IllegalStateException("Can't lock file: ${this.path.toAbsolutePath()} -- Nextflow needs to run in a file system that supports file locks") + error = new IllegalStateException("Can't lock file: ${FilesEx.toUriString(this.path)} - Nextflow needs to run in a file system that supports file locks") break } }