Merge branch 'master' into improve-progress-logging

Signed-off-by: Ben Sherman <[email protected]>
nextflow-io · Feb 12, 2024 · 112edde · 112edde
2 parents d082496 + 5e2ce9e
commit 112edde
Show file tree

Hide file tree

Showing 1,152 changed files with 3,753 additions and 1,957 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -12,6 +12,7 @@ on:
       - 'master'
       - 'test*'
       - 'dev*'
+      - 'STABLE-*'
   pull_request:
     types: [opened, reopened, synchronize]
   workflow_dispatch:
@@ -31,14 +32,14 @@ jobs:
         run: env | sort
 
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
           submodules: true
 
       - name: Get changed files
         id: changed-files
-        uses: tj-actions/changed-files@v35
+        uses: tj-actions/changed-files@v41
         with:
           files_ignore: docs
 
@@ -102,7 +103,7 @@ jobs:
         test_mode: ["test_integration", "test_docs", "test_aws", "test_azure", "test_google", "test_wave"]
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 1
           submodules: true

diff --git a/.github/workflows/cffconvert.yml b/.github/workflows/cffconvert.yml
@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out a copy of the repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Check whether the citation metadata from CITATION.cff is valid
         uses: citation-file-format/[email protected]

diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml
@@ -17,6 +17,6 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Codespell
         uses: codespell-project/actions-codespell@v2
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -10,7 +10,7 @@ jobs:
     name: Build
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
           python-version: '3.9'

diff --git a/Makefile b/Makefile
@@ -1,5 +1,5 @@
 #
-#  Copyright 2013-2023, Seqera Labs
+#  Copyright 2013-2024, Seqera Labs
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/NOTICE b/NOTICE
@@ -1,6 +1,6 @@
 NEXTFLOW
 
-Copyright 2013-2023, Seqera Labs
+Copyright 2013-2024, Seqera Labs
 
 
 This software includes source code and libraries developed by:
@@ -142,7 +142,7 @@ This software includes source code and libraries developed by:
 
     Spack
     https://github.com/spack/spack
-    Copyright 2013-2023 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details.
+    Copyright 2013-2024 Lawrence Livermore National Security, LLC and other Spack Project Developers. See the top-level COPYRIGHT file for details.
     Licensed under Apache License, Version 2.0
 
 Nextflow is a registered trademark of Seqera Labs, Spain.

diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-23.11.0-edge
+24.01.0-edge
diff --git a/build.gradle b/build.gradle
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -32,7 +32,7 @@ if (groovyVer) {
 
         configurations.all {
             resolutionStrategy.eachDependency { DependencyResolveDetails details ->
-                if (details.requested.group == 'org.codehaus.groovy') {
+                if (details.requested.group == 'org.apache.groovy') {
                     if( groovyVer.contains(':') )
                         details.useTarget(groovyVer)
                     else
@@ -95,8 +95,8 @@ allprojects {
 
     configurations {
         // see https://docs.gradle.org/4.1/userguide/dependency_management.html#sub:exclude_transitive_dependencies
-        all*.exclude group: 'org.codehaus.groovy', module: 'groovy-all'
-        all*.exclude group: 'org.codehaus.groovy', module: 'groovy-cli-picocli'
+        all*.exclude group: 'org.apache.groovy', module: 'groovy-all'
+        all*.exclude group: 'org.apache.groovy', module: 'groovy-cli-picocli'
         // groovydoc libs
         groovyDoc.extendsFrom runtime
     }
@@ -108,9 +108,9 @@ allprojects {
         }
 
         // Documentation required libraries
-        groovyDoc 'org.fusesource.jansi:jansi:1.11'
-        groovyDoc "org.codehaus.groovy:groovy-groovydoc:3.0.19"
-        groovyDoc "org.codehaus.groovy:groovy-ant:3.0.19"
+        groovyDoc 'org.fusesource.jansi:jansi:2.4.0'
+        groovyDoc "org.apache.groovy:groovy-groovydoc:4.0.18"
+        groovyDoc "org.apache.groovy:groovy-ant:4.0.18"
     }
 
     test {

diff --git a/buildSrc/src/main/groovy/io/nextflow/gradle/tasks/GithubUploader.groovy b/buildSrc/src/main/groovy/io/nextflow/gradle/tasks/GithubUploader.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

diff --git a/changelog.txt b/changelog.txt
@@ -1,5 +1,85 @@
 NEXTFLOW CHANGE-LOG
 ===================
+24.01.0-edge - 5 Feb 2024
+- Add support for custom fuse device plugin (#4612) [a1e33193]
+- Fix Ignore stored process message when ansi log is enabled (#4645) [f9ba47ef]
+- Fix Wave container replicable checksum [da382ddf]
+- Fix azure retry policy (#4638) [85bab699]
+- Fix handling of wave.s5cmdConfigUrl setting (#4707) [3a19386d]
+- Fix typo in Hyperqueue section (#4635) [d5a6a963]
+- Fix typo in OCI mode warning message (#4633) [e216a876]
+- Fix typo in the docs (#4636) [ci-skip] [4727f174]
+- Improve detection of sys home variable [8812138f]
+- Improve handling of publish error (#4703) [fc9f7685]
+- LocalSecretsProvider: invalid permissions tests made more robust (#4663) [682a6806]
+- Remove Glacier auto-retrieval (#4705) [5f0ec50d]
+- Remove preview from cmd secret [dc040226]
+- Set scratch default to false when using Fusion (#4675) [9d17e407]
+- Slight change in wording for error message about userEmulation (#4680) [a9c8a841]
+- Truncate max size exceeded error message (#4644) [36090b70]
+- Use AZURE_STORAGE_SAS_TOKEN environment variable (#4627) [2e02afbf]
+- Use canonical init method for plugins initialization [e7a20aff]
+- Bump [email protected] [8b9905c7]
+- Bump [email protected] [bfc94e46]
+- Bump [email protected] [01ff0353]
+- Bump [email protected] [3df34188]
+- Bump [email protected] [e90d73a7]
+- Bump [email protected] [4379c941]
+- Bump [email protected] [2bc698c7]
+- Bump [email protected] [07415ce1]
+- Bump [email protected] [b991e14b]
+- Bump Groovy 4 (#4443) [ci fast] [9d32503b]
+- Bump actions/checkout@v4 [d1b3195e]
+- Bump [email protected] + [email protected] [331ff425]
+- Bump nextflow 23.12.0-edge as min version [63e83702]
+- Bump tj-actions/changed-files from 35 to 41 in /.github/workflows (#4622) [d67aef5f]
+
+23.10.1 - 12 Jan 2023 
+- Fix bug with Fusion symlink resolution (#4593) [f28c9e48]
+- Fix Fusion symlinks when publishing files (#4348) [1fa5878a]
+- Fix Inspect command fails with Singularity [25883df3]
+- Fix Allow the use of error built-in function in onComplete handler (#4458) [4be10cd3]
+- Fix Harden regular expression to used to strip secrets in logs (#4563) [0102d4d6]
+- Fix custom notification template [40980bcb]
+- Fix container environment with special chars (#4594) [f4e00601]
+- Fix AZURE_STORAGE_SAS_TOKEN environment variable (#4627) [2e1cb413]
+- Fix azure retry policy (#4638) [2bc3cf0e]
+- Fix BitBucket get source API with custom branch [5ef75e32]
+- Improve error details for AbortOperationException [9e795a62]
+- Bump [email protected] [206f2614]
+- Bump [email protected] [50bcad59]
+
+23.04.5 - 12 Jan 2023
+- Fix container environment with special chars (#4594) [663b2936]
+
+23.12.0-edge - 20 Dec 2023 
+- Add AWS_SESSION_TOKEN to Fusion environment (#4581) [552f29b0]
+- Add ability to disable Cloudinfo service (#4606) [f7251895]
+- Add experimental support for Fargate compute type for AWS Batch  (#3474) [47cf335b]
+- Add support for Instance template to Google Batch [df7ed294]
+- Add support for Singularity/Apptainer auto pull mode for OCI containers [b7f1a192]
+- Fix BitBucket get source API with custom branch [58937831]
+- Fix Fusion tags documentation (#4551) [687e2e96]
+- Fix Harden regular expression to used to strip secrets in logs (#4563) [832bff24]
+- Fix bug with Fusion symlink resolution (#4593) [09e85582]
+- Fix container environment with special chars (#4594) [e0fe952f]
+- Fix custom notification template [ccf4f59e]
+- Fix fusion symlink test (#4604) [681ace86]
+- Fix smoke tests [d3c2f330]
+- Improve GLS tests [58590b1c]
+- Load nf-amazon when AWS SES is enabled [887f06f4]
+- Move build num & timestamp to BuildInfo class [ec8083d4]
+- Move app version to BuildInfo class [c7d749e8]
+- Remove deprecated Wave observer [0e009ef7]
+- Remove undocumented userEmulation mode (#4596) [f6c79788]
+- Remove unused DSL2 check [e9ee3b2c]
+- Replace each iterator with class for [f7662e68]
+- Bump [email protected] [0b40b7b9]
+- Bump [email protected] [bcb20fcf]
+- Bump [email protected] [aa981814]
+- Bump [email protected] and [email protected] [9cb50035]
+- Build optimizations (#4579) [5ad41e44]
+
 23.11.0-edge - 24 Nov 2023
 - Add `fusion.cacheSize` config option (#4518) [2faadc22]
 - Add Topic channel type (experimental) (#4459) [921313d1]
@@ -16,29 +96,29 @@ NEXTFLOW CHANGE-LOG
 - Add support for Azure low-priority pool (#4527) [8320ea10]
 - Add support for FUSION_AWS_REGION (#4481) [8f8b09fa]
 - Add support for Fusion when using Singularity OCI mode (#4508) [4f3aa631]
-- Add support for K8s schedulerName pod spec (#4485) [ci fast] [dfc7b7c8]
+- Add support for K8s schedulerName pod spec (#4485) [dfc7b7c8]
 - Add support for Singularity OCI mode (#4440) [f5362a7b]
-- Allow the use of error built-in function in onComplete handler (#4458) [ci fast] [35a4424b]
-- Fix Bug in JsonSplitter ordering [ci fast] [8ec14dd2]
+- Allow the use of error built-in function in onComplete handler (#4458) [35a4424b]
+- Fix Bug in JsonSplitter ordering [8ec14dd2]
 - Fix Bypass Google Batch Price query if task cpus and memory are defined (#4521) [7f8f20d3]
 - Fix Checkout remote tag if checkout remote branch fails (#4247) [b8907ccb]
 - Fix Fusion symlinks when publishing files (#4348) [89f09fe0]
 - Fix Inspect command fails with Singularity [f5bb829f]
 - Fix ParamsMap copyWith param aliases (#4188) [b480ee0e]
 - Fix Singularity docs [e952299f]
 - Fix container hashing for Singularity + Wave containers [4c6f2e85]
-- Fix detection of Conda local path made by Wave client [ci fast] (#4532) [4d5bc216]
+- Fix detection of Conda local path made by Wave client (#4532) [4d5bc216]
 - Fix doc tests to fail on test failure (#4505) [4d326551]
-- Fix errors when NXF_HOME contains spaces (#4456) [ci fast] [fe5bea99]
+- Fix errors when NXF_HOME contains spaces (#4456) [fe5bea99]
 - Fix Google Batch network/subnetwork docs (#4475) [27d132f3]
-- Fix rounding error with long durations (#4496) [ci fast] [0356178b]
+- Fix rounding error with long durations (#4496) [0356178b]
 - Fix security vulnerabilities (#4513) [a310c777]
 - Fix Use consistently NXF_TASK_WORKDIR (#4484) [48ee3c64]
 - Improve error details for AbortOperationException [35609cb0]
 - Improve operator docs (#4502) [38210e11]
 - Makefile clean to also remove buildSrc/build (#4517) [2ccb05d0]
-- Minor test improvements [ci fast] [171831ea]
-- Minor types improvement for mix operator [ci fast] [91c1ab15]
+- Minor test improvements [171831ea]
+- Minor types improvement for mix operator [91c1ab15]
 - Normalise channel docs [b641d677]
 - Remove deprecated TowerArchiver feature [ff8e06a3]
 - Remove dsl1 deprecated code (part 2) [159effb1]
@@ -138,6 +218,14 @@ NEXTFLOW CHANGE-LOG
 - Bump [email protected] [0cf2476c]
 - Bump [email protected] [d0c47d49]
 
+23.04.4 - 25 Sep 2023
+- Fix use of GITHUB_TOKEN variable to access GitHub repo [108c6b55]
+- Fix allow_other vulnerability preventing google-batch submissions (#4332) [4895d547]
+- Fix Prevent false positive resumable task [aae87715]
+- Fix Always emit publish event for cached task outputs (#4227) [c4cd53c2]
+- Fix Too long Http connection pool timeout [ce5e9930]
+- Bump [email protected] [d881728c]
+
 23.09.1-edge - 11 Sep 2023
 - Revert "Allow setting shell directive when using the trace file (#4210)" [9f9edcdc]
 

diff --git a/config/codenarc/codenarc.groovy b/config/codenarc/codenarc.groovy
@@ -1,5 +1,5 @@
 /*
- * Copyright 2013-2023, Seqera Labs
+ * Copyright 2013-2024, Seqera Labs
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.

diff --git a/config/codenarc/codenarc.xml b/config/codenarc/codenarc.xml
@@ -1,5 +1,5 @@
 <!--
-  ~ Copyright 2013-2023, Seqera Labs
+  ~ Copyright 2013-2024, Seqera Labs
   ~
   ~ Licensed under the Apache License, Version 2.0 (the "License");
   ~ you may not use this file except in compliance with the License.

diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -1,4 +1,4 @@
-FROM amazoncorretto:17.0.9
+FROM amazoncorretto:17.0.10
 RUN yum install -y procps-ng shadow-utils which
 
 ENV NXF_HOME=/.nextflow

diff --git a/docker/Makefile b/docker/Makefile
@@ -1,5 +1,5 @@
 #
-#  Copyright 2013-2023, Seqera Labs
+#  Copyright 2013-2024, Seqera Labs
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/docker/entry.sh b/docker/entry.sh
@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-#  Copyright 2013-2023, Seqera Labs
+#  Copyright 2013-2024, Seqera Labs
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
 #  you may not use this file except in compliance with the License.

diff --git a/docs/azure.md b/docs/azure.md
@@ -96,7 +96,7 @@ set in the configuration if a share is mounted.
 
 Nextflow provides built-in support for Azure Batch, allowing the seamless deployment of Nextflow pipelines in the cloud, in which tasks are offloaded as Batch jobs.
 
-Read the {ref}`Azore Batch executor <azurebatch-executor>` section to learn more about the `azurebatch` executor in Nextflow.
+Read the {ref}`Azure Batch executor <azurebatch-executor>` section to learn more about the `azurebatch` executor in Nextflow.
 
 ### Get started
 

diff --git a/docs/cli.md b/docs/cli.md
@@ -50,6 +50,9 @@ Available options:
 `-syslog`
 : Send logs to syslog server (e.g. localhost:514).
 
+`-trace`
+: Enable trace level logging for the specified packages. Multiple packages can be provided separating them with a comma, e.g. `-trace nextflow,io.seqera`.
+
 `-v, -version`
 : Print the program version.
 
@@ -121,7 +124,7 @@ The `-c` option is used to append a new configuration to the default configurati
 
 ### Docker driven execution
 
-:::{warning} *Experimental: not recommended for production environments.*
+:::{deprecated} 23.09.0-edge
 :::
 
 Launch Nextflow via Docker.
@@ -1129,7 +1132,7 @@ The `run` command is used to execute a local pipeline script or remote pipeline
 : Enable/disable ANSI console logging.
 
 `-bucket-dir`
-: Remote bucket where intermediate result files are stored.
+: Remote bucket where intermediate result files are stored. When running a hybrid workflow, `-bucket-dir` and `-work-dir` should define separate work directories for remote tasks and local tasks, respectively.
 
 `-cache`
 : Enable/disable processes caching.
@@ -1264,7 +1267,7 @@ The `run` command is used to execute a local pipeline script or remote pipeline
 : Create workflow execution timeline.
 
 `-with-tower` (`https://api.tower.nf`)
-: Monitor workflow execution with [Tower](https://cloud.tower.nf/).
+: Monitor workflow execution with [Seqera Platform](https://cloud.tower.nf/) (formerly Tower Cloud).
 
 `-with-trace` (`trace-<timestamp>.txt`)
 : Create workflow execution trace file.
@@ -1328,7 +1331,7 @@ The `run` command is used to execute a local pipeline script or remote pipeline
   $ nextflow run main.nf -entry workflow_A
   ```
 
-- Execute a pipeline with integrated monitoring in [Tower](https://cloud.tower.nf).
+- Execute a pipeline with integrated monitoring in [Seqera Platform](https://cloud.tower.nf).
 
   ```console
   $ nextflow run nextflow-io/hello -with-tower

diff --git a/docs/conda.md b/docs/conda.md
@@ -76,6 +76,20 @@ dependencies:
   - bwa=0.7.15
 ```
 
+This other example shows how to leverage a Conda environment file to install Python packages from the [PyPI repository](https://pypi.org/)), through the `pip` package manager (which must also be explicitly listed as a required package):
+
+```yaml
+name: my-env-2
+channels:
+  - defaults
+dependencies:
+  - pip
+  - pip:
+    - numpy
+    - pandas
+    - matplotlib
+```
+
 Read the Conda documentation for more details about how to create [environment files](https://conda.io/docs/user-guide/tasks/manage-environments.html#creating-an-environment-file-manually).
 
 The path of an environment file can be specified using the `conda` directive:

diff --git a/docs/conf.py b/docs/conf.py
@@ -64,9 +64,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '23.11'
+version = '24.01'
 # The full version, including alpha/beta/rc tags.
-release = '23.11.0-edge'
+release = '24.01.0-edge'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.