Merge branch 'master' into default-conda-channels

.github/workflows/build.yml

@@ -25,12 +25,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        java_version: [11, 17, 23]
+        java_version: [17, 23]
 
     steps:
-      - name: Environment
-        run: env | sort
-
       - name: Checkout
         uses: actions/checkout@v4
         with:
@@ -93,6 +90,7 @@ jobs:
       - name: Test
         if: steps.changed-files.outputs.any_changed == 'true'
         run: |
+            env | sort
             # configure test env
             if [[ "$GOOGLE_SECRET" ]]; then
             echo $GOOGLE_SECRET | base64 -d > $PWD/google_credentials.json
@@ -133,7 +131,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        java_version: [11, 23]
+        java_version: [17, 23]
         test_mode: ["test_integration", "test_docs", "test_aws", "test_azure", "test_google", "test_wave"]
     steps:
       - name: Checkout
@@ -160,6 +158,12 @@ jobs:
 
       - name: Run tests
         run: |
+          env | sort
+          # configure test env
+          if [[ "$GOOGLE_SECRET" ]]; then
+          echo $GOOGLE_SECRET | base64 -d > $PWD/google_credentials.json
+          export GOOGLE_APPLICATION_CREDENTIALS=$PWD/google_credentials.json
+          fi
           cat $HOME/.nextflow/scm
           make clean assemble install
           bash test-ci.sh
@@ -191,7 +195,8 @@ jobs:
         with:
           name: report-${{ matrix.test_mode }}-jdk-${{ matrix.java_version }}
           path: |
-            validation/**
+            validation/**/*
+            validation/**/.*
             integration-tests.tar
 
   test-e2e:

.github/workflows/generate-send-dependencies.yml

@@ -0,0 +1,27 @@
+name: Generate and submit dependency graph for nextflow
+on:
+    push:
+      branches: ['master']
+
+permissions:
+  contents: write
+
+jobs:
+  dependency-submission:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        components: ["nextflow", "plugins:nf-google", "plugins:nf-amazon", "plugins:nf-azure", "plugins:nf-cloudcache", "plugins:nf-codecommit", "plugins:nf-console", "plugins:nf-tower", "plugins:nf-wave"]
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-java@v4
+      with:
+        distribution: temurin
+        java-version: 21
+
+    - name: Generate and submit dependency graph for nextflow
+      uses: gradle/actions/dependency-submission@v4
+      with:
+        dependency-resolution-task: ":${{ matrix.components }}:dependencies"
+        additional-arguments: "--configuration runtimeClasspath"
+        dependency-graph: generate-and-submit

VERSION

@@ -1 +1 @@
-24.09.2-edge
+24.11.0-edge

build.gradle

@@ -15,7 +15,7 @@
  */
 
 plugins {
-    id "io.codearte.nexus-staging" version "0.21.2"
+    id "io.codearte.nexus-staging" version "0.30.0"
     id 'java'
     id 'idea'
 }
@@ -73,12 +73,12 @@ allprojects {
     }
 
     compileJava {
-        options.release.set(11)
+        options.release.set(17)
     }
 
     tasks.withType(GroovyCompile) {
-        sourceCompatibility = '11'
-        targetCompatibility = '11'
+        sourceCompatibility = '17'
+        targetCompatibility = '17'
     }
 
     idea {
@@ -147,7 +147,6 @@ allprojects {
     // Required to run tests on Java 9 and higher in compatibility mode
     tasks.withType(Test) {
         jvmArgs ([
-            '-Dorg.spockframework.mock.ignoreByteBuddy=true',
             '--enable-preview',
             '--add-opens=java.base/java.lang=ALL-UNNAMED',
             '--add-opens=java.base/java.io=ALL-UNNAMED',

buildSrc/build.gradle

@@ -11,7 +11,7 @@ version = "1.0.1"
 group = "io.nextflow"
 
 dependencies {
-    implementation ('com.amazonaws:aws-java-sdk-s3:1.12.129')
+    implementation ('com.amazonaws:aws-java-sdk-s3:1.12.777')
     implementation 'com.google.code.gson:gson:2.10.1'
 }
 

changelog.txt

@@ -1,5 +1,83 @@
 NEXTFLOW CHANGE-LOG
 ===================
+24.11.0-edge - 3 Dec 2024
+- Add GHA to submit dependencies to dependabot (#5440) [80395a6d]
+- Add NotFoundException to retry condition for Google Batch [aa4d19cc]
+- Add Rahel Hirsch to run name generator (#5442) [ff2bc6ae]
+- Add `env()` function (#5506) [fa0e8e0f]
+- Add more scientists to run name generator (#5447) [38d9eda0]
+- Add `singularity.libraryDir` to config page (#5498) [b5e31bb0]
+- Add RepositoryProvider.revision now public property (#5500) [f0a4c526]
+- Deprecate process `shell` block (#5508) [6f527551]
+- Detecting errors in data unstaging (#5345) [3c8e602d]
+- Disable virtual threads on CI tests [ci slip] [69d07dbc]
+- Fix Fargate warning on memory check (#5475) [bdf0ad00]
+- Fix `isContainerReady` when wave is disabled (#5509) [3215afa8]
+- Fix missing wave response (#5547) [ee252173]
+- Fix overlapping conda lock file (#5540) [9248c04d]
+- Fix overlapping conda lock exception (#5489) [eaaeb3de]
+- Fix possible deadlock in dynamic `maxRetry` resolution (#5474) [25bbb621]
+- Fix Wave GCP integration test (#5490) [ad56c89b]
+- Fixing bug when execution with stub and no stub defined (#5473) [f7fd56db]
+- Fix Incorrect CPU value in Azure example (#5549) [fc5e2c2a]
+- Improve docs for using the GPU accelerator directive (#5488) [4b908524]
+- Improve groupTuple docs with scatter/gather example (#5520) [b5c63a9f]
+- Prevent NPE with null AWS Batch response [12fc1d60]
+- Target Java 17 as minimal Java version (#5045) [0140f954]
+- Update 'nexus-staging' plugin to latest version (#5462) [07934513]
+- Update gradle 'shadow' plugin version to 8.3.5 (#5463) [2a5f15f0]
+- Update install docs to reflect change from 'all' to 'dist' (#5496) [c9115659]
+- Update process snippets to comply with strict syntax (#5526) [be1694bf]
+- Update Wave dependencies [09ccd295]
+- Bump aws sdk 1.12.777 (#5458) [8bad0b4b]
+- Bump bouncycastle to jdk18on:1.78.1  (#5467) [cd8c385f]
+- Bump com.azure:azure-identity from 1.11.3 to 1.12.2 (#5449) [cb70f1df]
+- Bump commons-io:2.15.1 [767e4a0a]
+- Bump groovy 4.0.24 [dd71ad31]
+- Bump netty-common to version 4.1.115.Final [d1bbd3d0]
+- Bump [email protected] [2b653b07]
+- Bump [email protected] [6af7198d]
+- Bump [email protected] [9494f970]
+- Bump [email protected] [7e2c8d82]
+- Bump protobuf-java:3.25.5 to nf-google [488b7906]
+
+24.10.2 - 27 Nov 2024
+- Prevent NPE with null AWS Batch response [3d491934]
+- Fix overlapping conda lock file (#5540) [df66deaa]
+- Fix missing wave response (#5547) [eb85cda8]
+- Bump [email protected] [93d09404]
+- Bump [email protected] [469a35dd]
+
+24.10.1 - 18 Nov 2024
+- Fix overlapping file lock exception (#5489) [a2566d54]
+- Fix isContainerReady when wave is disabled (#5509) [c69e3711]
+- Bump [email protected] [e7709a0f]
+- Bump [email protected] [54496ac4]
+- Bump [email protected] [fa227933]
+- Bump netty-common to version 4.1.115.Final [90623c1e]
+
+24.10.0 - 27 Oct 2024
+- Add `manifest.contributors` config option (#5322) [cf0f9690]
+- Add wave mirror and scan config [92e69776]
+- Add wave mirror vs module bundles conflicst warning [b37a8a5b]
+- Add Workflow output definition (second preview) (#5185) [ea128463]
+- Demote azure batch task status log level to trace (#5416) [d6c684bb]
+- Document formal script syntax (#5336) [a48ac589]
+- Add Expose trace record meta info in the task context (#5402) [ffdd3819]
+- Fix NextflowMeta compilation error [500f49c0]
+- Fix XFileSystem serialization issue (#5411) [711864fd]
+- Fix closure rendering in yaml and json config (#5408) [d3a85ceb]
+- Fix failing docs snippet (#5430) [a818f4b8]
+- Fix http files stream  (#5405) [e2e stage] [718dcbe6]
+- Fix nf-tower plugin to upload logs when early failures  (#5434) [4222442a]
+- Fix support for micromamba (#4302) [12431302]
+- Harmonise documentation for hybrid cloud execution (#5362) [a69407d2]
+- Prevent NPE when containerInfo is null [368a266a]
+- Remove invalid equals char (#5414) [7f39e6ed]
+- Bump [email protected] [f67e157a]
+- Bump [email protected] [7a572a12]
+- Bump [email protected] [6524d8dc]
+
 24.09.2-edge - 14 Oct 2024
 - Add Google LS deprecation notice (#5400) [0ee1d9bc]
 - Add hybrid workflow documentation for Azure (#5361) [f81f27de]

docs/_static/dag.mmd

@@ -1,3 +1,17 @@
+%%{
+    init: {
+        'theme': 'base',
+        'themeVariables': {
+            'primaryColor': '#B6ECE2',
+            'primaryTextColor': '#160F26',
+            'primaryBorderColor': '#065647',
+            'lineColor': '#545555',
+            'clusterBkg': '#BABCBD22',
+            'clusterBorder': '#DDDEDE',
+            'fontFamily': 'arial'
+        }
+    }
+}%%
 flowchart TB
     subgraph " "
     v0["Channel.fromFilePairs"]

docs/amazons3.md

@@ -16,7 +16,7 @@ s3://my-bucket/data/sequences.fa
 
 The usual file operations can be applied to a path handle with the above notation. For example, the content of an S3 file can be printed as follows:
 
-```groovy
+```nextflow
 println file('s3://my-bucket/data/sequences.fa').text
 ```
 

docs/aws.md

@@ -46,53 +46,54 @@ Minimal permissions policies to be attached to the AWS account used by Nextflow
 - To use AWS Batch:
 
   ```json
-  "batch:DescribeJobQueues"
   "batch:CancelJob"
-  "batch:SubmitJob"
-  "batch:ListJobs"
   "batch:DescribeComputeEnvironments"
-  "batch:TerminateJob"
+  "batch:DescribeJobDefinitions"
+  "batch:DescribeJobQueues"
   "batch:DescribeJobs"
+  "batch:ListJobs"
   "batch:RegisterJobDefinition"
-  "batch:DescribeJobDefinitions"
+  "batch:SubmitJob"
+  "batch:TagResource"
+  "batch:TerminateJob"
   ```
 
 - To view [EC2](https://aws.amazon.com/ec2/) instances:
 
   ```json
-  "ecs:DescribeTasks"
+  "ec2:DescribeInstanceAttribute"
   "ec2:DescribeInstances"
+  "ec2:DescribeInstanceStatus"
   "ec2:DescribeInstanceTypes"
-  "ec2:DescribeInstanceAttribute"
   "ecs:DescribeContainerInstances"
-  "ec2:DescribeInstanceStatus"
+  "ecs:DescribeTasks"
   ```
 
 - To pull container images from [ECR](https://aws.amazon.com/ecr/) repositories:
 
   ```json
-  "ecr:GetAuthorizationToken"
   "ecr:BatchCheckLayerAvailability"
-  "ecr:GetDownloadUrlForLayer"
-  "ecr:GetRepositoryPolicy"
-  "ecr:DescribeRepositories"
-  "ecr:ListImages"
-  "ecr:DescribeImages"
   "ecr:BatchGetImage"
+  "ecr:DescribeImages"
+  "ecr:DescribeImageScanFindings"
+  "ecr:DescribeRepositories"
+  "ecr:GetAuthorizationToken"
+  "ecr:GetDownloadUrlForLayer"
   "ecr:GetLifecyclePolicy"
   "ecr:GetLifecyclePolicyPreview"
+  "ecr:GetRepositoryPolicy"
+  "ecr:ListImages"
   "ecr:ListTagsForResource"
-  "ecr:DescribeImageScanFindings"
   ```
 
 :::{note}
 If you are running Fargate or Fargate Spot, you may need the following policies in addition to the listed above:
   ```json
+  "ec2:DescribeSubnets"
   "ecs:CreateCluster"
   "ecs:DeleteCluster"
   "ecs:DescribeClusters"
   "ecs:ListClusters"
-  "ec2:DescribeSubnets"
   ```
 :::
 
@@ -252,7 +253,7 @@ Container options may be passed in long form (e.g `--option value`) or short for
 
 Few examples:
 
-```groovy
+```nextflow
 containerOptions '--tmpfs /run:rw,noexec,nosuid,size=128 --tmpfs /app:ro,size=64'
 
 containerOptions '-e MYVAR1 --env MYVAR2=foo2 --env MYVAR3=foo3 --memory-swap 3240000 --memory-swappiness 20 --shm-size 16000000'
@@ -538,7 +539,8 @@ See the AWS documentation for details how to create the required AWS Batch queue
 and the Batch Execution Role.
 
 :::{note}
-This feature requires the use {ref}`Wave <wave-page>` container provisioning service.
+Nextflow uses [s5cmd](https://github.com/peak/s5cmd) to download the task input data and upload the task outputs.
+To enable this capability, you need to enable the Wave service in the Nextflow configuration, as shown in the above example. See {ref}Wave <wave-page> documentation for more details.
 :::
 
 ## Advanced configuration

docs/azure.md

@@ -163,16 +163,16 @@ By default, the `cpus` and `memory` directives are used to find the smallest mac
 
 To specify multiple Azure machine families, use a comma separated list with glob (`*`) values in the `machineType` directive. For example, the following will select any machine size from D or E v5 machines, with additional data disk, denoted by the `d` suffix:
 
-```config
+```groovy
 process.machineType = "Standard_D*d_v5,Standard_E*d_v5"
 ```
 
-For example, the following process will create a pool of `Standard_E4d_v5` machines based when using `autoPoolMode`:
+For example, the following process will create a pool of `Standard_E8d_v5` machines based when using `autoPoolMode`:
 
 ```nextflow
 process EXAMPLE_PROCESS {
     machineType "Standard_E*d_v5"
-    cpus 16
+    cpus 8
     memory 8.GB
 
     script: