From 71932f9d118a8c87634e4889cb99be5737ea2d37 Mon Sep 17 00:00:00 2001 From: Jorge Aguilera Date: Wed, 16 Nov 2022 13:24:45 +0100 Subject: [PATCH 01/13] remove capsule and create a single jar Signed-off-by: Jorge Aguilera --- modules/nextflow/build.gradle | 17 +++++ nextflow | 32 +++------ packing.gradle | 119 +--------------------------------- 3 files changed, 28 insertions(+), 140 deletions(-) diff --git a/modules/nextflow/build.gradle b/modules/nextflow/build.gradle index 3e3f156a7d..898a17080c 100644 --- a/modules/nextflow/build.gradle +++ b/modules/nextflow/build.gradle @@ -1,3 +1,6 @@ +plugins { + id "com.github.johnrengelman.shadow" version "7.1.2" +} apply plugin: 'groovy' apply plugin: 'application' @@ -70,3 +73,17 @@ application { run{ args( (project.hasProperty("runCmd") ? project.findProperty("runCmd") : "set a cmd to run").split(' ') ) } + +shadowJar { + archiveClassifier='one' + manifest { + attributes 'Main-Class': "$mainClassName" + } + mergeServiceFiles() + mergeGroovyExtensionModules() + doLast { + ant.copy(file: "$buildDir/libs/nextflow-${version}-one.jar", todir: releaseDir, overwrite: true) + ant.copy(file: "$buildDir/libs/nextflow-${version}-one.jar", todir: nextflowDir, overwrite: true) + println "\n+ Nextflow package copied to: $releaseDir" + } +} \ No newline at end of file diff --git a/nextflow b/nextflow index 0b9752caa0..e77dc5bbac 100755 --- a/nextflow +++ b/nextflow @@ -135,12 +135,12 @@ function install() { function launch_nextflow() { # the launch command line local cmdline=() - # remove leading and trailing double-quotes + # remove leading and trailing double-quotes for x in "${launcher[@]}"; do x="${x%\"}" x="${x#\"}" - cmdline+=("$x") - done + cmdline+=("$x") + done if [[ "$bg" ]]; then local pid_file="${NXF_PID_FILE:-.nextflow.pid}" @@ -237,9 +237,6 @@ if [[ "$dockerize" ]]; then exit 1 fi -CAPSULE_LOG=${CAPSULE_LOG:=''} -CAPSULE_RESET=${CAPSULE_RESET:=''} -CAPSULE_CACHE_DIR=${CAPSULE_CACHE_DIR:="$NXF_HOME/capsule"} NXF_PACK=one NXF_MODE=${NXF_MODE:-''} @@ -254,11 +251,11 @@ NXF_HOST=${HOSTNAME:-localhost} # Determine the path to this file if [[ $NXF_PACK = all ]]; then - NXF_BIN=$(which "$0" 2>/dev/null) - [ $? -gt 0 -a -f "$0" ] && NXF_BIN="./$0" + echo "NXF_PACK=all is deprecated" + exit 1 fi -# use nextflow custom java home path +# use nextflow custom java home path if [[ "$NXF_JAVA_HOME" ]]; then JAVA_HOME="$NXF_JAVA_HOME" unset JAVA_CMD @@ -333,14 +330,11 @@ fi COLUMNS=${COLUMNS:-`tty -s && tput cols 2>/dev/null || true`} declare -a JAVA_OPTS=() -JAVA_OPTS+=(-Dfile.encoding=UTF-8 -Dcapsule.trampoline -Dcapsule.java.cmd="$JAVA_CMD" -Dcom.sun.security.enableAIAcaIssuers=true) +JAVA_OPTS+=(-Dfile.encoding=UTF-8 -Dcom.sun.security.enableAIAcaIssuers=true) if [[ $cmd == console ]]; then bg=1; else JAVA_OPTS+=(-Djava.awt.headless=true) fi -[[ "$JAVA_HOME" ]] && JAVA_OPTS+=(-Dcapsule.java.home="$JAVA_HOME") -[[ "$CAPSULE_LOG" ]] && JAVA_OPTS+=(-Dcapsule.log=$CAPSULE_LOG) -[[ "$CAPSULE_RESET" ]] && JAVA_OPTS+=(-Dcapsule.reset=true) [[ "$cmd" != "run" && "$cmd" != "node" ]] && JAVA_OPTS+=(-XX:+TieredCompilation -XX:TieredStopAtLevel=1) [[ "$NXF_OPTS" ]] && JAVA_OPTS+=($NXF_OPTS) [[ "$NXF_CLASSPATH" ]] && export NXF_CLASSPATH @@ -349,7 +343,6 @@ fi [[ "$NXF_TEMP" ]] && JAVA_OPTS+=(-Djava.io.tmpdir="$NXF_TEMP") [[ "${jvmopts[@]}" ]] && JAVA_OPTS+=("${jvmopts[@]}") export JAVA_CMD -export CAPSULE_CACHE_DIR export NXF_PLUGINS_DIR export NXF_PLUGINS_MODE export NXF_PLUGINS_DEFAULT @@ -363,7 +356,7 @@ else MD5='' fi # when no md5 command is available fallback on default execution -if [ ! "$MD5" ] || [ "$CAPSULE_RESET" ]; then +if [ ! "$MD5" ]; then launcher=($("$JAVA_CMD" "${JAVA_OPTS[@]}" -jar "$NXF_BIN")) launch_nextflow exit 1 @@ -393,13 +386,8 @@ fi if [ -s "$LAUNCH_FILE" ] && [ "$LAUNCH_FILE" -nt "$NXF_BIN" ]; then declare -a launcher="($(cat "$LAUNCH_FILE"))" else - # otherwise run the capsule and get the result classpath in the 'launcher' and save it to a file - cli=($("$JAVA_CMD" "${JAVA_OPTS[@]}" -jar "$NXF_BIN")) - [[ $? -ne 0 ]] && echo_red 'Unable to initialize nextflow environment' && exit 1 - - # first string between double quotes is the full path to java, also blank spaces are included - # remainder string are arguments - # we extract first part into `cmd_base`` and remainder into `cmd_tail`` and convert them to array as previous version + # otherwise parse the command and get the result classpath in the 'launcher' and save it to a file + cli=("\"$JAVA_CMD\"" "${JAVA_OPTS[@]}" -jar "$NXF_BIN") cmd_pattern='"([^"]*)"(.*)' [[ "${cli[@]}" =~ $cmd_pattern ]] cmd_base=(${BASH_REMATCH[1]}) diff --git a/packing.gradle b/packing.gradle index a9b142f870..75e2f68784 100644 --- a/packing.gradle +++ b/packing.gradle @@ -1,5 +1,4 @@ configurations { - capsule defaultCfg.extendsFrom api //provided console.extendsFrom defaultCfg @@ -17,9 +16,6 @@ dependencies { defaultCfg "org.apache.ivy:ivy:2.3.0" // default cfg = runtime + httpfs + amazon + tower client defaultCfg project(':nf-httpfs') - // Capsule manages the fat jar building process - capsule 'io.nextflow:capsule:1.1.1' - capsule 'io.nextflow:capsule-maven:1.0.3.2' console project(':plugins:nf-console') ga4gh project(':plugins:nf-ga4gh') google project(':plugins:nf-google') @@ -91,124 +87,11 @@ protected coordinates( it ) { throw new IllegalArgumentException("Not a valid dependency object [${it?.class?.name}]: $it") } -/* - * Default nextflow package. It contains the capsule loader - */ -task packOne(type: Jar) { - dependsOn configurations.capsule, configurations.defaultCfg - archiveName = "nextflow-${version}-one.jar" - - from (configurations.capsule.collect { zipTree(it) }) - - // main manifest attributes - def deps = resolveDeps('defaultCfg') - - manifest.attributes( - 'Main-Class' : 'NextflowLoader', - 'Application-Name' : 'nextflow', - 'Application-Class' : mainClassName, - 'Application-Version': version, - 'Min-Java-Version' : '1.8.0', - 'Caplets' : 'MavenCapsule', - 'Dependencies' : deps - ) - - // enable snapshot dependencies lookup - if( version.endsWith('-SNAPSHOT') ) { - manifest.attributes 'Allow-Snapshots': true - manifest.attributes 'Repositories': 'local https://oss.sonatype.org/content/repositories/snapshots central seqera' - } - else { - manifest.attributes 'Repositories': 'central seqera' - } - - doLast { - ant.copy(file: "$buildDir/libs/nextflow-${version}-one.jar", todir: releaseDir, overwrite: true) - ant.copy(file: "$buildDir/libs/nextflow-${version}-one.jar", todir: nextflowDir, overwrite: true) - println "\n+ Nextflow package `ONE` copied to: $releaseDir" - } -} - -task packAll(type: Jar) { - dependsOn configurations.capsule, configurations.defaultCfg - archiveName = "nextflow-${version}-all.jar" - - from jar // embed our application jar - from (configurations.amazon + configurations.google + configurations.tower) - from (configurations.capsule.collect { zipTree(it) }) - duplicatesStrategy = DuplicatesStrategy.EXCLUDE - - manifest.attributes( 'Main-Class' : 'NextflowLoader', - 'Application-Name' : 'nextflow-all', - 'Application-Class' : mainClassName, - 'Application-Version': version, - 'Min-Java-Version' : '1.8.0' - ) - - manifest.attributes('Main-Class': 'NextflowLoader', 'amazon') - manifest.attributes('Main-Class': 'NextflowLoader', 'google') - - if( project.hasProperty('GA4GH') ) { - println "The build will include GA4GH dependencies." - from(configurations.ga4gh) - manifest.attributes('Main-Class': 'CapsuleLoader', 'ga4gh') - } - - doLast { - file(releaseDir).mkdir() - // cleanup - def source = file("$buildDir/libs/nextflow-${version}-all.jar") - def target = file("$releaseDir/nextflow-${version}-all"); target.delete() - // append the big jar - target.withOutputStream { - it << file('nextflow').text.replaceAll(/NXF_PACK\=.*/, 'NXF_PACK=all') - it << new FileInputStream(source) - } - // execute permission - "chmod +x $target".execute() - // done - println "+ Nextflow package `ALL` copied to: $releaseDir\n" - } -} - -task packCore(type: Jar) { - dependsOn configurations.capsule, configurations.defaultCfg - archiveName = "nextflow-${version}-core.jar" - - from jar // embed our application jar - from (configurations.defaultCfg) - from (configurations.capsule.collect { zipTree(it) }) - duplicatesStrategy = DuplicatesStrategy.EXCLUDE - - manifest.attributes( 'Main-Class' : 'NextflowLoader', - 'Application-Name' : 'nextflow-core', - 'Application-Class' : mainClassName, - 'Application-Version': version, - 'Min-Java-Version' : '1.8.0' - ) - - doLast { - file(releaseDir).mkdir() - // cleanup - def source = file("$buildDir/libs/nextflow-${version}-core.jar") - def target = file("$releaseDir/nextflow-${version}-core"); target.delete() - // append the big jar - target.withOutputStream { - it << file('nextflow').text.replaceAll(/NXF_PACK\=.*/, 'NXF_PACK=all') - it << new FileInputStream(source) - } - // execute permission - "chmod +x $target".execute() - // done - println "+ Nextflow package `CORE` copied to: $releaseDir\n" - } -} - /* * Compile and pack all packages */ -task pack( dependsOn: [packOne, packAll]) { +task pack( dependsOn: [clean, compile, ":nextflow:shadowJar"]) { } From fc02a6e41d3800283033efa61962964fc9968567 Mon Sep 17 00:00:00 2001 From: Jorge Aguilera Date: Wed, 16 Nov 2022 14:44:25 +0100 Subject: [PATCH 02/13] remove capsule and create a single jar Signed-off-by: Jorge Aguilera --- Makefile | 6 +++--- modules/nextflow/build.gradle | 5 ----- packing.gradle | 6 +++++- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index e0d25170a0..3ef2ca89c6 100644 --- a/Makefile +++ b/Makefile @@ -95,16 +95,16 @@ upload: # Create self-contained distribution package # pack: - BUILD_PACK=1 ./gradlew packAll + BUILD_PACK=1 ./gradlew pack packCore: - BUILD_PACK=1 ./gradlew packCore + BUILD_PACK=1 ./gradlew pack # # Create self-contained distribution package, including GA4GH support and associated dependencies # packGA4GH: - BUILD_PACK=1 ./gradlew -PGA4GH packAll + BUILD_PACK=1 ./gradlew -PGA4GH pack # # Upload NF launcher to nextflow.io web site diff --git a/modules/nextflow/build.gradle b/modules/nextflow/build.gradle index 40dba1abb6..677e20e9cb 100644 --- a/modules/nextflow/build.gradle +++ b/modules/nextflow/build.gradle @@ -81,9 +81,4 @@ shadowJar { } mergeServiceFiles() mergeGroovyExtensionModules() - doLast { - ant.copy(file: "$buildDir/libs/nextflow-${version}-one.jar", todir: releaseDir, overwrite: true) - ant.copy(file: "$buildDir/libs/nextflow-${version}-one.jar", todir: nextflowDir, overwrite: true) - println "\n+ Nextflow package copied to: $releaseDir" - } } \ No newline at end of file diff --git a/packing.gradle b/packing.gradle index 75e2f68784..4b7872110f 100644 --- a/packing.gradle +++ b/packing.gradle @@ -92,7 +92,11 @@ protected coordinates( it ) { * Compile and pack all packages */ task pack( dependsOn: [clean, compile, ":nextflow:shadowJar"]) { - + doLast { + ant.copy(file: "modules/nextflow/build/libs/nextflow-${version}-one.jar", todir: releaseDir, overwrite: true) + ant.copy(file: "modules/nextflow/build/libs/nextflow-${version}-one.jar", todir: nextflowDir, overwrite: true) + println "\n+ Nextflow package `ONE` copied to: $releaseDir" + } } From 4b1f00ad8b9c9af180451266183af055324b221e Mon Sep 17 00:00:00 2001 From: Jorge Aguilera Date: Wed, 16 Nov 2022 16:48:15 +0100 Subject: [PATCH 03/13] make launcher compatible with previous and new versions Signed-off-by: Jorge Aguilera --- nextflow | 42 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/nextflow b/nextflow index e77dc5bbac..035c7cc7e7 100755 --- a/nextflow +++ b/nextflow @@ -26,6 +26,14 @@ NXF_DIST=${NXF_DIST:-$NXF_HOME/framework} NXF_CLI="$0 $@" NXF_CLI_OPTS=${NXF_CLI_OPTS:-} +NXF_CAPSULE_LAUNCHER=1 +NXF_V=$(echo $NXF_VER| cut -d'.' -f1) +NXF_R=$(echo $NXF_VER| cut -d'.' -f2) +if [[ NXF_V -ge 22 ]] && [[ NXF_R -ge 11 ]]; then + # from version 22.11.X we will use a fat jar and remove capsule dependency + unset NXF_CAPSULE_LAUNCHER +fi + export NXF_CLI export NXF_ORG export NXF_HOME @@ -70,7 +78,7 @@ function get() { exit 1 fi - printf "Downloading nextflow dependencies. It may require a few seconds, please wait .. " + printf "Downloading nextflow dependencies using $GET. It may require a few seconds, please wait .. " eval $GET; status=$? printf "\r\033[K" if [ $status -ne 0 ]; then @@ -237,6 +245,9 @@ if [[ "$dockerize" ]]; then exit 1 fi +CAPSULE_LOG=${CAPSULE_LOG:=''} +CAPSULE_RESET=${CAPSULE_RESET:=''} +CAPSULE_CACHE_DIR=${CAPSULE_CACHE_DIR:="$NXF_HOME/capsule"} NXF_PACK=one NXF_MODE=${NXF_MODE:-''} @@ -251,8 +262,13 @@ NXF_HOST=${HOSTNAME:-localhost} # Determine the path to this file if [[ $NXF_PACK = all ]]; then + if [[ $NXF_CAPSULE_LAUNCHER ]]; then + NXF_BIN=$(which "$0" 2>/dev/null) + [ $? -gt 0 -a -f "$0" ] && NXF_BIN="./$0" + else echo "NXF_PACK=all is deprecated" exit 1 + fi fi # use nextflow custom java home path @@ -330,11 +346,16 @@ fi COLUMNS=${COLUMNS:-`tty -s && tput cols 2>/dev/null || true`} declare -a JAVA_OPTS=() -JAVA_OPTS+=(-Dfile.encoding=UTF-8 -Dcom.sun.security.enableAIAcaIssuers=true) +JAVA_OPTS+=(-Dfile.encoding=UTF-8 -Dcapsule.trampoline -Dcapsule.java.cmd="$JAVA_CMD" -Dcom.sun.security.enableAIAcaIssuers=true) if [[ $cmd == console ]]; then bg=1; else JAVA_OPTS+=(-Djava.awt.headless=true) fi +if [[ $NXF_CAPSULE_LAUNCHER ]]; then + [[ "$JAVA_HOME" ]] && JAVA_OPTS+=(-Dcapsule.java.home="$JAVA_HOME") + [[ "$CAPSULE_LOG" ]] && JAVA_OPTS+=(-Dcapsule.log=$CAPSULE_LOG) + [[ "$CAPSULE_RESET" ]] && JAVA_OPTS+=(-Dcapsule.reset=true) +fi [[ "$cmd" != "run" && "$cmd" != "node" ]] && JAVA_OPTS+=(-XX:+TieredCompilation -XX:TieredStopAtLevel=1) [[ "$NXF_OPTS" ]] && JAVA_OPTS+=($NXF_OPTS) [[ "$NXF_CLASSPATH" ]] && export NXF_CLASSPATH @@ -343,6 +364,7 @@ fi [[ "$NXF_TEMP" ]] && JAVA_OPTS+=(-Djava.io.tmpdir="$NXF_TEMP") [[ "${jvmopts[@]}" ]] && JAVA_OPTS+=("${jvmopts[@]}") export JAVA_CMD +export CAPSULE_CACHE_DIR export NXF_PLUGINS_DIR export NXF_PLUGINS_MODE export NXF_PLUGINS_DEFAULT @@ -356,7 +378,7 @@ else MD5='' fi # when no md5 command is available fallback on default execution -if [ ! "$MD5" ]; then +if [ ! "$MD5" ] || [ "$CAPSULE_RESET" ]; then launcher=($("$JAVA_CMD" "${JAVA_OPTS[@]}" -jar "$NXF_BIN")) launch_nextflow exit 1 @@ -386,8 +408,18 @@ fi if [ -s "$LAUNCH_FILE" ] && [ "$LAUNCH_FILE" -nt "$NXF_BIN" ]; then declare -a launcher="($(cat "$LAUNCH_FILE"))" else - # otherwise parse the command and get the result classpath in the 'launcher' and save it to a file - cli=("\"$JAVA_CMD\"" "${JAVA_OPTS[@]}" -jar "$NXF_BIN") + if [[ $NXF_CAPSULE_LAUNCHER ]]; then + # otherwise run the capsule and get the result classpath in the 'launcher' and save it to a file + cli=($("$JAVA_CMD" "${JAVA_OPTS[@]}" -jar "$NXF_BIN")) + [[ $? -ne 0 ]] && echo_red 'Unable to initialize nextflow environment' && exit 1 + else + # otherwise parse the command and get the result classpath in the 'launcher' and save it to a file + cli=("\"$JAVA_CMD\"" "${JAVA_OPTS[@]}" -jar "$NXF_BIN") + fi + + # first string between double quotes is the full path to java, also blank spaces are included + # remainder string are arguments + # we extract first part into `cmd_base`` and remainder into `cmd_tail`` and convert them to array as previous version cmd_pattern='"([^"]*)"(.*)' [[ "${cli[@]}" =~ $cmd_pattern ]] cmd_base=(${BASH_REMATCH[1]}) From 031a6194b360848969fda55408426f759fdbdddd Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sat, 8 Jun 2024 17:13:35 +0200 Subject: [PATCH 04/13] Bump version Signed-off-by: Paolo Di Tommaso --- VERSION | 2 +- nextflow | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/VERSION b/VERSION index 11300bc91f..4369e7eda7 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -24.04.2 +24.05.0-edge diff --git a/nextflow b/nextflow index 7b5ff0ac84..8ddb456368 100755 --- a/nextflow +++ b/nextflow @@ -15,7 +15,7 @@ # limitations under the License. [[ "$NXF_DEBUG" == 'x' ]] && set -x -NXF_VER=${NXF_VER:-'24.04.2'} +NXF_VER=${NXF_VER:-'24.05.0-edge'} NXF_ORG=${NXF_ORG:-'nextflow-io'} NXF_HOME=${NXF_HOME:-$HOME/.nextflow} NXF_PROT=${NXF_PROT:-'https'} @@ -29,8 +29,7 @@ NXF_REMOTE_DEBUG_PORT=${NXF_REMOTE_DEBUG_PORT:-5005} NXF_CAPSULE_LAUNCHER=1 NXF_V=$(echo $NXF_VER| cut -d'.' -f1) NXF_R=$(echo $NXF_VER| cut -d'.' -f2) -if [[ NXF_V -ge 22 ]] && [[ NXF_R -ge 11 ]]; then - # from version 22.11.X we will use a fat jar and remove capsule dependency +if [[ NXF_V -ge 24 ]] && [[ NXF_R -ge 5 ]]; then unset NXF_CAPSULE_LAUNCHER fi From ba16f24dc2640b5f3b632619ee69f84d1f7f90d4 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sat, 27 Jul 2024 17:10:32 +0200 Subject: [PATCH 05/13] Bump shadowJar 8.1.1 Signed-off-by: Paolo Di Tommaso --- modules/nextflow/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/nextflow/build.gradle b/modules/nextflow/build.gradle index 672840cc7d..ff7f48dc83 100644 --- a/modules/nextflow/build.gradle +++ b/modules/nextflow/build.gradle @@ -1,5 +1,5 @@ plugins { - id "com.github.johnrengelman.shadow" version "7.1.2" + id "com.github.johnrengelman.shadow" version "8.1.1" } apply plugin: 'groovy' apply plugin: 'application' @@ -83,4 +83,4 @@ shadowJar { } mergeServiceFiles() mergeGroovyExtensionModules() -} \ No newline at end of file +} From b518fae3c3927b8b0e143b979f7d49cb42b35f77 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Sat, 27 Jul 2024 20:18:11 +0200 Subject: [PATCH 06/13] wip Signed-off-by: Paolo Di Tommaso --- Makefile | 4 +-- VERSION | 2 +- modules/nf-all/build.gradle | 53 +++++++++++++++++++++++++++++++++++++ nextflow | 7 +---- packing.gradle | 19 +++++++++++++ settings.gradle | 1 + 6 files changed, 77 insertions(+), 9 deletions(-) create mode 100644 modules/nf-all/build.gradle diff --git a/Makefile b/Makefile index f07921c1e1..7e5f088f94 100644 --- a/Makefile +++ b/Makefile @@ -97,8 +97,8 @@ upload: pack: BUILD_PACK=1 ./gradlew pack -packCore: - BUILD_PACK=1 ./gradlew pack +packAll: + BUILD_PACK=1 ./gradlew packAll # # Upload NF launcher to nextflow.io web site diff --git a/VERSION b/VERSION index 4e8327ae10..d4c0237907 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -24.06.0-edge +24.07.0-edge diff --git a/modules/nf-all/build.gradle b/modules/nf-all/build.gradle new file mode 100644 index 0000000000..e153797d86 --- /dev/null +++ b/modules/nf-all/build.gradle @@ -0,0 +1,53 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +plugins { + id "com.github.johnrengelman.shadow" version "8.1.1" +} +apply plugin: 'groovy' +apply plugin: 'application' + +group = 'io.nextflow' +version = rootProject.file('VERSION').text.trim() + +dependencies { + implementation project(':nextflow') + implementation project(':plugins:nf-amazon') + implementation project(':plugins:nf-azure') + implementation project(':plugins:nf-cloudcache') + implementation project(':plugins:nf-codecommit') + implementation project(':plugins:nf-console') + implementation project(':plugins:nf-google') + implementation project(':plugins:nf-tower') + implementation project(':plugins:nf-wave') +} + +application { + mainClass = 'nextflow.cli.Launcher' +} + +shadowJar { + zip64 = true + archiveBaseName.set('nextflow') + archiveVersion.set(version as String) + archiveClassifier.set('all') + manifest { + attributes 'Main-Class': "$mainClassName" + } + mergeServiceFiles() + mergeGroovyExtensionModules() +} diff --git a/nextflow b/nextflow index 8af75f036a..449debafdc 100755 --- a/nextflow +++ b/nextflow @@ -15,7 +15,7 @@ # limitations under the License. [[ "$NXF_DEBUG" == 'x' ]] && set -x -NXF_VER=${NXF_VER:-'24.06.0-edge'} +NXF_VER=${NXF_VER:-'24.07.0-edge'} NXF_ORG=${NXF_ORG:-'nextflow-io'} NXF_HOME=${NXF_HOME:-$HOME/.nextflow} NXF_PROT=${NXF_PROT:-'https'} @@ -263,13 +263,8 @@ NXF_HOST=${HOSTNAME:-localhost} # Determine the path to this file if [[ $NXF_PACK = all ]]; then - if [[ $NXF_CAPSULE_LAUNCHER ]]; then NXF_BIN=$(which "$0" 2>/dev/null) [ $? -gt 0 -a -f "$0" ] && NXF_BIN="./$0" - else - echo "NXF_PACK=all is deprecated" - exit 1 - fi fi # use nextflow custom java home path diff --git a/packing.gradle b/packing.gradle index c41664738d..041b294d54 100644 --- a/packing.gradle +++ b/packing.gradle @@ -98,6 +98,25 @@ task pack( dependsOn: [clean, compile, ":nextflow:shadowJar"]) { } } +task packAll( dependsOn: [clean, compile, ":nf-all:shadowJar"]) { + + doLast { + file(releaseDir).mkdirs() + // cleanup + def source = file("modules/nf-all/build/libs/nextflow-${version}-all.jar") + def target = file("$releaseDir/nextflow-${version}-all"); target.delete() + // append the big jar + target.withOutputStream { + it << file('nextflow').text.replaceAll(/NXF_PACK\=.*/, 'NXF_PACK=all') + it << new FileInputStream(source) + } + // execute permission + "chmod +x $target".execute() + // done + println "+ Nextflow package `ALL` copied to: $releaseDir\n" + } +} + task deploy( type: Exec, dependsOn: [clean, compile, pack]) { diff --git a/settings.gradle b/settings.gradle index c9900fd00f..300028d13e 100644 --- a/settings.gradle +++ b/settings.gradle @@ -26,6 +26,7 @@ rootProject.name = 'nextflow-prj' include 'nextflow' include 'nf-commons' include 'nf-httpfs' +include 'nf-all' rootProject.children.each { prj -> prj.projectDir = new File("$rootDir/modules/$prj.name") From 3cae56995aa824a984b57aa185343915892b07bd Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Mon, 29 Jul 2024 12:25:26 +0200 Subject: [PATCH 07/13] wip Signed-off-by: Paolo Di Tommaso --- docs/config.md | 3 --- modules/nf-all/build.gradle | 53 ------------------------------------- nextflow | 5 ++-- settings.gradle | 1 - 4 files changed, 3 insertions(+), 59 deletions(-) delete mode 100644 modules/nf-all/build.gradle diff --git a/docs/config.md b/docs/config.md index ba97797cdb..e63aedd1dd 100644 --- a/docs/config.md +++ b/docs/config.md @@ -1894,9 +1894,6 @@ The following environment variables control the configuration of the Nextflow ru `NXF_CHARLIECLOUD_CACHEDIR` : Directory where remote Charliecloud images are stored. When using a computing cluster it must be a shared folder accessible from all compute nodes. -`NXF_CLASSPATH` -: Allows the extension of the Java runtime classpath with extra JAR files or class folders. - `NXF_CLOUDCACHE_PATH` : :::{versionadded} 23.07.0-edge ::: diff --git a/modules/nf-all/build.gradle b/modules/nf-all/build.gradle deleted file mode 100644 index e153797d86..0000000000 --- a/modules/nf-all/build.gradle +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2013-2024, Seqera Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -plugins { - id "com.github.johnrengelman.shadow" version "8.1.1" -} -apply plugin: 'groovy' -apply plugin: 'application' - -group = 'io.nextflow' -version = rootProject.file('VERSION').text.trim() - -dependencies { - implementation project(':nextflow') - implementation project(':plugins:nf-amazon') - implementation project(':plugins:nf-azure') - implementation project(':plugins:nf-cloudcache') - implementation project(':plugins:nf-codecommit') - implementation project(':plugins:nf-console') - implementation project(':plugins:nf-google') - implementation project(':plugins:nf-tower') - implementation project(':plugins:nf-wave') -} - -application { - mainClass = 'nextflow.cli.Launcher' -} - -shadowJar { - zip64 = true - archiveBaseName.set('nextflow') - archiveVersion.set(version as String) - archiveClassifier.set('all') - manifest { - attributes 'Main-Class': "$mainClassName" - } - mergeServiceFiles() - mergeGroovyExtensionModules() -} diff --git a/nextflow b/nextflow index 449debafdc..661df08986 100755 --- a/nextflow +++ b/nextflow @@ -256,10 +256,11 @@ NXF_JAR=${NXF_JAR:-nextflow-$NXF_VER-$NXF_PACK.jar} NXF_BIN=${NXF_BIN:-$NXF_DIST/$NXF_VER/$NXF_JAR} NXF_PATH=$(dirname "$NXF_BIN") NXF_URL=${NXF_URL:-$NXF_BASE/v$NXF_VER/$NXF_JAR} -NXF_GRAB=${NXF_GRAB:-''} -NXF_CLASSPATH=${NXF_CLASSPATH:-''} NXF_HOST=${HOSTNAME:-localhost} [[ $NXF_LAUNCHER ]] || NXF_LAUNCHER=${NXF_HOME}/tmp/launcher/nextflow-${NXF_PACK}_${NXF_VER}/${NXF_HOST} +# both NXF_GRAB and NXF_CLASSPATH are not supported any more as of version 24.04.7-edge +NXF_GRAB=${NXF_GRAB:-''} +NXF_CLASSPATH=${NXF_CLASSPATH:-''} # Determine the path to this file if [[ $NXF_PACK = all ]]; then diff --git a/settings.gradle b/settings.gradle index 300028d13e..c9900fd00f 100644 --- a/settings.gradle +++ b/settings.gradle @@ -26,7 +26,6 @@ rootProject.name = 'nextflow-prj' include 'nextflow' include 'nf-commons' include 'nf-httpfs' -include 'nf-all' rootProject.children.each { prj -> prj.projectDir = new File("$rootDir/modules/$prj.name") From 68895a39cf6acfc36770bd383690bb2f76e13dec Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Mon, 29 Jul 2024 12:48:04 +0200 Subject: [PATCH 08/13] Allow plugins in self-contained mode Signed-off-by: Paolo Di Tommaso --- .../main/nextflow/plugin/PluginsFacade.groovy | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/modules/nf-commons/src/main/nextflow/plugin/PluginsFacade.groovy b/modules/nf-commons/src/main/nextflow/plugin/PluginsFacade.groovy index 01df157273..4bf9640577 100644 --- a/modules/nf-commons/src/main/nextflow/plugin/PluginsFacade.groovy +++ b/modules/nf-commons/src/main/nextflow/plugin/PluginsFacade.groovy @@ -318,8 +318,8 @@ class PluginsFacade implements PluginStateListener { } void start( String pluginId ) { - if( isSelfContained() && defaultPlugins.hasPlugin(pluginId) ) { - log.debug "Plugin 'start' is not required in self-contained mode -- ignoring for plugin: $pluginId" + if( isEmbedded() && defaultPlugins.hasPlugin(pluginId) ) { + log.debug "Plugin 'start' is not required in embedded mode -- ignoring for plugin: $pluginId" return } @@ -327,8 +327,8 @@ class PluginsFacade implements PluginStateListener { } void start(PluginSpec plugin) { - if( isSelfContained() && defaultPlugins.hasPlugin(plugin.id) ) { - log.debug "Plugin 'start' is not required in self-contained mode -- ignoring for plugin: $plugin.id" + if( isEmbedded() && defaultPlugins.hasPlugin(plugin.id) ) { + log.debug "Plugin 'start' is not required in embedded mode -- ignoring for plugin: $plugin.id" return } @@ -346,19 +346,19 @@ class PluginsFacade implements PluginStateListener { } /** - * @return {@code true} when running in self-contained mode ie. the nextflow distribution + * @return {@code true} when running in embedded mode ie. the nextflow distribution * include also plugin libraries. When running is this mode, plugins should not be started * and cannot be updated. */ - protected boolean isSelfContained() { - return env.get('NXF_PACK')=='all' || embedded + protected boolean isEmbedded() { + return embedded } protected List pluginsRequirement(Map config) { def specs = parseConf(config) - if( isSelfContained() && specs ) { + if( isEmbedded() && specs ) { // custom plugins are not allowed for nextflow self-contained package - log.warn "Nextflow self-contained distribution allows only core plugins -- User config plugins will be ignored: ${specs.join(',')}" + log.warn "Nextflow embedded mode only core plugins -- User config plugins will be ignored: ${specs.join(',')}" return Collections.emptyList() } if( specs ) { @@ -441,7 +441,7 @@ class PluginsFacade implements PluginStateListener { boolean startIfMissing(String pluginId) { if( env.NXF_PLUGINS_DEFAULT == 'false' ) return false - if( isSelfContained() && defaultPlugins.hasPlugin(pluginId) ) + if( isEmbedded() && defaultPlugins.hasPlugin(pluginId) ) return false if( isStarted(pluginId) ) From d5d70d95cc7b661f53f50a9e71881a262298e11c Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Mon, 29 Jul 2024 13:12:20 +0200 Subject: [PATCH 09/13] Rename NXF_CAPSULE_LAUNCHER to NXF_LEGACY_LAUNCHER Signed-off-by: Paolo Di Tommaso --- nextflow | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/nextflow b/nextflow index 661df08986..36ff944087 100755 --- a/nextflow +++ b/nextflow @@ -26,11 +26,11 @@ NXF_CLI="$0 $@" NXF_CLI_OPTS=${NXF_CLI_OPTS:-} NXF_REMOTE_DEBUG_PORT=${NXF_REMOTE_DEBUG_PORT:-5005} -NXF_CAPSULE_LAUNCHER=1 -NXF_V=$(echo $NXF_VER| cut -d'.' -f1) -NXF_R=$(echo $NXF_VER| cut -d'.' -f2) -if [[ NXF_V -ge 24 ]] && [[ NXF_R -ge 5 ]]; then - unset NXF_CAPSULE_LAUNCHER +NXF_LEGACY_LAUNCHER=1 +NXF_V1=$(echo $NXF_VER| cut -d'.' -f1) +NXF_V2=$(echo $NXF_VER| cut -d'.' -f2) +if [[ NXF_V1 -ge 24 ]] && [[ NXF_V2 -ge 7 ]]; then + unset NXF_LEGACY_LAUNCHER fi export NXF_CLI @@ -354,7 +354,7 @@ if [[ $cmd == console ]]; then bg=1; else JAVA_OPTS+=(-Djava.awt.headless=true) fi -if [[ $NXF_CAPSULE_LAUNCHER ]]; then +if [[ $NXF_LEGACY_LAUNCHER ]]; then [[ "$JAVA_HOME" ]] && JAVA_OPTS+=(-Dcapsule.java.home="$JAVA_HOME") [[ "$CAPSULE_LOG" ]] && JAVA_OPTS+=(-Dcapsule.log=$CAPSULE_LOG) [[ "$CAPSULE_RESET" ]] && JAVA_OPTS+=(-Dcapsule.reset=true) @@ -411,10 +411,10 @@ LAUNCH_FILE="${NXF_LAUNCHER}/classpath-$(env_md5)" if [ -s "$LAUNCH_FILE" ] && [ "$LAUNCH_FILE" -nt "$NXF_BIN" ] && [[ "$remote_debug" -ne 1 ]]; then declare -a launcher="($(cat "$LAUNCH_FILE"))" else - if [[ $NXF_CAPSULE_LAUNCHER ]]; then + if [[ $NXF_LEGACY_LAUNCHER ]]; then # otherwise run the capsule and get the result classpath in the 'launcher' and save it to a file cli=($("$JAVA_CMD" "${JAVA_OPTS[@]}" -jar "$NXF_BIN")) - [[ $? -ne 0 ]] && echo_red 'Unable to initialize nextflow environment' && exit 1 + [[ $? -ne 0 ]] && echo_red 'Unable to initialize nextflow environment' && exit 1 else # otherwise parse the command and get the result classpath in the 'launcher' and save it to a file cli=("\"$JAVA_CMD\"" "${JAVA_OPTS[@]}" -jar "$NXF_BIN") @@ -460,6 +460,7 @@ else launcher+=("${cmd_tail[@]}") fi + if [[ $NXF_LEGACY_LAUNCHER ]]; then # Don't show errors if the LAUNCH_FILE can't be created if mkdir -p "${NXF_LAUNCHER}" 2>/dev/null; then STR='' @@ -470,6 +471,7 @@ else else echo_yellow "Warning: Couldn't create cached classpath folder: $NXF_LAUNCHER -- Maybe NXF_HOME is not writable?" fi + fi fi From 3aec12f4a806e42be55dd288719d244a87d063e7 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Mon, 29 Jul 2024 13:12:50 +0200 Subject: [PATCH 10/13] Rename pack "all" to "dist" Signed-off-by: Paolo Di Tommaso --- Makefile | 3 --- nextflow | 2 +- packing.gradle | 25 ++++++++++++++++--------- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 7e5f088f94..4a12211e00 100644 --- a/Makefile +++ b/Makefile @@ -97,9 +97,6 @@ upload: pack: BUILD_PACK=1 ./gradlew pack -packAll: - BUILD_PACK=1 ./gradlew packAll - # # Upload NF launcher to nextflow.io web site # diff --git a/nextflow b/nextflow index 36ff944087..d0c8c2d78d 100755 --- a/nextflow +++ b/nextflow @@ -263,7 +263,7 @@ NXF_GRAB=${NXF_GRAB:-''} NXF_CLASSPATH=${NXF_CLASSPATH:-''} # Determine the path to this file -if [[ $NXF_PACK = all ]]; then +if [[ $NXF_PACK = dist ]]; then NXF_BIN=$(which "$0" 2>/dev/null) [ $? -gt 0 -a -f "$0" ] && NXF_BIN="./$0" fi diff --git a/packing.gradle b/packing.gradle index 041b294d54..ccbfdaf40e 100644 --- a/packing.gradle +++ b/packing.gradle @@ -90,33 +90,40 @@ protected coordinates( it ) { /* * Compile and pack all packages */ -task pack( dependsOn: [clean, compile, ":nextflow:shadowJar"]) { +task packOne( dependsOn: [clean, compile, ":nextflow:shadowJar"]) { doLast { - ant.copy(file: "modules/nextflow/build/libs/nextflow-${version}-one.jar", todir: releaseDir, overwrite: true) - ant.copy(file: "modules/nextflow/build/libs/nextflow-${version}-one.jar", todir: nextflowDir, overwrite: true) - println "\n+ Nextflow package `ONE` copied to: $releaseDir" + def source = "modules/nextflow/build/libs/nextflow-${version}-one.jar" + ant.copy(file: source, todir: releaseDir, overwrite: true) + ant.copy(file: source, todir: nextflowDir, overwrite: true) + println "\n+ Nextflow package `ONE` copied to: $releaseDir/nextflow-${version}-one.jar" } } -task packAll( dependsOn: [clean, compile, ":nf-all:shadowJar"]) { +task packDist( dependsOn: [clean, compile, ":nextflow:shadowJar"]) { doLast { file(releaseDir).mkdirs() // cleanup - def source = file("modules/nf-all/build/libs/nextflow-${version}-all.jar") - def target = file("$releaseDir/nextflow-${version}-all"); target.delete() + def source = file("modules/nextflow/build/libs/nextflow-${version}-one.jar") + def target = file("$releaseDir/nextflow-${version}-dist"); target.delete() // append the big jar target.withOutputStream { - it << file('nextflow').text.replaceAll(/NXF_PACK\=.*/, 'NXF_PACK=all') + it << file('nextflow').text.replaceAll(/NXF_PACK\=.*/, 'NXF_PACK=dist') it << new FileInputStream(source) } // execute permission "chmod +x $target".execute() // done - println "+ Nextflow package `ALL` copied to: $releaseDir\n" + println "+ Nextflow package `ALL` copied to: $target\n" } } +/* + * Compile and pack all packages + */ +task pack( dependsOn: [packOne, packDist]) { + +} task deploy( type: Exec, dependsOn: [clean, compile, pack]) { From 6784c7026adf8f4008cb9d77586726f09990be03 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Mon, 29 Jul 2024 15:57:29 +0200 Subject: [PATCH 11/13] Update nextflow [ci skip] Signed-off-by: Paolo Di Tommaso --- nextflow | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nextflow b/nextflow index d0c8c2d78d..d3c74d5a6d 100755 --- a/nextflow +++ b/nextflow @@ -77,7 +77,7 @@ function get() { exit 1 fi - printf "Downloading nextflow dependencies using $GET. It may require a few seconds, please wait .. " + printf "Downloading nextflow dependencies. It may require a few seconds, please wait .. " eval $GET; status=$? printf "\r\033[K" if [ $status -ne 0 ]; then From bb4d02636f067c1d7d49dedb95b48f2d87284613 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Mon, 29 Jul 2024 16:12:26 +0200 Subject: [PATCH 12/13] Update nextflow [ci skip] Co-authored-by: Ben Sherman Signed-off-by: Paolo Di Tommaso --- nextflow | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nextflow b/nextflow index d3c74d5a6d..86214ec377 100755 --- a/nextflow +++ b/nextflow @@ -27,9 +27,9 @@ NXF_CLI_OPTS=${NXF_CLI_OPTS:-} NXF_REMOTE_DEBUG_PORT=${NXF_REMOTE_DEBUG_PORT:-5005} NXF_LEGACY_LAUNCHER=1 -NXF_V1=$(echo $NXF_VER| cut -d'.' -f1) -NXF_V2=$(echo $NXF_VER| cut -d'.' -f2) -if [[ NXF_V1 -ge 24 ]] && [[ NXF_V2 -ge 7 ]]; then +NXF_VER_MAJOR=$(echo $NXF_VER| cut -d'.' -f1) +NXF_VER_MINOR=$(echo $NXF_VER| cut -d'.' -f2) +if [[ NXF_VER_MAJOR -ge 24 ]] && [[ NXF_VER_MINOR -ge 7 ]]; then unset NXF_LEGACY_LAUNCHER fi From 909a9f3e5adb01e6bc7af7adbb77c71a4f255a89 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 30 Jul 2024 13:10:08 +0200 Subject: [PATCH 13/13] Minor change [ci skip] Signed-off-by: Paolo Di Tommaso --- nextflow | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nextflow b/nextflow index 86214ec377..9b7395b7a9 100755 --- a/nextflow +++ b/nextflow @@ -166,8 +166,8 @@ function launch_nextflow() { for x in "${launcher[@]}"; do x="${x%\"}" x="${x#\"}" - cmdline+=("$x") - done + cmdline+=("$x") + done if [[ "$bg" ]]; then local pid_file="${NXF_PID_FILE:-.nextflow.pid}" @@ -460,6 +460,7 @@ else launcher+=("${cmd_tail[@]}") fi + # create the launch file only when using the legacy launcher (capsule) if [[ $NXF_LEGACY_LAUNCHER ]]; then # Don't show errors if the LAUNCH_FILE can't be created if mkdir -p "${NXF_LAUNCHER}" 2>/dev/null; then