Skip to content

Commit

Permalink
Refactor Conda and Spack support for Wave to Java
Browse files Browse the repository at this point in the history
Signed-off-by: Paolo Di Tommaso <[email protected]>
  • Loading branch information
pditommaso committed Jun 4, 2023
1 parent f6e4b9b commit 36b9e22
Show file tree
Hide file tree
Showing 23 changed files with 985 additions and 516 deletions.
1 change: 1 addition & 0 deletions plugins/nf-wave/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ dependencies {
compileOnly 'org.slf4j:slf4j-api:2.0.7'
compileOnly 'org.pf4j:pf4j:3.4.1'
api 'org.apache.commons:commons-compress:1.21'
api 'org.apache.commons:commons-lang3:3.12.0'
api 'com.google.code.gson:gson:2.10.1'

testImplementation(testFixtures(project(":nextflow")))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,25 +15,27 @@
*
*/

package io.seqera.wave.plugin.config
package io.seqera.wave.config;

import java.util.List;
import java.util.Map;

/**
* Conda build options
*
* @author Paolo Di Tommaso <[email protected]>
*/
class CondaOpts {

final public static String DEFAULT_MAMBA_IMAGE = 'mambaorg/micromamba:1.4.2'
public class CondaOpts {
final public static String DEFAULT_MAMBA_IMAGE = "mambaorg/micromamba:1.4.2";

final String mambaImage
final List<String> commands
final String basePackages
final public String mambaImage;
final public List<String> commands;
final public String basePackages;

CondaOpts(Map opts) {
this.mambaImage = opts.mambaImage ?: DEFAULT_MAMBA_IMAGE
this.commands = opts.commands as List<String>
this.basePackages = opts.basePackages
public CondaOpts(Map<String,?> opts) {
this.mambaImage = opts.containsKey("mambaImage") ? opts.get("mambaImage").toString(): DEFAULT_MAMBA_IMAGE;
this.commands = opts.containsKey("commands") ? (List<String>)opts.get("commands") : null;
this.basePackages = (String)opts.get("basePackages");
}

}
58 changes: 58 additions & 0 deletions plugins/nf-wave/src/main/io/seqera/wave/config/SpackOpts.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
/*
* Copyright 2013-2023, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/

package io.seqera.wave.config;

import java.util.List;
import java.util.Map;

/**
* Spack build options
*
* @author Marco De La Pierre <[email protected]>
*/
public class SpackOpts {

final static public String DEFAULT_SPACK_BUILDER_IMAGE = "spack/ubuntu-jammy:v0.19.2";
final static public String DEFAULT_SPACK_RUNNER_IMAGE = "ubuntu:22.04";
final static public String DEFAULT_SPACK_OSPACKAGES = "";
final static public String DEFAULT_SPACK_FLAGS = "-O3";

public final Boolean checksum;
public final String builderImage;
public final String runnerImage;
public final String osPackages;
public final String cFlags;
public final String cxxFlags;
public final String fFlags;
public final List<String> commands;

public SpackOpts() {
this(Map.of());
}
public SpackOpts(Map<String,?> opts) {
this.checksum = opts.get("checksum") == null || Boolean.parseBoolean(opts.get("checksum").toString());
this.builderImage = opts.containsKey("builderImage") ? opts.get("builderImage").toString() : DEFAULT_SPACK_BUILDER_IMAGE;
this.runnerImage = opts.containsKey("runnerImage") ? opts.get("runnerImage").toString() : DEFAULT_SPACK_RUNNER_IMAGE;
this.osPackages = opts.containsKey("osPackages") ? opts.get("osPackages").toString() : DEFAULT_SPACK_OSPACKAGES;
this.cFlags = opts.containsKey("cFlags") ? opts.get("cFlags").toString() : DEFAULT_SPACK_FLAGS;
this.cxxFlags = opts.containsKey("cxxFlags") ? opts.get("cxxFlags").toString() : DEFAULT_SPACK_FLAGS;
this.fFlags = opts.containsKey("fFlags") ? opts.get("fFlags").toString() : DEFAULT_SPACK_FLAGS;
this.commands = opts.containsKey("commands") ? (List<String>)opts.get("commands") : null;
}

}
Original file line number Diff line number Diff line change
@@ -1,3 +1,20 @@
/*
* Copyright 2013-2023, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/

package io.seqera.wave.plugin

import java.time.Instant
Expand Down
110 changes: 6 additions & 104 deletions plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveClient.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

package io.seqera.wave.plugin

import static io.seqera.wave.util.DockerHelper.*

import java.net.http.HttpClient
import java.net.http.HttpRequest
Expand Down Expand Up @@ -54,12 +55,10 @@ import io.seqera.wave.plugin.packer.Packer
import nextflow.Session
import nextflow.SysEnv
import nextflow.container.resolver.ContainerInfo
import nextflow.executor.BashTemplateEngine
import nextflow.fusion.FusionConfig
import nextflow.processor.Architecture
import nextflow.processor.TaskRun
import nextflow.script.bundle.ResourcesBundle
import nextflow.util.MustacheTemplateEngine
import nextflow.util.SysHelper
import org.slf4j.Logger
import org.slf4j.LoggerFactory
Expand Down Expand Up @@ -380,10 +379,11 @@ class WaveClient {
// map the recipe to a dockerfile
if( isCondaLocalFile(attrs.conda) ) {
condaFile = Path.of(attrs.conda)
dockerScript = condaFileToDockerFile()
dockerScript = condaFileToDockerFile(config.condaOpts())
}
// 'conda' attributes is resolved as the conda packages to be used
else {
dockerScript = condaRecipeToDockerFile(attrs.conda)
dockerScript = condaPackagesToDockerFile(attrs.conda, condaChannels, config.condaOpts())
}
}

Expand All @@ -399,10 +399,10 @@ class WaveClient {
// map the recipe to a dockerfile
if( isSpackFile(attrs.spack) ) {
spackFile = Path.of(attrs.spack)
dockerScript = spackFileToDockerFile(spackArch)
dockerScript = spackFileToDockerFile(spackArch, config.spackOpts())
}
else {
dockerScript = spackRecipeToDockerFile(attrs.spack, spackArch)
dockerScript = spackPackagesToDockerFile(attrs.spack, spackArch, config.spackOpts())
}
}

Expand Down Expand Up @@ -464,105 +464,7 @@ class WaveClient {
}
}

protected String condaFileToDockerFile() {
final template = """\
FROM {{base_image}}
COPY --chown=\$MAMBA_USER:\$MAMBA_USER conda.yml /tmp/conda.yml
RUN micromamba install -y -n base -f /tmp/conda.yml && \\
{{base_packages}}
micromamba clean -a -y
""".stripIndent(true)
final image = config.condaOpts().mambaImage

final basePackage = config.condaOpts().basePackages ? "micromamba install -y -n base ${config.condaOpts().basePackages} && \\".toString() : null
final binding = ['base_image': image, 'base_packages': basePackage]
final result = new MustacheTemplateEngine().render(template, binding)

return addCommands(result)
}

// Dockerfile template adpated from the Spack package manager
// https://github.com/spack/spack/blob/develop/share/spack/templates/container/Dockerfile
// LICENSE APACHE 2.0
protected String spackFileToDockerFile(String spackArch) {

String cmd_template = ''
final binding = [
'builder_image': config.spackOpts().builderImage,
'c_flags': config.spackOpts().cFlags,
'cxx_flags': config.spackOpts().cxxFlags,
'f_flags': config.spackOpts().fFlags,
'spack_arch': spackArch,
'checksum_string': config.spackOpts().checksum ? '' : '-n ',
'runner_image': config.spackOpts().runnerImage,
'os_packages': config.spackOpts().osPackages,
'add_commands': addCommands(cmd_template),
]
final template = WaveClient.class.getResource('/templates/spack/dockerfile-spack-file.txt')
try(final reader = template.newReader()) {
final result = new BashTemplateEngine().render(reader, binding)
return result
}
}

protected String addCommands(String result) {
if( config.condaOpts().commands )
for( String cmd : config.condaOpts().commands ) {
result += cmd + "\n"
}
if( config.spackOpts().commands )
for( String cmd : config.spackOpts().commands ) {
result += cmd + "\n"
}
return result
}

protected String condaRecipeToDockerFile(String recipe) {
final template = """\
FROM {{base_image}}
RUN \\
micromamba install -y -n base {{channel_opts}} \\
{{target}} \\
{{base_packages}}
&& micromamba clean -a -y
""".stripIndent(true)

final channelsOpts = condaChannels.collect(it -> "-c $it").join(' ')
final image = config.condaOpts().mambaImage
final target = recipe.startsWith('http://') || recipe.startsWith('https://')
? "-f $recipe".toString()
: recipe
final basePackage = config.condaOpts().basePackages ? "&& micromamba install -y -n base ${config.condaOpts().basePackages} \\".toString() : null
final binding = [base_image: image, channel_opts: channelsOpts, target:target, base_packages: basePackage]
final result = new MustacheTemplateEngine().render(template, binding)
return addCommands(result)
}

// Dockerfile template adpated from the Spack package manager
// https://github.com/spack/spack/blob/develop/share/spack/templates/container/Dockerfile
// LICENSE APACHE 2.0
protected String spackRecipeToDockerFile(String recipe, String spackArch) {

String cmd_template = ''
final binding = [
'recipe': recipe,
'builder_image': config.spackOpts().builderImage,
'c_flags': config.spackOpts().cFlags,
'cxx_flags': config.spackOpts().cxxFlags,
'f_flags': config.spackOpts().fFlags,
'spack_arch': spackArch,
'checksum_string': config.spackOpts().checksum ? '' : '-n ',
'runner_image': config.spackOpts().runnerImage,
'os_packages': config.spackOpts().osPackages,
'add_commands': addCommands(cmd_template),
]
final template = WaveClient.class.getResource('/templates/spack/dockerfile-spack-recipe.txt')

try(final reader = template.newReader()) {
final result = new BashTemplateEngine().render(reader, binding)
return result
}
}

static protected boolean isCondaLocalFile(String value) {
if( value.contains('\n') )
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2020-2022, Seqera Labs
* Copyright 2013-2023, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2020-2022, Seqera Labs
* Copyright 2013-2023, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2020-2022, Seqera Labs
* Copyright 2013-2023, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2020-2022, Seqera Labs
* Copyright 2013-2023, Seqera Labs
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ package io.seqera.wave.plugin.config

import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import io.seqera.wave.config.CondaOpts
import io.seqera.wave.config.SpackOpts
import nextflow.util.Duration
/**
* Model Wave client configuration
Expand Down
Loading

0 comments on commit 36b9e22

Please sign in to comment.