From ab826041528a92a6e8b326bb4b4432cc00629505 Mon Sep 17 00:00:00 2001 From: tdayris Date: Mon, 21 Sep 2020 09:16:53 +0200 Subject: [PATCH 01/61] [fix] (template): Missing code in wrappers' doc. Error #187 --- docs/generate_docs.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/generate_docs.py b/docs/generate_docs.py index c471d06d2f..446fe83a9e 100644 --- a/docs/generate_docs.py +++ b/docs/generate_docs.py @@ -34,10 +34,10 @@ TOOL_TEMPLATE = Template(f.read()) with open(os.path.join(BASE_DIR, "_templates", "wrapper.rst")) as f: - TEMPLATE = Template(f.read()) + TEMPLATE_WRAPPER = Template(f.read()) with open(os.path.join(BASE_DIR, "_templates", "meta_wrapper.rst")) as f: - TEMPLATE = Template(f.read()) + TEMPLATE_META = Template(f.read()) def get_tool_dir(tool): @@ -88,7 +88,7 @@ def render_wrapper(path, target): name = meta["name"].replace(" ", "_") + ".rst" os.makedirs(os.path.dirname(target), exist_ok=True) with open(target, "w") as readme: - rst = TEMPLATE.render( + rst = TEMPLATE_WRAPPER.render( snakefile=snakefile, wrapper=wrapper, wrapper_lang=wrapper_lang, @@ -113,7 +113,7 @@ def render_meta(path, target): with open(os.path.join(path, "test", "Snakefile")) as snakefile: snakefile = textwrap.indent(snakefile.read(), " ").replace("master", TAG) - + wrappers = [] for uw in used_wrappers: wrapper = os.path.join(WRAPPER_DIR, uw, "wrapper.py") @@ -124,11 +124,11 @@ def render_meta(path, target): with open(wrapper) as wrapper: wrapper = textwrap.indent(wrapper.read(), " ") wrappers.append((wrapper, wrapper_lang, uw)) - + name = meta["name"].replace(" ", "_") + ".rst" os.makedirs(os.path.dirname(target), exist_ok=True) with open(target, "w") as readme: - rst = TEMPLATE.render( + rst = TEMPLATE_META.render( snakefile=snakefile, wrappers=wrappers, usedwrappers=used_wrappers, From ab5c6c7c57aed1e1c4ab28f8cb69b4ff2dc23cc0 Mon Sep 17 00:00:00 2001 From: tdayris Date: Mon, 21 Sep 2020 16:50:29 +0200 Subject: [PATCH 02/61] [dev] (Picard): Memory --- bio/picard/markduplicates/wrapper.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/bio/picard/markduplicates/wrapper.py b/bio/picard/markduplicates/wrapper.py index aa065b0f94..5fa4ac6ccb 100644 --- a/bio/picard/markduplicates/wrapper.py +++ b/bio/picard/markduplicates/wrapper.py @@ -8,8 +8,16 @@ log = snakemake.log_fmt_shell(stdout=True, stderr=True) +memory = "" +if "mem_mb" in snakemake.resources.keys(): + memory = "-Xmx{}M".format(str(snakemake.resources["mem_mb"])) + shell( - "picard MarkDuplicates {snakemake.params} INPUT={snakemake.input} " - "OUTPUT={snakemake.output.bam} METRICS_FILE={snakemake.output.metrics} " - "{log}" + "picard MarkDuplicates " # Tool and its subcommand + "{memory} " # Automatic Xmx java option + "{snakemake.params} " # User defined parmeters + "INPUT={snakemake.input} " # Input file + "OUTPUT={snakemake.output.bam} " # Output bam + "METRICS_FILE={snakemake.output.metrics} " # Output metrics + "{log}" # Logging ) From 9fb664a2507479b692dcc89b3e631886c9b3baf0 Mon Sep 17 00:00:00 2001 From: tdayris Date: Tue, 22 Sep 2020 08:44:45 +0200 Subject: [PATCH 03/61] [dev] (applybqsr): Automatic Xmx fill --- bio/gatk/applybqsr/wrapper.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bio/gatk/applybqsr/wrapper.py b/bio/gatk/applybqsr/wrapper.py index d38b1cd542..dfd838d01f 100644 --- a/bio/gatk/applybqsr/wrapper.py +++ b/bio/gatk/applybqsr/wrapper.py @@ -8,6 +8,8 @@ extra = snakemake.params.get("extra", "") java_opts = snakemake.params.get("java_opts", "") +if "mem_mb" in snakemake.resources.keys() and not "-Xmx" in java_opts: + java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) log = snakemake.log_fmt_shell(stdout=True, stderr=True, append=True) shell( From a54fd7683b2c3c0027da9c0970f58131a28f5fc1 Mon Sep 17 00:00:00 2001 From: tdayris Date: Tue, 22 Sep 2020 08:47:33 +0200 Subject: [PATCH 04/61] [dev] (baserecal): Automatic Xmx fill --- bio/gatk/baserecalibrator/wrapper.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bio/gatk/baserecalibrator/wrapper.py b/bio/gatk/baserecalibrator/wrapper.py index 70eb5ba33c..760070a6bd 100644 --- a/bio/gatk/baserecalibrator/wrapper.py +++ b/bio/gatk/baserecalibrator/wrapper.py @@ -8,6 +8,9 @@ extra = snakemake.params.get("extra", "") java_opts = snakemake.params.get("java_opts", "") +if "mem_mb" in snakemake.resources.keys() and not "-Xmx" in java_opts: + java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + log = snakemake.log_fmt_shell(stdout=True, stderr=True) known = snakemake.input.get("known", "") From 24b2a220f7f63c6e5f6a81b784d01fd25f1d6b64 Mon Sep 17 00:00:00 2001 From: tdayris Date: Tue, 22 Sep 2020 08:52:36 +0200 Subject: [PATCH 05/61] [dev] (baserecalspark): Automatic Xmx fill --- bio/gatk/baserecalibratorspark/wrapper.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bio/gatk/baserecalibratorspark/wrapper.py b/bio/gatk/baserecalibratorspark/wrapper.py index aa325dbde1..bb7e54e595 100644 --- a/bio/gatk/baserecalibratorspark/wrapper.py +++ b/bio/gatk/baserecalibratorspark/wrapper.py @@ -13,6 +13,8 @@ ) spark_extra = snakemake.params.get("spark_extra", "") java_opts = snakemake.params.get("java_opts", "") +if "mem_mb" in snakemake.resources.keys() and not "-Xmx" in java_opts: + java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) log = snakemake.log_fmt_shell(stdout=True, stderr=True) known = snakemake.input.get("known", "") From 93d8a528b245640ab9b8d4bc63575999079473fc Mon Sep 17 00:00:00 2001 From: tdayris Date: Tue, 22 Sep 2020 09:09:23 +0200 Subject: [PATCH 06/61] [dev] (combinegvcf): Automatic Xmx fill --- bio/gatk/combinegvcfs/wrapper.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bio/gatk/combinegvcfs/wrapper.py b/bio/gatk/combinegvcfs/wrapper.py index 5c0f96a55f..7766394d02 100644 --- a/bio/gatk/combinegvcfs/wrapper.py +++ b/bio/gatk/combinegvcfs/wrapper.py @@ -11,6 +11,9 @@ extra = snakemake.params.get("extra", "") java_opts = snakemake.params.get("java_opts", "") +if "mem_mb" in snakemake.resources.keys() and not "-Xmx" in java_opts: + java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + gvcfs = list(map("-V {}".format, snakemake.input.gvcfs)) log = snakemake.log_fmt_shell(stdout=True, stderr=True) From 56a6c78aab266b6625d6de210995e3db41290eed Mon Sep 17 00:00:00 2001 From: tdayris Date: Tue, 22 Sep 2020 09:11:32 +0200 Subject: [PATCH 07/61] [dev] (genotypegvcfs): Automatic Xmx fill --- bio/gatk/genotypegvcfs/wrapper.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bio/gatk/genotypegvcfs/wrapper.py b/bio/gatk/genotypegvcfs/wrapper.py index 581ced212b..334491dd6e 100644 --- a/bio/gatk/genotypegvcfs/wrapper.py +++ b/bio/gatk/genotypegvcfs/wrapper.py @@ -11,6 +11,8 @@ extra = snakemake.params.get("extra", "") java_opts = snakemake.params.get("java_opts", "") +if "mem_mb" in snakemake.resources.keys() and not "-Xmx" in java_opts: + java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) log = snakemake.log_fmt_shell(stdout=True, stderr=True) shell( From 3f0087079a028bd8c3c78e3271e0a12e56015c77 Mon Sep 17 00:00:00 2001 From: tdayris Date: Tue, 22 Sep 2020 09:12:08 +0200 Subject: [PATCH 08/61] [dev] (haplotypecaller): Automatic Xmx fill --- bio/gatk/haplotypecaller/wrapper.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bio/gatk/haplotypecaller/wrapper.py b/bio/gatk/haplotypecaller/wrapper.py index cdbd1a71da..9e675a8611 100644 --- a/bio/gatk/haplotypecaller/wrapper.py +++ b/bio/gatk/haplotypecaller/wrapper.py @@ -14,6 +14,9 @@ extra = snakemake.params.get("extra", "") java_opts = snakemake.params.get("java_opts", "") +if "mem_mb" in snakemake.resources.keys() and not "-Xmx" in java_opts: + java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + bams = snakemake.input.bam if isinstance(bams, str): bams = [bams] From bf61cfaba23f9ee3edd47daed80eca0d6a9451ae Mon Sep 17 00:00:00 2001 From: tdayris Date: Tue, 22 Sep 2020 09:15:17 +0200 Subject: [PATCH 09/61] [dev] (selectvariants): Automatic Xmx fill --- bio/gatk/selectvariants/wrapper.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bio/gatk/selectvariants/wrapper.py b/bio/gatk/selectvariants/wrapper.py index cd6df94062..eba819006c 100644 --- a/bio/gatk/selectvariants/wrapper.py +++ b/bio/gatk/selectvariants/wrapper.py @@ -8,6 +8,8 @@ extra = snakemake.params.get("extra", "") java_opts = snakemake.params.get("java_opts", "") +if "mem_mb" in snakemake.resources.keys() and not "-Xmx" in java_opts: + java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) log = snakemake.log_fmt_shell(stdout=True, stderr=True) shell( From 855d57c7a63b2acf6b132fe9c1247c9acc7d80e1 Mon Sep 17 00:00:00 2001 From: tdayris Date: Tue, 22 Sep 2020 09:19:12 +0200 Subject: [PATCH 10/61] [dev] (splitncigarreads): Automatic Xmx fill --- bio/gatk/splitncigarreads/wrapper.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bio/gatk/splitncigarreads/wrapper.py b/bio/gatk/splitncigarreads/wrapper.py index 7431abb14e..025aa9535a 100644 --- a/bio/gatk/splitncigarreads/wrapper.py +++ b/bio/gatk/splitncigarreads/wrapper.py @@ -9,6 +9,8 @@ extra = snakemake.params.get("extra", "") java_opts = snakemake.params.get("java_opts", "") +if "mem_mb" in snakemake.resources.keys() and not "-Xmx" in java_opts: + java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) log = snakemake.log_fmt_shell(stdout=True, stderr=True) shell( From 0869139067bf405daef4cbdfc8b84b19607a812a Mon Sep 17 00:00:00 2001 From: tdayris Date: Tue, 22 Sep 2020 09:21:11 +0200 Subject: [PATCH 11/61] [dev] (variantfiltration): Automatic Xmx fill --- bio/gatk/variantfiltration/wrapper.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bio/gatk/variantfiltration/wrapper.py b/bio/gatk/variantfiltration/wrapper.py index d2ca667f5d..ded75a1c4e 100644 --- a/bio/gatk/variantfiltration/wrapper.py +++ b/bio/gatk/variantfiltration/wrapper.py @@ -8,6 +8,9 @@ extra = snakemake.params.get("extra", "") java_opts = snakemake.params.get("java_opts", "") +if "mem_mb" in snakemake.resources.keys() and not "-Xmx" in java_opts: + java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + filters = [ "--filter-name {} --filter-expression '{}'".format(name, expr.replace("'", "\\'")) for name, expr in snakemake.params.filters.items() From 2c369d3cf7d036e2678f8f55db538e10e1c9df20 Mon Sep 17 00:00:00 2001 From: tdayris Date: Tue, 22 Sep 2020 09:24:54 +0200 Subject: [PATCH 12/61] [dev] (variantrecalibrator): Automatic Xmx fill --- bio/gatk/variantrecalibrator/wrapper.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bio/gatk/variantrecalibrator/wrapper.py b/bio/gatk/variantrecalibrator/wrapper.py index 9c53a7b957..208a5ae9b3 100644 --- a/bio/gatk/variantrecalibrator/wrapper.py +++ b/bio/gatk/variantrecalibrator/wrapper.py @@ -11,6 +11,8 @@ extra = snakemake.params.get("extra", "") java_opts = snakemake.params.get("java_opts", "") +if "mem_mb" in snakemake.resource.keys() and not "-Xmx" in java_opts: + java_opts += " -Xmx{}M".format(snakemake.resource["mem_mb"]) def fmt_res(resname, resparams): From 8dc5ed887388ad57f33911094ce6e0b0ef10b3b6 Mon Sep 17 00:00:00 2001 From: tdayris Date: Tue, 22 Sep 2020 09:53:29 +0200 Subject: [PATCH 13/61] [dev] (snpsift_annotate): Automatix Xmx fill --- bio/snpsift/annotate/wrapper.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bio/snpsift/annotate/wrapper.py b/bio/snpsift/annotate/wrapper.py index 77cbc2f451..0de6e98586 100644 --- a/bio/snpsift/annotate/wrapper.py +++ b/bio/snpsift/annotate/wrapper.py @@ -9,6 +9,9 @@ log = snakemake.log_fmt_shell(stdout=False, stderr=True) extra = snakemake.params.get("extra", "") +if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: + extra += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + min_threads = 1 incall = snakemake.input["call"] From 6148de2342066d7d421ea895d16893fd2211dee1 Mon Sep 17 00:00:00 2001 From: tdayris Date: Tue, 22 Sep 2020 10:15:50 +0200 Subject: [PATCH 14/61] [dev] (snpsift_dbnsfp): Automatic Xmx fill --- bio/snpsift/dbnsfp/wrapper.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bio/snpsift/dbnsfp/wrapper.py b/bio/snpsift/dbnsfp/wrapper.py index bcf15a36a0..ce82366941 100644 --- a/bio/snpsift/dbnsfp/wrapper.py +++ b/bio/snpsift/dbnsfp/wrapper.py @@ -9,6 +9,8 @@ log = snakemake.log_fmt_shell(stdout=False, stderr=True) extra = snakemake.params.get("extra", "") +if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: + extra += " -Xmx{}M".format(snakemake.resources["mem_mb"]) # Using user-defined file if requested db = snakemake.input.get("dbNSFP", "") From 6eb309fb95f5b15a2e8fe84311583ff7c2553c75 Mon Sep 17 00:00:00 2001 From: tdayris Date: Tue, 22 Sep 2020 10:19:39 +0200 Subject: [PATCH 15/61] [dev] (snpsift): genesets, gwascat and varType: automatic Xmx fill --- bio/snpsift/genesets/wrapper.py | 2 ++ bio/snpsift/gwascat/wrapper.py | 2 ++ bio/snpsift/varType/wrapper.py | 2 ++ 3 files changed, 6 insertions(+) diff --git a/bio/snpsift/genesets/wrapper.py b/bio/snpsift/genesets/wrapper.py index 59f33258c4..3185fe0324 100644 --- a/bio/snpsift/genesets/wrapper.py +++ b/bio/snpsift/genesets/wrapper.py @@ -9,6 +9,8 @@ log = snakemake.log_fmt_shell(stdout=False, stderr=True) extra = snakemake.params.get("extra", "") +if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: + extra += " -Xmx{}M".format(snakemake.resources["mem_mb"]) min_threads = 1 # Uncompression shall be done according to user-defined input diff --git a/bio/snpsift/gwascat/wrapper.py b/bio/snpsift/gwascat/wrapper.py index 3df41c9eb8..e7748cb159 100644 --- a/bio/snpsift/gwascat/wrapper.py +++ b/bio/snpsift/gwascat/wrapper.py @@ -9,6 +9,8 @@ log = snakemake.log_fmt_shell(stdout=False, stderr=True) extra = snakemake.params.get("extra", "") +if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: + extra += " -Xmx{}M".format(snakemake.resources["mem_mb"]) min_threads = 1 diff --git a/bio/snpsift/varType/wrapper.py b/bio/snpsift/varType/wrapper.py index 57132fdc96..994301a109 100644 --- a/bio/snpsift/varType/wrapper.py +++ b/bio/snpsift/varType/wrapper.py @@ -10,6 +10,8 @@ log = snakemake.log_fmt_shell(stdout=False, stderr=True) extra = snakemake.params.get("extra", "") +if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: + extra += " -Xmx{}M".format(snakemake.resources["mem_mb"]) shell( "SnpSift varType" # Tool and its subcommand From 6c8d6ab8fb03de87f6f758c3eb84b524f6355e65 Mon Sep 17 00:00:00 2001 From: tdayris Date: Tue, 22 Sep 2020 10:34:36 +0200 Subject: [PATCH 16/61] [dev] (trinity): Aotumatix Xmx fill --- bio/trinity/wrapper.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bio/trinity/wrapper.py b/bio/trinity/wrapper.py index fa6433662b..8758222f32 100644 --- a/bio/trinity/wrapper.py +++ b/bio/trinity/wrapper.py @@ -10,6 +10,10 @@ extra = snakemake.params.get("extra", "") max_memory = snakemake.params.get("max_memory", "10G") +if "mem_mb" in snakemake.resources.keys(): + if snakemake.resources["mem_mb"] > 10240: + rounded_mb_to_gb = int(snakemake.resources["mem_mb"] / 1024) + max_memory = "{}G".format(rounded_mb_to_gb) # allow multiple input files for single assembly left = snakemake.input.get("left") From 321a5dabb6636b3f6293db16c1e95ff802666846 Mon Sep 17 00:00:00 2001 From: tdayris Date: Tue, 22 Sep 2020 11:07:09 +0200 Subject: [PATCH 17/61] [dev] (Mutect2): Java options added, with automatic Xmx fill --- bio/gatk/mutect/wrapper.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bio/gatk/mutect/wrapper.py b/bio/gatk/mutect/wrapper.py index e25c557f33..d7e16452f3 100644 --- a/bio/gatk/mutect/wrapper.py +++ b/bio/gatk/mutect/wrapper.py @@ -11,9 +11,12 @@ log = snakemake.log_fmt_shell(stdout=True, stderr=True) extra = snakemake.params.get("extra", "") +java_opts = snakemake.params.get("java_opts", "") +if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in java_opts: + java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) shell( - "gatk Mutect2 " # Tool and its subprocess + "gatk --java-options '{java_opts}' Mutect2 " # Tool and its subprocess "--input {snakemake.input.map} " # Path to input mapping file "--output {snakemake.output.vcf} " # Path to output vcf file "--reference {snakemake.input.fasta} " # Path to reference fasta file From e6ee27b6f2fc5a3c26862b688116921251c663de Mon Sep 17 00:00:00 2001 From: tdayris Date: Fri, 25 Sep 2020 14:15:10 +0200 Subject: [PATCH 18/61] [fix] (Picard): Test if -Xmx was not set before --- bio/picard/markduplicates/wrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bio/picard/markduplicates/wrapper.py b/bio/picard/markduplicates/wrapper.py index 5fa4ac6ccb..c67fff39c0 100644 --- a/bio/picard/markduplicates/wrapper.py +++ b/bio/picard/markduplicates/wrapper.py @@ -9,7 +9,7 @@ log = snakemake.log_fmt_shell(stdout=True, stderr=True) memory = "" -if "mem_mb" in snakemake.resources.keys(): +if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in snakemake.params: memory = "-Xmx{}M".format(str(snakemake.resources["mem_mb"])) shell( From 2c089436b63a0902bfefe542c01c9b6831a77591 Mon Sep 17 00:00:00 2001 From: tdayris Date: Mon, 19 Oct 2020 08:05:34 +0200 Subject: [PATCH 19/61] [dev] (SnpEff): Automatic Xmx fill --- bio/snpeff/annotate/wrapper.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bio/snpeff/annotate/wrapper.py b/bio/snpeff/annotate/wrapper.py index ecd3645f25..c522a5a715 100644 --- a/bio/snpeff/annotate/wrapper.py +++ b/bio/snpeff/annotate/wrapper.py @@ -25,6 +25,8 @@ log = snakemake.log_fmt_shell(stdout=False, stderr=True) extra = snakemake.params.get("extra", "") +if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: + extra += " -Xmx{}M".format(snakemake.resources["mem_mb"]) data_dir = Path(snakemake.input.db).parent.resolve() From 5defcbfb2fe5b56e2f1b09485a5632901d9e5962 Mon Sep 17 00:00:00 2001 From: tdayris Date: Mon, 19 Oct 2020 08:06:49 +0200 Subject: [PATCH 20/61] [dev] (SnpEff): Automatic Xmx fill in download wrapper --- bio/snpeff/download/wrapper.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bio/snpeff/download/wrapper.py b/bio/snpeff/download/wrapper.py index 5832841637..2ac1b2bd84 100644 --- a/bio/snpeff/download/wrapper.py +++ b/bio/snpeff/download/wrapper.py @@ -10,4 +10,7 @@ outdir = Path(snakemake.output[0]).parent.resolve() log = snakemake.log_fmt_shell(stdout=False, stderr=True) +if "mem_mb" in snakemake.resources.keys(): + extra += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + shell("snpEff download -dataDir {outdir} {reference} {log}") From 8e4a9f3b4dc707ad255fe6a06931e78cdafba79b Mon Sep 17 00:00:00 2001 From: tdayris Date: Mon, 19 Oct 2020 08:21:47 +0200 Subject: [PATCH 21/61] [dev] (Jannovar): Automatic Xmx fill --- bio/jannovar/wrapper.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bio/jannovar/wrapper.py b/bio/jannovar/wrapper.py index db619f1d3f..8d6a8abbbc 100644 --- a/bio/jannovar/wrapper.py +++ b/bio/jannovar/wrapper.py @@ -11,6 +11,8 @@ log = snakemake.log_fmt_shell(stdout=False, stderr=True) extra = snakemake.params.get("extra", "") +if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: + extra += " -Xmx{}M".format(snakemake.resources["mem_mb"]) pedigree = snakemake.input.get("pedigree", "") if pedigree: From a9433c845ddedc09f589f12a7cd724b0a078ee09 Mon Sep 17 00:00:00 2001 From: tdayris Date: Mon, 19 Oct 2020 08:42:08 +0200 Subject: [PATCH 22/61] [dev] (Varscan): Automatic Xmx fill --- bio/varscan/mpileup2indel/wrapper.py | 2 ++ bio/varscan/mpileup2snp/wrapper.py | 2 ++ bio/varscan/somatic/wrapper.py | 2 ++ 3 files changed, 6 insertions(+) diff --git a/bio/varscan/mpileup2indel/wrapper.py b/bio/varscan/mpileup2indel/wrapper.py index d267c09496..31826418ab 100644 --- a/bio/varscan/mpileup2indel/wrapper.py +++ b/bio/varscan/mpileup2indel/wrapper.py @@ -12,6 +12,8 @@ # Gathering extra parameters and logging behaviour log = snakemake.log_fmt_shell(stdout=False, stderr=True) extra = snakemake.params.get("extra", "") +if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: + extra += " -Xmx{}M".format(snakemake.resources["mem_mb"]) # In case input files are gzipped mpileup files, # they are being unzipped and piped diff --git a/bio/varscan/mpileup2snp/wrapper.py b/bio/varscan/mpileup2snp/wrapper.py index 4129464e86..d8765e45bc 100644 --- a/bio/varscan/mpileup2snp/wrapper.py +++ b/bio/varscan/mpileup2snp/wrapper.py @@ -12,6 +12,8 @@ # Gathering extra parameters and logging behaviour log = snakemake.log_fmt_shell(stdout=False, stderr=True) extra = snakemake.params.get("extra", "") +if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: + extra += " -Xmx{}M".format(snakemake.resources["mem_mb"]) # In case input files are gzipped mpileup files, # they are being unzipped and piped diff --git a/bio/varscan/somatic/wrapper.py b/bio/varscan/somatic/wrapper.py index 4679762741..6ecfeb16e1 100644 --- a/bio/varscan/somatic/wrapper.py +++ b/bio/varscan/somatic/wrapper.py @@ -14,6 +14,8 @@ # Defining logging and gathering extra parameters log = snakemake.log_fmt_shell(stdout=True, stderr=True) extra = snakemake.params.get("extra", "") +if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: + extra += " -Xmx{}M".format(snakemake.resources["mem_mb"]) # Building output dirs makedirs(op.dirname(snakemake.output.snp)) From bab609f03d95ba9649605b58675ee9b32d17258d Mon Sep 17 00:00:00 2001 From: tdayris Date: Mon, 19 Oct 2020 08:48:48 +0200 Subject: [PATCH 23/61] [dev] (trimmomatic): Automatic Xmx fill --- bio/trimmomatic/pe/wrapper.py | 2 ++ bio/trimmomatic/se/wrapper.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/bio/trimmomatic/pe/wrapper.py b/bio/trimmomatic/pe/wrapper.py index 9bef8eff57..c9faaa6857 100644 --- a/bio/trimmomatic/pe/wrapper.py +++ b/bio/trimmomatic/pe/wrapper.py @@ -66,6 +66,8 @@ def compose_output_gz(filename, threads, compression_level): extra = snakemake.params.get("extra", "") +if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: + extra += " -Xmx{}M".format(snakemake.resources["mem_mb"]) log = snakemake.log_fmt_shell(stdout=True, stderr=True) compression_level = snakemake.params.get("compression_level", "-5") trimmer = " ".join(snakemake.params.trimmer) diff --git a/bio/trimmomatic/se/wrapper.py b/bio/trimmomatic/se/wrapper.py index f47fc2284d..76e2ab367d 100644 --- a/bio/trimmomatic/se/wrapper.py +++ b/bio/trimmomatic/se/wrapper.py @@ -66,6 +66,8 @@ def compose_output_gz(filename, threads, compression_level): extra = snakemake.params.get("extra", "") +if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: + extra += " -Xmx{}M".format(snakemake.resources["mem_mb"]) log = snakemake.log_fmt_shell(stdout=True, stderr=True) compression_level = snakemake.params.get("compression_level", "-5") trimmer = " ".join(snakemake.params.trimmer) From 43700aad656b9025b0c6f6d544a12510ac270cd0 Mon Sep 17 00:00:00 2001 From: tdayris Date: Tue, 20 Oct 2020 10:03:03 +0200 Subject: [PATCH 24/61] [dev] (gatk4): Update with comments from @dlaehnemann and java_tmp --- bio/gatk/applybqsr/wrapper.py | 17 ++++++++++++++++- bio/gatk/baserecalibrator/wrapper.py | 13 +++++++++++++ bio/gatk/baserecalibratorspark/wrapper.py | 13 +++++++++++++ bio/gatk/combinegvcfs/wrapper.py | 13 +++++++++++++ bio/gatk/genotypegvcfs/wrapper.py | 13 +++++++++++++ bio/gatk/haplotypecaller/wrapper.py | 13 +++++++++++++ bio/gatk/mutect/wrapper.py | 15 ++++++++++++++- bio/gatk/selectvariants/wrapper.py | 13 +++++++++++++ bio/gatk/splitncigarreads/wrapper.py | 13 +++++++++++++ bio/gatk/variantfiltration/wrapper.py | 13 +++++++++++++ bio/gatk/variantrecalibrator/wrapper.py | 17 +++++++++++++++-- 11 files changed, 149 insertions(+), 4 deletions(-) diff --git a/bio/gatk/applybqsr/wrapper.py b/bio/gatk/applybqsr/wrapper.py index dfd838d01f..784e972abd 100644 --- a/bio/gatk/applybqsr/wrapper.py +++ b/bio/gatk/applybqsr/wrapper.py @@ -8,12 +8,27 @@ extra = snakemake.params.get("extra", "") java_opts = snakemake.params.get("java_opts", "") + +# Getting memory in megabytes, if java opts is not filled with -Xmx parameter +# By doing so, backward compatibility is preserved if "mem_mb" in snakemake.resources.keys() and not "-Xmx" in java_opts: java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) +# Getting memory in gigabytes, for user convenience. Please prefer the use +# of mem_mb over mem_gb as advised in documentation. +elif "mem_gb" in snakemake.resources.keys() and not "-Xmx" in java_opts: + java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) + +# Getting java temp directory from output files list, if -Djava.io.tmpdir +# is not provided in java parameters. By doing so, backward compatibility is +# not borken. +if "java_temp" in snakemake.output.keys() and not "-Djava.io.tmpdir" in java_opts: + java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) + log = snakemake.log_fmt_shell(stdout=True, stderr=True, append=True) shell( - "gatk --java-options '{java_opts}' ApplyBQSR {extra} -R {snakemake.input.ref} -I {snakemake.input.bam} " + "gatk --java-options '{java_opts}' ApplyBQSR {extra} " + "-R {snakemake.input.ref} -I {snakemake.input.bam} " "--bqsr-recal-file {snakemake.input.recal_table} " "-O {snakemake.output.bam} {log}" ) diff --git a/bio/gatk/baserecalibrator/wrapper.py b/bio/gatk/baserecalibrator/wrapper.py index 760070a6bd..737d52c388 100644 --- a/bio/gatk/baserecalibrator/wrapper.py +++ b/bio/gatk/baserecalibrator/wrapper.py @@ -8,9 +8,22 @@ extra = snakemake.params.get("extra", "") java_opts = snakemake.params.get("java_opts", "") + +# Getting memory in megabytes, if java opts is not filled with -Xmx parameter +# By doing so, backward compatibility is preserved if "mem_mb" in snakemake.resources.keys() and not "-Xmx" in java_opts: java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) +# Getting memory in gigabytes, for user convenience. Please prefer the use +# of mem_mb over mem_gb as advised in documentation. +elif "mem_gb" in snakemake.resources.keys() and not "-Xmx" in java_opts: + java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) + +# Getting java temp directory from output files list, if -Djava.io.tmpdir +# is not provided in java parameters. By doing so, backward compatibility is +# not broken. +if "java_temp" in snakemake.output.keys() and not "-Djava.io.tmpdir" in java_opts: + java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) log = snakemake.log_fmt_shell(stdout=True, stderr=True) known = snakemake.input.get("known", "") diff --git a/bio/gatk/baserecalibratorspark/wrapper.py b/bio/gatk/baserecalibratorspark/wrapper.py index bb7e54e595..09146524a9 100644 --- a/bio/gatk/baserecalibratorspark/wrapper.py +++ b/bio/gatk/baserecalibratorspark/wrapper.py @@ -13,9 +13,22 @@ ) spark_extra = snakemake.params.get("spark_extra", "") java_opts = snakemake.params.get("java_opts", "") +# Getting memory in megabytes, if java opts is not filled with -Xmx parameter +# By doing so, backward compatibility is preserved if "mem_mb" in snakemake.resources.keys() and not "-Xmx" in java_opts: java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) +# Getting memory in gigabytes, for user convenience. Please prefer the use +# of mem_mb over mem_gb as advised in documentation. +elif "mem_gb" in snakemake.resources.keys() and not "-Xmx" in java_opts: + java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) + +# Getting java temp directory from output files list, if -Djava.io.tmpdir +# is not provided in java parameters. By doing so, backward compatibility is +# not broken. +if "java_temp" in snakemake.output.keys() and not "-Djava.io.tmpdir" in java_opts: + java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) + log = snakemake.log_fmt_shell(stdout=True, stderr=True) known = snakemake.input.get("known", "") if known: diff --git a/bio/gatk/combinegvcfs/wrapper.py b/bio/gatk/combinegvcfs/wrapper.py index 7766394d02..fcedf72dbb 100644 --- a/bio/gatk/combinegvcfs/wrapper.py +++ b/bio/gatk/combinegvcfs/wrapper.py @@ -11,9 +11,22 @@ extra = snakemake.params.get("extra", "") java_opts = snakemake.params.get("java_opts", "") +# Getting memory in megabytes, if java opts is not filled with -Xmx parameter +# By doing so, backward compatibility is preserved if "mem_mb" in snakemake.resources.keys() and not "-Xmx" in java_opts: java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) +# Getting memory in gigabytes, for user convenience. Please prefer the use +# of mem_mb over mem_gb as advised in documentation. +elif "mem_gb" in snakemake.resources.keys() and not "-Xmx" in java_opts: + java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) + +# Getting java temp directory from output files list, if -Djava.io.tmpdir +# is not provided in java parameters. By doing so, backward compatibility is +# not broken. +if "java_temp" in snakemake.output.keys() and not "-Djava.io.tmpdir" in java_opts: + java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) + gvcfs = list(map("-V {}".format, snakemake.input.gvcfs)) log = snakemake.log_fmt_shell(stdout=True, stderr=True) diff --git a/bio/gatk/genotypegvcfs/wrapper.py b/bio/gatk/genotypegvcfs/wrapper.py index 334491dd6e..5d580ca482 100644 --- a/bio/gatk/genotypegvcfs/wrapper.py +++ b/bio/gatk/genotypegvcfs/wrapper.py @@ -11,9 +11,22 @@ extra = snakemake.params.get("extra", "") java_opts = snakemake.params.get("java_opts", "") +# Getting memory in megabytes, if java opts is not filled with -Xmx parameter +# By doing so, backward compatibility is preserved if "mem_mb" in snakemake.resources.keys() and not "-Xmx" in java_opts: java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) +# Getting memory in gigabytes, for user convenience. Please prefer the use +# of mem_mb over mem_gb as advised in documentation. +elif "mem_gb" in snakemake.resources.keys() and not "-Xmx" in java_opts: + java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) + +# Getting java temp directory from output files list, if -Djava.io.tmpdir +# is not provided in java parameters. By doing so, backward compatibility is +# not broken. +if "java_temp" in snakemake.output.keys() and not "-Djava.io.tmpdir" in java_opts: + java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) + log = snakemake.log_fmt_shell(stdout=True, stderr=True) shell( "gatk --java-options '{java_opts}' GenotypeGVCFs {extra} " diff --git a/bio/gatk/haplotypecaller/wrapper.py b/bio/gatk/haplotypecaller/wrapper.py index 9e675a8611..d2d83552a2 100644 --- a/bio/gatk/haplotypecaller/wrapper.py +++ b/bio/gatk/haplotypecaller/wrapper.py @@ -14,9 +14,22 @@ extra = snakemake.params.get("extra", "") java_opts = snakemake.params.get("java_opts", "") +# Getting memory in megabytes, if java opts is not filled with -Xmx parameter +# By doing so, backward compatibility is preserved if "mem_mb" in snakemake.resources.keys() and not "-Xmx" in java_opts: java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) +# Getting memory in gigabytes, for user convenience. Please prefer the use +# of mem_mb over mem_gb as advised in documentation. +elif "mem_gb" in snakemake.resources.keys() and not "-Xmx" in java_opts: + java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) + +# Getting java temp directory from output files list, if -Djava.io.tmpdir +# is not provided in java parameters. By doing so, backward compatibility is +# not broken. +if "java_temp" in snakemake.output.keys() and not "-Djava.io.tmpdir" in java_opts: + java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) + bams = snakemake.input.bam if isinstance(bams, str): bams = [bams] diff --git a/bio/gatk/mutect/wrapper.py b/bio/gatk/mutect/wrapper.py index d7e16452f3..c457ca131b 100644 --- a/bio/gatk/mutect/wrapper.py +++ b/bio/gatk/mutect/wrapper.py @@ -12,9 +12,22 @@ extra = snakemake.params.get("extra", "") java_opts = snakemake.params.get("java_opts", "") -if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in java_opts: +# Getting memory in megabytes, if java opts is not filled with -Xmx parameter +# By doing so, backward compatibility is preserved +if "mem_mb" in snakemake.resources.keys() and not "-Xmx" in java_opts: java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) +# Getting memory in gigabytes, for user convenience. Please prefer the use +# of mem_mb over mem_gb as advised in documentation. +elif "mem_gb" in snakemake.resources.keys() and not "-Xmx" in java_opts: + java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) + +# Getting java temp directory from output files list, if -Djava.io.tmpdir +# is not provided in java parameters. By doing so, backward compatibility is +# not broken. +if "java_temp" in snakemake.output.keys() and not "-Djava.io.tmpdir" in java_opts: + java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) + shell( "gatk --java-options '{java_opts}' Mutect2 " # Tool and its subprocess "--input {snakemake.input.map} " # Path to input mapping file diff --git a/bio/gatk/selectvariants/wrapper.py b/bio/gatk/selectvariants/wrapper.py index eba819006c..f8e4781268 100644 --- a/bio/gatk/selectvariants/wrapper.py +++ b/bio/gatk/selectvariants/wrapper.py @@ -8,9 +8,22 @@ extra = snakemake.params.get("extra", "") java_opts = snakemake.params.get("java_opts", "") +# Getting memory in megabytes, if java opts is not filled with -Xmx parameter +# By doing so, backward compatibility is preserved if "mem_mb" in snakemake.resources.keys() and not "-Xmx" in java_opts: java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) +# Getting memory in gigabytes, for user convenience. Please prefer the use +# of mem_mb over mem_gb as advised in documentation. +elif "mem_gb" in snakemake.resources.keys() and not "-Xmx" in java_opts: + java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) + +# Getting java temp directory from output files list, if -Djava.io.tmpdir +# is not provided in java parameters. By doing so, backward compatibility is +# not broken. +if "java_temp" in snakemake.output.keys() and not "-Djava.io.tmpdir" in java_opts: + java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) + log = snakemake.log_fmt_shell(stdout=True, stderr=True) shell( "gatk --java-options '{java_opts}' SelectVariants -R {snakemake.input.ref} -V {snakemake.input.vcf} " diff --git a/bio/gatk/splitncigarreads/wrapper.py b/bio/gatk/splitncigarreads/wrapper.py index 025aa9535a..fe9eafbb6e 100644 --- a/bio/gatk/splitncigarreads/wrapper.py +++ b/bio/gatk/splitncigarreads/wrapper.py @@ -9,9 +9,22 @@ extra = snakemake.params.get("extra", "") java_opts = snakemake.params.get("java_opts", "") +# Getting memory in megabytes, if java opts is not filled with -Xmx parameter +# By doing so, backward compatibility is preserved if "mem_mb" in snakemake.resources.keys() and not "-Xmx" in java_opts: java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) +# Getting memory in gigabytes, for user convenience. Please prefer the use +# of mem_mb over mem_gb as advised in documentation. +elif "mem_gb" in snakemake.resources.keys() and not "-Xmx" in java_opts: + java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) + +# Getting java temp directory from output files list, if -Djava.io.tmpdir +# is not provided in java parameters. By doing so, backward compatibility is +# not broken. +if "java_temp" in snakemake.output.keys() and not "-Djava.io.tmpdir" in java_opts: + java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) + log = snakemake.log_fmt_shell(stdout=True, stderr=True) shell( "gatk --java-options '{java_opts}' SplitNCigarReads {extra} " diff --git a/bio/gatk/variantfiltration/wrapper.py b/bio/gatk/variantfiltration/wrapper.py index ded75a1c4e..17095907d9 100644 --- a/bio/gatk/variantfiltration/wrapper.py +++ b/bio/gatk/variantfiltration/wrapper.py @@ -8,9 +8,22 @@ extra = snakemake.params.get("extra", "") java_opts = snakemake.params.get("java_opts", "") +# Getting memory in megabytes, if java opts is not filled with -Xmx parameter +# By doing so, backward compatibility is preserved if "mem_mb" in snakemake.resources.keys() and not "-Xmx" in java_opts: java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) +# Getting memory in gigabytes, for user convenience. Please prefer the use +# of mem_mb over mem_gb as advised in documentation. +elif "mem_gb" in snakemake.resources.keys() and not "-Xmx" in java_opts: + java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) + +# Getting java temp directory from output files list, if -Djava.io.tmpdir +# is not provided in java parameters. By doing so, backward compatibility is +# not broken. +if "java_temp" in snakemake.output.keys() and not "-Djava.io.tmpdir" in java_opts: + java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) + filters = [ "--filter-name {} --filter-expression '{}'".format(name, expr.replace("'", "\\'")) for name, expr in snakemake.params.filters.items() diff --git a/bio/gatk/variantrecalibrator/wrapper.py b/bio/gatk/variantrecalibrator/wrapper.py index 208a5ae9b3..3e1c8dec1f 100644 --- a/bio/gatk/variantrecalibrator/wrapper.py +++ b/bio/gatk/variantrecalibrator/wrapper.py @@ -11,8 +11,21 @@ extra = snakemake.params.get("extra", "") java_opts = snakemake.params.get("java_opts", "") -if "mem_mb" in snakemake.resource.keys() and not "-Xmx" in java_opts: - java_opts += " -Xmx{}M".format(snakemake.resource["mem_mb"]) +# Getting memory in megabytes, if java opts is not filled with -Xmx parameter +# By doing so, backward compatibility is preserved +if "mem_mb" in snakemake.resources.keys() and not "-Xmx" in java_opts: + java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + +# Getting memory in gigabytes, for user convenience. Please prefer the use +# of mem_mb over mem_gb as advised in documentation. +elif "mem_gb" in snakemake.resources.keys() and not "-Xmx" in java_opts: + java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) + +# Getting java temp directory from output files list, if -Djava.io.tmpdir +# is not provided in java parameters. By doing so, backward compatibility is +# not broken. +if "java_temp" in snakemake.output.keys() and not "-Djava.io.tmpdir" in java_opts: + java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) def fmt_res(resname, resparams): From 56d581b3c1f3ece5ceabc2316feb11d88c510a10 Mon Sep 17 00:00:00 2001 From: tdayris Date: Tue, 20 Oct 2020 10:03:20 +0200 Subject: [PATCH 25/61] [dev] (jannovar): Update with comments from @dlaehnemann and java temp --- bio/jannovar/wrapper.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/bio/jannovar/wrapper.py b/bio/jannovar/wrapper.py index 8d6a8abbbc..6d957d306d 100644 --- a/bio/jannovar/wrapper.py +++ b/bio/jannovar/wrapper.py @@ -11,9 +11,22 @@ log = snakemake.log_fmt_shell(stdout=False, stderr=True) extra = snakemake.params.get("extra", "") +# Getting memory in megabytes, if java opts is not filled with -Xmx parameter +# By doing so, backward compatibility is preserved if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: extra += " -Xmx{}M".format(snakemake.resources["mem_mb"]) +# Getting memory in gigabytes, for user convenience. Please prefer the use +# of mem_mb over mem_gb as advised in documentation. +elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: + extra += " -Xmx{}G".format(snakemake.resources["mem_gb"]) + +# Getting java temp directory from output files list, if -Djava.io.tmpdir +# is not provided in java parameters. By doing so, backward compatibility is +# not broken. +if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: + extra += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) + pedigree = snakemake.input.get("pedigree", "") if pedigree: pedigree = '--pedigree-file "%s"' % pedigree From a3cb408cc6c82f0485680619ac3fde0ca2be7875 Mon Sep 17 00:00:00 2001 From: tdayris Date: Tue, 20 Oct 2020 10:04:02 +0200 Subject: [PATCH 26/61] [dev] (picard): AddOrReplaceGroups updated, missing test added. --- .../addorreplacereadgroups/test/mapped/a.bam | Bin 0 -> 184 bytes bio/picard/addorreplacereadgroups/wrapper.py | 22 ++++++++++++++++-- test.py | 6 +++++ 3 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 bio/picard/addorreplacereadgroups/test/mapped/a.bam diff --git a/bio/picard/addorreplacereadgroups/test/mapped/a.bam b/bio/picard/addorreplacereadgroups/test/mapped/a.bam new file mode 100644 index 0000000000000000000000000000000000000000..dba1268acbd8446e4fde54d7da33434597fbe635 GIT binary patch literal 184 zcmb2|=3rp}f&Xj_PR>jWb_~TuUs6R95)ukH_@3~5+q`PUgD)R98yP)FV(BtuE_7vW z=9s|5aI{h|P#vgC9!+};gK@G0Lz-KrbIh-tVq12fpEAOZjf CvNY8I literal 0 HcmV?d00001 diff --git a/bio/picard/addorreplacereadgroups/wrapper.py b/bio/picard/addorreplacereadgroups/wrapper.py index a71925cd68..f277bb5a7b 100644 --- a/bio/picard/addorreplacereadgroups/wrapper.py +++ b/bio/picard/addorreplacereadgroups/wrapper.py @@ -6,8 +6,26 @@ from snakemake.shell import shell +extra = snakemake.params +java_opts = "" +# Getting memory in megabytes, if java opts is not filled with -Xmx parameter +# By doing so, backward compatibility is preserved +if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: + java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + +# Getting memory in gigabytes, for user convenience. Please prefer the use +# of mem_mb over mem_gb as advised in documentation. +elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: + java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) + +# Getting java temp directory from output files list, if -Djava.io.tmpdir +# is not provided in java parameters. By doing so, backward compatibility is +# not broken. +if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: + java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) + shell( - "picard AddOrReplaceReadGroups {snakemake.params} I={snakemake.input} " - "O={snakemake.output} &> {snakemake.log}" + "picard AddOrReplaceReadGroups {java_opts} {snakemake.params} " + "I={snakemake.input} O={snakemake.output} &> {snakemake.log}" ) diff --git a/test.py b/test.py index d8c18dbcf2..1069106971 100644 --- a/test.py +++ b/test.py @@ -1084,6 +1084,12 @@ def test_optitype(): ["snakemake", "--cores", "1", "--use-conda", "-F", "optitype/a_result.tsv"], ) +def test_picard_addorreplacegroups(): + run( + "bio/picard/addorreplacereadgroups", + ["snakemake", "--cores", "1", "fixed-rg/a.bam", "--use-conda", "-F"], + ) + def test_picard_collectalignmentsummarymetrics(): run( From 438b97e59b30940684b1ecae38aff35b774ba77d Mon Sep 17 00:00:00 2001 From: tdayris Date: Tue, 20 Oct 2020 10:20:17 +0200 Subject: [PATCH 27/61] [dev] (picard): Update with comments from @dlaehnemann and java temp --- bio/picard/addorreplacereadgroups/wrapper.py | 2 +- bio/picard/bedtointervallist/wrapper.py | 21 ++++++++++++-- .../collectalignmentsummarymetrics/wrapper.py | 21 ++++++++++++-- bio/picard/collecthsmetrics/wrapper.py | 20 ++++++++++++- .../collectinsertsizemetrics/wrapper.py | 21 ++++++++++++-- bio/picard/collectmultiplemetrics/wrapper.py | 29 +++++++++++++++---- .../createsequencedictionary/wrapper.py | 19 +++++++++++- bio/picard/markduplicates/wrapper.py | 25 ++++++++++++---- bio/picard/mergesamfiles/wrapper.py | 20 ++++++++++++- bio/picard/mergevcfs/wrapper.py | 18 ++++++++++++ bio/picard/revertsam/wrapper.py | 18 ++++++++++++ bio/picard/samtofastq/wrapper.py | 29 +++++++++++++++++-- bio/picard/sortsam/wrapper.py | 18 ++++++++++++ 13 files changed, 238 insertions(+), 23 deletions(-) diff --git a/bio/picard/addorreplacereadgroups/wrapper.py b/bio/picard/addorreplacereadgroups/wrapper.py index f277bb5a7b..5f58b44ad8 100644 --- a/bio/picard/addorreplacereadgroups/wrapper.py +++ b/bio/picard/addorreplacereadgroups/wrapper.py @@ -26,6 +26,6 @@ shell( - "picard AddOrReplaceReadGroups {java_opts} {snakemake.params} " + "picard AddOrReplaceReadGroups {java_opts} {extra} " "I={snakemake.input} O={snakemake.output} &> {snakemake.log}" ) diff --git a/bio/picard/bedtointervallist/wrapper.py b/bio/picard/bedtointervallist/wrapper.py index d7447d16b4..7584ee9ee1 100644 --- a/bio/picard/bedtointervallist/wrapper.py +++ b/bio/picard/bedtointervallist/wrapper.py @@ -5,14 +5,31 @@ from snakemake.shell import shell +log = snakemake.log_fmt_shell() +extra = snakemake.params +java_opts = "" +# Getting memory in megabytes, if java opts is not filled with -Xmx parameter +# By doing so, backward compatibility is preserved +if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: + java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + +# Getting memory in gigabytes, for user convenience. Please prefer the use +# of mem_mb over mem_gb as advised in documentation. +elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: + java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) + +# Getting java temp directory from output files list, if -Djava.io.tmpdir +# is not provided in java parameters. By doing so, backward compatibility is +# not broken. +if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: + java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) -log = snakemake.log_fmt_shell() shell( "picard BedToIntervalList " - "{snakemake.params} " + "{java_opts} {extra} " "INPUT={snakemake.input.bed} " "SEQUENCE_DICTIONARY={snakemake.input.dict} " "OUTPUT={snakemake.output} " diff --git a/bio/picard/collectalignmentsummarymetrics/wrapper.py b/bio/picard/collectalignmentsummarymetrics/wrapper.py index 03e1c76d1f..609f39f879 100644 --- a/bio/picard/collectalignmentsummarymetrics/wrapper.py +++ b/bio/picard/collectalignmentsummarymetrics/wrapper.py @@ -5,13 +5,30 @@ from snakemake.shell import shell +log = snakemake.log_fmt_shell() +extra = snakemake.params +java_opts = "" +# Getting memory in megabytes, if java opts is not filled with -Xmx parameter +# By doing so, backward compatibility is preserved +if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: + java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + +# Getting memory in gigabytes, for user convenience. Please prefer the use +# of mem_mb over mem_gb as advised in documentation. +elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: + java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) + +# Getting java temp directory from output files list, if -Djava.io.tmpdir +# is not provided in java parameters. By doing so, backward compatibility is +# not broken. +if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: + java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) -log = snakemake.log_fmt_shell() shell( - "picard CollectAlignmentSummaryMetrics {snakemake.params} " + "picard CollectAlignmentSummaryMetrics {java_opts} {extra} " "INPUT={snakemake.input.bam} OUTPUT={snakemake.output[0]} " "REFERENCE_SEQUENCE={snakemake.input.ref} {log}" ) diff --git a/bio/picard/collecthsmetrics/wrapper.py b/bio/picard/collecthsmetrics/wrapper.py index f5adc9be49..0e94763ae4 100644 --- a/bio/picard/collecthsmetrics/wrapper.py +++ b/bio/picard/collecthsmetrics/wrapper.py @@ -13,9 +13,27 @@ extra = snakemake.params.get("extra", "") log = snakemake.log_fmt_shell(stdout=False, stderr=True) +java_opts = "" +# Getting memory in megabytes, if java opts is not filled with -Xmx parameter +# By doing so, backward compatibility is preserved +if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: + java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + +# Getting memory in gigabytes, for user convenience. Please prefer the use +# of mem_mb over mem_gb as advised in documentation. +elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: + java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) + +# Getting java temp directory from output files list, if -Djava.io.tmpdir +# is not provided in java parameters. By doing so, backward compatibility is +# not broken. +if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: + java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) + + shell( "picard CollectHsMetrics" - " {extra}" + " {java_opts} {extra}" " INPUT={snakemake.input.bam}" " OUTPUT={snakemake.output[0]}" " REFERENCE_SEQUENCE={snakemake.input.reference}" diff --git a/bio/picard/collectinsertsizemetrics/wrapper.py b/bio/picard/collectinsertsizemetrics/wrapper.py index 9bb9a107db..4783592bba 100644 --- a/bio/picard/collectinsertsizemetrics/wrapper.py +++ b/bio/picard/collectinsertsizemetrics/wrapper.py @@ -5,13 +5,30 @@ from snakemake.shell import shell +log = snakemake.log_fmt_shell() +extra = snakemake.params +java_opts = "" +# Getting memory in megabytes, if java opts is not filled with -Xmx parameter +# By doing so, backward compatibility is preserved +if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: + java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + +# Getting memory in gigabytes, for user convenience. Please prefer the use +# of mem_mb over mem_gb as advised in documentation. +elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: + java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) + +# Getting java temp directory from output files list, if -Djava.io.tmpdir +# is not provided in java parameters. By doing so, backward compatibility is +# not broken. +if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: + java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) -log = snakemake.log_fmt_shell() shell( - "picard CollectInsertSizeMetrics {snakemake.params} " + "picard CollectInsertSizeMetrics {java_opts} {extra} " "INPUT={snakemake.input} OUTPUT={snakemake.output.txt} " "HISTOGRAM_FILE={snakemake.output.pdf} {log}" ) diff --git a/bio/picard/collectmultiplemetrics/wrapper.py b/bio/picard/collectmultiplemetrics/wrapper.py index c1076c225e..7263b1c458 100644 --- a/bio/picard/collectmultiplemetrics/wrapper.py +++ b/bio/picard/collectmultiplemetrics/wrapper.py @@ -8,9 +8,28 @@ log = snakemake.log_fmt_shell(stdout=False, stderr=True) -res = snakemake.resources.get("mem_gb", "3") -if not res or res is None: - res = 3 +extra = snakemake.params +java_opts = "" +# Getting memory in megabytes, if java opts is not filled with -Xmx parameter +# By doing so, backward compatibility is preserved +if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: + java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + +# Getting memory in gigabytes, for user convenience. Please prefer the use +# of mem_mb over mem_gb as advised in documentation. +elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: + java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) + +# Previous wrapper set to 3 Gigabytes the default reserved memory. +# This behaviour is preseved below: +elif "-Xmx" not in extra: + java_opts += " -Xmx3G" + +# Getting java temp directory from output files list, if -Djava.io.tmpdir +# is not provided in java parameters. By doing so, backward compatibility is +# not broken. +if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: + java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) exts_to_prog = { ".alignment_summary_metrics": "CollectAlignmentSummaryMetrics", @@ -56,9 +75,9 @@ break shell( - "(picard -Xmx{res}g CollectMultipleMetrics " + "(picard CollectMultipleMetrics " "I={snakemake.input.bam} " "O={out} " "R={snakemake.input.ref} " - "{snakemake.params}{programs}) {log}" + "{extra} {programs} {java_opts}) {log}" ) diff --git a/bio/picard/createsequencedictionary/wrapper.py b/bio/picard/createsequencedictionary/wrapper.py index b8cb37eedc..0facbc9a0b 100644 --- a/bio/picard/createsequencedictionary/wrapper.py +++ b/bio/picard/createsequencedictionary/wrapper.py @@ -8,12 +8,29 @@ extra = snakemake.params.get("extra", "") +java_opts = "" +# Getting memory in megabytes, if java opts is not filled with -Xmx parameter +# By doing so, backward compatibility is preserved +if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: + java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + +# Getting memory in gigabytes, for user convenience. Please prefer the use +# of mem_mb over mem_gb as advised in documentation. +elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: + java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) + +# Getting java temp directory from output files list, if -Djava.io.tmpdir +# is not provided in java parameters. By doing so, backward compatibility is +# not broken. +if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: + java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) + log = snakemake.log_fmt_shell(stdout=False, stderr=True) shell( "picard " "CreateSequenceDictionary " - "{extra} " + "{java_opts} {extra} " "R={snakemake.input[0]} " "O={snakemake.output[0]} " "{log}" diff --git a/bio/picard/markduplicates/wrapper.py b/bio/picard/markduplicates/wrapper.py index c67fff39c0..78e2243573 100644 --- a/bio/picard/markduplicates/wrapper.py +++ b/bio/picard/markduplicates/wrapper.py @@ -8,14 +8,29 @@ log = snakemake.log_fmt_shell(stdout=True, stderr=True) -memory = "" -if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in snakemake.params: - memory = "-Xmx{}M".format(str(snakemake.resources["mem_mb"])) +extra = snakemake.params +java_opts = "" +# Getting memory in megabytes, if java opts is not filled with -Xmx parameter +# By doing so, backward compatibility is preserved +if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: + java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + +# Getting memory in gigabytes, for user convenience. Please prefer the use +# of mem_mb over mem_gb as advised in documentation. +elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: + java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) + +# Getting java temp directory from output files list, if -Djava.io.tmpdir +# is not provided in java parameters. By doing so, backward compatibility is +# not broken. +if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: + java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) + shell( "picard MarkDuplicates " # Tool and its subcommand - "{memory} " # Automatic Xmx java option - "{snakemake.params} " # User defined parmeters + "{java_opts} " # Automatic java option + "{extra} " # User defined parmeters "INPUT={snakemake.input} " # Input file "OUTPUT={snakemake.output.bam} " # Output bam "METRICS_FILE={snakemake.output.metrics} " # Output metrics diff --git a/bio/picard/mergesamfiles/wrapper.py b/bio/picard/mergesamfiles/wrapper.py index 7c9c7eafa4..dc09d6fe43 100644 --- a/bio/picard/mergesamfiles/wrapper.py +++ b/bio/picard/mergesamfiles/wrapper.py @@ -8,6 +8,24 @@ from snakemake.shell import shell +extra = snakemake.params +java_opts = "" +# Getting memory in megabytes, if java opts is not filled with -Xmx parameter +# By doing so, backward compatibility is preserved +if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: + java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + +# Getting memory in gigabytes, for user convenience. Please prefer the use +# of mem_mb over mem_gb as advised in documentation. +elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: + java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) + +# Getting java temp directory from output files list, if -Djava.io.tmpdir +# is not provided in java parameters. By doing so, backward compatibility is +# not broken. +if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: + java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) + inputs = " ".join("INPUT={}".format(in_) for in_ in snakemake.input) log = snakemake.log_fmt_shell(stdout=False, stderr=True) @@ -15,7 +33,7 @@ shell( "picard" " MergeSamFiles" - " {snakemake.params}" + " {java_opts} {extra}" " {inputs}" " OUTPUT={snakemake.output[0]}" " {log}" diff --git a/bio/picard/mergevcfs/wrapper.py b/bio/picard/mergevcfs/wrapper.py index 4443dd53f5..7b0eb47c96 100644 --- a/bio/picard/mergevcfs/wrapper.py +++ b/bio/picard/mergevcfs/wrapper.py @@ -12,10 +12,28 @@ inputs = " ".join("INPUT={}".format(f) for f in snakemake.input.vcfs) log = snakemake.log_fmt_shell(stdout=False, stderr=True) extra = snakemake.params.get("extra", "") +java_opts = "" +# Getting memory in megabytes, if java opts is not filled with -Xmx parameter +# By doing so, backward compatibility is preserved +if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: + java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + +# Getting memory in gigabytes, for user convenience. Please prefer the use +# of mem_mb over mem_gb as advised in documentation. +elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: + java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) + +# Getting java temp directory from output files list, if -Djava.io.tmpdir +# is not provided in java parameters. By doing so, backward compatibility is +# not broken. +if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: + java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) + shell( "picard" " MergeVcfs" + " {java_opts}" " {extra}" " {inputs}" " OUTPUT={snakemake.output[0]}" diff --git a/bio/picard/revertsam/wrapper.py b/bio/picard/revertsam/wrapper.py index ccf54fea60..b9b60c31b4 100644 --- a/bio/picard/revertsam/wrapper.py +++ b/bio/picard/revertsam/wrapper.py @@ -9,11 +9,29 @@ from snakemake.shell import shell extra = snakemake.params.get("extra", "") +java_opts = "" +# Getting memory in megabytes, if java opts is not filled with -Xmx parameter +# By doing so, backward compatibility is preserved +if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: + java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + +# Getting memory in gigabytes, for user convenience. Please prefer the use +# of mem_mb over mem_gb as advised in documentation. +elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: + java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) + +# Getting java temp directory from output files list, if -Djava.io.tmpdir +# is not provided in java parameters. By doing so, backward compatibility is +# not broken. +if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: + java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) + log = snakemake.log_fmt_shell(stdout=False, stderr=True) shell( "picard" " RevertSam" + " {java_opts}" " {extra}" " INPUT={snakemake.input[0]}" " OUTPUT={snakemake.output[0]}" diff --git a/bio/picard/samtofastq/wrapper.py b/bio/picard/samtofastq/wrapper.py index 9c5748a1e8..259333f234 100644 --- a/bio/picard/samtofastq/wrapper.py +++ b/bio/picard/samtofastq/wrapper.py @@ -10,6 +10,23 @@ extra = snakemake.params.get("extra", "") +java_opts = "" +# Getting memory in megabytes, if java opts is not filled with -Xmx parameter +# By doing so, backward compatibility is preserved +if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: + java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + +# Getting memory in gigabytes, for user convenience. Please prefer the use +# of mem_mb over mem_gb as advised in documentation. +elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: + java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) + +# Getting java temp directory from output files list, if -Djava.io.tmpdir +# is not provided in java parameters. By doing so, backward compatibility is +# not broken. +if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: + java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) + log = snakemake.log_fmt_shell(stdout=False, stderr=True) fastq1 = snakemake.output.fastq1 @@ -27,9 +44,15 @@ if isinstance(fastq_unpaired, str): if not isinstance(fastq2, str): raise ValueError("f2 is required if fastq_unpaired is set") - else: - output += " UNPAIRED_FASTQ=" + fastq_unpaired + + output += " UNPAIRED_FASTQ=" + fastq_unpaired shell( - "picard" " SamToFastq" " {extra}" " INPUT={snakemake.input[0]}" " {output}" " {log}" + "picard" + " SamToFastq" + " {java_opts}" + " {extra}" + " INPUT={snakemake.input[0]}" + " {output}" + " {log}" ) diff --git a/bio/picard/sortsam/wrapper.py b/bio/picard/sortsam/wrapper.py index f2b817da42..96e98c3c92 100644 --- a/bio/picard/sortsam/wrapper.py +++ b/bio/picard/sortsam/wrapper.py @@ -10,11 +10,29 @@ extra = snakemake.params.get("extra", "") +java_opts = "" +# Getting memory in megabytes, if java opts is not filled with -Xmx parameter +# By doing so, backward compatibility is preserved +if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: + java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + +# Getting memory in gigabytes, for user convenience. Please prefer the use +# of mem_mb over mem_gb as advised in documentation. +elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: + java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) + +# Getting java temp directory from output files list, if -Djava.io.tmpdir +# is not provided in java parameters. By doing so, backward compatibility is +# not broken. +if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: + java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) + log = snakemake.log_fmt_shell(stdout=False, stderr=True) shell( "picard" " SortSam" + " {java_opts}" " {extra}" " INPUT={snakemake.input[0]}" " OUTPUT={snakemake.output[0]}" From f17c6f4157f7d408b256a392128698db6f2901bf Mon Sep 17 00:00:00 2001 From: tdayris Date: Tue, 20 Oct 2020 10:20:47 +0200 Subject: [PATCH 28/61] [dev] (collecttargetedpcametrics): Updated and missing test added --- .../collecttargetedpcrmetrics/wrapper.py | 19 ++++++++++++++++++- test.py | 7 +++++++ 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/bio/picard/collecttargetedpcrmetrics/wrapper.py b/bio/picard/collecttargetedpcrmetrics/wrapper.py index d112fbab4d..cb0a1fc0f8 100644 --- a/bio/picard/collecttargetedpcrmetrics/wrapper.py +++ b/bio/picard/collecttargetedpcrmetrics/wrapper.py @@ -10,10 +10,27 @@ log = snakemake.log_fmt_shell() extra = snakemake.params.get("extra", "") +java_opts = "" +# Getting memory in megabytes, if java opts is not filled with -Xmx parameter +# By doing so, backward compatibility is preserved +if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: + java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + +# Getting memory in gigabytes, for user convenience. Please prefer the use +# of mem_mb over mem_gb as advised in documentation. +elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: + java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) + +# Getting java temp directory from output files list, if -Djava.io.tmpdir +# is not provided in java parameters. By doing so, backward compatibility is +# not broken. +if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: + java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) + shell( "picard CollectTargetedPcrMetrics " - "{extra} " + "{java_opts} {extra} " "INPUT={snakemake.input.bam} " "OUTPUT={snakemake.output[0]} " "AMPLICON_INTERVALS={snakemake.input.amplicon_intervals} " diff --git a/test.py b/test.py index 1069106971..1d826435ac 100644 --- a/test.py +++ b/test.py @@ -1158,6 +1158,13 @@ def test_picard_mergesamfiles(): ) +def test_picard_collecttargettedpcemetrics(): + run( + "bio/picard/collecttargetedpcrmetrics/", + ["snakemake", "--cores", "1", "stats/a.pcr.txt", "--use-conda", "-F"], + ) + + def test_picard_bam_to_fastq(): run( "bio/picard/samtofastq", From f176831f7a5dbe0ad45d10b5f03a9217b3f934b5 Mon Sep 17 00:00:00 2001 From: tdayris Date: Tue, 20 Oct 2020 10:33:17 +0200 Subject: [PATCH 29/61] [dev] (snpeff): Update with comments from @dlaehnemann and java temp --- bio/snpeff/annotate/test/Snakefile | 4 ++-- bio/snpeff/annotate/wrapper.py | 20 ++++++++++++++++++-- bio/snpeff/download/wrapper.py | 22 +++++++++++++++++++--- 3 files changed, 39 insertions(+), 7 deletions(-) diff --git a/bio/snpeff/annotate/test/Snakefile b/bio/snpeff/annotate/test/Snakefile index 9d2bb90e10..6c19eef040 100644 --- a/bio/snpeff/annotate/test/Snakefile +++ b/bio/snpeff/annotate/test/Snakefile @@ -8,7 +8,7 @@ rule snpeff: csvstats="snpeff/{sample}.csv" # summary statistics in CSV, optional log: "logs/snpeff/{sample}.log" - params: - extra="-Xmx4g" # optional parameters (e.g., max memory 4g) + resources: + mem_gb = 4 # Original tests was run with -Xmx4G wrapper: "master/bio/snpeff/annotate" diff --git a/bio/snpeff/annotate/wrapper.py b/bio/snpeff/annotate/wrapper.py index c522a5a715..00daf8afab 100644 --- a/bio/snpeff/annotate/wrapper.py +++ b/bio/snpeff/annotate/wrapper.py @@ -25,8 +25,23 @@ log = snakemake.log_fmt_shell(stdout=False, stderr=True) extra = snakemake.params.get("extra", "") +java_opts = "" +# Getting memory in megabytes, if java opts is not filled with -Xmx parameter +# By doing so, backward compatibility is preserved if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: - extra += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + +# Getting memory in gigabytes, for user convenience. Please prefer the use +# of mem_mb over mem_gb as advised in documentation. +elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: + java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) + +# Getting java temp directory from output files list, if -Djava.io.tmpdir +# is not provided in java parameters. By doing so, backward compatibility is +# not broken. +if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: + java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) + data_dir = Path(snakemake.input.db).parent.resolve() @@ -38,7 +53,8 @@ reference = path.basename(snakemake.input.db) shell( - "snpEff -dataDir {data_dir} {stats_opt} {csvstats_opt} {extra} " + "snpEff {java_opts} -dataDir {data_dir} " + "{stats_opt} {csvstats_opt} {extra} " "{reference} {incalls} " "{outprefix} > {outcalls} {log}" ) diff --git a/bio/snpeff/download/wrapper.py b/bio/snpeff/download/wrapper.py index 2ac1b2bd84..84534dcb57 100644 --- a/bio/snpeff/download/wrapper.py +++ b/bio/snpeff/download/wrapper.py @@ -10,7 +10,23 @@ outdir = Path(snakemake.output[0]).parent.resolve() log = snakemake.log_fmt_shell(stdout=False, stderr=True) -if "mem_mb" in snakemake.resources.keys(): - extra += " -Xmx{}M".format(snakemake.resources["mem_mb"]) +extra = snakemake.params +java_opts = "" +# Getting memory in megabytes, if java opts is not filled with -Xmx parameter +# By doing so, backward compatibility is preserved +if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: + java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) -shell("snpEff download -dataDir {outdir} {reference} {log}") +# Getting memory in gigabytes, for user convenience. Please prefer the use +# of mem_mb over mem_gb as advised in documentation. +elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: + java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) + +# Getting java temp directory from output files list, if -Djava.io.tmpdir +# is not provided in java parameters. By doing so, backward compatibility is +# not broken. +if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: + java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) + + +shell("snpEff download {java_opts} {extra} -dataDir {outdir} {reference} {log}") From b23777ca688b55183154bfc16b736fefe1047ab4 Mon Sep 17 00:00:00 2001 From: tdayris Date: Tue, 20 Oct 2020 10:35:52 +0200 Subject: [PATCH 30/61] [dev] (snpsift): Update with comments from @dlaehnemann and java temp --- bio/snpsift/annotate/wrapper.py | 19 +++++++++++++++++-- bio/snpsift/dbnsfp/wrapper.py | 19 +++++++++++++++++-- bio/snpsift/genesets/wrapper.py | 19 +++++++++++++++++-- bio/snpsift/gwascat/wrapper.py | 19 +++++++++++++++++-- bio/snpsift/varType/wrapper.py | 19 +++++++++++++++++-- 5 files changed, 85 insertions(+), 10 deletions(-) diff --git a/bio/snpsift/annotate/wrapper.py b/bio/snpsift/annotate/wrapper.py index 0de6e98586..a8fe4d1c39 100644 --- a/bio/snpsift/annotate/wrapper.py +++ b/bio/snpsift/annotate/wrapper.py @@ -9,8 +9,23 @@ log = snakemake.log_fmt_shell(stdout=False, stderr=True) extra = snakemake.params.get("extra", "") +java_opts = "" +# Getting memory in megabytes, if java opts is not filled with -Xmx parameter +# By doing so, backward compatibility is preserved if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: - extra += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + +# Getting memory in gigabytes, for user convenience. Please prefer the use +# of mem_mb over mem_gb as advised in documentation. +elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: + java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) + +# Getting java temp directory from output files list, if -Djava.io.tmpdir +# is not provided in java parameters. By doing so, backward compatibility is +# not broken. +if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: + java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) + min_threads = 1 @@ -41,7 +56,7 @@ shell( "SnpSift annotate" # Tool and its subcommand - " {extra}" # Extra parameters + " {java_opts} {extra}" # Extra parameters " {snakemake.input.database}" # Path to annotation vcf file " {incall} " # Path to input vcf file " {outcall} " # Path to output vcf file diff --git a/bio/snpsift/dbnsfp/wrapper.py b/bio/snpsift/dbnsfp/wrapper.py index ce82366941..db36e52103 100644 --- a/bio/snpsift/dbnsfp/wrapper.py +++ b/bio/snpsift/dbnsfp/wrapper.py @@ -9,8 +9,23 @@ log = snakemake.log_fmt_shell(stdout=False, stderr=True) extra = snakemake.params.get("extra", "") +java_opts = "" +# Getting memory in megabytes, if java opts is not filled with -Xmx parameter +# By doing so, backward compatibility is preserved if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: - extra += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + +# Getting memory in gigabytes, for user convenience. Please prefer the use +# of mem_mb over mem_gb as advised in documentation. +elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: + java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) + +# Getting java temp directory from output files list, if -Djava.io.tmpdir +# is not provided in java parameters. By doing so, backward compatibility is +# not broken. +if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: + java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) + # Using user-defined file if requested db = snakemake.input.get("dbNSFP", "") @@ -50,7 +65,7 @@ shell( "SnpSift dbnsfp" # Tool and its subcommand - " {extra}" # Extra parameters + " {java_opts} {extra}" # Extra parameters " {db}" # Path to annotation vcf file " {incall}" # Path to input vcf file " {outcall}" # Path to output vcf file diff --git a/bio/snpsift/genesets/wrapper.py b/bio/snpsift/genesets/wrapper.py index 3185fe0324..7cef8a7a0d 100644 --- a/bio/snpsift/genesets/wrapper.py +++ b/bio/snpsift/genesets/wrapper.py @@ -9,8 +9,23 @@ log = snakemake.log_fmt_shell(stdout=False, stderr=True) extra = snakemake.params.get("extra", "") +java_opts = "" +# Getting memory in megabytes, if java opts is not filled with -Xmx parameter +# By doing so, backward compatibility is preserved if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: - extra += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + +# Getting memory in gigabytes, for user convenience. Please prefer the use +# of mem_mb over mem_gb as advised in documentation. +elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: + java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) + +# Getting java temp directory from output files list, if -Djava.io.tmpdir +# is not provided in java parameters. By doing so, backward compatibility is +# not broken. +if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: + java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) + min_threads = 1 # Uncompression shall be done according to user-defined input @@ -44,7 +59,7 @@ shell( "SnpSift geneSets" # Tool and its subcommand - " {extra}" # Extra parameters + " {java_opts} {extra}" # Extra parameters " {snakemake.input.gmt}" # Path to annotation vcf file " {incall}" # Path to input vcf file " {outcall}" # Path to output vcf file diff --git a/bio/snpsift/gwascat/wrapper.py b/bio/snpsift/gwascat/wrapper.py index e7748cb159..5871419c4c 100644 --- a/bio/snpsift/gwascat/wrapper.py +++ b/bio/snpsift/gwascat/wrapper.py @@ -9,8 +9,23 @@ log = snakemake.log_fmt_shell(stdout=False, stderr=True) extra = snakemake.params.get("extra", "") +java_opts = "" +# Getting memory in megabytes, if java opts is not filled with -Xmx parameter +# By doing so, backward compatibility is preserved if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: - extra += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + +# Getting memory in gigabytes, for user convenience. Please prefer the use +# of mem_mb over mem_gb as advised in documentation. +elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: + java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) + +# Getting java temp directory from output files list, if -Djava.io.tmpdir +# is not provided in java parameters. By doing so, backward compatibility is +# not broken. +if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: + java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) + min_threads = 1 @@ -46,7 +61,7 @@ shell( "SnpSift gwasCat " # Tool and its subcommand - " {extra} " # Extra parameters + " {java_opts} {extra} " # Extra parameters " -db {snakemake.input.gwascat} " # Path to gwasCat file " {incall} " # Path to input vcf file " {outcall} " # Path to output vcf file diff --git a/bio/snpsift/varType/wrapper.py b/bio/snpsift/varType/wrapper.py index 994301a109..fff970a98d 100644 --- a/bio/snpsift/varType/wrapper.py +++ b/bio/snpsift/varType/wrapper.py @@ -10,12 +10,27 @@ log = snakemake.log_fmt_shell(stdout=False, stderr=True) extra = snakemake.params.get("extra", "") +java_opts = "" +# Getting memory in megabytes, if java opts is not filled with -Xmx parameter +# By doing so, backward compatibility is preserved if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: - extra += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + +# Getting memory in gigabytes, for user convenience. Please prefer the use +# of mem_mb over mem_gb as advised in documentation. +elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: + java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) + +# Getting java temp directory from output files list, if -Djava.io.tmpdir +# is not provided in java parameters. By doing so, backward compatibility is +# not broken. +if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: + java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) + shell( "SnpSift varType" # Tool and its subcommand - " {extra}" # Extra parameters + " {java_opts} {extra}" # Extra parameters " {snakemake.input.vcf}" # Path to input vcf file " > {snakemake.output.vcf}" # Path to output vcf file " {log}" # Logging behaviour From 6e6a69c6f2ef6d022048a65de5a6d2d4fec74857 Mon Sep 17 00:00:00 2001 From: tdayris Date: Tue, 20 Oct 2020 10:40:23 +0200 Subject: [PATCH 31/61] e[dev] (trimmomatic): Updatr with comments from @dlaehnemann, javatemp --- bio/trimmomatic/pe/wrapper.py | 19 +++++++++++++++++-- bio/trimmomatic/se/wrapper.py | 20 ++++++++++++++++++-- 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/bio/trimmomatic/pe/wrapper.py b/bio/trimmomatic/pe/wrapper.py index c9faaa6857..eca21c25dd 100644 --- a/bio/trimmomatic/pe/wrapper.py +++ b/bio/trimmomatic/pe/wrapper.py @@ -66,8 +66,23 @@ def compose_output_gz(filename, threads, compression_level): extra = snakemake.params.get("extra", "") +java_opts = "" +# Getting memory in megabytes, if java opts is not filled with -Xmx parameter +# By doing so, backward compatibility is preserved if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: - extra += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + +# Getting memory in gigabytes, for user convenience. Please prefer the use +# of mem_mb over mem_gb as advised in documentation. +elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: + java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) + +# Getting java temp directory from output files list, if -Djava.io.tmpdir +# is not provided in java parameters. By doing so, backward compatibility is +# not broken. +if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: + java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) + log = snakemake.log_fmt_shell(stdout=True, stderr=True) compression_level = snakemake.params.get("compression_level", "-5") trimmer = " ".join(snakemake.params.trimmer) @@ -95,7 +110,7 @@ def compose_output_gz(filename, threads, compression_level): ] shell( - "trimmomatic PE -threads {trimmomatic_threads} {extra} " + "trimmomatic PE -threads {trimmomatic_threads} {java_opts} {extra} " "{input_r1} {input_r2} " "{output_r1} {output_r1_unp} " "{output_r2} {output_r2_unp} " diff --git a/bio/trimmomatic/se/wrapper.py b/bio/trimmomatic/se/wrapper.py index 76e2ab367d..1823aa585e 100644 --- a/bio/trimmomatic/se/wrapper.py +++ b/bio/trimmomatic/se/wrapper.py @@ -66,8 +66,23 @@ def compose_output_gz(filename, threads, compression_level): extra = snakemake.params.get("extra", "") +java_opts = "" +# Getting memory in megabytes, if java opts is not filled with -Xmx parameter +# By doing so, backward compatibility is preserved if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: - extra += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + +# Getting memory in gigabytes, for user convenience. Please prefer the use +# of mem_mb over mem_gb as advised in documentation. +elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: + java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) + +# Getting java temp directory from output files list, if -Djava.io.tmpdir +# is not provided in java parameters. By doing so, backward compatibility is +# not broken. +if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: + java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) + log = snakemake.log_fmt_shell(stdout=True, stderr=True) compression_level = snakemake.params.get("compression_level", "-5") trimmer = " ".join(snakemake.params.trimmer) @@ -82,5 +97,6 @@ def compose_output_gz(filename, threads, compression_level): output = compose_output_gz(snakemake.output[0], output_threads, compression_level) shell( - "trimmomatic SE -threads {trimmomatic_threads} {extra} {input} {output} {trimmer} {log}" + "trimmomatic SE -threads {trimmomatic_threads} " + "{java_opts} {extra} {input} {output} {trimmer} {log}" ) From 2e27658f13fc9558c797c90c2c7a7a7f64a8e405 Mon Sep 17 00:00:00 2001 From: tdayris Date: Tue, 20 Oct 2020 10:52:08 +0200 Subject: [PATCH 32/61] [dev] (trinity): Update with comments from @dlaehnemann --- bio/trinity/wrapper.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/bio/trinity/wrapper.py b/bio/trinity/wrapper.py index 8758222f32..25fdbdee64 100644 --- a/bio/trinity/wrapper.py +++ b/bio/trinity/wrapper.py @@ -9,11 +9,22 @@ from snakemake.shell import shell extra = snakemake.params.get("extra", "") -max_memory = snakemake.params.get("max_memory", "10G") +# Previous wrapper reserved 10 Gigabytes by default. This behaviour is +# preserved below: +max_memory = "10G" + +# Getting memory in megabytes, if java opts is not filled with -Xmx parameter +# By doing so, backward compatibility is preserved if "mem_mb" in snakemake.resources.keys(): - if snakemake.resources["mem_mb"] > 10240: - rounded_mb_to_gb = int(snakemake.resources["mem_mb"] / 1024) - max_memory = "{}G".format(rounded_mb_to_gb) + # max_memory from trinity expects a value in gigabytes. + rounded_mb_to_gb = int(snakemake.resources["mem_mb"] / 1024) + max_memory = "{}G".format(rounded_mb_to_gb) + +# Getting memory in gigabytes, for user convenience. Please prefer the use +# of mem_mb over mem_gb as advised in documentation. +elif "mem_gb" in snakemake.resources.keys(): + max_memory = "{}G".format(snakemake.resources["mem_gb"]) + # allow multiple input files for single assembly left = snakemake.input.get("left") @@ -59,7 +70,7 @@ log = snakemake.log_fmt_shell(stdout=False, stderr=True) shell( - "Trinity {input_cmd} --CPU {snakemake.threads} " + "Trinity {input_cmd}--CPU {snakemake.threads} " " --max_memory {max_memory} --seqType {seqtype} " " --output {outdir} {snakemake.params.extra} " " {log}" From 9fe9385cf5556fe77a8b6a2351a2a29ebcc44db4 Mon Sep 17 00:00:00 2001 From: tdayris Date: Tue, 20 Oct 2020 10:55:46 +0200 Subject: [PATCH 33/61] [dev] (varscan): Update with comments from @dlaehnemann and java temp --- bio/varscan/mpileup2indel/wrapper.py | 18 ++++++++++++++++-- bio/varscan/mpileup2snp/wrapper.py | 18 ++++++++++++++++-- bio/varscan/somatic/wrapper.py | 18 ++++++++++++++++-- 3 files changed, 48 insertions(+), 6 deletions(-) diff --git a/bio/varscan/mpileup2indel/wrapper.py b/bio/varscan/mpileup2indel/wrapper.py index 31826418ab..c586ccdb50 100644 --- a/bio/varscan/mpileup2indel/wrapper.py +++ b/bio/varscan/mpileup2indel/wrapper.py @@ -12,8 +12,22 @@ # Gathering extra parameters and logging behaviour log = snakemake.log_fmt_shell(stdout=False, stderr=True) extra = snakemake.params.get("extra", "") +java_opts = "" +# Getting memory in megabytes, if java opts is not filled with -Xmx parameter +# By doing so, backward compatibility is preserved if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: - extra += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + +# Getting memory in gigabytes, for user convenience. Please prefer the use +# of mem_mb over mem_gb as advised in documentation. +elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: + java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) + +# Getting java temp directory from output files list, if -Djava.io.tmpdir +# is not provided in java parameters. By doing so, backward compatibility is +# not broken. +if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: + java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) # In case input files are gzipped mpileup files, # they are being unzipped and piped @@ -31,8 +45,8 @@ shell( "varscan mpileup2indel " # Tool and its subprocess - "{extra} " # Extra parameters "<( {pileup} ) " + "{java_opts} {extra} " # Extra parameters "> {snakemake.output[0]} " # Path to vcf file "{log}" # Logging behaviour ) diff --git a/bio/varscan/mpileup2snp/wrapper.py b/bio/varscan/mpileup2snp/wrapper.py index d8765e45bc..0a3558e85e 100644 --- a/bio/varscan/mpileup2snp/wrapper.py +++ b/bio/varscan/mpileup2snp/wrapper.py @@ -12,8 +12,22 @@ # Gathering extra parameters and logging behaviour log = snakemake.log_fmt_shell(stdout=False, stderr=True) extra = snakemake.params.get("extra", "") +java_opts = "" +# Getting memory in megabytes, if java opts is not filled with -Xmx parameter +# By doing so, backward compatibility is preserved if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: - extra += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + +# Getting memory in gigabytes, for user convenience. Please prefer the use +# of mem_mb over mem_gb as advised in documentation. +elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: + java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) + +# Getting java temp directory from output files list, if -Djava.io.tmpdir +# is not provided in java parameters. By doing so, backward compatibility is +# not broken. +if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: + java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) # In case input files are gzipped mpileup files, # they are being unzipped and piped @@ -31,8 +45,8 @@ shell( "varscan mpileup2snp " # Tool and its subprocess - "{extra} " # Extra parameters "<( {pileup} ) " + "{java_opts} {extra} " # Extra parameters "> {snakemake.output[0]} " # Path to vcf file "{log}" # Logging behaviour ) diff --git a/bio/varscan/somatic/wrapper.py b/bio/varscan/somatic/wrapper.py index 6ecfeb16e1..ae4f2d0090 100644 --- a/bio/varscan/somatic/wrapper.py +++ b/bio/varscan/somatic/wrapper.py @@ -14,8 +14,22 @@ # Defining logging and gathering extra parameters log = snakemake.log_fmt_shell(stdout=True, stderr=True) extra = snakemake.params.get("extra", "") +java_opts = "" +# Getting memory in megabytes, if java opts is not filled with -Xmx parameter +# By doing so, backward compatibility is preserved if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: - extra += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) + +# Getting memory in gigabytes, for user convenience. Please prefer the use +# of mem_mb over mem_gb as advised in documentation. +elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: + java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) + +# Getting java temp directory from output files list, if -Djava.io.tmpdir +# is not provided in java parameters. By doing so, backward compatibility is +# not broken. +if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: + java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) # Building output dirs makedirs(op.dirname(snakemake.output.snp)) @@ -43,7 +57,7 @@ "varscan somatic" # Tool and its subcommand " {in_pileup}" # Path to input file(s) " {prefix}" # Path to output - " {extra}" # Extra parameters + " {java_opts} {extra}" # Extra parameters " {mpileup}" " --output-snp {snakemake.output.snp}" # Path to snp output file " --output-indel {snakemake.output.indel}" # Path to indel output file From 9e84b42db977ea05dca56d403b547c006f4f86b5 Mon Sep 17 00:00:00 2001 From: tdayris Date: Tue, 20 Oct 2020 11:08:25 +0200 Subject: [PATCH 34/61] [fix] (black): Files now blacked --- bio/picard/bedtointervallist/wrapper.py | 2 +- bio/picard/collectalignmentsummarymetrics/wrapper.py | 2 +- bio/picard/collectinsertsizemetrics/wrapper.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bio/picard/bedtointervallist/wrapper.py b/bio/picard/bedtointervallist/wrapper.py index 7584ee9ee1..9e3f9dc9e3 100644 --- a/bio/picard/bedtointervallist/wrapper.py +++ b/bio/picard/bedtointervallist/wrapper.py @@ -5,6 +5,7 @@ from snakemake.shell import shell + log = snakemake.log_fmt_shell() extra = snakemake.params @@ -26,7 +27,6 @@ java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) - shell( "picard BedToIntervalList " "{java_opts} {extra} " diff --git a/bio/picard/collectalignmentsummarymetrics/wrapper.py b/bio/picard/collectalignmentsummarymetrics/wrapper.py index 609f39f879..3b89559e5d 100644 --- a/bio/picard/collectalignmentsummarymetrics/wrapper.py +++ b/bio/picard/collectalignmentsummarymetrics/wrapper.py @@ -5,6 +5,7 @@ from snakemake.shell import shell + log = snakemake.log_fmt_shell() extra = snakemake.params @@ -26,7 +27,6 @@ java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) - shell( "picard CollectAlignmentSummaryMetrics {java_opts} {extra} " "INPUT={snakemake.input.bam} OUTPUT={snakemake.output[0]} " diff --git a/bio/picard/collectinsertsizemetrics/wrapper.py b/bio/picard/collectinsertsizemetrics/wrapper.py index 4783592bba..7e2a293adf 100644 --- a/bio/picard/collectinsertsizemetrics/wrapper.py +++ b/bio/picard/collectinsertsizemetrics/wrapper.py @@ -5,6 +5,7 @@ from snakemake.shell import shell + log = snakemake.log_fmt_shell() extra = snakemake.params @@ -26,7 +27,6 @@ java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) - shell( "picard CollectInsertSizeMetrics {java_opts} {extra} " "INPUT={snakemake.input} OUTPUT={snakemake.output.txt} " From 58d7a2c8db7c17bb73b193ebd1d866f82e388e03 Mon Sep 17 00:00:00 2001 From: tdayris Date: Tue, 20 Oct 2020 11:09:08 +0200 Subject: [PATCH 35/61] [doc] (varscan): Snakefiles updated --- bio/varscan/mpileup2indel/test/Snakefile | 6 ++++++ bio/varscan/mpileup2snp/test/Snakefile | 6 ++++++ bio/varscan/somatic/test/Snakefile | 6 ++++++ 3 files changed, 18 insertions(+) diff --git a/bio/varscan/mpileup2indel/test/Snakefile b/bio/varscan/mpileup2indel/test/Snakefile index d3f18c4e67..b5ba30e5ae 100644 --- a/bio/varscan/mpileup2indel/test/Snakefile +++ b/bio/varscan/mpileup2indel/test/Snakefile @@ -9,6 +9,12 @@ rule mpileup_to_vcf: 1 # However, mpileup might have to be unzipped. # Keep threading value to one for unzipped mpileup input # Set it to two for zipped mipileup files + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties + resources: + mem_mb=1024 log: "logs/varscan_{sample}.log" wrapper: diff --git a/bio/varscan/mpileup2snp/test/Snakefile b/bio/varscan/mpileup2snp/test/Snakefile index 95d5b6b5f4..1468487568 100644 --- a/bio/varscan/mpileup2snp/test/Snakefile +++ b/bio/varscan/mpileup2snp/test/Snakefile @@ -9,6 +9,12 @@ rule mpileup_to_vcf: 1 # However, mpileup might have to be unzipped. # Keep threading value to one for unzipped mpileup input # Set it to two for zipped mipileup files + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties + resources: + mem_mb=1024 log: "logs/varscan_{sample}.log" wrapper: diff --git a/bio/varscan/somatic/test/Snakefile b/bio/varscan/somatic/test/Snakefile index 22dbf4e06b..831ed9a620 100644 --- a/bio/varscan/somatic/test/Snakefile +++ b/bio/varscan/somatic/test/Snakefile @@ -11,6 +11,12 @@ rule varscan_somatic: "Calling somatic variants {wildcards.sample}" threads: 1 + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties + resources: + mem_mb=1024 params: extra = "" wrapper: From 967580127c5e5799af9ad9d04f9cdf435a819cd8 Mon Sep 17 00:00:00 2001 From: tdayris Date: Tue, 20 Oct 2020 11:09:34 +0200 Subject: [PATCH 36/61] [fix] (varscan): Test for varscan somatic had a wrong name --- test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.py b/test.py index 1d826435ac..47f909cfb6 100644 --- a/test.py +++ b/test.py @@ -1929,7 +1929,7 @@ def test_varscan_mpileup2snp(): ) -def test_varscan_mpileup2snp(): +def test_varscan_somatic(): run( "bio/varscan/somatic", ["snakemake", "--cores", "1", "vcf/a.snp.vcf", "--use-conda", "-F"], From 62645567360012ad4f11d29755bd9b8b4bd5f751 Mon Sep 17 00:00:00 2001 From: tdayris Date: Tue, 20 Oct 2020 12:03:23 +0200 Subject: [PATCH 37/61] [doc] (trimmomatic): Snakefiles updates --- bio/trimmomatic/pe/test/Snakefile_fq_fq | 6 ++++++ bio/trimmomatic/pe/test/Snakefile_fq_gz | 6 ++++++ bio/trimmomatic/pe/test/Snakefile_gz_fq | 6 ++++++ bio/trimmomatic/pe/test/Snakefile_gz_gz | 6 ++++++ bio/trimmomatic/se/test/Snakefile_fq_fq | 6 ++++++ bio/trimmomatic/se/test/Snakefile_fq_gz | 6 ++++++ bio/trimmomatic/se/test/Snakefile_gz_fq | 6 ++++++ bio/trimmomatic/se/test/Snakefile_gz_gz | 6 ++++++ 8 files changed, 48 insertions(+) diff --git a/bio/trimmomatic/pe/test/Snakefile_fq_fq b/bio/trimmomatic/pe/test/Snakefile_fq_fq index 99397ef923..e27f80f2ce 100644 --- a/bio/trimmomatic/pe/test/Snakefile_fq_fq +++ b/bio/trimmomatic/pe/test/Snakefile_fq_fq @@ -17,5 +17,11 @@ rule trimmomatic_pe: extra="" threads: 32 + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties + resources: + mem_mb=1024 wrapper: "master/bio/trimmomatic/pe" diff --git a/bio/trimmomatic/pe/test/Snakefile_fq_gz b/bio/trimmomatic/pe/test/Snakefile_fq_gz index 293bb9c8fb..be06e639bf 100644 --- a/bio/trimmomatic/pe/test/Snakefile_fq_gz +++ b/bio/trimmomatic/pe/test/Snakefile_fq_gz @@ -18,5 +18,11 @@ rule trimmomatic_pe_fq_gz: compression_level="-9" threads: 32 + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties + resources: + mem_mb=1024 wrapper: "master/bio/trimmomatic/pe" diff --git a/bio/trimmomatic/pe/test/Snakefile_gz_fq b/bio/trimmomatic/pe/test/Snakefile_gz_fq index fc0cd05965..3981dfd5fb 100644 --- a/bio/trimmomatic/pe/test/Snakefile_gz_fq +++ b/bio/trimmomatic/pe/test/Snakefile_gz_fq @@ -17,5 +17,11 @@ rule trimmomatic_pe_gz_fq: extra="" threads: 32 + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties + resources: + mem_mb=1024 wrapper: "master/bio/trimmomatic/pe" diff --git a/bio/trimmomatic/pe/test/Snakefile_gz_gz b/bio/trimmomatic/pe/test/Snakefile_gz_gz index e1217ba52a..53c897499c 100644 --- a/bio/trimmomatic/pe/test/Snakefile_gz_gz +++ b/bio/trimmomatic/pe/test/Snakefile_gz_gz @@ -18,5 +18,11 @@ rule trimmomatic_pe: compression_level="-9" threads: 32 + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties + resources: + mem_mb=1024 wrapper: "master/bio/trimmomatic/pe" diff --git a/bio/trimmomatic/se/test/Snakefile_fq_fq b/bio/trimmomatic/se/test/Snakefile_fq_fq index 80259be19b..1a9f5fe67c 100644 --- a/bio/trimmomatic/se/test/Snakefile_fq_fq +++ b/bio/trimmomatic/se/test/Snakefile_fq_fq @@ -12,5 +12,11 @@ rule trimmomatic: extra="", threads: 32 + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties + resources: + mem_mb=1024 wrapper: "master/bio/trimmomatic/se" diff --git a/bio/trimmomatic/se/test/Snakefile_fq_gz b/bio/trimmomatic/se/test/Snakefile_fq_gz index 092efde9f8..1cd54c29d7 100644 --- a/bio/trimmomatic/se/test/Snakefile_fq_gz +++ b/bio/trimmomatic/se/test/Snakefile_fq_gz @@ -14,5 +14,11 @@ rule trimmomatic: extra="" threads: 32 + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties + resources: + mem_mb=1024 wrapper: "master/bio/trimmomatic/se" diff --git a/bio/trimmomatic/se/test/Snakefile_gz_fq b/bio/trimmomatic/se/test/Snakefile_gz_fq index 95c55bae9c..b5762c0264 100644 --- a/bio/trimmomatic/se/test/Snakefile_gz_fq +++ b/bio/trimmomatic/se/test/Snakefile_gz_fq @@ -12,5 +12,11 @@ rule trimmomatic: extra="" threads: 32 + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties + resources: + mem_mb=1024 wrapper: "master/bio/trimmomatic/se" diff --git a/bio/trimmomatic/se/test/Snakefile_gz_gz b/bio/trimmomatic/se/test/Snakefile_gz_gz index 890d03343d..bea06e3f50 100644 --- a/bio/trimmomatic/se/test/Snakefile_gz_gz +++ b/bio/trimmomatic/se/test/Snakefile_gz_gz @@ -14,5 +14,11 @@ rule trimmomatic: compression_level="-9" threads: 32 + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties + resources: + mem_mb=1024 wrapper: "master/bio/trimmomatic/se" From 915c01fd18d21d00e70eedbfc917d9b9b52262cd Mon Sep 17 00:00:00 2001 From: tdayris Date: Tue, 20 Oct 2020 12:03:37 +0200 Subject: [PATCH 38/61] [fix] (Trinity): Missing whitespace --- bio/trinity/test/Snakefile | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/bio/trinity/test/Snakefile b/bio/trinity/test/Snakefile index 90cc1f1d01..178ddf67bd 100644 --- a/bio/trinity/test/Snakefile +++ b/bio/trinity/test/Snakefile @@ -2,7 +2,7 @@ rule trinity: input: left=["reads/reads.left.fq.gz", "reads/reads2.left.fq.gz"], - right=["reads/reads.right.fq.gz", "reads/reads2.right.fq.gz"] + right=["reads/reads.right.fq.gz", "reads/reads2.right.fq.gz"] output: "trinity_out_dir/Trinity.fasta" log: @@ -10,5 +10,11 @@ rule trinity: params: extra="" threads: 4 + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties + resources: + mem_gb=10 wrapper: "master/bio/trinity" From 29480b04795dc56de595017786ecc66ffc42ec3c Mon Sep 17 00:00:00 2001 From: tdayris Date: Tue, 20 Oct 2020 12:04:05 +0200 Subject: [PATCH 39/61] [fix] (Trinity): Missing whitespace --- bio/trinity/wrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bio/trinity/wrapper.py b/bio/trinity/wrapper.py index 25fdbdee64..d814ea98a5 100644 --- a/bio/trinity/wrapper.py +++ b/bio/trinity/wrapper.py @@ -70,7 +70,7 @@ log = snakemake.log_fmt_shell(stdout=False, stderr=True) shell( - "Trinity {input_cmd}--CPU {snakemake.threads} " + "Trinity {input_cmd} --CPU {snakemake.threads} " " --max_memory {max_memory} --seqType {seqtype} " " --output {outdir} {snakemake.params.extra} " " {log}" From 79517029b28bb54f194ef6106242bc061f7c7e25 Mon Sep 17 00:00:00 2001 From: tdayris Date: Tue, 20 Oct 2020 12:05:30 +0200 Subject: [PATCH 40/61] [doc] (Snpsift): Snakefiles updated --- bio/snpsift/annotate/test/Snakefile | 6 ++++++ bio/snpsift/dbnsfp/test/Snakefile | 6 ++++++ bio/snpsift/genesets/test/Snakefile | 6 ++++++ bio/snpsift/gwascat/test/Snakefile | 6 ++++++ bio/snpsift/varType/test/Snakefile | 6 ++++++ 5 files changed, 30 insertions(+) diff --git a/bio/snpsift/annotate/test/Snakefile b/bio/snpsift/annotate/test/Snakefile index 9eb6b94849..dbbe791bd0 100644 --- a/bio/snpsift/annotate/test/Snakefile +++ b/bio/snpsift/annotate/test/Snakefile @@ -6,5 +6,11 @@ rule test_snpsift_annotate: call="annotated/out.vcf" log: "annotate.log" + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties + resources: + mem_mb=1024 wrapper: "master/bio/snpsift/annotate" diff --git a/bio/snpsift/dbnsfp/test/Snakefile b/bio/snpsift/dbnsfp/test/Snakefile index e14a921308..73b38da54a 100644 --- a/bio/snpsift/dbnsfp/test/Snakefile +++ b/bio/snpsift/dbnsfp/test/Snakefile @@ -4,5 +4,11 @@ rule test_snpsift_dbnsfp: dbNSFP = "dbNSFP.txt.gz" output: call = "out.vcf" + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties + resources: + mem_mb=1024 wrapper: "master/bio/snpsift/dbnsfp" diff --git a/bio/snpsift/genesets/test/Snakefile b/bio/snpsift/genesets/test/Snakefile index 7695e63415..30633e9544 100644 --- a/bio/snpsift/genesets/test/Snakefile +++ b/bio/snpsift/genesets/test/Snakefile @@ -4,5 +4,11 @@ rule test_snpsift_gmt: gmt = "fake_set.gmt" output: call = "annotated/out.vcf" + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties + resources: + mem_mb=1024 wrapper: "master/bio/snpsift/genesets" diff --git a/bio/snpsift/gwascat/test/Snakefile b/bio/snpsift/gwascat/test/Snakefile index 5c6b6478d3..d32f02642b 100644 --- a/bio/snpsift/gwascat/test/Snakefile +++ b/bio/snpsift/gwascat/test/Snakefile @@ -4,5 +4,11 @@ rule test_snpsift_gwascat: gwascat = "gwascatalog.txt" output: call = "annotated/out.vcf" + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties + resources: + mem_mb=1024 wrapper: "master/bio/snpsift/gwascat" diff --git a/bio/snpsift/varType/test/Snakefile b/bio/snpsift/varType/test/Snakefile index ac1ad69c0e..c6fa4a3a8e 100644 --- a/bio/snpsift/varType/test/Snakefile +++ b/bio/snpsift/varType/test/Snakefile @@ -5,6 +5,12 @@ rule test_snpsift_vartype: vcf="annotated/out.vcf" message: "Testing SnpSift varType" + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties + resources: + mem_mb=1024 log: "varType.log" wrapper: From f2fb34d475009338d68a83f9d9962cd152f205ab Mon Sep 17 00:00:00 2001 From: tdayris Date: Tue, 20 Oct 2020 12:34:02 +0200 Subject: [PATCH 41/61] [doc] (Snpeff): Snakefiles updated --- bio/snpeff/annotate/test/Snakefile | 6 +++++- bio/snpeff/annotate/test/Snakefile_nostats | 6 ++++++ bio/snpeff/download/test/Snakefile | 6 ++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/bio/snpeff/annotate/test/Snakefile b/bio/snpeff/annotate/test/Snakefile index 6c19eef040..2add04dfa0 100644 --- a/bio/snpeff/annotate/test/Snakefile +++ b/bio/snpeff/annotate/test/Snakefile @@ -8,7 +8,11 @@ rule snpeff: csvstats="snpeff/{sample}.csv" # summary statistics in CSV, optional log: "logs/snpeff/{sample}.log" + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties resources: - mem_gb = 4 # Original tests was run with -Xmx4G + mem_mb=4096 wrapper: "master/bio/snpeff/annotate" diff --git a/bio/snpeff/annotate/test/Snakefile_nostats b/bio/snpeff/annotate/test/Snakefile_nostats index ab6c2bdea8..19f7ab5f85 100644 --- a/bio/snpeff/annotate/test/Snakefile_nostats +++ b/bio/snpeff/annotate/test/Snakefile_nostats @@ -9,5 +9,11 @@ rule snpeff: "logs/snpeff_nostats/{sample}.log" params: extra="" # optional parameters + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties + resources: + mem_mb=1024 wrapper: "master/bio/snpeff/annotate" diff --git a/bio/snpeff/download/test/Snakefile b/bio/snpeff/download/test/Snakefile index 3bd28fb0c0..04bc374223 100644 --- a/bio/snpeff/download/test/Snakefile +++ b/bio/snpeff/download/test/Snakefile @@ -6,5 +6,11 @@ rule snpeff_download: "logs/snpeff/download/{reference}.log" params: reference="{reference}" + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties + resources: + mem_mb=1024 wrapper: "master/bio/snpeff/download" From 6e95a9231a8b79b19d439dc18cfd4d39cec44ba2 Mon Sep 17 00:00:00 2001 From: tdayris Date: Tue, 20 Oct 2020 12:34:28 +0200 Subject: [PATCH 42/61] [fix] (snpeff-download): Useless extra parameters removed --- bio/snpeff/download/wrapper.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/bio/snpeff/download/wrapper.py b/bio/snpeff/download/wrapper.py index 84534dcb57..49141c1c93 100644 --- a/bio/snpeff/download/wrapper.py +++ b/bio/snpeff/download/wrapper.py @@ -10,23 +10,22 @@ outdir = Path(snakemake.output[0]).parent.resolve() log = snakemake.log_fmt_shell(stdout=False, stderr=True) -extra = snakemake.params java_opts = "" # Getting memory in megabytes, if java opts is not filled with -Xmx parameter # By doing so, backward compatibility is preserved -if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: +if "mem_mb" in snakemake.resources.keys(): java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) # Getting memory in gigabytes, for user convenience. Please prefer the use # of mem_mb over mem_gb as advised in documentation. -elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: +elif "mem_gb" in snakemake.resources.keys(): java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) # Getting java temp directory from output files list, if -Djava.io.tmpdir # is not provided in java parameters. By doing so, backward compatibility is # not broken. -if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: +if "java_temp" in snakemake.output.keys(): java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) -shell("snpEff download {java_opts} {extra} -dataDir {outdir} {reference} {log}") +shell("snpEff download {java_opts} -dataDir {outdir} {reference} {log}") From 3062c26439f2cf995c2af569214d5bf76bd4416b Mon Sep 17 00:00:00 2001 From: tdayris Date: Tue, 20 Oct 2020 12:51:42 +0200 Subject: [PATCH 43/61] [doc] (picard): Snakefiles updated --- bio/picard/addorreplacereadgroups/test/Snakefile | 6 ++++++ bio/picard/bedtointervallist/test/Snakefile | 7 ++++++- bio/picard/collectalignmentsummarymetrics/test/Snakefile | 6 ++++++ bio/picard/collecthsmetrics/test/Snakefile | 6 ++++++ bio/picard/collectinsertsizemetrics/test/Snakefile | 6 ++++++ bio/picard/collecttargetedpcrmetrics/test/Snakefile | 6 ++++++ bio/picard/createsequencedictionary/test/Snakefile | 6 ++++++ bio/picard/markduplicates/test/Snakefile | 6 ++++++ bio/picard/mergesamfiles/test/Snakefile | 6 ++++++ bio/picard/mergevcfs/test/Snakefile | 6 ++++++ bio/picard/revertsam/test/Snakefile | 6 ++++++ bio/picard/samtofastq/test/Snakefile | 6 ++++++ bio/picard/sortsam/test/Snakefile | 6 ++++++ 13 files changed, 78 insertions(+), 1 deletion(-) diff --git a/bio/picard/addorreplacereadgroups/test/Snakefile b/bio/picard/addorreplacereadgroups/test/Snakefile index 0a9e97ddd8..7228942595 100644 --- a/bio/picard/addorreplacereadgroups/test/Snakefile +++ b/bio/picard/addorreplacereadgroups/test/Snakefile @@ -7,5 +7,11 @@ rule replace_rg: "logs/picard/replace_rg/{sample}.log" params: "RGLB=lib1 RGPL=illumina RGPU={sample} RGSM={sample}" + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties + resources: + mem_mb=1024 wrapper: "master/bio/picard/addorreplacereadgroups" diff --git a/bio/picard/bedtointervallist/test/Snakefile b/bio/picard/bedtointervallist/test/Snakefile index 315ffd08bd..d2a2abf4d3 100644 --- a/bio/picard/bedtointervallist/test/Snakefile +++ b/bio/picard/bedtointervallist/test/Snakefile @@ -9,6 +9,11 @@ rule bed_to_interval_list: params: # optional parameters "SORT=true " # sort output interval list before writing + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties + resources: + mem_mb=1024 wrapper: "master/bio/picard/bedtointervallist" - diff --git a/bio/picard/collectalignmentsummarymetrics/test/Snakefile b/bio/picard/collectalignmentsummarymetrics/test/Snakefile index c3d07e0f47..9512db88bc 100644 --- a/bio/picard/collectalignmentsummarymetrics/test/Snakefile +++ b/bio/picard/collectalignmentsummarymetrics/test/Snakefile @@ -11,5 +11,11 @@ rule alignment_summary: "VALIDATION_STRINGENCY=LENIENT " "METRIC_ACCUMULATION_LEVEL=null " "METRIC_ACCUMULATION_LEVEL=SAMPLE" + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties + resources: + mem_mb=1024 wrapper: "master/bio/picard/collectalignmentsummarymetrics" diff --git a/bio/picard/collecthsmetrics/test/Snakefile b/bio/picard/collecthsmetrics/test/Snakefile index 0ae4264c95..5d66ba801a 100644 --- a/bio/picard/collecthsmetrics/test/Snakefile +++ b/bio/picard/collecthsmetrics/test/Snakefile @@ -12,6 +12,12 @@ rule picard_collect_hs_metrics: # Optional extra arguments. Here we reduce sample size # to reduce the runtime in our unit test. extra="SAMPLE_SIZE=1000" + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties + resources: + mem_mb=1024 log: "logs/picard_collect_hs_metrics/{sample}.log" wrapper: diff --git a/bio/picard/collectinsertsizemetrics/test/Snakefile b/bio/picard/collectinsertsizemetrics/test/Snakefile index 4790b4162b..a9af368d49 100644 --- a/bio/picard/collectinsertsizemetrics/test/Snakefile +++ b/bio/picard/collectinsertsizemetrics/test/Snakefile @@ -11,5 +11,11 @@ rule insert_size: "VALIDATION_STRINGENCY=LENIENT " "METRIC_ACCUMULATION_LEVEL=null " "METRIC_ACCUMULATION_LEVEL=SAMPLE" + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties + resources: + mem_mb=1024 wrapper: "master/bio/picard/collectinsertsizemetrics" diff --git a/bio/picard/collecttargetedpcrmetrics/test/Snakefile b/bio/picard/collecttargetedpcrmetrics/test/Snakefile index c2de3454e7..54fcf562ac 100644 --- a/bio/picard/collecttargetedpcrmetrics/test/Snakefile +++ b/bio/picard/collecttargetedpcrmetrics/test/Snakefile @@ -9,5 +9,11 @@ rule CollectTargetedPcrMetrics: "logs/picard/collecttargetedpcrmetrics/{sample}.log" params: extra="" + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties + resources: + mem_mb=1024 wrapper: "master/bio/picard/collecttargetedpcrmetrics" diff --git a/bio/picard/createsequencedictionary/test/Snakefile b/bio/picard/createsequencedictionary/test/Snakefile index ddbd70ffb4..ddce54c1bb 100644 --- a/bio/picard/createsequencedictionary/test/Snakefile +++ b/bio/picard/createsequencedictionary/test/Snakefile @@ -7,5 +7,11 @@ rule create_dict: "logs/picard/create_dict.log" params: extra="" # optional: extra arguments for picard. + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties + resources: + mem_mb=1024 wrapper: "master/bio/picard/createsequencedictionary" diff --git a/bio/picard/markduplicates/test/Snakefile b/bio/picard/markduplicates/test/Snakefile index c9fc10dc89..03dca2a91d 100644 --- a/bio/picard/markduplicates/test/Snakefile +++ b/bio/picard/markduplicates/test/Snakefile @@ -8,5 +8,11 @@ rule mark_duplicates: "logs/picard/dedup/{sample}.log" params: "REMOVE_DUPLICATES=true" + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties + resources: + mem_mb=1024 wrapper: "master/bio/picard/markduplicates" diff --git a/bio/picard/mergesamfiles/test/Snakefile b/bio/picard/mergesamfiles/test/Snakefile index 4bdfb7828a..9f3df07e85 100644 --- a/bio/picard/mergesamfiles/test/Snakefile +++ b/bio/picard/mergesamfiles/test/Snakefile @@ -7,5 +7,11 @@ rule merge_bams: "logs/picard_mergesamfiles.log" params: "VALIDATION_STRINGENCY=LENIENT" + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties + resources: + mem_mb=1024 wrapper: "master/bio/picard/mergesamfiles" diff --git a/bio/picard/mergevcfs/test/Snakefile b/bio/picard/mergevcfs/test/Snakefile index c04291dd5a..f378488977 100644 --- a/bio/picard/mergevcfs/test/Snakefile +++ b/bio/picard/mergevcfs/test/Snakefile @@ -7,5 +7,11 @@ rule merge_vcfs: "logs/picard/mergevcfs.log" params: extra="" + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties + resources: + mem_mb=1024 wrapper: "master/bio/picard/mergevcfs" diff --git a/bio/picard/revertsam/test/Snakefile b/bio/picard/revertsam/test/Snakefile index 10fbe867c9..8ac10abd27 100644 --- a/bio/picard/revertsam/test/Snakefile +++ b/bio/picard/revertsam/test/Snakefile @@ -7,5 +7,11 @@ rule revert_bam: "logs/picard/revert_sam/{sample}.log" params: extra="SANITIZE=true" # optional: Extra arguments for picard. + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties + resources: + mem_mb=1024 wrapper: "master/bio/picard/revertsam" diff --git a/bio/picard/samtofastq/test/Snakefile b/bio/picard/samtofastq/test/Snakefile index 78f43cec4f..9ff5492641 100644 --- a/bio/picard/samtofastq/test/Snakefile +++ b/bio/picard/samtofastq/test/Snakefile @@ -8,5 +8,11 @@ rule bam_to_fastq: "logs/picard/sam_to_fastq/{sample}.log" params: extra="" # optional: Extra arguments for picard. + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties + resources: + mem_mb=1024 wrapper: "master/bio/picard/samtofastq" diff --git a/bio/picard/sortsam/test/Snakefile b/bio/picard/sortsam/test/Snakefile index e66ee13253..b4c1b4b4e7 100644 --- a/bio/picard/sortsam/test/Snakefile +++ b/bio/picard/sortsam/test/Snakefile @@ -8,5 +8,11 @@ rule sort_bam: params: sort_order="coordinate", extra="VALIDATION_STRINGENCY=LENIENT" # optional: Extra arguments for picard. + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties + resources: + mem_mb=1024 wrapper: "master/bio/picard/sortsam" From a0d73fe73b27c0853c7dc15c6e44fd4b51a86b11 Mon Sep 17 00:00:00 2001 From: tdayris Date: Tue, 20 Oct 2020 13:03:01 +0200 Subject: [PATCH 44/61] [doc] (GATK): Snakefiles updated --- bio/gatk/applybqsr/test/Snakefile | 6 ++++++ bio/gatk/baserecalibrator/test/Snakefile | 6 ++++++ bio/gatk/baserecalibratorspark/test/Snakefile | 6 ++++++ bio/gatk/combinegvcfs/test/Snakefile | 6 ++++++ bio/gatk/genotypegvcfs/test/Snakefile | 6 ++++++ bio/gatk/haplotypecaller/test/Snakefile | 6 ++++++ bio/gatk/mutect/test/Snakefile | 6 ++++++ bio/gatk/selectvariants/test/Snakefile | 8 +++++++- bio/gatk/splitncigarreads/test/Snakefile | 6 ++++++ bio/gatk/variantfiltration/test/Snakefile | 6 ++++++ bio/gatk/variantrecalibrator/test/Snakefile | 8 +++++++- 11 files changed, 68 insertions(+), 2 deletions(-) diff --git a/bio/gatk/applybqsr/test/Snakefile b/bio/gatk/applybqsr/test/Snakefile index 87a1f06942..08f4dc0fec 100644 --- a/bio/gatk/applybqsr/test/Snakefile +++ b/bio/gatk/applybqsr/test/Snakefile @@ -11,5 +11,11 @@ rule gatk_applybqsr: params: extra="", # optional java_opts="", # optional + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties + resources: + mem_mb=1024 wrapper: "master/bio/gatk/applybqsr" diff --git a/bio/gatk/baserecalibrator/test/Snakefile b/bio/gatk/baserecalibrator/test/Snakefile index 508ae11d08..a9a51ca8ea 100644 --- a/bio/gatk/baserecalibrator/test/Snakefile +++ b/bio/gatk/baserecalibrator/test/Snakefile @@ -11,5 +11,11 @@ rule gatk_baserecalibrator: params: extra="", # optional java_opts="", # optional + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties + resources: + mem_mb=1024 wrapper: "master/bio/gatk/baserecalibrator" diff --git a/bio/gatk/baserecalibratorspark/test/Snakefile b/bio/gatk/baserecalibratorspark/test/Snakefile index bec0e7c1c3..ea7bf6b34b 100644 --- a/bio/gatk/baserecalibratorspark/test/Snakefile +++ b/bio/gatk/baserecalibratorspark/test/Snakefile @@ -14,6 +14,12 @@ rule gatk_baserecalibratorspark: #spark_runner="", # optional, local by default #spark_master="", # optional #spark_extra="", # optional + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties + resources: + mem_mb=1024 threads: 8 wrapper: "master/bio/gatk/baserecalibratorspark" diff --git a/bio/gatk/combinegvcfs/test/Snakefile b/bio/gatk/combinegvcfs/test/Snakefile index fb5a9d3ec7..2f1ac2c4d2 100644 --- a/bio/gatk/combinegvcfs/test/Snakefile +++ b/bio/gatk/combinegvcfs/test/Snakefile @@ -9,5 +9,11 @@ rule genotype_gvcfs: params: extra="", # optional java_opts="", # optional + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties + resources: + mem_mb=1024 wrapper: "master/bio/gatk/combinegvcfs" diff --git a/bio/gatk/genotypegvcfs/test/Snakefile b/bio/gatk/genotypegvcfs/test/Snakefile index 9f7bf430e7..6e2d49047c 100644 --- a/bio/gatk/genotypegvcfs/test/Snakefile +++ b/bio/gatk/genotypegvcfs/test/Snakefile @@ -9,5 +9,11 @@ rule genotype_gvcfs: params: extra="", # optional java_opts="", # optional + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties + resources: + mem_mb=1024 wrapper: "master/bio/gatk/genotypegvcfs" diff --git a/bio/gatk/haplotypecaller/test/Snakefile b/bio/gatk/haplotypecaller/test/Snakefile index f65b9eb5c3..98fa4281ad 100644 --- a/bio/gatk/haplotypecaller/test/Snakefile +++ b/bio/gatk/haplotypecaller/test/Snakefile @@ -11,5 +11,11 @@ rule haplotype_caller: params: extra="", # optional java_opts="", # optional + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties + resources: + mem_mb=1024 wrapper: "master/bio/gatk/haplotypecaller" diff --git a/bio/gatk/mutect/test/Snakefile b/bio/gatk/mutect/test/Snakefile index 7f6341a3e7..ade6c0c6f3 100644 --- a/bio/gatk/mutect/test/Snakefile +++ b/bio/gatk/mutect/test/Snakefile @@ -8,6 +8,12 @@ rule mutect2: "Testing Mutect2 with {wildcards.sample}" threads: 1 + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties + resources: + mem_mb=1024 log: "logs/mutect_{sample}.log" wrapper: diff --git a/bio/gatk/selectvariants/test/Snakefile b/bio/gatk/selectvariants/test/Snakefile index 099bc53bf1..f1be8087b1 100644 --- a/bio/gatk/selectvariants/test/Snakefile +++ b/bio/gatk/selectvariants/test/Snakefile @@ -8,6 +8,12 @@ rule gatk_select: "logs/gatk/select/snvs.log" params: extra="--select-type-to-include SNP", # optional filter arguments, see GATK docs - java_opts="", # optional + java_opts="", # optional + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties + resources: + mem_mb=1024 wrapper: "master/bio/gatk/selectvariants" diff --git a/bio/gatk/splitncigarreads/test/Snakefile b/bio/gatk/splitncigarreads/test/Snakefile index 08ba6ab50a..1d7c5774fb 100644 --- a/bio/gatk/splitncigarreads/test/Snakefile +++ b/bio/gatk/splitncigarreads/test/Snakefile @@ -9,5 +9,11 @@ rule splitncigarreads: params: extra="", # optional java_opts="", # optional + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties + resources: + mem_mb=1024 wrapper: "master/bio/gatk/splitncigarreads" diff --git a/bio/gatk/variantfiltration/test/Snakefile b/bio/gatk/variantfiltration/test/Snakefile index a789065c0d..9c888dd9b6 100644 --- a/bio/gatk/variantfiltration/test/Snakefile +++ b/bio/gatk/variantfiltration/test/Snakefile @@ -10,5 +10,11 @@ rule gatk_filter: filters={"myfilter": "AB < 0.2 || MQ0 > 50"}, extra="", # optional arguments, see GATK docs java_opts="", # optional + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties + resources: + mem_mb=1024 wrapper: "master/bio/gatk/variantfiltration" diff --git a/bio/gatk/variantrecalibrator/test/Snakefile b/bio/gatk/variantrecalibrator/test/Snakefile index 8364f1fe3f..6d928cc370 100644 --- a/bio/gatk/variantrecalibrator/test/Snakefile +++ b/bio/gatk/variantrecalibrator/test/Snakefile @@ -19,7 +19,7 @@ rule haplotype_caller: g1k=gatk_bundle("1000G_phase1.snps.high_confidence.hg38.vcf.gz"), dbsnp=gatk_bundle("Homo_sapiens_assembly38.dbsnp138.vcf.gz"), # use aux to e.g. download other necessary file - aux=[gatk_bundle("hapmap_3.3.hg38.sites.vcf.gz.tbi"), + aux=[gatk_bundle("hapmap_3.3.hg38.sites.vcf.gz.tbi"), gatk_bundle("1000G_omni2.5.hg38.sites.vcf.gz.tbi"), gatk_bundle("1000G_phase1.snps.high_confidence.hg38.vcf.gz.tbi"), gatk_bundle("Homo_sapiens_assembly38.dbsnp138.vcf.gz.tbi")] @@ -38,5 +38,11 @@ rule haplotype_caller: annotation=["QD", "FisherStrand"], # which fields to use with -an (see VariantRecalibrator docs) extra="", # optional java_opts="", # optional + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mb}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties + resources: + mem_mb=1024 wrapper: "master/bio/gatk/haplotypecaller" From b5ea7aea7521021ba41514b0449c66b1b945f803 Mon Sep 17 00:00:00 2001 From: tdayris Date: Mon, 2 Nov 2020 12:09:47 +0100 Subject: [PATCH 45/61] [dev] (GATK4): Update with snakemake-wrappers-utils --- bio/gatk/applybqsr/environment.yaml | 3 ++- bio/gatk/applybqsr/wrapper.py | 19 ++----------------- bio/gatk/baserecalibrator/environment.yaml | 3 ++- bio/gatk/baserecalibrator/wrapper.py | 19 ++----------------- .../baserecalibratorspark/environment.yaml | 3 ++- bio/gatk/baserecalibratorspark/wrapper.py | 18 ++---------------- bio/gatk/combinegvcfs/environment.yaml | 1 + bio/gatk/combinegvcfs/wrapper.py | 18 ++---------------- bio/gatk/genotypegvcfs/environment.yaml | 1 + bio/gatk/genotypegvcfs/wrapper.py | 18 ++---------------- bio/gatk/haplotypecaller/environment.yaml | 1 + bio/gatk/haplotypecaller/wrapper.py | 18 ++---------------- bio/gatk/mutect/environment.yaml | 1 + bio/gatk/mutect/wrapper.py | 18 ++---------------- bio/gatk/selectvariants/environment.yaml | 1 + bio/gatk/selectvariants/wrapper.py | 18 ++---------------- bio/gatk/splitncigarreads/environment.yaml | 1 + bio/gatk/splitncigarreads/wrapper.py | 18 ++---------------- bio/gatk/variantfiltration/environment.yaml | 1 + bio/gatk/variantfiltration/wrapper.py | 18 ++---------------- bio/gatk/variantrecalibrator/environment.yaml | 1 + bio/gatk/variantrecalibrator/wrapper.py | 18 ++---------------- 22 files changed, 36 insertions(+), 181 deletions(-) diff --git a/bio/gatk/applybqsr/environment.yaml b/bio/gatk/applybqsr/environment.yaml index 3510cd8498..6fb2d81249 100644 --- a/bio/gatk/applybqsr/environment.yaml +++ b/bio/gatk/applybqsr/environment.yaml @@ -4,4 +4,5 @@ channels: - defaults dependencies: - gatk4 ==4.1.4.1 - - openjdk =8 \ No newline at end of file + - openjdk =8 + - snakemake-wrapper-utils ==0.1.2 diff --git a/bio/gatk/applybqsr/wrapper.py b/bio/gatk/applybqsr/wrapper.py index 784e972abd..1b2e4b5dca 100644 --- a/bio/gatk/applybqsr/wrapper.py +++ b/bio/gatk/applybqsr/wrapper.py @@ -5,25 +5,10 @@ from snakemake.shell import shell +from snakemake_wrapper_utils.java import get_java_opts extra = snakemake.params.get("extra", "") -java_opts = snakemake.params.get("java_opts", "") - -# Getting memory in megabytes, if java opts is not filled with -Xmx parameter -# By doing so, backward compatibility is preserved -if "mem_mb" in snakemake.resources.keys() and not "-Xmx" in java_opts: - java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) - -# Getting memory in gigabytes, for user convenience. Please prefer the use -# of mem_mb over mem_gb as advised in documentation. -elif "mem_gb" in snakemake.resources.keys() and not "-Xmx" in java_opts: - java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) - -# Getting java temp directory from output files list, if -Djava.io.tmpdir -# is not provided in java parameters. By doing so, backward compatibility is -# not borken. -if "java_temp" in snakemake.output.keys() and not "-Djava.io.tmpdir" in java_opts: - java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) +java_opts = get_java_opts(snakemake) log = snakemake.log_fmt_shell(stdout=True, stderr=True, append=True) shell( diff --git a/bio/gatk/baserecalibrator/environment.yaml b/bio/gatk/baserecalibrator/environment.yaml index 3510cd8498..6fb2d81249 100644 --- a/bio/gatk/baserecalibrator/environment.yaml +++ b/bio/gatk/baserecalibrator/environment.yaml @@ -4,4 +4,5 @@ channels: - defaults dependencies: - gatk4 ==4.1.4.1 - - openjdk =8 \ No newline at end of file + - openjdk =8 + - snakemake-wrapper-utils ==0.1.2 diff --git a/bio/gatk/baserecalibrator/wrapper.py b/bio/gatk/baserecalibrator/wrapper.py index 737d52c388..b0245f292e 100644 --- a/bio/gatk/baserecalibrator/wrapper.py +++ b/bio/gatk/baserecalibrator/wrapper.py @@ -5,25 +5,10 @@ from snakemake.shell import shell +from snakemake_wrapper_utils.java import get_java_opts extra = snakemake.params.get("extra", "") -java_opts = snakemake.params.get("java_opts", "") - -# Getting memory in megabytes, if java opts is not filled with -Xmx parameter -# By doing so, backward compatibility is preserved -if "mem_mb" in snakemake.resources.keys() and not "-Xmx" in java_opts: - java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) - -# Getting memory in gigabytes, for user convenience. Please prefer the use -# of mem_mb over mem_gb as advised in documentation. -elif "mem_gb" in snakemake.resources.keys() and not "-Xmx" in java_opts: - java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) - -# Getting java temp directory from output files list, if -Djava.io.tmpdir -# is not provided in java parameters. By doing so, backward compatibility is -# not broken. -if "java_temp" in snakemake.output.keys() and not "-Djava.io.tmpdir" in java_opts: - java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) +java_opts = get_java_opts(snakemake) log = snakemake.log_fmt_shell(stdout=True, stderr=True) known = snakemake.input.get("known", "") diff --git a/bio/gatk/baserecalibratorspark/environment.yaml b/bio/gatk/baserecalibratorspark/environment.yaml index 3510cd8498..6fb2d81249 100644 --- a/bio/gatk/baserecalibratorspark/environment.yaml +++ b/bio/gatk/baserecalibratorspark/environment.yaml @@ -4,4 +4,5 @@ channels: - defaults dependencies: - gatk4 ==4.1.4.1 - - openjdk =8 \ No newline at end of file + - openjdk =8 + - snakemake-wrapper-utils ==0.1.2 diff --git a/bio/gatk/baserecalibratorspark/wrapper.py b/bio/gatk/baserecalibratorspark/wrapper.py index 09146524a9..94e1b1f2ab 100644 --- a/bio/gatk/baserecalibratorspark/wrapper.py +++ b/bio/gatk/baserecalibratorspark/wrapper.py @@ -5,6 +5,7 @@ from snakemake.shell import shell +from snakemake_wrapper_utils.java import get_java_opts extra = snakemake.params.get("extra", "") spark_runner = snakemake.params.get("spark_runner", "LOCAL") @@ -12,22 +13,7 @@ "spark_master", "local[{}]".format(snakemake.threads) ) spark_extra = snakemake.params.get("spark_extra", "") -java_opts = snakemake.params.get("java_opts", "") -# Getting memory in megabytes, if java opts is not filled with -Xmx parameter -# By doing so, backward compatibility is preserved -if "mem_mb" in snakemake.resources.keys() and not "-Xmx" in java_opts: - java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) - -# Getting memory in gigabytes, for user convenience. Please prefer the use -# of mem_mb over mem_gb as advised in documentation. -elif "mem_gb" in snakemake.resources.keys() and not "-Xmx" in java_opts: - java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) - -# Getting java temp directory from output files list, if -Djava.io.tmpdir -# is not provided in java parameters. By doing so, backward compatibility is -# not broken. -if "java_temp" in snakemake.output.keys() and not "-Djava.io.tmpdir" in java_opts: - java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) +java_opts = get_java_opts(snakemake) log = snakemake.log_fmt_shell(stdout=True, stderr=True) known = snakemake.input.get("known", "") diff --git a/bio/gatk/combinegvcfs/environment.yaml b/bio/gatk/combinegvcfs/environment.yaml index 17b64b9e46..a92753a102 100644 --- a/bio/gatk/combinegvcfs/environment.yaml +++ b/bio/gatk/combinegvcfs/environment.yaml @@ -4,3 +4,4 @@ channels: - defaults dependencies: - gatk4 ==4.1.4.1 + - snakemake-wrapper-utils ==0.1.2 diff --git a/bio/gatk/combinegvcfs/wrapper.py b/bio/gatk/combinegvcfs/wrapper.py index fcedf72dbb..61a46161bd 100644 --- a/bio/gatk/combinegvcfs/wrapper.py +++ b/bio/gatk/combinegvcfs/wrapper.py @@ -7,25 +7,11 @@ import os from snakemake.shell import shell +from snakemake_wrapper_utils.java import get_java_opts extra = snakemake.params.get("extra", "") -java_opts = snakemake.params.get("java_opts", "") -# Getting memory in megabytes, if java opts is not filled with -Xmx parameter -# By doing so, backward compatibility is preserved -if "mem_mb" in snakemake.resources.keys() and not "-Xmx" in java_opts: - java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) - -# Getting memory in gigabytes, for user convenience. Please prefer the use -# of mem_mb over mem_gb as advised in documentation. -elif "mem_gb" in snakemake.resources.keys() and not "-Xmx" in java_opts: - java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) - -# Getting java temp directory from output files list, if -Djava.io.tmpdir -# is not provided in java parameters. By doing so, backward compatibility is -# not broken. -if "java_temp" in snakemake.output.keys() and not "-Djava.io.tmpdir" in java_opts: - java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) +java_opts = get_java_opts(snakemake) gvcfs = list(map("-V {}".format, snakemake.input.gvcfs)) diff --git a/bio/gatk/genotypegvcfs/environment.yaml b/bio/gatk/genotypegvcfs/environment.yaml index 17b64b9e46..a92753a102 100644 --- a/bio/gatk/genotypegvcfs/environment.yaml +++ b/bio/gatk/genotypegvcfs/environment.yaml @@ -4,3 +4,4 @@ channels: - defaults dependencies: - gatk4 ==4.1.4.1 + - snakemake-wrapper-utils ==0.1.2 diff --git a/bio/gatk/genotypegvcfs/wrapper.py b/bio/gatk/genotypegvcfs/wrapper.py index 5d580ca482..50a8c1f41b 100644 --- a/bio/gatk/genotypegvcfs/wrapper.py +++ b/bio/gatk/genotypegvcfs/wrapper.py @@ -7,25 +7,11 @@ import os from snakemake.shell import shell +from snakemake_wrapper_utils.java import get_java_opts extra = snakemake.params.get("extra", "") -java_opts = snakemake.params.get("java_opts", "") -# Getting memory in megabytes, if java opts is not filled with -Xmx parameter -# By doing so, backward compatibility is preserved -if "mem_mb" in snakemake.resources.keys() and not "-Xmx" in java_opts: - java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) - -# Getting memory in gigabytes, for user convenience. Please prefer the use -# of mem_mb over mem_gb as advised in documentation. -elif "mem_gb" in snakemake.resources.keys() and not "-Xmx" in java_opts: - java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) - -# Getting java temp directory from output files list, if -Djava.io.tmpdir -# is not provided in java parameters. By doing so, backward compatibility is -# not broken. -if "java_temp" in snakemake.output.keys() and not "-Djava.io.tmpdir" in java_opts: - java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) +java_opts = get_java_opts(snakemake) log = snakemake.log_fmt_shell(stdout=True, stderr=True) shell( diff --git a/bio/gatk/haplotypecaller/environment.yaml b/bio/gatk/haplotypecaller/environment.yaml index 17b64b9e46..a92753a102 100644 --- a/bio/gatk/haplotypecaller/environment.yaml +++ b/bio/gatk/haplotypecaller/environment.yaml @@ -4,3 +4,4 @@ channels: - defaults dependencies: - gatk4 ==4.1.4.1 + - snakemake-wrapper-utils ==0.1.2 diff --git a/bio/gatk/haplotypecaller/wrapper.py b/bio/gatk/haplotypecaller/wrapper.py index d2d83552a2..afaac11dfa 100644 --- a/bio/gatk/haplotypecaller/wrapper.py +++ b/bio/gatk/haplotypecaller/wrapper.py @@ -7,28 +7,14 @@ import os from snakemake.shell import shell +from snakemake_wrapper_utils.java import get_java_opts known = snakemake.input.get("known", "") if known: known = "--dbsnp " + known extra = snakemake.params.get("extra", "") -java_opts = snakemake.params.get("java_opts", "") -# Getting memory in megabytes, if java opts is not filled with -Xmx parameter -# By doing so, backward compatibility is preserved -if "mem_mb" in snakemake.resources.keys() and not "-Xmx" in java_opts: - java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) - -# Getting memory in gigabytes, for user convenience. Please prefer the use -# of mem_mb over mem_gb as advised in documentation. -elif "mem_gb" in snakemake.resources.keys() and not "-Xmx" in java_opts: - java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) - -# Getting java temp directory from output files list, if -Djava.io.tmpdir -# is not provided in java parameters. By doing so, backward compatibility is -# not broken. -if "java_temp" in snakemake.output.keys() and not "-Djava.io.tmpdir" in java_opts: - java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) +java_opts = get_java_opts(snakemake) bams = snakemake.input.bam if isinstance(bams, str): diff --git a/bio/gatk/mutect/environment.yaml b/bio/gatk/mutect/environment.yaml index 17b64b9e46..a92753a102 100644 --- a/bio/gatk/mutect/environment.yaml +++ b/bio/gatk/mutect/environment.yaml @@ -4,3 +4,4 @@ channels: - defaults dependencies: - gatk4 ==4.1.4.1 + - snakemake-wrapper-utils ==0.1.2 diff --git a/bio/gatk/mutect/wrapper.py b/bio/gatk/mutect/wrapper.py index c457ca131b..3a9d00bce0 100644 --- a/bio/gatk/mutect/wrapper.py +++ b/bio/gatk/mutect/wrapper.py @@ -7,26 +7,12 @@ from snakemake.shell import shell from snakemake.utils import makedirs +from snakemake_wrapper_utils.java import get_java_opts log = snakemake.log_fmt_shell(stdout=True, stderr=True) extra = snakemake.params.get("extra", "") -java_opts = snakemake.params.get("java_opts", "") -# Getting memory in megabytes, if java opts is not filled with -Xmx parameter -# By doing so, backward compatibility is preserved -if "mem_mb" in snakemake.resources.keys() and not "-Xmx" in java_opts: - java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) - -# Getting memory in gigabytes, for user convenience. Please prefer the use -# of mem_mb over mem_gb as advised in documentation. -elif "mem_gb" in snakemake.resources.keys() and not "-Xmx" in java_opts: - java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) - -# Getting java temp directory from output files list, if -Djava.io.tmpdir -# is not provided in java parameters. By doing so, backward compatibility is -# not broken. -if "java_temp" in snakemake.output.keys() and not "-Djava.io.tmpdir" in java_opts: - java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) +java_opts = get_java_opts(snakemake) shell( "gatk --java-options '{java_opts}' Mutect2 " # Tool and its subprocess diff --git a/bio/gatk/selectvariants/environment.yaml b/bio/gatk/selectvariants/environment.yaml index 17b64b9e46..a92753a102 100644 --- a/bio/gatk/selectvariants/environment.yaml +++ b/bio/gatk/selectvariants/environment.yaml @@ -4,3 +4,4 @@ channels: - defaults dependencies: - gatk4 ==4.1.4.1 + - snakemake-wrapper-utils ==0.1.2 diff --git a/bio/gatk/selectvariants/wrapper.py b/bio/gatk/selectvariants/wrapper.py index f8e4781268..a07bca351c 100644 --- a/bio/gatk/selectvariants/wrapper.py +++ b/bio/gatk/selectvariants/wrapper.py @@ -5,24 +5,10 @@ from snakemake.shell import shell +from snakemake_wrapper_utils.java import get_java_opts extra = snakemake.params.get("extra", "") -java_opts = snakemake.params.get("java_opts", "") -# Getting memory in megabytes, if java opts is not filled with -Xmx parameter -# By doing so, backward compatibility is preserved -if "mem_mb" in snakemake.resources.keys() and not "-Xmx" in java_opts: - java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) - -# Getting memory in gigabytes, for user convenience. Please prefer the use -# of mem_mb over mem_gb as advised in documentation. -elif "mem_gb" in snakemake.resources.keys() and not "-Xmx" in java_opts: - java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) - -# Getting java temp directory from output files list, if -Djava.io.tmpdir -# is not provided in java parameters. By doing so, backward compatibility is -# not broken. -if "java_temp" in snakemake.output.keys() and not "-Djava.io.tmpdir" in java_opts: - java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) +java_opts = get_java_opts(snakemake) log = snakemake.log_fmt_shell(stdout=True, stderr=True) shell( diff --git a/bio/gatk/splitncigarreads/environment.yaml b/bio/gatk/splitncigarreads/environment.yaml index 17b64b9e46..a92753a102 100644 --- a/bio/gatk/splitncigarreads/environment.yaml +++ b/bio/gatk/splitncigarreads/environment.yaml @@ -4,3 +4,4 @@ channels: - defaults dependencies: - gatk4 ==4.1.4.1 + - snakemake-wrapper-utils ==0.1.2 diff --git a/bio/gatk/splitncigarreads/wrapper.py b/bio/gatk/splitncigarreads/wrapper.py index fe9eafbb6e..9e3b1162d9 100644 --- a/bio/gatk/splitncigarreads/wrapper.py +++ b/bio/gatk/splitncigarreads/wrapper.py @@ -6,24 +6,10 @@ import os from snakemake.shell import shell +from snakemake_wrapper_utils.java import get_java_opts extra = snakemake.params.get("extra", "") -java_opts = snakemake.params.get("java_opts", "") -# Getting memory in megabytes, if java opts is not filled with -Xmx parameter -# By doing so, backward compatibility is preserved -if "mem_mb" in snakemake.resources.keys() and not "-Xmx" in java_opts: - java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) - -# Getting memory in gigabytes, for user convenience. Please prefer the use -# of mem_mb over mem_gb as advised in documentation. -elif "mem_gb" in snakemake.resources.keys() and not "-Xmx" in java_opts: - java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) - -# Getting java temp directory from output files list, if -Djava.io.tmpdir -# is not provided in java parameters. By doing so, backward compatibility is -# not broken. -if "java_temp" in snakemake.output.keys() and not "-Djava.io.tmpdir" in java_opts: - java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) +java_opts = get_java_opts(snakemake) log = snakemake.log_fmt_shell(stdout=True, stderr=True) shell( diff --git a/bio/gatk/variantfiltration/environment.yaml b/bio/gatk/variantfiltration/environment.yaml index 17b64b9e46..a92753a102 100644 --- a/bio/gatk/variantfiltration/environment.yaml +++ b/bio/gatk/variantfiltration/environment.yaml @@ -4,3 +4,4 @@ channels: - defaults dependencies: - gatk4 ==4.1.4.1 + - snakemake-wrapper-utils ==0.1.2 diff --git a/bio/gatk/variantfiltration/wrapper.py b/bio/gatk/variantfiltration/wrapper.py index 17095907d9..ffcc5f52e2 100644 --- a/bio/gatk/variantfiltration/wrapper.py +++ b/bio/gatk/variantfiltration/wrapper.py @@ -5,24 +5,10 @@ from snakemake.shell import shell +from snakemake_wrapper_utils.java import get_java_opts extra = snakemake.params.get("extra", "") -java_opts = snakemake.params.get("java_opts", "") -# Getting memory in megabytes, if java opts is not filled with -Xmx parameter -# By doing so, backward compatibility is preserved -if "mem_mb" in snakemake.resources.keys() and not "-Xmx" in java_opts: - java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) - -# Getting memory in gigabytes, for user convenience. Please prefer the use -# of mem_mb over mem_gb as advised in documentation. -elif "mem_gb" in snakemake.resources.keys() and not "-Xmx" in java_opts: - java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) - -# Getting java temp directory from output files list, if -Djava.io.tmpdir -# is not provided in java parameters. By doing so, backward compatibility is -# not broken. -if "java_temp" in snakemake.output.keys() and not "-Djava.io.tmpdir" in java_opts: - java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) +java_opts = get_java_opts(snakemake) filters = [ "--filter-name {} --filter-expression '{}'".format(name, expr.replace("'", "\\'")) diff --git a/bio/gatk/variantrecalibrator/environment.yaml b/bio/gatk/variantrecalibrator/environment.yaml index 17b64b9e46..a92753a102 100644 --- a/bio/gatk/variantrecalibrator/environment.yaml +++ b/bio/gatk/variantrecalibrator/environment.yaml @@ -4,3 +4,4 @@ channels: - defaults dependencies: - gatk4 ==4.1.4.1 + - snakemake-wrapper-utils ==0.1.2 diff --git a/bio/gatk/variantrecalibrator/wrapper.py b/bio/gatk/variantrecalibrator/wrapper.py index 3e1c8dec1f..cee0877786 100644 --- a/bio/gatk/variantrecalibrator/wrapper.py +++ b/bio/gatk/variantrecalibrator/wrapper.py @@ -7,25 +7,11 @@ import os from snakemake.shell import shell +from snakemake_wrapper_utils.java import get_java_opts extra = snakemake.params.get("extra", "") -java_opts = snakemake.params.get("java_opts", "") -# Getting memory in megabytes, if java opts is not filled with -Xmx parameter -# By doing so, backward compatibility is preserved -if "mem_mb" in snakemake.resources.keys() and not "-Xmx" in java_opts: - java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) - -# Getting memory in gigabytes, for user convenience. Please prefer the use -# of mem_mb over mem_gb as advised in documentation. -elif "mem_gb" in snakemake.resources.keys() and not "-Xmx" in java_opts: - java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) - -# Getting java temp directory from output files list, if -Djava.io.tmpdir -# is not provided in java parameters. By doing so, backward compatibility is -# not broken. -if "java_temp" in snakemake.output.keys() and not "-Djava.io.tmpdir" in java_opts: - java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) +java_opts = get_java_opts(snakemake) def fmt_res(resname, resparams): From a0a32d5a8d4867bc5184cf1b8e4f5d0f1cfceadf Mon Sep 17 00:00:00 2001 From: tdayris Date: Mon, 2 Nov 2020 12:10:03 +0100 Subject: [PATCH 46/61] [dev] (GATK3): Update with snakemake-wrappers-utils --- bio/gatk3/baserecalibrator/environment.yaml | 1 + bio/gatk3/baserecalibrator/wrapper.py | 3 ++- bio/gatk3/indelrealigner/environment.yaml | 1 + bio/gatk3/indelrealigner/wrapper.py | 3 ++- bio/gatk3/printreads/environment.yaml | 1 + bio/gatk3/printreads/wrapper.py | 3 ++- bio/gatk3/realignertargetcreator/environment.yaml | 1 + bio/gatk3/realignertargetcreator/wrapper.py | 3 ++- 8 files changed, 12 insertions(+), 4 deletions(-) diff --git a/bio/gatk3/baserecalibrator/environment.yaml b/bio/gatk3/baserecalibrator/environment.yaml index a143f0e89d..110a8be1e0 100644 --- a/bio/gatk3/baserecalibrator/environment.yaml +++ b/bio/gatk3/baserecalibrator/environment.yaml @@ -4,3 +4,4 @@ channels: - defaults dependencies: - gatk ==3.8 + - snakemake-wrapper-utils ==0.1.2 diff --git a/bio/gatk3/baserecalibrator/wrapper.py b/bio/gatk3/baserecalibrator/wrapper.py index e6bf19cc70..e70df99f3c 100644 --- a/bio/gatk3/baserecalibrator/wrapper.py +++ b/bio/gatk3/baserecalibrator/wrapper.py @@ -6,9 +6,10 @@ import os from snakemake.shell import shell +from snakemake_wrapper_utils.java import get_java_opts extra = snakemake.params.get("extra", "") -java_opts = snakemake.params.get("java_opts", "") +java_opts = get_java_opts(snakemake) input_bam = snakemake.input.bam input_known = snakemake.input.known diff --git a/bio/gatk3/indelrealigner/environment.yaml b/bio/gatk3/indelrealigner/environment.yaml index a143f0e89d..110a8be1e0 100644 --- a/bio/gatk3/indelrealigner/environment.yaml +++ b/bio/gatk3/indelrealigner/environment.yaml @@ -4,3 +4,4 @@ channels: - defaults dependencies: - gatk ==3.8 + - snakemake-wrapper-utils ==0.1.2 diff --git a/bio/gatk3/indelrealigner/wrapper.py b/bio/gatk3/indelrealigner/wrapper.py index f66bc8476d..90cbb13cd5 100644 --- a/bio/gatk3/indelrealigner/wrapper.py +++ b/bio/gatk3/indelrealigner/wrapper.py @@ -6,9 +6,10 @@ import os from snakemake.shell import shell +from snakemake_wrapper_utils.java import get_java_opts extra = snakemake.params.get("extra", "") -java_opts = snakemake.params.get("java_opts", "") +java_opts = get_java_opts(snakemake) input_bam = snakemake.input.bam input_known = snakemake.input.known diff --git a/bio/gatk3/printreads/environment.yaml b/bio/gatk3/printreads/environment.yaml index a143f0e89d..110a8be1e0 100644 --- a/bio/gatk3/printreads/environment.yaml +++ b/bio/gatk3/printreads/environment.yaml @@ -4,3 +4,4 @@ channels: - defaults dependencies: - gatk ==3.8 + - snakemake-wrapper-utils ==0.1.2 diff --git a/bio/gatk3/printreads/wrapper.py b/bio/gatk3/printreads/wrapper.py index d1a13daa7c..faa3619879 100644 --- a/bio/gatk3/printreads/wrapper.py +++ b/bio/gatk3/printreads/wrapper.py @@ -6,9 +6,10 @@ import os from snakemake.shell import shell +from snakemake_wrapper_utils.java import get_java_opts extra = snakemake.params.get("extra", "") -java_opts = snakemake.params.get("java_opts", "") +java_opts = get_java_opts(snakemake) input_bam = snakemake.input.bam input_recal_data = snakemake.input.recal_data diff --git a/bio/gatk3/realignertargetcreator/environment.yaml b/bio/gatk3/realignertargetcreator/environment.yaml index a143f0e89d..110a8be1e0 100644 --- a/bio/gatk3/realignertargetcreator/environment.yaml +++ b/bio/gatk3/realignertargetcreator/environment.yaml @@ -4,3 +4,4 @@ channels: - defaults dependencies: - gatk ==3.8 + - snakemake-wrapper-utils ==0.1.2 diff --git a/bio/gatk3/realignertargetcreator/wrapper.py b/bio/gatk3/realignertargetcreator/wrapper.py index 76489c1312..01860fc35d 100644 --- a/bio/gatk3/realignertargetcreator/wrapper.py +++ b/bio/gatk3/realignertargetcreator/wrapper.py @@ -6,9 +6,10 @@ import os from snakemake.shell import shell +from snakemake_wrapper_utils.java import get_java_opts extra = snakemake.params.get("extra", "") -java_opts = snakemake.params.get("java_opts", "") +java_opts = get_java_opts(snakemake) input_bam = snakemake.input.bam input_known = snakemake.input.known From 9503d7f3dc40d81003c66a28abcc18a93adcb74d Mon Sep 17 00:00:00 2001 From: tdayris Date: Mon, 2 Nov 2020 12:10:22 +0100 Subject: [PATCH 47/61] [dev] (Jannovar): Update with snakemake-wrappers-utils --- bio/jannovar/environment.yaml | 1 + bio/jannovar/test/Snakefile | 2 ++ bio/jannovar/wrapper.py | 19 +++---------------- 3 files changed, 6 insertions(+), 16 deletions(-) diff --git a/bio/jannovar/environment.yaml b/bio/jannovar/environment.yaml index c5bc591ea5..ab9f79f983 100644 --- a/bio/jannovar/environment.yaml +++ b/bio/jannovar/environment.yaml @@ -4,3 +4,4 @@ channels: - defaults dependencies: - jannovar-cli ==0.31 + - snakemake-wrapper-utils ==0.1.2 diff --git a/bio/jannovar/test/Snakefile b/bio/jannovar/test/Snakefile index a6ec2ebce1..e1d9ebb1ea 100644 --- a/bio/jannovar/test/Snakefile +++ b/bio/jannovar/test/Snakefile @@ -6,6 +6,8 @@ rule jannovar: "jannovar/{sample}.vcf.gz" log: "logs/jannovar/{sample}.log" + resources: + mem_gb = 1 params: database="hg19_small.ser", # path to jannovar reference dataset extra="--show-all" # optional parameters diff --git a/bio/jannovar/wrapper.py b/bio/jannovar/wrapper.py index 6d957d306d..6efd4d92f4 100644 --- a/bio/jannovar/wrapper.py +++ b/bio/jannovar/wrapper.py @@ -5,27 +5,14 @@ from snakemake.shell import shell +from snakemake_wrapper_utils.java import get_java_opts shell.executable("bash") log = snakemake.log_fmt_shell(stdout=False, stderr=True) extra = snakemake.params.get("extra", "") -# Getting memory in megabytes, if java opts is not filled with -Xmx parameter -# By doing so, backward compatibility is preserved -if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: - extra += " -Xmx{}M".format(snakemake.resources["mem_mb"]) - -# Getting memory in gigabytes, for user convenience. Please prefer the use -# of mem_mb over mem_gb as advised in documentation. -elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: - extra += " -Xmx{}G".format(snakemake.resources["mem_gb"]) - -# Getting java temp directory from output files list, if -Djava.io.tmpdir -# is not provided in java parameters. By doing so, backward compatibility is -# not broken. -if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: - extra += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) +java_opts = get_java_opts(snakemake) pedigree = snakemake.input.get("pedigree", "") if pedigree: @@ -34,5 +21,5 @@ shell( "jannovar annotate-vcf --database {snakemake.params.database}" " --input-vcf {snakemake.input.vcf} --output-vcf {snakemake.output}" - " {pedigree} {extra} {log}" + " {pedigree} {extra} {java_opts} {log}" ) From 8d7539bb114ef6d1b172c2405734552dbf62f372 Mon Sep 17 00:00:00 2001 From: tdayris Date: Mon, 2 Nov 2020 12:10:55 +0100 Subject: [PATCH 48/61] [dev] (Picard): Update with snakemake-wrappers-utils --- .../addorreplacereadgroups/environment.yaml | 1 + bio/picard/addorreplacereadgroups/wrapper.py | 19 ++------------- bio/picard/bedtointervallist/environment.yaml | 1 + bio/picard/bedtointervallist/wrapper.py | 19 ++------------- .../environment.yaml | 1 + .../collectalignmentsummarymetrics/wrapper.py | 19 ++------------- bio/picard/collecthsmetrics/environment.yaml | 1 + bio/picard/collecthsmetrics/wrapper.py | 20 ++-------------- .../collectinsertsizemetrics/environment.yaml | 1 + .../collectinsertsizemetrics/wrapper.py | 17 ++------------ .../collectmultiplemetrics/environment.yaml | 1 + bio/picard/collectmultiplemetrics/wrapper.py | 23 ++----------------- .../environment.yaml | 1 + .../collecttargetedpcrmetrics/wrapper.py | 19 ++------------- .../createsequencedictionary/environment.yaml | 1 + .../createsequencedictionary/wrapper.py | 19 ++------------- bio/picard/markduplicates/environment.yaml | 1 + bio/picard/markduplicates/wrapper.py | 19 ++------------- bio/picard/mergesamfiles/environment.yaml | 1 + bio/picard/mergesamfiles/wrapper.py | 19 ++------------- bio/picard/mergevcfs/environment.yaml | 1 + bio/picard/mergevcfs/wrapper.py | 19 ++------------- bio/picard/revertsam/environment.yaml | 1 + bio/picard/revertsam/wrapper.py | 18 ++------------- bio/picard/samtofastq/environment.yaml | 1 + bio/picard/samtofastq/wrapper.py | 18 ++------------- bio/picard/sortsam/environment.yaml | 1 + bio/picard/sortsam/wrapper.py | 18 ++------------- 28 files changed, 42 insertions(+), 238 deletions(-) diff --git a/bio/picard/addorreplacereadgroups/environment.yaml b/bio/picard/addorreplacereadgroups/environment.yaml index 21cd7a2eb2..0ba484ccb1 100644 --- a/bio/picard/addorreplacereadgroups/environment.yaml +++ b/bio/picard/addorreplacereadgroups/environment.yaml @@ -4,3 +4,4 @@ channels: - defaults dependencies: - picard ==2.22.1 + - snakemake-wrapper-utils ==0.1.2 diff --git a/bio/picard/addorreplacereadgroups/wrapper.py b/bio/picard/addorreplacereadgroups/wrapper.py index 5f58b44ad8..2ea40d2294 100644 --- a/bio/picard/addorreplacereadgroups/wrapper.py +++ b/bio/picard/addorreplacereadgroups/wrapper.py @@ -5,25 +5,10 @@ from snakemake.shell import shell +from snakemake_wrapper_utils.java import get_java_opts extra = snakemake.params -java_opts = "" -# Getting memory in megabytes, if java opts is not filled with -Xmx parameter -# By doing so, backward compatibility is preserved -if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) - -# Getting memory in gigabytes, for user convenience. Please prefer the use -# of mem_mb over mem_gb as advised in documentation. -elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) - -# Getting java temp directory from output files list, if -Djava.io.tmpdir -# is not provided in java parameters. By doing so, backward compatibility is -# not broken. -if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: - java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) - +java_opts = get_java_opts(snakemake) shell( "picard AddOrReplaceReadGroups {java_opts} {extra} " diff --git a/bio/picard/bedtointervallist/environment.yaml b/bio/picard/bedtointervallist/environment.yaml index 21cd7a2eb2..0ba484ccb1 100644 --- a/bio/picard/bedtointervallist/environment.yaml +++ b/bio/picard/bedtointervallist/environment.yaml @@ -4,3 +4,4 @@ channels: - defaults dependencies: - picard ==2.22.1 + - snakemake-wrapper-utils ==0.1.2 diff --git a/bio/picard/bedtointervallist/wrapper.py b/bio/picard/bedtointervallist/wrapper.py index 9e3f9dc9e3..57d9e55484 100644 --- a/bio/picard/bedtointervallist/wrapper.py +++ b/bio/picard/bedtointervallist/wrapper.py @@ -5,27 +5,12 @@ from snakemake.shell import shell +from snakemake_wrapper_utils.java import get_java_opts log = snakemake.log_fmt_shell() extra = snakemake.params -java_opts = "" -# Getting memory in megabytes, if java opts is not filled with -Xmx parameter -# By doing so, backward compatibility is preserved -if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) - -# Getting memory in gigabytes, for user convenience. Please prefer the use -# of mem_mb over mem_gb as advised in documentation. -elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) - -# Getting java temp directory from output files list, if -Djava.io.tmpdir -# is not provided in java parameters. By doing so, backward compatibility is -# not broken. -if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: - java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) - +java_opts = get_java_opts(snakemake) shell( "picard BedToIntervalList " diff --git a/bio/picard/collectalignmentsummarymetrics/environment.yaml b/bio/picard/collectalignmentsummarymetrics/environment.yaml index 21cd7a2eb2..0ba484ccb1 100644 --- a/bio/picard/collectalignmentsummarymetrics/environment.yaml +++ b/bio/picard/collectalignmentsummarymetrics/environment.yaml @@ -4,3 +4,4 @@ channels: - defaults dependencies: - picard ==2.22.1 + - snakemake-wrapper-utils ==0.1.2 diff --git a/bio/picard/collectalignmentsummarymetrics/wrapper.py b/bio/picard/collectalignmentsummarymetrics/wrapper.py index 3b89559e5d..abfa107c11 100644 --- a/bio/picard/collectalignmentsummarymetrics/wrapper.py +++ b/bio/picard/collectalignmentsummarymetrics/wrapper.py @@ -5,27 +5,12 @@ from snakemake.shell import shell +from snakemake_wrapper_utils.java import get_java_opts log = snakemake.log_fmt_shell() extra = snakemake.params -java_opts = "" -# Getting memory in megabytes, if java opts is not filled with -Xmx parameter -# By doing so, backward compatibility is preserved -if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) - -# Getting memory in gigabytes, for user convenience. Please prefer the use -# of mem_mb over mem_gb as advised in documentation. -elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) - -# Getting java temp directory from output files list, if -Djava.io.tmpdir -# is not provided in java parameters. By doing so, backward compatibility is -# not broken. -if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: - java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) - +java_opts = get_java_opts(snakemake) shell( "picard CollectAlignmentSummaryMetrics {java_opts} {extra} " diff --git a/bio/picard/collecthsmetrics/environment.yaml b/bio/picard/collecthsmetrics/environment.yaml index acc7bc8192..4ff61e3772 100644 --- a/bio/picard/collecthsmetrics/environment.yaml +++ b/bio/picard/collecthsmetrics/environment.yaml @@ -3,3 +3,4 @@ channels: - conda-forge dependencies: - picard ==2.22.1 + - snakemake-wrapper-utils ==0.1.2 diff --git a/bio/picard/collecthsmetrics/wrapper.py b/bio/picard/collecthsmetrics/wrapper.py index 0e94763ae4..53594ec2a4 100644 --- a/bio/picard/collecthsmetrics/wrapper.py +++ b/bio/picard/collecthsmetrics/wrapper.py @@ -7,29 +7,13 @@ from snakemake.shell import shell +from snakemake_wrapper_utils.java import get_java_opts inputs = " ".join("INPUT={}".format(in_) for in_ in snakemake.input) extra = snakemake.params.get("extra", "") log = snakemake.log_fmt_shell(stdout=False, stderr=True) - -java_opts = "" -# Getting memory in megabytes, if java opts is not filled with -Xmx parameter -# By doing so, backward compatibility is preserved -if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) - -# Getting memory in gigabytes, for user convenience. Please prefer the use -# of mem_mb over mem_gb as advised in documentation. -elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) - -# Getting java temp directory from output files list, if -Djava.io.tmpdir -# is not provided in java parameters. By doing so, backward compatibility is -# not broken. -if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: - java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) - +java_opts = get_java_opts(snakemake) shell( "picard CollectHsMetrics" diff --git a/bio/picard/collectinsertsizemetrics/environment.yaml b/bio/picard/collectinsertsizemetrics/environment.yaml index 967149ae58..f745497b4c 100644 --- a/bio/picard/collectinsertsizemetrics/environment.yaml +++ b/bio/picard/collectinsertsizemetrics/environment.yaml @@ -5,3 +5,4 @@ channels: dependencies: - picard ==2.22.1 - r-base ==3.6.2 + - snakemake-wrapper-utils ==0.1.2 diff --git a/bio/picard/collectinsertsizemetrics/wrapper.py b/bio/picard/collectinsertsizemetrics/wrapper.py index 7e2a293adf..93b88a0bd2 100644 --- a/bio/picard/collectinsertsizemetrics/wrapper.py +++ b/bio/picard/collectinsertsizemetrics/wrapper.py @@ -5,26 +5,13 @@ from snakemake.shell import shell +from snakemake_wrapper_utils.java import get_java_opts log = snakemake.log_fmt_shell() extra = snakemake.params -java_opts = "" -# Getting memory in megabytes, if java opts is not filled with -Xmx parameter -# By doing so, backward compatibility is preserved -if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) +java_opts = get_java_opts(snakemake) -# Getting memory in gigabytes, for user convenience. Please prefer the use -# of mem_mb over mem_gb as advised in documentation. -elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) - -# Getting java temp directory from output files list, if -Djava.io.tmpdir -# is not provided in java parameters. By doing so, backward compatibility is -# not broken. -if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: - java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) shell( diff --git a/bio/picard/collectmultiplemetrics/environment.yaml b/bio/picard/collectmultiplemetrics/environment.yaml index 82392faaa3..692b99ffe7 100644 --- a/bio/picard/collectmultiplemetrics/environment.yaml +++ b/bio/picard/collectmultiplemetrics/environment.yaml @@ -4,3 +4,4 @@ channels: - defaults dependencies: - picard ==2.23.0 + - snakemake-wrapper-utils ==0.1.2 diff --git a/bio/picard/collectmultiplemetrics/wrapper.py b/bio/picard/collectmultiplemetrics/wrapper.py index 7263b1c458..86a9dfb767 100644 --- a/bio/picard/collectmultiplemetrics/wrapper.py +++ b/bio/picard/collectmultiplemetrics/wrapper.py @@ -5,31 +5,12 @@ import sys from snakemake.shell import shell +from snakemake_wrapper_utils.java import get_java_opts log = snakemake.log_fmt_shell(stdout=False, stderr=True) extra = snakemake.params -java_opts = "" -# Getting memory in megabytes, if java opts is not filled with -Xmx parameter -# By doing so, backward compatibility is preserved -if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) - -# Getting memory in gigabytes, for user convenience. Please prefer the use -# of mem_mb over mem_gb as advised in documentation. -elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) - -# Previous wrapper set to 3 Gigabytes the default reserved memory. -# This behaviour is preseved below: -elif "-Xmx" not in extra: - java_opts += " -Xmx3G" - -# Getting java temp directory from output files list, if -Djava.io.tmpdir -# is not provided in java parameters. By doing so, backward compatibility is -# not broken. -if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: - java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) +java_opts = get_java_opts(snakemake) exts_to_prog = { ".alignment_summary_metrics": "CollectAlignmentSummaryMetrics", diff --git a/bio/picard/collecttargetedpcrmetrics/environment.yaml b/bio/picard/collecttargetedpcrmetrics/environment.yaml index 21cd7a2eb2..0ba484ccb1 100644 --- a/bio/picard/collecttargetedpcrmetrics/environment.yaml +++ b/bio/picard/collecttargetedpcrmetrics/environment.yaml @@ -4,3 +4,4 @@ channels: - defaults dependencies: - picard ==2.22.1 + - snakemake-wrapper-utils ==0.1.2 diff --git a/bio/picard/collecttargetedpcrmetrics/wrapper.py b/bio/picard/collecttargetedpcrmetrics/wrapper.py index cb0a1fc0f8..befef23fc4 100644 --- a/bio/picard/collecttargetedpcrmetrics/wrapper.py +++ b/bio/picard/collecttargetedpcrmetrics/wrapper.py @@ -5,28 +5,13 @@ from snakemake.shell import shell +from snakemake_wrapper_utils.java import get_java_opts log = snakemake.log_fmt_shell() extra = snakemake.params.get("extra", "") -java_opts = "" -# Getting memory in megabytes, if java opts is not filled with -Xmx parameter -# By doing so, backward compatibility is preserved -if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) - -# Getting memory in gigabytes, for user convenience. Please prefer the use -# of mem_mb over mem_gb as advised in documentation. -elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) - -# Getting java temp directory from output files list, if -Djava.io.tmpdir -# is not provided in java parameters. By doing so, backward compatibility is -# not broken. -if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: - java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) - +java_opts = get_java_opts(snakemake) shell( "picard CollectTargetedPcrMetrics " diff --git a/bio/picard/createsequencedictionary/environment.yaml b/bio/picard/createsequencedictionary/environment.yaml index 21cd7a2eb2..0ba484ccb1 100644 --- a/bio/picard/createsequencedictionary/environment.yaml +++ b/bio/picard/createsequencedictionary/environment.yaml @@ -4,3 +4,4 @@ channels: - defaults dependencies: - picard ==2.22.1 + - snakemake-wrapper-utils ==0.1.2 diff --git a/bio/picard/createsequencedictionary/wrapper.py b/bio/picard/createsequencedictionary/wrapper.py index 0facbc9a0b..e368347c34 100644 --- a/bio/picard/createsequencedictionary/wrapper.py +++ b/bio/picard/createsequencedictionary/wrapper.py @@ -5,26 +5,11 @@ from snakemake.shell import shell +from snakemake_wrapper_utils.java import get_java_opts extra = snakemake.params.get("extra", "") -java_opts = "" -# Getting memory in megabytes, if java opts is not filled with -Xmx parameter -# By doing so, backward compatibility is preserved -if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) - -# Getting memory in gigabytes, for user convenience. Please prefer the use -# of mem_mb over mem_gb as advised in documentation. -elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) - -# Getting java temp directory from output files list, if -Djava.io.tmpdir -# is not provided in java parameters. By doing so, backward compatibility is -# not broken. -if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: - java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) - +java_opts = get_java_opts(snakemake) log = snakemake.log_fmt_shell(stdout=False, stderr=True) shell( diff --git a/bio/picard/markduplicates/environment.yaml b/bio/picard/markduplicates/environment.yaml index 21cd7a2eb2..0ba484ccb1 100644 --- a/bio/picard/markduplicates/environment.yaml +++ b/bio/picard/markduplicates/environment.yaml @@ -4,3 +4,4 @@ channels: - defaults dependencies: - picard ==2.22.1 + - snakemake-wrapper-utils ==0.1.2 diff --git a/bio/picard/markduplicates/wrapper.py b/bio/picard/markduplicates/wrapper.py index 78e2243573..ac9c3be5e2 100644 --- a/bio/picard/markduplicates/wrapper.py +++ b/bio/picard/markduplicates/wrapper.py @@ -5,27 +5,12 @@ from snakemake.shell import shell +from snakemake_wrapper_utils.java import get_java_opts log = snakemake.log_fmt_shell(stdout=True, stderr=True) extra = snakemake.params -java_opts = "" -# Getting memory in megabytes, if java opts is not filled with -Xmx parameter -# By doing so, backward compatibility is preserved -if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) - -# Getting memory in gigabytes, for user convenience. Please prefer the use -# of mem_mb over mem_gb as advised in documentation. -elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) - -# Getting java temp directory from output files list, if -Djava.io.tmpdir -# is not provided in java parameters. By doing so, backward compatibility is -# not broken. -if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: - java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) - +java_opts = get_java_opts(snakemake) shell( "picard MarkDuplicates " # Tool and its subcommand diff --git a/bio/picard/mergesamfiles/environment.yaml b/bio/picard/mergesamfiles/environment.yaml index 21cd7a2eb2..0ba484ccb1 100644 --- a/bio/picard/mergesamfiles/environment.yaml +++ b/bio/picard/mergesamfiles/environment.yaml @@ -4,3 +4,4 @@ channels: - defaults dependencies: - picard ==2.22.1 + - snakemake-wrapper-utils ==0.1.2 diff --git a/bio/picard/mergesamfiles/wrapper.py b/bio/picard/mergesamfiles/wrapper.py index dc09d6fe43..e3072a869c 100644 --- a/bio/picard/mergesamfiles/wrapper.py +++ b/bio/picard/mergesamfiles/wrapper.py @@ -7,25 +7,10 @@ from snakemake.shell import shell +from snakemake_wrapper_utils.java import get_java_opts extra = snakemake.params -java_opts = "" -# Getting memory in megabytes, if java opts is not filled with -Xmx parameter -# By doing so, backward compatibility is preserved -if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) - -# Getting memory in gigabytes, for user convenience. Please prefer the use -# of mem_mb over mem_gb as advised in documentation. -elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) - -# Getting java temp directory from output files list, if -Djava.io.tmpdir -# is not provided in java parameters. By doing so, backward compatibility is -# not broken. -if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: - java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) - +java_opts = get_java_opts(snakemake) inputs = " ".join("INPUT={}".format(in_) for in_ in snakemake.input) log = snakemake.log_fmt_shell(stdout=False, stderr=True) diff --git a/bio/picard/mergevcfs/environment.yaml b/bio/picard/mergevcfs/environment.yaml index 21cd7a2eb2..0ba484ccb1 100644 --- a/bio/picard/mergevcfs/environment.yaml +++ b/bio/picard/mergevcfs/environment.yaml @@ -4,3 +4,4 @@ channels: - defaults dependencies: - picard ==2.22.1 + - snakemake-wrapper-utils ==0.1.2 diff --git a/bio/picard/mergevcfs/wrapper.py b/bio/picard/mergevcfs/wrapper.py index 7b0eb47c96..f47f243100 100644 --- a/bio/picard/mergevcfs/wrapper.py +++ b/bio/picard/mergevcfs/wrapper.py @@ -7,28 +7,13 @@ from snakemake.shell import shell +from snakemake_wrapper_utils.java import get_java_opts inputs = " ".join("INPUT={}".format(f) for f in snakemake.input.vcfs) log = snakemake.log_fmt_shell(stdout=False, stderr=True) extra = snakemake.params.get("extra", "") -java_opts = "" -# Getting memory in megabytes, if java opts is not filled with -Xmx parameter -# By doing so, backward compatibility is preserved -if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) - -# Getting memory in gigabytes, for user convenience. Please prefer the use -# of mem_mb over mem_gb as advised in documentation. -elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) - -# Getting java temp directory from output files list, if -Djava.io.tmpdir -# is not provided in java parameters. By doing so, backward compatibility is -# not broken. -if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: - java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) - +java_opts = get_java_opts(snakemake) shell( "picard" diff --git a/bio/picard/revertsam/environment.yaml b/bio/picard/revertsam/environment.yaml index 21cd7a2eb2..0ba484ccb1 100644 --- a/bio/picard/revertsam/environment.yaml +++ b/bio/picard/revertsam/environment.yaml @@ -4,3 +4,4 @@ channels: - defaults dependencies: - picard ==2.22.1 + - snakemake-wrapper-utils ==0.1.2 diff --git a/bio/picard/revertsam/wrapper.py b/bio/picard/revertsam/wrapper.py index b9b60c31b4..e161ead372 100644 --- a/bio/picard/revertsam/wrapper.py +++ b/bio/picard/revertsam/wrapper.py @@ -7,24 +7,10 @@ from snakemake.shell import shell +from snakemake_wrapper_utils.java import get_java_opts extra = snakemake.params.get("extra", "") -java_opts = "" -# Getting memory in megabytes, if java opts is not filled with -Xmx parameter -# By doing so, backward compatibility is preserved -if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) - -# Getting memory in gigabytes, for user convenience. Please prefer the use -# of mem_mb over mem_gb as advised in documentation. -elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) - -# Getting java temp directory from output files list, if -Djava.io.tmpdir -# is not provided in java parameters. By doing so, backward compatibility is -# not broken. -if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: - java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) +java_opts = get_java_opts(snakemake) log = snakemake.log_fmt_shell(stdout=False, stderr=True) diff --git a/bio/picard/samtofastq/environment.yaml b/bio/picard/samtofastq/environment.yaml index 21cd7a2eb2..0ba484ccb1 100644 --- a/bio/picard/samtofastq/environment.yaml +++ b/bio/picard/samtofastq/environment.yaml @@ -4,3 +4,4 @@ channels: - defaults dependencies: - picard ==2.22.1 + - snakemake-wrapper-utils ==0.1.2 diff --git a/bio/picard/samtofastq/wrapper.py b/bio/picard/samtofastq/wrapper.py index 259333f234..77ddd4884b 100644 --- a/bio/picard/samtofastq/wrapper.py +++ b/bio/picard/samtofastq/wrapper.py @@ -7,25 +7,11 @@ from snakemake.shell import shell +from snakemake_wrapper_utils.java import get_java_opts extra = snakemake.params.get("extra", "") -java_opts = "" -# Getting memory in megabytes, if java opts is not filled with -Xmx parameter -# By doing so, backward compatibility is preserved -if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) - -# Getting memory in gigabytes, for user convenience. Please prefer the use -# of mem_mb over mem_gb as advised in documentation. -elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) - -# Getting java temp directory from output files list, if -Djava.io.tmpdir -# is not provided in java parameters. By doing so, backward compatibility is -# not broken. -if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: - java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) +java_opts = get_java_opts(snakemake) log = snakemake.log_fmt_shell(stdout=False, stderr=True) diff --git a/bio/picard/sortsam/environment.yaml b/bio/picard/sortsam/environment.yaml index 21cd7a2eb2..0ba484ccb1 100644 --- a/bio/picard/sortsam/environment.yaml +++ b/bio/picard/sortsam/environment.yaml @@ -4,3 +4,4 @@ channels: - defaults dependencies: - picard ==2.22.1 + - snakemake-wrapper-utils ==0.1.2 diff --git a/bio/picard/sortsam/wrapper.py b/bio/picard/sortsam/wrapper.py index 96e98c3c92..029f2143ad 100644 --- a/bio/picard/sortsam/wrapper.py +++ b/bio/picard/sortsam/wrapper.py @@ -7,25 +7,11 @@ from snakemake.shell import shell +from snakemake_wrapper_utils.java import get_java_opts extra = snakemake.params.get("extra", "") -java_opts = "" -# Getting memory in megabytes, if java opts is not filled with -Xmx parameter -# By doing so, backward compatibility is preserved -if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) - -# Getting memory in gigabytes, for user convenience. Please prefer the use -# of mem_mb over mem_gb as advised in documentation. -elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) - -# Getting java temp directory from output files list, if -Djava.io.tmpdir -# is not provided in java parameters. By doing so, backward compatibility is -# not broken. -if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: - java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) +java_opts = get_java_opts(snakemake) log = snakemake.log_fmt_shell(stdout=False, stderr=True) From 4b85044125e11174385134a4bcb206efedc137aa Mon Sep 17 00:00:00 2001 From: tdayris Date: Mon, 2 Nov 2020 12:11:08 +0100 Subject: [PATCH 49/61] [dev] (SnpEff): Update with snakemake-wrappers-utils --- bio/snpeff/annotate/environment.yaml | 1 + bio/snpeff/annotate/wrapper.py | 24 +++++------------------- bio/snpeff/download/environment.yaml | 1 + bio/snpeff/download/wrapper.py | 21 +++------------------ 4 files changed, 10 insertions(+), 37 deletions(-) diff --git a/bio/snpeff/annotate/environment.yaml b/bio/snpeff/annotate/environment.yaml index 5518f1df01..cd33b621bb 100644 --- a/bio/snpeff/annotate/environment.yaml +++ b/bio/snpeff/annotate/environment.yaml @@ -4,3 +4,4 @@ channels: dependencies: - snpeff ==4.3.1t - bcftools =1.10 + - snakemake-wrapper-utils ==0.1.2 diff --git a/bio/snpeff/annotate/wrapper.py b/bio/snpeff/annotate/wrapper.py index 00daf8afab..5e89fdbc85 100644 --- a/bio/snpeff/annotate/wrapper.py +++ b/bio/snpeff/annotate/wrapper.py @@ -9,6 +9,11 @@ import shutil import tempfile from pathlib import Path +from snakemake_wrapper_utils.java import get_java_opts + + +extra = snakemake.params.get("extra", "") +java_opts = get_java_opts(snakemake) outcalls = snakemake.output.calls if outcalls.endswith(".vcf.gz"): @@ -24,25 +29,6 @@ log = snakemake.log_fmt_shell(stdout=False, stderr=True) -extra = snakemake.params.get("extra", "") -java_opts = "" -# Getting memory in megabytes, if java opts is not filled with -Xmx parameter -# By doing so, backward compatibility is preserved -if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) - -# Getting memory in gigabytes, for user convenience. Please prefer the use -# of mem_mb over mem_gb as advised in documentation. -elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) - -# Getting java temp directory from output files list, if -Djava.io.tmpdir -# is not provided in java parameters. By doing so, backward compatibility is -# not broken. -if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: - java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) - - data_dir = Path(snakemake.input.db).parent.resolve() stats = snakemake.output.get("stats", "") diff --git a/bio/snpeff/download/environment.yaml b/bio/snpeff/download/environment.yaml index 5518f1df01..cd33b621bb 100644 --- a/bio/snpeff/download/environment.yaml +++ b/bio/snpeff/download/environment.yaml @@ -4,3 +4,4 @@ channels: dependencies: - snpeff ==4.3.1t - bcftools =1.10 + - snakemake-wrapper-utils ==0.1.2 diff --git a/bio/snpeff/download/wrapper.py b/bio/snpeff/download/wrapper.py index 49141c1c93..61e30688ec 100644 --- a/bio/snpeff/download/wrapper.py +++ b/bio/snpeff/download/wrapper.py @@ -5,27 +5,12 @@ from snakemake.shell import shell from pathlib import Path +from snakemake_wrapper_utils.java import get_java_opts + +java_opts = get_java_opts(snakemake) reference = snakemake.params.reference outdir = Path(snakemake.output[0]).parent.resolve() log = snakemake.log_fmt_shell(stdout=False, stderr=True) -java_opts = "" -# Getting memory in megabytes, if java opts is not filled with -Xmx parameter -# By doing so, backward compatibility is preserved -if "mem_mb" in snakemake.resources.keys(): - java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) - -# Getting memory in gigabytes, for user convenience. Please prefer the use -# of mem_mb over mem_gb as advised in documentation. -elif "mem_gb" in snakemake.resources.keys(): - java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) - -# Getting java temp directory from output files list, if -Djava.io.tmpdir -# is not provided in java parameters. By doing so, backward compatibility is -# not broken. -if "java_temp" in snakemake.output.keys(): - java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) - - shell("snpEff download {java_opts} -dataDir {outdir} {reference} {log}") From 37326152334412a1bbd9b78eea66a73441fcf6a8 Mon Sep 17 00:00:00 2001 From: tdayris Date: Mon, 2 Nov 2020 12:11:26 +0100 Subject: [PATCH 50/61] [dev] (SnpSift): Update with snakemake-wrappers-utils --- bio/snpsift/annotate/environment.yaml | 1 + bio/snpsift/annotate/wrapper.py | 21 +++------------------ bio/snpsift/dbnsfp/environment.yaml | 1 + bio/snpsift/dbnsfp/wrapper.py | 20 ++------------------ bio/snpsift/genesets/environment.yaml | 1 + bio/snpsift/genesets/wrapper.py | 20 +++----------------- bio/snpsift/gwascat/environment.yaml | 1 + bio/snpsift/gwascat/wrapper.py | 21 +++------------------ bio/snpsift/varType/environment.yaml | 1 + bio/snpsift/varType/wrapper.py | 20 ++------------------ 10 files changed, 18 insertions(+), 89 deletions(-) diff --git a/bio/snpsift/annotate/environment.yaml b/bio/snpsift/annotate/environment.yaml index 26a757a04d..8430bcc6f9 100644 --- a/bio/snpsift/annotate/environment.yaml +++ b/bio/snpsift/annotate/environment.yaml @@ -5,3 +5,4 @@ dependencies: - snpsift ==4.3.1t - bcftools ==1.10.2 - pbgzip ==2016.08.04 + - snakemake-wrapper-utils ==0.1.2 diff --git a/bio/snpsift/annotate/wrapper.py b/bio/snpsift/annotate/wrapper.py index a8fe4d1c39..43cebb3a54 100644 --- a/bio/snpsift/annotate/wrapper.py +++ b/bio/snpsift/annotate/wrapper.py @@ -6,27 +6,12 @@ __license__ = "MIT" from snakemake.shell import shell +from snakemake_wrapper_utils.java import get_java_opts + +java_opts = get_java_opts(snakemake) log = snakemake.log_fmt_shell(stdout=False, stderr=True) extra = snakemake.params.get("extra", "") -java_opts = "" -# Getting memory in megabytes, if java opts is not filled with -Xmx parameter -# By doing so, backward compatibility is preserved -if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) - -# Getting memory in gigabytes, for user convenience. Please prefer the use -# of mem_mb over mem_gb as advised in documentation. -elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) - -# Getting java temp directory from output files list, if -Djava.io.tmpdir -# is not provided in java parameters. By doing so, backward compatibility is -# not broken. -if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: - java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) - - min_threads = 1 incall = snakemake.input["call"] diff --git a/bio/snpsift/dbnsfp/environment.yaml b/bio/snpsift/dbnsfp/environment.yaml index 1d8a54b362..b47e8db16d 100644 --- a/bio/snpsift/dbnsfp/environment.yaml +++ b/bio/snpsift/dbnsfp/environment.yaml @@ -4,3 +4,4 @@ channels: dependencies: - snpsift=4.3.1t - bcftools ==1.10.2 + - snakemake-wrapper-utils ==0.1.2 diff --git a/bio/snpsift/dbnsfp/wrapper.py b/bio/snpsift/dbnsfp/wrapper.py index db36e52103..f24d60eb93 100644 --- a/bio/snpsift/dbnsfp/wrapper.py +++ b/bio/snpsift/dbnsfp/wrapper.py @@ -6,26 +6,10 @@ __license__ = "MIT" from snakemake.shell import shell +from snakemake_wrapper_utils.java import get_java_opts +java_opts = get_java_opts(snakemake) log = snakemake.log_fmt_shell(stdout=False, stderr=True) -extra = snakemake.params.get("extra", "") -java_opts = "" -# Getting memory in megabytes, if java opts is not filled with -Xmx parameter -# By doing so, backward compatibility is preserved -if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) - -# Getting memory in gigabytes, for user convenience. Please prefer the use -# of mem_mb over mem_gb as advised in documentation. -elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) - -# Getting java temp directory from output files list, if -Djava.io.tmpdir -# is not provided in java parameters. By doing so, backward compatibility is -# not broken. -if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: - java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) - # Using user-defined file if requested db = snakemake.input.get("dbNSFP", "") diff --git a/bio/snpsift/genesets/environment.yaml b/bio/snpsift/genesets/environment.yaml index 92638c9cb7..192c3d1f79 100644 --- a/bio/snpsift/genesets/environment.yaml +++ b/bio/snpsift/genesets/environment.yaml @@ -5,3 +5,4 @@ channels: dependencies: - snpsift ==4.3.1t - bcftools ==1.10.2 + - snakemake-wrapper-utils ==0.1.2 diff --git a/bio/snpsift/genesets/wrapper.py b/bio/snpsift/genesets/wrapper.py index 7cef8a7a0d..01f060aa20 100644 --- a/bio/snpsift/genesets/wrapper.py +++ b/bio/snpsift/genesets/wrapper.py @@ -6,26 +6,12 @@ __license__ = "MIT" from snakemake.shell import shell +from snakemake_wrapper_utils.java import get_java_opts + +java_opts = get_java_opts(snakemake) log = snakemake.log_fmt_shell(stdout=False, stderr=True) extra = snakemake.params.get("extra", "") -java_opts = "" -# Getting memory in megabytes, if java opts is not filled with -Xmx parameter -# By doing so, backward compatibility is preserved -if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) - -# Getting memory in gigabytes, for user convenience. Please prefer the use -# of mem_mb over mem_gb as advised in documentation. -elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) - -# Getting java temp directory from output files list, if -Djava.io.tmpdir -# is not provided in java parameters. By doing so, backward compatibility is -# not broken. -if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: - java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) - min_threads = 1 # Uncompression shall be done according to user-defined input diff --git a/bio/snpsift/gwascat/environment.yaml b/bio/snpsift/gwascat/environment.yaml index 43efdcf0ca..b6079c237c 100644 --- a/bio/snpsift/gwascat/environment.yaml +++ b/bio/snpsift/gwascat/environment.yaml @@ -4,3 +4,4 @@ channels: dependencies: - snpsift ==4.3.1t - bcftools ==1.10.2 + - snakemake-wrapper-utils ==0.1.2 diff --git a/bio/snpsift/gwascat/wrapper.py b/bio/snpsift/gwascat/wrapper.py index 5871419c4c..65005b0501 100644 --- a/bio/snpsift/gwascat/wrapper.py +++ b/bio/snpsift/gwascat/wrapper.py @@ -6,27 +6,12 @@ __license__ = "MIT" from snakemake.shell import shell +from snakemake_wrapper_utils.java import get_java_opts + +java_opts = get_java_opts(snakemake) log = snakemake.log_fmt_shell(stdout=False, stderr=True) extra = snakemake.params.get("extra", "") -java_opts = "" -# Getting memory in megabytes, if java opts is not filled with -Xmx parameter -# By doing so, backward compatibility is preserved -if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) - -# Getting memory in gigabytes, for user convenience. Please prefer the use -# of mem_mb over mem_gb as advised in documentation. -elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) - -# Getting java temp directory from output files list, if -Djava.io.tmpdir -# is not provided in java parameters. By doing so, backward compatibility is -# not broken. -if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: - java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) - - min_threads = 1 # Uncompression shall be done based on user input diff --git a/bio/snpsift/varType/environment.yaml b/bio/snpsift/varType/environment.yaml index ec177f0e81..cc7992c051 100644 --- a/bio/snpsift/varType/environment.yaml +++ b/bio/snpsift/varType/environment.yaml @@ -3,3 +3,4 @@ channels: - conda-forge dependencies: - snpsift =4.3.1t + - snakemake-wrapper-utils ==0.1.2 diff --git a/bio/snpsift/varType/wrapper.py b/bio/snpsift/varType/wrapper.py index fff970a98d..f4445176ec 100644 --- a/bio/snpsift/varType/wrapper.py +++ b/bio/snpsift/varType/wrapper.py @@ -6,27 +6,11 @@ __license__ = "MIT" from snakemake.shell import shell +from snakemake_wrapper_utils.java import get_java_opts +java_opts = get_java_opts(snakemake) log = snakemake.log_fmt_shell(stdout=False, stderr=True) - extra = snakemake.params.get("extra", "") -java_opts = "" -# Getting memory in megabytes, if java opts is not filled with -Xmx parameter -# By doing so, backward compatibility is preserved -if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) - -# Getting memory in gigabytes, for user convenience. Please prefer the use -# of mem_mb over mem_gb as advised in documentation. -elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) - -# Getting java temp directory from output files list, if -Djava.io.tmpdir -# is not provided in java parameters. By doing so, backward compatibility is -# not broken. -if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: - java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) - shell( "SnpSift varType" # Tool and its subcommand From 71db432bb2a423b8396d342958e20340ec1a5abe Mon Sep 17 00:00:00 2001 From: tdayris Date: Mon, 2 Nov 2020 12:11:36 +0100 Subject: [PATCH 51/61] [dev] (Trimmomatic): Update with snakemake-wrappers-utils --- bio/trimmomatic/pe/environment.yaml | 1 + bio/trimmomatic/pe/wrapper.py | 20 ++------------------ bio/trimmomatic/se/environment.yaml | 1 + bio/trimmomatic/se/wrapper.py | 20 ++------------------ 4 files changed, 6 insertions(+), 36 deletions(-) diff --git a/bio/trimmomatic/pe/environment.yaml b/bio/trimmomatic/pe/environment.yaml index e405fa6868..ee3a7ac0be 100644 --- a/bio/trimmomatic/pe/environment.yaml +++ b/bio/trimmomatic/pe/environment.yaml @@ -5,3 +5,4 @@ channels: dependencies: - trimmomatic ==0.36 - pigz ==2.3.4 + - snakemake-wrapper-utils ==0.1.2 diff --git a/bio/trimmomatic/pe/wrapper.py b/bio/trimmomatic/pe/wrapper.py index eca21c25dd..6e339d6ddf 100644 --- a/bio/trimmomatic/pe/wrapper.py +++ b/bio/trimmomatic/pe/wrapper.py @@ -16,7 +16,7 @@ from snakemake.shell import shell - +from snakemake_wrapper_utils.java import get_java_opts # Distribute available threads between trimmomatic itself and any potential pigz instances def distribute_threads(input_files, output_files, available_threads): @@ -66,23 +66,7 @@ def compose_output_gz(filename, threads, compression_level): extra = snakemake.params.get("extra", "") -java_opts = "" -# Getting memory in megabytes, if java opts is not filled with -Xmx parameter -# By doing so, backward compatibility is preserved -if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) - -# Getting memory in gigabytes, for user convenience. Please prefer the use -# of mem_mb over mem_gb as advised in documentation. -elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) - -# Getting java temp directory from output files list, if -Djava.io.tmpdir -# is not provided in java parameters. By doing so, backward compatibility is -# not broken. -if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: - java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) - +java_opts = get_java_opts(snakemake) log = snakemake.log_fmt_shell(stdout=True, stderr=True) compression_level = snakemake.params.get("compression_level", "-5") trimmer = " ".join(snakemake.params.trimmer) diff --git a/bio/trimmomatic/se/environment.yaml b/bio/trimmomatic/se/environment.yaml index e405fa6868..ee3a7ac0be 100644 --- a/bio/trimmomatic/se/environment.yaml +++ b/bio/trimmomatic/se/environment.yaml @@ -5,3 +5,4 @@ channels: dependencies: - trimmomatic ==0.36 - pigz ==2.3.4 + - snakemake-wrapper-utils ==0.1.2 diff --git a/bio/trimmomatic/se/wrapper.py b/bio/trimmomatic/se/wrapper.py index 1823aa585e..0ab2c21c85 100644 --- a/bio/trimmomatic/se/wrapper.py +++ b/bio/trimmomatic/se/wrapper.py @@ -16,7 +16,7 @@ from snakemake.shell import shell - +from snakemake_wrapper_utils.java import get_java_opts # Distribute available threads between trimmomatic itself and any potential pigz instances def distribute_threads(input_file, output_file, available_threads): @@ -66,23 +66,7 @@ def compose_output_gz(filename, threads, compression_level): extra = snakemake.params.get("extra", "") -java_opts = "" -# Getting memory in megabytes, if java opts is not filled with -Xmx parameter -# By doing so, backward compatibility is preserved -if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) - -# Getting memory in gigabytes, for user convenience. Please prefer the use -# of mem_mb over mem_gb as advised in documentation. -elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) - -# Getting java temp directory from output files list, if -Djava.io.tmpdir -# is not provided in java parameters. By doing so, backward compatibility is -# not broken. -if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: - java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) - +java_opts = get_java_opts(snakemake) log = snakemake.log_fmt_shell(stdout=True, stderr=True) compression_level = snakemake.params.get("compression_level", "-5") trimmer = " ".join(snakemake.params.trimmer) From e11bc710e6ae222eea76872f4ec9326c42140ec3 Mon Sep 17 00:00:00 2001 From: tdayris Date: Mon, 2 Nov 2020 12:11:51 +0100 Subject: [PATCH 52/61] [dev] (Varscan): Update with snakemake-wrappers-utils --- bio/varscan/mpileup2indel/environment.yaml | 1 + bio/varscan/mpileup2indel/wrapper.py | 18 ++---------------- bio/varscan/mpileup2snp/environment.yaml | 1 + bio/varscan/mpileup2snp/wrapper.py | 18 ++---------------- bio/varscan/somatic/environment.yaml | 1 + bio/varscan/somatic/wrapper.py | 18 ++---------------- 6 files changed, 9 insertions(+), 48 deletions(-) diff --git a/bio/varscan/mpileup2indel/environment.yaml b/bio/varscan/mpileup2indel/environment.yaml index d758fb6f95..0defe1b9ab 100644 --- a/bio/varscan/mpileup2indel/environment.yaml +++ b/bio/varscan/mpileup2indel/environment.yaml @@ -5,3 +5,4 @@ channels: - defaults dependencies: - varscan ==2.4.3 + - snakemake-wrapper-utils ==0.1.2 diff --git a/bio/varscan/mpileup2indel/wrapper.py b/bio/varscan/mpileup2indel/wrapper.py index c586ccdb50..825baaf70f 100644 --- a/bio/varscan/mpileup2indel/wrapper.py +++ b/bio/varscan/mpileup2indel/wrapper.py @@ -8,26 +8,12 @@ import os.path as op from snakemake.shell import shell from snakemake.utils import makedirs +from snakemake_wrapper_utils.java import get_java_opts # Gathering extra parameters and logging behaviour log = snakemake.log_fmt_shell(stdout=False, stderr=True) extra = snakemake.params.get("extra", "") -java_opts = "" -# Getting memory in megabytes, if java opts is not filled with -Xmx parameter -# By doing so, backward compatibility is preserved -if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) - -# Getting memory in gigabytes, for user convenience. Please prefer the use -# of mem_mb over mem_gb as advised in documentation. -elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) - -# Getting java temp directory from output files list, if -Djava.io.tmpdir -# is not provided in java parameters. By doing so, backward compatibility is -# not broken. -if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: - java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) +java_opts = get_java_opts(snakemake) # In case input files are gzipped mpileup files, # they are being unzipped and piped diff --git a/bio/varscan/mpileup2snp/environment.yaml b/bio/varscan/mpileup2snp/environment.yaml index c18ea01901..4d3a103529 100644 --- a/bio/varscan/mpileup2snp/environment.yaml +++ b/bio/varscan/mpileup2snp/environment.yaml @@ -5,3 +5,4 @@ channels: - defaults dependencies: - varscan ==2.4.3 + - snakemake-wrapper-utils ==0.1.2 diff --git a/bio/varscan/mpileup2snp/wrapper.py b/bio/varscan/mpileup2snp/wrapper.py index 0a3558e85e..d6d74f71bc 100644 --- a/bio/varscan/mpileup2snp/wrapper.py +++ b/bio/varscan/mpileup2snp/wrapper.py @@ -8,26 +8,12 @@ import os.path as op from snakemake.shell import shell from snakemake.utils import makedirs +from snakemake_wrapper_utils.java import get_java_opts # Gathering extra parameters and logging behaviour log = snakemake.log_fmt_shell(stdout=False, stderr=True) extra = snakemake.params.get("extra", "") -java_opts = "" -# Getting memory in megabytes, if java opts is not filled with -Xmx parameter -# By doing so, backward compatibility is preserved -if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) - -# Getting memory in gigabytes, for user convenience. Please prefer the use -# of mem_mb over mem_gb as advised in documentation. -elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) - -# Getting java temp directory from output files list, if -Djava.io.tmpdir -# is not provided in java parameters. By doing so, backward compatibility is -# not broken. -if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: - java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) +java_opts = get_java_opts(snakemake) # In case input files are gzipped mpileup files, # they are being unzipped and piped diff --git a/bio/varscan/somatic/environment.yaml b/bio/varscan/somatic/environment.yaml index 337ebec9ab..89df172f90 100644 --- a/bio/varscan/somatic/environment.yaml +++ b/bio/varscan/somatic/environment.yaml @@ -5,3 +5,4 @@ channels: - defaults dependencies: - varscan ==2.4.3 + - snakemake-wrapper-utils ==0.1.2 diff --git a/bio/varscan/somatic/wrapper.py b/bio/varscan/somatic/wrapper.py index ae4f2d0090..5b6227b00f 100644 --- a/bio/varscan/somatic/wrapper.py +++ b/bio/varscan/somatic/wrapper.py @@ -10,26 +10,12 @@ from snakemake.shell import shell from snakemake.utils import makedirs +from snakemake_wrapper_utils.java import get_java_opts # Defining logging and gathering extra parameters log = snakemake.log_fmt_shell(stdout=True, stderr=True) extra = snakemake.params.get("extra", "") -java_opts = "" -# Getting memory in megabytes, if java opts is not filled with -Xmx parameter -# By doing so, backward compatibility is preserved -if "mem_mb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}M".format(snakemake.resources["mem_mb"]) - -# Getting memory in gigabytes, for user convenience. Please prefer the use -# of mem_mb over mem_gb as advised in documentation. -elif "mem_gb" in snakemake.resources.keys() and "-Xmx" not in extra: - java_opts += " -Xmx{}G".format(snakemake.resources["mem_gb"]) - -# Getting java temp directory from output files list, if -Djava.io.tmpdir -# is not provided in java parameters. By doing so, backward compatibility is -# not broken. -if "java_temp" in snakemake.output.keys() and "-Djava.io.tmpdir" not in extra: - java_opts += " -Djava.io.tmpdir={}".format(snakemake.output["java_temp"]) +java_opts = get_java_opts(snakemake) # Building output dirs makedirs(op.dirname(snakemake.output.snp)) From 3afc38a47b723e076530a3df7a674026ace8034a Mon Sep 17 00:00:00 2001 From: tdayris Date: Thu, 12 Nov 2020 14:12:42 +0100 Subject: [PATCH 53/61] [fix] (NameError): snakemake-wrapper-utils updated --- bio/gatk/applybqsr/environment.yaml | 2 +- bio/gatk/baserecalibrator/environment.yaml | 2 +- bio/gatk/baserecalibratorspark/environment.yaml | 2 +- bio/gatk/combinegvcfs/environment.yaml | 2 +- bio/gatk/genotypegvcfs/environment.yaml | 2 +- bio/gatk/haplotypecaller/environment.yaml | 2 +- bio/gatk/mutect/environment.yaml | 2 +- bio/gatk/selectvariants/environment.yaml | 2 +- bio/gatk/splitncigarreads/environment.yaml | 2 +- bio/gatk/variantfiltration/environment.yaml | 2 +- bio/gatk/variantrecalibrator/environment.yaml | 2 +- bio/gatk3/baserecalibrator/environment.yaml | 2 +- bio/gatk3/indelrealigner/environment.yaml | 2 +- bio/gatk3/printreads/environment.yaml | 2 +- bio/gatk3/realignertargetcreator/environment.yaml | 2 +- bio/jannovar/environment.yaml | 2 +- bio/picard/addorreplacereadgroups/environment.yaml | 2 +- bio/picard/bedtointervallist/environment.yaml | 2 +- bio/picard/collectalignmentsummarymetrics/environment.yaml | 2 +- bio/picard/collecthsmetrics/environment.yaml | 2 +- bio/picard/collectinsertsizemetrics/environment.yaml | 2 +- bio/picard/collectmultiplemetrics/environment.yaml | 2 +- bio/picard/collecttargetedpcrmetrics/environment.yaml | 2 +- bio/picard/createsequencedictionary/environment.yaml | 2 +- bio/picard/markduplicates/environment.yaml | 2 +- bio/picard/mergesamfiles/environment.yaml | 2 +- bio/picard/mergevcfs/environment.yaml | 2 +- bio/picard/revertsam/environment.yaml | 2 +- bio/picard/samtofastq/environment.yaml | 2 +- bio/picard/sortsam/environment.yaml | 2 +- bio/snpeff/annotate/environment.yaml | 2 +- bio/snpeff/download/environment.yaml | 2 +- bio/snpsift/annotate/environment.yaml | 2 +- bio/snpsift/dbnsfp/environment.yaml | 2 +- bio/snpsift/genesets/environment.yaml | 2 +- bio/snpsift/gwascat/environment.yaml | 2 +- bio/snpsift/varType/environment.yaml | 2 +- bio/trimmomatic/pe/environment.yaml | 2 +- bio/trimmomatic/se/environment.yaml | 2 +- bio/varscan/mpileup2indel/environment.yaml | 2 +- bio/varscan/mpileup2snp/environment.yaml | 2 +- bio/varscan/somatic/environment.yaml | 2 +- 42 files changed, 42 insertions(+), 42 deletions(-) diff --git a/bio/gatk/applybqsr/environment.yaml b/bio/gatk/applybqsr/environment.yaml index 6fb2d81249..344bef71d4 100644 --- a/bio/gatk/applybqsr/environment.yaml +++ b/bio/gatk/applybqsr/environment.yaml @@ -5,4 +5,4 @@ channels: dependencies: - gatk4 ==4.1.4.1 - openjdk =8 - - snakemake-wrapper-utils ==0.1.2 + - snakemake-wrapper-utils ==0.1.3 diff --git a/bio/gatk/baserecalibrator/environment.yaml b/bio/gatk/baserecalibrator/environment.yaml index 6fb2d81249..344bef71d4 100644 --- a/bio/gatk/baserecalibrator/environment.yaml +++ b/bio/gatk/baserecalibrator/environment.yaml @@ -5,4 +5,4 @@ channels: dependencies: - gatk4 ==4.1.4.1 - openjdk =8 - - snakemake-wrapper-utils ==0.1.2 + - snakemake-wrapper-utils ==0.1.3 diff --git a/bio/gatk/baserecalibratorspark/environment.yaml b/bio/gatk/baserecalibratorspark/environment.yaml index 6fb2d81249..344bef71d4 100644 --- a/bio/gatk/baserecalibratorspark/environment.yaml +++ b/bio/gatk/baserecalibratorspark/environment.yaml @@ -5,4 +5,4 @@ channels: dependencies: - gatk4 ==4.1.4.1 - openjdk =8 - - snakemake-wrapper-utils ==0.1.2 + - snakemake-wrapper-utils ==0.1.3 diff --git a/bio/gatk/combinegvcfs/environment.yaml b/bio/gatk/combinegvcfs/environment.yaml index a92753a102..8584e883e8 100644 --- a/bio/gatk/combinegvcfs/environment.yaml +++ b/bio/gatk/combinegvcfs/environment.yaml @@ -4,4 +4,4 @@ channels: - defaults dependencies: - gatk4 ==4.1.4.1 - - snakemake-wrapper-utils ==0.1.2 + - snakemake-wrapper-utils ==0.1.3 diff --git a/bio/gatk/genotypegvcfs/environment.yaml b/bio/gatk/genotypegvcfs/environment.yaml index a92753a102..8584e883e8 100644 --- a/bio/gatk/genotypegvcfs/environment.yaml +++ b/bio/gatk/genotypegvcfs/environment.yaml @@ -4,4 +4,4 @@ channels: - defaults dependencies: - gatk4 ==4.1.4.1 - - snakemake-wrapper-utils ==0.1.2 + - snakemake-wrapper-utils ==0.1.3 diff --git a/bio/gatk/haplotypecaller/environment.yaml b/bio/gatk/haplotypecaller/environment.yaml index a92753a102..8584e883e8 100644 --- a/bio/gatk/haplotypecaller/environment.yaml +++ b/bio/gatk/haplotypecaller/environment.yaml @@ -4,4 +4,4 @@ channels: - defaults dependencies: - gatk4 ==4.1.4.1 - - snakemake-wrapper-utils ==0.1.2 + - snakemake-wrapper-utils ==0.1.3 diff --git a/bio/gatk/mutect/environment.yaml b/bio/gatk/mutect/environment.yaml index a92753a102..8584e883e8 100644 --- a/bio/gatk/mutect/environment.yaml +++ b/bio/gatk/mutect/environment.yaml @@ -4,4 +4,4 @@ channels: - defaults dependencies: - gatk4 ==4.1.4.1 - - snakemake-wrapper-utils ==0.1.2 + - snakemake-wrapper-utils ==0.1.3 diff --git a/bio/gatk/selectvariants/environment.yaml b/bio/gatk/selectvariants/environment.yaml index a92753a102..8584e883e8 100644 --- a/bio/gatk/selectvariants/environment.yaml +++ b/bio/gatk/selectvariants/environment.yaml @@ -4,4 +4,4 @@ channels: - defaults dependencies: - gatk4 ==4.1.4.1 - - snakemake-wrapper-utils ==0.1.2 + - snakemake-wrapper-utils ==0.1.3 diff --git a/bio/gatk/splitncigarreads/environment.yaml b/bio/gatk/splitncigarreads/environment.yaml index a92753a102..8584e883e8 100644 --- a/bio/gatk/splitncigarreads/environment.yaml +++ b/bio/gatk/splitncigarreads/environment.yaml @@ -4,4 +4,4 @@ channels: - defaults dependencies: - gatk4 ==4.1.4.1 - - snakemake-wrapper-utils ==0.1.2 + - snakemake-wrapper-utils ==0.1.3 diff --git a/bio/gatk/variantfiltration/environment.yaml b/bio/gatk/variantfiltration/environment.yaml index a92753a102..8584e883e8 100644 --- a/bio/gatk/variantfiltration/environment.yaml +++ b/bio/gatk/variantfiltration/environment.yaml @@ -4,4 +4,4 @@ channels: - defaults dependencies: - gatk4 ==4.1.4.1 - - snakemake-wrapper-utils ==0.1.2 + - snakemake-wrapper-utils ==0.1.3 diff --git a/bio/gatk/variantrecalibrator/environment.yaml b/bio/gatk/variantrecalibrator/environment.yaml index a92753a102..8584e883e8 100644 --- a/bio/gatk/variantrecalibrator/environment.yaml +++ b/bio/gatk/variantrecalibrator/environment.yaml @@ -4,4 +4,4 @@ channels: - defaults dependencies: - gatk4 ==4.1.4.1 - - snakemake-wrapper-utils ==0.1.2 + - snakemake-wrapper-utils ==0.1.3 diff --git a/bio/gatk3/baserecalibrator/environment.yaml b/bio/gatk3/baserecalibrator/environment.yaml index 110a8be1e0..87fe20c0d1 100644 --- a/bio/gatk3/baserecalibrator/environment.yaml +++ b/bio/gatk3/baserecalibrator/environment.yaml @@ -4,4 +4,4 @@ channels: - defaults dependencies: - gatk ==3.8 - - snakemake-wrapper-utils ==0.1.2 + - snakemake-wrapper-utils ==0.1.3 diff --git a/bio/gatk3/indelrealigner/environment.yaml b/bio/gatk3/indelrealigner/environment.yaml index 110a8be1e0..87fe20c0d1 100644 --- a/bio/gatk3/indelrealigner/environment.yaml +++ b/bio/gatk3/indelrealigner/environment.yaml @@ -4,4 +4,4 @@ channels: - defaults dependencies: - gatk ==3.8 - - snakemake-wrapper-utils ==0.1.2 + - snakemake-wrapper-utils ==0.1.3 diff --git a/bio/gatk3/printreads/environment.yaml b/bio/gatk3/printreads/environment.yaml index 110a8be1e0..87fe20c0d1 100644 --- a/bio/gatk3/printreads/environment.yaml +++ b/bio/gatk3/printreads/environment.yaml @@ -4,4 +4,4 @@ channels: - defaults dependencies: - gatk ==3.8 - - snakemake-wrapper-utils ==0.1.2 + - snakemake-wrapper-utils ==0.1.3 diff --git a/bio/gatk3/realignertargetcreator/environment.yaml b/bio/gatk3/realignertargetcreator/environment.yaml index 110a8be1e0..87fe20c0d1 100644 --- a/bio/gatk3/realignertargetcreator/environment.yaml +++ b/bio/gatk3/realignertargetcreator/environment.yaml @@ -4,4 +4,4 @@ channels: - defaults dependencies: - gatk ==3.8 - - snakemake-wrapper-utils ==0.1.2 + - snakemake-wrapper-utils ==0.1.3 diff --git a/bio/jannovar/environment.yaml b/bio/jannovar/environment.yaml index ab9f79f983..51e9634aa2 100644 --- a/bio/jannovar/environment.yaml +++ b/bio/jannovar/environment.yaml @@ -4,4 +4,4 @@ channels: - defaults dependencies: - jannovar-cli ==0.31 - - snakemake-wrapper-utils ==0.1.2 + - snakemake-wrapper-utils ==0.1.3 diff --git a/bio/picard/addorreplacereadgroups/environment.yaml b/bio/picard/addorreplacereadgroups/environment.yaml index 0ba484ccb1..15cd88cb5e 100644 --- a/bio/picard/addorreplacereadgroups/environment.yaml +++ b/bio/picard/addorreplacereadgroups/environment.yaml @@ -4,4 +4,4 @@ channels: - defaults dependencies: - picard ==2.22.1 - - snakemake-wrapper-utils ==0.1.2 + - snakemake-wrapper-utils ==0.1.3 diff --git a/bio/picard/bedtointervallist/environment.yaml b/bio/picard/bedtointervallist/environment.yaml index 0ba484ccb1..15cd88cb5e 100644 --- a/bio/picard/bedtointervallist/environment.yaml +++ b/bio/picard/bedtointervallist/environment.yaml @@ -4,4 +4,4 @@ channels: - defaults dependencies: - picard ==2.22.1 - - snakemake-wrapper-utils ==0.1.2 + - snakemake-wrapper-utils ==0.1.3 diff --git a/bio/picard/collectalignmentsummarymetrics/environment.yaml b/bio/picard/collectalignmentsummarymetrics/environment.yaml index 0ba484ccb1..15cd88cb5e 100644 --- a/bio/picard/collectalignmentsummarymetrics/environment.yaml +++ b/bio/picard/collectalignmentsummarymetrics/environment.yaml @@ -4,4 +4,4 @@ channels: - defaults dependencies: - picard ==2.22.1 - - snakemake-wrapper-utils ==0.1.2 + - snakemake-wrapper-utils ==0.1.3 diff --git a/bio/picard/collecthsmetrics/environment.yaml b/bio/picard/collecthsmetrics/environment.yaml index 4ff61e3772..ccaa7a0e45 100644 --- a/bio/picard/collecthsmetrics/environment.yaml +++ b/bio/picard/collecthsmetrics/environment.yaml @@ -3,4 +3,4 @@ channels: - conda-forge dependencies: - picard ==2.22.1 - - snakemake-wrapper-utils ==0.1.2 + - snakemake-wrapper-utils ==0.1.3 diff --git a/bio/picard/collectinsertsizemetrics/environment.yaml b/bio/picard/collectinsertsizemetrics/environment.yaml index f745497b4c..7bf2bf3134 100644 --- a/bio/picard/collectinsertsizemetrics/environment.yaml +++ b/bio/picard/collectinsertsizemetrics/environment.yaml @@ -5,4 +5,4 @@ channels: dependencies: - picard ==2.22.1 - r-base ==3.6.2 - - snakemake-wrapper-utils ==0.1.2 + - snakemake-wrapper-utils ==0.1.3 diff --git a/bio/picard/collectmultiplemetrics/environment.yaml b/bio/picard/collectmultiplemetrics/environment.yaml index 692b99ffe7..ba2514f5c7 100644 --- a/bio/picard/collectmultiplemetrics/environment.yaml +++ b/bio/picard/collectmultiplemetrics/environment.yaml @@ -4,4 +4,4 @@ channels: - defaults dependencies: - picard ==2.23.0 - - snakemake-wrapper-utils ==0.1.2 + - snakemake-wrapper-utils ==0.1.3 diff --git a/bio/picard/collecttargetedpcrmetrics/environment.yaml b/bio/picard/collecttargetedpcrmetrics/environment.yaml index 0ba484ccb1..15cd88cb5e 100644 --- a/bio/picard/collecttargetedpcrmetrics/environment.yaml +++ b/bio/picard/collecttargetedpcrmetrics/environment.yaml @@ -4,4 +4,4 @@ channels: - defaults dependencies: - picard ==2.22.1 - - snakemake-wrapper-utils ==0.1.2 + - snakemake-wrapper-utils ==0.1.3 diff --git a/bio/picard/createsequencedictionary/environment.yaml b/bio/picard/createsequencedictionary/environment.yaml index 0ba484ccb1..15cd88cb5e 100644 --- a/bio/picard/createsequencedictionary/environment.yaml +++ b/bio/picard/createsequencedictionary/environment.yaml @@ -4,4 +4,4 @@ channels: - defaults dependencies: - picard ==2.22.1 - - snakemake-wrapper-utils ==0.1.2 + - snakemake-wrapper-utils ==0.1.3 diff --git a/bio/picard/markduplicates/environment.yaml b/bio/picard/markduplicates/environment.yaml index 0ba484ccb1..15cd88cb5e 100644 --- a/bio/picard/markduplicates/environment.yaml +++ b/bio/picard/markduplicates/environment.yaml @@ -4,4 +4,4 @@ channels: - defaults dependencies: - picard ==2.22.1 - - snakemake-wrapper-utils ==0.1.2 + - snakemake-wrapper-utils ==0.1.3 diff --git a/bio/picard/mergesamfiles/environment.yaml b/bio/picard/mergesamfiles/environment.yaml index 0ba484ccb1..15cd88cb5e 100644 --- a/bio/picard/mergesamfiles/environment.yaml +++ b/bio/picard/mergesamfiles/environment.yaml @@ -4,4 +4,4 @@ channels: - defaults dependencies: - picard ==2.22.1 - - snakemake-wrapper-utils ==0.1.2 + - snakemake-wrapper-utils ==0.1.3 diff --git a/bio/picard/mergevcfs/environment.yaml b/bio/picard/mergevcfs/environment.yaml index 0ba484ccb1..15cd88cb5e 100644 --- a/bio/picard/mergevcfs/environment.yaml +++ b/bio/picard/mergevcfs/environment.yaml @@ -4,4 +4,4 @@ channels: - defaults dependencies: - picard ==2.22.1 - - snakemake-wrapper-utils ==0.1.2 + - snakemake-wrapper-utils ==0.1.3 diff --git a/bio/picard/revertsam/environment.yaml b/bio/picard/revertsam/environment.yaml index 0ba484ccb1..15cd88cb5e 100644 --- a/bio/picard/revertsam/environment.yaml +++ b/bio/picard/revertsam/environment.yaml @@ -4,4 +4,4 @@ channels: - defaults dependencies: - picard ==2.22.1 - - snakemake-wrapper-utils ==0.1.2 + - snakemake-wrapper-utils ==0.1.3 diff --git a/bio/picard/samtofastq/environment.yaml b/bio/picard/samtofastq/environment.yaml index 0ba484ccb1..15cd88cb5e 100644 --- a/bio/picard/samtofastq/environment.yaml +++ b/bio/picard/samtofastq/environment.yaml @@ -4,4 +4,4 @@ channels: - defaults dependencies: - picard ==2.22.1 - - snakemake-wrapper-utils ==0.1.2 + - snakemake-wrapper-utils ==0.1.3 diff --git a/bio/picard/sortsam/environment.yaml b/bio/picard/sortsam/environment.yaml index 0ba484ccb1..15cd88cb5e 100644 --- a/bio/picard/sortsam/environment.yaml +++ b/bio/picard/sortsam/environment.yaml @@ -4,4 +4,4 @@ channels: - defaults dependencies: - picard ==2.22.1 - - snakemake-wrapper-utils ==0.1.2 + - snakemake-wrapper-utils ==0.1.3 diff --git a/bio/snpeff/annotate/environment.yaml b/bio/snpeff/annotate/environment.yaml index cd33b621bb..3efd2e9a20 100644 --- a/bio/snpeff/annotate/environment.yaml +++ b/bio/snpeff/annotate/environment.yaml @@ -4,4 +4,4 @@ channels: dependencies: - snpeff ==4.3.1t - bcftools =1.10 - - snakemake-wrapper-utils ==0.1.2 + - snakemake-wrapper-utils ==0.1.3 diff --git a/bio/snpeff/download/environment.yaml b/bio/snpeff/download/environment.yaml index cd33b621bb..3efd2e9a20 100644 --- a/bio/snpeff/download/environment.yaml +++ b/bio/snpeff/download/environment.yaml @@ -4,4 +4,4 @@ channels: dependencies: - snpeff ==4.3.1t - bcftools =1.10 - - snakemake-wrapper-utils ==0.1.2 + - snakemake-wrapper-utils ==0.1.3 diff --git a/bio/snpsift/annotate/environment.yaml b/bio/snpsift/annotate/environment.yaml index 8430bcc6f9..1e63b8350d 100644 --- a/bio/snpsift/annotate/environment.yaml +++ b/bio/snpsift/annotate/environment.yaml @@ -5,4 +5,4 @@ dependencies: - snpsift ==4.3.1t - bcftools ==1.10.2 - pbgzip ==2016.08.04 - - snakemake-wrapper-utils ==0.1.2 + - snakemake-wrapper-utils ==0.1.3 diff --git a/bio/snpsift/dbnsfp/environment.yaml b/bio/snpsift/dbnsfp/environment.yaml index b47e8db16d..6aaa4fcf4a 100644 --- a/bio/snpsift/dbnsfp/environment.yaml +++ b/bio/snpsift/dbnsfp/environment.yaml @@ -4,4 +4,4 @@ channels: dependencies: - snpsift=4.3.1t - bcftools ==1.10.2 - - snakemake-wrapper-utils ==0.1.2 + - snakemake-wrapper-utils ==0.1.3 diff --git a/bio/snpsift/genesets/environment.yaml b/bio/snpsift/genesets/environment.yaml index 192c3d1f79..e5911a684a 100644 --- a/bio/snpsift/genesets/environment.yaml +++ b/bio/snpsift/genesets/environment.yaml @@ -5,4 +5,4 @@ channels: dependencies: - snpsift ==4.3.1t - bcftools ==1.10.2 - - snakemake-wrapper-utils ==0.1.2 + - snakemake-wrapper-utils ==0.1.3 diff --git a/bio/snpsift/gwascat/environment.yaml b/bio/snpsift/gwascat/environment.yaml index b6079c237c..279b5f8649 100644 --- a/bio/snpsift/gwascat/environment.yaml +++ b/bio/snpsift/gwascat/environment.yaml @@ -4,4 +4,4 @@ channels: dependencies: - snpsift ==4.3.1t - bcftools ==1.10.2 - - snakemake-wrapper-utils ==0.1.2 + - snakemake-wrapper-utils ==0.1.3 diff --git a/bio/snpsift/varType/environment.yaml b/bio/snpsift/varType/environment.yaml index cc7992c051..23e47baad5 100644 --- a/bio/snpsift/varType/environment.yaml +++ b/bio/snpsift/varType/environment.yaml @@ -3,4 +3,4 @@ channels: - conda-forge dependencies: - snpsift =4.3.1t - - snakemake-wrapper-utils ==0.1.2 + - snakemake-wrapper-utils ==0.1.3 diff --git a/bio/trimmomatic/pe/environment.yaml b/bio/trimmomatic/pe/environment.yaml index ee3a7ac0be..eca75970ee 100644 --- a/bio/trimmomatic/pe/environment.yaml +++ b/bio/trimmomatic/pe/environment.yaml @@ -5,4 +5,4 @@ channels: dependencies: - trimmomatic ==0.36 - pigz ==2.3.4 - - snakemake-wrapper-utils ==0.1.2 + - snakemake-wrapper-utils ==0.1.3 diff --git a/bio/trimmomatic/se/environment.yaml b/bio/trimmomatic/se/environment.yaml index ee3a7ac0be..eca75970ee 100644 --- a/bio/trimmomatic/se/environment.yaml +++ b/bio/trimmomatic/se/environment.yaml @@ -5,4 +5,4 @@ channels: dependencies: - trimmomatic ==0.36 - pigz ==2.3.4 - - snakemake-wrapper-utils ==0.1.2 + - snakemake-wrapper-utils ==0.1.3 diff --git a/bio/varscan/mpileup2indel/environment.yaml b/bio/varscan/mpileup2indel/environment.yaml index 0defe1b9ab..b79d63dac5 100644 --- a/bio/varscan/mpileup2indel/environment.yaml +++ b/bio/varscan/mpileup2indel/environment.yaml @@ -5,4 +5,4 @@ channels: - defaults dependencies: - varscan ==2.4.3 - - snakemake-wrapper-utils ==0.1.2 + - snakemake-wrapper-utils ==0.1.3 diff --git a/bio/varscan/mpileup2snp/environment.yaml b/bio/varscan/mpileup2snp/environment.yaml index 4d3a103529..3b326fad0d 100644 --- a/bio/varscan/mpileup2snp/environment.yaml +++ b/bio/varscan/mpileup2snp/environment.yaml @@ -5,4 +5,4 @@ channels: - defaults dependencies: - varscan ==2.4.3 - - snakemake-wrapper-utils ==0.1.2 + - snakemake-wrapper-utils ==0.1.3 diff --git a/bio/varscan/somatic/environment.yaml b/bio/varscan/somatic/environment.yaml index 89df172f90..919105cff1 100644 --- a/bio/varscan/somatic/environment.yaml +++ b/bio/varscan/somatic/environment.yaml @@ -5,4 +5,4 @@ channels: - defaults dependencies: - varscan ==2.4.3 - - snakemake-wrapper-utils ==0.1.2 + - snakemake-wrapper-utils ==0.1.3 From f87c02945794fa9b085565c11ae0e49ed7397f45 Mon Sep 17 00:00:00 2001 From: tdayris Date: Thu, 12 Nov 2020 14:30:46 +0100 Subject: [PATCH 54/61] [doc] (Documentations): Links to todumentation added --- bio/jannovar/meta.yaml | 2 +- bio/snpeff/annotate/meta.yaml | 2 +- bio/snpeff/download/meta.yaml | 2 +- bio/snpsift/annotate/meta.yaml | 2 +- bio/snpsift/dbnsfp/meta.yaml | 4 ++-- bio/snpsift/genesets/meta.yaml | 2 +- bio/snpsift/gwascat/meta.yaml | 2 +- bio/snpsift/varType/meta.yaml | 2 +- bio/trimmomatic/pe/meta.yaml | 2 +- bio/trimmomatic/se/meta.yaml | 2 +- bio/varscan/mpileup2indel/meta.yaml | 2 +- bio/varscan/mpileup2snp/meta.yaml | 2 +- bio/varscan/somatic/meta.yaml | 2 +- 13 files changed, 14 insertions(+), 14 deletions(-) diff --git a/bio/jannovar/meta.yaml b/bio/jannovar/meta.yaml index 817fd866de..5cf5ecf06d 100644 --- a/bio/jannovar/meta.yaml +++ b/bio/jannovar/meta.yaml @@ -1,4 +1,4 @@ name: "jannovar" -description: Annotate predicted effect of nucleotide changes with Jannovar +description: Annotate predicted effect of nucleotide changes with `Jannovar`_ authors: - Bradford Powell diff --git a/bio/snpeff/annotate/meta.yaml b/bio/snpeff/annotate/meta.yaml index 96b6f335a4..97d0fec7b7 100644 --- a/bio/snpeff/annotate/meta.yaml +++ b/bio/snpeff/annotate/meta.yaml @@ -1,4 +1,4 @@ name: "snpeff" -description: Annotate predicted effect of nucleotide changes with SnpEff +description: Annotate predicted effect of nucleotide changes with `SnpEff `_ authors: - Bradford Powell diff --git a/bio/snpeff/download/meta.yaml b/bio/snpeff/download/meta.yaml index 2c0ec0ff5c..05427213d0 100644 --- a/bio/snpeff/download/meta.yaml +++ b/bio/snpeff/download/meta.yaml @@ -1,4 +1,4 @@ name: "snpeff download" -description: Download snpeff DB for a given species. +description: Download `snpeff `_ DB for a given species. authors: - Johannes Köster diff --git a/bio/snpsift/annotate/meta.yaml b/bio/snpsift/annotate/meta.yaml index 3c1ddc44cd..f7a16c5c1a 100644 --- a/bio/snpsift/annotate/meta.yaml +++ b/bio/snpsift/annotate/meta.yaml @@ -1,5 +1,5 @@ name: SnpSift annotate -description: Annotate using fields from another VCF file. +description: Annotate using fields from another VCF file with `SnpSift `_ authors: - Thibault Dayris input: diff --git a/bio/snpsift/dbnsfp/meta.yaml b/bio/snpsift/dbnsfp/meta.yaml index 1e934f4ad3..fae1fec929 100644 --- a/bio/snpsift/dbnsfp/meta.yaml +++ b/bio/snpsift/dbnsfp/meta.yaml @@ -1,6 +1,6 @@ --- -name: SnpSift dnNSFP -description: Annotate using integrated annotation from dbNSFP. +name: SnpSift dbNSFP +description: Annotate using integrated annotation from dbNSFP with `SnpSift `_ authors: - Thibault Dayris input: diff --git a/bio/snpsift/genesets/meta.yaml b/bio/snpsift/genesets/meta.yaml index 8c80382765..1bfd63f1f6 100644 --- a/bio/snpsift/genesets/meta.yaml +++ b/bio/snpsift/genesets/meta.yaml @@ -1,5 +1,5 @@ name: SnpSift Genes Sets -description: Annotate using GMT genes sets. +description: Annotate using GMT genes sets with `SnpSift `_ authors: - Thibault Dayris input: diff --git a/bio/snpsift/gwascat/meta.yaml b/bio/snpsift/gwascat/meta.yaml index 542b706274..982b769ba6 100644 --- a/bio/snpsift/gwascat/meta.yaml +++ b/bio/snpsift/gwascat/meta.yaml @@ -1,5 +1,5 @@ name: SnpSift GWAS Catalog -description: Annotate using GWAS catalog. +description: Annotate using GWAS catalog with `SnpSift `_ authors: - Thibault Dayris input: diff --git a/bio/snpsift/varType/meta.yaml b/bio/snpsift/varType/meta.yaml index 3420693fab..159c73efae 100644 --- a/bio/snpsift/varType/meta.yaml +++ b/bio/snpsift/varType/meta.yaml @@ -1,5 +1,5 @@ name: SnpSift varType -description: Add an INFO field denoting variant type. +description: Add an INFO field denoting variant type with `SnpSift `_ authors: - Thibault Dayris input: diff --git a/bio/trimmomatic/pe/meta.yaml b/bio/trimmomatic/pe/meta.yaml index 7f1d143fa6..d9b1a49527 100644 --- a/bio/trimmomatic/pe/meta.yaml +++ b/bio/trimmomatic/pe/meta.yaml @@ -1,5 +1,5 @@ name: "trimmomatic pe" -description: Trim paired-end reads with trimmomatic. (De)compress with pigz. +description: Trim paired-end reads with `trimmomatic `_ . (De)compress with pigz. authors: - Johannes Köster - Jorge Langa diff --git a/bio/trimmomatic/se/meta.yaml b/bio/trimmomatic/se/meta.yaml index e39e94da78..dca915fc54 100644 --- a/bio/trimmomatic/se/meta.yaml +++ b/bio/trimmomatic/se/meta.yaml @@ -1,5 +1,5 @@ name: "trimmomatic se" -description: Trim single-end reads with trimmomatic. (De)compress with pigz. +description: Trim single-end reads with `trimmomatic `_. (De)compress with pigz. authors: - Johannes Köster - Jorge Langa diff --git a/bio/varscan/mpileup2indel/meta.yaml b/bio/varscan/mpileup2indel/meta.yaml index 1ac43dce3e..1e0d7ad6bf 100644 --- a/bio/varscan/mpileup2indel/meta.yaml +++ b/bio/varscan/mpileup2indel/meta.yaml @@ -1,5 +1,5 @@ name: varscan mpileup2indel -description: Detect indel in NGS data from mpileup files +description: Detect indel in NGS data from mpileup files with `VarScan `_ authors: - Thibault Dayris input: diff --git a/bio/varscan/mpileup2snp/meta.yaml b/bio/varscan/mpileup2snp/meta.yaml index bb8979286f..27721e875b 100644 --- a/bio/varscan/mpileup2snp/meta.yaml +++ b/bio/varscan/mpileup2snp/meta.yaml @@ -1,5 +1,5 @@ name: varscan mpileup2snp -description: Detect variants in NGS data from Samtools mpileup +description: Detect variants in NGS data from Samtools mpileup with `VarScan `_ authors: - Thibault Dayris input: diff --git a/bio/varscan/somatic/meta.yaml b/bio/varscan/somatic/meta.yaml index f9b31612a7..d9062b56f4 100644 --- a/bio/varscan/somatic/meta.yaml +++ b/bio/varscan/somatic/meta.yaml @@ -1,5 +1,5 @@ name: varscan somatic -description: Varscan Somatic calls variants and identifies their somatic status (Germline/LOH/Somatic) using pileup files from a matched tumor-normal pair. +description: `Varscan Somatic`_` calls variants and identifies their somatic status (Germline/LOH/Somatic) using pileup files from a matched tumor-normal pair. authors: - Thibault Dayris input: From 4a08dc1d037d15494b2ea5676c73b08269684637 Mon Sep 17 00:00:00 2001 From: tdayris Date: Thu, 12 Nov 2020 14:32:22 +0100 Subject: [PATCH 55/61] [fix] (Black): Formatting error fixed --- bio/picard/collectinsertsizemetrics/wrapper.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bio/picard/collectinsertsizemetrics/wrapper.py b/bio/picard/collectinsertsizemetrics/wrapper.py index 93b88a0bd2..cf2a6f5b85 100644 --- a/bio/picard/collectinsertsizemetrics/wrapper.py +++ b/bio/picard/collectinsertsizemetrics/wrapper.py @@ -13,7 +13,6 @@ java_opts = get_java_opts(snakemake) - shell( "picard CollectInsertSizeMetrics {java_opts} {extra} " "INPUT={snakemake.input} OUTPUT={snakemake.output.txt} " From 6e643e597bbadda0a412f547b698325097981627 Mon Sep 17 00:00:00 2001 From: tdayris Date: Thu, 12 Nov 2020 14:37:22 +0100 Subject: [PATCH 56/61] [fix] (SyntaxError): trailing quote removed --- bio/varscan/somatic/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bio/varscan/somatic/meta.yaml b/bio/varscan/somatic/meta.yaml index d9062b56f4..ea6c257c58 100644 --- a/bio/varscan/somatic/meta.yaml +++ b/bio/varscan/somatic/meta.yaml @@ -1,5 +1,5 @@ name: varscan somatic -description: `Varscan Somatic`_` calls variants and identifies their somatic status (Germline/LOH/Somatic) using pileup files from a matched tumor-normal pair. +description: `Varscan Somatic`_ calls variants and identifies their somatic status (Germline/LOH/Somatic) using pileup files from a matched tumor-normal pair. authors: - Thibault Dayris input: From 172b278e4a046a95b48438fa6b3da243c45fa2e5 Mon Sep 17 00:00:00 2001 From: tdayris Date: Thu, 12 Nov 2020 14:40:02 +0100 Subject: [PATCH 57/61] fix] (SyntaxError): Starting quote removed --- bio/varscan/somatic/meta.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bio/varscan/somatic/meta.yaml b/bio/varscan/somatic/meta.yaml index ea6c257c58..98d7194b96 100644 --- a/bio/varscan/somatic/meta.yaml +++ b/bio/varscan/somatic/meta.yaml @@ -1,5 +1,6 @@ name: varscan somatic -description: `Varscan Somatic`_ calls variants and identifies their somatic status (Germline/LOH/Somatic) using pileup files from a matched tumor-normal pair. +description: | + `Varscan Somatic `_ calls variants and identifies their somatic status (Germline/LOH/Somatic) using pileup files from a matched tumor-normal pair. authors: - Thibault Dayris input: From 2bc41886cd492c2ce5f9ffcd727eca066fa7b18f Mon Sep 17 00:00:00 2001 From: tdayris Date: Thu, 12 Nov 2020 15:18:01 +0100 Subject: [PATCH 58/61] [fix] (KeyError): missing extra definition --- bio/snpsift/dbnsfp/wrapper.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bio/snpsift/dbnsfp/wrapper.py b/bio/snpsift/dbnsfp/wrapper.py index f24d60eb93..d0a348515a 100644 --- a/bio/snpsift/dbnsfp/wrapper.py +++ b/bio/snpsift/dbnsfp/wrapper.py @@ -8,6 +8,7 @@ from snakemake.shell import shell from snakemake_wrapper_utils.java import get_java_opts +extra = snakemake.params.get("extra", "") java_opts = get_java_opts(snakemake) log = snakemake.log_fmt_shell(stdout=False, stderr=True) From 791322e6ef804b3bf5781a9f7f873311a482e14b Mon Sep 17 00:00:00 2001 From: David Laehnemann Date: Thu, 12 Nov 2020 16:29:29 +0100 Subject: [PATCH 59/61] add mem_gb spec explanation comment in bio/jannovar/test/Snakefile --- bio/jannovar/test/Snakefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bio/jannovar/test/Snakefile b/bio/jannovar/test/Snakefile index e1d9ebb1ea..dcbb67b557 100644 --- a/bio/jannovar/test/Snakefile +++ b/bio/jannovar/test/Snakefile @@ -6,6 +6,10 @@ rule jannovar: "jannovar/{sample}.vcf.gz" log: "logs/jannovar/{sample}.log" + # optional specification of memory usage of the JVM that snakemake will respect with global + # resource restrictions (https://snakemake.readthedocs.io/en/latest/snakefiles/rules.html#resources) + # and which can be used to request RAM during cluster job submission as `{resources.mem_mg}`: + # https://snakemake.readthedocs.io/en/latest/executing/cluster.html#job-properties resources: mem_gb = 1 params: From 1eb2c9fef3dd8192cbbb947d7afde500c82e59fc Mon Sep 17 00:00:00 2001 From: tdayris Date: Thu, 12 Nov 2020 17:39:52 +0100 Subject: [PATCH 60/61] [dev] (gatk3): Snakefile resources specifications --- bio/gatk3/baserecalibrator/test/Snakefile | 3 ++- bio/gatk3/indelrealigner/test/Snakefile | 3 ++- bio/gatk3/printreads/test/Snakefile | 3 ++- bio/gatk3/realignertargetcreator/test/Snakefile | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/bio/gatk3/baserecalibrator/test/Snakefile b/bio/gatk3/baserecalibrator/test/Snakefile index d498da35f7..1fcccb5094 100644 --- a/bio/gatk3/baserecalibrator/test/Snakefile +++ b/bio/gatk3/baserecalibrator/test/Snakefile @@ -9,7 +9,8 @@ rule baserecalibrator: "logs/gatk3/bqsr/{sample}.log" params: extra="", # optional - java_opts="", # optional + resources: + mem_mb = 1024 threads: 16 wrapper: "bio/gatk/baserecalibrator" diff --git a/bio/gatk3/indelrealigner/test/Snakefile b/bio/gatk3/indelrealigner/test/Snakefile index 35d28ffdf3..42417870e3 100644 --- a/bio/gatk3/indelrealigner/test/Snakefile +++ b/bio/gatk3/indelrealigner/test/Snakefile @@ -10,7 +10,8 @@ rule indelrealigner: "logs/gatk3/indelrealigner/{sample}.log" params: extra="", # optional - java_opts="", # optional + resources: + mem_mb = 1024 threads: 16 wrapper: "bio/gatk/indelrealigner" diff --git a/bio/gatk3/printreads/test/Snakefile b/bio/gatk3/printreads/test/Snakefile index cd69869619..df8381c4f1 100644 --- a/bio/gatk3/printreads/test/Snakefile +++ b/bio/gatk3/printreads/test/Snakefile @@ -9,7 +9,8 @@ rule printreads: "logs/gatk/bqsr/{sample}..log" params: extra="", # optional - java_opts="", + resources: + mem_mb = 1024 threads: 16 wrapper: "bio/gatk3/printreads" diff --git a/bio/gatk3/realignertargetcreator/test/Snakefile b/bio/gatk3/realignertargetcreator/test/Snakefile index 0bfddf21e3..251444e611 100644 --- a/bio/gatk3/realignertargetcreator/test/Snakefile +++ b/bio/gatk3/realignertargetcreator/test/Snakefile @@ -9,7 +9,8 @@ rule realignertargetcreator: "logs/gatk/realignertargetcreator/{sample}.log" params: extra="", # optional - java_opts="", + resources: + mem_mb = 1024 threads: 16 wrapper: "bio/gatk3/realignertargetcreator" From fbb9baf5bf7e59446c22f07cb78c01ed7d22ba2c Mon Sep 17 00:00:00 2001 From: tdayris Date: Thu, 12 Nov 2020 17:41:49 +0100 Subject: [PATCH 61/61] [fix] (gatk3): Removing trailing comma --- bio/gatk3/baserecalibrator/test/Snakefile | 2 +- bio/gatk3/indelrealigner/test/Snakefile | 2 +- bio/gatk3/printreads/test/Snakefile | 2 +- bio/gatk3/realignertargetcreator/test/Snakefile | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bio/gatk3/baserecalibrator/test/Snakefile b/bio/gatk3/baserecalibrator/test/Snakefile index 1fcccb5094..e8dcad7e25 100644 --- a/bio/gatk3/baserecalibrator/test/Snakefile +++ b/bio/gatk3/baserecalibrator/test/Snakefile @@ -8,7 +8,7 @@ rule baserecalibrator: log: "logs/gatk3/bqsr/{sample}.log" params: - extra="", # optional + extra="" # optional resources: mem_mb = 1024 threads: 16 diff --git a/bio/gatk3/indelrealigner/test/Snakefile b/bio/gatk3/indelrealigner/test/Snakefile index 42417870e3..0dd75ef377 100644 --- a/bio/gatk3/indelrealigner/test/Snakefile +++ b/bio/gatk3/indelrealigner/test/Snakefile @@ -9,7 +9,7 @@ rule indelrealigner: log: "logs/gatk3/indelrealigner/{sample}.log" params: - extra="", # optional + extra="" # optional resources: mem_mb = 1024 threads: 16 diff --git a/bio/gatk3/printreads/test/Snakefile b/bio/gatk3/printreads/test/Snakefile index df8381c4f1..b88b415fba 100644 --- a/bio/gatk3/printreads/test/Snakefile +++ b/bio/gatk3/printreads/test/Snakefile @@ -8,7 +8,7 @@ rule printreads: log: "logs/gatk/bqsr/{sample}..log" params: - extra="", # optional + extra="" # optional resources: mem_mb = 1024 threads: 16 diff --git a/bio/gatk3/realignertargetcreator/test/Snakefile b/bio/gatk3/realignertargetcreator/test/Snakefile index 251444e611..bd21d0b30c 100644 --- a/bio/gatk3/realignertargetcreator/test/Snakefile +++ b/bio/gatk3/realignertargetcreator/test/Snakefile @@ -8,7 +8,7 @@ rule realignertargetcreator: log: "logs/gatk/realignertargetcreator/{sample}.log" params: - extra="", # optional + extra="" # optional resources: mem_mb = 1024 threads: 16