Make shell scripts work on Windows

augur/align.py

@@ -6,7 +6,7 @@
 from shutil import copyfile
 import numpy as np
 from Bio import AlignIO, SeqIO, Seq
-from .utils import run_shell_command, nthreads_value
+from .utils import run_shell_command, nthreads_value, shquote
 
 def make_gaps_ambiguous(aln):
     '''
@@ -94,7 +94,9 @@ def run(args):
 
     # align
     if args.method=='mafft':
-        cmd = "mafft --reorder --anysymbol --thread %d %s 1> %s 2> %s.log"%(args.nthreads, seq_fname, output, output)
+        shoutput = shquote(output)
+        shname = shquote(seq_fname)
+        cmd = "mafft --reorder --anysymbol --thread %d %s 1> %s 2> %s.log"%(args.nthreads, shname, shoutput, shoutput)
         print("\nusing mafft to align via:\n\t" + cmd +
               " \n\n\tKatoh et al, Nucleic Acid Research, vol 30, issue 14"
               "\n\thttps://doi.org/10.1093%2Fnar%2Fgkf436\n")

augur/filter.py

@@ -8,7 +8,7 @@
 import random, os, re
 import numpy as np
 import sys
-from .utils import read_metadata, get_numerical_dates, run_shell_command
+from .utils import read_metadata, get_numerical_dates, run_shell_command, shquote
 
 comment_char = '#'
 
@@ -30,8 +30,8 @@ def write_vcf(compressed, input_file, output_file, dropped_samps):
     inCall = "--gzvcf" if compressed else "--vcf"
     outCall = "| gzip -c" if output_file.lower().endswith('.gz') else ""
 
-    toDrop = " ".join(["--remove-indv "+s for s in dropped_samps])
-    call = ["vcftools", toDrop, inCall, input_file, "--recode --stdout", outCall, ">", output_file]
+    toDrop = " ".join(["--remove-indv "+shquote(s) for s in dropped_samps])
+    call = ["vcftools", toDrop, inCall, shquote(input_file), "--recode --stdout", outCall, ">", shquote(output_file)]
 
     print("Filtering samples using VCFTools with the call:")
     print(" ".join(call))

augur/mask.py

@@ -6,7 +6,7 @@
 import pandas as pd
 import os
 import numpy as np
-from .utils import run_shell_command
+from .utils import run_shell_command, shquote
 
 def get_mask_sites(vcf_file, mask_file):
     '''
@@ -99,7 +99,7 @@ def run(args):
         in_file = args.sequences+"_temp"
         copyfile(args.sequences, in_file)
 
-    call = ["vcftools", "--exclude-positions", tempMaskFile, inCall, in_file, "--recode --stdout", outCall, ">", out_file]
+    call = ["vcftools", "--exclude-positions", shquote(tempMaskFile), inCall, shquote(in_file), "--recode --stdout", outCall, ">", shquote(out_file)]
     print("Removing masked sites from VCF file using vcftools... this may take some time. Call:")
     print(" ".join(call))
     run_shell_command(" ".join(call), raise_errors = True)

augur/tree.py

@@ -13,7 +13,7 @@
 from treetime.vcf_utils import read_vcf
 from pathlib import Path
 
-from .utils import run_shell_command, nthreads_value
+from .utils import run_shell_command, nthreads_value, shquote
 
 def find_executable(names, default = None):
     """
@@ -58,7 +58,7 @@ def build_raxml(aln_file, out_file, clean_up=True, nthreads=1, tree_builder_args
     # RAxML_bestTree.4ed91a, RAxML_info.4ed91a, RAxML_parsimonyTree.4ed91a, RAxML_result.4ed91a
     random_string = uuid.uuid4().hex[0:6]
 
-    call = [raxml,"-T",str(nthreads)," -f d -m GTRCAT -c 25 -p 235813 -n %s -s"%(random_string), aln_file, tree_builder_args, "> RAxML_log.%s"%(random_string)]
+    call = [raxml,"-T",str(nthreads)," -f d -m GTRCAT -c 25 -p 235813 -n %s -s"%(random_string), shquote(aln_file), tree_builder_args, "> RAxML_log.%s"%(random_string)]
     cmd = " ".join(call)
     print("Building a tree via:\n\t" + cmd +
           "\n\tStamatakis, A: RAxML Version 8: A tool for Phylogenetic Analysis and Post-Analysis of Large Phylogenies."
@@ -110,7 +110,7 @@ def build_fasttree(aln_file, out_file, clean_up=True, nthreads=1, tree_builder_a
         "OMP_NUM_THREADS": str(nthreads),
     }
 
-    call = [fasttree, "-nosupport", "-nt", aln_file, tree_builder_args, "1>", out_file, "2>", log_file]
+    call = [fasttree, "-nosupport", "-nt", shquote(aln_file), tree_builder_args, "1>", shquote(out_file), "2>", shquote(log_file)]
     cmd = " ".join(call)
     print("Building a tree via:\n\t" + cmd +
           "\n\tPrice et al: FastTree 2 - Approximately Maximum-Likelihood Trees for Large Alignments." +
@@ -164,10 +164,10 @@ def build_iqtree(aln_file, out_file, substitution_model="GTR", clean_up=True, nt
     ]
 
     if substitution_model.lower() != "none":
-        call = ["iqtree", *fast_opts, "-nt", str(nthreads), "-s", tmp_aln_file,
+        call = ["iqtree", *fast_opts, "-nt", str(nthreads), "-s", shquote(tmp_aln_file),
                 "-m", substitution_model, tree_builder_args, ">", log_file]
     else:
-        call = ["iqtree", *fast_opts, "-nt", str(nthreads), "-s", tmp_aln_file, tree_builder_args, ">", log_file]
+        call = ["iqtree", *fast_opts, "-nt", str(nthreads), "-s", shquote(tmp_aln_file), tree_builder_args, ">", shquote(log_file)]
 
     cmd = " ".join(call)
 

augur/utils.py

@@ -4,6 +4,7 @@
 import os, json, sys
 import pandas as pd
 import subprocess
+import shlex
 from treetime.utils import numeric_date
 from collections import defaultdict
 from pkg_resources import resource_stream
@@ -531,6 +532,7 @@ def write_VCF_translation(prot_dict, vcf_file_name, ref_file_name):
         call = ["gzip", vcf_file_name[:-3]]
         run_shell_command(" ".join(call), raise_errors = True)
 
+shquote = shlex.quote
 
 def run_shell_command(cmd, raise_errors = False, extra_env = None):
     """
@@ -544,21 +546,30 @@ def run_shell_command(cmd, raise_errors = False, extra_env = None):
     overlayed onto the default subprocess environment.
     """
     env = os.environ.copy()
+    out = ''
 
     if extra_env:
         env.update(extra_env)
 
     try:
+        shargs = ['-c', "set -euo pipefail; " + cmd]
+
+        if os.name == 'posix':
+            shellexec = ['/bin/bash']
+        else:
+            # We try best effort on other systems. For now that means nt/java.
+            shellexec = ['env', 'bash']
+
         # Use check_call() instead of run() since the latter was added only in Python 3.5.
-        subprocess.check_call(
-            "set -euo pipefail; " + cmd,
-            shell = True,
-            executable = "/bin/bash",
-            env = env)
+        out = subprocess.check_output(
+            shellexec + shargs,
+            shell = False,
+            stderr=subprocess.STDOUT)
 
     except subprocess.CalledProcessError as error:
         print_error(
-            "shell exited {rc} when running: {cmd}{extra}",
+            "{out}\nshell exited {rc} when running: {cmd}{extra}",
+            out = out,
             rc  = error.returncode,
             cmd = cmd,
             extra = "\nAre you sure this program is installed?" if error.returncode==127 else "",
@@ -571,11 +582,13 @@ def run_shell_command(cmd, raise_errors = False, extra_env = None):
     except FileNotFoundError as error:
         print_error(
             """
+            {out}
             Unable to run shell commands using {shell}!
 
             Augur requires {shell} to be installed.  Please open an issue on GitHub
             <https://github.com/nextstrain/augur/issues/new> if you need assistance.
             """,
+            out   = out,
             shell = error.filename
         )
         if raise_errors: