Merge pull request #114 from databio/dev

Release 0.9.1

Author: nsheff

doc/source/changelog.rst

@@ -1,5 +1,10 @@
 Changelog
 ******************************
+- **v0.9.1** (*2019-01-29*):
+
+    - Fixed a bug in NGSTk that caused errors for read counting functions on 
+    MACOS. MACOS ``wc`` returns leading whitespace, which caused these functions
+    to fail.
 
 - **v0.9.0** (*2018-11-19*):
 

pypiper/_version.py

@@ -1 +1 @@
-__version__ = "0.9.0"
+__version__ = "0.9.1"

pypiper/manager.py

@@ -26,7 +26,8 @@
 from .flags import *
 from .utils import \
     check_shell, check_shell_pipes, checkpoint_filepath, clear_flags, flag_name, \
-    make_lock_name, pipeline_filepath, CHECKPOINT_SPECIFICATIONS, split_by_pipes
+    is_multi_target, make_lock_name, pipeline_filepath, \
+    CHECKPOINT_SPECIFICATIONS, split_by_pipes
 from ._version import __version__
 import __main__
 
@@ -553,8 +554,8 @@ def run(self, cmd, target=None, lock_name=None, shell="guess",
 
         :param cmd: Shell command(s) to be run.
         :type cmd: str or list
-        :param target: Output file to be produced. Optional.
-        :type target: str or None
+        :param target: Output file(s) to produce, optional
+        :type target: None or str or Sequence of str
         :param lock_name: Name of lock file. Optional.
         :type lock_name: str or None
         :param shell: If command requires should be run in its own shell.
@@ -612,7 +613,7 @@ def run(self, cmd, target=None, lock_name=None, shell="guess",
 
         # If the target is a list, for now let's just strip it to the first target.
         # Really, it should just check for all of them.
-        if isinstance(target, list):
+        if is_multi_target(target):
             target = target[0]
             #primary_target = target[0]
         # Create lock file:
@@ -1171,7 +1172,7 @@ def report_object(self, key, filename, anchor_text=None, anchor_image=None,
             relative_anchor_image = os.path.relpath(anchor_image, self.outfolder) \
                 if os.path.isabs(anchor_image) else anchor_image
         else:
-            relative_anchor_image = ""
+            relative_anchor_image = "None"
 
 
         message_raw = "{key}\t{filename}\t{anchor_text}\t{anchor_image}\t{annotation}".format(

pypiper/ngstk.py

@@ -655,22 +655,22 @@ def merge_fastq(self, inputs, output, run=False, remove_inputs=False):
 
     def count_lines(self, file_name):
         """
-        Uses the command-line utility wc to count the number of lines in a file.
+        Uses the command-line utility wc to count the number of lines in a file. For MacOS, must strip leading whitespace from wc.
 
         :param file_name: name of file whose lines are to be counted
         :type file_name: str
         """
-        x = subprocess.check_output("wc -l " + file_name + " | cut -f1 -d' '", shell=True)
-        return x
+        x = subprocess.check_output("wc -l " + file_name + " | sed -E 's/^[[:space:]]+//' | cut -f1 -d' '", shell=True)
+        return x.strip()
 
     def count_lines_zip(self, file_name):
         """
-        Uses the command-line utility wc to count the number of lines in a file.
+        Uses the command-line utility wc to count the number of lines in a file. For MacOS, must strip leading whitespace from wc.
         For compressed files.
         :param file: file_name
         """
-        x = subprocess.check_output("gunzip -c " + file_name + " | wc -l | cut -f1 -d' '", shell=True)
-        return x
+        x = subprocess.check_output("gunzip -c " + file_name + " | wc -l | sed -E 's/^[[:space:]]+//' | cut -f1 -d' '", shell=True)
+        return x.strip()
 
     def get_chrs_from_bam(self, file_name):
         """
@@ -701,10 +701,10 @@ def count_unique_reads(self, file_name, paired_end):
         if file_name.endswith("bam"):
             param = ""
         if paired_end:
-            r1 = self.samtools_view(file_name, param=param + " -f64", postpend=" | cut -f1 | sort -k1,1 -u | wc -l ")
-            r2 = self.samtools_view(file_name, param=param + " -f128", postpend=" | cut -f1 | sort -k1,1 -u | wc -l ")
+            r1 = self.samtools_view(file_name, param=param + " -f64", postpend=" | cut -f1 | sort -k1,1 -u | wc -l | sed -E 's/^[[:space:]]+//'")
+            r2 = self.samtools_view(file_name, param=param + " -f128", postpend=" | cut -f1 | sort -k1,1 -u | wc -l | sed -E 's/^[[:space:]]+//'")
         else:
-            r1 = self.samtools_view(file_name, param=param + "", postpend=" | cut -f1 | sort -k1,1 -u | wc -l ")
+            r1 = self.samtools_view(file_name, param=param + "", postpend=" | cut -f1 | sort -k1,1 -u | wc -l | sed -E 's/^[[:space:]]+//'")
             r2 = 0
         return int(r1) + int(r2)
 
@@ -733,11 +733,11 @@ def count_unique_mapped_reads(self, file_name, paired_end):
         else:
             raise ValueError("Not a SAM or BAM: '{}'".format(file_name))
 
-        if paired_end:
-            r1 = self.samtools_view(file_name, param=param + " -f64", postpend=" | cut -f1 | sort -k1,1 -u | wc -l ")
-            r2 = self.samtools_view(file_name, param=param + " -f128", postpend=" | cut -f1 | sort -k1,1 -u | wc -l ")
+        if paired_end: 
+            r1 = self.samtools_view(file_name, param=param + " -f64", postpend=" | cut -f1 | sort -k1,1 -u | wc -l | sed -E 's/^[[:space:]]+//'")
+            r2 = self.samtools_view(file_name, param=param + " -f128", postpend=" | cut -f1 | sort -k1,1 -u | wc -l | sed -E 's/^[[:space:]]+//'")
         else:
-            r1 = self.samtools_view(file_name, param=param + "", postpend=" | cut -f1 | sort -k1,1 -u | wc -l ")
+            r1 = self.samtools_view(file_name, param=param + "", postpend=" | cut -f1 | sort -k1,1 -u | wc -l | sed -E 's/^[[:space:]]+//'")
             r2 = 0
 
         return int(r1) + int(r2)
@@ -867,7 +867,8 @@ def count_concordant(self, aligned_bam):
         :type aligned_bam: str
         """
         cmd = self.tools.samtools + " view " + aligned_bam + " | "
-        cmd += "grep 'YT:Z:CP'" + " | uniq -u | wc -l"
+        cmd += "grep 'YT:Z:CP'" + " | uniq -u | wc -l | sed -E 's/^[[:space:]]+//'"
+        
         return subprocess.check_output(cmd, shell=True)
 
 

pypiper/utils.py

@@ -430,6 +430,27 @@ def make_lock_name(original_path, path_base_folder):
     return original_path.replace(path_base_folder, "").replace(os.sep, "__")
 
 
+def is_multi_target(target):
+    """
+    Determine if pipeline manager's run target is multiple.
+
+    :param None or str or Sequence of str target: 0, 1, or multiple targets
+    :return bool: Whether there are multiple targets
+    :raise TypeError: if the argument is neither None nor string nor Sequence
+    """
+    if sys.version_info < (3, 3):
+        from collections import Sequence
+    else:
+        from collections.abc import Sequence
+    if target is None or isinstance(target, str):
+        return False
+    elif isinstance(target, Sequence):
+        return len(target) > 1
+    else:
+        raise TypeError("Could not interpret argument as a target: {} ({})".
+                        format(target, type(target)))
+
+
 def parse_cores(cores, pm, default):
     """
     Framework to finalize number of cores for an operation.