improve stat hiding filters in lineplot_and_heatmap (#182)

In `lineplot_and_heatmap`, mutations that are hidden are by one slider filtered even if they fail other sliders.

Previously, they were dropped rather than hidden.

Addresses this issue: dms-vep/dms-vep-pipeline-3#96

Author: jbloom

CHANGELOG.rst

@@ -6,6 +6,10 @@ All notable changes to this project will be documented in this file.
 
 The format is based on `Keep a Changelog <https://keepachangelog.com>`_.
 
+6.10
+----
+- In ``lineplot_and_heatmap``, mutations that are hidden are by one slider filtered even if they fail other sliders. Addresses `this issue <https://github.com/dms-vep/dms-vep-pipeline-3/issues/96>`_
+
 6.9
 ---
 - Added ``rename_std`` option to ``lineplot_and_heatmaps``, which fixes a quasi-bug introduced in the ``rename_stat_col`` option by the changes in version 6.8.

notebooks/real_LyCoV1404.ipynb

@@ -31,7 +31,7 @@
 
 __author__ = "`the Bloom lab <https://research.fhcrc.org/bloom/en.html>`_"
 __email__ = "[email protected]"
-__version__ = "6.9"
+__version__ = "6.10"
 __url__ = "https://github.com/jbloomlab/polyclonal"
 
 from polyclonal.alphabets import AAS

polyclonal/__init__.py

@@ -7,7 +7,6 @@
 
 """
 
-
 import collections  # noqa: F401
 import itertools
 import os  # noqa: F401

polyclonal/pdb_utils.py

@@ -7,7 +7,6 @@
 
 """
 
-
 import functools
 import math
 import operator
@@ -515,8 +514,10 @@ def lineplot_and_heatmap(
         indicated in the slider.
     addtl_slider_stats_hide_not_filter : None or list
         By default, `addtl_slider_stats` are filtered entirely from data set. If you just
-        them excluded from lineplot calculation but marked as filtered on heat map,
-        add names of stats to this list.
+        them excluded from lineplot but marked as hidden on heat map (eg, gray box),
+        add names of stats to this list. Mutations that fail one of these hiding filters
+        are always shown as hidden on the heat map rather than fully excluded, even if
+        they fail other filters in `addtl_slider_stats`.
     init_floor_at_zero : bool
         Initial value for option to put floor of zero on value is `stat_col`.
     init_site_statistic : {'sum', 'mean', 'max', 'min'}
@@ -657,7 +658,6 @@ def replace_std(col):
         .assign(**{stat_col: lambda x: x[stat_col] * scale_stat_col})
     )
 
-    # filter `data_df` by any minimums in `slider_binding_range_kwargs`
     if slider_binding_range_kwargs is None:
         slider_binding_range_kwargs = {}
 
@@ -723,9 +723,11 @@ def replace_std(col):
     # get tooltips for heatmap
     float_cols = [c for c in req_cols if data_df[c].dtype == float]
     heatmap_tooltips = [
-        alt.Tooltip(c, type="quantitative", format=".3g")
-        if c in float_cols
-        else alt.Tooltip(c, type="nominal")
+        (
+            alt.Tooltip(c, type="quantitative", format=".3g")
+            if c in float_cols
+            else alt.Tooltip(c, type="nominal")
+        )
         for c in req_cols
         if c != category_col or show_category_label
     ]
@@ -954,11 +956,13 @@ def replace_std(col):
                 base_chart = base_chart.transform_filter(
                     (alt.datum[slider_stat] <= (slider + 1e-6))  # round tol
                     | ~alt.expr.isFinite(alt.datum[slider_stat])  # do not filter null
+                    | alt.datum["_stat_hide"]  # do not filter hidden sites
                 )
             else:
                 base_chart = base_chart.transform_filter(
                     (alt.datum[slider_stat] >= (slider - 1e-6))  # round tol
                     | ~alt.expr.isFinite(alt.datum[slider_stat])  # do not filter null
+                    | alt.datum["_stat_hide"]  # do not filter hidden sites
                 )
     # Remove any sites that are only wildtype and filter with site zoom brush
     base_chart = (
@@ -1165,9 +1169,9 @@ def replace_std(col):
         heatmaps.append(heatmap_bg + heatmap_hide + heatmap + heatmap_wildtype)
     heatmaps = alt.vconcat(*heatmaps, spacing=10).resolve_scale(
         x="shared",
-        color="shared"
-        if heatmap_color_scheme or len(categories) == 1
-        else "independent",
+        color=(
+            "shared" if heatmap_color_scheme or len(categories) == 1 else "independent"
+        ),
     )
 
     chartlist = []

polyclonal/plot.py

@@ -7,7 +7,6 @@
 
 """
 
-
 import collections
 import copy  # noqa: F401
 import inspect
@@ -387,33 +386,33 @@ class Polyclonal:
        barcode aa_substitutions   IC50
     0       AA                   0.085
     1       AC              M1C  0.230
-    2       GA              M1C  0.230
-    3       AG              G2A  0.296
-    4       AT              A4K  0.128
-    5       TA              A4L  0.117
-    6       CA          M1C G2A  0.355
-    7       CG          M1C A4K  0.722
-    8       CC          G2A A4K  1.414
-    9       TC          G2A A4L  0.858
-    10      CT      M1C G2A A4K  3.237
-    11      TG      M1C G2A A4L  1.430
+    2       AG              G2A  0.296
+    3       AT              A4K  0.128
+    4       TA              A4L  0.117
+    5       CA          M1C G2A  0.355
+    6       CG          M1C A4K  0.722
+    7       CC          G2A A4K  1.414
+    8       TC          G2A A4L  0.858
+    9       CT      M1C G2A A4K  3.237
+    10      TG      M1C G2A A4L  1.430
+    11      GA              M1C  0.230
 
     Or the IC90s:
 
     >>> model.icXX(variants_df, x=0.9, col='IC90').round(3)
        barcode aa_substitutions    IC90
     0       AA                    0.464
     1       AC              M1C   1.260
-    2       GA              M1C   1.260
-    3       AG              G2A   1.831
-    4       AT              A4K   0.976
-    5       TA              A4L   0.782
-    6       CA          M1C G2A   2.853
-    7       CG          M1C A4K   4.176
-    8       CC          G2A A4K   7.473
-    9       TC          G2A A4L   4.532
-    10      CT      M1C G2A A4K  18.717
-    11      TG      M1C G2A A4L   9.532
+    2       AG              G2A   1.831
+    3       AT              A4K   0.976
+    4       TA              A4L   0.782
+    5       CA          M1C G2A   2.853
+    6       CG          M1C A4K   4.176
+    7       CC          G2A A4K   7.473
+    8       TC          G2A A4L   4.532
+    9       CT      M1C G2A A4K  18.717
+    10      TG      M1C G2A A4L   9.532
+    11      GA              M1C   1.260
 
     Or the fold change IC90s of all mutations:
     >>> model.mut_icXX_df(
@@ -2127,10 +2126,9 @@ def fit(
             fit_kwargs["fix_hill_coefficient"] = True
             fit_kwargs["fix_non_neutralized_frac"] = True
             fit_kwargs["reg_activity_weight"] = fit_fixed_first_reg_activity_weight
-            fit_kwargs[
-                "log_desc"
-            ] = "fixed Hill coefficient and non-neutralized frac" + (
-                f" {log_desc}" if log_desc else ""
+            fit_kwargs["log_desc"] = (
+                "fixed Hill coefficient and non-neutralized frac"
+                + (f" {log_desc}" if log_desc else "")
             )
             self.fit(**fit_kwargs)
         elif fit_site_level_first:

polyclonal/polyclonal.py

@@ -7,7 +7,6 @@
 
 """
 
-
 import re
 
 import pandas as pd  # noqa: F401

polyclonal/utils.py

@@ -1,6 +1,5 @@
 """Setup script for ``polyclonal``."""
 
-
 import re
 import sys
 
@@ -52,6 +51,7 @@
         "natsort>=8.0",
         "numpy>=1.17",
         "pandas>=1.5",
+        "pyarrow",
         "requests",
         "scipy>=1.7.1",
         "urllib3==1.26.15",  # https://github.com/googleapis/python-bigquery/issues/1565