Fix #1467 Use dataframe_image to render large dataframes as image in …

…icephys tutorial

docs/gallery/domain/plot_icephys_pandas.py

@@ -10,6 +10,13 @@
 from the :py:meth:`~pynwb.icephys` module. See the :ref:`icephys_tutorial_new`
 tutorial for an introduction to the intracellular electrophysiology metadata
 tables and how to create an NWBFile for intracellular electrophysiology data.
+
+.. note::
+
+    To enhance display of large pandas DataFrames, we save and render large tables
+    as images in this tutorial. Simply click on the rendered table to view the
+    full-size image.
+
 '''
 
 #####################################################################
@@ -20,6 +27,9 @@
 # Standard Python imports
 import numpy as np
 import pandas
+import dataframe_image
+df_basedir = "../../source/tutorials/domain/images/"  # directory to save rendered dataframe images for display
+
 # Set pandas rendering option to avoid very wide tables in the html docs
 pandas.set_option("display.max_colwidth", 30)
 pandas.set_option("display.max_rows", 10)
@@ -154,18 +164,26 @@
 # ``electrodes``, ``stimuli``, and ``responses``. For convenience, the
 # :py:meth:`~pynwb.icephys.IntracellularRecordingsTable.to_dataframe` of the
 # :py:class:`~pynwb.icephys.IntracellularRecordingsTable` provides a few
-# additonal optional parameters to optionally ignore the ids of the category tables
+# additional optional parameters to ignore the ids of the category tables
 # (via ``ignore_category_ids=True``) or to convert the electrode, stimulus, and
 # response references to ObjectIds. For example:
 #
-
-nwbfile.intracellular_recordings.to_dataframe(
+ir_df = nwbfile.intracellular_recordings.to_dataframe(
     ignore_category_ids=True,
     electrode_refs_as_objectids=True,
     stimulus_refs_as_objectids=True,
     response_refs_as_objectids=True
 )
 
+# save the table as image to display in the docs
+dataframe_image.export(obj=ir_df, filename=df_basedir +'intracellular_recordings_dataframe.png')
+
+#####################################################################
+# .. image:: images/intracellular_recordings_dataframe.png
+#     :width: 100%
+#     :alt: intracellular_recordings_dataframe.png
+#     :align: center
+
 #####################################################################
 # Using indexed DataFrames
 # ^^^^^^^^^^^^^^^^^^^^^^^^
@@ -224,12 +242,14 @@
 from hdmf.common.hierarchicaltable import to_hierarchical_dataframe
 icephys_meta_df = to_hierarchical_dataframe(root_table)
 
-#####################################################################
-#
+# save table as image to display in the docs
+dataframe_image.export(icephys_meta_df, df_basedir + 'icephys_meta_dataframe.png')
 
-# To avoid a too wide display in the online docs we here only show 4 select rows of the
-# table and transpose the table to show the large MultiIndex as columns instead of rows
-icephys_meta_df.iloc[[0, 1, 18, 19]].transpose()
+#####################################################################
+# .. image:: images/icephys_meta_dataframe.png
+#     :width: 100%
+#     :alt: icephys_meta_dataframe.png
+#     :align: center
 
 #####################################################################
 # Depending on the analysis, it can be useful to further process our `DataFrame`_. Using the standard
@@ -252,7 +272,18 @@
 flatten_column_index(dataframe=icephys_meta_df, max_levels=2, inplace=True)
 # Remove the id columns. By setting inplace=False allows us to visualize the result of this
 # action while keeping the id columns in our main icephys_meta_df table
-drop_id_columns(dataframe=icephys_meta_df, inplace=False)
+drid_icephys_meta_df = drop_id_columns(dataframe=icephys_meta_df, inplace=False)
+
+# save the table as image to display in the docs
+dataframe_image.export(
+    obj=drid_icephys_meta_df,
+    filename=df_basedir +'icephys_meta_dataframe_drop_id.png')
+
+#####################################################################
+# .. image:: images/icephys_meta_dataframe_drop_id.png
+#     :width: 100%
+#     :alt: icephys_meta_dataframe_drop_id.png
+#     :align: center
 
 #####################################################################
 # Useful additional data preparations
@@ -278,8 +309,17 @@
 # icephys_meta_df.drop(labels=[('stimuli', 'stimulus'), ], axis=1, inplace=True)
 # Add our expanded columns to the icephys_meta_df dataframe
 icephys_meta_df = pandas.concat([icephys_meta_df, stimulus_df], axis=1)
-# display the table for the HTML docs
-icephys_meta_df
+
+# save the table as image to display in the docs
+dataframe_image.export(
+    obj=icephys_meta_df,
+    filename=df_basedir +'icephys_meta_dataframe_expand_tsr.png')
+
+#####################################################################
+# .. image:: images/icephys_meta_dataframe_expand_tsr.png
+#     :width: 100%
+#     :alt: icephys_meta_dataframe_expand_tsr.png
+#     :align: center
 
 #####################################################################
 # We can then easily expand also the ``(responses, response)`` column in the same way
@@ -303,17 +343,29 @@
 # Here we show a few examples.
 
 # Add a column with the name of the stimulus TimeSeries object.
-# Note: We use getattr here to easily deal with missing values, i.e., cases where no stimulus is present
+# Note: We use getattr here to easily deal with missing values,
+#       i.e., here the cases where no stimulus is present
 col = ('stimuli', 'name')
 icephys_meta_df[col] = [getattr(s, 'name', None)
                         for s in icephys_meta_df[('stimuli', 'timeseries')]]
 
-# Often we can easily do this in bulk-fashion by specifing the collection of fields of interest
+# Often we can easily do this in a bulk-fashion by specifying
+# the collection of fields of interest
 for field in ['neurodata_type', 'gain', 'rate', 'starting_time', 'object_id']:
     col = ('stimuli', field)
     icephys_meta_df[col] = [getattr(s, field, None)
                             for s in icephys_meta_df[('stimuli', 'timeseries')]]
-icephys_meta_df
+# save the table as image to display in the docs
+dataframe_image.export(
+    obj=icephys_meta_df,
+    filename=df_basedir +'icephys_meta_dataframe_add_stimres.png',
+    max_cols=10)
+
+#####################################################################
+# .. image:: images/icephys_meta_dataframe_add_stimres.png
+#     :width: 100%
+#     :alt: icephys_meta_dataframe_add_stimres.png
+#     :align: center
 
 #####################################################################
 # Naturally we can again do the same also for our response columns
@@ -404,7 +456,8 @@
 
 # Get a response 'vcs_9' from the file
 response = nwbfile.get_acquisition('vcs_9')
-# Return all data related to that response, including the stimulus as part of ('stimuli', 'stimulus') column
+# Return all data related to that response, including the stimulus
+# as part of ('stimuli', 'stimulus') column
 icephys_meta_df[icephys_meta_df[('responses', 'object_id')] == response.object_id]
 
 
@@ -438,4 +491,20 @@
 # Given a stimulus type, get all corresponding intracellular recordings
 # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-icephys_meta_df[icephys_meta_df[('sequential_recordings', 'stimulus_type')] == 'StimType_1']
+query_res_df = icephys_meta_df[
+    icephys_meta_df[('sequential_recordings', 'stimulus_type')] == 'StimType_1'
+]
+
+# save the table as image to display in the docs
+dataframe_image.export(
+    obj=query_res_df,
+    filename=df_basedir +'icephys_meta_query_result_dataframe.png',
+    max_cols=10)
+
+#####################################################################
+# .. image:: images/icephys_meta_query_result_dataframe.png
+#     :width: 100%
+#     :alt: icephys_meta_query_result_dataframe.png
+#     :align: center
+
+

requirements-doc.txt

@@ -7,3 +7,4 @@ allensdk>=2.11.0  # python 3.8 is not supported in allensdk<2.11
 MarkupSafe==2.0.1  # resolve incompatibility between jinja2 and markupsafe: https://github.com/AllenInstitute/AllenSDK/issues/2308
 Pillow
 sphinx-copybutton
+dataframe_image   # used to render large dataframe as image in the sphinx gallery to improve html display