Move versioning page generation into its own executable

bin/plotting/pycbc_page_foreground

@@ -9,7 +9,7 @@ from pycbc.io import hdf
 import h5py, logging
 from pycbc.pnutils import mass1_mass2_to_mchirp_eta
 import pycbc.results, pycbc.results.followup
-from pycbc.results.versioning import save_fig_with_metadata
+from pycbc.results import save_fig_with_metadata
 import pycbc.version
 import sys
 

bin/plotting/pycbc_page_versioning

@@ -0,0 +1,73 @@
+#!/bin/env python
+
+"""
+Create the html files needed to describe the versioning
+information for a set of libraries and executables in
+pycbc results pages
+"""
+
+import argparse, logging
+import pycbc.version
+from pycbc import init_logging
+from pycbc.results import save_fig_with_metadata, html_escape, get_library_version_info, get_code_version_numbers
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--version', action="version",
+                    version=pycbc.version.git_verbose_msg)
+parser.add_argument('--verbose', action='store_true')
+parser.add_argument('--executables-files', nargs='+', required=True,
+                    help="List of executable files to provide version "
+                         "information for")
+parser.add_argument('--executables-names', nargs='+', required=True,
+                    help="List of executable names, must be in the "
+                         "same order as --executables-files")
+parser.add_argument("--output-file", required=True,
+                    help="The directory for output html snippets")
+args = parser.parse_args()
+
+init_logging(args.verbose)
+
+if not len(args.executables_files) == len(args.executables_names):
+    raise parser.error("--executables-files and executables-names must be "
+                       "the same number of arguments")
+
+
+logging.info("Getting version information for libraries")
+library_list = get_library_version_info()
+html_text = ''
+for curr_lib in library_list:
+    lib_name = curr_lib['Name']
+    logging.info(f"Getting {lib_name} information")
+    html_text += f'<h2>{lib_name} Version Information</h2>:<br>\n'
+    for key, value in curr_lib.items():
+        html_text += '<li> %s : %s </li>\n' % (key, value)
+
+
+code_version_dict = get_code_version_numbers(
+    args.executables_names,
+    args.executables_files
+)
+
+html_text += f'<h2>Version Information from Executables</h2>:<br>\n'
+for key, value in code_version_dict.items():
+    # value might be a str or a bytes object in python3. python2 is happy
+    # to combine these objects (or uniocde and str, their equivalents)
+    # but python3 is not.
+    try:
+        value = value.decode()
+    except AttributeError:
+        pass
+    html_text += '<li><b>%s</b>:<br><pre>%s</pre></li><hr><br><br>\n' \
+        % (key, str(value).replace('@', '&#64;'))
+
+kwds = {
+    'render-function' : 'render_text',
+    'title' : 'Version Information',
+}
+
+save_fig_with_metadata(
+    html_escape(html_text),
+    args.output_file,
+    **kwds
+)
+logging.info("Done")

bin/pygrb/pycbc_pygrb_pp_workflow

@@ -32,9 +32,9 @@ import os
 import pycbc.version
 from pycbc import init_logging
 import pycbc.workflow as _workflow
-from pycbc.results import create_versioning_page, layout
+from pycbc.results import layout
 from pycbc.results.pygrb_postprocessing_utils import extract_ifos
-from pycbc.results.versioning import save_fig_with_metadata
+from pycbc.results import save_fig_with_metadata
 
 __author__ = "Francesco Pannarale  <[email protected]>"
 __version__ = pycbc.version.git_verbose_msg
@@ -385,9 +385,13 @@ ini_file = _workflow.FileList([_workflow.File(wflow.ifos, '',
                                               wflow.analysis_time,
                                               file_url='file://' + ini_file_path)])
 layout.single_layout(base, ini_file)
-
+ 
 # Create versioning information
-create_versioning_page(rdir['workflow/version'], wflow.cp)
+wf.make_versioning_page(
+    _workflow,
+    wflow.cp,
+    rdir['workflow/version'],
+)
 
 # Create the final log file
 log_file_html = _workflow.File(wflow.ifos, 'WORKFLOW-LOG', wflow.analysis_time,

bin/workflows/pycbc_make_bank_verifier_workflow

@@ -31,7 +31,7 @@ from ligo import segments
 
 import pycbc.version
 import pycbc.workflow as wf
-from pycbc.results import (create_versioning_page, static_table, layout)
+from pycbc.results import (static_table, layout)
 from pycbc.workflow.jobsetup import (select_generic_executable,
                                      int_gps_time_to_str,
                                      PycbcCreateInjectionsExecutable,
@@ -382,7 +382,12 @@ for tag in sorted(output_pointinjs):
 layout.single_layout(conf_dir, conf_file)
 
 # Create versioning information
-create_versioning_page(rdir['workflow/version'], workflow.cp)
+wf.make_versioning_page(
+    workflow,
+    workflow.cp,
+    rdir['workflow/version'],
+)
+
 
 wf.make_results_web_page(workflow, os.path.join(os.getcwd(), rdir.base),
                          explicit_dependencies=plotting_nodes)

bin/workflows/pycbc_make_inference_inj_workflow

@@ -355,8 +355,12 @@ if do_pp_test:
     # add to the main workflow
     workflow += pp_workflow
 
-# create versioning HTML pages
-results.create_versioning_page(rdir["workflow/version"], workflow.cp)
+# Create versioning information
+wf.make_versioning_page(
+    workflow,
+    workflow.cp,
+    rdir['workflow/version'],
+)
 
 # create node for making HTML pages
 plotting.make_results_web_page(finalize_workflow,

bin/workflows/pycbc_make_inference_plots_workflow

@@ -246,9 +246,13 @@ for num_event, event in enumerate(events):
     layout.single_layout(rdir['detector_sensitivity'], [psd_plot],
                          unique=str(num_event).zfill(zpad),
                          title=label, collapse=True)
-
-# create versioning HTML pages
-results.create_versioning_page(rdir["workflow/version"], container.cp)
+
+# Create versioning information
+wf.make_versioning_page(
+    workflow,
+    container.cp,
+    rdir['workflow/version'],
+)
 
 # create node for making HTML pages
 plotting.make_results_web_page(finalize_workflow,

bin/workflows/pycbc_make_inference_workflow

@@ -37,6 +37,7 @@ from pycbc.workflow import configuration
 from pycbc.workflow import core
 from pycbc.workflow import datafind
 from pycbc.workflow import plotting
+from pycbc.workflow import versioning
 from pycbc import __version__
 import pycbc.workflow.inference_followups as inffu
 from pycbc.workflow.jobsetup import PycbcInferenceExecutable
@@ -307,8 +308,12 @@ for num_event, event in enumerate(events):
     workflow += sub_workflow
 
 
-# create versioning HTML pages
-results.create_versioning_page(rdir["workflow/version"], container.cp)
+# Create versioning information
+versioning.make_versioning_page(
+    workflow,
+    container.cp,
+    rdir['workflow/version'],
+)
 
 # create node for making HTML pages
 plotting.make_results_web_page(finalize_workflow,

bin/workflows/pycbc_make_offline_search_workflow

@@ -34,8 +34,7 @@ import os, argparse, logging
 import configparser as ConfigParser
 from ligo import segments
 import numpy, lal, datetime, itertools
-from pycbc.results import create_versioning_page, static_table, layout
-from pycbc.results.versioning import save_fig_with_metadata
+from pycbc.results import static_table, layout, save_fig_with_metadata
 from pycbc.results.metadata import html_escape
 
 
@@ -804,10 +803,12 @@ ini_file = wf.FileList([wf.File(workflow.ifos, '', workflow.analysis_time,
                         file_url='file://' + ini_file_path)])
 layout.single_layout(base, ini_file)
 
-
 # Create versioning information
-create_versioning_page(rdir['workflow/version'], container.cp)
-
+wf.make_versioning_page(
+    workflow,
+    container.cp,
+    rdir['workflow/version'],
+)
 
 ############################ Finalization ####################################
 

examples/search/executables.ini

@@ -23,6 +23,7 @@ page_ifar_catalog = ${which:pycbc_ifar_catalog}
 page_injections = ${which:pycbc_page_injtable}
 page_segplot = ${which:pycbc_page_segplot}
 page_segtable = ${which:pycbc_page_segtable}
+page_versioning = ${which:pycbc_page_versioning}
 page_vetotable = ${which:pycbc_page_vetotable}
 plot_bank = ${which:pycbc_plot_bank_bins}
 plot_binnedhist = ${which:pycbc_fit_sngls_binned}

examples/search/plotting.ini

@@ -242,3 +242,4 @@ num-to-write = 2
 [plot_gating]
 [plot_snrratehist]
 [plot_throughput]
+[page_versioning]

examples/workflow/inference/small_test/workflow.ini

@@ -61,6 +61,7 @@ table_summary = ${which:pycbc_inference_table_summary}
 create_fits_file = ${which:pycbc_inference_create_fits}
 plot_spectrum = ${which:pycbc_plot_psd_file}
 results_page = ${which:pycbc_make_html_page}
+page_versioning = ${which:pycbc_page_versioning}
 ; diagnostic plots: at the moment, there are none for Dynesty
 
 [pegasus_profile]
@@ -106,6 +107,10 @@ z-arg = snr
 ; page. The dyn-range-factor needs to be set to 1.
 dyn-range-factor = 1
 
+[page_versioning]
+; This finds version information for all libraries and scripts used by the
+; workflow, and puts them into a file for use in results pages
+
 [results_page]
 ; This sets settings for creating the results page. You may want to change
 ; the analysis title, to make it more descriptive.

pycbc/results/versioning.py

@@ -17,10 +17,8 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 
 import logging
-import os
 import subprocess
 import urllib.parse
-from pycbc.results import save_fig_with_metadata, html_escape
 
 import lal, lalframe
 import pycbc.version, glue.git_version
@@ -118,21 +116,7 @@ def add_info_new_version(info_dct, curr_module, extra_str):
 
     return library_list
 
-def write_library_information(path):
-    library_list = get_library_version_info()
-    for curr_lib in library_list:
-        lib_name = curr_lib['Name']
-        text = ''
-        for key, value in curr_lib.items():
-            text+='<li> %s : %s </li>\n' %(key,value)
-        kwds = {'render-function' : 'render_text',
-                'title' : '%s Version Information'%lib_name,
-        }
-
-        save_fig_with_metadata(html_escape(text),
-          os.path.join(path,'%s_version_information.html' %(lib_name)), **kwds)
-
-def get_code_version_numbers(cp):
+def get_code_version_numbers(executable_names, executable_files):
     """Will extract the version information from the executables listed in
     the executable section of the supplied ConfigParser object.
 
@@ -143,9 +127,9 @@ def get_code_version_numbers(cp):
         version string for each executable.
     """
     code_version_dict = {}
-    for _, value in cp.items('executables'):
+    for exe_name, value in zip(executable_names, executable_files):
         value = urllib.parse.urlparse(value)
-        _, exe_name = os.path.split(value.path)
+        logging.info("Getting version info for %s", exe_name)
         version_string = None
         if value.scheme in ['gsiftp', 'http', 'https']:
             code_version_dict[exe_name] = "Using bundle downloaded from %s" % value
@@ -168,30 +152,3 @@ def get_code_version_numbers(cp):
                 version_string = "Executable doesn't seem to exist(!)"
             code_version_dict[exe_name] = version_string
     return code_version_dict
-
-def write_code_versions(path, cp):
-    code_version_dict = get_code_version_numbers(cp)
-    html_text = ''
-    for key,value in code_version_dict.items():
-        # value might be a str or a bytes object in python3. python2 is happy
-        # to combine these objects (or uniocde and str, their equivalents)
-        # but python3 is not.
-        try:
-            value = value.decode()
-        except AttributeError:
-            pass
-        html_text+= '<li><b>%s</b>:<br><pre>%s</pre></li><hr><br><br>\n' \
-            % (key, str(value).replace('@', '&#64;'))
-    kwds = {'render-function' : 'render_text',
-            'title' : 'Version Information from Executables',
-    }
-    save_fig_with_metadata(html_escape(html_text),
-        os.path.join(path,'version_information_from_executables.html'), **kwds)
-
-def create_versioning_page(path, cp):
-    logging.info("Entering versioning module")
-    if not os.path.exists(path):
-        os.mkdir(path)
-    write_library_information(path)
-    write_code_versions(path, cp)
-    logging.info("Leaving versioning module")

pycbc/workflow/__init__.py

@@ -44,6 +44,7 @@
 from pycbc.workflow.plotting import *
 from pycbc.workflow.minifollowups import *
 from pycbc.workflow.dq import *
+from pycbc.workflow.versioning import *
 
 # Set the pycbc workflow specific pegasus configuration and planning files
 from pycbc.workflow.pegasus_workflow import PEGASUS_FILE_DIRECTORY