diff --git a/bin/minifollowups/pycbc_page_coincinfo b/bin/minifollowups/pycbc_page_coincinfo index 74c7250d268..0648074eebb 100644 --- a/bin/minifollowups/pycbc_page_coincinfo +++ b/bin/minifollowups/pycbc_page_coincinfo @@ -14,7 +14,7 @@ # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. -""" Make tables describing a coincident foreground event""" +""" Make tables describing a foreground event""" import h5py, argparse, logging, sys import matplotlib; matplotlib.use('Agg') @@ -33,7 +33,7 @@ parser.add_argument('--bank-file', help="HDF format template bank file") parser.add_argument('--output-file') parser.add_argument('--statmap-file', required=True, - help="HDF format clustered coincident statmap file containing the result " + help="HDF format clustered statmap file containing the result " "triggers. Required") parser.add_argument('--statmap-file-subspace-name', default='background_exc', help="If given look in this 'sub-directory' of the HDF file for triggers, " @@ -71,27 +71,27 @@ if args.n_loudest is not None: if args.sort_order == 'descending': sorting = sorting[::-1] n = sorting[args.n_loudest] - title = 'Parameters of coincident event ranked %s' % (args.n_loudest + 1) + title = 'Parameters of event ranked %s' % (args.n_loudest + 1) caption = ('Parameters of event ranked %s by %s %s in the search. The figures below' ' show the mini-followup data for this event.' % (args.n_loudest + 1, args.sort_order, args.sort_variable)) elif args.trigger_id is not None: n = args.trigger_id - title = 'Details of coincident trigger' - caption = ('Parameters of coincident event. The figures below show the ' + title = 'Details of trigger' + caption = ('Parameters of event. The figures below show the ' 'mini-followup data for this event.') else: # It shouldn't be possible to get here! raise ValueError() -# Make a table for the coincident information ################################# +# Make a table for the event information ################################# -hdrs = ["Coincident ranking statistic", - "Inclusive IFAR (yr)", - "Inclusive FAP", - "Exclusive IFAR (yr)", - "Exclusive FAP" - ] +hdrs = ["Ranking statistic", + "Inclusive IFAR (yr)", + "Inclusive FAP", + "Exclusive IFAR (yr)", + "Exclusive FAP" + ] dsets = ['stat', 'ifar', 'fap', 'ifar_exc', 'fap_exc'] formats = ['%5.2f', '%5.2f', '%5.2e', '%5.2f', '%5.2e'] diff --git a/bin/plotting/pycbc_page_ifar b/bin/plotting/pycbc_page_ifar index 5fd1b7b71a0..b9c61e81ac3 100644 --- a/bin/plotting/pycbc_page_ifar +++ b/bin/plotting/pycbc_page_ifar @@ -191,7 +191,6 @@ fig = pylab.figure(1) # get a unique list of timeslide_ids and loop over them interval = fp.attrs['timeslide_interval'] -pifo, fifo = fp.attrs['pivot'], fp.attrs['fixed'] ifo_joined = fp.attrs['ifos'].replace(' ','') p_starts = fp['segments'][ifo_joined]['start'][:] p_ends = fp['segments'][ifo_joined]['end'][:] diff --git a/bin/plotting/pycbc_page_snrifar b/bin/plotting/pycbc_page_snrifar index cfa2b4b6616..1d6978dff56 100644 --- a/bin/plotting/pycbc_page_snrifar +++ b/bin/plotting/pycbc_page_snrifar @@ -334,7 +334,7 @@ if not args.cumulative: ax2.plot([],[]) pylab.sca(ax1) -pylab.xlabel(r'Coincident Ranking Statistic') +pylab.xlabel(r'Ranking Statistic') pylab.yscale('log') pylab.ylim(plot_ymin, plot_ymax * 10.0) pylab.xlim(plot_xmin, plot_xmax) diff --git a/bin/plotting/pycbc_page_snrratehist b/bin/plotting/pycbc_page_snrratehist index 8c1d63727ab..db4f52c553c 100755 --- a/bin/plotting/pycbc_page_snrratehist +++ b/bin/plotting/pycbc_page_snrratehist @@ -205,7 +205,7 @@ if fstat is not None and not args.closed_box: label='Foreground', mec='none', fmt='o', ms=1, capthick=0, elinewidth=4, color='#ff6600') -pylab.xlabel('Coincident ranking statistic (bin size = %.2f)' % bin_size) +pylab.xlabel('Ranking statistic (bin size = %.2f)' % bin_size) pylab.ylabel('Trigger Rate (yr$^{-1})$') if args.x_min is not None: pylab.xlim(xmin=args.x_min) diff --git a/bin/plotting/pycbc_plot_singles_vs_params b/bin/plotting/pycbc_plot_singles_vs_params index 980e7a63741..8a76a677826 100644 --- a/bin/plotting/pycbc_plot_singles_vs_params +++ b/bin/plotting/pycbc_plot_singles_vs_params @@ -116,18 +116,17 @@ if opts.log_x: hexbin_style['xscale'] = 'log' if opts.log_y: hexbin_style['yscale'] = 'log' -if opts.min_z is not None: - hexbin_style['vmin'] = opts.min_z -if opts.max_z is not None: - hexbin_style['vmax'] = opts.max_z +minz = opts.min_z if opts.min_z else 1 +maxz = opts.max_z +hexbin_style['norm'] = LogNorm(vmin=minz, vmax=maxz) logging.info('Plotting') fig = pl.figure() ax = fig.gca() + if opts.z_var == 'density': - norm = LogNorm() - hb = ax.hexbin(x, y, norm=norm, vmin=1, **hexbin_style) + hb = ax.hexbin(x, y, **hexbin_style) fig.colorbar(hb, ticks=LogLocator(subs=range(10))) elif opts.z_var in ranking.sngls_ranking_function_dict: cb_style = {} @@ -137,7 +136,6 @@ elif opts.z_var in ranking.sngls_ranking_function_dict: min_z = z.min() if opts.min_z is None else opts.min_z max_z = z.max() if opts.max_z is None else opts.max_z if max_z / min_z > 10: - hexbin_style['norm'] = LogNorm() cb_style['ticks'] = LogLocator(subs=range(10)) hb = ax.hexbin(x, y, C=z, reduce_C_function=max, **hexbin_style) fig.colorbar(hb, **cb_style) diff --git a/bin/workflows/pycbc_make_coinc_search_workflow b/bin/workflows/pycbc_make_offline_search_workflow similarity index 92% rename from bin/workflows/pycbc_make_coinc_search_workflow rename to bin/workflows/pycbc_make_offline_search_workflow index b92ec4bf34a..41aab72d2d9 100755 --- a/bin/workflows/pycbc_make_coinc_search_workflow +++ b/bin/workflows/pycbc_make_offline_search_workflow @@ -1,7 +1,7 @@ #!/usr/bin/env python -# Copyright (C) 2013-2019 Ian W. Harry, Alex Nitz, Marton Tapai, -# Gareth Davies +# Copyright (C) 2013-2023, Ian W. Harry, Alex Nitz, Marton Tapai, +# Gareth Cabourn Davies # # This program is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by the @@ -17,12 +17,12 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. """ -Program for running multi-detector workflow analysis through coincidence and -then generate post-processing and plots. +Program for running offline analysis through event +finding and ranking then generate post-processing +and plots. """ import pycbc import pycbc.version -__author__ = "Alex Nitz " __version__ = pycbc.version.git_verbose_msg __date__ = pycbc.version.date __program__ = "pycbc_offline" @@ -69,13 +69,11 @@ wf.add_workflow_command_line_group(parser) wf.add_workflow_settings_cli(parser) args = parser.parse_args() -# FIXME: opts.tags is currently unused here. +container = wf.Workflow(args, args.workflow_name) +workflow = wf.Workflow(args, args.workflow_name + '-main') +finalize_workflow = wf.Workflow(args, args.workflow_name + '-finalization') wf.makedir(args.output_dir) - -container = wf.Workflow(args, name=args.workflow_name) -workflow = wf.Workflow(args, name=args.workflow_name + '-main') -finalize_workflow = wf.Workflow(args, name=args.workflow_name + '-finalization') os.chdir(args.output_dir) rdir = layout.SectionNumber('results', ['analysis_time', @@ -139,6 +137,7 @@ for ifo in workflow.ifos: hoft_tags=[] if 'hoft' in workflow.cp.get_subsections('workflow-datafind'): hoft_tags=['hoft'] + datafind_files, analyzable_file, analyzable_segs, analyzable_name = \ wf.setup_datafind_workflow(workflow, ssegs, "datafind", @@ -173,7 +172,6 @@ bank_plot = wf.make_template_plot(workflow, hdfbank, rdir['coincident_triggers'], tags=bank_tags) - ######################## Setup the FULL DATA run ############################## output_dir = "full_data" @@ -202,6 +200,7 @@ for dq_label in dq_labels: statfiles += dq_label_files dqfiles += dq_label_files dqfile_labels += len(dq_label_files) * [dq_label] + statfiles += wf.setup_trigger_fitting(workflow, insps, hdfbank, final_veto_file, final_veto_name, output_dir=output_dir, @@ -219,9 +218,7 @@ ifo_precedence_list = workflow.cp.get_opt_tags('workflow-coincidence', 'timeslid for ifo, _ in zip(*insps.categorize_by_attr('ifo')): ifo_ids[ifo] = ifo_precedence_list.index(ifo) -# Generate the possible detector combinations from 2 detectors -# up to the number of trigger files - +# Generate the possible detector combinations if workflow.cp.has_option_tags('workflow-data_quality', 'no-coinc-veto', tags=None): logging.info("no-coinc-veto option enabled, " + @@ -250,26 +247,50 @@ for ifocomb in ifo_combos(ifo_ids.keys()): tags=ctagcomb) -if len(insps) == 2: - final_bg_files = no_fg_exc_files -else: - final_bg_files = {} - for ifocomb in ifo_combos(ifo_ids.keys()): +# Are we analysing single-detector candidates? +analyze_singles = workflow.cp.has_section('workflow-singles') \ + and workflow.cp.has_option_tags('workflow-singles', + 'analyze', tags=None) + +# The single-detector findtrigs and statmap jobs work differently +# - set these up here + +for ifo in ifo_ids.keys(): + if not analyze_singles: + continue + inspcomb = wf.select_files_by_ifo_combination([ifo], insps) + # Create coinc tag, and set up the findtrigs job for the combination + ctagsngl = ['full_data', '1det'] + no_fg_exc_files[ifo] = wf.setup_sngls( + workflow, hdfbank, inspcomb, statfiles, final_veto_file, + final_veto_name, output_dir, tags=ctagsngl) + +ifo_sets = list(ifo_combos(ifo_ids.keys())) +if analyze_singles: + ifo_sets += [(ifo,) for ifo in ifo_ids.keys()] + +final_bg_files = {} +# set up exclude-zerolag jobs for each ifo combination +for ifocomb in ifo_sets: + if len(ifocomb) > 1: _, _, ordered_ifo_list = wf.get_ordered_ifo_list(ifocomb, ifo_ids) # Create coinc tag coinctag = '{}det'.format(len(ifocomb)) - ctagcomb = ['full_data', coinctag] - other_ifo_keys = list(no_fg_exc_files.keys()) - other_ifo_keys.remove(ordered_ifo_list) - other_bg_files = {ctype: no_fg_exc_files[ctype] - for ctype in other_ifo_keys} - final_bg_files[ordered_ifo_list] = wf.setup_exclude_zerolag( - workflow, - no_fg_exc_files[ordered_ifo_list], - wf.FileList(other_bg_files.values()), - output_dir, ordered_ifo_list, - tags=ctagcomb - ) + else: + ordered_ifo_list = ifocomb[0] + coinctag= '1det' + other_ifo_keys = list(no_fg_exc_files.keys()) + other_ifo_keys.remove(ordered_ifo_list) + ctagcomb = ['full_data', coinctag] + other_bg_files = {ctype: no_fg_exc_files[ctype] + for ctype in other_ifo_keys} + final_bg_files[ordered_ifo_list] = wf.setup_exclude_zerolag( + workflow, + no_fg_exc_files[ordered_ifo_list], + wf.FileList(other_bg_files.values()), + output_dir, ordered_ifo_list, + tags=ctagcomb + ) combined_bg_file = wf.setup_combine_statmap( workflow, @@ -433,29 +454,10 @@ for insp_file in full_insps: # Main results with combined file (we mix open and closed box here, but # separate them in the result page) -# FIXME: COMMENTED OUT JOBS ARE FAILING ... NEED FIXING!! -# (Currently that's most of the jobs :-( ) -#snrifar = wf.make_snrifar_plot(workflow, combined_bg_file, -# rdir['open_box_result'], -# tags=combined_bg_file.tags) -#snrifar_cb = wf.make_snrifar_plot(workflow, combined_bg_file, -# rdir['coincident_triggers'], closed_box=True, -# tags=combined_bg_file.tags + ['closed']) -#ratehist = wf.make_snrratehist_plot(workflow, combined_bg_file, -# rdir['open_box_result'], -# tags=combined_bg_file.tags) -#snrifar_ifar = wf.make_snrifar_plot(workflow, combined_bg_file, -# rdir['open_box_result/significance'], -# cumulative=False, -# tags=combined_bg_file.tags + ['ifar']) ifar_ob = wf.make_ifar_plot(workflow, combined_bg_file, rdir['open_box_result'], tags=combined_bg_file.tags + ['open_box'], executable='page_ifar_catalog') -#ifar_cb = wf.make_ifar_plot(workflow, combined_bg_file, -# rdir['coincident_triggers'], -# tags=combined_bg_file.tags + ['closed_box'], -# executable='page_ifar_catalog') table = wf.make_foreground_table(workflow, combined_bg_file, hdfbank, rdir['open_box_result'], singles=insps, extension='.html', @@ -468,8 +470,6 @@ fore_xmlloudest = wf.make_foreground_table(workflow, combined_bg_file, hdfbank, rdir['open_box_result'], singles=insps, extension='.xml', tags=["xmlloudest"]) -#symlink_result(snrifar, 'open_box_result/significance') -#symlink_result(ratehist, 'open_box_result/significance') symlink_result(table, 'open_box_result/significance') # Set html pages @@ -501,7 +501,6 @@ wf.setup_foreground_minifollowups(workflow, combined_bg_file, snrifar_summ = [] for key in final_bg_files: - # FIXME: Stop obfuscating this file! bg_file = final_bg_files[key] open_dir = rdir['open_box_result/{}_coincidences'.format(key)] closed_dir = rdir['coincident_triggers/{}_coincidences'.format(key)] @@ -517,8 +516,11 @@ for key in final_bg_files: tags=bg_file.tags + ['ifar']) ifar_ob = wf.make_ifar_plot(workflow, bg_file, open_dir, tags=bg_file.tags + ['open_box']) - ifar_cb = wf.make_ifar_plot(workflow, bg_file, closed_dir, - tags=bg_file.tags + ['closed_box']) + if len(key) > 2: + # don't do the background plot for single-detector stuff, + # as it is just blank + ifar_cb = wf.make_ifar_plot(workflow, bg_file, closed_dir, + tags=bg_file.tags + ['closed_box']) table = wf.make_foreground_table(workflow, bg_file, hdfbank, open_dir, singles=insps, extension='.html', tags=bg_file.tags) @@ -560,6 +562,7 @@ splitbank_files_inj = wf.setup_splittable_workflow(workflow, [hdfbank], # setup the injection files inj_files_base, inj_tags = wf.setup_injection_workflow(workflow, output_dir="inj_files") + inj_files = [] for inj_file, tag in zip(inj_files_base, inj_tags): inj_files.append(wf.inj_to_hdf(workflow, inj_file, 'inj_files', [tag])) @@ -630,8 +633,28 @@ for inj_file, tag in zip(inj_files, inj_tags): inj_coinc[ordered_ifo_list] = curr_out + # get sngls for injections + for ifo in ifo_ids.keys(): + if not analyze_singles: + continue + inspcomb = wf.select_files_by_ifo_combination([ifo], insps) + # Create sngls tag, and set up the findtrigs job for the combination + ctagsngl = [tag, 'injections', '1det'] + inj_coinc[ifo] = wf.setup_sngls_inj( + workflow, + hdfbank, + inspcomb, + statfiles, + final_bg_files[ifo], + final_veto_file, + final_veto_name, + output_dir, + tags=ctagsngl + ) + combctags = [tag, 'injections'] final_inj_bg_file_list = wf.FileList(inj_coinc.values()) + combined_inj_bg_file = wf.setup_combine_statmap( workflow, final_inj_bg_file_list, @@ -726,7 +749,6 @@ for inj_file, tag in zip(inj_files, inj_tags): wf.make_throughput_plot(workflow, insps, rdir['workflow/throughput'], tags=[tag]) - ######################## Make combined injection plots ########################## if len(files_for_combined_injfind) > 0: sen_all = wf.make_sensitivity_plot(workflow, found_inj_comb, @@ -747,14 +769,12 @@ if len(files_for_combined_injfind) > 0: require='summ') inj_summ = list(layout.grouper(inj_s + sen_s, 2)) - # Make analysis time summary analysis_time_summ = [time_file, seg_summ_plot] for f in analysis_time_summ: symlink_result(f, 'analysis_time') layout.single_layout(rdir['analysis_time'], (analysis_time_summ)) - ########################## Make full summary #################################### if len(files_for_combined_injfind) > 0: summ = ([(time_file,)] + [(seg_summ_plot,)] + diff --git a/docs/apps.rst b/docs/apps.rst index d8c7aeb9257..df3fd0d1257 100644 --- a/docs/apps.rst +++ b/docs/apps.rst @@ -13,7 +13,7 @@ template banks) should read the documentation at: inference workflow/pycbc_make_psd_estimation_workflow - workflow/pycbc_make_coinc_search_workflow + workflow/pycbc_make_offline_search_workflow workflow/pygrb.rst tmpltbank hwinj diff --git a/docs/workflow/pycbc_make_coinc_search_workflow.rst b/docs/workflow/pycbc_make_offline_search_workflow.rst similarity index 98% rename from docs/workflow/pycbc_make_coinc_search_workflow.rst rename to docs/workflow/pycbc_make_offline_search_workflow.rst index 93f8f8ad9ac..3a195b9f640 100644 --- a/docs/workflow/pycbc_make_coinc_search_workflow.rst +++ b/docs/workflow/pycbc_make_offline_search_workflow.rst @@ -1,14 +1,14 @@ .. _search_workflow: #################################################################################### -``pycbc_make_coinc_search_workflow``: A workflow to search for gravitational waves +``pycbc_make_offline_search_workflow``: A workflow to search for gravitational waves #################################################################################### =============== Introduction =============== -The executable ``pycbc_make_coinc_search_workflow`` is a tool used to search +The executable ``pycbc_make_offline_search_workflow`` is a tool used to search for gravitational waves using data from a network of gravitational-wave detectors. It performs all of the necessary steps in the workflow, including data management, template bank construction (if required), matched filtering @@ -449,16 +449,16 @@ The rest of the config file concerns plotting formats Generating the workflow ======================= -The workflow is generated by running the script ``pycbc_make_coinc_search_workflow``. This program takes the command line arguments +The workflow is generated by running the script ``pycbc_make_offline_search_workflow``. This program takes the command line arguments -.. command-output:: pycbc_make_coinc_search_workflow --help +.. command-output:: pycbc_make_offline_search_workflow --help The configuration files can either be passes as local files, or given as URLs to specific configuration files managed for an analysis. For example, to generate a workflow to search two weeks of S6D data and place the results in your ``public_html`` directory, run the command:: - pycbc_make_coinc_search_workflow --workflow-name s6d_chunk3 --output-dir output \ + pycbc_make_offline_search_workflow --workflow-name s6d_chunk3 --output-dir output \ --config-files https://code.pycbc.phy.syr.edu/ligo-cbc/pycbc-config/download/master/S6/pipeline/s6_run_pycbc_er8_pre_release.ini \ https://code.pycbc.phy.syr.edu/ligo-cbc/pycbc-config/download/master/S6/pipeline/executables.ini \ https://code.pycbc.phy.syr.edu/ligo-cbc/pycbc-config/download/master/S6/pipeline/injections.ini \ @@ -495,7 +495,7 @@ determine the correct tags. These can be applied by adding the following line to your submit invocation. For example, to plan and submit the workflow in the example above, change to the directory that you specified with the ``--output`` -command line option to ``pycbc_make_coinc_search_workflow`` and plan and submit +command line option to ``pycbc_make_offline_search_workflow`` and plan and submit the workflow:: cd output @@ -621,7 +621,7 @@ Setting up a workflow for data reuse The first step is to generate a new workflow that performs the analysis that you would like to do. This workflow should be generated in a new directory so that it does not overwrite data from your previous workflows. Data reuse happens at the ``pycbc_submit_dax`` step, so -first run ``pycbc_make_coinc_search_workflow`` to build a new workflow, +first run ``pycbc_make_offline_search_workflow`` to build a new workflow, following the instructions in the section :ref:`coincworkflowgenerate` of this page. @@ -756,7 +756,7 @@ days. This assumes that: * There are no changes to the workflow configuration file, other than incrementing the end time of the workflow. -In this case, first re-run ``pycbc_make_coinc_search_workflow`` to build the +In this case, first re-run ``pycbc_make_offline_search_workflow`` to build the new workflow. The normal file retention level will copy a lot of reused data from the previous workflow directory into the new workflow directory. If you do not want to do this, use a ``--config-override`` to change the value of @@ -795,7 +795,7 @@ Data reuse can be used to re-running a workflow with a new veto definer file, as * The previous workflow completed successfully. * No changes to the configuration file are made, other than changing the ``segments-veto-definer-url`` in the ``[workflow-segments]`` section of the workflow configration file (although the GPS end time can also be extended at the same time, if necessary). -In this case, first re-run ``pycbc_make_coinc_search_workflow`` to build the +In this case, first re-run ``pycbc_make_offline_search_workflow`` to build the new workflow. The normal file retention level will copy a lot of reused data from the previous workflow directory into the new workflow directory. If you do not want to do this, use a ``--config-override`` to change the value of @@ -928,7 +928,7 @@ server). If you have LIGO.org credentials, you should execute:: export LIGO_DATAFIND_SERVER="datafind.ligo.org" -before you run ``pycbc_make_coinc_search_workflow``. Otherwise, contact your local system +before you run ``pycbc_make_offline_search_workflow``. Otherwise, contact your local system administrator for a valid datafind server that points to publicly available frame files. In order to run ``pycbc_inspiral`` on OSG worker nodes, it must be available @@ -936,14 +936,14 @@ in a Singularity container served from CVMFS. Releases of PyCBC build such conta and publish them to CVMFS, but the workflow needs to be told to only run on nodes that have Singularity available, and it needs the location of the ``pycbc_inspiral`` executable inside of the Singularity image. To do this, add the following to the ``--config-overrides`` -given to ``pycbc_make_coinc_search_workflow``:: +given to ``pycbc_make_offline_search_workflow``:: "pegasus_profile-inspiral:pycbc|site:osg" \ "pegasus_profile-inspiral:hints|execution.site:osg" \ "pegasus_profile-inspiral:condor|Requirements:(HAS_SINGULARITY =?= TRUE) && (IS_GLIDEIN =?= True)" \ "executables:inspiral:/bin/pycbc_inspiral" \ -These lines tell ``pycbc_make_coinc_search_workflow`` that the inspiral jobs will +These lines tell ``pycbc_make_offline_search_workflow`` that the inspiral jobs will run on the Open Science Grid, and that such jobs need to run at sites where Singularity is installed. They also tell them that the path to ``pycbc_inspiral`` inside the container is ``/bin/pycbc_inspiral``. If you are an LVK user, and you will be accessing non-public @@ -955,7 +955,7 @@ to read instead:: Because the data that the inspiral jobs will need in addition to the frame files must come from the submit machine, you also need a ``--config-overrides`` argument to -``pycbc_make_coinc_search_workflow`` that sets the staging site for the main workflow to be +``pycbc_make_offline_search_workflow`` that sets the staging site for the main workflow to be the local site. To do this, add the following argument, replacing ``${WORKFLOW_NAME}`` with the string that is given as the argument to the option ``--workflow-name``:: diff --git a/examples/search/analysis.ini b/examples/search/analysis.ini index 7a405001444..548b0e83508 100644 --- a/examples/search/analysis.ini +++ b/examples/search/analysis.ini @@ -167,14 +167,17 @@ f-lower = ${inspiral|low-frequency-cutoff} log-param = True smoothing-width = 0.4 -[coinc] -coinc-threshold = 0.002 +[workflow-singles] +analyze = + +[coinc&sngls] ranking-statistic = phasetd_exp_fit_fgbg_norm sngl-ranking = newsnr_sgveto_psdvar randomize-template-order = statistic-files = ${resolve:./statHL.hdf} ${resolve:./statLV.hdf} ${resolve:./statHV.hdf} ${resolve:./statHLV.hdf} [coinc-defaultvalues] +coinc-threshold = 0.002 timeslide-interval = 0.1 [coinc-full_data] @@ -192,14 +195,33 @@ loudest-keep-values = 15.0:9999999999999 [coinc-injinj] -[statmap] +[sngls] +trigger-cuts = newsnr:5.5:lower traditional_chisq:12:upper sigma_multiple:10:upper + +[statmap&sngls_statmap] max-hierarchical-removal = ${workflow-results|max-hierarchical-removal} hierarchical-removal-against = exclusive -[statmap&statmap_inj] +[statmap&statmap_inj&sngls_statmap&sngls_statmap_inj] veto-window = 0.100 cluster-window = 10.0 +[statmap] + +[sngls_statmap] + +[sngls_statmap&sngls_statmap_inj] +far-calculation-method = H1:trigger_fit L1:trigger_fit V1:trigger_fit +fit-function = H1:exponential L1:exponential V1:exponential +fit-threshold = H1:-10 L1:-8 V1:-12.5 + +[exclude_zerolag] + +[exclude_zerolag-1det] +far-calculation-method = ${sngls_statmap|far-calculation-method} +fit-function = ${sngls_statmap|fit-function} +fit-threshold = ${sngls_statmap|fit-threshold} + [combine_statmap] cluster-window = ${statmap|cluster-window} diff --git a/examples/search/executables.ini b/examples/search/executables.ini index 57fffb21d7c..4fd71bcfdff 100644 --- a/examples/search/executables.ini +++ b/examples/search/executables.ini @@ -40,6 +40,9 @@ plot_snrifar = ${which:pycbc_page_snrifar} plot_spectrum = ${which:pycbc_plot_psd_file} plot_throughput = ${which:pycbc_plot_throughput} results_page = ${which:pycbc_make_html_page} +sngls = ${which:pycbc_sngls_findtrigs} +sngls_statmap = ${which:pycbc_sngls_statmap} +sngls_statmap_inj = ${which:pycbc_sngls_statmap_inj} splitbank = ${which:pycbc_hdf5_splitbank} statmap = ${which:pycbc_coinc_statmap} statmap_inj = ${which:pycbc_coinc_statmap_inj} diff --git a/examples/search/gen.sh b/examples/search/gen.sh index 6a6a4cccd07..1c94e68e21d 100644 --- a/examples/search/gen.sh +++ b/examples/search/gen.sh @@ -1,7 +1,7 @@ #!/bin/bash set -e -pycbc_make_coinc_search_workflow \ +pycbc_make_offline_search_workflow \ --workflow-name gw \ --output-dir output \ --config-files analysis.ini plotting.ini executables.ini injections_minimal.ini \ diff --git a/pycbc/events/stat.py b/pycbc/events/stat.py index 7006fe88d89..c50ddd1aca0 100644 --- a/pycbc/events/stat.py +++ b/pycbc/events/stat.py @@ -121,7 +121,8 @@ def single(self, trigs): # pylint:disable=unused-argument err_msg += "sub-classes. You shouldn't be seeing this error!" raise NotImplementedError(err_msg) - def rank_stat_single(self, single_info): + def rank_stat_single(self, single_info, + **kwargs): # pylint:disable=unused-argument """ Calculate the statistic for a single detector candidate @@ -205,7 +206,8 @@ def single(self, trigs): """ return self.get_sngl_ranking(trigs) - def rank_stat_single(self, single_info): + def rank_stat_single(self, single_info, + **kwargs): # pylint:disable=unused-argument """ Calculate the statistic for a single detector candidate @@ -658,7 +660,8 @@ def single(self, trigs): singles['snr'] = trigs['snr'][:] return numpy.array(singles, ndmin=1) - def rank_stat_single(self, single_info): + def rank_stat_single(self, single_info, + **kwargs): # pylint:disable=unused-argument """ Calculate the statistic for a single detector candidate @@ -900,7 +903,8 @@ def single(self, trigs): return self.lognoiserate(trigs) - def rank_stat_single(self, single_info): + def rank_stat_single(self, single_info, + **kwargs): # pylint:disable=unused-argument """ Calculate the statistic for a single detector candidate @@ -1039,7 +1043,8 @@ def single(self, trigs): stat = thresh - (logr_n / self.alpharef) return numpy.array(stat, ndmin=1, dtype=numpy.float32) - def rank_stat_single(self, single_info): + def rank_stat_single(self, single_info, + **kwargs): # pylint:disable=unused-argument """ Calculate the statistic for single detector candidates @@ -1170,7 +1175,8 @@ def single(self, trigs): singles['snr'] = trigs['snr'][:] return numpy.array(singles, ndmin=1) - def rank_stat_single(self, single_info): + def rank_stat_single(self, single_info, + **kwargs): # pylint:disable=unused-argument """ Calculate the statistic for a single detector candidate @@ -1488,7 +1494,8 @@ def single(self, trigs): singles['benchmark_logvol'] = self.benchmark_logvol[tnum] return numpy.array(singles, ndmin=1) - def rank_stat_single(self, single_info): + def rank_stat_single(self, single_info, + **kwargs): # pylint:disable=unused-argument """ Calculate the statistic for single detector candidates diff --git a/pycbc/workflow/coincidence.py b/pycbc/workflow/coincidence.py index 00511b360b4..2e2942c9787 100644 --- a/pycbc/workflow/coincidence.py +++ b/pycbc/workflow/coincidence.py @@ -108,9 +108,31 @@ def create_node(self, trig_files, bank_file, stat_files, veto_file, node.new_output_file_opt(seg, '.hdf', '--output-file', tags=tags) return node +class PyCBCFindSnglsExecutable(Executable): + """Calculate single-detector ranking statistic for triggers""" + + current_retention_level = Executable.ALL_TRIGGERS + file_input_options = ['--statistic-files'] + def create_node(self, trig_files, bank_file, stat_files, veto_file, + veto_name, template_str, tags=None): + if tags is None: + tags = [] + segs = trig_files.get_times_covered_by_files() + seg = segments.segment(segs[0][0], segs[-1][1]) + node = Node(self) + node.add_input_opt('--template-bank', bank_file) + node.add_input_list_opt('--trigger-files', trig_files) + if len(stat_files) > 0: + node.add_input_list_opt('--statistic-files', stat_files) + if veto_file is not None: + node.add_input_opt('--veto-files', veto_file) + node.add_opt('--segment-name', veto_name) + node.add_opt('--template-fraction-range', template_str) + node.new_output_file_opt(seg, '.hdf', '--output-file', tags=tags) + return node class PyCBCStatMapExecutable(Executable): - """Calculate FAP, IFAR, etc""" + """Calculate FAP, IFAR, etc for coincs""" current_retention_level = Executable.MERGED_TRIGGERS def create_node(self, coinc_files, ifos, tags=None): @@ -125,9 +147,25 @@ def create_node(self, coinc_files, ifos, tags=None): node.new_output_file_opt(seg, '.hdf', '--output-file', tags=tags) return node +class PyCBCSnglsStatMapExecutable(Executable): + """Calculate FAP, IFAR, etc for singles""" + + current_retention_level = Executable.MERGED_TRIGGERS + def create_node(self, sngls_files, ifo, tags=None): + if tags is None: + tags = [] + segs = sngls_files.get_times_covered_by_files() + seg = segments.segment(segs[0][0], segs[-1][1]) + + node = Node(self) + node.add_input_list_opt('--sngls-files', sngls_files) + node.add_opt('--ifos', ifo) + node.new_output_file_opt(seg, '.hdf', '--output-file', tags=tags) + return node + class PyCBCStatMapInjExecutable(Executable): - """Calculate FAP, IFAR, etc""" + """Calculate FAP, IFAR, etc for coincs for injections""" current_retention_level = Executable.MERGED_TRIGGERS def create_node(self, zerolag, full_data, @@ -151,6 +189,25 @@ def create_node(self, zerolag, full_data, node.new_output_file_opt(seg, '.hdf', '--output-file', tags=tags) return node +class PyCBCSnglsStatMapInjExecutable(Executable): + """Calculate FAP, IFAR, etc for singles for injections""" + + current_retention_level = Executable.MERGED_TRIGGERS + def create_node(self, sngls_files, background_file, + ifos, tags=None): + if tags is None: + tags = [] + segs = sngls_files.get_times_covered_by_files() + seg = segments.segment(segs[0][0], segs[-1][1]) + + node = Node(self) + node.add_input_list_opt('--sngls-files', sngls_files) + node.add_input_opt('--full-data-background', background_file) + + node.add_opt('--ifos', ifos) + node.new_output_file_opt(seg, '.hdf', '--output-file', tags=tags) + return node + class PyCBCHDFInjFindExecutable(Executable): """Find injections in the hdf files output""" @@ -373,6 +430,18 @@ def setup_statmap(workflow, ifos, coinc_files, out_dir, tags=None): return stat_node.output_file +def setup_sngls_statmap(workflow, ifo, sngls_files, out_dir, tags=None): + tags = [] if tags is None else tags + + statmap_exe = PyCBCSnglsStatMapExecutable(workflow.cp, 'sngls_statmap', + ifos=ifo, + tags=tags, out_dir=out_dir) + + stat_node = statmap_exe.create_node(sngls_files, ifo) + workflow.add_node(stat_node) + return stat_node.output_file + + def setup_statmap_inj(workflow, ifos, coinc_files, background_file, out_dir, tags=None): tags = [] if tags is None else tags @@ -392,6 +461,23 @@ def setup_statmap_inj(workflow, ifos, coinc_files, background_file, return stat_node.output_files[0] +def setup_sngls_statmap_inj(workflow, ifo, sngls_inj_files, background_file, + out_dir, tags=None): + tags = [] if tags is None else tags + + statmap_exe = PyCBCSnglsStatMapInjExecutable(workflow.cp, + 'sngls_statmap_inj', + ifos=ifo, + tags=tags, + out_dir=out_dir) + + stat_node = statmap_exe.create_node(sngls_inj_files, + background_file, + ifo) + workflow.add_node(stat_node) + return stat_node.output_files[0] + + def setup_interval_coinc_inj(workflow, hdfbank, full_data_trig_files, inj_trig_files, stat_files, background_file, veto_file, veto_name, @@ -504,6 +590,74 @@ def setup_interval_coinc(workflow, hdfbank, trig_files, stat_files, return statmap_files +def setup_sngls(workflow, hdfbank, trig_files, stat_files, + veto_file, veto_name, out_dir, tags=None): + """ + This function sets up getting statistic values for single-detector triggers + """ + ifos, _ = trig_files.categorize_by_attr('ifo') + findsngls_exe = PyCBCFindSnglsExecutable(workflow.cp, 'sngls', ifos=ifos, + tags=tags, out_dir=out_dir) + # Wall time knob and memory knob + factor = int(workflow.cp.get_opt_tags('workflow-coincidence', + 'parallelization-factor', + [findsngls_exe.ifo_string] + tags)) + + statmap_files = [] + bg_files = FileList() + for i in range(factor): + group_str = '%s/%s' % (i, factor) + sngls_node = findsngls_exe.create_node(trig_files, hdfbank, + stat_files, + veto_file, veto_name, + group_str, + tags=['JOB'+str(i)]) + bg_files += sngls_node.output_files + workflow.add_node(sngls_node) + + statmap_files = setup_sngls_statmap(workflow, ifos[0], bg_files, + out_dir, tags=tags) + + logging.info('...leaving coincidence ') + return statmap_files + + +def setup_sngls_inj(workflow, hdfbank, inj_trig_files, + stat_files, background_file, veto_file, veto_name, + out_dir, tags=None): + """ + This function sets up getting statistic values for single-detector triggers + from injections + """ + ifos, _ = inj_trig_files.categorize_by_attr('ifo') + findsnglsinj_exe = PyCBCFindSnglsExecutable(workflow.cp, 'sngls', ifos=ifos, + tags=tags, out_dir=out_dir) + # Wall time knob and memory knob + exe_str_tags = [findsnglsinj_exe.ifo_string] + tags + factor = int(workflow.cp.get_opt_tags('workflow-coincidence', + 'parallelization-factor', + exe_str_tags)) + + statmap_files = [] + bg_files = FileList() + for i in range(factor): + group_str = '%s/%s' % (i, factor) + sngls_node = findsnglsinj_exe.create_node(inj_trig_files, hdfbank, + stat_files, + veto_file, veto_name, + group_str, + tags=['JOB'+str(i)]) + bg_files += sngls_node.output_files + workflow.add_node(sngls_node) + + statmap_files = setup_sngls_statmap_inj(workflow, ifos[0], bg_files, + background_file, + out_dir, tags=tags) + + logging.info('...leaving coincidence ') + return statmap_files + + def select_files_by_ifo_combination(ifocomb, insps): """ This function selects single-detector files ('insps') for a given ifo combination @@ -522,6 +676,10 @@ def get_ordered_ifo_list(ifocomb, ifo_ids): precedence list (ifo_ids dictionary) and returns the first ifo as pivot the second ifo as fixed, and the ordered list joined as a string. """ + if len(ifocomb) == 1: + # Single-detector combinations don't have fixed/pivot IFOs + return None, None, ifocomb[0] + # combination_prec stores precedence info for the detectors in the combination combination_prec = {ifo: ifo_ids[ifo] for ifo in ifocomb} ordered_ifo_list = sorted(combination_prec, key = combination_prec.get) diff --git a/tools/static/needs_full_build b/tools/static/needs_full_build index 92a4fb6ce9e..941d3e9cf4b 100644 --- a/tools/static/needs_full_build +++ b/tools/static/needs_full_build @@ -31,7 +31,7 @@ pycbc_page_segplot pycbc_plot_bank_bins pycbc_plot_singles_timefreq pycbc_plot_hist -pycbc_make_coinc_search_workflow +pycbc_make_offline_search_workflow pycbc_page_recovery pycbc_page_coinc_snrchi pycbc_page_injtable diff --git a/tools/test_coinc_search_workflow.sh b/tools/test_coinc_search_workflow.sh index fd7e28c6b55..1f1784853ed 100755 --- a/tools/test_coinc_search_workflow.sh +++ b/tools/test_coinc_search_workflow.sh @@ -1,6 +1,6 @@ #!/bin/bash set -e -echo -e "\\n>> [`date`] Testing pycbc_make_coinc_search_workflow" +echo -e "\\n>> [`date`] Testing pycbc_make_offline_search_workflow" VENV_PATH=${1} TRAVIS_TAG=${2} @@ -39,7 +39,7 @@ echo "DUMMY STAT FILE" > statHLV.hdf echo -e "\\n>> [`date`] Building test workflow $WORKFLOWNAME" -pycbc_make_coinc_search_workflow \ +pycbc_make_offline_search_workflow \ --workflow-name ${WORKFLOW_NAME} --output-dir output \ --config-files \ /pycbc/examples/search/analysis.ini \