pyhf_from_dataframe.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "cutflow_path = \"output/cuts_EventSelection-weighted.csv\"\n",
    "events_path = \"output/tbl_dataset.jet_pt_binned--JetPt.csv\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### show the cutflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Unnamed: 1</th>\n",
       "      <th>Unnamed: 2</th>\n",
       "      <th>passed_only_cut</th>\n",
       "      <th>passed_only_cut.1</th>\n",
       "      <th>passed_incl</th>\n",
       "      <th>passed_incl.1</th>\n",
       "      <th>totals_incl</th>\n",
       "      <th>totals_incl.1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>unweighted</td>\n",
       "      <td>weight</td>\n",
       "      <td>unweighted</td>\n",
       "      <td>weight</td>\n",
       "      <td>unweighted</td>\n",
       "      <td>weight</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>dataset</td>\n",
       "      <td>depth</td>\n",
       "      <td>cut</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Background</td>\n",
       "      <td>0</td>\n",
       "      <td>All</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Background</td>\n",
       "      <td>1</td>\n",
       "      <td>jet_pt &gt; 200</td>\n",
       "      <td>84508</td>\n",
       "      <td>84510.59613378995</td>\n",
       "      <td>84508</td>\n",
       "      <td>84510.59613378995</td>\n",
       "      <td>100000</td>\n",
       "      <td>100000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Background</td>\n",
       "      <td>1</td>\n",
       "      <td>jet_pt &lt; 800</td>\n",
       "      <td>97837</td>\n",
       "      <td>97834.59955586787</td>\n",
       "      <td>82345</td>\n",
       "      <td>82345.195689657805</td>\n",
       "      <td>84508</td>\n",
       "      <td>84510.59613378995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Signal</td>\n",
       "      <td>0</td>\n",
       "      <td>All</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Signal</td>\n",
       "      <td>1</td>\n",
       "      <td>jet_pt &gt; 200</td>\n",
       "      <td>100000</td>\n",
       "      <td>12500</td>\n",
       "      <td>100000</td>\n",
       "      <td>12500</td>\n",
       "      <td>100000</td>\n",
       "      <td>12500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Signal</td>\n",
       "      <td>1</td>\n",
       "      <td>jet_pt &lt; 800</td>\n",
       "      <td>99394</td>\n",
       "      <td>12424.158414084641</td>\n",
       "      <td>99394</td>\n",
       "      <td>12424.158414084641</td>\n",
       "      <td>100000</td>\n",
       "      <td>12500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Data</td>\n",
       "      <td>0</td>\n",
       "      <td>All</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Data</td>\n",
       "      <td>1</td>\n",
       "      <td>jet_pt &gt; 200</td>\n",
       "      <td>99478</td>\n",
       "      <td>99478</td>\n",
       "      <td>99478</td>\n",
       "      <td>99478</td>\n",
       "      <td>115000</td>\n",
       "      <td>115000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Data</td>\n",
       "      <td>1</td>\n",
       "      <td>jet_pt &lt; 800</td>\n",
       "      <td>112810</td>\n",
       "      <td>112810</td>\n",
       "      <td>97288</td>\n",
       "      <td>97288</td>\n",
       "      <td>99478</td>\n",
       "      <td>99478</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Unnamed: 0 Unnamed: 1    Unnamed: 2 passed_only_cut   passed_only_cut.1  \\\n",
       "0          NaN        NaN           NaN      unweighted              weight   \n",
       "1      dataset      depth           cut             NaN                 NaN   \n",
       "2   Background          0           All               0                   0   \n",
       "3   Background          1  jet_pt > 200           84508   84510.59613378995   \n",
       "4   Background          1  jet_pt < 800           97837   97834.59955586787   \n",
       "5       Signal          0           All               0                   0   \n",
       "6       Signal          1  jet_pt > 200          100000               12500   \n",
       "7       Signal          1  jet_pt < 800           99394  12424.158414084641   \n",
       "8         Data          0           All               0                   0   \n",
       "9         Data          1  jet_pt > 200           99478               99478   \n",
       "10        Data          1  jet_pt < 800          112810              112810   \n",
       "\n",
       "   passed_incl       passed_incl.1 totals_incl      totals_incl.1  \n",
       "0   unweighted              weight  unweighted             weight  \n",
       "1          NaN                 NaN         NaN                NaN  \n",
       "2            0                   0           0                  0  \n",
       "3        84508   84510.59613378995      100000             100000  \n",
       "4        82345  82345.195689657805       84508  84510.59613378995  \n",
       "5            0                   0           0                  0  \n",
       "6       100000               12500      100000              12500  \n",
       "7        99394  12424.158414084641      100000              12500  \n",
       "8            0                   0           0                  0  \n",
       "9        99478               99478      115000             115000  \n",
       "10       97288               97288       99478              99478  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cutflow = pd.read_csv(cutflow_path)\n",
    "cutflow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### events that passed the cut"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>dataset</th>\n",
       "      <th>jet_pt_binned</th>\n",
       "      <th>n</th>\n",
       "      <th>weighted:sumw</th>\n",
       "      <th>weighted:sumw2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Background</td>\n",
       "      <td>[-inf, 200.0)</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Background</td>\n",
       "      <td>[200.0, 300.0)</td>\n",
       "      <td>22318</td>\n",
       "      <td>22326.030978</td>\n",
       "      <td>22554.532085</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Background</td>\n",
       "      <td>[300.0, 400.0)</td>\n",
       "      <td>22801</td>\n",
       "      <td>22798.138083</td>\n",
       "      <td>23024.127428</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Background</td>\n",
       "      <td>[400.0, 500.0)</td>\n",
       "      <td>17148</td>\n",
       "      <td>17155.294128</td>\n",
       "      <td>17336.270468</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Background</td>\n",
       "      <td>[500.0, 600.0)</td>\n",
       "      <td>11215</td>\n",
       "      <td>11204.588788</td>\n",
       "      <td>11305.493463</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Background</td>\n",
       "      <td>[600.0, 700.0)</td>\n",
       "      <td>5939</td>\n",
       "      <td>5931.339805</td>\n",
       "      <td>5982.373416</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Background</td>\n",
       "      <td>[700.0, 800.0)</td>\n",
       "      <td>2924</td>\n",
       "      <td>2929.803908</td>\n",
       "      <td>2964.685214</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Background</td>\n",
       "      <td>[800.0, inf)</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Signal</td>\n",
       "      <td>[-inf, 200.0)</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Signal</td>\n",
       "      <td>[200.0, 300.0)</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Signal</td>\n",
       "      <td>[300.0, 400.0)</td>\n",
       "      <td>2618</td>\n",
       "      <td>328.277192</td>\n",
       "      <td>41.566276</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Signal</td>\n",
       "      <td>[400.0, 500.0)</td>\n",
       "      <td>38234</td>\n",
       "      <td>4780.356486</td>\n",
       "      <td>603.717933</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Signal</td>\n",
       "      <td>[500.0, 600.0)</td>\n",
       "      <td>39979</td>\n",
       "      <td>4996.195347</td>\n",
       "      <td>630.557479</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Signal</td>\n",
       "      <td>[600.0, 700.0)</td>\n",
       "      <td>15125</td>\n",
       "      <td>1888.882307</td>\n",
       "      <td>238.271410</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Signal</td>\n",
       "      <td>[700.0, 800.0)</td>\n",
       "      <td>3438</td>\n",
       "      <td>430.447081</td>\n",
       "      <td>54.441405</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Signal</td>\n",
       "      <td>[800.0, inf)</td>\n",
       "      <td>0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Data</td>\n",
       "      <td>[-inf, 200.0)</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Data</td>\n",
       "      <td>[200.0, 300.0)</td>\n",
       "      <td>22389</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Data</td>\n",
       "      <td>[300.0, 400.0)</td>\n",
       "      <td>23182</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Data</td>\n",
       "      <td>[400.0, 500.0)</td>\n",
       "      <td>23046</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Data</td>\n",
       "      <td>[500.0, 600.0)</td>\n",
       "      <td>17071</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Data</td>\n",
       "      <td>[600.0, 700.0)</td>\n",
       "      <td>8171</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Data</td>\n",
       "      <td>[700.0, 800.0)</td>\n",
       "      <td>3429</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Data</td>\n",
       "      <td>[800.0, inf)</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       dataset   jet_pt_binned      n  weighted:sumw  weighted:sumw2\n",
       "0   Background   [-inf, 200.0)      0       0.000000        0.000000\n",
       "1   Background  [200.0, 300.0)  22318   22326.030978    22554.532085\n",
       "2   Background  [300.0, 400.0)  22801   22798.138083    23024.127428\n",
       "3   Background  [400.0, 500.0)  17148   17155.294128    17336.270468\n",
       "4   Background  [500.0, 600.0)  11215   11204.588788    11305.493463\n",
       "5   Background  [600.0, 700.0)   5939    5931.339805     5982.373416\n",
       "6   Background  [700.0, 800.0)   2924    2929.803908     2964.685214\n",
       "7   Background    [800.0, inf)      0       0.000000        0.000000\n",
       "8       Signal   [-inf, 200.0)      0       0.000000        0.000000\n",
       "9       Signal  [200.0, 300.0)      0       0.000000        0.000000\n",
       "10      Signal  [300.0, 400.0)   2618     328.277192       41.566276\n",
       "11      Signal  [400.0, 500.0)  38234    4780.356486      603.717933\n",
       "12      Signal  [500.0, 600.0)  39979    4996.195347      630.557479\n",
       "13      Signal  [600.0, 700.0)  15125    1888.882307      238.271410\n",
       "14      Signal  [700.0, 800.0)   3438     430.447081       54.441405\n",
       "15      Signal    [800.0, inf)      0       0.000000        0.000000\n",
       "16        Data   [-inf, 200.0)      0            NaN             NaN\n",
       "17        Data  [200.0, 300.0)  22389            NaN             NaN\n",
       "18        Data  [300.0, 400.0)  23182            NaN             NaN\n",
       "19        Data  [400.0, 500.0)  23046            NaN             NaN\n",
       "20        Data  [500.0, 600.0)  17071            NaN             NaN\n",
       "21        Data  [600.0, 700.0)   8171            NaN             NaN\n",
       "22        Data  [700.0, 800.0)   3429            NaN             NaN\n",
       "23        Data    [800.0, inf)      0            NaN             NaN"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# read into dataframe\n",
    "df = pd.read_csv(events_path)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_yields_from_dataframe(dataset_name):\n",
    "    yield_column = \"n\" if (dataset_name == \"Data\") else \"weighted:sumw\"\n",
    "    yields = df.loc[df[\"dataset\"] == dataset_name, yield_column].astype(float).values\n",
    "    # remove under/overflow bins\n",
    "    yields = yields[1:-1]\n",
    "    return yields.tolist()\n",
    "\n",
    "def get_errors_from_dataframe(dataset_name):\n",
    "    error_column = \"weighted:sumw2\"\n",
    "    errors = np.sqrt(df.loc[df[\"dataset\"] == dataset_name, error_column].astype(float).values)\n",
    "    # remove under/overflow bins\n",
    "    errors = errors[1:-1]\n",
    "    return errors.tolist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### create the workspace"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "regions = [\"Signal_region\"]\n",
    "samplenames = [\"Background\", \"Signal\"]\n",
    "measurements_list = [\"minimal_example\"]\n",
    "\n",
    "ws = {} # the workspace\n",
    "\n",
    "# channels\n",
    "channels = []\n",
    "for reg in regions:\n",
    "    channel = {}\n",
    "    channel.update({\"name\": reg})\n",
    "    samples = []\n",
    "    for sam in samplenames:\n",
    "        current_sample = {}\n",
    "        current_sample.update({\"name\": sam})\n",
    "        current_sample.update({\"data\": get_yields_from_dataframe(sam)})\n",
    "        \n",
    "        # build all the modifiers\n",
    "        modifiers = [{}, {}, {}]\n",
    "        lumi = {\"data\": None,\n",
    "                \"name\": \"lumi\",\n",
    "                \"type\": \"lumi\"}\n",
    "        gammas = {\"data\": get_errors_from_dataframe(sam),\n",
    "                  \"type\": \"staterror\",\n",
    "                  \"name\": \"staterror_\" + reg}\n",
    "        custom_lumi = {\"data\": {\"hi\": 1.02,\n",
    "                                \"lo\": 0.98},\n",
    "                       \"name\": \"Luminosity\",\n",
    "                       \"type\": \"normsys\"}\n",
    "        modifiers = [lumi, gammas, custom_lumi]\n",
    "        if sam == \"Signal\":\n",
    "            # add the normalization uncertainty for signal\n",
    "            modifiers.append({\"data\": None, \n",
    "                              \"name\": \"Signal_norm\",\n",
    "                              \"type\": \"normfactor\"})\n",
    "        \n",
    "        current_sample.update({\"modifiers\": modifiers})\n",
    "        samples.append(current_sample)\n",
    "    channel.update({\"samples\": samples})\n",
    "    channels.append(channel)\n",
    "\n",
    "ws.update({\"channels\": channels})\n",
    "\n",
    "# measurements\n",
    "measurements = []\n",
    "for m in measurements_list:\n",
    "    measurement = {}\n",
    "    measurement.update({\"name\": m})\n",
    "    config = {}\n",
    "    parameters = {\"parameters\": [{\"auxdata\": [1.0],\n",
    "                                  \"bounds\": [[0.5, 1.5]],\n",
    "                                  \"fixed\": True,\n",
    "                                  \"inits\": [1.0],\n",
    "                                  \"name\": \"lumi\",\n",
    "                                  \"sigmas\": [0.1]}]}\n",
    "    config.update(parameters)\n",
    "    config.update({\"poi\": \"Signal_norm\"})\n",
    "    measurement.update({\"config\": config})\n",
    "    measurements.append(measurement)\n",
    "ws.update({\"measurements\": measurements})\n",
    "\n",
    "# observations\n",
    "observations = []\n",
    "observation = {}\n",
    "for reg in regions:\n",
    "    observation.update({\"name\": reg})\n",
    "    observation.update({\"data\": get_yields_from_dataframe(\"Data\")})\n",
    "    observations.append(observation)\n",
    "ws.update({\"observations\": observations})\n",
    "\n",
    "# version\n",
    "ws.update({\"version\": \"1.0.0\"})\n",
    "\n",
    "# save to file\n",
    "with open(\"workspace_from_dataframe.json\", \"w\") as f:\n",
    "    json.dump(ws, f, sort_keys=True, indent=4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The resulting workspace should be identical to the one obtained when converting the `.xml` workspace produced by TRExFitter."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (fasthep)",
   "language": "python",
   "name": "fasthep"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}