result.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from pathlib import Path\n",
    "from collections import defaultdict\n",
    "import re\n",
    "import numpy as np\n",
    "# HOME = Path.cwd().parent\n",
    "HOME = Path('.')\n",
    "workspace = HOME / 'workspaces'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Helper functions\n",
    "def format_scores(scores, show_max_min: bool = True):\n",
    "    mean = np.nanmean(scores)\n",
    "    std = np.std(scores)\n",
    "    if show_max_min:\n",
    "        min_ = np.min(scores)\n",
    "        max_ = np.max(scores)\n",
    "        return f'{mean:.0f}±{std:.0f}'\n",
    "    else:\n",
    "        return f'{mean:.0f}±{std:.0f}'\n",
    "\n",
    "def format_df(res_dict, agg_fn):\n",
    "    new_res = {\n",
    "        k: {\n",
    "            key: agg_fn(val) for key, val in v.items()\n",
    "        } for k, v in res_dict.items()\n",
    "    }\n",
    "    return pd.DataFrame(new_res)\n",
    "\n",
    "def extract_scores(log_filename):\n",
    "    with open(log_filename, 'r') as f:\n",
    "        last_line = f.read().splitlines()[-1]\n",
    "    scores = {}\n",
    "    try:\n",
    "        rmse, mae, mape = re.findall(r'[^:]+: (\\d+\\.\\d+)', last_line)\n",
    "    except:\n",
    "        print(log_filename, last_line)\n",
    "        raise\n",
    "    scores['RMSE'] = float(rmse) \n",
    "    scores['MAE'] = float(mae)\n",
    "    scores['MAPE'] = float(mape) * 100\n",
    "    return scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "#workspace\n",
    "datasets_to_use = ['clo','cruh','crush','snl','matr_1', 'matr_2', 'hust', 'mix']\n",
    "sklearn_baseline_names = [\n",
    "    'dummy',\n",
    "    'variance_model',\n",
    "    'discharge_model',\n",
    "    'full_model',\n",
    "    'ridge',\n",
    "    'pcr',\n",
    "    'plsr',\n",
    "    'gpr',\n",
    "    'xgb',\n",
    "    'rf',\n",
    "]\n",
    "nn_baseline_names = ['mlp', 'cnn', 'lstm', 'transformer']\n",
    "\n",
    "def collect_results(dataset):\n",
    "    sklearn_results = [\n",
    "        sklearn_baselines[dataset][name] for name in sklearn_baseline_names\n",
    "    ]\n",
    "    nn_results = [\n",
    "        nn_baselines[dataset][name] for name in nn_baseline_names\n",
    "    ]\n",
    "    # our_results = [ours[dataset]['Ours']]\n",
    "    return sklearn_results + nn_results\n",
    "    # return sklearn_results + nn_results + our_results\n",
    "\n",
    "sklearn_baselines = defaultdict(dict)\n",
    "for method_path in workspace.glob('baselines/sklearn/*'):\n",
    "    method = method_path.name\n",
    "    for dataset_res in method_path.glob('*'):\n",
    "        dataset = dataset_res.name\n",
    "        score = extract_scores(dataset_res / 'log.0')['RMSE']\n",
    "        sklearn_baselines[dataset][method] = f'{int(score)}'\n",
    "\n",
    "\n",
    "    \n",
    "\n",
    "nn_baselines = defaultdict(dict)\n",
    "for method_path in workspace.glob('baselines/nn_models/*'):\n",
    "    method = method_path.name\n",
    "    for dataset_res in method_path.glob('*'):\n",
    "        dataset = dataset_res.name\n",
    "        scores = []\n",
    "        for i in range(10):\n",
    "            try:\n",
    "                score = extract_scores(dataset_res / f'log.{i}')['RMSE']\n",
    "            except:\n",
    "                score = None\n",
    "            scores.append(score)\n",
    "        try:\n",
    "            nn_baselines[dataset][method] = format_scores(scores)\n",
    "        except:\n",
    "            nn_baselines[dataset][method] = None\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MATR-1</th>\n",
       "      <th>MATR-2</th>\n",
       "      <th>HUST</th>\n",
       "      <th>SNL</th>\n",
       "      <th>CLO</th>\n",
       "      <th>CRUH</th>\n",
       "      <th>CRUSH</th>\n",
       "      <th>MIX</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Training Mean</th>\n",
       "      <td>398</td>\n",
       "      <td>510</td>\n",
       "      <td>419</td>\n",
       "      <td>466</td>\n",
       "      <td>331</td>\n",
       "      <td>239</td>\n",
       "      <td>576</td>\n",
       "      <td>573</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>``Variance'' Model</th>\n",
       "      <td>136</td>\n",
       "      <td>211</td>\n",
       "      <td>398</td>\n",
       "      <td>360</td>\n",
       "      <td>179</td>\n",
       "      <td>118</td>\n",
       "      <td>506</td>\n",
       "      <td>521</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>``Discharge'' Model</th>\n",
       "      <td>329</td>\n",
       "      <td>148</td>\n",
       "      <td>321</td>\n",
       "      <td>266</td>\n",
       "      <td>143</td>\n",
       "      <td>76</td>\n",
       "      <td>13514</td>\n",
       "      <td>1743</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>``Full'' Model</th>\n",
       "      <td>166</td>\n",
       "      <td>1074</td>\n",
       "      <td>335</td>\n",
       "      <td>433</td>\n",
       "      <td>138</td>\n",
       "      <td>93</td>\n",
       "      <td>1051</td>\n",
       "      <td>330</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Ridge Regression</th>\n",
       "      <td>115</td>\n",
       "      <td>183</td>\n",
       "      <td>54194</td>\n",
       "      <td>242</td>\n",
       "      <td>169</td>\n",
       "      <td>65</td>\n",
       "      <td>39804073</td>\n",
       "      <td>372</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PCR</th>\n",
       "      <td>123</td>\n",
       "      <td>197</td>\n",
       "      <td>434</td>\n",
       "      <td>199</td>\n",
       "      <td>197</td>\n",
       "      <td>68</td>\n",
       "      <td>559</td>\n",
       "      <td>376</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PLSR</th>\n",
       "      <td>103</td>\n",
       "      <td>180</td>\n",
       "      <td>431</td>\n",
       "      <td>241</td>\n",
       "      <td>176</td>\n",
       "      <td>60</td>\n",
       "      <td>535</td>\n",
       "      <td>383</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Guassian process</th>\n",
       "      <td>153</td>\n",
       "      <td>223</td>\n",
       "      <td>54202</td>\n",
       "      <td>250</td>\n",
       "      <td>203</td>\n",
       "      <td>115</td>\n",
       "      <td>42861399732314</td>\n",
       "      <td>573</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>XGBoost</th>\n",
       "      <td>333</td>\n",
       "      <td>798</td>\n",
       "      <td>394</td>\n",
       "      <td>547</td>\n",
       "      <td>214</td>\n",
       "      <td>119</td>\n",
       "      <td>330</td>\n",
       "      <td>205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Random Forest</th>\n",
       "      <td>164</td>\n",
       "      <td>231</td>\n",
       "      <td>358</td>\n",
       "      <td>558</td>\n",
       "      <td>192</td>\n",
       "      <td>81</td>\n",
       "      <td>416</td>\n",
       "      <td>197</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MLP</th>\n",
       "      <td>149±3</td>\n",
       "      <td>275±26</td>\n",
       "      <td>457±8</td>\n",
       "      <td>386±82</td>\n",
       "      <td>160±7</td>\n",
       "      <td>103±4</td>\n",
       "      <td>565±9</td>\n",
       "      <td>451±42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CNN</th>\n",
       "      <td>105±88</td>\n",
       "      <td>225±99</td>\n",
       "      <td>491±91</td>\n",
       "      <td>864±174</td>\n",
       "      <td>342±499</td>\n",
       "      <td>174±92</td>\n",
       "      <td>545±11</td>\n",
       "      <td>272±101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LSTM</th>\n",
       "      <td>113±12</td>\n",
       "      <td>227±31</td>\n",
       "      <td>446±27</td>\n",
       "      <td>535±31</td>\n",
       "      <td>204±13</td>\n",
       "      <td>105±10</td>\n",
       "      <td>519±39</td>\n",
       "      <td>268±9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Transformer</th>\n",
       "      <td>89±16</td>\n",
       "      <td>233±36</td>\n",
       "      <td>412±18</td>\n",
       "      <td>326±23</td>\n",
       "      <td>181±23</td>\n",
       "      <td>81±8</td>\n",
       "      <td>550±21</td>\n",
       "      <td>271±16</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     MATR-1  MATR-2    HUST      SNL      CLO    CRUH  \\\n",
       "Training Mean           398     510     419      466      331     239   \n",
       "``Variance'' Model      136     211     398      360      179     118   \n",
       "``Discharge'' Model     329     148     321      266      143      76   \n",
       "``Full'' Model          166    1074     335      433      138      93   \n",
       "Ridge Regression        115     183   54194      242      169      65   \n",
       "PCR                     123     197     434      199      197      68   \n",
       "PLSR                    103     180     431      241      176      60   \n",
       "Guassian process        153     223   54202      250      203     115   \n",
       "XGBoost                 333     798     394      547      214     119   \n",
       "Random Forest           164     231     358      558      192      81   \n",
       "MLP                   149±3  275±26   457±8   386±82    160±7   103±4   \n",
       "CNN                  105±88  225±99  491±91  864±174  342±499  174±92   \n",
       "LSTM                 113±12  227±31  446±27   535±31   204±13  105±10   \n",
       "Transformer           89±16  233±36  412±18   326±23   181±23    81±8   \n",
       "\n",
       "                              CRUSH      MIX  \n",
       "Training Mean                   576      573  \n",
       "``Variance'' Model              506      521  \n",
       "``Discharge'' Model           13514     1743  \n",
       "``Full'' Model                 1051      330  \n",
       "Ridge Regression           39804073      372  \n",
       "PCR                             559      376  \n",
       "PLSR                            535      383  \n",
       "Guassian process     42861399732314      573  \n",
       "XGBoost                         330      205  \n",
       "Random Forest                   416      197  \n",
       "MLP                           565±9   451±42  \n",
       "CNN                          545±11  272±101  \n",
       "LSTM                         519±39    268±9  \n",
       "Transformer                  550±21   271±16  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# new\n",
    "main_table_df = pd.DataFrame({\n",
    "    dataset: collect_results(dataset) for dataset in datasets_to_use\n",
    "}, index=sklearn_baseline_names+nn_baseline_names)\n",
    "\n",
    "\n",
    "main_table_df = main_table_df.rename(columns={\n",
    "    'snl': 'SNL',\n",
    "    'clo': 'CLO',\n",
    "    'hust': 'HUST',\n",
    "    'matr_1': 'MATR-1',\n",
    "    'matr_2': 'MATR-2',\n",
    "    'cruh': 'CRUH',\n",
    "    'crush': 'CRUSH',\n",
    "    'mix': 'MIX'\n",
    "}, index={\n",
    "    'dummy': 'Training Mean',\n",
    "    'variance_model': '``Variance\\'\\' Model',\n",
    "    'discharge_model': '``Discharge\\'\\' Model',\n",
    "    'full_model': '``Full\\'\\' Model',\n",
    "    'ridge': 'Ridge Regression',\n",
    "    'pcr': 'PCR',\n",
    "    'plsr': 'PLSR',\n",
    "    'gpr': 'Guassian process',\n",
    "    'xgb': 'XGBoost',\n",
    "    'rf': 'Random Forest',\n",
    "    'mlp': 'MLP',\n",
    "    'cnn': 'CNN',\n",
    "    'lstm': 'LSTM',\n",
    "    'transformer': 'Transformer',\n",
    "})[['MATR-1', 'MATR-2', 'HUST','SNL', 'CLO',  'CRUH', 'CRUSH', 'MIX']]\n",
    "main_table_df"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "test",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}