cancer_evaluate_markers.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.metrics import roc_auc_score, roc_curve\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from artifact_detector_model import MARKER_NAMES\n",
    "plt.rcParams[\"font.family\"] = \"Serif\"\n",
    "from sklearn.metrics import confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_markers = pd.read_csv(\"predicted_all_embed.csv\")[[\"image_path\"] + MARKER_NAMES]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_main = pd.read_csv(\"/vol/biomedic3/data/EMBED/tables/mammo-net-csv/embed-non-negative.csv\")\n",
    "output_dir = \"output/non-negative-baseline/version_0\"\n",
    "print(len(df_main))\n",
    "df_main[\"image_id\"] = df_main[\"image_path\"].apply(\n",
    "    lambda img_path: img_path.split(\"/\")[-1]\n",
    ")\n",
    "df_markers[\"image_id\"] = df_markers[\"image_path\"].apply(\n",
    "    lambda img_path: img_path.split(\"/\")[-1]\n",
    ")\n",
    "df_main.drop(columns=\"image_path\", inplace=True)\n",
    "df_full = df_main.merge(df_markers, how=\"inner\")\n",
    "print(len(df_full))\n",
    "df_full = df_full.loc[df_full[\"compression\"] == 0]\n",
    "print(len(df_full))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions = pd.read_csv(output_dir + \"/predictions.csv\")\n",
    "predictions[\"image_id\"] = predictions[\"image_path\"].apply(\n",
    "    lambda img_path: img_path.split(\"/\")[-1]\n",
    ")\n",
    "img_ids = predictions[\"image_id\"].values\n",
    "preds = predictions[\"probability\"].values\n",
    "targets = predictions[\"label\"].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "circle_image_id = df_full.loc[\n",
    "    (df_full[\"image_id\"].isin(img_ids)) & (df_full[\"circle marker\"] == 1), \"image_id\"\n",
    "]\n",
    "circle_idx = np.where([img_id in circle_image_id.values for img_id in img_ids])[0]\n",
    "triangle_image_id = df_full.loc[\n",
    "    (df_full[\"image_id\"].isin(img_ids)) & (df_full[\"triangle marker\"] == 1), \"image_id\"\n",
    "]\n",
    "triangle_idx = np.where([img_id in triangle_image_id.values for img_id in img_ids])[0]\n",
    "pacemaker_image_id = df_full.loc[\n",
    "    (df_full[\"image_id\"].isin(img_ids)) & (df_full[\"devices\"] == 1), \"image_id\"\n",
    "]\n",
    "pacemaker_idx = np.where([img_id in pacemaker_image_id.values for img_id in img_ids])[0]\n",
    "breast_implant_image_id = df_full.loc[\n",
    "    (df_full[\"image_id\"].isin(img_ids)) & (df_full[\"breast implant\"] == 1), \"image_id\"\n",
    "]\n",
    "breast_implant_idx = np.where(\n",
    "    [img_id in breast_implant_image_id.values for img_id in img_ids]\n",
    ")[0]\n",
    "compression_image_id = df_full.loc[\n",
    "    (df_full[\"image_id\"].isin(img_ids)) & (df_full[\"compression\"] == 1), \"image_id\"\n",
    "]\n",
    "compression_idx = np.where(\n",
    "    [img_id in compression_image_id.values for img_id in img_ids]\n",
    ")[0]\n",
    "normal_image_id = df_full.loc[\n",
    "    (df_full[\"image_id\"].isin(img_ids))\n",
    "    & (df_full[\"compression\"] == 0)\n",
    "    & (df_full[\"devices\"] == 0)\n",
    "    & (df_full[\"circle marker\"] == 0)\n",
    "    & (df_full[\"triangle marker\"] == 0)\n",
    "    & (df_full[\"breast implant\"] == 0),\n",
    "    \"image_id\",\n",
    "]\n",
    "normal_idx = np.where([img_id in normal_image_id.values for img_id in img_ids])[0]\n",
    "circle_idx.shape, triangle_idx.shape, pacemaker_idx.shape, breast_implant_idx.shape, compression_idx.shape, normal_idx.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(f\"All: {roc_auc_score(targets, preds):.3f}\")\n",
    "print(\n",
    "    f\"Normal images: {roc_auc_score(targets[normal_idx], preds[normal_idx]):.3f} ({targets[normal_idx].sum()} positives out of {targets[normal_idx].shape[0]})\"\n",
    ")\n",
    "print(\n",
    "    f\"Images with circle: {roc_auc_score(targets[circle_idx], preds[circle_idx]):.3f} ({targets[circle_idx].sum()} positives out of {targets[circle_idx].shape[0]})\"\n",
    ")\n",
    "print(\n",
    "    f\"Images with triangle: {roc_auc_score(targets[triangle_idx], preds[triangle_idx]):.3f} ({targets[triangle_idx].sum()} positives out of {targets[triangle_idx].shape[0]})\"\n",
    ")\n",
    "print(\n",
    "    f\"Images with implant: {roc_auc_score(targets[breast_implant_idx], preds[breast_implant_idx]):.3f} ({targets[breast_implant_idx].sum()} positives out of {targets[breast_implant_idx].shape[0]})\"\n",
    ")\n",
    "print(\n",
    "    f\"Pacemaker: {roc_auc_score(targets[pacemaker_idx], preds[pacemaker_idx]):.3f} ({targets[pacemaker_idx].sum()} positives out of {targets[pacemaker_idx].shape[0]})\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "predictions_full = predictions.merge(df_markers, on ='image_id')\n",
    "predictions_full[\"no marker\"] = (\n",
    "    (predictions_full[\"circle marker\"] == 0)\n",
    "    & (predictions_full[\"triangle marker\"] == 0)\n",
    "    & (predictions_full[\"breast implant\"] == 0)\n",
    "    & (predictions_full[\"devices\"] == 0)\n",
    ").astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for marker_col in MARKER_NAMES[:-1]:\n",
    "    f, ax = plt.subplots(1, 2, figsize=(7, 3))\n",
    "    for i in range(2):\n",
    "        df = predictions_full.loc[(predictions_full[\"no marker\"] == 1) | (predictions_full[marker_col] == 1)]\n",
    "        df.loc[df[marker_col] == 1, 'Artefact'] = marker_col.capitalize()\n",
    "        df.loc[df[marker_col] == 0, 'Artefact'] = 'No artefact'\n",
    "        df['Model output'] = df['probability']\n",
    "        g = sns.histplot(\n",
    "            data=df.loc[df.label == i].sort_values(by=marker_col),\n",
    "            x='Model output',\n",
    "            hue='Artefact',\n",
    "            common_norm=False,\n",
    "            stat=\"density\",\n",
    "            ax=ax[i],\n",
    "            bins=25,\n",
    "            kde=True,\n",
    "        )\n",
    "        g.legend_.set_title(None)\n",
    "        # handles, labels = ax[i].get_legend_handles_labels()\n",
    "        # ax[i].legend(handles=handles[1:], labels=labels[1:])\n",
    "        if marker_col == 'triangle marker':\n",
    "            ax[i].set_title(f\"Cancer screening status:\\nno finding\" if i == 0 else \"Cancer screening status:\\nnon negative\")\n",
    "        if i == 1:\n",
    "            ax[i].set_ylabel(\"\")\n",
    "    plt.savefig(f\"output/distribution_cancer_{marker_col}.pdf\", bbox_inches=\"tight\")\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_op_threshold(true, pred, operating_point=\"diag\"):\n",
    "    fpr, tpr, threshold = roc_curve(true, pred)\n",
    "    if operating_point == \"diag\":\n",
    "        op = np.argmin(np.abs(tpr - (1 - fpr)))\n",
    "    elif operating_point == \"spec90\":\n",
    "        op = np.argmin(np.abs(fpr - 0.10))\n",
    "    else:\n",
    "        raise ValueError(\"Operating point has to be diag, spec90\")\n",
    "    return threshold[op]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "op_global = get_op_threshold(targets, preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "plt.rcParams[\"font.family\"] = \"Serif\"\n",
    "f, ax = plt.subplots(2, 3, figsize=(8, 5), facecolor=\"none\")\n",
    "f.subplots_adjust(hspace=0.4, wspace=0.3)\n",
    "ax = ax.ravel()\n",
    "results = {\n",
    "    'all': [np.arange(targets.shape[0]), \"All images\"],\n",
    "    'no markers': [normal_idx, \"No markers\"],\n",
    "    'circle': [circle_idx, \"Circle markers\"],\n",
    "    'triangles': [triangle_idx, \"Triangle markers\"],\n",
    "    'implants': [breast_implant_idx, \"Breast implants\"],\n",
    "    'devices': [pacemaker_idx, \"Devices\"]\n",
    "}\n",
    "for i, (select_idx, title) in enumerate(results.values()):\n",
    "    cf_matrix = confusion_matrix(targets[select_idx], preds[select_idx] > op_global)\n",
    "    group_counts = [f\"{value:0.0f}\" for value in cf_matrix.flatten()]\n",
    "    group_percentages = [\n",
    "        f\"{value:.2%}\"\n",
    "        for value in (cf_matrix / np.sum(cf_matrix, 1, keepdims=True)).flatten()\n",
    "    ]\n",
    "    labels = [f\"{v3}\\n(N={v2})\" for v2, v3 in zip(group_counts, group_percentages)]\n",
    "    labels = np.asarray(labels).reshape(2, 2)\n",
    "    print(labels)\n",
    "    sns.heatmap(\n",
    "        (cf_matrix / np.sum(cf_matrix, 1, keepdims=True)),\n",
    "        annot=labels,\n",
    "        fmt=\"\",\n",
    "        cmap=\"Blues\",\n",
    "        ax=ax[i],\n",
    "        vmin=0,\n",
    "        vmax=1,\n",
    "        cbar=False,\n",
    "    )\n",
    "    ax[i].set_title(' '.join([r'$\\bf{' + t +'}$' for t in title.split(' ')]), fontsize=14)\n",
    "    # ax[i].set_title(title, fontsize=15)\n",
    "# [ax[i].get_images()[0].set_clim(0, 1) for i in range(5)]\n",
    "[ax[i].set_xlabel('') for i in range(3)]\n",
    "[ax[i].set_ylabel('') for i in [1, 2, 4, 5]]\n",
    "[ax[i].set_ylabel('True label', fontsize=14) for i in [0, 3]]\n",
    "[ax[i].set_xlabel('Predicted label', fontsize=14) for i in [3,4,5]]\n",
    "\n",
    "plt.savefig(\"output/confusion_cancer.pdf\", bbox_inches=\"tight\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}