august_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2.0.3'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from glob import glob\n",
    "from IPython.display import display, Markdown\n",
    "from tqdm import tqdm\n",
    "import csv\n",
    "import re\n",
    "import os\n",
    "pd.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "156777637it [02:03, 1274158.50it/s]\n"
     ]
    }
   ],
   "source": [
    "sabs = set()\n",
    "with open(\"dd_data/20230802/neo4j/import/CUI-CUIs.csv\") as o:\n",
    "    csv_reader = csv.reader(o)\n",
    "    headers = None\n",
    "    for row in tqdm(csv_reader):\n",
    "        if not headers:\n",
    "            headers = row\n",
    "        else:\n",
    "            sabs.add(row[3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"output/august_sabs.txt\", \"w\") as o:\n",
    "    o.write(\"\\n\".join(sorted(sabs)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "116"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open(\"output/august_sabs.txt\") as o:\n",
    "    dcc_sabs = o.read().strip().split(\"\\n\")\n",
    "len(dcc_sabs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Nodes\n",
    "## Concept"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>C0000097</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C0000359</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C0000610</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C0000739</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C0000873</th>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: [C0000097, C0000359, C0000610, C0000739, C0000873]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concepts = pd.read_csv(\"dd_data/20230802/neo4j/import/CUIs.csv\")\n",
    "concepts = pd.DataFrame(index=concepts[\"CUI:ID\"].unique())\n",
    "concepts.index.name = \"id\"\n",
    "concepts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(15527671, 0)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concepts.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Semantics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>STN</th>\n",
       "      <th>DEF</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TUI:ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>T012</th>\n",
       "      <td>Bird</td>\n",
       "      <td>A1.1.3.1.1.2</td>\n",
       "      <td>A vertebrate having a constant body temperatur...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T014</th>\n",
       "      <td>Reptile</td>\n",
       "      <td>A1.1.3.1.1.5</td>\n",
       "      <td>A cold-blooded vertebrate having an external c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T015</th>\n",
       "      <td>Mammal</td>\n",
       "      <td>A1.1.3.1.1.4</td>\n",
       "      <td>A vertebrate having a constant body temperatur...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T021</th>\n",
       "      <td>Fully Formed Anatomical Structure</td>\n",
       "      <td>A1.2.3</td>\n",
       "      <td>An anatomical structure in a fully formed orga...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T022</th>\n",
       "      <td>Body System</td>\n",
       "      <td>A2.1.4.1</td>\n",
       "      <td>A complex of anatomical structures that perfor...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     name           STN  \\\n",
       "TUI:ID                                                    \n",
       "T012                                 Bird  A1.1.3.1.1.2   \n",
       "T014                              Reptile  A1.1.3.1.1.5   \n",
       "T015                               Mammal  A1.1.3.1.1.4   \n",
       "T021    Fully Formed Anatomical Structure        A1.2.3   \n",
       "T022                          Body System      A2.1.4.1   \n",
       "\n",
       "                                                      DEF  \n",
       "TUI:ID                                                     \n",
       "T012    A vertebrate having a constant body temperatur...  \n",
       "T014    A cold-blooded vertebrate having an external c...  \n",
       "T015    A vertebrate having a constant body temperatur...  \n",
       "T021    An anatomical structure in a fully formed orga...  \n",
       "T022    A complex of anatomical structures that perfor...  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "semantics = pd.read_csv(\"dd_data/20230802/neo4j/import/TUIs.csv\", index_col=0)\n",
    "semantics.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>:START_ID</th>\n",
       "      <th>:END_ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C0000132</td>\n",
       "      <td>T126</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C0000246</td>\n",
       "      <td>T116</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C0000895</td>\n",
       "      <td>T060</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C0000908</td>\n",
       "      <td>T037</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C0000931</td>\n",
       "      <td>T067</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  :START_ID :END_ID\n",
       "0  C0000132    T126\n",
       "1  C0000246    T116\n",
       "2  C0000895    T060\n",
       "3  C0000908    T037\n",
       "4  C0000931    T067"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concept_semantics = pd.read_csv(\"dd_data/20230802/neo4j/import/CUI-TUIs.csv\")\n",
    "concept_semantics.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12264238"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "no_semantics = set(concepts.index) - set(concept_semantics[':START_ID'])\n",
    "len(no_semantics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('out/0623/semantics_ranked.tsv') as o:\n",
    "    ranked_type = [i.strip() for i in o.read().strip().split(\"\\n\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>TUI:ID</th>\n",
       "      <th>type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C0000132</td>\n",
       "      <td>T126</td>\n",
       "      <td>Enzyme</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C0000246</td>\n",
       "      <td>T116</td>\n",
       "      <td>Amino Acid, Peptide, or Protein</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C0000895</td>\n",
       "      <td>T060</td>\n",
       "      <td>Diagnostic Procedure</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C0000908</td>\n",
       "      <td>T037</td>\n",
       "      <td>Injury or Poisoning</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C0000931</td>\n",
       "      <td>T067</td>\n",
       "      <td>Phenomenon or Process</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         id TUI:ID                             type\n",
       "0  C0000132   T126                           Enzyme\n",
       "1  C0000246   T116  Amino Acid, Peptide, or Protein\n",
       "2  C0000895   T060             Diagnostic Procedure\n",
       "3  C0000908   T037              Injury or Poisoning\n",
       "4  C0000931   T067            Phenomenon or Process"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concept_semantics.columns = [\"id\", \"TUI:ID\"]\n",
    "concept_semantics[\"type\"] = [semantics.at[i, 'name'] for i in concept_semantics['TUI:ID']]\n",
    "concept_semantics.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fetch_type(v):\n",
    "    cat = \"\"\n",
    "    rank = len(ranked_type)\n",
    "    for i in v:\n",
    "        r = ranked_type.index(i)\n",
    "        if r < rank:\n",
    "            cat = i\n",
    "            rank = r\n",
    "    return cat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id\n",
       "C0000005    Pharmacologic Substance; Indicator, Reagent, o...\n",
       "C0000039            Pharmacologic Substance; Organic Chemical\n",
       "C0000052              Enzyme; Amino Acid, Peptide, or Protein\n",
       "C0000074                                     Organic Chemical\n",
       "C0000084    Biologically Active Substance; Amino Acid, Pep...\n",
       "Name: type, dtype: object"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cs = concept_semantics.groupby('id')['type'].apply(lambda x: \"; \".join(set(x)))\n",
    "cs.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id\n",
       "C0000005    Amino Acid, Peptide, or Protein\n",
       "C0000039                   Organic Chemical\n",
       "C0000052                             Enzyme\n",
       "C0000074                   Organic Chemical\n",
       "C0000084    Amino Acid, Peptide, or Protein\n",
       "Name: type, dtype: object"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cs_ranked = concept_semantics.groupby('id')['type'].apply(fetch_type)\n",
    "cs_ranked.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id\n",
       "C2149007    Therapeutic or Preventive Procedure\n",
       "C2920276                         Medical Device\n",
       "C4734693                          Clinical Drug\n",
       "C2476818    Therapeutic or Preventive Procedure\n",
       "C1611435                       Organic Chemical\n",
       "Name: type, dtype: object"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "common = list(set(concepts.index).intersection(cs.index))\n",
    "cs[common].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>type</th>\n",
       "      <th>type_combined</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>C0000097</th>\n",
       "      <td>Organic Chemical</td>\n",
       "      <td>Hazardous or Poisonous Substance; Organic Chem...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C0000359</th>\n",
       "      <td>Enzyme</td>\n",
       "      <td>Enzyme; Amino Acid, Peptide, or Protein</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C0000610</th>\n",
       "      <td>Pharmacologic Substance</td>\n",
       "      <td>Pharmacologic Substance; Vitamin</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C0000739</th>\n",
       "      <td>Body Part, Organ, or Organ Component</td>\n",
       "      <td>Body Part, Organ, or Organ Component</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C0000873</th>\n",
       "      <td>Finding</td>\n",
       "      <td>Finding</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                          type  \\\n",
       "id                                               \n",
       "C0000097                      Organic Chemical   \n",
       "C0000359                                Enzyme   \n",
       "C0000610               Pharmacologic Substance   \n",
       "C0000739  Body Part, Organ, or Organ Component   \n",
       "C0000873                               Finding   \n",
       "\n",
       "                                              type_combined  \n",
       "id                                                           \n",
       "C0000097  Hazardous or Poisonous Substance; Organic Chem...  \n",
       "C0000359            Enzyme; Amino Acid, Peptide, or Protein  \n",
       "C0000610                   Pharmacologic Substance; Vitamin  \n",
       "C0000739               Body Part, Organ, or Organ Component  \n",
       "C0000873                                            Finding  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concepts.loc[common, 'type'] = cs_ranked[common]\n",
    "concepts.loc[common, 'type_combined'] = cs[common]\n",
    "concepts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(15527671, 2)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concepts.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Terms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SUI:ID</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>S0009776</th>\n",
       "      <td>Acid, 2-Aminohexanedioic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>S7249234</th>\n",
       "      <td>BR CAMP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>S11872577</th>\n",
       "      <td>cramps abdominal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>S14680596</th>\n",
       "      <td>Retained tissue after pregnancy loss</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>S3417882</th>\n",
       "      <td>Missed miscarriage</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           name\n",
       "SUI:ID                                         \n",
       "S0009776               Acid, 2-Aminohexanedioic\n",
       "S7249234                                BR CAMP\n",
       "S11872577                      cramps abdominal\n",
       "S14680596  Retained tissue after pregnancy loss\n",
       "S3417882                     Missed miscarriage"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "terms = pd.read_csv(\"dd_data/20230802/neo4j/import/SUIs.csv\", index_col=0)\n",
    "terms.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>:START_ID</th>\n",
       "      <th>:END_ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C0000163</td>\n",
       "      <td>S0007806</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C0000427</td>\n",
       "      <td>S0008336</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C0000665</td>\n",
       "      <td>S11922611</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C0000992</td>\n",
       "      <td>S1577859</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C0001021</td>\n",
       "      <td>S0797872</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  :START_ID    :END_ID\n",
       "0  C0000163   S0007806\n",
       "1  C0000427   S0008336\n",
       "2  C0000665  S11922611\n",
       "3  C0000992   S1577859\n",
       "4  C0001021   S0797872"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concept_term = pd.read_csv(\"dd_data/20230802/neo4j/import/CUI-SUIs.csv\")\n",
    "concept_term.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7923747, 2)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concept_term.columns = [\"CUI:ID\", \"SUI:ID\"]\n",
    "concept_term.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SUI:ID</th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CUI:ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4DND:4DNES1JP4KZ1 CUI</th>\n",
       "      <td>aW4gc2l0dSBIaS1DIG9uIEhDVDExNiBjZWxscyAoY29udG...</td>\n",
       "      <td>in situ Hi-C on HCT116 cells (containing AID-t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4DND:4DNES21D8SP8 CUI</th>\n",
       "      <td>TWljcm8tQyBvbiBIMS1FU0MgY2VsbHMuSDEtRVND</td>\n",
       "      <td>Micro-C on H1-ESC cells.H1-ESC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4DND:4DNES2M5JIGV CUI</th>\n",
       "      <td>aW4gc2l0dSBIaS1DIG9uICBIaS1FU0MgY2VsbHMuSDEtRVND</td>\n",
       "      <td>in situ Hi-C on  Hi-ESC cells.H1-ESC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4DND:4DNES2R6PUEK CUI</th>\n",
       "      <td>aW4gc2l0dSBIaS1DIG9uIEhGRmM2IGNlbGxzLkhGRmM2</td>\n",
       "      <td>in situ Hi-C on HFFc6 cells.HFFc6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4DND:4DNES3QAGOZZ CUI</th>\n",
       "      <td>aW4gc2l0dSBIaS1DIG9uIEhDVDExNiBjZWxscyAoY29udG...</td>\n",
       "      <td>in situ Hi-C on HCT116 cells (containing AID-t...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                  SUI:ID  \\\n",
       "CUI:ID                                                                     \n",
       "4DND:4DNES1JP4KZ1 CUI  aW4gc2l0dSBIaS1DIG9uIEhDVDExNiBjZWxscyAoY29udG...   \n",
       "4DND:4DNES21D8SP8 CUI           TWljcm8tQyBvbiBIMS1FU0MgY2VsbHMuSDEtRVND   \n",
       "4DND:4DNES2M5JIGV CUI   aW4gc2l0dSBIaS1DIG9uICBIaS1FU0MgY2VsbHMuSDEtRVND   \n",
       "4DND:4DNES2R6PUEK CUI       aW4gc2l0dSBIaS1DIG9uIEhGRmM2IGNlbGxzLkhGRmM2   \n",
       "4DND:4DNES3QAGOZZ CUI  aW4gc2l0dSBIaS1DIG9uIEhDVDExNiBjZWxscyAoY29udG...   \n",
       "\n",
       "                                                                    name  \n",
       "CUI:ID                                                                    \n",
       "4DND:4DNES1JP4KZ1 CUI  in situ Hi-C on HCT116 cells (containing AID-t...  \n",
       "4DND:4DNES21D8SP8 CUI                     Micro-C on H1-ESC cells.H1-ESC  \n",
       "4DND:4DNES2M5JIGV CUI               in situ Hi-C on  Hi-ESC cells.H1-ESC  \n",
       "4DND:4DNES2R6PUEK CUI                  in situ Hi-C on HFFc6 cells.HFFc6  \n",
       "4DND:4DNES3QAGOZZ CUI  in situ Hi-C on HCT116 cells (containing AID-t...  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concept_term = pd.merge(concept_term, terms, on=\"SUI:ID\", how='outer')\n",
    "concept_term = concept_term.groupby('CUI:ID').first()\n",
    "concept_term.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7923727, 2)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concept_term.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>type</th>\n",
       "      <th>type_combined</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>C0000097</th>\n",
       "      <td>Organic Chemical</td>\n",
       "      <td>Hazardous or Poisonous Substance; Organic Chem...</td>\n",
       "      <td>1-Methyl-4-phenyl-1,2,3,6-tetrahydropyridine</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C0000359</th>\n",
       "      <td>Enzyme</td>\n",
       "      <td>Enzyme; Amino Acid, Peptide, or Protein</td>\n",
       "      <td>3',5'-Cyclic-Nucleotide Phosphodiesterase</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C0000610</th>\n",
       "      <td>Pharmacologic Substance</td>\n",
       "      <td>Pharmacologic Substance; Vitamin</td>\n",
       "      <td>6-Aminonicotinamide</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C0000739</th>\n",
       "      <td>Body Part, Organ, or Organ Component</td>\n",
       "      <td>Body Part, Organ, or Organ Component</td>\n",
       "      <td>Abdominal Muscles</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C0000873</th>\n",
       "      <td>Finding</td>\n",
       "      <td>Finding</td>\n",
       "      <td>Academic Problem</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                          type  \\\n",
       "id                                               \n",
       "C0000097                      Organic Chemical   \n",
       "C0000359                                Enzyme   \n",
       "C0000610               Pharmacologic Substance   \n",
       "C0000739  Body Part, Organ, or Organ Component   \n",
       "C0000873                               Finding   \n",
       "\n",
       "                                              type_combined  \\\n",
       "id                                                            \n",
       "C0000097  Hazardous or Poisonous Substance; Organic Chem...   \n",
       "C0000359            Enzyme; Amino Acid, Peptide, or Protein   \n",
       "C0000610                   Pharmacologic Substance; Vitamin   \n",
       "C0000739               Body Part, Organ, or Organ Component   \n",
       "C0000873                                            Finding   \n",
       "\n",
       "                                                 label  \n",
       "id                                                      \n",
       "C0000097  1-Methyl-4-phenyl-1,2,3,6-tetrahydropyridine  \n",
       "C0000359     3',5'-Cyclic-Nucleotide Phosphodiesterase  \n",
       "C0000610                           6-Aminonicotinamide  \n",
       "C0000739                             Abdominal Muscles  \n",
       "C0000873                              Academic Problem  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concepts.loc[concept_term.index, 'label'] = concept_term.loc[concept_term.index, 'name']\n",
    "concepts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(15527671, 3)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concepts.label = concepts.label.fillna(\"-\")\n",
    "concepts.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7603946"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# how many -\n",
    "len([i for i in concepts.label if i == \"-\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/wq/7bkxhy8j7ql194c1fbby4gy40000gn/T/ipykernel_9116/3416969763.py:1: DtypeWarning: Columns (2) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  codes = pd.read_csv(\"dd_data/20230802/neo4j/import/CODEs.csv\", index_col=0)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SAB</th>\n",
       "      <th>CODE</th>\n",
       "      <th>value:float</th>\n",
       "      <th>lowerbound:float</th>\n",
       "      <th>upperbound:float</th>\n",
       "      <th>unit</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CodeID:ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>MTHSPL:J7A92W69L7</th>\n",
       "      <td>MTHSPL</td>\n",
       "      <td>J7A92W69L7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NCI:C76777</th>\n",
       "      <td>NCI</td>\n",
       "      <td>C76777</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ATC:N07XX07</th>\n",
       "      <td>ATC</td>\n",
       "      <td>N07XX07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GS:1946</th>\n",
       "      <td>GS</td>\n",
       "      <td>1946</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NOC:040413</th>\n",
       "      <td>NOC</td>\n",
       "      <td>040413</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      SAB        CODE  value:float  lowerbound:float  \\\n",
       "CodeID:ID                                                              \n",
       "MTHSPL:J7A92W69L7  MTHSPL  J7A92W69L7          NaN               NaN   \n",
       "NCI:C76777            NCI      C76777          NaN               NaN   \n",
       "ATC:N07XX07           ATC     N07XX07          NaN               NaN   \n",
       "GS:1946                GS        1946          NaN               NaN   \n",
       "NOC:040413            NOC      040413          NaN               NaN   \n",
       "\n",
       "                   upperbound:float  unit  \n",
       "CodeID:ID                                  \n",
       "MTHSPL:J7A92W69L7               NaN   NaN  \n",
       "NCI:C76777                      NaN   NaN  \n",
       "ATC:N07XX07                     NaN   NaN  \n",
       "GS:1946                         NaN   NaN  \n",
       "NOC:040413                      NaN   NaN  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "codes = pd.read_csv(\"dd_data/20230802/neo4j/import/CODEs.csv\", index_col=0)\n",
    "codes.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>:START_ID</th>\n",
       "      <th>:END_ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C0000294</td>\n",
       "      <td>ATC:V03AF01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C0000481</td>\n",
       "      <td>CHV:0000000513</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C0000661</td>\n",
       "      <td>MSH:D015124</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C0000665</td>\n",
       "      <td>VANDF:4020847</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C0000737</td>\n",
       "      <td>LNC:LA15468-4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  :START_ID         :END_ID\n",
       "0  C0000294     ATC:V03AF01\n",
       "1  C0000481  CHV:0000000513\n",
       "2  C0000661     MSH:D015124\n",
       "3  C0000665   VANDF:4020847\n",
       "4  C0000737   LNC:LA15468-4"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concept_code = pd.read_csv(\"dd_data/20230802/neo4j/import/CUI-CODEs.csv\")\n",
    "concept_code.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>CodeID:ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C0000294</td>\n",
       "      <td>ATC:V03AF01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C0000481</td>\n",
       "      <td>CHV:0000000513</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C0000661</td>\n",
       "      <td>MSH:D015124</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C0000665</td>\n",
       "      <td>VANDF:4020847</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C0000737</td>\n",
       "      <td>LNC:LA15468-4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         id       CodeID:ID\n",
       "0  C0000294     ATC:V03AF01\n",
       "1  C0000481  CHV:0000000513\n",
       "2  C0000661     MSH:D015124\n",
       "3  C0000665   VANDF:4020847\n",
       "4  C0000737   LNC:LA15468-4"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concept_code.columns = [\"id\", \"CodeID:ID\"]\n",
    "concept_code.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>CodeID:ID</th>\n",
       "      <th>SAB</th>\n",
       "      <th>CODE</th>\n",
       "      <th>value:float</th>\n",
       "      <th>lowerbound:float</th>\n",
       "      <th>upperbound:float</th>\n",
       "      <th>unit</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C0000294</td>\n",
       "      <td>ATC:V03AF01</td>\n",
       "      <td>ATC</td>\n",
       "      <td>V03AF01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C0000481</td>\n",
       "      <td>CHV:0000000513</td>\n",
       "      <td>CHV</td>\n",
       "      <td>0000000513</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C0000661</td>\n",
       "      <td>MSH:D015124</td>\n",
       "      <td>MSH</td>\n",
       "      <td>D015124</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C0000665</td>\n",
       "      <td>VANDF:4020847</td>\n",
       "      <td>VANDF</td>\n",
       "      <td>4020847</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C0000737</td>\n",
       "      <td>LNC:LA15468-4</td>\n",
       "      <td>LNC</td>\n",
       "      <td>LA15468-4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         id       CodeID:ID    SAB        CODE  value:float  lowerbound:float  \\\n",
       "0  C0000294     ATC:V03AF01    ATC     V03AF01          NaN               NaN   \n",
       "1  C0000481  CHV:0000000513    CHV  0000000513          NaN               NaN   \n",
       "2  C0000661     MSH:D015124    MSH     D015124          NaN               NaN   \n",
       "3  C0000665   VANDF:4020847  VANDF     4020847          NaN               NaN   \n",
       "4  C0000737   LNC:LA15468-4    LNC   LA15468-4          NaN               NaN   \n",
       "\n",
       "   upperbound:float  unit  \n",
       "0               NaN   NaN  \n",
       "1               NaN   NaN  \n",
       "2               NaN   NaN  \n",
       "3               NaN   NaN  \n",
       "4               NaN   NaN  "
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concept_code = pd.merge(concept_code, codes, on=\"CodeID:ID\", how='left')\n",
    "concept_code.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>CodeID:ID</th>\n",
       "      <th>SAB</th>\n",
       "      <th>CODE</th>\n",
       "      <th>value:float</th>\n",
       "      <th>lowerbound:float</th>\n",
       "      <th>upperbound:float</th>\n",
       "      <th>unit</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>623690</th>\n",
       "      <td>C0000097</td>\n",
       "      <td>LCH_NW:sh86002892</td>\n",
       "      <td>LCH_NW</td>\n",
       "      <td>sh86002892</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1108107</th>\n",
       "      <td>C0000097</td>\n",
       "      <td>CSP:2511-0411</td>\n",
       "      <td>CSP</td>\n",
       "      <td>2511-0411</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1246274</th>\n",
       "      <td>C0000097</td>\n",
       "      <td>PSY:31213</td>\n",
       "      <td>PSY</td>\n",
       "      <td>31213</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2179442</th>\n",
       "      <td>C0000097</td>\n",
       "      <td>MSH:D015632</td>\n",
       "      <td>MSH</td>\n",
       "      <td>D015632</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3529094</th>\n",
       "      <td>C0000097</td>\n",
       "      <td>CHV:0000000501</td>\n",
       "      <td>CHV</td>\n",
       "      <td>0000000501</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3736237</th>\n",
       "      <td>C0000097</td>\n",
       "      <td>PSY:32433</td>\n",
       "      <td>PSY</td>\n",
       "      <td>32433</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4116966</th>\n",
       "      <td>C0000097</td>\n",
       "      <td>SNOMEDCT_US:285407008</td>\n",
       "      <td>SNOMEDCT_US</td>\n",
       "      <td>285407008</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               id              CodeID:ID          SAB        CODE  \\\n",
       "623690   C0000097      LCH_NW:sh86002892       LCH_NW  sh86002892   \n",
       "1108107  C0000097          CSP:2511-0411          CSP   2511-0411   \n",
       "1246274  C0000097              PSY:31213          PSY       31213   \n",
       "2179442  C0000097            MSH:D015632          MSH     D015632   \n",
       "3529094  C0000097         CHV:0000000501          CHV  0000000501   \n",
       "3736237  C0000097              PSY:32433          PSY       32433   \n",
       "4116966  C0000097  SNOMEDCT_US:285407008  SNOMEDCT_US   285407008   \n",
       "\n",
       "         value:float  lowerbound:float  upperbound:float  unit  \n",
       "623690           NaN               NaN               NaN   NaN  \n",
       "1108107          NaN               NaN               NaN   NaN  \n",
       "1246274          NaN               NaN               NaN   NaN  \n",
       "2179442          NaN               NaN               NaN   NaN  \n",
       "3529094          NaN               NaN               NaN   NaN  \n",
       "3736237          NaN               NaN               NaN   NaN  \n",
       "4116966          NaN               NaN               NaN   NaN  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concept_code[concept_code.id == 'C0000097']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"output/unique_SABS_of_Concept.txt\", \"w\") as o:\n",
    "    sabs = [str(i) for i in concept_code[concept_code.id.isin(concepts[concepts.type.isna()].index)].SAB.unique()]\n",
    "    o.write(\"\\n\".join(sabs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(159, 'Drug')"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type_mapper = {}\n",
    "with open(\"output/unique_SABS_of_Concept_Mapper.txt\") as o:\n",
    "    for line in o:\n",
    "        r = line.strip().split(\":\")\n",
    "        if len(r) == 2:\n",
    "            type_mapper[r[0]] = r[1]\n",
    "        elif \"MSIGDB\" in r[0]:\n",
    "            type_mapper[r[0]] = \"MSIGDB\"\n",
    "        else:\n",
    "            type_mapper[r[0]] = r[0]\n",
    "len((set(type_mapper.values()))), type_mapper[\"PUBCHEM\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "15721067it [07:33, 34691.41it/s]\n"
     ]
    }
   ],
   "source": [
    "for i,row in tqdm(concept_code[concept_code.id.isin(concepts[concepts.type.isna()].index)].iterrows()):\n",
    "    sab = row[\"SAB\"]\n",
    "    ind = row[\"id\"]\n",
    "    if type(sab) == str:\n",
    "        tp = type_mapper[sab]\n",
    "        if tp:\n",
    "            concepts.at[ind, \"type\"] = tp\n",
    "            concepts.at[ind, \"type_combined\"] = tp\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "concepts.loc[:, \"type\"] = [i.replace(\".\", \"_\") for i in concepts.type]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 284/284 [16:36<00:00,  3.51s/it] \n"
     ]
    }
   ],
   "source": [
    "for type in tqdm(concepts.type.unique()):\n",
    "    type = type.replace(\".\", \"_\")\n",
    "    con = concepts[concepts.type==type].copy()\n",
    "    cc = concept_code[concept_code.id.isin(con.index)]\n",
    "    if 'MSIGDB' in cc.SAB.unique()[0]:\n",
    "        sab = \"MSIGDB\"\n",
    "        cc.loc[:,'SAB'] = sab\n",
    "    for sab in cc.SAB.unique():\n",
    "        c = cc[cc.SAB == sab]\n",
    "        c = c.groupby('id').first()\n",
    "        common = list(set(con.index).intersection(c.index))\n",
    "        con.loc[common, sab] = c.loc[common, \"CodeID:ID\"]\n",
    "        if c.loc[common, \"value:float\"].isna().sum() != len(common):\n",
    "            con.loc[common, \"%s value\"%sab] = c.loc[common, \"value:float\"]\n",
    "        if c.loc[common, \"lowerbound:float\"].isna().sum() != len(common):\n",
    "            con.loc[common, \"%s lowerbound\"%sab] = c.loc[common, \"lowerbound:float\"]\n",
    "        if c.loc[common, \"upperbound:float\"].isna().sum() != len(common):\n",
    "            con.loc[common, \"%s upperbound\"%sab] = c.loc[common, \"upperbound:float\"]\n",
    "        if c.loc[common, \"unit\"].isna().sum() != len(common):\n",
    "            con.loc[common, \"%s unit\"%sab] = c.loc[common, \"unit\"]\n",
    "        if \"-\" in list(con[\"label\"]):\n",
    "            tmp = con[con.label == \"-\"]\n",
    "            ind = set(tmp.index).intersection(con.index)\n",
    "            ind2 = set(tmp.index).intersection(c.index)\n",
    "            if len(ind.intersection(ind2)) > 0:\n",
    "                l = list(ind.intersection(ind2))\n",
    "                con.loc[l, \"label\"] = c.loc[l, 'CodeID:ID']\n",
    "        con.to_csv(\"out/0802/serialization/nodes/%s.nodes.csv\"%(type))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"output/august_dcc_sabs.txt\") as o:\n",
    "    sabs_to_keep = set(o.read().strip().split(\"\\n\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "156777637it [1:09:36, 37540.23it/s] \n"
     ]
    }
   ],
   "source": [
    "row_headers = [\"source\", \"relation\", \"target\", \"source_label\", \"target_label\", \"SAB\", \"evidence\"]\n",
    "inverse = set()\n",
    "with open(\"dd_data/20230802/neo4j/import/CUI-CUIs.csv\") as o:\n",
    "    csv_reader = csv.reader(o)\n",
    "    headers = None\n",
    "    for row in tqdm(csv_reader):\n",
    "        if not headers:\n",
    "            headers = row\n",
    "        else:\n",
    "            source = row[0]\n",
    "            target = row[1]\n",
    "            if source in concepts.index and target in concepts.index:\n",
    "                source_label = concepts.at[source, 'label']\n",
    "                source_type = concepts.at[source, 'type']\n",
    "                \n",
    "                target_label = concepts.at[target, 'label']\n",
    "                target_type = concepts.at[target, 'type']\n",
    "                relation = row[2]\n",
    "                sab = row[3]\n",
    "                evidence = ''\n",
    "                if len(row) > 4:\n",
    "                    evidence = row[4]\n",
    "                if \"inverse\" in relation or \"_by\" in relation:\n",
    "                    inverse.add(relation)\n",
    "                if sab in sabs_to_keep and \"inverse\" not in relation and \"_by\" not in relation:\n",
    "                    filename = 'out/0802/serialization/edges/%s.%s.%s.edges.csv'%(source_type, relation, target_type)\n",
    "                    write_header = False\n",
    "                    operation = \"a\"\n",
    "                    if not os.path.isfile(filename):\n",
    "                        write_header = True\n",
    "                        operation = \"w\"\n",
    "                    with open(filename, operation) as w:\n",
    "                        csv_writer = csv.writer(w)\n",
    "                        if write_header:\n",
    "                            csv_writer.writerow(row_headers)\n",
    "                        csv_writer.writerow([source, relation, target, source_label, target_label, sab, evidence])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "edge_pattern = \"(?P<directory>.+)/(?P<source_type>.+)\\.(?P<relation>.+)\\.(?P<target_type>.+)\\.(?P<entity>.+)\\.csv\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "node_base = \"out/0802/serialization/nodes/%s.nodes.csv\"\n",
    "new_node_base = \"out/0802/filtered/nodes/%s.nodes.csv\"\n",
    "new_edge_base = \"out/0802/filtered/edges/%s.%s.%s.edges.csv\"\n",
    "ids_base = \"out/0802/filtered/ids/%s.txt\"\n",
    "node_ids = {}\n",
    "sab_relations = {}\n",
    "processed = set()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 586/586 [12:51<00:00,  1.32s/it]\n"
     ]
    }
   ],
   "source": [
    "for filename in tqdm(glob(\"out/0802/serialization/edges/*.csv\")):\n",
    "    if filename not in processed:\n",
    "        match = re.match(edge_pattern, filename).groupdict()\n",
    "        entity = match[\"entity\"]\n",
    "        source_type = match[\"source_type\"]\n",
    "        relation = match[\"relation\"].replace(\"_\", \" \")\n",
    "        target_type = match[\"target_type\"]\n",
    "        if \"inverse\" not in relation:\n",
    "            edge_df = pd.read_csv(filename, low_memory=False)\n",
    "            # filter for SAB\n",
    "            sabs = sabs_to_keep.intersection(edge_df.SAB.unique())\n",
    "            for sab in sabs:\n",
    "                if sab not in sab_relations:\n",
    "                    sab_relations[sab] = set()\n",
    "                sab_relations[sab].add(relation)\n",
    "            if len(sabs) > 0:\n",
    "                edge_df = edge_df[edge_df.SAB.isin(sabs)]\n",
    "                source_df = pd.read_csv(node_base%source_type, index_col=0, low_memory=False)\n",
    "                if not os.path.isfile(new_node_base%(source_type)):\n",
    "                    # new_source_df = pd.read_csv(new_node_base%(source_type), index_col=0, low_memory=False)\n",
    "                    # pd.concat([new_source_df, source_df]).dropna(axis=1).to_csv(new_node_base%(source_type))\n",
    "                    source_df.to_csv(new_node_base%(source_type))\n",
    "                # else:\n",
    "                #     source_df.dropna(axis=1).to_csv(new_node_base%(source_type))\n",
    "                \n",
    "                target_df = pd.read_csv(node_base%target_type, index_col=0, low_memory=False)\n",
    "                if not os.path.isfile(new_node_base%(target_type)):\n",
    "                    # new_target_df = pd.read_csv(new_node_base%(target_type), index_col=0, low_memory=False)\n",
    "                    # pd.concat([new_target_df, target_df]).dropna(axis=1).to_csv(new_node_base%(target_type))\n",
    "                    target_df.to_csv(new_node_base%(target_type))\n",
    "                # else:\n",
    "                #     target_df.dropna(axis=1).to_csv(new_node_base%(target_type))\n",
    "                edge_df.to_csv(new_edge_base%(source_type, relation, target_type), index=False)\n",
    "        processed.add(filename)\n",
    "                "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.read_csv(\"out/0802/filtered/edges/GTEXEQTL.p value.PVALUEBINS.edges.csv\").head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "gtex = 'EXPBINS'\n",
    "for i in glob(\"out/0802/filtered/edges/*%s*edges.csv\"%gtex):\n",
    "    df = pd.read_csv(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1573920, 7)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>relation</th>\n",
       "      <th>target</th>\n",
       "      <th>source_label</th>\n",
       "      <th>target_label</th>\n",
       "      <th>SAB</th>\n",
       "      <th>evidence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Testis CUI</td>\n",
       "      <td>has_expression</td>\n",
       "      <td>EXPBINS:0.1.0.2 CUI</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>GTEXEXP:ENSG00000227232-5-Adipose-Subcutaneous...</td>\n",
       "      <td>has_expression</td>\n",
       "      <td>EXPBINS:4.0.5.0 CUI</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>GTEXEXP:ENSG00000227232-5-Adrenal-Gland CUI</td>\n",
       "      <td>has_expression</td>\n",
       "      <td>EXPBINS:2.0.3.0 CUI</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>GTEXEXP:ENSG00000227232-5-Artery-Aorta CUI</td>\n",
       "      <td>has_expression</td>\n",
       "      <td>EXPBINS:4.0.5.0 CUI</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>GTEXEXP:ENSG00000227232-5-Artery-Coronary CUI</td>\n",
       "      <td>has_expression</td>\n",
       "      <td>EXPBINS:3.0.4.0 CUI</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              source        relation  \\\n",
       "0               GTEXEXP:ENSG00000223972-5-Testis CUI  has_expression   \n",
       "1  GTEXEXP:ENSG00000227232-5-Adipose-Subcutaneous...  has_expression   \n",
       "2        GTEXEXP:ENSG00000227232-5-Adrenal-Gland CUI  has_expression   \n",
       "3         GTEXEXP:ENSG00000227232-5-Artery-Aorta CUI  has_expression   \n",
       "4      GTEXEXP:ENSG00000227232-5-Artery-Coronary CUI  has_expression   \n",
       "\n",
       "                target source_label target_label      SAB  evidence  \n",
       "0  EXPBINS:0.1.0.2 CUI            -            -  GTEXEXP       NaN  \n",
       "1  EXPBINS:4.0.5.0 CUI            -            -  GTEXEXP       NaN  \n",
       "2  EXPBINS:2.0.3.0 CUI            -            -  GTEXEXP       NaN  \n",
       "3  EXPBINS:4.0.5.0 CUI            -            -  GTEXEXP       NaN  \n",
       "4  EXPBINS:3.0.4.0 CUI            -            -  GTEXEXP       NaN  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'out/0802/filtered/edges/GTEXEXP.has expression.EXPBINS.edges.csv'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "out/0802/filtered/edges/GTEXEXP.expressed in.Tissue.edges.csv\n",
      "out/0802/filtered/edges/Gene or Genome.expresses.GTEXEXP.edges.csv\n",
      "out/0802/filtered/edges/GTEXEXP.has expression.EXPBINS.edges.csv\n",
      "out/0802/filtered/edges/GTEXEXP.expressed in.Body Location or Region.edges.csv\n",
      "out/0802/filtered/edges/Tissue.expresses.GTEXEXP.edges.csv\n",
      "out/0802/filtered/edges/Body Part, Organ, or Organ Component.expresses.GTEXEXP.edges.csv\n",
      "out/0802/filtered/edges/GTEXEXP.expressed in.Body Part, Organ, or Organ Component.edges.csv\n",
      "out/0802/filtered/edges/Body Location or Region.expresses.GTEXEXP.edges.csv\n",
      "out/0802/filtered/edges/GTEXEXP.expressed in.Gene or Genome.edges.csv\n"
     ]
    }
   ],
   "source": [
    "gtex = 'GTEXEXP'\n",
    "for i in glob(\"out/0802/filtered/edges/*%s*edges.csv\"%gtex):\n",
    "    print(i)\n",
    "    df = pd.read_csv(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>relation</th>\n",
       "      <th>target</th>\n",
       "      <th>source_label</th>\n",
       "      <th>target_label</th>\n",
       "      <th>SAB</th>\n",
       "      <th>evidence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Testis CUI</td>\n",
       "      <td>has_expression</td>\n",
       "      <td>EXPBINS:0.1.0.2 CUI</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>GTEXEXP:ENSG00000227232-5-Adipose-Subcutaneous...</td>\n",
       "      <td>has_expression</td>\n",
       "      <td>EXPBINS:4.0.5.0 CUI</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>GTEXEXP:ENSG00000227232-5-Adrenal-Gland CUI</td>\n",
       "      <td>has_expression</td>\n",
       "      <td>EXPBINS:2.0.3.0 CUI</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>GTEXEXP:ENSG00000227232-5-Artery-Aorta CUI</td>\n",
       "      <td>has_expression</td>\n",
       "      <td>EXPBINS:4.0.5.0 CUI</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>GTEXEXP:ENSG00000227232-5-Artery-Coronary CUI</td>\n",
       "      <td>has_expression</td>\n",
       "      <td>EXPBINS:3.0.4.0 CUI</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              source        relation  \\\n",
       "0               GTEXEXP:ENSG00000223972-5-Testis CUI  has_expression   \n",
       "1  GTEXEXP:ENSG00000227232-5-Adipose-Subcutaneous...  has_expression   \n",
       "2        GTEXEXP:ENSG00000227232-5-Adrenal-Gland CUI  has_expression   \n",
       "3         GTEXEXP:ENSG00000227232-5-Artery-Aorta CUI  has_expression   \n",
       "4      GTEXEXP:ENSG00000227232-5-Artery-Coronary CUI  has_expression   \n",
       "\n",
       "                target source_label target_label      SAB  evidence  \n",
       "0  EXPBINS:0.1.0.2 CUI            -            -  GTEXEXP       NaN  \n",
       "1  EXPBINS:4.0.5.0 CUI            -            -  GTEXEXP       NaN  \n",
       "2  EXPBINS:2.0.3.0 CUI            -            -  GTEXEXP       NaN  \n",
       "3  EXPBINS:4.0.5.0 CUI            -            -  GTEXEXP       NaN  \n",
       "4  EXPBINS:3.0.4.0 CUI            -            -  GTEXEXP       NaN  "
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filename = \"out/0802/filtered/edges/GTEXEXP.has expression.EXPBINS.edges.csv\"\n",
    "scores = pd.read_csv(filename, index_col=0)\n",
    "scores.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores.evidence = ''\n",
    "scores.to_csv(filename, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['source', 'relation', 'target', 'source_label', 'target_label', 'SAB',\n",
       "       'evidence'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "score_dict = {}\n",
    "for k,v in scores.iterrows():\n",
    "    source = v[\"source\"]\n",
    "    score = v[\"target\"]\n",
    "    score_dict[source] = float(\".\".join(score.replace(\"CUI\", \"\").strip().split(\".\")[2:]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1573920"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(score_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "out/0802/filtered/edges/GTEXEXP.expressed in.Tissue.edges.csv\n",
      "out/0802/filtered/edges/Gene or Genome.expresses.GTEXEXP.edges.csv\n",
      "out/0802/filtered/edges/GTEXEXP.has expression.EXPBINS.edges.csv\n",
      "out/0802/filtered/edges/GTEXEXP.expressed in.Body Location or Region.edges.csv\n",
      "out/0802/filtered/edges/Tissue.expresses.GTEXEXP.edges.csv\n",
      "out/0802/filtered/edges/Body Part, Organ, or Organ Component.expresses.GTEXEXP.edges.csv\n",
      "out/0802/filtered/edges/GTEXEXP.expressed in.Body Part, Organ, or Organ Component.edges.csv\n",
      "out/0802/filtered/edges/Body Location or Region.expresses.GTEXEXP.edges.csv\n",
      "out/0802/filtered/edges/GTEXEXP.expressed in.Gene or Genome.edges.csv\n"
     ]
    }
   ],
   "source": [
    "cols = ['source', 'relation', 'target', 'source_label', 'target_label', 'SAB',\n",
    "       'evidence']\n",
    "gtex = 'GTEXEXP'\n",
    "for i in glob(\"out/0802/filtered/edges/*%s.*edges.csv\"%gtex):\n",
    "    print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "gene_df = pd.read_csv(\"out/0802/filtered/edges/GTEXEXP.expressed in.Gene or Genome.edges.csv\", index_col=0)\n",
    "tissue_df = pd.read_csv(\"out/0802/filtered/edges/Tissue.expresses.GTEXEXP.edges.csv\", index_col=0)\n",
    "organ_df = pd.read_csv(\"out/0802/filtered/edges/Body Part, Organ, or Organ Component.expresses.GTEXEXP.edges.csv\", index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "gene_df.columns = ['source', 'relation', 'gene_id', 'source_label', 'gene_label', 'SAB',\n",
    "       'evidence']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>relation</th>\n",
       "      <th>target</th>\n",
       "      <th>source_label</th>\n",
       "      <th>target_label</th>\n",
       "      <th>SAB</th>\n",
       "      <th>evidence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Testis CUI</td>\n",
       "      <td>has_expression</td>\n",
       "      <td>EXPBINS:0.1.0.2 CUI</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>GTEXEXP:ENSG00000227232-5-Adipose-Subcutaneous...</td>\n",
       "      <td>has_expression</td>\n",
       "      <td>EXPBINS:4.0.5.0 CUI</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>GTEXEXP:ENSG00000227232-5-Adrenal-Gland CUI</td>\n",
       "      <td>has_expression</td>\n",
       "      <td>EXPBINS:2.0.3.0 CUI</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>GTEXEXP:ENSG00000227232-5-Artery-Aorta CUI</td>\n",
       "      <td>has_expression</td>\n",
       "      <td>EXPBINS:4.0.5.0 CUI</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>GTEXEXP:ENSG00000227232-5-Artery-Coronary CUI</td>\n",
       "      <td>has_expression</td>\n",
       "      <td>EXPBINS:3.0.4.0 CUI</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              source        relation  \\\n",
       "0               GTEXEXP:ENSG00000223972-5-Testis CUI  has_expression   \n",
       "1  GTEXEXP:ENSG00000227232-5-Adipose-Subcutaneous...  has_expression   \n",
       "2        GTEXEXP:ENSG00000227232-5-Adrenal-Gland CUI  has_expression   \n",
       "3         GTEXEXP:ENSG00000227232-5-Artery-Aorta CUI  has_expression   \n",
       "4      GTEXEXP:ENSG00000227232-5-Artery-Coronary CUI  has_expression   \n",
       "\n",
       "                target source_label target_label      SAB  evidence  \n",
       "0  EXPBINS:0.1.0.2 CUI            -            -  GTEXEXP       NaN  \n",
       "1  EXPBINS:4.0.5.0 CUI            -            -  GTEXEXP       NaN  \n",
       "2  EXPBINS:2.0.3.0 CUI            -            -  GTEXEXP       NaN  \n",
       "3  EXPBINS:4.0.5.0 CUI            -            -  GTEXEXP       NaN  \n",
       "4  EXPBINS:3.0.4.0 CUI            -            -  GTEXEXP       NaN  "
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>relation</th>\n",
       "      <th>gene_id</th>\n",
       "      <th>source_label</th>\n",
       "      <th>gene_label</th>\n",
       "      <th>SAB</th>\n",
       "      <th>evidence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Adipose-Subcutaneous...</td>\n",
       "      <td>expressed_in</td>\n",
       "      <td>C2239334</td>\n",
       "      <td>-</td>\n",
       "      <td>DDX11L1 gene</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Adrenal-Gland CUI</td>\n",
       "      <td>expressed_in</td>\n",
       "      <td>C2239334</td>\n",
       "      <td>-</td>\n",
       "      <td>DDX11L1 gene</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Artery-Aorta CUI</td>\n",
       "      <td>expressed_in</td>\n",
       "      <td>C2239334</td>\n",
       "      <td>-</td>\n",
       "      <td>DDX11L1 gene</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Artery-Coronary CUI</td>\n",
       "      <td>expressed_in</td>\n",
       "      <td>C2239334</td>\n",
       "      <td>-</td>\n",
       "      <td>DDX11L1 gene</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Artery-Tibial CUI</td>\n",
       "      <td>expressed_in</td>\n",
       "      <td>C2239334</td>\n",
       "      <td>-</td>\n",
       "      <td>DDX11L1 gene</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              source      relation   gene_id  \\\n",
       "0  GTEXEXP:ENSG00000223972-5-Adipose-Subcutaneous...  expressed_in  C2239334   \n",
       "1        GTEXEXP:ENSG00000223972-5-Adrenal-Gland CUI  expressed_in  C2239334   \n",
       "2         GTEXEXP:ENSG00000223972-5-Artery-Aorta CUI  expressed_in  C2239334   \n",
       "3      GTEXEXP:ENSG00000223972-5-Artery-Coronary CUI  expressed_in  C2239334   \n",
       "4        GTEXEXP:ENSG00000223972-5-Artery-Tibial CUI  expressed_in  C2239334   \n",
       "\n",
       "  source_label    gene_label      SAB  evidence  \n",
       "0            -  DDX11L1 gene  GTEXEXP       0.0  \n",
       "1            -  DDX11L1 gene  GTEXEXP       0.0  \n",
       "2            -  DDX11L1 gene  GTEXEXP       0.0  \n",
       "3            -  DDX11L1 gene  GTEXEXP       0.0  \n",
       "4            -  DDX11L1 gene  GTEXEXP       0.0  "
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gene_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>relation</th>\n",
       "      <th>target</th>\n",
       "      <th>source_label</th>\n",
       "      <th>target_label</th>\n",
       "      <th>SAB</th>\n",
       "      <th>evidence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C0222331</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Adipose-Subcutaneous...</td>\n",
       "      <td>Subcutaneous Fat</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>UBERON:0008367 CUI</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Breast-Mammary-Tissu...</td>\n",
       "      <td>breast epithelium</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>UBERON:EFO 0002009 CUI</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Cells-Cultured-fibro...</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>UBERON:EFO 0000572 CUI</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Cells-EBV-transforme...</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C1707950</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Esophagus-Mucosa CUI</td>\n",
       "      <td>Esophageal Squamous Epithelium</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   source   relation  \\\n",
       "0                C0222331  expresses   \n",
       "1      UBERON:0008367 CUI  expresses   \n",
       "2  UBERON:EFO 0002009 CUI  expresses   \n",
       "3  UBERON:EFO 0000572 CUI  expresses   \n",
       "4                C1707950  expresses   \n",
       "\n",
       "                                              target  \\\n",
       "0  GTEXEXP:ENSG00000223972-5-Adipose-Subcutaneous...   \n",
       "1  GTEXEXP:ENSG00000223972-5-Breast-Mammary-Tissu...   \n",
       "2  GTEXEXP:ENSG00000223972-5-Cells-Cultured-fibro...   \n",
       "3  GTEXEXP:ENSG00000223972-5-Cells-EBV-transforme...   \n",
       "4     GTEXEXP:ENSG00000223972-5-Esophagus-Mucosa CUI   \n",
       "\n",
       "                     source_label target_label      SAB  evidence  \n",
       "0                Subcutaneous Fat            -  GTEXEXP       0.0  \n",
       "1               breast epithelium            -  GTEXEXP       0.0  \n",
       "2                               -            -  GTEXEXP       0.0  \n",
       "3                               -            -  GTEXEXP       0.0  \n",
       "4  Esophageal Squamous Epithelium            -  GTEXEXP       0.0  "
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tissue_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(244832, 244832)"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tissue_df.target), len(set(tissue_df.target))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "244811"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(set(tissue_df.target).intersection(set(gene_df.source)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1573785, 1573785)"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(gene_df.source), len(set(gene_df.source))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tissue_df.merge(gene_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/wq/7bkxhy8j7ql194c1fbby4gy40000gn/T/ipykernel_9890/759924432.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  tissue_df[tissue_df.target == source].target = gene_id\n",
      "/var/folders/wq/7bkxhy8j7ql194c1fbby4gy40000gn/T/ipykernel_9890/759924432.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  tissue_df[tissue_df.target == source].target_label = gene_label\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[69], line 5\u001b[0m\n\u001b[1;32m      3\u001b[0m gene_id \u001b[39m=\u001b[39m v[\u001b[39m\"\u001b[39m\u001b[39mtarget\u001b[39m\u001b[39m\"\u001b[39m]\n\u001b[1;32m      4\u001b[0m gene_label \u001b[39m=\u001b[39m v[\u001b[39m'\u001b[39m\u001b[39mtarget_label\u001b[39m\u001b[39m'\u001b[39m]\n\u001b[0;32m----> 5\u001b[0m tissue_df[tissue_df\u001b[39m.\u001b[39;49mtarget \u001b[39m==\u001b[39;49m source]\u001b[39m.\u001b[39mtarget \u001b[39m=\u001b[39m gene_id\n\u001b[1;32m      6\u001b[0m tissue_df[tissue_df\u001b[39m.\u001b[39mtarget \u001b[39m==\u001b[39m source]\u001b[39m.\u001b[39mtarget_label \u001b[39m=\u001b[39m gene_label\n",
      "File \u001b[0;32m~/Documents/codebase/DataDistillery/venv/lib/python3.10/site-packages/pandas/core/ops/common.py:81\u001b[0m, in \u001b[0;36m_unpack_zerodim_and_defer.<locals>.new_method\u001b[0;34m(self, other)\u001b[0m\n\u001b[1;32m     77\u001b[0m             \u001b[39mreturn\u001b[39;00m \u001b[39mNotImplemented\u001b[39m\n\u001b[1;32m     79\u001b[0m other \u001b[39m=\u001b[39m item_from_zerodim(other)\n\u001b[0;32m---> 81\u001b[0m \u001b[39mreturn\u001b[39;00m method(\u001b[39mself\u001b[39;49m, other)\n",
      "File \u001b[0;32m~/Documents/codebase/DataDistillery/venv/lib/python3.10/site-packages/pandas/core/arraylike.py:40\u001b[0m, in \u001b[0;36mOpsMixin.__eq__\u001b[0;34m(self, other)\u001b[0m\n\u001b[1;32m     38\u001b[0m \u001b[39m@unpack_zerodim_and_defer\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39m__eq__\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m     39\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m__eq__\u001b[39m(\u001b[39mself\u001b[39m, other):\n\u001b[0;32m---> 40\u001b[0m     \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_cmp_method(other, operator\u001b[39m.\u001b[39;49meq)\n",
      "File \u001b[0;32m~/Documents/codebase/DataDistillery/venv/lib/python3.10/site-packages/pandas/core/series.py:6096\u001b[0m, in \u001b[0;36mSeries._cmp_method\u001b[0;34m(self, other, op)\u001b[0m\n\u001b[1;32m   6093\u001b[0m rvalues \u001b[39m=\u001b[39m extract_array(other, extract_numpy\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m, extract_range\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m)\n\u001b[1;32m   6095\u001b[0m \u001b[39mwith\u001b[39;00m np\u001b[39m.\u001b[39merrstate(\u001b[39mall\u001b[39m\u001b[39m=\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mignore\u001b[39m\u001b[39m\"\u001b[39m):\n\u001b[0;32m-> 6096\u001b[0m     res_values \u001b[39m=\u001b[39m ops\u001b[39m.\u001b[39;49mcomparison_op(lvalues, rvalues, op)\n\u001b[1;32m   6098\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_construct_result(res_values, name\u001b[39m=\u001b[39mres_name)\n",
      "File \u001b[0;32m~/Documents/codebase/DataDistillery/venv/lib/python3.10/site-packages/pandas/core/ops/array_ops.py:293\u001b[0m, in \u001b[0;36mcomparison_op\u001b[0;34m(left, right, op)\u001b[0m\n\u001b[1;32m    290\u001b[0m     \u001b[39mreturn\u001b[39;00m invalid_comparison(lvalues, rvalues, op)\n\u001b[1;32m    292\u001b[0m \u001b[39melif\u001b[39;00m is_object_dtype(lvalues\u001b[39m.\u001b[39mdtype) \u001b[39mor\u001b[39;00m \u001b[39misinstance\u001b[39m(rvalues, \u001b[39mstr\u001b[39m):\n\u001b[0;32m--> 293\u001b[0m     res_values \u001b[39m=\u001b[39m comp_method_OBJECT_ARRAY(op, lvalues, rvalues)\n\u001b[1;32m    295\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m    296\u001b[0m     res_values \u001b[39m=\u001b[39m _na_arithmetic_op(lvalues, rvalues, op, is_cmp\u001b[39m=\u001b[39m\u001b[39mTrue\u001b[39;00m)\n",
      "File \u001b[0;32m~/Documents/codebase/DataDistillery/venv/lib/python3.10/site-packages/pandas/core/ops/array_ops.py:82\u001b[0m, in \u001b[0;36mcomp_method_OBJECT_ARRAY\u001b[0;34m(op, x, y)\u001b[0m\n\u001b[1;32m     80\u001b[0m     result \u001b[39m=\u001b[39m libops\u001b[39m.\u001b[39mvec_compare(x\u001b[39m.\u001b[39mravel(), y\u001b[39m.\u001b[39mravel(), op)\n\u001b[1;32m     81\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m---> 82\u001b[0m     result \u001b[39m=\u001b[39m libops\u001b[39m.\u001b[39;49mscalar_compare(x\u001b[39m.\u001b[39;49mravel(), y, op)\n\u001b[1;32m     83\u001b[0m \u001b[39mreturn\u001b[39;00m result\u001b[39m.\u001b[39mreshape(x\u001b[39m.\u001b[39mshape)\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "for i, v in gene_df.iterrows():\n",
    "    source = v[\"source\"]\n",
    "    gene_id = v[\"target\"]\n",
    "    gene_label = v['target_label']\n",
    "    tissue_df[tissue_df.target == source].target = gene_id\n",
    "    tissue_df[tissue_df.target == source].target_label = gene_label\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "out/0802/filtered/edges/GTEXEXP.expressed in.Tissue.edges.csv\n",
      "out/0802/filtered/edges/GTEXEXP.has expression.EXPBINS.edges.csv\n",
      "out/0802/filtered/edges/GTEXEXP.expressed in.Body Location or Region.edges.csv\n",
      "out/0802/filtered/edges/GTEXEXP.expressed in.Body Part, Organ, or Organ Component.edges.csv\n",
      "out/0802/filtered/edges/GTEXEXP.expressed in.Gene or Genome.edges.csv\n",
      "out/0802/filtered/edges/Gene or Genome.expresses.GTEXEXP.edges.csv\n",
      "out/0802/filtered/edges/Tissue.expresses.GTEXEXP.edges.csv\n",
      "out/0802/filtered/edges/Body Part, Organ, or Organ Component.expresses.GTEXEXP.edges.csv\n",
      "out/0802/filtered/edges/Body Location or Region.expresses.GTEXEXP.edges.csv\n"
     ]
    }
   ],
   "source": [
    "cols = ['source', 'relation', 'target', 'source_label', 'target_label', 'SAB',\n",
    "       'evidence']\n",
    "gtex = 'GTEXEXP'\n",
    "for i in glob(\"out/0802/filtered/edges/%s.*edges.csv\"%gtex):\n",
    "    print(i)\n",
    "    df = pd.read_csv(i)\n",
    "    df[cols].to_csv(i)\n",
    "\n",
    "for i in glob(\"out/0802/filtered/edges/*.%s.*edges.csv\"%gtex):\n",
    "    print(i)\n",
    "    df = pd.read_csv(i)\n",
    "    df[cols].to_csv(i)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "out/0802/filtered/edges/GTEXEXP.expressed in.Tissue.edges.csv\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>source</th>\n",
       "      <th>relation</th>\n",
       "      <th>target</th>\n",
       "      <th>source_label</th>\n",
       "      <th>target_label</th>\n",
       "      <th>SAB</th>\n",
       "      <th>evidence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Adipose-Subcutaneous...</td>\n",
       "      <td>expressed_in</td>\n",
       "      <td>C0222331</td>\n",
       "      <td>-</td>\n",
       "      <td>Subcutaneous Fat</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Breast-Mammary-Tissu...</td>\n",
       "      <td>expressed_in</td>\n",
       "      <td>UBERON:0008367 CUI</td>\n",
       "      <td>-</td>\n",
       "      <td>breast epithelium</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Cells-Cultured-fibro...</td>\n",
       "      <td>expressed_in</td>\n",
       "      <td>UBERON:EFO 0002009 CUI</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Cells-EBV-transforme...</td>\n",
       "      <td>expressed_in</td>\n",
       "      <td>UBERON:EFO 0000572 CUI</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Esophagus-Mucosa CUI</td>\n",
       "      <td>expressed_in</td>\n",
       "      <td>C1707950</td>\n",
       "      <td>-</td>\n",
       "      <td>Esophageal Squamous Epithelium</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0                                             source  \\\n",
       "0           0  GTEXEXP:ENSG00000223972-5-Adipose-Subcutaneous...   \n",
       "1           1  GTEXEXP:ENSG00000223972-5-Breast-Mammary-Tissu...   \n",
       "2           2  GTEXEXP:ENSG00000223972-5-Cells-Cultured-fibro...   \n",
       "3           3  GTEXEXP:ENSG00000223972-5-Cells-EBV-transforme...   \n",
       "4           4     GTEXEXP:ENSG00000223972-5-Esophagus-Mucosa CUI   \n",
       "\n",
       "       relation                  target source_label  \\\n",
       "0  expressed_in                C0222331            -   \n",
       "1  expressed_in      UBERON:0008367 CUI            -   \n",
       "2  expressed_in  UBERON:EFO 0002009 CUI            -   \n",
       "3  expressed_in  UBERON:EFO 0000572 CUI            -   \n",
       "4  expressed_in                C1707950            -   \n",
       "\n",
       "                     target_label      SAB  evidence  \n",
       "0                Subcutaneous Fat  GTEXEXP       0.0  \n",
       "1               breast epithelium  GTEXEXP       0.0  \n",
       "2                               -  GTEXEXP       0.0  \n",
       "3                               -  GTEXEXP       0.0  \n",
       "4  Esophageal Squamous Epithelium  GTEXEXP       0.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "out/0802/filtered/edges/GTEXEXP.has expression.EXPBINS.edges.csv\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>source</th>\n",
       "      <th>relation</th>\n",
       "      <th>target</th>\n",
       "      <th>source_label</th>\n",
       "      <th>target_label</th>\n",
       "      <th>SAB</th>\n",
       "      <th>evidence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Testis CUI</td>\n",
       "      <td>has_expression</td>\n",
       "      <td>EXPBINS:0.1.0.2 CUI</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>GTEXEXP:ENSG00000227232-5-Adipose-Subcutaneous...</td>\n",
       "      <td>has_expression</td>\n",
       "      <td>EXPBINS:4.0.5.0 CUI</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>GTEXEXP:ENSG00000227232-5-Adrenal-Gland CUI</td>\n",
       "      <td>has_expression</td>\n",
       "      <td>EXPBINS:2.0.3.0 CUI</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>GTEXEXP:ENSG00000227232-5-Artery-Aorta CUI</td>\n",
       "      <td>has_expression</td>\n",
       "      <td>EXPBINS:4.0.5.0 CUI</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>GTEXEXP:ENSG00000227232-5-Artery-Coronary CUI</td>\n",
       "      <td>has_expression</td>\n",
       "      <td>EXPBINS:3.0.4.0 CUI</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0                                             source  \\\n",
       "0           0               GTEXEXP:ENSG00000223972-5-Testis CUI   \n",
       "1           1  GTEXEXP:ENSG00000227232-5-Adipose-Subcutaneous...   \n",
       "2           2        GTEXEXP:ENSG00000227232-5-Adrenal-Gland CUI   \n",
       "3           3         GTEXEXP:ENSG00000227232-5-Artery-Aorta CUI   \n",
       "4           4      GTEXEXP:ENSG00000227232-5-Artery-Coronary CUI   \n",
       "\n",
       "         relation               target source_label target_label      SAB  \\\n",
       "0  has_expression  EXPBINS:0.1.0.2 CUI            -            -  GTEXEXP   \n",
       "1  has_expression  EXPBINS:4.0.5.0 CUI            -            -  GTEXEXP   \n",
       "2  has_expression  EXPBINS:2.0.3.0 CUI            -            -  GTEXEXP   \n",
       "3  has_expression  EXPBINS:4.0.5.0 CUI            -            -  GTEXEXP   \n",
       "4  has_expression  EXPBINS:3.0.4.0 CUI            -            -  GTEXEXP   \n",
       "\n",
       "   evidence  \n",
       "0       0.2  \n",
       "1       5.0  \n",
       "2       3.0  \n",
       "3       5.0  \n",
       "4       4.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "out/0802/filtered/edges/GTEXEXP.expressed in.Body Location or Region.edges.csv\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>source</th>\n",
       "      <th>relation</th>\n",
       "      <th>target</th>\n",
       "      <th>source_label</th>\n",
       "      <th>target_label</th>\n",
       "      <th>SAB</th>\n",
       "      <th>evidence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Kidney-Medulla CUI</td>\n",
       "      <td>expressed_in</td>\n",
       "      <td>C0736435</td>\n",
       "      <td>-</td>\n",
       "      <td>Set of outer region of renal pyramids</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>GTEXEXP:ENSG00000227232-5-Kidney-Medulla CUI</td>\n",
       "      <td>expressed_in</td>\n",
       "      <td>C0736435</td>\n",
       "      <td>-</td>\n",
       "      <td>Set of outer region of renal pyramids</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>GTEXEXP:ENSG00000278267-1-Kidney-Medulla CUI</td>\n",
       "      <td>expressed_in</td>\n",
       "      <td>C0736435</td>\n",
       "      <td>-</td>\n",
       "      <td>Set of outer region of renal pyramids</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>GTEXEXP:ENSG00000243485-5-Kidney-Medulla CUI</td>\n",
       "      <td>expressed_in</td>\n",
       "      <td>C0736435</td>\n",
       "      <td>-</td>\n",
       "      <td>Set of outer region of renal pyramids</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>GTEXEXP:ENSG00000237613-2-Kidney-Medulla CUI</td>\n",
       "      <td>expressed_in</td>\n",
       "      <td>C0736435</td>\n",
       "      <td>-</td>\n",
       "      <td>Set of outer region of renal pyramids</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0                                        source      relation  \\\n",
       "0           0  GTEXEXP:ENSG00000223972-5-Kidney-Medulla CUI  expressed_in   \n",
       "1           1  GTEXEXP:ENSG00000227232-5-Kidney-Medulla CUI  expressed_in   \n",
       "2           2  GTEXEXP:ENSG00000278267-1-Kidney-Medulla CUI  expressed_in   \n",
       "3           3  GTEXEXP:ENSG00000243485-5-Kidney-Medulla CUI  expressed_in   \n",
       "4           4  GTEXEXP:ENSG00000237613-2-Kidney-Medulla CUI  expressed_in   \n",
       "\n",
       "     target source_label                           target_label      SAB  \\\n",
       "0  C0736435            -  Set of outer region of renal pyramids  GTEXEXP   \n",
       "1  C0736435            -  Set of outer region of renal pyramids  GTEXEXP   \n",
       "2  C0736435            -  Set of outer region of renal pyramids  GTEXEXP   \n",
       "3  C0736435            -  Set of outer region of renal pyramids  GTEXEXP   \n",
       "4  C0736435            -  Set of outer region of renal pyramids  GTEXEXP   \n",
       "\n",
       "   evidence  \n",
       "0       0.0  \n",
       "1       3.0  \n",
       "2       0.0  \n",
       "3       0.0  \n",
       "4       0.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "out/0802/filtered/edges/GTEXEXP.expressed in.Body Part, Organ, or Organ Component.edges.csv\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>relation</th>\n",
       "      <th>target</th>\n",
       "      <th>source_label</th>\n",
       "      <th>target_label</th>\n",
       "      <th>SAB</th>\n",
       "      <th>evidence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Adrenal-Gland CUI</td>\n",
       "      <td>expressed_in</td>\n",
       "      <td>C0001625</td>\n",
       "      <td>-</td>\n",
       "      <td>Adrenal Glands</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Artery-Aorta CUI</td>\n",
       "      <td>expressed_in</td>\n",
       "      <td>C0003956</td>\n",
       "      <td>-</td>\n",
       "      <td>Ascending aorta structure</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Artery-Coronary CUI</td>\n",
       "      <td>expressed_in</td>\n",
       "      <td>C0205042</td>\n",
       "      <td>-</td>\n",
       "      <td>Coronary artery</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Artery-Tibial CUI</td>\n",
       "      <td>expressed_in</td>\n",
       "      <td>C0085427</td>\n",
       "      <td>-</td>\n",
       "      <td>Tibial Arteries</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Bladder CUI</td>\n",
       "      <td>expressed_in</td>\n",
       "      <td>C0005682</td>\n",
       "      <td>-</td>\n",
       "      <td>Urinary Bladder</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                          source      relation    target  \\\n",
       "0    GTEXEXP:ENSG00000223972-5-Adrenal-Gland CUI  expressed_in  C0001625   \n",
       "1     GTEXEXP:ENSG00000223972-5-Artery-Aorta CUI  expressed_in  C0003956   \n",
       "2  GTEXEXP:ENSG00000223972-5-Artery-Coronary CUI  expressed_in  C0205042   \n",
       "3    GTEXEXP:ENSG00000223972-5-Artery-Tibial CUI  expressed_in  C0085427   \n",
       "4          GTEXEXP:ENSG00000223972-5-Bladder CUI  expressed_in  C0005682   \n",
       "\n",
       "  source_label               target_label      SAB  evidence  \n",
       "0            -             Adrenal Glands  GTEXEXP       0.0  \n",
       "1            -  Ascending aorta structure  GTEXEXP       0.0  \n",
       "2            -            Coronary artery  GTEXEXP       0.0  \n",
       "3            -            Tibial Arteries  GTEXEXP       0.0  \n",
       "4            -            Urinary Bladder  GTEXEXP       0.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "out/0802/filtered/edges/GTEXEXP.expressed in.Gene or Genome.edges.csv\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>relation</th>\n",
       "      <th>target</th>\n",
       "      <th>source_label</th>\n",
       "      <th>target_label</th>\n",
       "      <th>SAB</th>\n",
       "      <th>evidence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Adipose-Subcutaneous...</td>\n",
       "      <td>expressed_in</td>\n",
       "      <td>C2239334</td>\n",
       "      <td>-</td>\n",
       "      <td>DDX11L1 gene</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Adrenal-Gland CUI</td>\n",
       "      <td>expressed_in</td>\n",
       "      <td>C2239334</td>\n",
       "      <td>-</td>\n",
       "      <td>DDX11L1 gene</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Artery-Aorta CUI</td>\n",
       "      <td>expressed_in</td>\n",
       "      <td>C2239334</td>\n",
       "      <td>-</td>\n",
       "      <td>DDX11L1 gene</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Artery-Coronary CUI</td>\n",
       "      <td>expressed_in</td>\n",
       "      <td>C2239334</td>\n",
       "      <td>-</td>\n",
       "      <td>DDX11L1 gene</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Artery-Tibial CUI</td>\n",
       "      <td>expressed_in</td>\n",
       "      <td>C2239334</td>\n",
       "      <td>-</td>\n",
       "      <td>DDX11L1 gene</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              source      relation    target  \\\n",
       "0  GTEXEXP:ENSG00000223972-5-Adipose-Subcutaneous...  expressed_in  C2239334   \n",
       "1        GTEXEXP:ENSG00000223972-5-Adrenal-Gland CUI  expressed_in  C2239334   \n",
       "2         GTEXEXP:ENSG00000223972-5-Artery-Aorta CUI  expressed_in  C2239334   \n",
       "3      GTEXEXP:ENSG00000223972-5-Artery-Coronary CUI  expressed_in  C2239334   \n",
       "4        GTEXEXP:ENSG00000223972-5-Artery-Tibial CUI  expressed_in  C2239334   \n",
       "\n",
       "  source_label  target_label      SAB  evidence  \n",
       "0            -  DDX11L1 gene  GTEXEXP       0.0  \n",
       "1            -  DDX11L1 gene  GTEXEXP       0.0  \n",
       "2            -  DDX11L1 gene  GTEXEXP       0.0  \n",
       "3            -  DDX11L1 gene  GTEXEXP       0.0  \n",
       "4            -  DDX11L1 gene  GTEXEXP       0.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "gtex = 'GTEXEXP'\n",
    "for i in glob(\"out/0802/filtered/edges/%s.*edges.csv\"%gtex):\n",
    "    print(i)\n",
    "    df = pd.read_csv(i)\n",
    "    for k,v in df.iterrows():\n",
    "        gid = v[\"source\"]\n",
    "        if gid in score_dict:\n",
    "            df.at[k, 'evidence'] = score_dict[gid]\n",
    "    display(df.head())\n",
    "    df.to_csv(i)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "out/0802/filtered/edges/Gene or Genome.expresses.GTEXEXP.edges.csv\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>relation</th>\n",
       "      <th>target</th>\n",
       "      <th>source_label</th>\n",
       "      <th>target_label</th>\n",
       "      <th>SAB</th>\n",
       "      <th>evidence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C2239334</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Adipose-Subcutaneous...</td>\n",
       "      <td>DDX11L1 gene</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C2239334</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Adrenal-Gland CUI</td>\n",
       "      <td>DDX11L1 gene</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C2239334</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Artery-Aorta CUI</td>\n",
       "      <td>DDX11L1 gene</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C2239334</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Artery-Coronary CUI</td>\n",
       "      <td>DDX11L1 gene</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C2239334</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Artery-Tibial CUI</td>\n",
       "      <td>DDX11L1 gene</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     source   relation                                             target  \\\n",
       "0  C2239334  expresses  GTEXEXP:ENSG00000223972-5-Adipose-Subcutaneous...   \n",
       "1  C2239334  expresses        GTEXEXP:ENSG00000223972-5-Adrenal-Gland CUI   \n",
       "2  C2239334  expresses         GTEXEXP:ENSG00000223972-5-Artery-Aorta CUI   \n",
       "3  C2239334  expresses      GTEXEXP:ENSG00000223972-5-Artery-Coronary CUI   \n",
       "4  C2239334  expresses        GTEXEXP:ENSG00000223972-5-Artery-Tibial CUI   \n",
       "\n",
       "   source_label target_label      SAB  evidence  \n",
       "0  DDX11L1 gene            -  GTEXEXP       0.0  \n",
       "1  DDX11L1 gene            -  GTEXEXP       0.0  \n",
       "2  DDX11L1 gene            -  GTEXEXP       0.0  \n",
       "3  DDX11L1 gene            -  GTEXEXP       0.0  \n",
       "4  DDX11L1 gene            -  GTEXEXP       0.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "out/0802/filtered/edges/Tissue.expresses.GTEXEXP.edges.csv\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>relation</th>\n",
       "      <th>target</th>\n",
       "      <th>source_label</th>\n",
       "      <th>target_label</th>\n",
       "      <th>SAB</th>\n",
       "      <th>evidence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C0222331</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Adipose-Subcutaneous...</td>\n",
       "      <td>Subcutaneous Fat</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>UBERON:0008367 CUI</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Breast-Mammary-Tissu...</td>\n",
       "      <td>breast epithelium</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>UBERON:EFO 0002009 CUI</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Cells-Cultured-fibro...</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>UBERON:EFO 0000572 CUI</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Cells-EBV-transforme...</td>\n",
       "      <td>-</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C1707950</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Esophagus-Mucosa CUI</td>\n",
       "      <td>Esophageal Squamous Epithelium</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   source   relation  \\\n",
       "0                C0222331  expresses   \n",
       "1      UBERON:0008367 CUI  expresses   \n",
       "2  UBERON:EFO 0002009 CUI  expresses   \n",
       "3  UBERON:EFO 0000572 CUI  expresses   \n",
       "4                C1707950  expresses   \n",
       "\n",
       "                                              target  \\\n",
       "0  GTEXEXP:ENSG00000223972-5-Adipose-Subcutaneous...   \n",
       "1  GTEXEXP:ENSG00000223972-5-Breast-Mammary-Tissu...   \n",
       "2  GTEXEXP:ENSG00000223972-5-Cells-Cultured-fibro...   \n",
       "3  GTEXEXP:ENSG00000223972-5-Cells-EBV-transforme...   \n",
       "4     GTEXEXP:ENSG00000223972-5-Esophagus-Mucosa CUI   \n",
       "\n",
       "                     source_label target_label      SAB  evidence  \n",
       "0                Subcutaneous Fat            -  GTEXEXP       0.0  \n",
       "1               breast epithelium            -  GTEXEXP       0.0  \n",
       "2                               -            -  GTEXEXP       0.0  \n",
       "3                               -            -  GTEXEXP       0.0  \n",
       "4  Esophageal Squamous Epithelium            -  GTEXEXP       0.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "out/0802/filtered/edges/Body Part, Organ, or Organ Component.expresses.GTEXEXP.edges.csv\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>relation</th>\n",
       "      <th>target</th>\n",
       "      <th>source_label</th>\n",
       "      <th>target_label</th>\n",
       "      <th>SAB</th>\n",
       "      <th>evidence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C0001625</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Adrenal-Gland CUI</td>\n",
       "      <td>Adrenal Glands</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C0003956</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Artery-Aorta CUI</td>\n",
       "      <td>Ascending aorta structure</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C0205042</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Artery-Coronary CUI</td>\n",
       "      <td>Coronary artery</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C0085427</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Artery-Tibial CUI</td>\n",
       "      <td>Tibial Arteries</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C0005682</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Bladder CUI</td>\n",
       "      <td>Urinary Bladder</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     source   relation                                         target  \\\n",
       "0  C0001625  expresses    GTEXEXP:ENSG00000223972-5-Adrenal-Gland CUI   \n",
       "1  C0003956  expresses     GTEXEXP:ENSG00000223972-5-Artery-Aorta CUI   \n",
       "2  C0205042  expresses  GTEXEXP:ENSG00000223972-5-Artery-Coronary CUI   \n",
       "3  C0085427  expresses    GTEXEXP:ENSG00000223972-5-Artery-Tibial CUI   \n",
       "4  C0005682  expresses          GTEXEXP:ENSG00000223972-5-Bladder CUI   \n",
       "\n",
       "                source_label target_label      SAB  evidence  \n",
       "0             Adrenal Glands            -  GTEXEXP       0.0  \n",
       "1  Ascending aorta structure            -  GTEXEXP       0.0  \n",
       "2            Coronary artery            -  GTEXEXP       0.0  \n",
       "3            Tibial Arteries            -  GTEXEXP       0.0  \n",
       "4            Urinary Bladder            -  GTEXEXP       0.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "out/0802/filtered/edges/Body Location or Region.expresses.GTEXEXP.edges.csv\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>relation</th>\n",
       "      <th>target</th>\n",
       "      <th>source_label</th>\n",
       "      <th>target_label</th>\n",
       "      <th>SAB</th>\n",
       "      <th>evidence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C0736435</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Kidney-Medulla CUI</td>\n",
       "      <td>Set of outer region of renal pyramids</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C0736435</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000227232-5-Kidney-Medulla CUI</td>\n",
       "      <td>Set of outer region of renal pyramids</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C0736435</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000278267-1-Kidney-Medulla CUI</td>\n",
       "      <td>Set of outer region of renal pyramids</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C0736435</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000243485-5-Kidney-Medulla CUI</td>\n",
       "      <td>Set of outer region of renal pyramids</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C0736435</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000237613-2-Kidney-Medulla CUI</td>\n",
       "      <td>Set of outer region of renal pyramids</td>\n",
       "      <td>-</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     source   relation                                        target  \\\n",
       "0  C0736435  expresses  GTEXEXP:ENSG00000223972-5-Kidney-Medulla CUI   \n",
       "1  C0736435  expresses  GTEXEXP:ENSG00000227232-5-Kidney-Medulla CUI   \n",
       "2  C0736435  expresses  GTEXEXP:ENSG00000278267-1-Kidney-Medulla CUI   \n",
       "3  C0736435  expresses  GTEXEXP:ENSG00000243485-5-Kidney-Medulla CUI   \n",
       "4  C0736435  expresses  GTEXEXP:ENSG00000237613-2-Kidney-Medulla CUI   \n",
       "\n",
       "                            source_label target_label      SAB  evidence  \n",
       "0  Set of outer region of renal pyramids            -  GTEXEXP       0.0  \n",
       "1  Set of outer region of renal pyramids            -  GTEXEXP       3.0  \n",
       "2  Set of outer region of renal pyramids            -  GTEXEXP       0.0  \n",
       "3  Set of outer region of renal pyramids            -  GTEXEXP       0.0  \n",
       "4  Set of outer region of renal pyramids            -  GTEXEXP       0.0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "gtex = 'GTEXEXP'\n",
    "for i in glob(\"out/0802/filtered/edges/*.%s.*edges.csv\"%gtex):\n",
    "    print(i)\n",
    "    df = pd.read_csv(i)\n",
    "    for k,v in df.iterrows():\n",
    "        gid = v[\"target\"]\n",
    "        if gid in score_dict:\n",
    "            df.at[k, 'evidence'] = score_dict[gid]\n",
    "    display(df.head())\n",
    "    df.to_csv(i)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filename = \"out/0802/filtered/edges/GTEXEQTL\"\n",
    "scores = pd.read_csv(filename)\n",
    "scores.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "out/0802/filtered/edges/GTEXEQTL.has part.Tissue.edges.csv\n",
      "out/0802/filtered/edges/GTEXEQTL.located in.Body Part, Organ, or Organ Component.edges.csv\n",
      "out/0802/filtered/edges/GTEXEQTL.positively regulates.ENSEMBL.edges.csv\n",
      "out/0802/filtered/edges/GTEXEQTL.has part.Body Part, Organ, or Organ Component.edges.csv\n",
      "out/0802/filtered/edges/GTEXEQTL.negatively regulates.Gene.edges.csv\n",
      "out/0802/filtered/edges/GTEXEQTL.located in.Cell.edges.csv\n",
      "out/0802/filtered/edges/GTEXEQTL.p value.PVALUEBINS.edges.csv\n",
      "out/0802/filtered/edges/GTEXEQTL.negatively regulates.ENSEMBL.edges.csv\n",
      "out/0802/filtered/edges/GTEXEQTL.positively regulates.ENTREZ.edges.csv\n",
      "out/0802/filtered/edges/GTEXEQTL.positively regulates.Gene.edges.csv\n",
      "out/0802/filtered/edges/GTEXEQTL.has part.CLINGEN_ALLELE_REGISTRY.edges.csv\n",
      "out/0802/filtered/edges/GTEXEQTL.located in.CHLO.edges.csv\n",
      "out/0802/filtered/edges/GTEXEQTL.positively regulates.Gene or Genome.edges.csv\n",
      "out/0802/filtered/edges/GTEXEQTL.located in.Tissue.edges.csv\n",
      "out/0802/filtered/edges/GTEXEQTL.located in.Gene or Genome.edges.csv\n",
      "out/0802/filtered/edges/GTEXEQTL.negatively regulates.Gene or Genome.edges.csv\n",
      "out/0802/filtered/edges/GTEXEQTL.located in.EFO.edges.csv\n"
     ]
    }
   ],
   "source": [
    "gtex = 'GTEXEQTL'\n",
    "for i in glob(\"out/0802/filtered/edges/%s.*edges.csv\"%gtex):\n",
    "    print(i)\n",
    "    # df = pd.read_csv(i)\n",
    "    # for k,v in df.iterrows():\n",
    "    #     gid = v[\"source\"]\n",
    "    #     if gid in score_dict:\n",
    "    #         df.at[k, 'evidence'] = score_dict[gid]\n",
    "    # display(df.head())\n",
    "    # df.to_csv(i)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "source          GTEXEQTL:eQTL.chr2.112358021.G.A.b38.Skin.Sun....\n",
       "relation                                                  p_value\n",
       "target                                 PVALUEBINS:1e-12.1e-11 CUI\n",
       "source_label                                                    -\n",
       "target_label                                                    -\n",
       "SAB                                                      GTEXEQTL\n",
       "evidence                                                      NaN\n",
       "Name: 1, dtype: object"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_csv(\"out/0802/filtered/edges/GTEXEQTL.p value.PVALUEBINS.edges.csv\").head().loc[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}