module3/s1_keywords.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extraction de Keywords"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import yake"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Extraire les mots clés d'un document avec Yake"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://github.com/LIAAD/yake"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'bruxelles', 'conseil communal', 'conseil général'}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Création d'une liste de mots à ignorer\n",
    "# set([]) dedouble la liste\n",
    "ignored = set([\"conseil communal\", \"conseil général\", \"bruxelles\"])\n",
    "ignored"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<yake.yake.KeywordExtractor at 0x7f33863debe0>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Instantier l'extracteur de mots clés\n",
    "# KeyWord Extractor contiens une fonction\n",
    "# top = 50 est arbritraire\n",
    "kw_extractor = yake.KeywordExtractor(lan=\"fr\", top=50)\n",
    "kw_extractor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lister les Fichiers\n",
    "data_path = \"../data/txt/\"\n",
    "files = os.listdir(data_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2026"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Imprimer le nombre de fichiers identifiés\n",
    "len(files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Bxl_1847_Tome_I1_Part_1.txt',\n",
       " 'Bxl_1847_Tome_I1_Part_2.txt',\n",
       " 'Bxl_1847_Tome_I1_Part_3.txt',\n",
       " 'Bxl_1847_Tome_I1_Part_4.txt',\n",
       " 'Bxl_1847_Tome_I1_Part_5.txt',\n",
       " 'Bxl_1848_Tome_I1_Part_1.txt',\n",
       " 'Bxl_1848_Tome_I1_Part_2.txt',\n",
       " 'Bxl_1848_Tome_I1_Part_3.txt',\n",
       " 'Bxl_1849_Tome_I1_Part_1.txt',\n",
       " 'Bxl_1849_Tome_I1_Part_2.txt',\n",
       " 'Bxl_1849_Tome_I1_Part_3.txt',\n",
       " 'Bxl_1849_Tome_I1_Part_4.txt',\n",
       " 'Bxl_1849_Tome_I1_Part_5.txt',\n",
       " 'Bxl_1849_Tome_II1_Part_1.txt',\n",
       " 'Bxl_1849_Tome_II1_Part_2.txt',\n",
       " 'Bxl_1849_Tome_II1_Part_3.txt',\n",
       " 'Bxl_1849_Tome_II1_Part_4.txt',\n",
       " 'Bxl_1849_Tome_II1_Part_5.txt',\n",
       " 'Bxl_1849_Tome_II1_Part_6.txt',\n",
       " 'Bxl_1849_Tome_II1_Part_7.txt']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Les dix premiers fichiers\n",
    "files[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1992"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Enlever les fichiers qui ne commencent pas par Bxl_\n",
    "bxl_files = [f for f in files if f.startswith('Bxl_')]\n",
    "len(bxl_files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Bxl_1847_Tome_I1_Part_1.txt'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Choisir un fichier\n",
    "this_file = bxl_files[0]\n",
    "this_file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"V I L L E DE\\n\\nBRUXELLES\\n\\nbulletin ires 8éanas\\nDl!\\n\\nCONSEIL\\n\\nCOMMUNAL\\n\\nANNÉE\\n\\n1847.\\n\\n\\x0cAU\\n\\n\\x0cVILLE DE B R U X E L L E S .\\n\\nbulletin\\n\\nCONSEIL\\n\\nàes\\n\\nSéances\\n\\nCOMMUNAL.\\n\\nANNÉE\\n\\n1847.\\n\\nBRUXELLES,\\nIMPRIMERIE\\n\\nD E J . H. B R I A R D ,\\n\\nRITE N E U V E , 3 1 , FAUBOURG DE N A M U R ,\\n\\n1 84 8\\n\\n\\x0cDE!\\n\\nDU CONSEI\\nDîBÏ\\n\\nE. - Communication\\nconclusions de la section des\\ndu nouvel hospice pour les av\\n\\nEnraisonde l'absence &\\nmaladie.le Conseil ajourne\\nleurs de pierre el marchai\\ncles des taxes communale'\\nbieniàance e\""
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Récupérer le texte du fichier\n",
    "text = open(os.path.join(data_path, this_file), 'r').read()\n",
    "text[:500]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Extraire les mots clés de ce texte\n",
    "keywords = kw_extractor.extract_keywords(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('marché', 0.00015029560074715333),\n",
       " ('dans', 0.00025307923380123577),\n",
       " ('CONSEIL', 0.0002896026299095025),\n",
       " (\"d'un marché couvert\", 0.00038300149546329404),\n",
       " ('marchés couverts', 0.0004276960207809479),\n",
       " ('marchés', 0.0004407998491711812),\n",
       " (\"d'un marché dans\", 0.00047380845640108494),\n",
       " (\"d'un marché\", 0.0005054375296920362),\n",
       " ('VILLE', 0.0005449628477985323),\n",
       " ('marché couvert', 0.0005583809160195709),\n",
       " ('nouveau marché', 0.0006935528782713473),\n",
       " (\"l'établissement d'un marché\", 0.0007124503616443153),\n",
       " ('marché dans', 0.0009249108286977143),\n",
       " ('Bortier', 0.0009602384499736716),\n",
       " (\"qu'il\", 0.0010280312600004049),\n",
       " ('rue', 0.0011547164914727408),\n",
       " ('BRUXELLES', 0.0012658247679985801),\n",
       " (\"d'un\", 0.0014453642815040189),\n",
       " (\"construction d'un marché\", 0.00152548007392674),\n",
       " ('Conseil général', 0.0015984204266513492),\n",
       " ('marché Saint-Jean', 0.0016222814416333386),\n",
       " ('CONSEIL COMMUNAL', 0.0017261255673432948),\n",
       " (\"c'est\", 0.0019181549803940444),\n",
       " (\"ville d'un marché\", 0.0019310426378027572),\n",
       " ('faire', 0.0019366844354443231),\n",
       " ('marchés couverts dans', 0.0020351416144506365),\n",
       " ('marché projeté', 0.0020533980187456393),\n",
       " ('Madeleine', 0.0021592341049713965),\n",
       " ('Conseil central', 0.002195880736607183),\n",
       " ('mètres', 0.0022384542389980453),\n",
       " ('section', 0.0022425380056325726),\n",
       " ('rue Royale', 0.0025678714980423597),\n",
       " ('marché nouveau dans', 0.0026908818596917713),\n",
       " ('marché projeté rue', 0.0026916034436206154),\n",
       " ('terrains', 0.0028406857031578327),\n",
       " ('serait', 0.003031470763632522),\n",
       " (\"propriétaire d'un marché\", 0.0030914310274931986),\n",
       " ('Hospices', 0.003175516202119258),\n",
       " ('Art', 0.003204225810114583),\n",
       " ('rapport', 0.0032565253901411712),\n",
       " ('terrain', 0.0033214171298460815),\n",
       " ('droit', 0.0033373164338846694),\n",
       " ('mètre', 0.0033949889291470347),\n",
       " ('centimes', 0.003447134874572461),\n",
       " (\"l'administration\", 0.0034747917125426504),\n",
       " ('produit', 0.0035509718775508453),\n",
       " ('sections', 0.003604078937623777),\n",
       " (\"faire d'un marché\", 0.003619220026532306),\n",
       " ('Société Philanthropique', 0.0036541711120919858),\n",
       " (\"J'ai\", 0.0036767997420869567)]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "keywords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['marchés couverts',\n",
       " \"d'un marché\",\n",
       " 'marché couvert',\n",
       " 'nouveau marché',\n",
       " 'marché dans',\n",
       " 'marché Saint-Jean',\n",
       " 'marché projeté',\n",
       " 'Conseil central',\n",
       " 'rue Royale',\n",
       " 'Société Philanthropique']"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Ne garder que les bigrammes\n",
    "kept = []\n",
    "for kw, score in keywords:\n",
    "    words = kw.split()\n",
    "    if len(words) == 2 and kw.lower() not in ignored:\n",
    "        kept.append(kw)\n",
    "kept"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Faire la même opération sur tous les documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Bxl_1847_Tome_I1_Part_1.txt\n",
      "Bxl_1847_Tome_I1_Part_1.txt mentions these keywords: marchés couverts, d'un marché, marché couvert, nouveau marché, marché dans, marché saint-jean, marché projeté, marchés actuels, marchés marché...\n",
      "Bxl_1847_Tome_I1_Part_2.txt\n",
      "Bxl_1847_Tome_I1_Part_2.txt mentions these keywords: belgique communale, l'administration communale...\n",
      "Bxl_1847_Tome_I1_Part_3.txt\n",
      "Bxl_1847_Tome_I1_Part_3.txt mentions these keywords: rue royale, bons communaux, d'un marché, d'une place, qu'il faut...\n"
     ]
    }
   ],
   "source": [
    "for f in sorted(bxl_files)[:3]:\n",
    "    print(f)\n",
    "    text = open(os.path.join(data_path, f), 'r').read()\n",
    "    keywords = kw_extractor.extract_keywords(text.lower())\n",
    "    kept = []\n",
    "    for kw, score in keywords:\n",
    "        words = kw.split()\n",
    "        # condition pour pouvoir les ajouter dans la liste\n",
    "        if len(words) == 2 and kw.lower() not in ignored:\n",
    "            kept.append(kw)\n",
    "    print(f\"{f} mentions these keywords: {', '.join(kept)}...\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}