custom_ner.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Legal Annotator"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this document you find the first steps to annotate legel documents form the Belgian official Gazette. This part needs to be expanded upon to be viable, maybe in a later stage."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import random\n",
    "import spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load('nl_core_news_sm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_file(num):\n",
    "    path = './data/clean_articles/'\n",
    "\n",
    "    names = os.listdir(path)\n",
    "    file = open(path+names[num],'rt')\n",
    "    raw_text = file.read()\n",
    "    file.close()\n",
    "    return raw_text"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Personen en ministers"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We create an annotated text to train the algorithm to analyze legal texts."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "Train_people = [\n",
    "    ('De Minister van de Brusselse Hoofdstedelijke Regering, belast met Financiën, Begroting, Openbaar ambt en de Promotie van Meertaligheid, S. GATZ', { 'entities': [(136, 153, 'PERSON')]}),\n",
    "    ('De minister is belast met de uitvoering van dit besluit. Brussel, 28 mei 2020. Voor de Regering : De Minister-President, R. VERVOORT De Minister van Economie, A. MARON', {'entities': [(121, 132, 'PERSON'), (159, 167, 'PERSON')]}),\n",
    "    ('Gezien om gevoegd te worden bij Ons besluit van 22 juni 2020. FILIP Van Koningswege : De Vice-Eerste Minister en Minister van Financiën, A. DE CROO', { 'entities':[(48, 60, 'DATE'),(62, 67, 'PERSON'), (137, 147, 'PERSON')]}),\n",
    "    ('V. GLATIGNY, Minister van Hoger Onderwijs, Onderwijs voor sociale promotie, Universitaire ziekenhuizen, Hulpverlening aan de Jeugd, Justitiehuizen, Jeugd, Sport en Promotie van Brussel ', {'entities': [(0, 11, 'PERSON')]}),\n",
    "    ('FILIP Van Koningswege', {'entities': [(0,5, 'PERSON')]}),\n",
    "    ('De Minister van Middenstand, Zelfstandigen, KMO, Landbouw en Maatschappelijke Integratie D. DUCARME', {'entities': [(89, 99, 'PERSON')]}),\n",
    "    ('De Minister van Financiën, A. DE CROO', {'entities': [(27, 37, 'PERSON')]}),\n",
    "    ('De Minister van Werk, Economie en Consumenten, N. MUYLLE', {'entities': [(47,56, 'PERSON')]}),\n",
    "    ('De minister-president van de Vlaamse Regering, J. JAMBON', {'entities': [(47,56, 'PERSON')]}),\n",
    "    ('De Vlaamse minister van Economie, Innovatie, Werk, Sociale economie en Landbouw, H. CREVITS', {'entities': [(81, 91, 'PERSON')]}),\n",
    "    ('De Minister van Werk, Vorming, Gezondheid, Sociale Actie, Gelijke Kansen en Vrouwenrechten, Ch. MORREALE', {'entities': [(92, 104, 'PERSON')]}),\n",
    "    ('De Minister-President, E. DI RUPO', {'entities': [(23, 33, 'PERSON')]}),\n",
    "    ('De Minister van Defensie, Ph. GOFFIN', {'entities': [(26, 36, 'PERSON')]}),\n",
    "    ('Hans D\\'HONDT Voorzitter van het Directiecomité', {'entities': [(0, 12, 'PERSON')]}),\n",
    "    ('De Minister van Economie, Buitenlandse Handel, Onderzoek, Innovatie, Digitale Technologieën, Ruimtelijke Ordening, Landbouw, het \"IFAPME\", en de Vaardigheidscentra, W. BORSUS', {'entities': [(165, 174, 'PERSON')] }),\n",
    "    ('De Vice-eersteminister en Minister van Financiën, V. VAN PETEGHEM', {'entities': [(50, 65, 'PERSON')]}),\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "S. GATZ\n",
      "R. VERVOORT\n",
      "A. MARON\n",
      "22 juni 2020\n",
      "FILIP\n",
      "A. DE CROO\n",
      "V. GLATIGNY\n",
      "FILIP\n",
      "D. DUCARME\n",
      "A. DE CROO\n",
      "N. MUYLLE\n",
      "J. JAMBON\n",
      "H. CREVITS\n",
      "Ch. MORREALE\n",
      "E. DI RUPO\n",
      "Ph. GOFFIN\n",
      "Hans D'HONDT\n",
      "W. BORSUS\n",
      "V. VAN PETEGHEM\n"
     ]
    }
   ],
   "source": [
    "# quick check if annotations are set correctly\n",
    "\n",
    "for sent in Train_people:\n",
    "    # print(sent[0][:10])\n",
    "    for ent in sent[1]['entities']:\n",
    "        print(sent[0][ent[0]:ent[1]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from spacy.training import Example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "other_pipes = [pipe for pipe in nlp.pipe_names if pipe != 'ner']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'ner': 0.12171862882431664}\n",
      "{'ner': 6.23167423408818e-05}\n",
      "{'ner': 0.945803488746967}\n",
      "{'ner': 1.2430904164734548e-06}\n",
      "{'ner': 1.027207034757528e-09}\n",
      "{'ner': 5.707490897301728e-09}\n",
      "{'ner': 4.442415159760653e-06}\n",
      "{'ner': 6.754015743684301e-10}\n",
      "{'ner': 1.4418883084647198e-14}\n",
      "{'ner': 1.1353891543057905e-09}\n",
      "{'ner': 6.960589141791904e-06}\n",
      "{'ner': 6.119560557018491e-11}\n",
      "{'ner': 6.536301013904508e-12}\n",
      "{'ner': 1.842800501655478e-09}\n",
      "{'ner': 1.0873874034221245e-09}\n",
      "{'ner': 0.004099525976963241}\n",
      "{'ner': 1.2463475072180033e-11}\n",
      "{'ner': 1.2267756097100325e-12}\n",
      "{'ner': 2.2968688406789127e-08}\n",
      "{'ner': 1.4088111752219956e-17}\n",
      "{'ner': 1.2670568036427502e-08}\n",
      "{'ner': 4.586841254611908e-18}\n",
      "{'ner': 6.50517474337259e-12}\n",
      "{'ner': 6.679771690702079e-06}\n",
      "{'ner': 1.1336367667296384e-13}\n",
      "{'ner': 3.202645932274119e-12}\n",
      "{'ner': 1.5194055011677475e-07}\n",
      "{'ner': 3.4752921297191987e-09}\n",
      "{'ner': 1.8177787392776475e-15}\n",
      "{'ner': 6.626632455485992e-15}\n",
      "{'ner': 1.4788060020118653e-07}\n",
      "{'ner': 3.802104892444596e-16}\n",
      "{'ner': 8.494231784676005e-13}\n",
      "{'ner': 9.425507788811303e-18}\n",
      "{'ner': 3.015689926626068e-16}\n",
      "{'ner': 1.1690531954463743e-10}\n",
      "{'ner': 6.999622739069965e-14}\n",
      "{'ner': 1.906700905961294e-15}\n",
      "{'ner': 2.108575008697702e-13}\n",
      "{'ner': 9.166539852007355e-08}\n",
      "{'ner': 8.01156474002941e-15}\n",
      "{'ner': 1.4867896278476355e-12}\n",
      "{'ner': 4.529087270407155e-14}\n",
      "{'ner': 8.600823944232834e-17}\n",
      "{'ner': 1.7960095113652e-17}\n",
      "{'ner': 6.264384857729181e-13}\n",
      "{'ner': 2.0509030078500768e-16}\n",
      "{'ner': 4.786439293271183e-09}\n",
      "{'ner': 3.206793632687907e-18}\n",
      "{'ner': 2.6003812893984924e-13}\n",
      "{'ner': 6.012877239060591e-14}\n",
      "{'ner': 1.373683479840943e-13}\n",
      "{'ner': 2.600063583879603e-11}\n",
      "{'ner': 1.8398421652437514e-13}\n",
      "{'ner': 2.987951240646975e-08}\n",
      "{'ner': 2.65039814748646e-17}\n",
      "{'ner': 1.8836790269307268e-17}\n",
      "{'ner': 7.082957462843974e-13}\n",
      "{'ner': 3.191223663767112e-16}\n",
      "{'ner': 1.7051851843425005e-14}\n",
      "{'ner': 4.927647047706368e-17}\n",
      "{'ner': 2.148791131749633e-10}\n",
      "{'ner': 1.6077393882050851e-06}\n",
      "{'ner': 2.0658984275904676e-16}\n",
      "{'ner': 2.8910076498409e-16}\n",
      "{'ner': 6.680177157836957e-16}\n",
      "{'ner': 4.405393129793282e-11}\n",
      "{'ner': 3.690455452792696e-10}\n",
      "{'ner': 5.549073310199924e-20}\n",
      "{'ner': 2.101652533956319e-14}\n",
      "{'ner': 2.3652723830675867e-17}\n",
      "{'ner': 2.473586206315087e-17}\n",
      "{'ner': 4.7358012713241525e-18}\n",
      "{'ner': 5.799757204342654e-11}\n",
      "{'ner': 3.737915935318252e-18}\n",
      "{'ner': 2.3208704979485155e-17}\n",
      "{'ner': 3.731923901411915e-16}\n",
      "{'ner': 1.9051032873165682e-16}\n",
      "{'ner': 1.1134306198733881e-21}\n",
      "{'ner': 3.1063195473590606e-20}\n",
      "{'ner': 4.936711709686724e-19}\n",
      "{'ner': 1.1786716207465258e-09}\n",
      "{'ner': 4.68492903522736e-12}\n",
      "{'ner': 2.1920055452527102e-16}\n",
      "{'ner': 1.5753041072383702e-16}\n",
      "{'ner': 4.221173813653411e-15}\n",
      "{'ner': 5.22620209554756e-14}\n",
      "{'ner': 3.8847118169639423e-22}\n",
      "{'ner': 6.673631996643582e-21}\n",
      "{'ner': 4.634778977059173e-19}\n",
      "{'ner': 2.597611256889982e-23}\n",
      "{'ner': 2.2974470488605716e-24}\n",
      "{'ner': 9.139806057487477e-20}\n",
      "{'ner': 9.129307652208035e-18}\n",
      "{'ner': 1.4422814246653333e-16}\n",
      "{'ner': 1.2408386340672425e-16}\n",
      "{'ner': 1.8135206284326226e-17}\n",
      "{'ner': 1.365176440374688e-16}\n",
      "{'ner': 1.7230906431115685e-12}\n",
      "{'ner': 5.318515593217453e-12}\n"
     ]
    }
   ],
   "source": [
    "for pipe in other_pipes:\n",
    "    nlp.disable_pipe(pipe)\n",
    "for i in range(100):\n",
    "        random.shuffle(Train_people)\n",
    "        losses = {}\n",
    "        batches = spacy.util.minibatch(Train_people, size=2)\n",
    "        for batch in batches:\n",
    "                for text, annotation in batch:\n",
    "                        losses = {}\n",
    "                        # create Example\n",
    "                        example = Example.from_dict(nlp.make_doc(text), annotation)\n",
    "                        # Update the model\n",
    "                        nlp.update([example], losses=losses, drop=0.3)\n",
    "        print(losses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "for pipe in other_pipes:\n",
    "    nlp.enable_pipe(pipe)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Testing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "One file is loaded to check the annotations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "i =  20\n",
    "doc = nlp(get_file(i))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Koning der Belgen PERSON\n",
      "Aan \n",
      " PERSON\n",
      "Onze Groet PERSON\n",
      "1 CARDINAL\n",
      "74 CARDINAL\n",
      "2 CARDINAL\n",
      "1 CARDINAL\n",
      "6 april \n",
      "2020 DATE\n",
      "3 CARDINAL\n",
      "2 CARDINAL\n",
      "9 april 2020 DATE\n",
      "28 april 2020 DATE\n",
      "2 CARDINAL\n",
      "9 april 2020 DATE\n",
      "3 CARDINAL\n",
      "9 april 2020 DATE\n",
      "28 april 2020 DATE\n",
      "3 CARDINAL\n",
      "9 april 2020 DATE\n",
      "3 CARDINAL\n",
      "13 \n",
      " DATE\n",
      "3 CARDINAL\n",
      "9 april 2020 DATE\n",
      "5 CARDINAL\n",
      "1 CARDINAL\n",
      "4 CARDINAL\n",
      "9 april 2020 DATE\n",
      "mede-eigendom \n",
      " PERSON\n",
      "28 april 2020 DATE\n",
      "4 CARDINAL\n",
      "9 april 2020 DATE\n",
      "mede-eigendom \n",
      " PERSON\n",
      "6 CARDINAL\n",
      "5 CARDINAL\n",
      "9 april 2020 DATE\n",
      "27 maart 2020 DATE\n",
      "2 CARDINAL\n",
      "31bis CARDINAL\n",
      "28 november 1969 DATE\n",
      "27 juni 1969 DATE\n",
      "28 december 1944 DATE\n",
      "7 CARDINAL\n",
      "6 CARDINAL\n",
      "16 april 2020 DATE\n",
      "8 CARDINAL\n",
      "7 CARDINAL\n",
      "19 april 2020 DATE\n",
      "9 CARDINAL\n",
      "19 april 2020 DATE\n",
      "10 juli 2008 DATE\n",
      "10 CARDINAL\n",
      "19 april 2020 DATE\n",
      "35 CARDINAL\n",
      "24 juni 2020 DATE\n",
      "19 april 2020 DATE\n",
      "11 CARDINAL\n",
      "11 CARDINAL\n",
      "22 april 2020 DATE\n",
      "38 CARDINAL\n",
      "24 juni 2020 DATE\n",
      "27 april 2020 DATE\n",
      "23 maart 2020 DATE\n",
      "22 december 2016 DATE\n",
      "13 CARDINAL\n",
      "14 CARDINAL\n",
      "27 april 2020 DATE\n",
      "27 maart 2020 DATE\n",
      "4 juni 2020 DATE\n",
      "14 CARDINAL\n",
      "15 CARDINAL\n",
      "24 april 2020 DATE\n",
      "13 mei 2020 DATE\n",
      "15 CARDINAL\n",
      "24 april 2020 DATE\n",
      "15 CARDINAL\n",
      "4 mei 2020 DATE\n",
      "3 CARDINAL\n",
      "27 maart 2020 DATE\n",
      "4 juni 2020 DATE\n",
      "4 mei 2020 DATE\n",
      "3 CARDINAL\n",
      "27 maart 2020 DATE\n",
      "20 CARDINAL\n",
      "3 juni 2020 DATE\n",
      "20 CARDINAL\n",
      "2 CARDINAL\n",
      "72 CARDINAL\n",
      "20 CARDINAL\n",
      "31 december 2025 DATE\n",
      "17 CARDINAL\n",
      "1 CARDINAL\n",
      "14 mei 2020 DATE\n",
      "2 CARDINAL\n",
      "14 mei 2020 DATE\n",
      "31 december 2025 DATE\n",
      "18 CARDINAL\n",
      "4 juni \n",
      "2020 DATE\n",
      "40 CARDINAL\n",
      "26 juni 2020 DATE\n",
      "4 juni 2020 DATE\n",
      "19 CARDINAL\n",
      "23 CARDINAL\n",
      "27 maart 2020 DATE\n",
      "20 CARDINAL\n",
      "3 juli 2005 DATE\n",
      "36 CARDINAL\n",
      "23 juni 2020 DATE\n",
      "3 juli 2005 DATE\n",
      "5 juni 2020 DATE\n",
      "2° PERSON\n",
      "27 maart 2020 DATE\n",
      "22 CARDINAL\n",
      "31 van 16 juni 2020 DATE\n",
      "15 juni 2020 DATE\n",
      "33 CARDINAL\n",
      "23 juni 2020 DATE\n",
      "25 CARDINAL\n",
      "34 CARDINAL\n",
      "23 juni 2020 DATE\n",
      "1° PERSON\n",
      "27 maart 2020 DATE\n",
      "26 CARDINAL\n",
      "24 \n",
      "juni 2020 DATE\n",
      "5 CARDINAL\n",
      "27 maart 2020 DATE\n",
      "26 juni 2020 DATE\n",
      "2 CARDINAL\n",
      "1 CARDINAL\n",
      "41 CARDINAL\n",
      "26 juni 2020 DATE\n",
      "23 maart 2020 DATE\n",
      "22 december 2016 DATE\n",
      "42 CARDINAL\n",
      "27 juni 2020 DATE\n",
      "43 CARDINAL\n",
      "26 juni 2020 DATE\n",
      "26 mei 2002 DATE\n",
      "45 van 26 juni 2020 DATE\n",
      "27 maart 2020 DATE\n",
      "46 CARDINAL\n",
      "26 juni 2020 DATE\n",
      "27 maart 2020 DATE\n",
      "33 CARDINAL\n",
      "47 CARDINAL\n",
      "26 juni 2020 DATE\n",
      "3 CARDINAL\n",
      "27 maart 2020 DATE\n",
      "24 december 2020 DATE\n",
      "FILIP PERSON\n",
      "A. DE CROO PERSON\n",
      "P.-Y. DERMAGNE PERSON\n",
      "S. WILMES PERSON\n",
      "G. GILKINET PERSON\n",
      "V. VAN PETEGHEM PERSON\n",
      "Fr. VANDENBROUCKE PERSON\n",
      "P. \n",
      "DE PERSON\n",
      "V. \n",
      "VAN QUICKENBORNE PERSON\n",
      "KMO's en Landbouw PERSON\n",
      "D. CLARINVAL PERSON\n",
      "K. \n",
      "LALIEUX PERSON\n",
      "L. DEDONDER PERSON\n",
      "Z. KHATTABI PERSON\n",
      "A. VERLINDEN PERSON\n",
      "M. KITIR PERSON\n",
      "T. \n",
      "VAN DER STRAETEN PERSON\n",
      "Th. DERMINE PERSON\n",
      "M. MICHEL PERSON\n",
      "S. SCHLITZ PERSON\n",
      "S. MAHDI PERSON\n",
      "E. DE BLEEKER PERSON\n",
      "V. VAN QUICKENBORNE PERSON\n",
      "Nota PERSON\n",
      "55 CARDINAL\n",
      "3 december 2020 DATE\n"
     ]
    }
   ],
   "source": [
    "for ent in doc.ents:\n",
    "    print(ent.text, ent.label_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['tok2vec', 'morphologizer', 'tagger', 'parser', 'lemmatizer', 'attribute_ruler', 'ner']\n"
     ]
    }
   ],
   "source": [
    "print(nlp.pipe_names)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### test on a small doc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this part I want to have a list of tokens were tokens of one entity are combined"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc_small = nlp('Th. DERMINE De Staatssecretaris voor Digitalisering, belast met \\nAdministratieve Vereenvoudiging, Privacy en de Regie der gebouwen, M. MICHEL De Staatssecretaris \\nvoor Gendergelijkheid Gelijke Kansen en Diversiteit, S. SCHLITZ De Staatssecretaris \\nvoor Asiel en Migratie, belast met de Nationale Loterij')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Th. DERMINE PERSON\n",
      "M. MICHEL PERSON\n",
      "S. SCHLITZ PERSON\n"
     ]
    }
   ],
   "source": [
    "for ent in doc_small.ents:\n",
    "    print(ent.text, ent.label_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['DERMINE',\n",
       " 'staatssecretaris',\n",
       " 'digitalisering',\n",
       " 'Administratieve',\n",
       " 'Vereenvoudiging',\n",
       " 'Privacy',\n",
       " 'regie',\n",
       " 'gebouw',\n",
       " 'MICHEL',\n",
       " 'staatssecretaris',\n",
       " 'Gendergelijkheid',\n",
       " 'Gelijke',\n",
       " 'Kansen',\n",
       " 'Diversiteit',\n",
       " 'SCHLITZ',\n",
       " 'staatssecretaris',\n",
       " 'Asiel',\n",
       " 'Migratie',\n",
       " 'nationaal',\n",
       " 'loterij']"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[token.lemma_ for token in doc_small if ((token.is_alpha) and (token.pos_ in ['NOUN', 'ADJ' ,'PROPN'])) or (token.ent_id_ != '')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "ents = [ent for ent in doc_small.ents]\n",
    "ents.reverse()\n",
    "for ent in ents:    \n",
    "    with doc_small.retokenize() as retokenizer: \n",
    "        attrs = {'LEMMA': ent.text, 'is_alpha': True}\n",
    "        retokenizer.merge(ent, attrs=attrs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Th. DERMINE',\n",
       " 'staatssecretaris',\n",
       " 'digitalisering',\n",
       " 'belasten',\n",
       " 'Administratieve',\n",
       " 'Vereenvoudiging',\n",
       " 'Privacy',\n",
       " 'regie',\n",
       " 'gebouw',\n",
       " 'M. MICHEL',\n",
       " 'staatssecretaris',\n",
       " 'Gendergelijkheid',\n",
       " 'Gelijke',\n",
       " 'Kansen',\n",
       " 'Diversiteit',\n",
       " 'S. SCHLITZ',\n",
       " 'staatssecretaris',\n",
       " 'Asiel',\n",
       " 'Migratie',\n",
       " 'belasten',\n",
       " 'nationaal',\n",
       " 'loterij']"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[token.lemma_ for token in doc_small if token.is_alpha and not token.is_stop]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### return to the original text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "i =  20\n",
    "doc = nlp(get_file(i))\n",
    "ents = [ent for ent in doc.ents]\n",
    "ents.reverse()\n",
    "for ent in ents:\n",
    "    if ent.label_ == 'PERSON':    \n",
    "        with doc.retokenize() as retokenizer: \n",
    "            attrs = {'LEMMA': ent.text, 'is_alpha': True}\n",
    "            retokenizer.merge(ent, attrs=attrs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokens = [token.lemma_ for token in doc if token.is_alpha]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Green',\n",
       " 'Deal',\n",
       " 'Z. KHATTABI',\n",
       " 'de',\n",
       " 'minister',\n",
       " 'van',\n",
       " 'Binnenlandse',\n",
       " 'Zaken',\n",
       " 'Institutionele',\n",
       " 'Hervormingen',\n",
       " 'en',\n",
       " 'Democratisch',\n",
       " 'vernieuwing',\n",
       " 'A. VERLINDEN',\n",
       " 'de',\n",
       " 'minister',\n",
       " 'van',\n",
       " 'ontwikkelingssamenwerking',\n",
       " 'belasten',\n",
       " 'met',\n",
       " 'groot',\n",
       " 'Steden',\n",
       " 'M. KITIR',\n",
       " 'de',\n",
       " 'minister',\n",
       " 'van',\n",
       " 'Energie',\n",
       " 'T. \\nVAN DER STRAETEN',\n",
       " 'de',\n",
       " 'staatssecretaris',\n",
       " 'voor',\n",
       " 'Relance',\n",
       " 'en',\n",
       " 'Strategisch',\n",
       " 'Investering',\n",
       " 'belasten',\n",
       " 'met',\n",
       " 'Wetenschapsbeleid',\n",
       " 'Th. DERMINE',\n",
       " 'de',\n",
       " 'staatssecretaris',\n",
       " 'voor',\n",
       " 'digitalisering',\n",
       " 'belasten',\n",
       " 'met',\n",
       " 'Administratieve',\n",
       " 'Vereenvoudiging',\n",
       " 'Privacy',\n",
       " 'en',\n",
       " 'de',\n",
       " 'regie',\n",
       " 'de',\n",
       " 'gebouw',\n",
       " 'M. MICHEL',\n",
       " 'de',\n",
       " 'staatssecretaris',\n",
       " 'voor',\n",
       " 'Gendergelijkheid',\n",
       " 'Gelijke',\n",
       " 'Kansen',\n",
       " 'en',\n",
       " 'Diversiteit',\n",
       " 'S. SCHLITZ',\n",
       " 'de',\n",
       " 'staatssecretaris',\n",
       " 'voor',\n",
       " 'Asiel',\n",
       " 'en',\n",
       " 'Migratie',\n",
       " 'belasten',\n",
       " 'met',\n",
       " 'de',\n",
       " 'nationaal',\n",
       " 'loterij',\n",
       " 'S. MAHDI',\n",
       " 'de',\n",
       " 'staatssecretaris',\n",
       " 'voor',\n",
       " 'Begroting',\n",
       " 'en',\n",
       " 'Consumentenbescherming',\n",
       " 'E. DE BLEEKER',\n",
       " 'met',\n",
       " 'Lands',\n",
       " 'zegel',\n",
       " 'zegelen',\n",
       " 'de',\n",
       " 'minister',\n",
       " 'van',\n",
       " 'justitie',\n",
       " 'V. VAN QUICKENBORNE',\n",
       " 'Nota',\n",
       " 'kamer',\n",
       " 'van',\n",
       " 'volksvertegenwoordiger',\n",
       " 'Documenten',\n",
       " '55',\n",
       " 'verslag',\n",
       " '3',\n",
       " 'december']"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokens[-100:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Koning der Belgen\n",
      "Aan \n",
      "\n",
      "Onze Groet\n",
      "mede-eigendom \n",
      "\n",
      "mede-eigendom \n",
      "\n",
      "2°\n",
      "1°\n",
      "FILIP\n",
      "A. DE CROO\n",
      "P.-Y. DERMAGNE\n",
      "S. WILMES\n",
      "G. GILKINET\n",
      "V. VAN PETEGHEM\n",
      "Fr. VANDENBROUCKE\n",
      "P. \n",
      "DE\n",
      "V. \n",
      "VAN QUICKENBORNE\n",
      "KMO's en Landbouw\n",
      "D. CLARINVAL\n",
      "K. \n",
      "LALIEUX\n",
      "L. DEDONDER\n",
      "Z. KHATTABI\n",
      "A. VERLINDEN\n",
      "M. KITIR\n",
      "T. \n",
      "VAN DER STRAETEN\n",
      "Th. DERMINE\n",
      "M. MICHEL\n",
      "S. SCHLITZ\n",
      "S. MAHDI\n",
      "E. DE BLEEKER\n",
      "V. VAN QUICKENBORNE\n",
      "Nota\n"
     ]
    }
   ],
   "source": [
    "for ent in doc.ents:\n",
    "    if ent.label_ == 'PERSON':\n",
    "        print(ent.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "35c267216a8a9a5de802732b368407c9e859165faf5cda25a6a24bab33258039"
  },
  "kernelspec": {
   "display_name": "Python 3.8.8 64-bit ('3.8.8')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}