SimplifyUR.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "cZlY2Is9J9wh"
   },
   "source": [
    "# SimplifyUR: Unsupervised Lexical Text Simplification for Urdu"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's start by loading in the packages."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 122
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 27390,
     "status": "ok",
     "timestamp": 1574060846687,
     "user": {
      "displayName": "Namoos Hayat Qasmi",
      "photoUrl": "",
      "userId": "05460789366919507294"
     },
     "user_tz": -300
    },
    "id": "QrS83ldGWXYt",
    "outputId": "5e49a358-d6a8-4cf5-e0ff-d5580292990f"
   },
   "outputs": [],
   "source": [
    "from __future__ import division\n",
    "import gensim\n",
    "from gensim.models import Word2Vec\n",
    "import codecs\n",
    "from pickle import load\n",
    "import collections\n",
    "from nltk.translate.bleu_score import sentence_bleu,SmoothingFunction\n",
    "import unicodedata\n",
    "import re\n",
    "from collections import Counter\n",
    "import sys\n",
    "import copy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Execute the main functions that take the complex sentence as input and output its simplified form."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def handle_grammatical_case(backBackBack, backBack, back, word, forw, forwForw, forwForwForw, caseMarkers, lm):\n",
    "    \n",
    "    forwardWithProb = {}\n",
    "    backwardWithProb = {}\n",
    "    middleWithProb = {}\n",
    "    if (backBack, back, word) in lm.keys():\n",
    "        if (word, forw, forwForw) in lm.keys():\n",
    "            if (back, word, forw) in lm.keys():\n",
    "                return [back, forw]\n",
    "            else:\n",
    "                if forw in caseMarkers:\n",
    "                    for fc in caseMarkers:\n",
    "                        if (word, fc, forwForw) in lm.keys():\n",
    "                            forwardWithProb[lm[(word, fc, forwForw)]] = \\\n",
    "                                (word, fc, forwForw)\n",
    "                    sortForwProb = sorted(forwardWithProb.keys(),\n",
    "                            reverse=True)\n",
    "                    if forwardWithProb:\n",
    "                        for f in sortForwProb:\n",
    "                            if (word, forwardWithProb[f][1], forwForw) \\\n",
    "                                in lm.keys():\n",
    "                                if (back, word, forwardWithProb[f][1]) \\\n",
    "                                    in lm.keys():\n",
    "\n",
    "                                    return [back, forwardWithProb[f][1]]\n",
    "                else:\n",
    "                    return False\n",
    "        else:\n",
    "            if forw in caseMarkers:\n",
    "                for fc in caseMarkers:\n",
    "                    if (word, fc, forwForw) in lm.keys():\n",
    "                        forwardWithProb[lm[(word, fc, forwForw)]] = \\\n",
    "                            (word, fc, forwForw)\n",
    "                sortForwProb = sorted(forwardWithProb.keys(),\n",
    "                        reverse=True)\n",
    "                if forwardWithProb:\n",
    "                    for f in sortForwProb:\n",
    "                        if (word, forwardWithProb[f][1], forwForw) \\\n",
    "                            in lm.keys():\n",
    "                            if (back, word, forwardWithProb[f][1]) \\\n",
    "                                in lm.keys():\n",
    "                                return [back, forwardWithProb[f][1]]\n",
    "            else:\n",
    "                return False\n",
    "        if forw in caseMarkers and (word, forwForw, forwForwForw) \\\n",
    "            in lm.keys():\n",
    "            return [back, '']\n",
    "        else:\n",
    "            return False\n",
    "\n",
    "    if back in caseMarkers:\n",
    "        for c in caseMarkers:\n",
    "            if (backBack, c, word) in lm.keys():\n",
    "                backwardWithProb[lm[(backBack, c, word)]] = (backBack,\n",
    "                        c, word)\n",
    "        sortBackProb = sorted(backwardWithProb.keys(), reverse=True)\n",
    "        if backwardWithProb:\n",
    "            for b in sortBackProb:\n",
    "                if (word, forw, forwForw) in lm.keys():\n",
    "                    if (backwardWithProb[b][1], word, forw) \\\n",
    "                        in lm.keys():\n",
    "                        return [backwardWithProb[b][1], forw]\n",
    "                else:\n",
    "                    if forw in caseMarkers:\n",
    "                        for fc in caseMarkers:\n",
    "                            if (word, fc, forwForw) in lm.keys():\n",
    "                                forwardWithProb[lm[(word, fc,\n",
    "                                        forwForw)]] = (word, fc,\n",
    "                                        forwForw)\n",
    "                        sortForwProb = sorted(forwardWithProb.keys(),\n",
    "                                reverse=True)\n",
    "                        if forwardWithProb:\n",
    "                            for f in sortForwProb:\n",
    "                                if (word, forwardWithProb[f][1],\n",
    "                                    forwForw) in lm.keys():\n",
    "                                    if (backwardWithProb[b][1], word,\n",
    "        forwardWithProb[f][1]) in lm.keys():\n",
    "                                        return [backwardWithProb[b][1],\n",
    "        forwardWithProb[f][1]]\n",
    "                if forw in caseMarkers and (word, forwForw,\n",
    "                        forwForwForw) in lm.keys():\n",
    "                    if (backwardWithProb[b][1], word, forwForw) \\\n",
    "                        in lm.keys():\n",
    "                        return [backwardWithProb[b][1], '']\n",
    "        if back in caseMarkers:\n",
    "            if (backBackBack, backBack, word) in lm.keys():\n",
    "                if (word, forw, forwForw) in lm.keys():\n",
    "                    if (backBack, word, forw) in lm.keys():\n",
    "                        return ['', forw]\n",
    "                    else:\n",
    "                        if forw in caseMarkers:\n",
    "                            for fc in caseMarkers:\n",
    "                                if (word, fc, forwForw) in lm.keys():\n",
    "                                    forwardWithProb[lm[(word, fc,\n",
    "        forwForw)]] = (word, fc, forwForw)\n",
    "                            sortForwProb = \\\n",
    "                                sorted(forwardWithProb.keys(),\n",
    "                                    reverse=True)\n",
    "                            if forwardWithProb:\n",
    "                                for f in sortForwProb:\n",
    "                                    if (word, forwardWithProb[f][1],\n",
    "        forwForw) in lm.keys():\n",
    "                                        if (backBack, word,\n",
    "        forwardWithProb[f][1]) in lm.keys():\n",
    "                                            return ['',\n",
    "        forwardWithProb[f][1]]\n",
    "                else:\n",
    "                    if forw in caseMarkers:\n",
    "                        for fc in caseMarkers:\n",
    "                            if (word, fc, forwForw) in lm.keys():\n",
    "                                forwardWithProb[lm[(word, fc,\n",
    "                                        forwForw)]] = (word, fc,\n",
    "                                        forwForw)\n",
    "                        sortForwProb = sorted(forwardWithProb.keys(),\n",
    "                                reverse=True)\n",
    "                        if forwardWithProb:\n",
    "                            for f in sortForwProb:\n",
    "                                if (word, forwardWithProb[f][1],\n",
    "                                    forwForw) in lm.keys():\n",
    "                                    if (backBack, word,\n",
    "        forwardWithProb[f][1]) in lm.keys():\n",
    "                                        return ['',\n",
    "        forwardWithProb[f][1]]\n",
    "                if forw in caseMarkers and (word, forwForw,\n",
    "                        forwForwForw) in lm.keys():\n",
    "                    return ['', '']\n",
    "                else:\n",
    "                    return False\n",
    "    return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def simplify(content, synModel, tagger, lm, sortedMaxToMinWords, dontRemove):\n",
    "\n",
    "    final = []\n",
    "    orig = []\n",
    "\n",
    "\n",
    "    caseMarkers=[\"تک\",\"تلک\",\"پر\",\"سی\",\"سا\",\"کےلۓ\",\"کے لۓ\",\"کے لیے\",\n",
    "                   \"کےلیے\",\"کیلیے\",\"پاس\",\"پار\",\"ساتھ\",\"سے\",\"کے\",\"کی\",\"کو\",\"کا\",\"میں\",\"نے\",\"کیلئے\"]\n",
    "\n",
    "    temp = \"\"\n",
    "    for sent in content:\n",
    "        sent = unicodedata.normalize('NFC',sent)\n",
    "        orig = sent.split(' ')\n",
    "        li = sent.split(' ')\n",
    "        liTag = tagger.tag(li)\n",
    "\n",
    "        i = -1\n",
    "        indexes = []\n",
    "        for currentWord in liTag:\n",
    "            i += 1\n",
    "            if (currentWord[0] not in dontRemove) and (currentWord[1] == 'NN' or currentWord[1] == 'ADJ' or currentWord[1] == \"VB\" or currentWord[1] == \"ADV\" or currentWord[1] == \"Q\"):\n",
    "                if currentWord[0] in synModel.wv.vocab:\n",
    "                    n = synModel.wv.most_similar(positive=(currentWord[0]), topn = 10)#10)\n",
    "                    index = []\n",
    "                    for oneSyn in n:\n",
    "                        if oneSyn[0] in sortedMaxToMinWords:\n",
    "                            if oneSyn[0] not in currentWord[0] and currentWord[0] not in oneSyn[0] and currentWord[0][0:len(currentWord[0])-1] not in oneSyn[0] and oneSyn[0][0:len(oneSyn[0])-1] not in currentWord[0] and currentWord[0][1:len(currentWord[0])] not in oneSyn[0] and oneSyn[0][1:len(oneSyn[0])] not in currentWord[0]:\n",
    "                                index.append(sortedMaxToMinWords.index(oneSyn[0]))\n",
    "                    index = sorted(index)\n",
    "                    replaceIndex = li.index(currentWord[0])\n",
    "\n",
    "                    for ind in index:\n",
    "                        backBackBack = \"\"\n",
    "                        backBack = \"\"\n",
    "                        back = \"\"\n",
    "                        forw = \"\"\n",
    "                        forwForw = \"\"\n",
    "                        forwForwForw = \"\"\n",
    "\n",
    "                        if i-3 in range(len(orig)):\n",
    "                            backBackBack = orig[i-3]\n",
    "                        else:\n",
    "                            backBackBack = \"<s>\"\n",
    "                        if i-2 in range(len(orig)):\n",
    "                            backBack = orig[i-2]\n",
    "                        else:\n",
    "                            backBack = \"<s>\"\n",
    "                        if i-1 in range(len(orig)):\n",
    "                            back = orig[i-1]\n",
    "                        else:\n",
    "                            back = \"<s>\"\n",
    "                        if i+3 in range(len(orig)):\n",
    "                            forwForwForw = orig[i+3]\n",
    "                        else:\n",
    "                            forwForwForw = \"</s>\"\n",
    "                        if i+2 in range(len(orig)):\n",
    "                            forwForw = orig[i+2]\n",
    "                        else:\n",
    "                            forwForw = \"</s>\"\n",
    "                        if i+1 in range(len(orig)):\n",
    "                            forw = orig[i+1]\n",
    "                        else:\n",
    "                            forw = \"</s>\"\n",
    "                        case = handle_grammatical_case(backBackBack,backBack,back,sortedMaxToMinWords[ind],forw,forwForw,forwForwForw,caseMarkers,lm)\n",
    "                        if case != False:\n",
    "                            orig[i] = sortedMaxToMinWords[ind]\n",
    "                            if i-1 in range(len(orig)):\n",
    "                                orig[i-1] = case[0]\n",
    "                            if i+1 in range(len(orig)):\n",
    "                                orig[i+1] = case[1]\n",
    "                            break\n",
    "\n",
    "        temp = str.join(' ',orig)\n",
    "        final.append(re.sub(r\"\\s{2,}\", \" \", temp))\n",
    "    return final"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "NpAO4aJ0fEeL"
   },
   "source": [
    "## Import necessary models and data files"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Word2Vec Model: A CBOW model trained on different Urdu corpora (over 240 million words) with a window size of 5 words to obtain 300 dimensional vector representations of Urdu words. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 71
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 66351,
     "status": "ok",
     "timestamp": 1574060921616,
     "user": {
      "displayName": "Namoos Hayat Qasmi",
      "photoUrl": "",
      "userId": "05460789366919507294"
     },
     "user_tz": -300
    },
    "id": "twCvQQDNJ8th",
    "outputId": "9ab7e502-46c1-4153-868f-57b5e1f96225"
   },
   "outputs": [],
   "source": [
    "synModel  = Word2Vec.load(\"Models/extended.model\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "F0XnL50YKGns"
   },
   "source": [
    "Parts of Speech (PoS) Tagger: Trained on an Urdu corpus of 95.4 million tokens with 41 tags."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "8pfa1GFDKKxu"
   },
   "outputs": [],
   "source": [
    "file = open('Models/trigramPOS.pkl', 'rb')\n",
    "tagger = load(file)\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "1YTD4M_rKLsk"
   },
   "source": [
    "Language Model: A trigram language model trained on 120.9 million tokens."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "CF5pegAUKPFJ"
   },
   "outputs": [],
   "source": [
    "file = open('Models/trigramsLM.pkl', 'rb')\n",
    "lm = load(file)\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Common_Words.txt: A list of 370 frequently occuring Urdu words."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = codecs.open(\"Data/Common_Words.txt\", \"r\",encoding=\"utf8\")\n",
    "dontRemove = []\n",
    "for line in file:\n",
    "    dontRemove.append(line.strip())\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sorted_Words.txt: A list 14 lac Urdu words sorted (max to min) on their frequency of occurrence. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = codecs.open(\"Data/Sorted_Words.txt\", \"r\",encoding=\"utf8\")\n",
    "sortedMaxToMinWords = []\n",
    "for line in file:\n",
    "    sortedMaxToMinWords.append(line.strip())\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##  Make predictions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's simplify a sentence to see how it works."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Complex Sentence: میرے بلانے پر وہ ٹھٹھک کر مڑے\n",
      "Simple Sentence: میرے بلانے پر وہ چونک کر مڑے\n"
     ]
    }
   ],
   "source": [
    "complex_sentence = ['میرے بلانے پر وہ ٹھٹھک کر مڑے']\n",
    "simple_sentence = simplify(complex_sentence, synModel, tagger, lm, sortedMaxToMinWords, dontRemove)\n",
    "print(\"Complex Sentence: \" + str(complex_sentence[0]))\n",
    "print(\"Simple Sentence: \" + str(simple_sentence[0]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "aJYRoEpAKPvT"
   },
   "source": [
    "## Evaluate the model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Simplify all sentences in our test set. Complex.txt contains 500 complex sentences taken from newspapers, magazines, books and literary journals."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = codecs.open(\"Data/Complex.txt\",'r',encoding='utf8')\n",
    "content = []\n",
    "for line in file:\n",
    "    line = line.replace(u'\\ufeff','')\n",
    "    line = line.strip()\n",
    "    if \"--\" not in line:\n",
    "        content.append(line)\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "final = simplify(content, synModel, tagger, lm, sortedMaxToMinWords, dontRemove)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save the system output in a file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = codecs.open(\"System_Output.txt\", 'w', encoding=\"utf8\")\n",
    "for f in final:\n",
    "    file.write(f.strip()+\"\\n\"+\"\\n\")\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load the system output."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = codecs.open(\"System_Output.txt\",'r',encoding='utf8')\n",
    "candidates = []\n",
    "for line in file:\n",
    "    line = line.replace(u'\\ufeff','')\n",
    "    line = line.strip()\n",
    "    if line != '':\n",
    "        candidates.append(line)\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load the reference simplifications. The simplifications were created by an expert linguist."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 71
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 40114,
     "status": "ok",
     "timestamp": 1574060967158,
     "user": {
      "displayName": "Namoos Hayat Qasmi",
      "photoUrl": "",
      "userId": "05460789366919507294"
     },
     "user_tz": -300
    },
    "id": "ghRwjhaFKdy1",
    "outputId": "e781fd91-daa0-4296-e69c-59b983a31431"
   },
   "outputs": [],
   "source": [
    "file = codecs.open(\"Data/Simplified.txt\",'r',encoding='utf8')\n",
    "temp = []\n",
    "references = []\n",
    "for line in file:\n",
    "    line = line.replace(u'\\ufeff','')\n",
    "    line = line.strip()\n",
    "    if \"--\" not in line:\n",
    "        temp.append(line)\n",
    "    else:\n",
    "        references.append(temp)\n",
    "        temp = []\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "xutSAowJHDDf"
   },
   "source": [
    "Calculate BLEU score."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 2025,
     "status": "ok",
     "timestamp": 1574061178325,
     "user": {
      "displayName": "Namoos Hayat Qasmi",
      "photoUrl": "",
      "userId": "05460789366919507294"
     },
     "user_tz": -300
    },
    "id": "ujBoudrAHFla",
    "outputId": "6b7b5bcd-0a7a-4b7e-fc38-b651f4a535d0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average BLEU Score: 80.15\n"
     ]
    }
   ],
   "source": [
    "references_words = copy.deepcopy(references)\n",
    "\n",
    "i = 0\n",
    "for ref in references_words:\n",
    "    ind = 0\n",
    "    for r in ref:\n",
    "        references_words[i][ind] = r.split(' ')\n",
    "        ind += 1\n",
    "    i += 1\n",
    "\n",
    "average = 0\n",
    "index = 0\n",
    "cc = SmoothingFunction()\n",
    "for candidate in candidates:\n",
    "    BLEU = sentence_bleu(references_words[index], candidate.split(' '), weights=(1, 0, 0, 0), smoothing_function=cc.method1)\n",
    "    average += BLEU\n",
    "    index += 1\n",
    "\n",
    "den = len(candidates)\n",
    "print(\"Average BLEU Score: \" + \"{:.2f}\".format((average/den)*100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "UZ2BghM5IO1O"
   },
   "source": [
    "Calculate SARI score. Implementation taken form https://github.com/cocoxu/simplification/blob/master/SARI.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1585,
     "status": "ok",
     "timestamp": 1574061199021,
     "user": {
      "displayName": "Namoos Hayat Qasmi",
      "photoUrl": "",
      "userId": "05460789366919507294"
     },
     "user_tz": -300
    },
    "id": "mLu3MwD1Hzzx",
    "outputId": "53f4452b-4fd4-4e7f-bb96-d594112b5300"
   },
   "outputs": [],
   "source": [
    "def ReadInFile(filename):\n",
    "    with open(filename) as f:\n",
    "        lines = f.readlines()\n",
    "        lines = [x.strip() for x in lines]\n",
    "    return lines\n",
    "\n",
    "\n",
    "def SARIngram(sgrams, cgrams, rgramslist, numref):\n",
    "    rgramsall = [rgram for rgrams in rgramslist for rgram in rgrams]\n",
    "    rgramcounter = Counter(rgramsall)\n",
    "\n",
    "    sgramcounter = Counter(sgrams)\n",
    "    sgramcounter_rep = Counter()\n",
    "    for sgram, scount in sgramcounter.items():\n",
    "        sgramcounter_rep[sgram] = scount * numref\n",
    "\n",
    "    cgramcounter = Counter(cgrams)\n",
    "    cgramcounter_rep = Counter()\n",
    "    for cgram, ccount in cgramcounter.items():\n",
    "        cgramcounter_rep[cgram] = ccount * numref\n",
    "\n",
    "    # KEEP\n",
    "    keepgramcounter_rep = sgramcounter_rep & cgramcounter_rep\n",
    "    keepgramcountergood_rep = keepgramcounter_rep & rgramcounter\n",
    "    keepgramcounterall_rep = sgramcounter_rep & rgramcounter\n",
    "\n",
    "    keeptmpscore1 = 0\n",
    "    keeptmpscore2 = 0\n",
    "    for keepgram in keepgramcountergood_rep:\n",
    "        keeptmpscore1 += keepgramcountergood_rep[keepgram] / keepgramcounter_rep[keepgram]\n",
    "        keeptmpscore2 += keepgramcountergood_rep[keepgram] / keepgramcounterall_rep[keepgram]\n",
    "        # print \"KEEP\", keepgram, keepscore, cgramcounter[keepgram], sgramcounter[keepgram], rgramcounter[keepgram]\n",
    "    keepscore_precision = 0\n",
    "    if len(keepgramcounter_rep) > 0:\n",
    "        keepscore_precision = keeptmpscore1 / len(keepgramcounter_rep)\n",
    "    keepscore_recall = 0\n",
    "    if len(keepgramcounterall_rep) > 0:\n",
    "        keepscore_recall = keeptmpscore2 / len(keepgramcounterall_rep)\n",
    "    keepscore = 0\n",
    "    if keepscore_precision > 0 or keepscore_recall > 0:\n",
    "        keepscore = 2 * keepscore_precision * keepscore_recall / (keepscore_precision + keepscore_recall)\n",
    "\n",
    "    # DELETION\n",
    "    delgramcounter_rep = sgramcounter_rep - cgramcounter_rep\n",
    "    delgramcountergood_rep = delgramcounter_rep - rgramcounter\n",
    "    delgramcounterall_rep = sgramcounter_rep - rgramcounter\n",
    "    deltmpscore1 = 0\n",
    "    deltmpscore2 = 0\n",
    "    for delgram in delgramcountergood_rep:\n",
    "        deltmpscore1 += delgramcountergood_rep[delgram] / delgramcounter_rep[delgram]\n",
    "        deltmpscore2 += delgramcountergood_rep[delgram] / delgramcounterall_rep[delgram]\n",
    "    delscore_precision = 0\n",
    "    if len(delgramcounter_rep) > 0:\n",
    "        delscore_precision = deltmpscore1 / len(delgramcounter_rep)\n",
    "    delscore_recall = 0\n",
    "    if len(delgramcounterall_rep) > 0:\n",
    "        delscore_recall = deltmpscore1 / len(delgramcounterall_rep)\n",
    "    delscore = 0\n",
    "    if delscore_precision > 0 or delscore_recall > 0:\n",
    "        delscore = 2 * delscore_precision * delscore_recall / (delscore_precision + delscore_recall)\n",
    "\n",
    "    # ADDITION\n",
    "    addgramcounter = set(cgramcounter) - set(sgramcounter)\n",
    "    addgramcountergood = set(addgramcounter) & set(rgramcounter)\n",
    "    addgramcounterall = set(rgramcounter) - set(sgramcounter)\n",
    "\n",
    "    addtmpscore = 0\n",
    "    for addgram in addgramcountergood:\n",
    "        addtmpscore += 1\n",
    "\n",
    "    addscore_precision = 0\n",
    "    addscore_recall = 0\n",
    "    if len(addgramcounter) > 0:\n",
    "        addscore_precision = addtmpscore / len(addgramcounter)\n",
    "    if len(addgramcounterall) > 0:\n",
    "        addscore_recall = addtmpscore / len(addgramcounterall)\n",
    "    addscore = 0\n",
    "    if addscore_precision > 0 or addscore_recall > 0:\n",
    "        addscore = 2 * addscore_precision * addscore_recall / (addscore_precision + addscore_recall)\n",
    "\n",
    "    return (keepscore, delscore_precision, addscore)\n",
    "\n",
    "\n",
    "def SARIsent(ssent, csent, rsents):\n",
    "    numref = len(rsents)\n",
    "\n",
    "    s1grams = ssent.lower().split(\" \")\n",
    "    c1grams = csent.lower().split(\" \")\n",
    "    s2grams = []\n",
    "    c2grams = []\n",
    "    s3grams = []\n",
    "    c3grams = []\n",
    "    s4grams = []\n",
    "    c4grams = []\n",
    "\n",
    "    r1gramslist = []\n",
    "    r2gramslist = []\n",
    "    r3gramslist = []\n",
    "    r4gramslist = []\n",
    "    for rsent in rsents:\n",
    "        r1grams = rsent.lower().split(\" \")\n",
    "        r2grams = []\n",
    "        r3grams = []\n",
    "        r4grams = []\n",
    "        r1gramslist.append(r1grams)\n",
    "        for i in range(0, len(r1grams) - 1):\n",
    "            if i < len(r1grams) - 1:\n",
    "                r2gram = r1grams[i] + \" \" + r1grams[i + 1]\n",
    "                r2grams.append(r2gram)\n",
    "            if i < len(r1grams) - 2:\n",
    "                r3gram = r1grams[i] + \" \" + r1grams[i + 1] + \" \" + r1grams[i + 2]\n",
    "                r3grams.append(r3gram)\n",
    "            if i < len(r1grams) - 3:\n",
    "                r4gram = r1grams[i] + \" \" + r1grams[i + 1] + \" \" + r1grams[i + 2] + \" \" + r1grams[i + 3]\n",
    "                r4grams.append(r4gram)\n",
    "        r2gramslist.append(r2grams)\n",
    "        r3gramslist.append(r3grams)\n",
    "        r4gramslist.append(r4grams)\n",
    "\n",
    "    for i in range(0, len(s1grams) - 1):\n",
    "        if i < len(s1grams) - 1:\n",
    "            s2gram = s1grams[i] + \" \" + s1grams[i + 1]\n",
    "            s2grams.append(s2gram)\n",
    "        if i < len(s1grams) - 2:\n",
    "            s3gram = s1grams[i] + \" \" + s1grams[i + 1] + \" \" + s1grams[i + 2]\n",
    "            s3grams.append(s3gram)\n",
    "        if i < len(s1grams) - 3:\n",
    "            s4gram = s1grams[i] + \" \" + s1grams[i + 1] + \" \" + s1grams[i + 2] + \" \" + s1grams[i + 3]\n",
    "            s4grams.append(s4gram)\n",
    "\n",
    "    for i in range(0, len(c1grams) - 1):\n",
    "        if i < len(c1grams) - 1:\n",
    "            c2gram = c1grams[i] + \" \" + c1grams[i + 1]\n",
    "            c2grams.append(c2gram)\n",
    "        if i < len(c1grams) - 2:\n",
    "            c3gram = c1grams[i] + \" \" + c1grams[i + 1] + \" \" + c1grams[i + 2]\n",
    "            c3grams.append(c3gram)\n",
    "        if i < len(c1grams) - 3:\n",
    "            c4gram = c1grams[i] + \" \" + c1grams[i + 1] + \" \" + c1grams[i + 2] + \" \" + c1grams[i + 3]\n",
    "            c4grams.append(c4gram)\n",
    "\n",
    "    (keep1score, del1score, add1score) = SARIngram(s1grams, c1grams, r1gramslist, numref)\n",
    "    (keep2score, del2score, add2score) = SARIngram(s2grams, c2grams, r2gramslist, numref)\n",
    "    (keep3score, del3score, add3score) = SARIngram(s3grams, c3grams, r3gramslist, numref)\n",
    "    (keep4score, del4score, add4score) = SARIngram(s4grams, c4grams, r4gramslist, numref)\n",
    "    avgkeepscore = sum([keep1score, keep2score, keep3score, keep4score]) / 4\n",
    "    avgdelscore = sum([del1score, del2score, del3score, del4score]) / 4\n",
    "    avgaddscore = sum([add1score, add2score, add3score, add4score]) / 4\n",
    "    finalscore = (avgkeepscore + avgdelscore + avgaddscore) / 3\n",
    "\n",
    "    return finalscore"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average SARI Score: 42.02\n"
     ]
    }
   ],
   "source": [
    "average = 0\n",
    "i = 0\n",
    "SARI = 0\n",
    "\n",
    "for s in content:\n",
    "    SARI = SARIsent(s, candidates[i], references[i])\n",
    "    average += SARI\n",
    "    i += 1\n",
    "\n",
    "print(\"Average SARI Score: \" + \"{:.2f}\".format((average/len(content))*100))"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "machine_shape": "hm",
   "name": "SimplifyUR_Automatic_Text_Simplification_System_for_Urdu.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}