ngam_language_model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Odia n-gram language model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import copy\n",
    "import os\n",
    "import random\n",
    "from collections import defaultdict\n",
    "from typing import List\n",
    "\n",
    "import dill\n",
    "import numpy as np\n",
    "from indicnlp.tokenize.indic_tokenize import trivial_tokenize_indic\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# set random seed\n",
    "random_seed = 123\n",
    "random.seed(random_seed)\n",
    "np.random.seed(random_seed)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "read lines from file: 100%|██████████| 3594672/3594672 [00:01<00:00, 1968188.57it/s]\n"
     ]
    }
   ],
   "source": [
    "data_filepath = os.path.join('data/or')\n",
    "assert os.path.isfile(data_filepath)  # sanity check\n",
    "with open(data_filepath, 'r', encoding='utf-8') as f:\n",
    "    lines = [s.strip() for s in tqdm(f.readlines(), desc='read lines from file')]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def tokenize_text(text: List[str]) -> List[List[str]]:\n",
    "    \"\"\"Tokenize text\"\"\"\n",
    "    return [trivial_tokenize_indic(sample) for sample in tqdm(text, desc='tokenize', unit=' samples')]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "tokenize: 100%|██████████| 3594672/3594672 [01:14<00:00, 47932.80 samples/s]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "tokenized_text = tokenize_text(lines)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Split into training and validation data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "\n",
    "num_val = 500\n",
    "\n",
    "# shuffle\n",
    "random.shuffle(tokenized_text)\n",
    "\n",
    "# split\n",
    "tokenized_train, tokenized_val = tokenized_text[:-num_val], tokenized_text[-num_val:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Compute vocabulary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "def compute_vocab(tok_text: List[List[str]]) -> List[str]:\n",
    "    return list(set(\n",
    "        [tok for tokens in tqdm(tok_text, unit=' samples') for tok in tokens]\n",
    "    ))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 3594672/3594672 [00:05<00:00, 639556.79 samples/s]\n"
     ]
    }
   ],
   "source": [
    "odia_vocab = compute_vocab(tokenized_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab size: 778862\n"
     ]
    }
   ],
   "source": [
    "print('vocab size:', len(odia_vocab))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## N-Gram language model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class NGramLM(object):\n",
    "    def __init__(self, n: int, delta: float, vocab: List[str]):\n",
    "        self.n = n\n",
    "        self.delta = delta\n",
    "        self.count = defaultdict(lambda: defaultdict(float))\n",
    "        self.total = defaultdict(float)\n",
    "        self.vocab = vocab\n",
    "        if '<eos>' not in self.vocab:\n",
    "            self.vocab.append('<eos>')\n",
    "        self.vsize = len(vocab)\n",
    "\n",
    "    def estimate(self, sequences: List[List[str]]) -> None:\n",
    "        for sequence_raw in tqdm(sequences, unit=' sequences', desc='LM estimate'):\n",
    "            sequence = ['<bos>'] * (self.n - 1) + sequence_raw + ['<eos>']\n",
    "            for i in range(len(sequence) - self.n + 1):\n",
    "                ngram = tuple(sequence[i:i + self.n])\n",
    "                prefix, word = ngram[:-1], ngram[-1]\n",
    "                self.count[prefix][word] += 1\n",
    "                self.total[prefix] += 1\n",
    "\n",
    "    def sequence_logp(self, sequence_raw: List[str]) -> float:\n",
    "        \"\"\"Compute perplexity for a sequence\"\"\"\n",
    "        sequence = ['<bos>'] * (self.n - 1) + sequence_raw + ['<eos>']\n",
    "        total_logp = 0\n",
    "        for i in range(len(sequence) - self.n + 1):\n",
    "            ngram = tuple(sequence[i:i + self.n])\n",
    "            prefix = ngram[:-1]\n",
    "            word = ngram[-1]\n",
    "            logp = np.log2((self.delta + self.count[prefix][word]) /\n",
    "                           (self.total[prefix] + self.delta * self.vsize))\n",
    "            total_logp += logp\n",
    "        return total_logp\n",
    "\n",
    "    def perplexity(self, sequences: List[List[str]]) -> float:\n",
    "        \"\"\"Compute perplexity for multiple sequences\"\"\"\n",
    "        n_total = 0\n",
    "        logp_total = 0\n",
    "        for sequence_raw in tqdm(sequences, unit=' sequences'):\n",
    "            logp_total += self.sequence_logp(sequence_raw)\n",
    "            n_total += len(sequence_raw) + 1  # add 1 for <eos>\n",
    "        ppl = 2 ** (- (1.0 / n_total) * logp_total)  # the log needs to be in base 2!\n",
    "        return ppl\n",
    "\n",
    "    def generate(self, context: List[str] = None) -> str:\n",
    "        \"\"\"Generate text\"\"\"\n",
    "\n",
    "        if context is None:\n",
    "            prefix = None\n",
    "            context = ['<bos>'] * (self.n - 1)\n",
    "        elif len(context) < self.n - 1:\n",
    "            prefix = copy.deepcopy(context)\n",
    "            context = ['<bos>'] * (self.n - 1 - len(context)) + context\n",
    "        elif len(context) > self.n - 1:\n",
    "            prefix = copy.deepcopy(context)\n",
    "            context = context[-(self.n - 1):]\n",
    "        else:  # len(context) = self.n - 1\n",
    "            prefix = None\n",
    "\n",
    "        output = context\n",
    "        while output[-1] != '<eos>':\n",
    "            # Form conditional distribution to sample from\n",
    "            probs, tokens = [], []\n",
    "            for token in self.count[tuple(context)]:\n",
    "                p = self.count[tuple(context)][token] / self.total[tuple(context)]\n",
    "                probs.append(p)\n",
    "                tokens.append(token)\n",
    "            # Sample\n",
    "            wt = np.random.choice(tokens, p=probs)\n",
    "            output = output + [wt]\n",
    "            context = context[1:] + [wt]\n",
    "        if prefix is not None:\n",
    "            return ' '.join(prefix + output[self.n - 1:])\n",
    "        else:\n",
    "            return ' '.join(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "LM estimate: 100%|██████████| 3594172/3594172 [04:08<00:00, 14486.51 sequences/s]\n",
      "100%|██████████| 500/500 [00:00<00:00, 16002.93 sequences/s]\n"
     ]
    }
   ],
   "source": [
    "ns = [3]\n",
    "deltas = [0.001]\n",
    "\n",
    "lm_odia = {}\n",
    "ppl_odia = {}\n",
    "\n",
    "for n in ns:\n",
    "    for delta in deltas:\n",
    "        lm_odia[n, delta] = NGramLM(n=n, delta=delta, vocab=odia_vocab, )\n",
    "\n",
    "        # estimate\n",
    "        lm_odia[n, delta].estimate(tokenized_train)\n",
    "\n",
    "        # compute perplexity\n",
    "        ppl_odia[n, delta] = lm_odia[n, delta].perplexity(tokenized_val)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "## Generate Odia text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------------------------------------------------------------------------\n",
      "Odia LM, n = 3, delta = 0.0010\n",
      "<bos> <bos> ରୋହିତ ଶର୍ମା ଏହି କାରଣରୁ ବୈତରଣୀ , ବିପଦ ଆଶଙ୍କା , ଏହି ରେସ୍ରେ ପାଣ୍ଡ୍ୟାଙ୍କ ଦମ୍ ବାହାରି ଯାଇଥିଲା । <eos>\n",
      "<bos> <bos> କିନ୍ତୁ ଆଇନକୁ ଆଖିଠାର ମାରି ଏହାର ଚମଡ଼ା ବିକ୍ରୀ କରିବା ପାଇଁ ଯାଉଛନ୍ତି । <eos>\n",
      "<bos> <bos> ଅନ୍ୟପଟେ , ବିଜେପି ସହ ଯେଉଁ ସ୍ତରଦେଇ ରାସ୍ତାରେ ପିଚୁ ନିର୍ମାଣ ଶେଷ ହେବ । <eos>\n",
      "<bos> <bos> ପୂର୍ବରୁ ୨ଟି କେନ୍ଦ୍ର ଶାସିତ ଅଞ୍ଚଳରେ ଅପ୍ରୀତିକର ପରିସ୍ଥିତି ନ ଉପୁଜେ ଏଥିପ୍ରତି ଧ୍ୟାନ ଦିଅନ୍ତୁ । <eos>\n",
      "<bos> <bos> ଆପଣଙ୍କୁ ଟିକିଏ ଭାଗ୍ୟବାନ ହେବାକୁ ପଡିବ । <eos>\n",
      "-----------------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "for n in ns:\n",
    "    for delta in deltas:\n",
    "        print(89 * '-')\n",
    "        print(f'Odia LM, n = {n}, delta = {delta:.4f}')\n",
    "        for _ in range(5):\n",
    "            print(lm_odia[n, delta].generate())\n",
    "        print(89 * '-')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": "'ଆସି ଚାଲିଯିବ ପଛେ ଆମେ ଜାଣିପାରିବାନି । <eos>'"
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lm_odia[3, deltas[0]].generate(context=['ଆସି'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# dump lm to file\n",
    "with open(os.path.join('ngram.lm.pkl'), 'wb') as f:\n",
    "    s = dill.dumps(lm_odia[3, deltas[0]])\n",
    "    f.write(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}