Text Preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://www.kaggle.com/datasets/lakshmi25npathi/imdb-dataset-of-50k-movie-reviews"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('IMDB Dataset.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>review</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>One of the other reviewers has mentioned that ...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A wonderful little production. &lt;br /&gt;&lt;br /&gt;The...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>I thought this was a wonderful way to spend ti...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Basically there's a family where a little boy ...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Petter Mattei's \"Love in the Time of Money\" is...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              review sentiment\n",
       "0  One of the other reviewers has mentioned that ...  positive\n",
       "1  A wonderful little production. <br /><br />The...  positive\n",
       "2  I thought this was a wonderful way to spend ti...  positive\n",
       "3  Basically there's a family where a little boy ...  negative\n",
       "4  Petter Mattei's \"Love in the Time of Money\" is...  positive"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0        One of the other reviewers has mentioned that ...\n",
       "1        A wonderful little production. <br /><br />The...\n",
       "2        I thought this was a wonderful way to spend ti...\n",
       "3        Basically there's a family where a little boy ...\n",
       "4        Petter Mattei's \"Love in the Time of Money\" is...\n",
       "                               ...                        \n",
       "49995    I thought this movie did a down right good job...\n",
       "49996    Bad plot, bad dialogue, bad acting, idiotic di...\n",
       "49997    I am a Catholic taught in parochial elementary...\n",
       "49998    I'm going to have to disagree with the previou...\n",
       "49999    No one expects the Star Trek movies to be high...\n",
       "Name: review, Length: 50000, dtype: object"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['review']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"Basically there's a family where a little boy (Jake) thinks there's a zombie in his closet & his parents are fighting all the time.<br /><br />This movie is slower than a soap opera... and suddenly, Jake decides to become Rambo and kill the zombie.<br /><br />OK, first of all when you're going to make a film you must Decide if its a thriller or a drama! As a drama the movie is watchable. Parents are divorcing & arguing like in real life. And then we have Jake with his closet which totally ruins all the film! I expected to see a BOOGEYMAN similar movie, and instead i watched a drama with some meaningless thriller spots.<br /><br />3 out of 10 just for the well playing parents & descent dialogs. As for the shots with Jake: just ignore them.\""
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['review'][3]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"basically there's a family where a little boy (jake) thinks there's a zombie in his closet & his parents are fighting all the time.<br /><br />this movie is slower than a soap opera... and suddenly, jake decides to become rambo and kill the zombie.<br /><br />ok, first of all when you're going to make a film you must decide if its a thriller or a drama! as a drama the movie is watchable. parents are divorcing & arguing like in real life. and then we have jake with his closet which totally ruins all the film! i expected to see a boogeyman similar movie, and instead i watched a drama with some meaningless thriller spots.<br /><br />3 out of 10 just for the well playing parents & descent dialogs. as for the shots with jake: just ignore them.\""
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['review'][3].lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<pandas.core.strings.StringMethods at 0x20535d1bd90>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['review'].str"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0        one of the other reviewers has mentioned that ...\n",
       "1        a wonderful little production. <br /><br />the...\n",
       "2        i thought this was a wonderful way to spend ti...\n",
       "3        basically there's a family where a little boy ...\n",
       "4        petter mattei's \"love in the time of money\" is...\n",
       "                               ...                        \n",
       "49995    i thought this movie did a down right good job...\n",
       "49996    bad plot, bad dialogue, bad acting, idiotic di...\n",
       "49997    i am a catholic taught in parochial elementary...\n",
       "49998    i'm going to have to disagree with the previou...\n",
       "49999    no one expects the star trek movies to be high...\n",
       "Name: review, Length: 50000, dtype: object"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['review'].str.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['review'] =  df['review'].str.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>review</th>\n",
       "      <th>sentiment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>one of the other reviewers has mentioned that ...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>a wonderful little production. &lt;br /&gt;&lt;br /&gt;the...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>i thought this was a wonderful way to spend ti...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>basically there's a family where a little boy ...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>petter mattei's \"love in the time of money\" is...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49995</th>\n",
       "      <td>i thought this movie did a down right good job...</td>\n",
       "      <td>positive</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49996</th>\n",
       "      <td>bad plot, bad dialogue, bad acting, idiotic di...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49997</th>\n",
       "      <td>i am a catholic taught in parochial elementary...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49998</th>\n",
       "      <td>i'm going to have to disagree with the previou...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49999</th>\n",
       "      <td>no one expects the star trek movies to be high...</td>\n",
       "      <td>negative</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>50000 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  review sentiment\n",
       "0      one of the other reviewers has mentioned that ...  positive\n",
       "1      a wonderful little production. <br /><br />the...  positive\n",
       "2      i thought this was a wonderful way to spend ti...  positive\n",
       "3      basically there's a family where a little boy ...  negative\n",
       "4      petter mattei's \"love in the time of money\" is...  positive\n",
       "...                                                  ...       ...\n",
       "49995  i thought this movie did a down right good job...  positive\n",
       "49996  bad plot, bad dialogue, bad acting, idiotic di...  negative\n",
       "49997  i am a catholic taught in parochial elementary...  negative\n",
       "49998  i'm going to have to disagree with the previou...  negative\n",
       "49999  no one expects the star trek movies to be high...  negative\n",
       "\n",
       "[50000 rows x 2 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### remove HTML Tags"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "use python regular expression "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_html_tags(text):\n",
    "    pattern = re.compile('<.*?>')\n",
    "    return pattern.sub(r'',text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "text_1 = \"basically there's a family where a little boy (jake) thinks there's a zombie in his closet & his parents are fighting all the time.<br /><br />this movie is slower than a soap opera... and suddenly, jake decides to become rambo and kill the zombie.<br /><br />ok, first of all when you're going to make a film you must decide if its a thriller or a drama! as a drama the movie is watchable. parents are divorcing & arguing like in real life. and then we have jake with his closet which totally ruins all the film! i expected to see a boogeyman similar movie, and instead i watched a drama with some meaningless thriller spots.<br /><br />3 out of 10 just for the well playing parents & descent dialogs. as for the shots with jake: just ignore them.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"basically there's a family where a little boy (jake) thinks there's a zombie in his closet & his parents are fighting all the time.this movie is slower than a soap opera... and suddenly, jake decides to become rambo and kill the zombie.ok, first of all when you're going to make a film you must decide if its a thriller or a drama! as a drama the movie is watchable. parents are divorcing & arguing like in real life. and then we have jake with his closet which totally ruins all the film! i expected to see a boogeyman similar movie, and instead i watched a drama with some meaningless thriller spots.3 out of 10 just for the well playing parents & descent dialogs. as for the shots with jake: just ignore them.\""
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "remove_html_tags(\"basically there's a family where a little boy (jake) thinks there's a zombie in his closet & his parents are fighting all the time.<br /><br />this movie is slower than a soap opera... and suddenly, jake decides to become rambo and kill the zombie.<br /><br />ok, first of all when you're going to make a film you must decide if its a thriller or a drama! as a drama the movie is watchable. parents are divorcing & arguing like in real life. and then we have jake with his closet which totally ruins all the film! i expected to see a boogeyman similar movie, and instead i watched a drama with some meaningless thriller spots.<br /><br />3 out of 10 just for the well playing parents & descent dialogs. as for the shots with jake: just ignore them.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0        one of the other reviewers has mentioned that ...\n",
       "1        a wonderful little production. the filming tec...\n",
       "2        i thought this was a wonderful way to spend ti...\n",
       "3        basically there's a family where a little boy ...\n",
       "4        petter mattei's \"love in the time of money\" is...\n",
       "                               ...                        \n",
       "49995    i thought this movie did a down right good job...\n",
       "49996    bad plot, bad dialogue, bad acting, idiotic di...\n",
       "49997    i am a catholic taught in parochial elementary...\n",
       "49998    i'm going to have to disagree with the previou...\n",
       "49999    no one expects the star trek movies to be high...\n",
       "Name: review, Length: 50000, dtype: object"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['review'].apply(remove_html_tags)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['review'] = df['review'].apply(remove_html_tags)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"basically there's a family where a little boy (jake) thinks there's a zombie in his closet & his parents are fighting all the time.this movie is slower than a soap opera... and suddenly, jake decides to become rambo and kill the zombie.ok, first of all when you're going to make a film you must decide if its a thriller or a drama! as a drama the movie is watchable. parents are divorcing & arguing like in real life. and then we have jake with his closet which totally ruins all the film! i expected to see a boogeyman similar movie, and instead i watched a drama with some meaningless thriller spots.3 out of 10 just for the well playing parents & descent dialogs. as for the shots with jake: just ignore them.\""
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['review'][3]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Remove urls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "##### not present in this dataset "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_url(text):\n",
    "    pattern = re.compile(r'https?://\\S+|www\\.\\S+')\n",
    "    return pattern.sub(r'',text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "text1 = ' check out my notebook at https://www.youtube.com/watch?v=6C0sLtw5ctc&list=PLKnIA16_RmvZo7fp5kkIth6nRTeQQsjfX&index=3&ab_channel=CampusX '"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' check out my notebook at https://www.youtube.com/watch?v=6C0sLtw5ctc&list=PLKnIA16_RmvZo7fp5kkIth6nRTeQQsjfX&index=3&ab_channel=CampusX '"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' check out my notebook at  '"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "remove_url(text1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove punctuation marks\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "during time of tokenization , punutation marks are considered or if there is so pace then it would concatenate it with any other text thus hello and hello! are different  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "import string , time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'!\"#$%&\\'()*+,-./:;<=>?@[\\\\]^_`{|}~'"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "string.punctuation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'!\"#$%&\\'()*+,-./:;<=>?@[\\\\]^_`{|}~'"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "exclude = string.punctuation\n",
    "exclude"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_punc(text):\n",
    "    print(text)\n",
    "    for char in text:\n",
    "        if char in exclude:\n",
    "            text = text.replace(char, '')\n",
    "    \n",
    "    return text \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' check ? // .  ok kiusydf $ hhsuahfw %%! '"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_text = ' check ? // . '' ok kiusydf $ hhsuahfw %%! ' \n",
    "test_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " check ? // .  ok kiusydf $ hhsuahfw %%! \n",
      " check     ok kiusydf  hhsuahfw  \n"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "\n",
    "print(remove_punc(test_text))\n",
    "\n",
    "close_time = time.time()\n",
    "\n",
    "time1 =  close_time - start_time\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "# faster method "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_punc2(text):\n",
    "    return text.translate(str.maketrans('','',exclude))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " check     ok kiusydf  hhsuahfw  \n"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "\n",
    "print(remove_punc2(test_text))\n",
    "\n",
    "close_time = time.time()\n",
    "time2 =  close_time - start_time\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0        one of the other reviewers has mentioned that ...\n",
       "1        a wonderful little production the filming tech...\n",
       "2        i thought this was a wonderful way to spend ti...\n",
       "3        basically theres a family where a little boy j...\n",
       "4        petter matteis love in the time of money is a ...\n",
       "                               ...                        \n",
       "49995    i thought this movie did a down right good job...\n",
       "49996    bad plot bad dialogue bad acting idiotic direc...\n",
       "49997    i am a catholic taught in parochial elementary...\n",
       "49998    im going to have to disagree with the previous...\n",
       "49999    no one expects the star trek movies to be high...\n",
       "Name: review, Length: 50000, dtype: object"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['review'].apply(remove_punc2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['review']  =  df['review'].apply(remove_punc2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### chat word treatment "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "# rofl , lmao , fyi , asap , gn "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### spelling correction "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting textblob\n",
      "  Downloading textblob-0.17.1-py2.py3-none-any.whl (636 kB)\n",
      "Requirement already satisfied: nltk>=3.1; python_version >= \"3\" in c:\\programdata\\anaconda3\\lib\\site-packages (from textblob) (3.5)\n",
      "Requirement already satisfied: joblib in c:\\programdata\\anaconda3\\lib\\site-packages (from nltk>=3.1; python_version >= \"3\"->textblob) (0.17.0)\n",
      "Requirement already satisfied: tqdm in c:\\programdata\\anaconda3\\lib\\site-packages (from nltk>=3.1; python_version >= \"3\"->textblob) (4.50.2)\n",
      "Requirement already satisfied: click in c:\\programdata\\anaconda3\\lib\\site-packages (from nltk>=3.1; python_version >= \"3\"->textblob) (7.1.2)\n",
      "Requirement already satisfied: regex in c:\\programdata\\anaconda3\\lib\\site-packages (from nltk>=3.1; python_version >= \"3\"->textblob) (2020.10.15)\n",
      "Installing collected packages: textblob\n",
      "Successfully installed textblob-0.17.1\n"
     ]
    }
   ],
   "source": [
    "!pip install textblob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [],
   "source": [
    "from textblob import TextBlob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [],
   "source": [
    "inorrect_text = \" certtain woorld peeople \""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "correct_text = TextBlob(inorrect_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TextBlob(\" certtain woorld peeople \")"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "correct_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TextBlob(\" certain world people \")"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "correct_text.correct()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' certain world people '"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "correct_text.correct().string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [],
   "source": [
    "# nltk spacey , py spell checkef "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### remove stop word "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "# a , the , of , are , way "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "# don1t remove in parts of speech tagging "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'nltk' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-156-1db350491a93>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mnltk\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdownload\u001b[0m \u001b[1;33m(\u001b[0m\u001b[1;34m'stopwords'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'nltk' is not defined"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\pranj\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Unzipping corpora\\stopwords.zip.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 160,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download ('stopwords')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "stop_words = set(stopwords.words ('english'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'a',\n",
       " 'about',\n",
       " 'above',\n",
       " 'after',\n",
       " 'again',\n",
       " 'against',\n",
       " 'ain',\n",
       " 'all',\n",
       " 'am',\n",
       " 'an',\n",
       " 'and',\n",
       " 'any',\n",
       " 'are',\n",
       " 'aren',\n",
       " \"aren't\",\n",
       " 'as',\n",
       " 'at',\n",
       " 'be',\n",
       " 'because',\n",
       " 'been',\n",
       " 'before',\n",
       " 'being',\n",
       " 'below',\n",
       " 'between',\n",
       " 'both',\n",
       " 'but',\n",
       " 'by',\n",
       " 'can',\n",
       " 'couldn',\n",
       " \"couldn't\",\n",
       " 'd',\n",
       " 'did',\n",
       " 'didn',\n",
       " \"didn't\",\n",
       " 'do',\n",
       " 'does',\n",
       " 'doesn',\n",
       " \"doesn't\",\n",
       " 'doing',\n",
       " 'don',\n",
       " \"don't\",\n",
       " 'down',\n",
       " 'during',\n",
       " 'each',\n",
       " 'few',\n",
       " 'for',\n",
       " 'from',\n",
       " 'further',\n",
       " 'had',\n",
       " 'hadn',\n",
       " \"hadn't\",\n",
       " 'has',\n",
       " 'hasn',\n",
       " \"hasn't\",\n",
       " 'have',\n",
       " 'haven',\n",
       " \"haven't\",\n",
       " 'having',\n",
       " 'he',\n",
       " 'her',\n",
       " 'here',\n",
       " 'hers',\n",
       " 'herself',\n",
       " 'him',\n",
       " 'himself',\n",
       " 'his',\n",
       " 'how',\n",
       " 'i',\n",
       " 'if',\n",
       " 'in',\n",
       " 'into',\n",
       " 'is',\n",
       " 'isn',\n",
       " \"isn't\",\n",
       " 'it',\n",
       " \"it's\",\n",
       " 'its',\n",
       " 'itself',\n",
       " 'just',\n",
       " 'll',\n",
       " 'm',\n",
       " 'ma',\n",
       " 'me',\n",
       " 'mightn',\n",
       " \"mightn't\",\n",
       " 'more',\n",
       " 'most',\n",
       " 'mustn',\n",
       " \"mustn't\",\n",
       " 'my',\n",
       " 'myself',\n",
       " 'needn',\n",
       " \"needn't\",\n",
       " 'no',\n",
       " 'nor',\n",
       " 'not',\n",
       " 'now',\n",
       " 'o',\n",
       " 'of',\n",
       " 'off',\n",
       " 'on',\n",
       " 'once',\n",
       " 'only',\n",
       " 'or',\n",
       " 'other',\n",
       " 'our',\n",
       " 'ours',\n",
       " 'ourselves',\n",
       " 'out',\n",
       " 'over',\n",
       " 'own',\n",
       " 're',\n",
       " 's',\n",
       " 'same',\n",
       " 'shan',\n",
       " \"shan't\",\n",
       " 'she',\n",
       " \"she's\",\n",
       " 'should',\n",
       " \"should've\",\n",
       " 'shouldn',\n",
       " \"shouldn't\",\n",
       " 'so',\n",
       " 'some',\n",
       " 'such',\n",
       " 't',\n",
       " 'than',\n",
       " 'that',\n",
       " \"that'll\",\n",
       " 'the',\n",
       " 'their',\n",
       " 'theirs',\n",
       " 'them',\n",
       " 'themselves',\n",
       " 'then',\n",
       " 'there',\n",
       " 'these',\n",
       " 'they',\n",
       " 'this',\n",
       " 'those',\n",
       " 'through',\n",
       " 'to',\n",
       " 'too',\n",
       " 'under',\n",
       " 'until',\n",
       " 'up',\n",
       " 've',\n",
       " 'very',\n",
       " 'was',\n",
       " 'wasn',\n",
       " \"wasn't\",\n",
       " 'we',\n",
       " 'were',\n",
       " 'weren',\n",
       " \"weren't\",\n",
       " 'what',\n",
       " 'when',\n",
       " 'where',\n",
       " 'which',\n",
       " 'while',\n",
       " 'who',\n",
       " 'whom',\n",
       " 'why',\n",
       " 'will',\n",
       " 'with',\n",
       " 'won',\n",
       " \"won't\",\n",
       " 'wouldn',\n",
       " \"wouldn't\",\n",
       " 'y',\n",
       " 'you',\n",
       " \"you'd\",\n",
       " \"you'll\",\n",
       " \"you're\",\n",
       " \"you've\",\n",
       " 'your',\n",
       " 'yours',\n",
       " 'yourself',\n",
       " 'yourselves'}"
      ]
     },
     "execution_count": 162,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stop_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_stopwords(text):\n",
    "    \n",
    "    new_text = []\n",
    "    \n",
    "    \n",
    "    for word in text.split():\n",
    "        if word in stop_words:\n",
    "            new_text.append(\" \")\n",
    "        else:\n",
    "            new_text.append(word)\n",
    "    \n",
    "    x = new_text[:]\n",
    "    new_text.clear()\n",
    "    \n",
    "    return \" \".join(x)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [],
   "source": [
    "check_text = \" my was if opposite\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'      opposite'"
      ]
     },
     "execution_count": 166,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "remove_stopwords(check_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['review']  =  df['review'].apply(remove_stopwords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0        one       reviewers   mentioned     watching  ...\n",
       "1          wonderful little production   filming techni...\n",
       "2          thought       wonderful way   spend time    ...\n",
       "3        basically theres   family     little boy jake ...\n",
       "4        petter matteis love     time   money     visua...\n",
       "                               ...                        \n",
       "49995      thought   movie       right good job   wasnt...\n",
       "49996    bad plot bad dialogue bad acting idiotic direc...\n",
       "49997          catholic taught   parochial elementary s...\n",
       "49998    im going       disagree     previous comment  ...\n",
       "49999      one expects   star trek movies     high art ...\n",
       "Name: review, Length: 50000, dtype: object"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['review']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### handling emojis "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "def remove_emoji(text):\n",
    "    emoji_pattern = re.compile(\"[\"\n",
    "                           u\"\\U0001F600-\\U0001F64F\"  # emoticons\n",
    "                           u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n",
    "                           u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n",
    "                           u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n",
    "                           u\"\\U00002702-\\U000027B0\"\n",
    "                           u\"\\U000024C2-\\U0001F251\"\n",
    "                           \"]+\", flags=re.UNICODE)\n",
    "    return emoji_pattern.sub(r'', text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Loved the movie. It was '"
      ]
     },
     "execution_count": 170,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "remove_emoji(\"Loved the movie. It was 😘😘\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting emoji\n",
      "  Downloading emoji-2.2.0.tar.gz (240 kB)\n",
      "Building wheels for collected packages: emoji\n",
      "  Building wheel for emoji (setup.py): started\n",
      "  Building wheel for emoji (setup.py): finished with status 'done'\n",
      "  Created wheel for emoji: filename=emoji-2.2.0-py3-none-any.whl size=234912 sha256=835ab8f80146203c19977b501844eab7e9443324119ec600fbcb3db83fd639a2\n",
      "  Stored in directory: c:\\users\\pranj\\appdata\\local\\pip\\cache\\wheels\\86\\62\\9e\\a6b27a681abcde69970dbc0326ff51955f3beac72f15696984\n",
      "Successfully built emoji\n",
      "Installing collected packages: emoji\n",
      "Successfully installed emoji-2.2.0\n"
     ]
    }
   ],
   "source": [
    "!pip install emoji"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Python is :fire:\n"
     ]
    }
   ],
   "source": [
    "import emoji\n",
    "print(emoji.demojize('Python is 🔥'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tokenization "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [],
   "source": [
    "# break text into smaller parts whcih could be words , senetences etc "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [],
   "source": [
    "# usually we do word tokensization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [],
   "source": [
    "#### 1,split fuction of python"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['I', 'am', 'going', 'to', 'delhi']"
      ]
     },
     "execution_count": 178,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sent1 = 'I am going to delhi'\n",
    "sent1.split()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['I am going to delhi',\n",
       " ' I will stay there for 3 days',\n",
       " \" Let's hope the trip to be great\"]"
      ]
     },
     "execution_count": 179,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# sentence tokenization\n",
    "sent2 = 'I am going to delhi. I will stay there for 3 days. Let\\'s hope the trip to be great'\n",
    "sent2.split('.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['I', 'am', 'going', 'to', 'delhi!']"
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Problems with split function\n",
    "sent3 = 'I am going to delhi!'\n",
    "sent3.split()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Where do think I should go? I have 3 day holiday']"
      ]
     },
     "execution_count": 181,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sent4 = 'Where do think I should go? I have 3 day holiday'\n",
    "sent4.split('.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Regular Expression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['I', 'am', 'going', 'to', 'delhi']"
      ]
     },
     "execution_count": 183,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "sent3 = 'I am going to delhi!'\n",
    "tokens = re.findall(\"[\\w']+\", sent3)\n",
    "tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "text = \"\"\"Lorem Ipsum is simply dummy text of the printing and typesetting industry? \n",
    "Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, \n",
    "when an unknown printer took a galley of type and scrambled it to make a type specimen book.\"\"\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences = re.compile('[.!?] ').split(text)\n",
    "#https://www.w3schools.com/python/trypython.asp?filename=demo_regex_set1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: nltk in c:\\programdata\\anaconda3\\lib\\site-packages (3.5)\n",
      "Requirement already satisfied: tqdm in c:\\programdata\\anaconda3\\lib\\site-packages (from nltk) (4.50.2)\n",
      "Requirement already satisfied: click in c:\\programdata\\anaconda3\\lib\\site-packages (from nltk) (7.1.2)\n",
      "Requirement already satisfied: joblib in c:\\programdata\\anaconda3\\lib\\site-packages (from nltk) (0.17.0)\n",
      "Requirement already satisfied: regex in c:\\programdata\\anaconda3\\lib\\site-packages (from nltk) (2020.10.15)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install nltk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     C:\\Users\\pranj\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('punkt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.tokenize import word_tokenize,sent_tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['I', 'am', 'going', 'to', 'visit', 'delhi', '!']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sent1 = 'I am going to visit delhi!'\n",
    "word_tokenize(sent1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Lorem Ipsum is simply dummy text of the printing and typesetting industry?',\n",
       " \"Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, \\nwhen an unknown printer took a galley of type and scrambled it to make a type specimen book.\"]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text = \"\"\"Lorem Ipsum is simply dummy text of the printing and typesetting industry? \n",
    "Lorem Ipsum has been the industry's standard dummy text ever since the 1500s, \n",
    "when an unknown printer took a galley of type and scrambled it to make a type specimen book.\"\"\"\n",
    "\n",
    "sent_tokenize(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['I', 'have', 'a', 'Ph.D', 'in', 'A.I']\n",
      "['We', \"'re\", 'here', 'to', 'help', '!', 'mail', 'us', 'at', 'nks', '@', 'gmail.com']\n",
      "['A', '5km', 'ride', 'cost', '$', '10.50']\n"
     ]
    }
   ],
   "source": [
    "sent5 = 'I have a Ph.D in A.I'\n",
    "sent6 = \"We're here to help! mail us at nks@gmail.com\"\n",
    "sent7 = 'A 5km ride cost $10.50'\n",
    "\n",
    "print(word_tokenize(sent5))\n",
    "print(word_tokenize(sent6))\n",
    "print(word_tokenize(sent7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#### spacy "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting spacy\n",
      "  Using cached spacy-3.4.4-cp38-cp38-win_amd64.whl (12.2 MB)\n",
      "Requirement already satisfied: smart-open<7.0.0,>=5.2.1 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (6.3.0)\n",
      "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (3.3.0)\n",
      "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (1.0.9)\n",
      "Collecting pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4\n",
      "  Using cached pydantic-1.10.4-cp38-cp38-win_amd64.whl (2.2 MB)\n",
      "Requirement already satisfied: pathy>=0.3.5 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (0.10.1)\n",
      "Requirement already satisfied: numpy>=1.15.0 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (1.19.2)\n",
      "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (3.0.8)\n",
      "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (2.0.8)\n",
      "Requirement already satisfied: requests<3.0.0,>=2.13.0 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (2.24.0)\n",
      "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (2.0.7)\n",
      "Collecting thinc<8.2.0,>=8.1.0\n",
      "  Using cached thinc-8.1.7-cp38-cp38-win_amd64.whl (1.3 MB)\n",
      "Collecting spacy-loggers<2.0.0,>=1.0.0\n",
      "  Using cached spacy_loggers-1.0.4-py3-none-any.whl (11 kB)\n",
      "Requirement already satisfied: typer<0.8.0,>=0.3.0 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (0.7.0)\n",
      "Requirement already satisfied: setuptools in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (50.3.1.post20201107)\n",
      "Requirement already satisfied: jinja2 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (2.11.2)\n",
      "Requirement already satisfied: wasabi<1.1.0,>=0.9.1 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (0.10.1)\n",
      "Requirement already satisfied: packaging>=20.0 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (20.4)\n",
      "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (4.50.2)\n",
      "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (2.4.5)\n",
      "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.10 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (3.0.11)\n",
      "Collecting typing-extensions>=4.2.0\n",
      "  Using cached typing_extensions-4.4.0-py3-none-any.whl (26 kB)\n",
      "Requirement already satisfied: idna<3,>=2.5 in c:\\programdata\\anaconda3\\lib\\site-packages (from requests<3.0.0,>=2.13.0->spacy) (2.10)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\programdata\\anaconda3\\lib\\site-packages (from requests<3.0.0,>=2.13.0->spacy) (2020.6.20)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in c:\\programdata\\anaconda3\\lib\\site-packages (from requests<3.0.0,>=2.13.0->spacy) (1.25.11)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in c:\\programdata\\anaconda3\\lib\\site-packages (from requests<3.0.0,>=2.13.0->spacy) (3.0.4)\n",
      "Requirement already satisfied: blis<0.8.0,>=0.7.8 in c:\\programdata\\anaconda3\\lib\\site-packages (from thinc<8.2.0,>=8.1.0->spacy) (0.7.9)\n",
      "Collecting confection<1.0.0,>=0.0.1\n",
      "  Using cached confection-0.0.4-py3-none-any.whl (32 kB)\n",
      "Requirement already satisfied: click<9.0.0,>=7.1.1 in c:\\programdata\\anaconda3\\lib\\site-packages (from typer<0.8.0,>=0.3.0->spacy) (7.1.2)\n",
      "Requirement already satisfied: MarkupSafe>=0.23 in c:\\programdata\\anaconda3\\lib\\site-packages (from jinja2->spacy) (1.1.1)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in c:\\programdata\\anaconda3\\lib\\site-packages (from packaging>=20.0->spacy) (2.4.7)\n",
      "Requirement already satisfied: six in c:\\programdata\\anaconda3\\lib\\site-packages (from packaging>=20.0->spacy) (1.15.0)\n",
      "Installing collected packages: typing-extensions, pydantic, confection, thinc, spacy-loggers, spacy\n",
      "  Attempting uninstall: typing-extensions\n",
      "    Found existing installation: typing-extensions 3.7.4.3\n",
      "    Uninstalling typing-extensions-3.7.4.3:\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "ERROR: Could not install packages due to an EnvironmentError: [WinError 5] Access is denied: 'c:\\\\programdata\\\\anaconda3\\\\lib\\\\site-packages\\\\__pycache__\\\\typing_extensions.cpython-38.pyc'\n",
      "Consider using the `--user` option or check the permissions.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "pip install spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: spacy in c:\\users\\pranj\\appdata\\roaming\\python\\python38\\site-packages (3.4.4)\n",
      "Requirement already satisfied: jinja2 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (2.11.2)\n",
      "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4 in c:\\users\\pranj\\appdata\\roaming\\python\\python38\\site-packages (from spacy) (1.10.4)\n",
      "Requirement already satisfied: numpy>=1.15.0 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (1.19.2)\n",
      "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (2.0.7)\n",
      "Requirement already satisfied: thinc<8.2.0,>=8.1.0 in c:\\users\\pranj\\appdata\\roaming\\python\\python38\\site-packages (from spacy) (8.1.7)\n",
      "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (2.0.8)\n",
      "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (4.50.2)\n",
      "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (3.0.8)\n",
      "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (1.0.9)\n",
      "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in c:\\users\\pranj\\appdata\\roaming\\python\\python38\\site-packages (from spacy) (1.0.4)\n",
      "Requirement already satisfied: typer<0.8.0,>=0.3.0 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (0.7.0)\n",
      "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.10 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (3.0.11)\n",
      "Requirement already satisfied: smart-open<7.0.0,>=5.2.1 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (6.3.0)\n",
      "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (3.3.0)\n",
      "Requirement already satisfied: wasabi<1.1.0,>=0.9.1 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (0.10.1)\n",
      "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (2.4.5)\n",
      "Requirement already satisfied: pathy>=0.3.5 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (0.10.1)\n",
      "Requirement already satisfied: requests<3.0.0,>=2.13.0 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (2.24.0)\n",
      "Requirement already satisfied: packaging>=20.0 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (20.4)\n",
      "Requirement already satisfied: setuptools in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (50.3.1.post20201107)\n",
      "Requirement already satisfied: MarkupSafe>=0.23 in c:\\programdata\\anaconda3\\lib\\site-packages (from jinja2->spacy) (1.1.1)\n",
      "Requirement already satisfied: typing-extensions>=4.2.0 in c:\\users\\pranj\\appdata\\roaming\\python\\python38\\site-packages (from pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4->spacy) (4.4.0)\n",
      "Requirement already satisfied: blis<0.8.0,>=0.7.8 in c:\\programdata\\anaconda3\\lib\\site-packages (from thinc<8.2.0,>=8.1.0->spacy) (0.7.9)\n",
      "Requirement already satisfied: confection<1.0.0,>=0.0.1 in c:\\users\\pranj\\appdata\\roaming\\python\\python38\\site-packages (from thinc<8.2.0,>=8.1.0->spacy) (0.0.4)\n",
      "Requirement already satisfied: click<9.0.0,>=7.1.1 in c:\\programdata\\anaconda3\\lib\\site-packages (from typer<0.8.0,>=0.3.0->spacy) (7.1.2)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in c:\\programdata\\anaconda3\\lib\\site-packages (from requests<3.0.0,>=2.13.0->spacy) (3.0.4)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in c:\\programdata\\anaconda3\\lib\\site-packages (from requests<3.0.0,>=2.13.0->spacy) (1.25.11)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\programdata\\anaconda3\\lib\\site-packages (from requests<3.0.0,>=2.13.0->spacy) (2020.6.20)\n",
      "Requirement already satisfied: idna<3,>=2.5 in c:\\programdata\\anaconda3\\lib\\site-packages (from requests<3.0.0,>=2.13.0->spacy) (2.10)\n",
      "Requirement already satisfied: six in c:\\programdata\\anaconda3\\lib\\site-packages (from packaging>=20.0->spacy) (1.15.0)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in c:\\programdata\\anaconda3\\lib\\site-packages (from packaging>=20.0->spacy) (2.4.7)\n"
     ]
    }
   ],
   "source": [
    "!pip install spacy --user"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (<ipython-input-13-25a08b6b9a52>, line 1)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;36m  File \u001b[1;32m\"<ipython-input-13-25a08b6b9a52>\"\u001b[1;36m, line \u001b[1;32m1\u001b[0m\n\u001b[1;33m    pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.2.0/en_core_web_sm-2.2.5.tar.gz\u001b[0m\n\u001b[1;37m        ^\u001b[0m\n\u001b[1;31mSyntaxError\u001b[0m\u001b[1;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "pip install https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-2.2.0/en_core_web_sm-2.2.5.tar.gz\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\pranj\\AppData\\Roaming\\Python\\Python38\\site-packages\\spacy\\util.py:885: UserWarning: [W094] Model 'en_core_web_sm' (2.2.0) specifies an under-constrained spaCy version requirement: >=2.2.0. This can lead to compatibility problems with older versions, or as new spaCy versions are released, because the model may say it's compatible when it's not. Consider changing the \"spacy_version\" in your meta.json to a version range, with a lower and upper pin. For example: >=3.4.4,<3.5.0\n",
      "  warnings.warn(warn_msg)\n"
     ]
    },
    {
     "ename": "OSError",
     "evalue": "[E053] Could not read config file from C:\\ProgramData\\Anaconda3\\lib\\site-packages\\en_core_web_sm\\en_core_web_sm-2.2.0\\config.cfg",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mOSError\u001b[0m                                   Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-9-12102ff9e1a9>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mnlp\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mspacy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mload\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'en_core_web_sm'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\spacy\\__init__.py\u001b[0m in \u001b[0;36mload\u001b[1;34m(name, vocab, disable, enable, exclude, config)\u001b[0m\n\u001b[0;32m     52\u001b[0m     \u001b[0mRETURNS\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mLanguage\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mThe\u001b[0m \u001b[0mloaded\u001b[0m \u001b[0mnlp\u001b[0m \u001b[0mobject\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     53\u001b[0m     \"\"\"\n\u001b[1;32m---> 54\u001b[1;33m     return util.load_model(\n\u001b[0m\u001b[0;32m     55\u001b[0m         \u001b[0mname\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     56\u001b[0m         \u001b[0mvocab\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mvocab\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\spacy\\util.py\u001b[0m in \u001b[0;36mload_model\u001b[1;34m(name, vocab, disable, enable, exclude, config)\u001b[0m\n\u001b[0;32m    430\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mget_lang_class\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"blank:\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    431\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mis_package\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m  \u001b[1;31m# installed as package\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 432\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mload_model_from_package\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# type: ignore[arg-type]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    433\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mPath\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexists\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m  \u001b[1;31m# path to model data directory\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    434\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mload_model_from_path\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mPath\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# type: ignore[arg-type]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\spacy\\util.py\u001b[0m in \u001b[0;36mload_model_from_package\u001b[1;34m(name, vocab, disable, enable, exclude, config)\u001b[0m\n\u001b[0;32m    466\u001b[0m     \"\"\"\n\u001b[0;32m    467\u001b[0m     \u001b[0mcls\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mimportlib\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mimport_module\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mname\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 468\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mcls\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mload\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvocab\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mvocab\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdisable\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdisable\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0menable\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0menable\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mexclude\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mexclude\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mconfig\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mconfig\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# type: ignore[attr-defined]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    469\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    470\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\en_core_web_sm\\__init__.py\u001b[0m in \u001b[0;36mload\u001b[1;34m(**overrides)\u001b[0m\n\u001b[0;32m     10\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     11\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mload\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m**\u001b[0m\u001b[0moverrides\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 12\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mload_model_from_init_py\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m__file__\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0moverrides\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\spacy\\util.py\u001b[0m in \u001b[0;36mload_model_from_init_py\u001b[1;34m(init_file, vocab, disable, enable, exclude, config)\u001b[0m\n\u001b[0;32m    647\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mmodel_path\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexists\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    648\u001b[0m         \u001b[1;32mraise\u001b[0m \u001b[0mIOError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mErrors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mE052\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdata_path\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 649\u001b[1;33m     return load_model_from_path(\n\u001b[0m\u001b[0;32m    650\u001b[0m         \u001b[0mdata_path\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    651\u001b[0m         \u001b[0mvocab\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mvocab\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\spacy\\util.py\u001b[0m in \u001b[0;36mload_model_from_path\u001b[1;34m(model_path, meta, vocab, disable, enable, exclude, config)\u001b[0m\n\u001b[0;32m    503\u001b[0m     \u001b[0mconfig_path\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mmodel_path\u001b[0m \u001b[1;33m/\u001b[0m \u001b[1;34m\"config.cfg\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    504\u001b[0m     \u001b[0moverrides\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdict_to_dot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 505\u001b[1;33m     \u001b[0mconfig\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mload_config\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mconfig_path\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moverrides\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0moverrides\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    506\u001b[0m     nlp = load_model_from_config(\n\u001b[0;32m    507\u001b[0m         \u001b[0mconfig\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\AppData\\Roaming\\Python\\Python38\\site-packages\\spacy\\util.py\u001b[0m in \u001b[0;36mload_config\u001b[1;34m(path, overrides, interpolate)\u001b[0m\n\u001b[0;32m    679\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    680\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mconfig_path\u001b[0m \u001b[1;32mor\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mconfig_path\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mis_file\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 681\u001b[1;33m             \u001b[1;32mraise\u001b[0m \u001b[0mIOError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mErrors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mE053\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mconfig_path\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"config file\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    682\u001b[0m         return config.from_disk(\n\u001b[0;32m    683\u001b[0m             \u001b[0mconfig_path\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moverrides\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0moverrides\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0minterpolate\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0minterpolate\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mOSError\u001b[0m: [E053] Could not read config file from C:\\ProgramData\\Anaconda3\\lib\\site-packages\\en_core_web_sm\\en_core_web_sm-2.2.0\\config.cfg"
     ]
    }
   ],
   "source": [
    "nlp = spacy.load('en_core_web_sm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: spacy in c:\\users\\pranj\\appdata\\roaming\\python\\python38\\site-packages (3.4.4)\n",
      "Requirement already satisfied: wasabi<1.1.0,>=0.9.1 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (0.10.1)\n",
      "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4 in c:\\users\\pranj\\appdata\\roaming\\python\\python38\\site-packages (from spacy) (1.10.4)\n",
      "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (1.0.9)\n",
      "Requirement already satisfied: setuptools in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (50.3.1.post20201107)\n",
      "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (4.50.2)\n",
      "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (3.0.8)\n",
      "Requirement already satisfied: pathy>=0.3.5 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (0.10.1)\n",
      "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in c:\\users\\pranj\\appdata\\roaming\\python\\python38\\site-packages (from spacy) (1.0.4)\n",
      "Requirement already satisfied: smart-open<7.0.0,>=5.2.1 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (6.3.0)\n",
      "Requirement already satisfied: thinc<8.2.0,>=8.1.0 in c:\\users\\pranj\\appdata\\roaming\\python\\python38\\site-packages (from spacy) (8.1.7)\n",
      "Requirement already satisfied: jinja2 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (2.11.2)\n",
      "Requirement already satisfied: packaging>=20.0 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (20.4)\n",
      "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (2.0.8)\n",
      "Requirement already satisfied: typer<0.8.0,>=0.3.0 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (0.7.0)\n",
      "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (2.0.7)\n",
      "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (2.4.5)\n",
      "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.10 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (3.0.11)\n",
      "Requirement already satisfied: numpy>=1.15.0 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (1.19.2)\n",
      "Requirement already satisfied: requests<3.0.0,>=2.13.0 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (2.24.0)\n",
      "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in c:\\programdata\\anaconda3\\lib\\site-packages (from spacy) (3.3.0)\n",
      "Requirement already satisfied: typing-extensions>=4.2.0 in c:\\users\\pranj\\appdata\\roaming\\python\\python38\\site-packages (from pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4->spacy) (4.4.0)\n",
      "Requirement already satisfied: confection<1.0.0,>=0.0.1 in c:\\users\\pranj\\appdata\\roaming\\python\\python38\\site-packages (from thinc<8.2.0,>=8.1.0->spacy) (0.0.4)\n",
      "Requirement already satisfied: blis<0.8.0,>=0.7.8 in c:\\programdata\\anaconda3\\lib\\site-packages (from thinc<8.2.0,>=8.1.0->spacy) (0.7.9)\n",
      "Requirement already satisfied: MarkupSafe>=0.23 in c:\\programdata\\anaconda3\\lib\\site-packages (from jinja2->spacy) (1.1.1)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in c:\\programdata\\anaconda3\\lib\\site-packages (from packaging>=20.0->spacy) (2.4.7)\n",
      "Requirement already satisfied: six in c:\\programdata\\anaconda3\\lib\\site-packages (from packaging>=20.0->spacy) (1.15.0)\n",
      "Requirement already satisfied: click<9.0.0,>=7.1.1 in c:\\programdata\\anaconda3\\lib\\site-packages (from typer<0.8.0,>=0.3.0->spacy) (7.1.2)\n",
      "Requirement already satisfied: idna<3,>=2.5 in c:\\programdata\\anaconda3\\lib\\site-packages (from requests<3.0.0,>=2.13.0->spacy) (2.10)\n",
      "Requirement already satisfied: chardet<4,>=3.0.2 in c:\\programdata\\anaconda3\\lib\\site-packages (from requests<3.0.0,>=2.13.0->spacy) (3.0.4)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in c:\\programdata\\anaconda3\\lib\\site-packages (from requests<3.0.0,>=2.13.0->spacy) (2020.6.20)\n",
      "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in c:\\programdata\\anaconda3\\lib\\site-packages (from requests<3.0.0,>=2.13.0->spacy) (1.25.11)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Traceback (most recent call last):\n",
      "  File \"C:\\ProgramData\\Anaconda3\\lib\\runpy.py\", line 194, in _run_module_as_main\n",
      "    return _run_code(code, main_globals, None,\n",
      "  File \"C:\\ProgramData\\Anaconda3\\lib\\runpy.py\", line 87, in _run_code\n",
      "    exec(code, run_globals)\n",
      "  File \"C:\\Users\\pranj\\AppData\\Roaming\\Python\\Python38\\site-packages\\spacy\\__main__.py\", line 4, in <module>\n",
      "    setup_cli()\n",
      "  File \"C:\\Users\\pranj\\AppData\\Roaming\\Python\\Python38\\site-packages\\spacy\\cli\\_util.py\", line 71, in setup_cli\n",
      "    command(prog_name=COMMAND)\n",
      "  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\click\\core.py\", line 829, in __call__\n",
      "    return self.main(*args, **kwargs)\n",
      "  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\typer\\core.py\", line 778, in main\n",
      "    return _main(\n",
      "  File \"C:\\ProgramData\\Anaconda3\\lib\\site-packages\\typer\\core.py\", line 190, in _main\n",
      "    args = click.utils._expand_args(args)\n",
      "AttributeError: module 'click.utils' has no attribute '_expand_args'\n"
     ]
    }
   ],
   "source": [
    "!pip install spacy\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "✔ Download and installation successful\n",
      "You can now load the package via spacy.load('en_core_web_sm')\n"
     ]
    }
   ],
   "source": [
    "from spacy.cli import download\n",
    "\n",
    "download(\"en_core_web_sm\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "import spacy\n",
    "nlp = spacy.load('en_core_web_sm')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "doc1 = nlp(sent5)\n",
    "doc2 = nlp(sent6)\n",
    "doc3 = nlp(sent7)\n",
    "doc4 = nlp(sent1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I\n",
      "am\n",
      "going\n",
      "to\n",
      "visit\n",
      "delhi\n",
      "!\n"
     ]
    }
   ],
   "source": [
    "for token in doc4:\n",
    "    print(token)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### stemming "
   ]
  },
  {
   "attachments": {},
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "### walk , walking , walked , -> inflection -> base word walk , we can add prefic and suffix to the word "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "#### stemming reduce inflection  , mapping to root word ,  used escpecially  for informational retreival systems \n",
    "#### stemer is a algo which does stemming for us made by linguist experts , many stemmer possible eg snowball "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.stem.porter import PorterStemmer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "ps = PorterStemmer()\n",
    "def stem_words(text):\n",
    "    return \" \".join([ps.stem(word) for word in text.split()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'walk walk walk walk'"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample = \"walk walks walking walked\"\n",
    "stem_words(sample)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "probably my alltime favorite movie a story of selflessness sacrifice and dedication to a noble cause but its not preachy or boring it just never gets old despite my having seen it some 15 or more times in the last 25 years paul lukas performance brings tears to my eyes and bette davis in one of her very few truly sympathetic roles is a delight the kids are as grandma says more like dressedup midgets than children but that only makes them more fun to watch and the mothers slow awakening to whats happening in the world and under her own roof is believable and startling if i had a dozen thumbs theyd all be up for this movie\n"
     ]
    }
   ],
   "source": [
    "text = 'probably my alltime favorite movie a story of selflessness sacrifice and dedication to a noble cause but its not preachy or boring it just never gets old despite my having seen it some 15 or more times in the last 25 years paul lukas performance brings tears to my eyes and bette davis in one of her very few truly sympathetic roles is a delight the kids are as grandma says more like dressedup midgets than children but that only makes them more fun to watch and the mothers slow awakening to whats happening in the world and under her own roof is believable and startling if i had a dozen thumbs theyd all be up for this movie'\n",
    "print(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'probabl my alltim favorit movi a stori of selfless sacrific and dedic to a nobl caus but it not preachi or bore it just never get old despit my have seen it some 15 or more time in the last 25 year paul luka perform bring tear to my eye and bett davi in one of her veri few truli sympathet role is a delight the kid are as grandma say more like dressedup midget than children but that onli make them more fun to watch and the mother slow awaken to what happen in the world and under her own roof is believ and startl if i had a dozen thumb theyd all be up for thi movi'"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stem_words(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "#### base word might not be a word in eglish lanuage and hence might not make sense , stem might not be a valid word in english lanuage eg \n",
    "### probably , thus can`t be used to give input to user "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "#### lemmatizing -> slower but gives english words back  , root word belongs to lanuage , lemma , use this when slow speed does`nt matters but \n",
    "### we need to give output to user "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "wordnet_lemmatizer = WordNetLemmatizer()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     C:\\Users\\pranj\\AppData\\Roaming\\nltk_data...\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import nltk\n",
    "nltk.download('wordnet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Word                Lemma               \n",
      "He                  He                  \n",
      "was                 be                  \n",
      "running             run                 \n",
      "and                 and                 \n",
      "eating              eat                 \n",
      "at                  at                  \n",
      "same                same                \n",
      "time                time                \n",
      "He                  He                  \n",
      "has                 have                \n",
      "bad                 bad                 \n",
      "habit               habit               \n",
      "of                  of                  \n",
      "swimming            swim                \n",
      "after               after               \n",
      "playing             play                \n",
      "long                long                \n",
      "hours               hours               \n",
      "in                  in                  \n",
      "the                 the                 \n",
      "Sun                 Sun                 \n"
     ]
    }
   ],
   "source": [
    "sentence = \"He was running and eating at same time. He has bad habit of swimming after playing long hours in the Sun.\"\n",
    "punctuations=\"?:!.,;\"\n",
    "sentence_words = nltk.word_tokenize(sentence)\n",
    "for word in sentence_words:\n",
    "    if word in punctuations:\n",
    "        sentence_words.remove(word)\n",
    "\n",
    "sentence_words\n",
    "print(\"{0:20}{1:20}\".format(\"Word\",\"Lemma\"))\n",
    "for word in sentence_words:\n",
    "    print (\"{0:20}{1:20}\".format(word,wordnet_lemmatizer.lemmatize(word,pos='v'))) # pos = v means verb , specifying in whcih\n",
    "    # part of speech we want to do lemmatization "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#### we search in a dictornary to find root word "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}