Language_Identifier.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JZX2-XzMwnmC"
      },
      "source": [
        "We use google colab to implement our langauge Identifier.     "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GiJkZyFMx4xz",
        "outputId": "e2484313-41dd-4b49-e4f3-19eef1950512"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jqZJUDKIwnmG"
      },
      "source": [
        "### 1. Importing libraries"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "q0wXYefPwnmI"
      },
      "outputs": [],
      "source": [
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np \n",
        "import pandas as pd\n",
        "import itertools\n",
        "import re\n",
        "import codecs\n",
        "import string\n",
        "import pickle\n",
        "from sklearn import feature_extraction\n",
        "from sklearn import linear_model\n",
        "from sklearn.naive_bayes import MultinomialNB\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.tree import DecisionTreeClassifier\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn import pipeline\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn import metrics"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hKaXDo6hwnmK"
      },
      "source": [
        "### 2. Load dataset\n",
        "To train the model, we use the text data from the famous [European Parliament Proceedings Parallel Corpus](https://www.statmt.org/europarl/). For our convenience, we use just the following four langauges to train and test our Language Identification system: English, German, French and Spanish.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 299
        },
        "id": "mjdPRwajwnmK",
        "outputId": "c73ede0c-8151-4382-c692-681636622a4c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/IPython/core/interactiveshell.py:2882: FutureWarning: In a future version of pandas all arguments of read_csv except for the argument 'filepath_or_buffer' will be keyword-only\n",
            "  exec(code_obj, self.user_global_ns, self.user_ns)\n",
            "/usr/local/lib/python3.7/dist-packages/pandas/util/_decorators.py:311: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n",
            "  return func(*args, **kwargs)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                             English\n",
              "0                          Resumption of the session\n",
              "1  I declare resumed the session of the European ...\n",
              "2  Although, as you will have seen, the dreaded '...\n",
              "3  You have requested a debate on this subject in...\n",
              "4  In the meantime, I should like to observe a mi..."
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-4cc0f0fe-d30c-4ab2-be14-7911e708a99c\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>English</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Resumption of the session</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>I declare resumed the session of the European ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Although, as you will have seen, the dreaded '...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>You have requested a debate on this subject in...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>In the meantime, I should like to observe a mi...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4cc0f0fe-d30c-4ab2-be14-7911e708a99c')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-4cc0f0fe-d30c-4ab2-be14-7911e708a99c button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-4cc0f0fe-d30c-4ab2-be14-7911e708a99c');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ],
      "source": [
        "# English data\n",
        "\n",
        "english_data = pd.read_csv(\"/content/drive/MyDrive/Language_Detection/data/europarl-v7.de-en.en\", \"ut-8\", header=None, names=[\"English\"])\n",
        "english_data.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 299
        },
        "id": "3X3kB5aYwnmN",
        "outputId": "7012ded5-c463-4991-eee8-b12dee85d84f"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/IPython/core/interactiveshell.py:2882: FutureWarning: In a future version of pandas all arguments of read_csv except for the argument 'filepath_or_buffer' will be keyword-only\n",
            "  exec(code_obj, self.user_global_ns, self.user_ns)\n",
            "/usr/local/lib/python3.7/dist-packages/pandas/util/_decorators.py:311: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n",
            "  return func(*args, **kwargs)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                              German\n",
              "0                 Wiederaufnahme der Sitzungsperiode\n",
              "1  Ich erkläre die am Freitag, dem 17. Dezember u...\n",
              "2  Wie Sie feststellen konnten, ist der gefürchte...\n",
              "3  Im Parlament besteht der Wunsch nach einer Aus...\n",
              "4  Heute möchte ich Sie bitten - das ist auch der..."
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-ad55ac57-8f11-43be-b922-18fce0a03afb\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>German</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Wiederaufnahme der Sitzungsperiode</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Ich erkläre die am Freitag, dem 17. Dezember u...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Wie Sie feststellen konnten, ist der gefürchte...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Im Parlament besteht der Wunsch nach einer Aus...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Heute möchte ich Sie bitten - das ist auch der...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-ad55ac57-8f11-43be-b922-18fce0a03afb')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-ad55ac57-8f11-43be-b922-18fce0a03afb button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-ad55ac57-8f11-43be-b922-18fce0a03afb');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ],
      "source": [
        "# German data\n",
        "german_data = pd.read_csv(\"/content/drive/MyDrive/Language_Detection/data/europarl-v7.de-en.de\", \"utf-8\", header=None, names=[\"German\"])\n",
        "german_data.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 299
        },
        "id": "b_beS80UwnmQ",
        "outputId": "3ea181a3-bc63-48f8-a342-760b078f3cc7"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/IPython/core/interactiveshell.py:2882: FutureWarning: In a future version of pandas all arguments of read_csv except for the argument 'filepath_or_buffer' will be keyword-only\n",
            "  exec(code_obj, self.user_global_ns, self.user_ns)\n",
            "/usr/local/lib/python3.7/dist-packages/pandas/util/_decorators.py:311: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n",
            "  return func(*args, **kwargs)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                             Spanish\n",
              "0                Reanudación del período de sesiones\n",
              "1  Declaro reanudado el período de sesiones del P...\n",
              "2  Como todos han podido comprobar, el gran \"efec...\n",
              "3  Sus Señorías han solicitado un debate sobre el...\n",
              "4  A la espera de que se produzca, de acuerdo con..."
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-3a573d50-b8eb-4764-9430-6da8b3693c9a\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Spanish</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Reanudación del período de sesiones</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Declaro reanudado el período de sesiones del P...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Como todos han podido comprobar, el gran \"efec...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Sus Señorías han solicitado un debate sobre el...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>A la espera de que se produzca, de acuerdo con...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-3a573d50-b8eb-4764-9430-6da8b3693c9a')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-3a573d50-b8eb-4764-9430-6da8b3693c9a button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-3a573d50-b8eb-4764-9430-6da8b3693c9a');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ],
      "source": [
        "# Spanish data\n",
        "\n",
        "spanish_data = pd.read_csv(\"/content/drive/MyDrive/Language_Detection/data/europarl-v7.es-en.es\", \"utf-8\", header=None, names=[\"Spanish\"])\n",
        "spanish_data.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 299
        },
        "id": "kNIbJUBJwnmO",
        "outputId": "8b664931-1d93-4619-c4eb-1fdecd7e77cf"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/IPython/core/interactiveshell.py:2882: FutureWarning: In a future version of pandas all arguments of read_csv except for the argument 'filepath_or_buffer' will be keyword-only\n",
            "  exec(code_obj, self.user_global_ns, self.user_ns)\n",
            "/usr/local/lib/python3.7/dist-packages/pandas/util/_decorators.py:311: ParserWarning: Falling back to the 'python' engine because the 'c' engine does not support regex separators (separators > 1 char and different from '\\s+' are interpreted as regex); you can avoid this warning by specifying engine='python'.\n",
            "  return func(*args, **kwargs)\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                              French\n",
              "0                              Reprise de la session\n",
              "1  Je déclare reprise la session du Parlement eur...\n",
              "2  Comme vous avez pu le constater, le grand \"bog...\n",
              "3  Vous avez souhaité un débat à ce sujet dans le...\n",
              "4  En attendant, je souhaiterais, comme un certai..."
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-c52e86af-0e3b-4b0d-9d05-9ed6777d2fad\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>French</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Reprise de la session</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Je déclare reprise la session du Parlement eur...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Comme vous avez pu le constater, le grand \"bog...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>Vous avez souhaité un débat à ce sujet dans le...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>En attendant, je souhaiterais, comme un certai...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-c52e86af-0e3b-4b0d-9d05-9ed6777d2fad')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-c52e86af-0e3b-4b0d-9d05-9ed6777d2fad button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-c52e86af-0e3b-4b0d-9d05-9ed6777d2fad');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ],
      "source": [
        "# French data\n",
        "\n",
        "french_data = pd.read_csv(\"/content/drive/MyDrive/Language_Detection/data/europarl-v7.fr-en.fr\", \"utf-8\", header=None, names=[\"French\"])\n",
        "french_data.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FkEh5U9Ownmb"
      },
      "source": [
        "### 3. Data Preprocessing"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wyc-HKXsdGh9"
      },
      "source": [
        "We preprocess our data by removing the unwanted characters and punctuations. After tokenizing a sentence into words, we convert all the text into lower case. Then remove stopwords, numerical values and punctuations."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "l5puk79fwnmb",
        "outputId": "3d52d459-33f3-40a1-c5e6-99b5d51275c9",
        "scrolled": true
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "! \" # $ % & ' ( ) * + , - . / : ; < = > ? @ [ \\ ] ^ _ ` { | } ~ "
          ]
        }
      ],
      "source": [
        "for character in string.punctuation:\n",
        "    print(character, end = ' ')\n",
        "translate_table = dict((ord(character), None) for character in string.punctuation)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UQhJc1K4g2-3"
      },
      "outputs": [],
      "source": [
        "# Cleaning english dataset\n",
        "data_eng = []\n",
        "lang_eng = []\n",
        "\n",
        "for i, line in english_data.iterrows():\n",
        "    line = line['English']\n",
        "    if len(line) !=0:\n",
        "        line = line.lower()\n",
        "        line = re.sub(r\"\\d+\", \"\", line)\n",
        "        line = line.translate(translate_table)\n",
        "        data_eng.append(line)\n",
        "        lang_eng.append(\"English\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "v0GCecJrwnmd"
      },
      "outputs": [],
      "source": [
        "# Cleaning german dataset\n",
        "data_ger = []\n",
        "lang_ger = []\n",
        "\n",
        "for i, line in german_data.iterrows():\n",
        "    line = line['German']\n",
        "    if len(line) !=0:\n",
        "        line = line.lower()\n",
        "        line = re.sub(r\"\\d+\", \"\", line)\n",
        "        line = line.translate(translate_table)\n",
        "        data_ger.append(line)\n",
        "        lang_ger.append(\"German\")\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_O7DPyZ0wnmd"
      },
      "outputs": [],
      "source": [
        "# Cleaning spanish dataset\n",
        "data_esp = []\n",
        "lang_esp = []\n",
        "\n",
        "for i, line in spanish_data.iterrows():\n",
        "    line = line['Spanish']\n",
        "    if len(line) !=0:\n",
        "        line = line.lower()\n",
        "        line = re.sub(r\"\\d+\", \"\", line)\n",
        "        line = line.translate(translate_table)\n",
        "        data_esp.append(line)\n",
        "        lang_esp.append(\"Spanish\")\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Le9fZSPxwnmd"
      },
      "outputs": [],
      "source": [
        "# Cleaning french dataset\n",
        "data_fre = []\n",
        "lang_fre = []\n",
        "\n",
        "for i, line in french_data.iterrows():\n",
        "    line = line['French']\n",
        "    if len(line) !=0:\n",
        "        line = line.lower()\n",
        "        line = re.sub(r\"\\d+\", \"\", line)\n",
        "        line = line.translate(translate_table)\n",
        "        data_fre.append(line)\n",
        "        lang_fre.append(\"French\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VP2jG7Fiwnmi"
      },
      "source": [
        "### 4. Transforming the data into a single dataset"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "P9EZqk46wnmj",
        "outputId": "7ef11f87-8337-4533-9095-72c993ff9633"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "(7796133, 2)\n"
          ]
        }
      ],
      "source": [
        "df = pd.DataFrame({\n",
        "    \"Text\" : data_eng+data_ger+data_esp+data_fre,\n",
        "    \"Language\" : lang_eng+lang_ger+lang_esp+lang_fre\n",
        "})\n",
        "\n",
        "print(df.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "WCOc02tYwnmj",
        "outputId": "8ffdf145-6bd8-43c8-d2cb-46244f1f3728"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                Text Language\n",
              "0                          resumption of the session  English\n",
              "1  i declare resumed the session of the european ...  English\n",
              "2  although as you will have seen the dreaded mil...  English\n",
              "3  you have requested a debate on this subject in...  English\n",
              "4  in the meantime i should like to observe a min...  English"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-f8f982d8-c517-4071-97c0-f56273e4ae60\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Text</th>\n",
              "      <th>Language</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>resumption of the session</td>\n",
              "      <td>English</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>i declare resumed the session of the european ...</td>\n",
              "      <td>English</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>although as you will have seen the dreaded mil...</td>\n",
              "      <td>English</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>you have requested a debate on this subject in...</td>\n",
              "      <td>English</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>in the meantime i should like to observe a min...</td>\n",
              "      <td>English</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-f8f982d8-c517-4071-97c0-f56273e4ae60')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-f8f982d8-c517-4071-97c0-f56273e4ae60 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-f8f982d8-c517-4071-97c0-f56273e4ae60');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ],
      "source": [
        "df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "sChOPH4bwnmk",
        "outputId": "c30cebb8-7411-4ef6-e879-7719db988a89"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                      Text Language\n",
              "7796128  je me permettrai même bien quils soient absent...   French\n",
              "7796129  je ne rouvrirai pas le débat sur le millénaire...   French\n",
              "7796130                         interruption de la session   French\n",
              "7796131  je déclare interrompue la session du parlement...   French\n",
              "7796132                            la séance est levée à h   French"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-7c0ec28b-5d84-4746-8501-92ba8b32f13e\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Text</th>\n",
              "      <th>Language</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>7796128</th>\n",
              "      <td>je me permettrai même bien quils soient absent...</td>\n",
              "      <td>French</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7796129</th>\n",
              "      <td>je ne rouvrirai pas le débat sur le millénaire...</td>\n",
              "      <td>French</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7796130</th>\n",
              "      <td>interruption de la session</td>\n",
              "      <td>French</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7796131</th>\n",
              "      <td>je déclare interrompue la session du parlement...</td>\n",
              "      <td>French</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7796132</th>\n",
              "      <td>la séance est levée à h</td>\n",
              "      <td>French</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-7c0ec28b-5d84-4746-8501-92ba8b32f13e')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-7c0ec28b-5d84-4746-8501-92ba8b32f13e button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-7c0ec28b-5d84-4746-8501-92ba8b32f13e');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ],
      "source": [
        "df.tail()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_N6XNV6Bwnmk",
        "outputId": "e2f73800-a274-4214-f5cb-22aff561d8b6"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "French     2004791\n",
              "Spanish    1962213\n",
              "German     1917286\n",
              "English    1911843\n",
              "Name: Language, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ],
      "source": [
        "df['Language'].value_counts()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "yQbqpwOKwnml",
        "outputId": "081ce259-ae67-400f-ebd8-15dd38afd6f6"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.axes._subplots.AxesSubplot at 0x7f0ddc1e3c10>"
            ]
          },
          "metadata": {},
          "execution_count": 16
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 1440x1440 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAABJUAAAR3CAYAAABOuFVXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzda6xld3nf8d8DYxIlIY3pnNxswyDkNKUhNuTI0IDARKoxUVO3URLZJcGlpFNVFDVVi5T2Ba6M+iq9JUBiWe3EclUcNSWmruQAVpvilODUM8gB4zTEcmgZi8pTj7nk0qZGT1+cPeF0PJfz2LNmnzPz+UhbZ+//f619ntdfrb1WdXcAAAAAYOJ56x4AAAAAgL1HVAIAAABgTFQCAAAAYExUAgAAAGBMVAIAAABgTFQCAAAAYOyCi0pVdaiqnqiqh3d4/I9V1SNV9Zmq+sDS8wEAAABcCKq71z3DOVVVr0/y+0nu7O7vOcuxVyb5t0l+oLufqqpv7e4nzsecAAAAAHvZBXelUnffn+T49rWqellVfbiqjlTVr1fVd6+2/kaS93f3U6tzBSUAAACAHbjgotJp3J7knd39fUn+fpKfX61/V5LvqqqPV9UDVXX92iYEAAAA2EP2rXuApVXVNyX5/iS/XFUnlr9u9XdfkiuTXJvk8iT3V9UruvuL53tOAAAAgL3kgo9K2boa64vdffUp9o4m+c3u/r9Jfq+qPputyPTg+RwQAAAAYK+54H/+1t1fzlYw+tEkqS1XrbY/lK2rlFJV+7P1c7jH1jEnAAAAwF5ywUWlqrorySeS/JmqOlpVb0/yliRvr6rfSvKZJDesDv9Ikier6pEkv5bkXd395DrmBgAAANhLqrvXPQMAAAAAe8wFd6USAAAAAMsTlQAAAAAYu6Ce/rZ///4+cODAuscAAAAAuGAcOXLkf3X3xsnrF1RUOnDgQA4fPrzuMQAAAAAuGFX130+17udvAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjC0Wlarqiqr6tap6pKo+U1V/5xTHVFX9XFU9WlWfqqpXbdu7uap+d/W6eak5AQAAAJjbt+B3P53k73X3J6vqhUmOVNV93f3ItmPenOTK1evVSX4hyaur6kVJbkmymaRX597T3U8tOC8AAAA79LHXv2HdI8BF6w33f2zdIyRZ8Eql7v5Cd39y9f4rSX47yWUnHXZDkjt7ywNJvqWqviPJm5Lc193HVyHpviTXLzUrAAAAADPn5Z5KVXUgySuT/OZJW5cl+fy2z0dXa6dbBwAAAGAXWDwqVdU3Jflgkp/q7i8v8P0Hq+pwVR0+duzYuf56AAAAAE5h0ahUVZdkKyj9m+7+lVMc8niSK7Z9vny1drr1Z+ju27t7s7s3NzY2zs3gAAAAAJzRkk9/qyT/Kslvd/c/O81h9yR56+opcK9J8qXu/kKSjyS5rqourapLk1y3WgMAAABgF1jy6W+vTfITST5dVQ+t1v5hkhcnSXffluTeJD+Y5NEkf5jkbau941X1niQPrs67tbuPLzgrAAAAAAOLRaXu/i9J6izHdJJ3nGbvUJJDC4wGAAAAwHN0Xp7+BgAAAMCFRVQCAAAAYExUAgAAAGBMVAIAAABgTFQCAAAAYExUAgAAAGBMVAIAAABgTFQCAAAAYExUAgAAAGBMVAIAAABgTFQCAAAAYExUAgAAAGBMVAIAAABgTFQCAAAAYGzfugcAAGBZr33va9c9Aly0Pv7Oj697BIDFuFIJAAAAgDFRCQAAAIAxUQkAAACAMVEJAAAAgDFRCQAAAIAxUQkAAACAMVEJAAAAgDFRCQAAAIAxUQkAAACAMVEJAAAAgDFRCQAAAIAxUQkAAACAMVEJAAAAgDFRCQAAAIAxUQkAAACAMVEJAAAAgDFRCQAAAIAxUQkAAACAMVEJAAAAgDFRCQAAAIAxUQkAAACAMVEJAAAAgDFRCQAAAICxfeseAICv+R+3vmLdI8BF68Xv/vS6RwAA2FNcqQQAAADAmCuVnqXve9ed6x4BLlpHfuat6x4BAADgoudKJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxvYt9cVVdSjJX0zyRHd/zyn235XkLdvm+LNJNrr7eFV9LslXknw1ydPdvbnUnAAAAADMLXml0h1Jrj/dZnf/THdf3d1XJ/kHST7W3ce3HfLG1b6gBAAAALDLLBaVuvv+JMfPeuCWm5LctdQsAAAAAJxba7+nUlV9Q7auaPrgtuVO8tGqOlJVB89y/sGqOlxVh48dO7bkqAAAAACsrD0qJfmhJB8/6advr+vuVyV5c5J3VNXrT3dyd9/e3ZvdvbmxsbH0rAAAAABkd0SlG3PST9+6+/HV3yeS3J3kmjXMBQAAAMBprDUqVdWfSvKGJP9+29o3VtULT7xPcl2Sh9czIQAAAACnsm+pL66qu5Jcm2R/VR1NckuSS5Kku29bHfZXkny0u/9g26nfluTuqjox3we6+8NLzQkAAADA3GJRqbtv2sExdyS546S1x5JctcxUAAAAAJwLu+GeSgAAAADsMaISAAAAAGOiEgAAAABjohIAAAAAY6ISAAAAAGOiEgAAAABjohIAAAAAY6ISAAAAAGOiEgAAAABjohIAAAAAY6ISAAAAAGOiEgAAAABjohIAAAAAY6ISAAAAAGOiEgAAAABjohIAAAAAY6ISAAAAAGOiEgAAAABjohIAAAAAY6ISAAAAAGOiEgAAAABjohIAAAAAY6ISAAAAAGOiEgAAAABjohIAAAAAY6ISAAAAAGOiEgAAAABjohIAAAAAY6ISAAAAAGOiEgAAAABjohIAAAAAY6ISAAAAAGOiEgAAAABjohIAAAAAY6ISAAAAAGOiEgAAAABjohIAAAAAY6ISAAAAAGOiEgAAAABjohIAAAAAY6ISAAAAAGOiEgAAAABjohIAAAAAY6ISAAAAAGOiEgAAAABjohIAAAAAY6ISAAAAAGOiEgAAAABjohIAAAAAY6ISAAAAAGOiEgAAAABjohIAAAAAY6ISAAAAAGOiEgAAAABjohIAAAAAY6ISAAAAAGOiEgAAAABjohIAAAAAY6ISAAAAAGOiEgAAAABjohIAAAAAY6ISAAAAAGOiEgAAAABjohIAAAAAY6ISAAAAAGOiEgAAAABjohIAAAAAY6ISAAAAAGOiEgAAAABjohIAAAAAY6ISAAAAAGOiEgAAAABjohIAAAAAY6ISAAAAAGOiEgAAAABjohIAAAAAY6ISAAAAAGOiEgAAAABjohIAAAAAY6ISAAAAAGOiEgAAAABjohIAAAAAY6ISAAAAAGOiEgAAAABjohIAAAAAY6ISAAAAAGOiEgAAAABjohIAAAAAY6ISAAAAAGOiEgAAAABjohIAAAAAY6ISAAAAAGOiEgAAAABjohIAAAAAY6ISAAAAAGOiEgAAAABjohIAAAAAY6ISAAAAAGOiEgAAAABjohIAAAAAY6ISAAAAAGOiEgAAAABjohIAAAAAY6ISAAAAAGOiEgAAAABjohIAAAAAY6ISAAAAAGOiEgAAAABjohIAAAAAY6ISAAAAAGOiEgAAAABjohIAAAAAY6ISAAAAAGOiEgAAAABjohIAAAAAY6ISAAAAAGOiEgAAAABjohIAAAAAY6ISAAAAAGOiEgAAAABji0WlqjpUVU9U1cOn2b+2qr5UVQ+tXu/etnd9Vf1OVT1aVT+91IwAAAAAPDtLXql0R5Lrz3LMr3f31avXrUlSVc9P8v4kb07y8iQ3VdXLF5wTAAAAgKHFolJ335/k+LM49Zokj3b3Y939x0l+KckN53Q4AAAAAJ6Tdd9T6c9X1W9V1a9W1Z9brV2W5PPbjjm6WjulqjpYVYer6vCxY8eWnBUAAACAlXVGpU8meUl3X5XkvUk+9Gy+pLtv7+7N7t7c2Ng4pwMCAAAAcGpri0rd/eXu/v3V+3uTXFJV+5M8nuSKbYdevloDAAAAYJdYW1Sqqm+vqlq9v2Y1y5NJHkxyZVW9tKpekOTGJPesa04AAAAAnmnfUl9cVXcluTbJ/qo6muSWJJckSXffluRHkvytqno6yR8lubG7O8nTVfW3k3wkyfOTHOruzyw1JwAAAABzi0Wl7r7pLPvvS/K+0+zdm+TeJeYCAAAA4Llb99PfAAAAANiDRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMYWi0pVdaiqnqiqh0+z/5aq+lRVfbqqfqOqrtq297nV+kNVdXipGQEAAAB4dpa8UumOJNefYf/3kryhu1+R5D1Jbj9p/43dfXV3by40HwAAAADP0r6lvri776+qA2fY/41tHx9IcvlSswAAAABwbu2Weyq9PcmvbvvcST5aVUeq6uCZTqyqg1V1uKoOHzt2bNEhAQAAANiy2JVKO1VVb8xWVHrdtuXXdffjVfWtSe6rqv/W3fef6vzuvj2rn85tbm724gMDAAAAsN4rlarqe5P8yyQ3dPeTJ9a7+/HV3yeS3J3kmvVMCAAAAMCprC0qVdWLk/xKkp/o7s9uW//GqnrhifdJrktyyifIAQAAALAei/38raruSnJtkv1VdTTJLUkuSZLuvi3Ju5P86SQ/X1VJ8vTqSW/fluTu1dq+JB/o7g8vNScAAAAAc0s+/e2ms+z/ZJKfPMX6Y0muWmouAAAAAJ673fL0NwAAAAD2EFEJAAAAgDFRCQAAAIAxUQkAAACAMVEJAAAAgDFRCQAAAIAxUQkAAACAMVEJAAAAgDFRCQAAAIAxUQkAAACAMVEJAAAAgDFRCQAAAIAxUQkAAACAMVEJAAAAgDFRCQAAAIAxUQkAAACAMVEJAAAAgDFRCQAAAIAxUQkAAACAMVEJAAAAgDFRCQAAAIAxUQkAAACAMVEJAAAAgDFRCQAAAIAxUQkAAACAMVEJAAAAgDFRCQAAAIAxUQkAAACAMVEJAAAAgDFRCQAAAIAxUQkAAACAMVEJAAAAgDFRCQAAAIAxUQkAAACAMVEJAAAAgDFRCQAAAIAxUQkAAACAMVEJAAAAgDFRCQAAAIAxUQkAAACAMVEJAAAAgDFRCQAAAIAxUQkAAACAMVEJAAAAgDFRCQAAAIAxUQkAAACAMVEJAAAAgDFRCQAAAIAxUQkAAACAMVEJAAAAgDFRCQAAAIAxUQkAAACAMVEJAAAAgDFRCQAAAIAxUQkAAACAMVEJAAAAgDFRCQAAAIAxUQkAAACAMVEJAAAAgDFRCQAAAIAxUQkAAACAMVEJAAAAgDFRCQAAAIAxUQkAAACAMVEJAAAAgDFRCQAAAIAxUQkAAACAMVEJAAAAgDFRCQAAAIAxUQkAAACAMVEJAAAAgDFRCQAAAIAxUQkAAACAMVEJAAAAgDFRCQAAAIAxUQkAAACAMVEJAAAAgDFRCQAAAIAxUQkAAACAMVEJAAAAgDFRCQAAAIAxUQkAAACAMVEJAAAAgDFRCQAAAIAxUQkAAACAsR1Fpar6jztZAwAAAODisO9Mm1X19Um+Icn+qro0Sa22vjnJZQvPBgAAAMAudcaolORvJvmpJN+Z5Ei+FpW+nOR9C84FAAAAwC52xqjU3T+b5Ger6p3d/d7zNBMAAAAAu9zZrlRKknT3e6vq+5Mc2H5Od9+50FwAAAAA7GI7ikpV9a+TvCzJQ0m+ulruJKISAAAAwEVoR1EpyWaSl3d3LzkMAAAAAHvD83Z43MNJvn3JQQAAAADYO3Z6pdL+JI9U1X9N8n9OLHb3X1pkKgAAAAB2tZ1GpX+05BAAAAAA7C07ffrbx5YeBAAAAIC9Y6dPf/tKtp72liQvSHJJkj/o7m9eajAAAAAAdq+dXqn0whPvq6qS3JDkNUsNBQAAAMDuttOnv/2J3vKhJG9aYB4AAAAA9oCd/vzth7d9fF6SzST/e5GJAAAAANj1dvr0tx/a9v7pJJ/L1k/gAAAAALgI7fSeSm9behAAAAAA9o4d3VOpqi6vqrur6onV64NVdfnSwwEAAACwO+30Rt2/mOSeJN+5ev2H1RoAAAAAF6GdRqWN7v7F7n569bojycaCcwEAAACwi+00Kj1ZVT9eVc9fvX48yZNLDgYAAADA7rXTqPTXk/xYkv+Z5AtJfiTJX1toJgAAAAB2uR09/S3JrUlu7u6nkqSqXpTkn2QrNgEAAABwkdnplUrfeyIoJUl3H0/yymVGAgAAAGC322lUel5VXXriw+pKpZ1e5QQAAADABWanYeifJvlEVf3y6vOPJvnHy4wEAAAAwG63o6jU3XdW1eEkP7Ba+uHufmS5sQAAAADYzXb8E7ZVRBKSAAAAANjxPZUAAAAA4E+ISgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIwtGpWq6lBVPVFVD59mv6rq56rq0ar6VFW9atvezVX1u6vXzUvOCQAAAMDM0lcq3ZHk+jPsvznJlavXwSS/kCRV9aIktyR5dZJrktxSVZcuOikAAAAAO7ZoVOru+5McP8MhNyS5s7c8kORbquo7krwpyX3dfby7n0pyX84cpwAAAAA4j9Z9T6XLknx+2+ejq7XTrT9DVR2sqsNVdfjYsWOLDQoAAADA16w7Kj1n3X17d2929+bGxsa6xwEAAAC4KKw7Kj2e5Iptny9frZ1uHQAAAIBdYN1R6Z4kb109Be41Sb7U3V9I8pEk11XVpasbdF+3WgMAAABgF9i35JdX1V1Jrk2yv6qOZuuJbpckSXffluTeJD+Y5NEkf5jkbau941X1niQPrr7q1u4+0w2/AQAAADiPFo1K3X3TWfY7yTtOs3coyaEl5gIAAADguVn3z98AAAAA2INEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxkQlAAAAAMZEJQAAAADGRCUAAAAAxhaNSlV1fVX9TlU9WlU/fYr9f15VD61en62qL27b++q2vXuWnBMAAACAmX1LfXFVPT/J+5P8hSRHkzxYVfd09yMnjunuv7vt+HcmeeW2r/ij7r56qfkAAAAAePaWvFLpmiSPdvdj3f3HSX4pyQ1nOP6mJHctOA8AAAAA58iSUemyJJ/f9vnoau0ZquolSV6a5D9tW/76qjpcVQ9U1V8+3T+pqoOr4w4fO3bsXMwNAAAAwFnslht135jk33X3V7etvaS7N5P81ST/oqpedqoTu/v27t7s7s2NjY3zMSsAAADARW/JqPR4kiu2fb58tXYqN+akn7519+Orv48l+c/5/++3BAAAAMAaLRmVHkxyZVW9tKpekK1w9IynuFXVdye5NMkntq1dWlVft3q/P8lrkzxy8rkA/L/27j348ruu7/jrTcLVtEhNmqEESJBwCeWaJRXwQlUuzshFoYLWGKfUWMZIW0cpDFMS4tjiZUorJApiRtoqQajKqtEQi4jILRdCINHUGEA2giRCGaOQsMm7f5zvksN2N9n37h5+u8njMXPmd76f7+V8z/7xne957vd8DwAAwNbY2K+/dffOqjojyYVJjkhyXndfWVVnJ7mku3cFphcmOb+7e231RyZ5fVXdmlX4evX6r8YBAAAAsLU2FpWSpLsvSHLBbmOv3G36rD2s994kj97kvgEAAACw/w6VG3UDAAAAcBgRlQAAAAAYE5UAAAAAGBOVAAAAABgTlQAAAAAYE5UAAAAAGBOVAAAAABgTlQAAAAAYE5UAAAAAGBOVAAAAABgTlQAAAAAYE5UAAAAAGBOVAAAAABgTlQAAAAAYE5UAAAAAGBOVAAAAABgTlQAAAAAYE5UAAAAAGBOVAAAAABgTlQAAAAAYE5UAAAAAGBOVAAAAABgTlQAAAAAYE5UAAAAAGBOVAAAAABgTlQAAAAAYE5UAAAAAGBOVAAAAABgTlQAAABHwX2QAABcxSURBVAAYE5UAAAAAGBOVAAAAABgTlQAAAAAYE5UAAAAAGBOVAAAAABgTlQAAAAAYE5UAAAAAGBOVAAAAABgTlQAAAAAYE5UAAAAAGBOVAAAAABgTlQAAAAAYE5UAAAAAGBOVAAAAABgTlQAAAAAYE5UAAAAAGBOVAAAAABgTlQAAAAAYE5UAAAAAGBOVAAAAABgTlQAAAAAYE5UAAAAAGBOVAAAAABgTlQAAAAAYE5UAAAAAGBOVAAAAABgTlQAAAAAYE5UAAAAAGBOVAAAAABgTlQAAAAAYE5UAAAAAGBOVAAAAABgTlQAAAAAYE5UAAAAAGBOVAAAAABgTlQAAAAAYE5UAAAAAGBOVAAAAABgTlQAAAAAYE5UAAAAAGBOVAAAAABgTlQAAAAAYE5UAAAAAGBOVAAAAABgTlQAAAAAYE5UAAAAAGBOVAAAAABgTlQAAAAAYE5UAAAAAGBOVAAAAABgTlQAAAAAYE5UAAAAAGBOVAAAAABgTlQAAAAAYE5UAAAAAGBOVAAAAABgTlQAAAAAYE5UAAAAAGBOVAAAAABgTlQAAAAAYE5UAAAAAGBOVAAAAABgTlQAAAAAYE5UAAAAAGBOVAAAAABgTlQAAAAAYE5UAAAAAGBOVAAAAABgTlQAAAAAYE5UAAAAAGBOVAAAAABgTlQAAAAAYE5UAAAAAGBOVAAAAABgTlQAAAAAYE5UAAAAAGBOVAAAAABgTlQAAAAAYE5UAAAAAGBOVAAAAABgTlQAAAAAYE5UAAAAAGBOVAAAAABgTlQAAAAAYE5UAAAAAGBOVAAAAABgTlQAAAAAYE5UAAAAAGBOVAAAAABgTlQAAAAAYE5UAAAAAGNtoVKqqZ1bV1VV1TVW9bA/zf7Cqrq+qy5fHv16bd1pV/fnyOG2T+wkAAADAzJGb2nBVHZHknCRPS7IjycVVtb27r9pt0bd09xm7rfuPkpyZZFuSTnLpsu7nNrW/AAAAAOy7TV6pdEqSa7r72u6+Ocn5SZ6zj+s+I8lF3f3ZJSRdlOSZG9pPAAAAAIY2GZUekOSTa9M7lrHdPa+qrqiqt1XVA4frAgAAALAFtvpG3b+d5PjufkxWVyO9abqBqjq9qi6pqkuuv/76g76DAAAAAPz/NhmVrkvywLXp45axL+vuv+num5bJNyY5eV/XXdvGG7p7W3dvO+aYYw7KjgMAAABw+zYZlS5OcmJVnVBV90jywiTb1xeoqvuvTT47yZ8uzy9M8vSqul9V3S/J05cxAAAAAA4BG/v1t+7eWVVnZBWDjkhyXndfWVVnJ7mku7cneUlVPTvJziSfTfKDy7qfraqfzCpMJcnZ3f3ZTe0rAAAAADMbi0pJ0t0XJLlgt7FXrj1/eZKX72Xd85Kct8n9AwAAAGD/bPWNugEAAAA4DIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMiUoAAAAAjIlKAAAAAIyJSgAAAACMbTQqVdUzq+rqqrqmql62h/k/VlVXVdUVVfW/q+rBa/NuqarLl8f2Te4nAAAAADNHbmrDVXVEknOSPC3JjiQXV9X27r5qbbEPJdnW3X9fVS9O8jNJXrDM+0J3P25T+wcAAADA/tvklUqnJLmmu6/t7puTnJ/kOesLdPcfdvffL5PvT3LcBvcHAAAAgINkk1HpAUk+uTa9Yxnbmxcl+b216XtV1SVV9f6qeu4mdhAAAACA/bOxr79NVNX3J9mW5FvWhh/c3ddV1UOSvLOqPtLdf7GHdU9PcnqSPOhBD/qq7C8AAADAXd0mr1S6LskD16aPW8a+QlV9e5JXJHl2d9+0a7y7r1v+XpvkXUkev6cX6e43dPe27t52zDHHHLy9BwAAAGCvNhmVLk5yYlWdUFX3SPLCJF/xK25V9fgkr88qKH1mbfx+VXXP5fnRSZ6SZP0G3wAAAABsoY19/a27d1bVGUkuTHJEkvO6+8qqOjvJJd29PcnPJjkqyVurKkn+srufneSRSV5fVbdmFb5evduvxgEAAACwhTZ6T6XuviDJBbuNvXLt+bfvZb33Jnn0JvcNAAAAgP23ya+/AQAAAHAnJSoBAAAAMCYqAQAAADAmKgEAAAAwJioBAAAAMCYqAQAAADAmKgEAAAAwJioBAAAAMCYqAQAAADAmKgEAAAAwJioBAAAAMCYqAQAAADAmKgEAAAAwJioBAAAAMCYqAQAAADAmKgEAAAAwJioBAAAAMCYqAQAAADAmKgEAAAAwJioBAAAAMCYqAQAAADAmKgEAAAAwJioBAAAAMCYqAQAAADAmKgEAAAAwJioBAAAAMCYqAQAAADAmKgEAAAAwJioBAAAAMCYqAQAAADAmKgEAAAAwJioBAAAAMCYqAQAAADAmKgEAAAAwJioBAAAAMCYqAQAAADAmKgEAAAAwJioBAAAAMCYqAQAAADAmKgEAAAAwJioBAAAAMCYqAQAAADAmKgEAAAAwJioBAAAAMCYqAQAAADAmKgEAAAAwJioBAAAAMCYqAQAAADAmKgEAAAAwJioBAAAAMCYqAQAAADAmKgEAAAAwJioBAAAAMCYqAQAAADAmKgEAAAAwJioBAAAAMCYqAQAAADAmKgEAAAAwJioBAAAAMCYqAQAAADAmKgEAAAAwJioBAAAAMCYqAQAAADAmKgEAAAAwJioBAAAAMCYqAQAAADAmKgEAAAAwJioBAAAAMCYqAQAAADAmKgEAAAAwJioBAAAAMCYqAQAAADAmKgEAAAAwJioBAAAAMCYqAQAAADAmKgEAAAAwJioBAAAAMCYqAQAAADAmKgEAAAAwJioBAAAAMCYqAQAAADAmKgEAAAAwJioBAAAAMCYqAQAAADAmKgEAAAAwJioBAAAAMCYqAQAAADAmKgEAAAAwJioBAAAAMCYqAQAAADAmKgEAAAAwJioBAAAAMCYqAQAAADAmKgEAAAAwJioBAAAAMCYqAQAAADAmKgEAAAAwJioBAAAAMCYqAQAAADAmKgEAAAAwJioBAAAAMCYqAQAAADAmKgEAAAAwJioBAAAAMCYqAQAAADAmKgEAAAAwJioBAAAAMCYqAQAAADAmKgEAAAAwJioBAAAAMCYqAQAAADAmKgEAAAAwJioBAAAAMCYqAQAAADAmKgEAAAAwJioBAAAAMCYqAQAAADAmKgEAAAAwJioBAAAAMCYqAQAAADC20ahUVc+sqqur6pqqetke5t+zqt6yzP9AVR2/Nu/ly/jVVfWMTe4nAAAAADMbi0pVdUSSc5J8R5KTknxvVZ2022IvSvK57n5oktck+ell3ZOSvDDJo5I8M8m5y/YAAAAAOARs8kqlU5Jc093XdvfNSc5P8pzdlnlOkjctz9+W5Nuqqpbx87v7pu7+WJJrlu0BAAAAcAjYZFR6QJJPrk3vWMb2uEx370zy+SRft4/rAgAAALBFjtzqHThQVXV6ktOXyRur6uqt3B8OG0cnuWGrd4L9Uz932lbvAuyNY8vh7Mza6j2AvXFsOYzVSxxbOGQ5thzO6qt+bHnwngY3GZWuS/LAtenjlrE9LbOjqo5Mct8kf7OP6yZJuvsNSd5wkPaZu4iquqS7t231fgB3Lo4twCY4tgCb4NjCwbDJr79dnOTEqjqhqu6R1Y23t++2zPYkuy45eH6Sd3Z3L+MvXH4d7oQkJyb54Ab3FQAAAICBjV2p1N07q+qMJBcmOSLJed19ZVWdneSS7t6e5JeT/I+quibJZ7MKT1mW+/UkVyXZmeRHuvuWTe0rAAAAADO1ujAI7lqq6vTlq5MAB41jC7AJji3AJji2cDCISgAAAACMbfKeSgAAAADcSYlKHHaq6paqunzt8bID2NaNy99/UlVvu53ljq+qj+7v6wCHtqo6tqp+raqurapLq+p9VfVdW71fwOGtql5RVVdW1RXLOcs/O8jbf+8dzL/xYL4ecGjZw+ei4zf0Ok+tqt/ZxLY5/G3sRt2wQV/o7scdzA12919l9QuEwF1MVVWS30rypu7+vmXswUmevY/rH9ndOze4i8BhqKqelOQ7kzyhu2+qqqOT3ONgvkZ3P/lgbg847Oz1c9FyflPdfetXeZ+4i3GlEncaVfXxqnpVVV1WVR+pqkcs48dU1UXL/xS+sao+sZzYra/75SuRqupRVfXBpfZfUVUnLosdUVW/tGznHVV176/yWwQ241uT3Nzdv7hroLs/0d2vraojqupnq+ri5Xjww8mX/8fuj6tqe5Krluk/qqq3L1c7vbqq/uVyLPlIVX39st6zquoDVfWhqvqDqjp2GT+rqs6rqnct679kK/4hgIPq/klu6O6bkqS7b+juv1rOV35mOTZ8sKoemuzf8WHtiuv7V9W7l3OXj1bVN60t81NV9eGqev+ubQJ3Tstnmqur6r8n+WiSB1bVT6ydx7xqbbk/3dNnm6p66HIM+vDyuerrl80fVVVvq6o/q6pfXaIViEoclu6922WeL1ibd0N3PyHJLyT58WXszCTv7O5HJXlbkgfdwfb/TZL/tlT/bUl2LOMnJjln2c7/TfK8g/R+gK31qCSX7WXei5J8vrufmOSJSX6oqk5Y5j0hyb/t7oct04/N6vjxyCSnJnlYd5+S5I1JfnRZ5j1JvqG7H5/k/CQvXXutRyR5RpJTkpxZVXc/GG8O2DLvyOoD3f+pqnOr6lvW5n2+ux+d5HVJ/usydiDHh+9LcuFy7vLYJJcv41+T5P3d/dgk707yQwfv7QGHgPXPRb+5jJ2Y5NzlM8vDl+lTkjwuyclV9c1ry+3ps82vLuOPTfLkJJ9axh+f5N8lOSnJQ5I8ZbNvjcOFr79xOLq9r7/9xvL30iTfvTz/xiTflSTd/ftV9bk72P77kryiqo5L8hvd/edLiP9Yd+86Sbs0yfH7uf/AIayqzsnquHFzkk8keUxV7fp67H2zOgm7OckHu/tja6te3N2fWrbxF1l9oEySjyT558vz45K8parun9XXYNbX/93lioabquozSY7NbVEbOMx0941VdXKSb8rqGPCWuu0+kG9e+/ua5fmBHB8uTnLeEpt+a+185eYku+6DcmmSpx20NwgcCr7ic1Gt7qn0ie5+/zL09OXxoWX6qKzOY/4ye/hsU1X/IMkDuvs3k6S7v7hsN1md9+xYpi/P6rPQezb1xjh8uFKJO5ublr+3ZD+jaXf/Wlb3UvlCkguq6lt32/YBbR845FyZ1VVHSZLu/pEk35bkmCSV5Ee7+3HL44Tu3hWL/m637awfI25dm741tx0vXpvkdcsVCj+c5F57Wd8xBu4EuvuW7n5Xd5+Z5IzcdiVAry+2/N3v40N3vzvJNye5LsmvVNUPLLO+1N29t/WAO6X185NK8p/XzmMe2t2/vMybnnc4T2GPRCXuCv4kyfckSVU9Pcn9bm/hqnpIkmu7++eTvD3JYza+h8BWemeSe1XVi9fG7rP8vTDJi3d91aSqHlZVX3MAr3XfrD70JclpB7Ad4BBXVQ9fuy9jsvrqySeW5y9Y+/u+5fl+Hx9q9eMCf93dv5TVV26fcAerAHcNFyb5V1V1VJJU1QOq6h/vbeHu/tskO6rqucvy96yq++xteUjURQ5P914uudzl97v7ZXtdOnlVkjdX1alZnbh9Osnf3s7y35Pk1Kr60rLsf0ryDw9wn4FDVHf3cvL0mqp6aZLrs/pfvv+Q5K1ZXd592XJDyuuTPPcAXu6sJG9dvob7ziQn3P7iwGHsqCSvraqvTbIzyTVJTs/qF+HuV1VXZPU//9+7LH9W9v/48NQkP7Gcu9yY5Aduf3HgrqC731FVj0zyvuUrbDcm+f6srjTam1OTvL6qzk7ypST/YuM7ymGtbrsiFu6cquqeSW7p7p21+nnfX7idezIBAGxMVX08ybbuvmGr9wUADpQrlbgreFCSX6+qu2V1w0q/fAIAAAAHyJVKAAAAAIy5UTcAAAAAY6ISAAAAAGOiEgAAAABjohIAwB5U1Y1bvQ8AAIcyUQkAAACAMVEJAGAfVdWzquoDVfWhqvqDqjp2GT+rqs6rqndV1bVV9ZK1df5jVV1dVe+pqjdX1Y8v4++qqm3L86Or6uPL8+Or6o+r6rLl8eRl/G5VdW5V/VlVXVRVF1TV85d5J1fVH1XVpVV1YVXd/6v8TwMA3AWJSgAA++49Sb6hux+f5PwkL12b94gkz0hySpIzq+ruVfXEJM9L8tgk35Fk2z68xmeSPK27n5DkBUl+fhn/7iTHJzkpyalJnpQkVXX3JK9N8vzuPjnJeUl+6gDeIwDAPjlyq3cAAOAwclyStyxXAt0jycfW5v1ud9+U5Kaq+kySY5M8Jcnbu/uLSb5YVb+9D69x9ySvq6rHJbklycOW8W9M8tbuvjXJp6vqD5fxhyf5p0kuqqokOSLJpw7kTQIA7AtRCQBg3702yX/p7u1V9dQkZ63Nu2nt+S254/OsnbntqvF7rY3/+yR/ndXVTXdL8sU72E4lubK7n3QHywEAHFS+/gYAsO/um+S65flp+7D8nyR5VlXdq6qOSvKda/M+nuTk5fnzd3uNTy1XJJ2a1ZVHu7b1vOXeSscmeeoyfnWSY6rqy1+Hq6pHjd4VAMB+EJUAAPbsPlW1Y+3xY1ldmfTWqro0yQ13tIHuvjjJ9iRXJPm9JB9J8vll9s8leXFVfSjJ0WurnZvktKr6cFb3afq7Zfx/JdmR5Kok/zPJZUk+3903ZxWlfnpZ5/IkT97/tw0AsG+qu7d6HwAA7rSq6qjuvrGq7pPk3UlO7+7LDnBbX5fkg0me0t2fPpj7CwCwr9xTCQBgs95QVSdldd+kN+1vUFr8TlV9bVY3Cf9JQQkA2EquVAIAAABgzD2VAAAAABgTlQAAAAAYE5UAAAAAGBOVAAAAABgTlQAAAAAYE5UAAAAAGPt/rObl+Jr1/n4AAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "plt.figure(figsize=(20,20))\n",
        "sns.countplot(x = df['Language'], data=df)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EDr3gRjjwnml"
      },
      "source": [
        "### 5. Splitting the dataset\n",
        "\n",
        "* Splitting the dataset into Independent and Dependent variables"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cOV214gBwnml"
      },
      "outputs": [],
      "source": [
        "x = df.iloc[:,0] # Independent Variable\n",
        "y = df.iloc[:,1] # Dependent Variable"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3kNI9e8Ewnmm",
        "outputId": "9e8ec8e8-a029-4fff-a09e-613f3f374c0e"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0    English\n",
              "1    English\n",
              "2    English\n",
              "3    English\n",
              "4    English\n",
              "Name: Language, dtype: object"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ],
      "source": [
        "y.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HWB3gQUawnmn",
        "outputId": "4a166e04-65f9-4c31-ac33-a17b86c29256"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0                            resumption of the session\n",
              "1    i declare resumed the session of the european ...\n",
              "2    although as you will have seen the dreaded mil...\n",
              "3    you have requested a debate on this subject in...\n",
              "4    in the meantime i should like to observe a min...\n",
              "Name: Text, dtype: object"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ],
      "source": [
        "x.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CxKbV4Dnwnmn"
      },
      "outputs": [],
      "source": [
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.2, random_state=42)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qiJ-XsI_wnmo"
      },
      "source": [
        "## Applying TF-IDF Vectorizer and training the model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oCSCL869wnmo"
      },
      "source": [
        "* `ngram_range` : It collets one , one two, one two three words \n",
        "* `analyzer` : We are not going word by word here we are going character by character that why we have used **char**."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Ri0QPrmKwnmo",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5998d957-1802-4441-e338-e4cfe83e05b6"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "TfidfVectorizer(analyzer='char', ngram_range=(1, 3))\n"
          ]
        }
      ],
      "source": [
        "vectorizer = feature_extraction.text.TfidfVectorizer(ngram_range=(1,3), analyzer='char')\n",
        "print(vectorizer)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ahnbCGYAwnmp"
      },
      "outputs": [],
      "source": [
        "pipe_mnb = pipeline.Pipeline([\n",
        "    ('vectorizer', vectorizer),\n",
        "    ('clf', MultinomialNB())\n",
        "])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "nkGXoppywnmp"
      },
      "outputs": [],
      "source": [
        "pipe_mnb.fit(x_train, y_train)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ViLgtPxlwnmp"
      },
      "outputs": [],
      "source": [
        "pipe_lr = pipeline.Pipeline([\n",
        "    ('vectorizer', vectorizer),\n",
        "    ('lr_clf', LogisticRegression())\n",
        "])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ahyFYUJ-wnmq"
      },
      "outputs": [],
      "source": [
        "pipe_lr.fit(x_train, y_train)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3Wm_cxWJwnmq"
      },
      "source": [
        "## Model Prediction for Logistic Regression\n",
        "\n",
        "---\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "L4CuWbiBwnmq"
      },
      "outputs": [],
      "source": [
        "lr_predicted = pipe_lr.predict(x_test)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "upjA0rhdwnmq"
      },
      "source": [
        "## Model evaluation for Logistic Regression"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "L-1GbkX3wnmr"
      },
      "outputs": [],
      "source": [
        "lr_acc = (metrics.accuracy_score(y_test, lr_predicted))*100\n",
        "print('The logistic regression has:',lr_acc,'% accuracy')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lp9imU7lwnmr"
      },
      "source": [
        "## Model Prediction for MultinomialNB"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "wWhUCeRhwnms"
      },
      "outputs": [],
      "source": [
        "mnb_predicted = pipe_mnb.predict(x_test)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VWnTmGOwwnms"
      },
      "source": [
        "## Model Evaluation for MultinomialNB"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "R_AeySCMwnms"
      },
      "outputs": [],
      "source": [
        "mnb_acc = (metrics.accuracy_score(y_test, mnb_predicted))*100\n",
        "print('The MultinomialNB has :',mnb_acc,'% accuracy')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "V7tBSCFFwnmt"
      },
      "source": [
        "## Evaluation matrix for Logistic Regression"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "asQKuTM3wnmt"
      },
      "outputs": [],
      "source": [
        "matrix = metrics.confusion_matrix(y_test, lr_predicted)\n",
        "print('Confusion matrix: \\n', matrix)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bMnbqgD0wnmu"
      },
      "outputs": [],
      "source": [
        "def plot_confusion_matrix(cm, classes,\n",
        "                          normalize=False,\n",
        "                          title='Actual labels',\n",
        "                          cmap=plt.cm.Blues):\n",
        "    \"\"\"\n",
        "    See full source and example: \n",
        "    http://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html\n",
        "    \n",
        "    This function prints and plots the confusion matrix.\n",
        "    Normalization can be applied by setting `normalize=True`.\n",
        "    \"\"\"\n",
        "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
        "    plt.title(title)\n",
        "    plt.colorbar()\n",
        "    tick_marks = np.arange(len(classes))\n",
        "    plt.xticks(tick_marks, classes, rotation=45)\n",
        "    plt.yticks(tick_marks, classes)\n",
        "\n",
        "    if normalize:\n",
        "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
        "        print(\"Normalized confusion matrix\")\n",
        "    else:\n",
        "        print('Confusion matrix, without normalization')\n",
        "\n",
        "    thresh = cm.max() / 2.\n",
        "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
        "        plt.text(j, i, cm[i, j],\n",
        "                 horizontalalignment=\"center\",\n",
        "                 color=\"white\" if cm[i, j] > thresh else \"black\")\n",
        "\n",
        "    plt.tight_layout()\n",
        "    plt.ylabel('True label')\n",
        "    plt.xlabel('Predicted label')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "nlErIYj6wnmv"
      },
      "outputs": [],
      "source": [
        "plt.figure(figsize=(20,20))\n",
        "plot_confusion_matrix(matrix, classes = ['ENG', 'GER', 'ESP', 'FRA'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "viuD8hdrwnmv"
      },
      "outputs": [],
      "source": [
        "# saving the model. The model can be further loaded and be used to identify the language of the document (any of the above 4 langauges).\n",
        "lrfile = open('lrmodel.pckl', 'wb')\n",
        "pickle.dump(pipe_lr, lrfile)\n",
        "lrfile.close()"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "collapsed_sections": [],
      "name": "Language Identification.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.3"
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "nbformat": 4,
  "nbformat_minor": 0
}