get_predictions.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "f8fc18fe-abfb-43bc-8a0e-c34ada617ebd",
   "metadata": {},
   "source": [
    "# Imports & setup"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab328785-35ed-421e-9bdc-7adc3d0da18c",
   "metadata": {},
   "source": [
    "### Imports - general"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "4c99ff22-0d3d-472d-9dcc-e14ed61df596",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import re\n",
    "\n",
    "import pandas as pd\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "from sklearn.metrics import f1_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "475e4b02-da1e-4ae8-9b02-59e136a475ae",
   "metadata": {},
   "outputs": [],
   "source": [
    "from create_Tatoeba_train_test import get_language_mappings, get_sentence_word_char_len"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "90594c53-135c-4a1a-b290-df730f1f17bd",
   "metadata": {},
   "source": [
    "### Imports & setup - langid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6e589b45-4e08-43f9-9070-28cf3a66e241",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langid.langid import LanguageIdentifier, model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "04e11668-3149-4a59-b42c-7c257a950a45",
   "metadata": {},
   "outputs": [],
   "source": [
    "lang_identifier = LanguageIdentifier.from_modelstring(model, norm_probs=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0203c7de-3ca7-49b8-81cc-cf7279cd89c7",
   "metadata": {},
   "source": [
    "### Imports & setup - langdetect"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "29f10f5c-d960-4fd5-a9fc-282ce788590a",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langdetect import DetectorFactory, detect_langs\n",
    "DetectorFactory.seed = 7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "1c3b9bb9-dfbf-479e-8270-62010a895ea0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_langdetect_prediction(this_string: str):\n",
    "    \"\"\"\n",
    "    Run langdetect on a sentence and get the results.\n",
    "    \n",
    "    Args:\n",
    "    this_string - str; the sentence to run through langdetect.\n",
    "    \n",
    "    Returns:\n",
    "    result - langdetect.language.Language; the langdetect output.\n",
    "    \"\"\"\n",
    "    assert isinstance(this_string, str), \"this_string is not of type str.\"\n",
    "    \n",
    "    result = None\n",
    "    if len(this_string)>0:\n",
    "        try:\n",
    "            result = detect_langs(this_string)[0]\n",
    "        except:\n",
    "            result = \"Error\"\n",
    "            print(f\"Error processing sentence: {this_string}\")\n",
    "    return result\n",
    "\n",
    "def get_langdetect_lang(this_result):\n",
    "    \"\"\"\n",
    "    Get the language-component from the langdetect results.\n",
    "    \n",
    "    Args:\n",
    "    this_result - langdetect.language.Language; the langdetect output.\n",
    "    \n",
    "    Returns:\n",
    "    result - str; 2-character identifier for the predicted language.\n",
    "    \"\"\"\n",
    "    result = \"!!\"\n",
    "    if this_result:\n",
    "        try:\n",
    "            result = this_result.lang\n",
    "            result = result.split(\"-\")[0] # mainly for zh-cn\n",
    "        except:\n",
    "            pass\n",
    "    return result\n",
    "\n",
    "def get_langdetect_prob(this_result):\n",
    "    \"\"\"\n",
    "    Get the probability-component from the langdetect results.\n",
    "    \n",
    "    Args:\n",
    "    this_result - langdetect.language.Language; the langdetect output.\n",
    "    \n",
    "    Returns:\n",
    "    result - float; the probability for the predicted language.\n",
    "    \"\"\"\n",
    "    result = 0.0\n",
    "    if this_result:\n",
    "        try:\n",
    "            result = this_result.prob\n",
    "        except:\n",
    "            pass\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3aa3b681-2682-4b94-a10d-47bc706d284b",
   "metadata": {},
   "outputs": [],
   "source": [
    "language_mappings = get_language_mappings(\"language_mappings.csv\")\n",
    "iso_639_2_English = language_mappings[[\"ISO 639-2\", \"English Name\"]].set_index(keys=\"ISO 639-2\")[\"English Name\"].to_dict()\n",
    "iso_639_2_iso_639_1 = language_mappings[[\"ISO 639-2\", \"ISO 639-1\"]].set_index(keys=\"ISO 639-2\")[\"ISO 639-1\"].to_dict()\n",
    "\n",
    "ISO_639_2 = language_mappings.set_index(\"ISO 639-2\")\n",
    "ISO_639_2 = ISO_639_2[\"English Name\"].to_dict()\n",
    "\n",
    "ISO_639_1 = language_mappings.set_index(\"ISO 639-1\")\n",
    "ISO_639_1 = ISO_639_1[\"English Name\"].to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "573cb215-6756-4456-8651-d321fe5360b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "bin_labels = [\"1\",\"2\",\"3\",\"4\",\"5\",\"6\",\"7\",\"8\",\"9\",\"10\",\"11 to 16\", \"17 to 27\", \"28 to 48\", \"49 to 99\"]\n",
    "counter = 0\n",
    "results = []"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b1b10eb3-0912-449a-9565-82d5bb2f3ae6",
   "metadata": {},
   "source": [
    "# Get predictions & calculate F1 scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d39facc3-63a2-456c-96b4-7fb3b412c250",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing - file:1 - Tatoeba_stratify_train_2021-8-3_2122.csv\n",
      "Finished - file:1 - len(results):336\n",
      "\n",
      "Processing - file:2 - Tatoeba_stratify_train_2021-8-3_2123.csv\n",
      "Finished - file:2 - len(results):672\n",
      "\n",
      "Processing - file:3 - Tatoeba_stratify_train_2021-8-3_2124.csv\n",
      "Error processing sentence: 咦！？\n",
      "Finished - file:3 - len(results):1008\n",
      "\n",
      "Processing - file:4 - Tatoeba_stratify_train_2021-8-3_2125.csv\n",
      "Error processing sentence: 啥？\n",
      "Finished - file:4 - len(results):1344\n",
      "\n",
      "Processing - file:5 - Tatoeba_stratify_train_2021-8-3_2126.csv\n",
      "Finished - file:5 - len(results):1680\n",
      "\n",
      "Processing - file:6 - Tatoeba_stratify_train_2021-8-3_2127.csv\n",
      "Error processing sentence: 嗯..\n",
      "Finished - file:6 - len(results):2016\n",
      "\n",
      "Processing - file:7 - Tatoeba_stratify_train_2021-8-3_2128.csv\n",
      "Finished - file:7 - len(results):2352\n",
      "\n",
      "Processing - file:8 - Tatoeba_stratify_train_2021-8-3_2129.csv\n",
      "Finished - file:8 - len(results):2688\n",
      "\n",
      "Processing - file:9 - Tatoeba_stratify_train_2021-8-3_2130.csv\n",
      "Finished - file:9 - len(results):3024\n",
      "\n",
      "Processing - file:10 - Tatoeba_stratify_train_2021-8-3_2131.csv\n",
      "Finished - file:10 - len(results):3360\n",
      "\n",
      "Processing - file:11 - Tatoeba_stratify_train_2021-8-3_2136.csv\n",
      "Finished - file:11 - len(results):3696\n",
      "\n",
      "Processing - file:12 - Tatoeba_stratify_train_2021-8-3_2137.csv\n",
      "Finished - file:12 - len(results):4032\n",
      "\n",
      "Processing - file:13 - Tatoeba_stratify_train_2021-8-3_2138.csv\n",
      "Finished - file:13 - len(results):4368\n",
      "\n",
      "Processing - file:14 - Tatoeba_stratify_train_2021-8-3_2139.csv\n",
      "Finished - file:14 - len(results):4704\n",
      "\n",
      "Processing - file:15 - Tatoeba_stratify_train_2021-8-3_2140.csv\n",
      "Finished - file:15 - len(results):5040\n",
      "\n",
      "Processing - file:16 - Tatoeba_stratify_train_2021-8-3_2141.csv\n",
      "Error processing sentence: 啥？\n",
      "Finished - file:16 - len(results):5376\n",
      "\n",
      "Processing - file:17 - Tatoeba_stratify_train_2021-8-3_2142.csv\n",
      "Finished - file:17 - len(results):5712\n",
      "\n",
      "Processing - file:18 - Tatoeba_stratify_train_2021-8-3_2143.csv\n",
      "Finished - file:18 - len(results):6048\n",
      "\n",
      "Processing - file:19 - Tatoeba_stratify_train_2021-8-3_2144.csv\n",
      "Error processing sentence: 啥？\n",
      "Finished - file:19 - len(results):6384\n",
      "\n",
      "Processing - file:20 - Tatoeba_stratify_train_2021-8-3_2145.csv\n",
      "Finished - file:20 - len(results):6720\n",
      "\n",
      "Processing - file:21 - Tatoeba_stratify_train_2021-8-3_2147.csv\n",
      "Finished - file:21 - len(results):7056\n",
      "\n",
      "Processing - file:22 - Tatoeba_stratify_train_2021-8-3_2148.csv\n",
      "Error processing sentence: 咦！？\n",
      "Finished - file:22 - len(results):7392\n",
      "\n",
      "Processing - file:23 - Tatoeba_stratify_train_2021-8-3_2149.csv\n",
      "Finished - file:23 - len(results):7728\n",
      "\n",
      "Processing - file:24 - Tatoeba_stratify_train_2021-8-3_2150.csv\n",
      "Error processing sentence: 嗯..\n",
      "Finished - file:24 - len(results):8064\n",
      "\n",
      "Processing - file:25 - Tatoeba_stratify_train_2021-8-3_2151.csv\n",
      "Error processing sentence: 啥？\n",
      "Finished - file:25 - len(results):8400\n",
      "\n",
      "Processing - file:26 - Tatoeba_stratify_train_2021-8-3_2152.csv\n",
      "Error processing sentence: 咦！？\n",
      "Finished - file:26 - len(results):8736\n",
      "\n",
      "Processing - file:27 - Tatoeba_stratify_train_2021-8-3_2153.csv\n",
      "Finished - file:27 - len(results):9072\n",
      "\n",
      "Processing - file:28 - Tatoeba_stratify_train_2021-8-3_2154.csv\n",
      "Error processing sentence: 嗯..\n",
      "Finished - file:28 - len(results):9408\n",
      "\n",
      "Processing - file:29 - Tatoeba_stratify_train_2021-8-3_2155.csv\n",
      "Finished - file:29 - len(results):9744\n",
      "\n",
      "Processing - file:30 - Tatoeba_stratify_train_2021-8-3_2156.csv\n",
      "Finished - file:30 - len(results):10080\n",
      "\n",
      "Processing - file:31 - Tatoeba_stratify_train_2021-8-3_2157.csv\n",
      "Error processing sentence: 咦！？\n",
      "Finished - file:31 - len(results):10416\n",
      "\n",
      "Processing - file:32 - Tatoeba_stratify_train_2021-8-3_2158.csv\n",
      "Error processing sentence: 嗯..\n",
      "Error processing sentence: 咦！？\n",
      "Finished - file:32 - len(results):10752\n",
      "\n",
      "Processing - file:33 - Tatoeba_stratify_train_2021-8-3_2159.csv\n",
      "Finished - file:33 - len(results):11088\n",
      "\n",
      "Processing - file:34 - Tatoeba_stratify_train_2021-8-3_220.csv\n",
      "Error processing sentence: 嗯..\n",
      "Finished - file:34 - len(results):11424\n",
      "\n",
      "Processing - file:35 - Tatoeba_stratify_train_2021-8-3_221.csv\n",
      "Error processing sentence: 啥？\n",
      "Finished - file:35 - len(results):11760\n",
      "\n",
      "Processing - file:36 - Tatoeba_stratify_train_2021-8-3_2210.csv\n",
      "Finished - file:36 - len(results):12096\n",
      "\n",
      "Processing - file:37 - Tatoeba_stratify_train_2021-8-3_2211.csv\n",
      "Error processing sentence: 嗯..\n",
      "Finished - file:37 - len(results):12432\n",
      "\n",
      "Processing - file:38 - Tatoeba_stratify_train_2021-8-3_2212.csv\n",
      "Error processing sentence: 啥？\n",
      "Finished - file:38 - len(results):12768\n",
      "\n",
      "Processing - file:39 - Tatoeba_stratify_train_2021-8-3_2213.csv\n",
      "Error processing sentence: 咦！？\n",
      "Finished - file:39 - len(results):13104\n",
      "\n",
      "Processing - file:40 - Tatoeba_stratify_train_2021-8-3_2214.csv\n",
      "Error processing sentence: 咦！？\n",
      "Error processing sentence: 嗯..\n",
      "Finished - file:40 - len(results):13440\n",
      "\n",
      "Processing - file:41 - Tatoeba_stratify_train_2021-8-3_2215.csv\n",
      "Finished - file:41 - len(results):13776\n",
      "\n",
      "Processing - file:42 - Tatoeba_stratify_train_2021-8-3_2216.csv\n",
      "Finished - file:42 - len(results):14112\n",
      "\n",
      "Processing - file:43 - Tatoeba_stratify_train_2021-8-3_2217.csv\n",
      "Error processing sentence: 咦！？\n",
      "Finished - file:43 - len(results):14448\n",
      "\n",
      "Processing - file:44 - Tatoeba_stratify_train_2021-8-3_2218.csv\n",
      "Error processing sentence: 嗯..\n",
      "Finished - file:44 - len(results):14784\n",
      "\n",
      "Processing - file:45 - Tatoeba_stratify_train_2021-8-3_2219.csv\n",
      "Finished - file:45 - len(results):15120\n",
      "\n",
      "Processing - file:46 - Tatoeba_stratify_train_2021-8-3_222.csv\n",
      "Finished - file:46 - len(results):15456\n",
      "\n",
      "Processing - file:47 - Tatoeba_stratify_train_2021-8-3_2220.csv\n",
      "Finished - file:47 - len(results):15792\n",
      "\n",
      "Processing - file:48 - Tatoeba_stratify_train_2021-8-3_2221.csv\n",
      "Finished - file:48 - len(results):16128\n",
      "\n",
      "Processing - file:49 - Tatoeba_stratify_train_2021-8-3_2222.csv\n",
      "Error processing sentence: 嗯..\n",
      "Finished - file:49 - len(results):16464\n",
      "\n",
      "Processing - file:50 - Tatoeba_stratify_train_2021-8-3_2223.csv\n",
      "Finished - file:50 - len(results):16800\n",
      "\n",
      "Processing - file:51 - Tatoeba_stratify_train_2021-8-3_2224.csv\n",
      "Error processing sentence: 啥？\n",
      "Error processing sentence: 咦！？\n",
      "Finished - file:51 - len(results):17136\n",
      "\n",
      "Processing - file:52 - Tatoeba_stratify_train_2021-8-3_2225.csv\n",
      "Finished - file:52 - len(results):17472\n",
      "\n",
      "Processing - file:53 - Tatoeba_stratify_train_2021-8-3_2226.csv\n",
      "Error processing sentence: 嗯..\n",
      "Finished - file:53 - len(results):17808\n",
      "\n",
      "Processing - file:54 - Tatoeba_stratify_train_2021-8-3_2227.csv\n",
      "Finished - file:54 - len(results):18144\n",
      "\n",
      "Processing - file:55 - Tatoeba_stratify_train_2021-8-3_2228.csv\n",
      "Finished - file:55 - len(results):18480\n",
      "\n",
      "Processing - file:56 - Tatoeba_stratify_train_2021-8-3_2229.csv\n",
      "Finished - file:56 - len(results):18816\n",
      "\n",
      "Processing - file:57 - Tatoeba_stratify_train_2021-8-3_223.csv\n",
      "Finished - file:57 - len(results):19152\n",
      "\n",
      "Processing - file:58 - Tatoeba_stratify_train_2021-8-3_2230.csv\n",
      "Finished - file:58 - len(results):19488\n",
      "\n",
      "Processing - file:59 - Tatoeba_stratify_train_2021-8-3_2231.csv\n",
      "Error processing sentence: 啥？\n",
      "Finished - file:59 - len(results):19824\n",
      "\n",
      "Processing - file:60 - Tatoeba_stratify_train_2021-8-3_2232.csv\n",
      "Finished - file:60 - len(results):20160\n",
      "\n",
      "Processing - file:61 - Tatoeba_stratify_train_2021-8-3_2233.csv\n",
      "Finished - file:61 - len(results):20496\n",
      "\n",
      "Processing - file:62 - Tatoeba_stratify_train_2021-8-3_2234.csv\n",
      "Finished - file:62 - len(results):20832\n",
      "\n",
      "Processing - file:63 - Tatoeba_stratify_train_2021-8-3_2235.csv\n",
      "Finished - file:63 - len(results):21168\n",
      "\n",
      "Processing - file:64 - Tatoeba_stratify_train_2021-8-3_2237.csv\n",
      "Finished - file:64 - len(results):21504\n",
      "\n",
      "Processing - file:65 - Tatoeba_stratify_train_2021-8-3_2238.csv\n",
      "Finished - file:65 - len(results):21840\n",
      "\n",
      "Processing - file:66 - Tatoeba_stratify_train_2021-8-3_2239.csv\n",
      "Finished - file:66 - len(results):22176\n",
      "\n",
      "Processing - file:67 - Tatoeba_stratify_train_2021-8-3_2240.csv\n",
      "Finished - file:67 - len(results):22512\n",
      "\n",
      "Processing - file:68 - Tatoeba_stratify_train_2021-8-3_2241.csv\n",
      "Finished - file:68 - len(results):22848\n",
      "\n",
      "Processing - file:69 - Tatoeba_stratify_train_2021-8-3_2242.csv\n",
      "Finished - file:69 - len(results):23184\n",
      "\n",
      "Processing - file:70 - Tatoeba_stratify_train_2021-8-3_2243.csv\n",
      "Finished - file:70 - len(results):23520\n",
      "\n",
      "Processing - file:71 - Tatoeba_stratify_train_2021-8-3_2244.csv\n",
      "Finished - file:71 - len(results):23856\n",
      "\n",
      "Processing - file:72 - Tatoeba_stratify_train_2021-8-3_2245.csv\n",
      "Error processing sentence: 嗯..\n",
      "Finished - file:72 - len(results):24192\n",
      "\n",
      "Processing - file:73 - Tatoeba_stratify_train_2021-8-3_2247.csv\n",
      "Finished - file:73 - len(results):24528\n",
      "\n",
      "Processing - file:74 - Tatoeba_stratify_train_2021-8-3_2248.csv\n",
      "Error processing sentence: 咦！？\n",
      "Finished - file:74 - len(results):24864\n",
      "\n",
      "Processing - file:75 - Tatoeba_stratify_train_2021-8-3_2249.csv\n",
      "Finished - file:75 - len(results):25200\n",
      "\n",
      "Processing - file:76 - Tatoeba_stratify_train_2021-8-3_225.csv\n",
      "Error processing sentence: 啥？\n",
      "Error processing sentence: 咦！？\n",
      "Finished - file:76 - len(results):25536\n",
      "\n",
      "Processing - file:77 - Tatoeba_stratify_train_2021-8-3_2250.csv\n",
      "Error processing sentence: 啥？\n",
      "Error processing sentence: 咦！？\n",
      "Finished - file:77 - len(results):25872\n",
      "\n",
      "Processing - file:78 - Tatoeba_stratify_train_2021-8-3_2251.csv\n",
      "Error processing sentence: 啥？\n",
      "Finished - file:78 - len(results):26208\n",
      "\n",
      "Processing - file:79 - Tatoeba_stratify_train_2021-8-3_2252.csv\n",
      "Finished - file:79 - len(results):26544\n",
      "\n",
      "Processing - file:80 - Tatoeba_stratify_train_2021-8-3_2253.csv\n",
      "Error processing sentence: 咦！？\n",
      "Finished - file:80 - len(results):26880\n",
      "\n",
      "Processing - file:81 - Tatoeba_stratify_train_2021-8-3_2254.csv\n",
      "Finished - file:81 - len(results):27216\n",
      "\n",
      "Processing - file:82 - Tatoeba_stratify_train_2021-8-3_2255.csv\n",
      "Finished - file:82 - len(results):27552\n",
      "\n",
      "Processing - file:83 - Tatoeba_stratify_train_2021-8-3_2257.csv\n",
      "Error processing sentence: 嗯..\n",
      "Finished - file:83 - len(results):27888\n",
      "\n",
      "Processing - file:84 - Tatoeba_stratify_train_2021-8-3_2258.csv\n",
      "Error processing sentence: 嗯..\n",
      "Finished - file:84 - len(results):28224\n",
      "\n",
      "Processing - file:85 - Tatoeba_stratify_train_2021-8-3_2259.csv\n",
      "Finished - file:85 - len(results):28560\n",
      "\n",
      "Processing - file:86 - Tatoeba_stratify_train_2021-8-3_226.csv\n",
      "Error processing sentence: 啥？\n",
      "Finished - file:86 - len(results):28896\n",
      "\n",
      "Processing - file:87 - Tatoeba_stratify_train_2021-8-3_227.csv\n",
      "Error processing sentence: 啥？\n",
      "Finished - file:87 - len(results):29232\n",
      "\n",
      "Processing - file:88 - Tatoeba_stratify_train_2021-8-3_228.csv\n",
      "Finished - file:88 - len(results):29568\n",
      "\n",
      "Processing - file:89 - Tatoeba_stratify_train_2021-8-3_229.csv\n",
      "Finished - file:89 - len(results):29904\n",
      "\n",
      "Processing - file:90 - Tatoeba_stratify_train_2021-8-3_230.csv\n",
      "Finished - file:90 - len(results):30240\n",
      "\n",
      "Processing - file:91 - Tatoeba_stratify_train_2021-8-3_231.csv\n",
      "Error processing sentence: 啥？\n",
      "Finished - file:91 - len(results):30576\n",
      "\n",
      "Processing - file:92 - Tatoeba_stratify_train_2021-8-3_232.csv\n",
      "Finished - file:92 - len(results):30912\n",
      "\n",
      "Processing - file:93 - Tatoeba_stratify_train_2021-8-3_233.csv\n",
      "Finished - file:93 - len(results):31248\n",
      "\n",
      "Processing - file:94 - Tatoeba_stratify_train_2021-8-3_234.csv\n",
      "Finished - file:94 - len(results):31584\n",
      "\n",
      "Processing - file:95 - Tatoeba_stratify_train_2021-8-3_235.csv\n",
      "Finished - file:95 - len(results):31920\n",
      "\n",
      "Processing - file:96 - Tatoeba_stratify_train_2021-8-3_236.csv\n",
      "Finished - file:96 - len(results):32256\n",
      "\n",
      "Processing - file:97 - Tatoeba_stratify_train_2021-8-3_238.csv\n",
      "Finished - file:97 - len(results):32592\n",
      "\n",
      "Processing - file:98 - Tatoeba_stratify_train_2021-8-3_239.csv\n",
      "Error processing sentence: 嗯..\n",
      "Finished - file:98 - len(results):32928\n",
      "\n",
      "Processing - file:99 - Tatoeba_stratify_train_2021-8-4_1048.csv\n",
      "Finished - file:99 - len(results):33264\n",
      "\n",
      "Processing - file:100 - Tatoeba_stratify_train_2021-8-4_1049.csv\n",
      "Error processing sentence: 咦！？\n",
      "Finished - file:100 - len(results):33600\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# %%time\n",
    "# 11min for 2 samples\n",
    "# overnight run (8+ hours) for 100 samples\n",
    "for root, dirs, file in os.walk(\"output/\"):\n",
    "    if root == \"output/\":\n",
    "        for i in file:\n",
    "            train_sample = re.search(r\"train\", i)\n",
    "            if train_sample:\n",
    "                counter += 1\n",
    "                print(f\"Processing - file:{counter} - {i}\")\n",
    "                sample = pd.read_csv(f\"output/{i}\")\n",
    "                sample[\"Language_English_Name\"] = sample[\"Language\"].map(iso_639_2_English)\n",
    "                sample[\"Sentence_len\"] = sample.apply(lambda row: get_sentence_word_char_len(row), axis=1)\n",
    "                \n",
    "                sample[\"Sentence_len_bin\"] = pd.cut(\n",
    "                    x=sample[\"Sentence_len\"],\n",
    "                    bins=[0,1,2,3,4,5,6,7,8,9,10,16,27,48,99],\n",
    "                    labels=bin_labels\n",
    "                )\n",
    "                \n",
    "                ### LANGID ###\n",
    "                ##############\n",
    "                \n",
    "                sample[\"langid\"] = sample.apply(lambda row: list(lang_identifier.classify(row[\"Sentence\"])), axis=1)\n",
    "                sample[\"langid_language\"], sample[\"langid_score\"] = sample[\"langid\"].str\n",
    "                sample.drop(columns=[\"langid\"], inplace=True)\n",
    "                # map Language to English Name & langid_language to English Name\n",
    "                sample[\"Language_English_Name\"] = sample[\"Language\"].map(ISO_639_2)\n",
    "                sample[\"langid_language_English_Name\"] = sample[\"langid_language\"].map(ISO_639_1)\n",
    "                \n",
    "                ### LANGDETECT ###\n",
    "                ##################\n",
    "                \n",
    "                sample[\"langdetect\"] = sample.apply(lambda row: get_langdetect_prediction(row[\"Sentence\"]), axis=1)\n",
    "                sample[\"langdetect_language\"] = sample.apply(lambda row: get_langdetect_lang(row[\"langdetect\"]), axis=1)\n",
    "                sample[\"langdetect_score\"] = sample.apply(lambda row: get_langdetect_prob(row[\"langdetect\"]), axis=1)\n",
    "                sample.drop(columns=[\"langdetect\"], inplace=True)\n",
    "                sample[\"langdetect_language_English_Name\"] = sample[\"langdetect_language\"].map(ISO_639_1)\n",
    "                \n",
    "                ### Get F1 ###\n",
    "                ##############\n",
    "                \n",
    "                for this_bin in bin_labels:\n",
    "                    bin_results = sample[sample[\"Sentence_len_bin\"]==this_bin]\n",
    "                    for lang in bin_results[\"Language_English_Name\"].unique().tolist():\n",
    "                        these_results = bin_results[(bin_results[\"Language_English_Name\"]==lang)]\n",
    "                        for prediction in [\"langid\", \"langdetect\"]:\n",
    "                            these_labels = these_results[\"Language_English_Name\"].unique().tolist()\n",
    "                            this_f1 = f1_score(these_results[\"Language_English_Name\"], these_results[f\"{prediction}_language_English_Name\"], labels=these_labels, average=\"weighted\")\n",
    "                            result = (this_bin, lang, prediction, round(this_f1,2))\n",
    "                            results.append(result)\n",
    "                \n",
    "                print(f\"Finished - file:{counter} - len(results):{len(results)}\\n\")\n",
    "                \n",
    "del file, bin_results, i, lang, prediction, result, root, sample, these_labels, these_results, this_bin, this_f1 # clean up vars & free up memory"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "8aa542a3-f9b0-4526-9857-7ed872e03304",
   "metadata": {},
   "outputs": [],
   "source": [
    "results = pd.DataFrame(data=results, columns=[\"Bin\", \"Language\", \"Source\", \"F1\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "da16f36b-4e53-4178-b15d-a5a8cffbd766",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "90efcf58-4c81-40c2-bf63-a181219252df",
   "metadata": {},
   "outputs": [],
   "source": [
    "now = datetime.now()\n",
    "datetime_stamp = f\"{now.year}-{now.month}-{now.day}_{now.hour}{now.minute}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "1cb5dff3-ad39-4d86-8b58-dfc335d8d6d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "results.to_csv(f\"output/results_df_{datetime_stamp}.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6252cd20-76d4-4239-b61c-26ab458a5c99",
   "metadata": {},
   "source": [
    "# End"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}