export_sentences_for_lm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "bad14518",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "import datasets as hfd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a12b10b2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "853bccf4",
   "metadata": {},
   "outputs": [],
   "source": [
    "DATA_ROOT_DP = os.environ['DATA_HOME']\n",
    "CV_DP = f'{DATA_ROOT_DP}/datasets/cv-corpus-8.0-2022-01-19/be'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b01404c8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['dev.tsv',\n",
       " 'validated.tsv',\n",
       " 'train.tsv',\n",
       " 'clips',\n",
       " 'reported.tsv',\n",
       " 'invalidated.tsv',\n",
       " 'other.tsv',\n",
       " 'test.tsv']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.listdir(CV_DP)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9cf7c8d9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "df_train.shape: (314305, 10)\n",
      "df_dev.shape: (15803, 10)\n",
      "df_test.shape: (15801, 10)\n",
      "df_other.shape: (40596, 10)\n"
     ]
    }
   ],
   "source": [
    "df_train = pd.read_csv(os.path.join(CV_DP, 'train.tsv'), delimiter='\\t')\n",
    "df_dev = pd.read_csv(os.path.join(CV_DP, 'dev.tsv'), delimiter='\\t')\n",
    "df_test = pd.read_csv(os.path.join(CV_DP, 'test.tsv'), delimiter='\\t')\n",
    "df_other = pd.read_csv(os.path.join(CV_DP, 'other.tsv'), delimiter='\\t')\n",
    "\n",
    "print(f'df_train.shape: {df_train.shape}')\n",
    "print(f'df_dev.shape: {df_dev.shape}')\n",
    "print(f'df_test.shape: {df_test.shape}')\n",
    "print(f'df_other.shape: {df_other.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c3b2725f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "9cb41c79",
   "metadata": {},
   "source": [
    "## combine sentences from df_train and df_other to build LM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "d4a713aa",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "26906\n"
     ]
    }
   ],
   "source": [
    "s_from_other = set(df_other['sentence']).difference(df_dev['sentence']).difference(df_test['sentence'])\n",
    "print(len(s_from_other))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "33fc8339",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "len(s_train_other): 314676\n"
     ]
    }
   ],
   "source": [
    "s_train_other = list(set(df_train['sentence']).union(s_from_other))\n",
    "print(f'len(s_train_other): {len(s_train_other)}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2b418ade",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "a919b079",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "df.shape: (314676, 2)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "train    314305\n",
       "other       371\n",
       "Name: split, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.DataFrame({'sentence': s_train_other})\n",
    "df.loc[df['sentence'].isin(df_train['sentence']), 'split'] = 'train'\n",
    "df.loc[~df['sentence'].isin(df_train['sentence']), 'split'] = 'other'\n",
    "print(f'df.shape: {df.shape}')\n",
    "df['split'].value_counts(dropna=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "eaa7f3cf",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "071bd49e",
   "metadata": {},
   "source": [
    "## save"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "600398c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "fp = 'artifacts/lm/sentences_for_lm.tsv'\n",
    "os.makedirs(os.path.dirname(fp), exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "66096e73",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(fp, index=False, sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b1fd11d6",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "c0ecb2c4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sentence\tsplit\r\n",
      "Вы ўжо ўсе, відавочна, адчулі, што нешта не так?\ttrain\r\n",
      "Зойдзе да яе, а яна не ведае, з чаго пачынаць гаворку.\ttrain\r\n",
      "І трэба істотна прызавыя і іншае дадаваць за вынік на ўнутраным чэмпіянаце.\ttrain\r\n",
      "Каб дзевяць кампазіцый на канцэрце гучалі амаль як на пласцінцы.\ttrain\r\n",
      "Але маюцца межавыя ўмовы.\ttrain\r\n",
      "Гэта можа быць пералом.\ttrain\r\n",
      "Сапраўды, цяпер гістарычны помнік выглядае інакш, чым месяц таму.\ttrain\r\n",
      "І вось прыехаў да вас.\ttrain\r\n",
      "Апроч таго, мы яшчэ не абнавілі свой авіяцыйны парк.\ttrain\r\n"
     ]
    }
   ],
   "source": [
    "!head -10 $fp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "e59714df",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  314677  2700184 30778659 artifacts/lm/sentences_for_lm.tsv\r\n"
     ]
    }
   ],
   "source": [
    "!wc $fp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "efb93882",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f9733b18",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "58ab854a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb7fac19",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}