part1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas\n",
    "pandas.set_option('max_colwidth', 200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nWe will use lenta dataset from https://github.com/natasha/corus\\n'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "We will use lenta dataset from https://github.com/natasha/corus\n",
    "\"\"\"\n",
    "#!mkdir -p data\n",
    "#!wget https://github.com/yutkin/Lenta.Ru-News-Dataset/releases/download/v1.0/lenta-ru-news.csv.gz -O data/lenta-ru-news.csv.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LentaRecord(\n",
       "    url='https://lenta.ru/news/2018/12/14/cancer/',\n",
       "    title='Названы регионы России с\\xa0самой высокой смертностью от\\xa0рака',\n",
       "    text='Вице-премьер по социальным вопросам Татьяна Голикова рассказала, в каких регионах России зафиксирована наиболее высокая смертность от рака, сообщает РИА Новости. По словам Голиковой, чаще всего онкологические заболевания становились причиной смерти в Псковской, Тверской, Тульской и Орловской областях, а также в Севастополе. Вице-премьер напомнила, что главные факторы смертности в России — рак и болезни системы кровообращения. В начале года стало известно, что смертность от онкологических заболеваний среди россиян снизилась впервые за три года. По данным Росстата, в 2017 году от рака умерли 289 тысяч человек. Это на 3,5 процента меньше, чем годом ранее.',\n",
       "    topic='Россия',\n",
       "    tags='Общество'\n",
       ")"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from corus import load_lenta\n",
    "path = 'data/lenta-ru-news.csv.gz'\n",
    "records = load_lenta(path)\n",
    "next(records)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#preprocessing: lower-case, commas, lemmatization\n",
    "#PRACTICE: implement preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['выплата', 'на', 'второй', 'ребенок']\n",
      "['выплата', 'на', 'второй', 'ребенок']\n"
     ]
    }
   ],
   "source": [
    "import razdel\n",
    "from pymystem3 import Mystem\n",
    "#from pymorphy2 import MorphAnalyzer\n",
    "\n",
    "_LEMMATIZER = None\n",
    "def tag(word):\n",
    "    global _LEMMATIZER\n",
    "    if _LEMMATIZER is None:\n",
    "        _LEMMATIZER = Mystem()\n",
    "        #_LEMMATIZER = MorphAnalyzer('ru')\n",
    "    try:\n",
    "        processed = _LEMMATIZER.analyze(word)[0]\n",
    "    except:\n",
    "        return word\n",
    "    if not processed.get('analysis'):\n",
    "        return word.strip().lower()\n",
    "    lemma = processed[\"analysis\"][0][\"lex\"].lower().strip()\n",
    "    return lemma\n",
    "\n",
    "import re\n",
    "def tokenize_re(doc):\n",
    "    return [tag(x) for x in re.split('(\\W+)', doc) if x.strip()]\n",
    "\n",
    "def tokenize(doc):\n",
    "    # упомянуть регулярные выражения\n",
    "    return [tag(t.text) for t in razdel.tokenize(doc)]\n",
    "\n",
    "import sentencepiece\n",
    "def tokenize_bpe(doc):\n",
    "    #FIXME\n",
    "    pass\n",
    "\n",
    "print(tokenize_re('Выплаты на второго ребёнка'))\n",
    "print(tokenize('Выплаты на второго ребёнка'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['называть', 'регион', 'россия', 'с', 'самый', 'высокий', 'смертность', ',', 'от', 'рак', '...', 'привет', 'все', '!!!']\n",
      "['называть', 'регион', 'россия', 'с', 'самый', 'высокий', 'смертность', ',', 'от', 'рак', '...', 'привет', 'все', '!!!']\n",
      "['это', 'на', '3', ',', '5', 'процент', 'мало', ',', 'что', 'год', 'ранее', '.']\n",
      "['это', 'на', '3,5', 'процент', 'мало', ',', 'что', 'год', 'ранее', '.']\n"
     ]
    }
   ],
   "source": [
    "print(tokenize_re('Названы регионы России с\\xa0самой высокой смертностью, от\\xa0рака... Привет всем!!!'))\n",
    "print(tokenize('Названы регионы России с\\xa0самой высокой смертностью, от\\xa0рака... Привет всем!!!'))\n",
    "print(tokenize_re('Это на 3,5 процента меньше, чем годом ранее.'))\n",
    "print(tokenize('Это на 3,5 процента меньше, чем годом ранее.'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "096b3be2b20c4dd9a521d2245d71a13f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=25000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25000\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>[называть, регион, россия, с, самый, высокий, смертность, от, рак]</td>\n",
       "      <td>Россия</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>[австрия, не, представлять, доказательство, вина, российский, биатлонист]</td>\n",
       "      <td>Спорт</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>[обнаруживать, самый, счастливый, место, на, планета]</td>\n",
       "      <td>Путешествия</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>[в, сша, раскрывать, сумма, расход, на, расследование, «, российский, дело, »]</td>\n",
       "      <td>Мир</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>[хакер, рассказывать, о, план, великобритания, заминировать, севастополь]</td>\n",
       "      <td>Мир</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                0  \\\n",
       "0              [называть, регион, россия, с, самый, высокий, смертность, от, рак]   \n",
       "1       [австрия, не, представлять, доказательство, вина, российский, биатлонист]   \n",
       "2                           [обнаруживать, самый, счастливый, место, на, планета]   \n",
       "3  [в, сша, раскрывать, сумма, расход, на, расследование, «, российский, дело, »]   \n",
       "4       [хакер, рассказывать, о, план, великобритания, заминировать, севастополь]   \n",
       "\n",
       "             1  \n",
       "0       Россия  \n",
       "1        Спорт  \n",
       "2  Путешествия  \n",
       "3          Мир  \n",
       "4          Мир  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas\n",
    "from tqdm import tqdm_notebook as tq\n",
    "\n",
    "records = load_lenta(path)\n",
    "# overfitting and underfitting\n",
    "# splitting the dataset. how many examples to take for a test?\n",
    "dataset = []\n",
    "for i, r in tq(enumerate(records), total=25000):\n",
    "    if not r.topic:\n",
    "        # есть примеры в датасете, где нет категории\n",
    "        continue\n",
    "    dataset.append((tokenize(r.title), r.topic))\n",
    "    if len(dataset) >= 25000:\n",
    "        break\n",
    "df = pandas.DataFrame(dataset)\n",
    "print(len(df))\n",
    "df.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "17 ['69-я параллель', 'Бизнес', 'Бывший СССР', 'Дом', 'Из жизни', 'Интернет и СМИ', 'Крым', 'Культпросвет ', 'Культура', 'Мир', 'Наука и техника', 'Путешествия', 'Россия', 'Силовые структуры', 'Спорт', 'Ценности', 'Экономика']\n"
     ]
    }
   ],
   "source": [
    "classes = sorted(set([_[1] for _ in dataset]))\n",
    "print(len(classes), classes)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Dice with N sides STD formula: <img src=\"files/dice-std.png\">"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "4.898979485566356"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "9.797958971132712"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "2514.6055943493534"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Calculator: https://select-statistics.co.uk/calculators/sample-size-calculator-two-proportions/\n",
    "import math\n",
    "def std_n_classes(n):\n",
    "    return math.sqrt((n*n-1) / 12)\n",
    "display(\n",
    "    std_n_classes(2),\n",
    "    std_n_classes(17),\n",
    "    std_n_classes(17) / std_n_classes(2),\n",
    "    24638 / 9.797958)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20000 20000 2500 2500 2500 2500\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x_train</th>\n",
       "      <th>y_train</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>[в, россия, закрывать, чемпионат, мир, по, футбол]</td>\n",
       "      <td>Спорт</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>[назначать, орешкин, неизвестный, таджик, обретать, миссия]</td>\n",
       "      <td>Экономика</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>[в, киев, массово, лопнуть, труба, после, подача, горячий, вода]</td>\n",
       "      <td>Бывший СССР</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>[россиянин, избегать, тюрьма, за, убийство, оскорблять, он, девушка]</td>\n",
       "      <td>Россия</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>[умирать, режиссер, «, чегемский, детектив, », александр, светлов]</td>\n",
       "      <td>Культура</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                x_train  \\\n",
       "0                    [в, россия, закрывать, чемпионат, мир, по, футбол]   \n",
       "1           [назначать, орешкин, неизвестный, таджик, обретать, миссия]   \n",
       "2      [в, киев, массово, лопнуть, труба, после, подача, горячий, вода]   \n",
       "3  [россиянин, избегать, тюрьма, за, убийство, оскорблять, он, девушка]   \n",
       "4    [умирать, режиссер, «, чегемский, детектив, », александр, светлов]   \n",
       "\n",
       "       y_train  \n",
       "0        Спорт  \n",
       "1    Экономика  \n",
       "2  Бывший СССР  \n",
       "3       Россия  \n",
       "4     Культура  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>x_dev</th>\n",
       "      <th>y_dev</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>[телеведущая, отказывать, в, работа, из-за, «, недостаточно, большой, грудь, »]</td>\n",
       "      <td>Интернет и СМИ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>[шарапов, отвечать, на, отказ, серена, уильямс, играть, против, она]</td>\n",
       "      <td>Спорт</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>[bvlgari, представлять, новый, вариация, на, змеиный, тема]</td>\n",
       "      <td>Ценности</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>[находить, новый, способ, побеждать, особо, опасный, рак]</td>\n",
       "      <td>Наука и техника</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>[анестезиолог, изнасиловать, россиянка, под, наркоз, и, пойти, под, суд]</td>\n",
       "      <td>Силовые структуры</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                             x_dev  \\\n",
       "0  [телеведущая, отказывать, в, работа, из-за, «, недостаточно, большой, грудь, »]   \n",
       "1             [шарапов, отвечать, на, отказ, серена, уильямс, играть, против, она]   \n",
       "2                      [bvlgari, представлять, новый, вариация, на, змеиный, тема]   \n",
       "3                        [находить, новый, способ, побеждать, особо, опасный, рак]   \n",
       "4         [анестезиолог, изнасиловать, россиянка, под, наркоз, и, пойти, под, суд]   \n",
       "\n",
       "               y_dev  \n",
       "0     Интернет и СМИ  \n",
       "1              Спорт  \n",
       "2           Ценности  \n",
       "3    Наука и техника  \n",
       "4  Силовые структуры  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#train, dev, test\n",
    "#20000, 2500, 2500 \n",
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_dev, y_train, y_dev = train_test_split([_[0] for _ in dataset], \n",
    "                                                  [_[1] for _ in dataset], test_size=2500, random_state=42)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_train, \n",
    "                                                    y_train, test_size=2500, random_state=42)\n",
    "print(len(X_train), len(y_train), len(X_dev), len(y_dev), len(X_test), len(y_test))\n",
    "display(pandas.DataFrame(zip(X_train[:5], y_train[:5]), columns=['x_train', 'y_train']))\n",
    "display(pandas.DataFrame(zip(X_dev[:5], y_dev[:5]), columns=['x_dev', 'y_dev']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(0, ','),\n",
      " (1, ', ха-ха'),\n",
      " (2, '.'),\n",
      " (3, '. это'),\n",
      " (4, '...'),\n",
      " (5, 'а'),\n",
      " (6, 'а это'),\n",
      " (7, 'второй'),\n",
      " (8, 'второй предложение'),\n",
      " (9, 'выплата'),\n",
      " (10, 'выплата на'),\n",
      " (11, 'на'),\n",
      " (12, 'на ребенок'),\n",
      " (13, 'оно'),\n",
      " (14, 'оно снова'),\n",
      " (15, 'первый'),\n",
      " (16, 'первый предложение'),\n",
      " (17, 'предложение'),\n",
      " (18, 'предложение ,'),\n",
      " (19, 'предложение .'),\n",
      " (20, 'пример'),\n",
      " (21, 'пример второй'),\n",
      " (22, 'ребенок'),\n",
      " (23, 'снова'),\n",
      " (24, 'снова ...'),\n",
      " (25, 'третий'),\n",
      " (26, 'третий предложение'),\n",
      " (27, 'ха-ха'),\n",
      " (28, 'это'),\n",
      " (29, 'это оно'),\n",
      " (30, 'это первый'),\n",
      " (31, 'это пример'),\n",
      " (32, 'это третий')]\n"
     ]
    }
   ],
   "source": [
    "TEST_ITEMS = {\n",
    "    6: 'Это первое предложение. Это оно снова...', \n",
    "    8: 'Это пример второго предложения, ха-ха-ха',\n",
    "    9: 'А это третье предложение',\n",
    "    10: 'выплата на ребенка'}\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "vectorizer = CountVectorizer(ngram_range=(1, 2), tokenizer=tokenize)\n",
    "vectorizer.fit_transform(TEST_ITEMS.values())\n",
    "import pprint; pprint.pprint(list(enumerate(vectorizer.get_feature_names())))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['выплата', 'на', 'второй', 'ребенок']\n",
      "Example:\n",
      "  (0, 9)\t1\n",
      "  (1, 11)\t1\n",
      "  (2, 7)\t1\n",
      "  (3, 22)\t1\n",
      "Dense:\n",
      "[[0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n",
      " [0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n",
      " [0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n",
      " [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0]]\n"
     ]
    }
   ],
   "source": [
    "tokenized = tokenize_re('Выплаты на второго ребёнка')\n",
    "print(tokenized)\n",
    "feats_example = vectorizer.transform(tokenized)\n",
    "print(\"Example:\")\n",
    "print(feats_example)\n",
    "print(\"Dense:\")\n",
    "print(feats_example.todense())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test example:\n",
      "  (0, 27)\t1\n",
      "  (1, 0)\t1\n",
      "  (2, 15)\t1\n",
      "  (3, 17)\t1\n",
      "  (5, 20)\t1\n",
      "Test dense:\n",
      "Dense:\n",
      "[[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0]\n",
      " [1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n",
      " [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n",
      " [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n",
      " [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]\n",
      " [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 0 0 0 0 0 0 0 0 0 0]\n",
      " [0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0]]\n"
     ]
    }
   ],
   "source": [
    "TEST_QUERY = 'ха-ха, первое предложение для примера!'\n",
    "print(\"Test example:\")\n",
    "feats_test = vectorizer.transform(tokenize(TEST_QUERY))\n",
    "print(feats_test)\n",
    "print(\"Test dense:\")\n",
    "print(\"Dense:\")\n",
    "print(feats_test.todense())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(0, '!'),\n",
      " (1, 'британский разведка'),\n",
      " (2, 'дворцовый переворот'),\n",
      " (3, 'игра с'),\n",
      " (4, 'между мат'),\n",
      " (5, 'обнаженный азия'),\n",
      " (6, 'позволять'),\n",
      " (7, 'разрушать стадион'),\n",
      " (8, 'соглашаться'),\n",
      " (9, 'факт »')]\n",
      "Vocabulary size: 95391\n"
     ]
    }
   ],
   "source": [
    "vec_x = CountVectorizer(ngram_range=(1, 2), tokenizer=lambda x: x, lowercase=False, min_df=1)\n",
    "vec_x.fit_transform(X_train)\n",
    "import pprint; pprint.pprint(list(enumerate(vec_x.get_feature_names()[::10000])))\n",
    "print(\"Vocabulary size:\", len(vec_x.vocabulary_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(0, '!'), (1, 'машина и'), (2, 'совместно')]\n",
      "Vocabulary size: 23708\n"
     ]
    }
   ],
   "source": [
    "vec_x = CountVectorizer(ngram_range=(1, 2), tokenizer=lambda x: x, lowercase=False, min_df=2)\n",
    "vec_x.fit_transform(X_train)\n",
    "import pprint; pprint.pprint(list(enumerate(vec_x.get_feature_names()[::10000])))\n",
    "print(\"Vocabulary size:\", len(vec_x.vocabulary_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 20000 x 17000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'69-я параллель': 0,\n",
       " 'Бизнес': 1,\n",
       " 'Бывший СССР': 2,\n",
       " 'Дом': 3,\n",
       " 'Из жизни': 4,\n",
       " 'Интернет и СМИ': 5,\n",
       " 'Крым': 6,\n",
       " 'Культпросвет ': 7,\n",
       " 'Культура': 8,\n",
       " 'Мир': 9,\n",
       " 'Наука и техника': 10,\n",
       " 'Путешествия': 11,\n",
       " 'Россия': 12,\n",
       " 'Силовые структуры': 13,\n",
       " 'Спорт': 14,\n",
       " 'Ценности': 15,\n",
       " 'Экономика': 16}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vec_y = {c: i for i, c in enumerate(classes)}\n",
    "vec_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x_vec = vec_x.transform(X_train)\n",
    "dev_x_vec = vec_x.transform(X_dev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearSVC(C=1.0, class_weight=None, dual=True, fit_intercept=True,\n",
       "          intercept_scaling=1, loss='squared_hinge', max_iter=1000,\n",
       "          multi_class='ovr', penalty='l2', random_state=0, tol=1e-05,\n",
       "          verbose=0)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.svm import LinearSVC\n",
    "clf = LinearSVC(random_state=0, tol=1e-5)\n",
    "clf.fit(train_x_vec, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>predicted</th>\n",
       "      <th>expected</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>Интернет и СМИ</td>\n",
       "      <td>Интернет и СМИ</td>\n",
       "      <td>[телеведущая, отказывать, в, работа, из-за, «, недостаточно, большой, грудь, »]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>Спорт</td>\n",
       "      <td>Спорт</td>\n",
       "      <td>[шарапов, отвечать, на, отказ, серена, уильямс, играть, против, она]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>Ценности</td>\n",
       "      <td>Ценности</td>\n",
       "      <td>[bvlgari, представлять, новый, вариация, на, змеиный, тема]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>Наука и техника</td>\n",
       "      <td>Наука и техника</td>\n",
       "      <td>[находить, новый, способ, побеждать, особо, опасный, рак]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>Россия</td>\n",
       "      <td>Силовые структуры</td>\n",
       "      <td>[анестезиолог, изнасиловать, россиянка, под, наркоз, и, пойти, под, суд]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2495</td>\n",
       "      <td>Спорт</td>\n",
       "      <td>Спорт</td>\n",
       "      <td>[смалывать, поведывать, о, смерть, на, футбольный, поле]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2496</td>\n",
       "      <td>Мир</td>\n",
       "      <td>Интернет и СМИ</td>\n",
       "      <td>[plus-size-блогер, рассказывать, о, нелепый, комплимент, мужчина]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2497</td>\n",
       "      <td>Интернет и СМИ</td>\n",
       "      <td>Экономика</td>\n",
       "      <td>[оператор, находить, способ, частично, окупать, «, закон, яровой, »]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2498</td>\n",
       "      <td>Россия</td>\n",
       "      <td>Россия</td>\n",
       "      <td>[россиянка, давать, отпор, избивать, она, козленок, полицейский, и, быть, наказывать]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2499</td>\n",
       "      <td>Бывший СССР</td>\n",
       "      <td>Бывший СССР</td>\n",
       "      <td>[в, киев, находить, способ, заставлять, россия, пойти, на, уступка]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2500 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            predicted           expected  \\\n",
       "0      Интернет и СМИ     Интернет и СМИ   \n",
       "1               Спорт              Спорт   \n",
       "2            Ценности           Ценности   \n",
       "3     Наука и техника    Наука и техника   \n",
       "4              Россия  Силовые структуры   \n",
       "...               ...                ...   \n",
       "2495            Спорт              Спорт   \n",
       "2496              Мир     Интернет и СМИ   \n",
       "2497   Интернет и СМИ          Экономика   \n",
       "2498           Россия             Россия   \n",
       "2499      Бывший СССР        Бывший СССР   \n",
       "\n",
       "                                                                                      title  \n",
       "0           [телеведущая, отказывать, в, работа, из-за, «, недостаточно, большой, грудь, »]  \n",
       "1                      [шарапов, отвечать, на, отказ, серена, уильямс, играть, против, она]  \n",
       "2                               [bvlgari, представлять, новый, вариация, на, змеиный, тема]  \n",
       "3                                 [находить, новый, способ, побеждать, особо, опасный, рак]  \n",
       "4                  [анестезиолог, изнасиловать, россиянка, под, наркоз, и, пойти, под, суд]  \n",
       "...                                                                                     ...  \n",
       "2495                               [смалывать, поведывать, о, смерть, на, футбольный, поле]  \n",
       "2496                      [plus-size-блогер, рассказывать, о, нелепый, комплимент, мужчина]  \n",
       "2497                   [оператор, находить, способ, частично, окупать, «, закон, яровой, »]  \n",
       "2498  [россиянка, давать, отпор, избивать, она, козленок, полицейский, и, быть, наказывать]  \n",
       "2499                    [в, киев, находить, способ, заставлять, россия, пойти, на, уступка]  \n",
       "\n",
       "[2500 rows x 3 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predict_dev = clf.predict(dev_x_vec)\n",
    "pandas.DataFrame(zip(predict_dev, y_dev, X_dev), columns=['predicted', 'expected', 'title'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7248"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.score(dev_x_vec, y_dev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report, confusion_matrix, accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                   precision    recall  f1-score   support\n",
      "\n",
      "   69-я параллель      0.545     0.923     0.686        13\n",
      "      Бывший СССР      0.789     0.759     0.774       158\n",
      "              Дом      0.768     0.768     0.768        82\n",
      "         Из жизни      0.654     0.651     0.652       186\n",
      "   Интернет и СМИ      0.595     0.685     0.637       165\n",
      "             Крым      0.000     0.000     0.000         0\n",
      "         Культура      0.744     0.726     0.735       164\n",
      "              Мир      0.726     0.689     0.707       347\n",
      "  Наука и техника      0.798     0.753     0.775       178\n",
      "      Путешествия      0.662     0.707     0.684        75\n",
      "           Россия      0.695     0.644     0.668       385\n",
      "Силовые структуры      0.625     0.616     0.620       138\n",
      "            Спорт      0.920     0.906     0.913       266\n",
      "         Ценности      0.713     0.750     0.731        96\n",
      "        Экономика      0.698     0.777     0.736       247\n",
      "\n",
      "         accuracy                          0.725      2500\n",
      "        macro avg      0.662     0.690     0.672      2500\n",
      "     weighted avg      0.728     0.725     0.725      2500\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Projects/ai/nlp_workshop/.venv/lib/python3.6/site-packages/sklearn/metrics/classification.py:1439: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.\n",
      "  'recall', 'true', average, warn_for)\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(predict_dev, y_dev, digits=3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 12   0   0   1   0   0   0   0   0   0   0   0   0   0   0]\n",
      " [  0 120   0   0   4   0   4   2   3   1   6   2   1   1  14]\n",
      " [  0   1  63   2   1   0   0   2   0   0   5   4   0   0   4]\n",
      " [  2   1   2 121   8   0   3  21   2   3   9   4   3   5   2]\n",
      " [  1   0   0  10 113   0   5   7   1   5   8   1   4   3   7]\n",
      " [  0   0   0   0   0   0   0   0   0   0   0   0   0   0   0]\n",
      " [  3   1   1   5  10   0 119   6   0   2   6   4   2   4   1]\n",
      " [  0   8   0  13  13   0   6 239  12   4  24   4   3   4  17]\n",
      " [  0   4   0   4   6   1   5   6 134   3   3   1   1   5   5]\n",
      " [  0   2   2   3   3   0   1   4   2  53   3   0   0   0   2]\n",
      " [  2   7   9   7  13   0  10  23   7   5 248  24   4   2  24]\n",
      " [  0   4   2   5   3   0   3   5   0   1  27  85   1   0   2]\n",
      " [  0   0   0   1   5   0   2   3   1   1   2   4 241   3   3]\n",
      " [  0   0   0   9   4   0   2   4   0   1   0   1   1  72   2]\n",
      " [  2   4   3   4   7   0   0   7   6   1  16   2   1   2 192]]\n"
     ]
    }
   ],
   "source": [
    "print(confusion_matrix(predict_dev, y_dev))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1466148"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lines = [' '.join(['__label__'+y.replace(' ','_')] + x) for x, y in zip(X_train, y_train)]\n",
    "lines[:5]\n",
    "from pathlib import Path\n",
    "Path('data/train.txt').write_text('\\n'.join(lines))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "183526"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lines = [' '.join(['__label__'+y] + x) for x, y in zip(X_dev, y_dev)]\n",
    "Path('data/dev.txt').write_text('\\n'.join(lines))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Read 0M words\n",
      "Number of words:  8505\n",
      "Number of labels: 17\n",
      "Progress: 100.0% words/sec/thread:  801783 lr:  0.000000 loss:  0.927016 ETA:   0h 0m\n"
     ]
    }
   ],
   "source": [
    "!dist/fastText-0.9.1/fasttext supervised -input data/train.txt -output data/ft.bin -minCount 2 -epoch 10 -wordNgrams 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "N\t1647\r\n",
      "P@1\t0.738\r\n",
      "R@1\t0.738\r\n"
     ]
    }
   ],
   "source": [
    "!dist/fastText-0.9.1/fasttext test data/ft.bin.bin data/dev.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "import fasttext\n",
    "model = fasttext.load_model('data/ft.bin.bin')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(['телеведущая',\n",
       "  'отказывать',\n",
       "  'в',\n",
       "  'работа',\n",
       "  'из-за',\n",
       "  '«',\n",
       "  'недостаточно',\n",
       "  'большой',\n",
       "  'грудь',\n",
       "  '»'],\n",
       " 'Интернет и СМИ')"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_dev[0], y_dev[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('__label__Интернет_и_СМИ', 0.8731516599655151)]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "16da06a1c4604e09a223e1800fc223f7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2500), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[5, 14, 15, 10, 13, 13, 10, 12, 2, 4]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "[5, 14, 15, 10, 13, 12, 10, 9, 16, 4]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "0.726"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "display(list(zip(*model.predict(' '.join(X_dev[0]), k=1))))\n",
    "predicted = []\n",
    "for x in tq(X_dev):\n",
    "    predicts = model.predict(' '.join(x), k=1)[0][0]\n",
    "    #print(predicts)\n",
    "    predicted.append(vec_y[predicts[9:].replace('_', ' ')])\n",
    "    expected = [vec_y[y] for y in y_dev]\n",
    "display(predicted[:10], expected[:10])\n",
    "accuracy_score(predicted, expected)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# NN\n",
    "# more complex models"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}