tutorials/vectorizer_usage.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('../')\n",
    "import soynlp"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Build tokenizer\n",
    "\n",
    "Vectorizer 가 이용할 토크나이저를 학습합니다. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from soynlp import DoublespaceLineCorpus\n",
    "from soynlp.word import WordExtractor\n",
    "from soynlp.tokenizer import LTokenizer\n",
    "from soynlp.vectorizer import BaseVectorizer\n",
    "\n",
    "corpus_path = 'YOURS'\n",
    "corpus = DoublespaceLineCorpus(corpus_path, iter_sent=True)\n",
    "word_extractor = WordExtractor()\n",
    "word_extractor.train(corpus)\n",
    "\n",
    "# (leftside cohesion, rightside cohesion)\n",
    "cohesion_scores = word_extractor.all_cohesion_scores()\n",
    "\n",
    "# use only leftside cohesion\n",
    "scores = {word:score[0] for word, score in cohesion_scores.items()}\n",
    "tokenizer = LTokenizer(scores=scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Vectorizing\n",
    "\n",
    "corpus 를 term frequency matrix 로 변환합니다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vectorizer = BaseVectorizer(\n",
    "    tokenizer=tokenizer,\n",
    "    min_tf=0,\n",
    "    max_tf=10000,\n",
    "    min_df=0,\n",
    "    max_df=1.0,\n",
    "    stopwords=None,\n",
    "    lowercase=True,\n",
    "    verbose=True\n",
    ")\n",
    "\n",
    "corpus.iter_sent = False\n",
    "x = vectorizer.fit_transform(corpus)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Save matrix market file without loading matrix on memory\n",
    "\n",
    "대량의 문서에 대한 sparse matrix 를 메모리에 올리지 않고 곧바로 파일에 저장합니다. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "scanning was done                                        \n",
      "8193 terms are recognized\n",
      "writing to file was done. 100 docs                  \n"
     ]
    }
   ],
   "source": [
    "vectorizer = BaseVectorizer(min_tf=1, tokenizer=tokenizer)\n",
    "corpus.iter_sent = False\n",
    "\n",
    "matrix_path = 'YOURS'\n",
    "vectorizer.fit_to_file(corpus, matrix_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{3: 1, 258: 1, 428: 1, 1814: 1}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer.encode_a_doc_to_bow('오늘 뉴스는 이것이 전부다')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'뉴스': 1, '는': 1, '오늘': 1, '이것이': 1}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer.decode_from_bow({3: 1, 258: 1, 428: 1, 1814: 1})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[258, 4, 428, 3, 333]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer.encode_a_doc_to_list('오늘의 뉴스는 매우 심각합니다')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['오늘', '의', '뉴스', '는', '매우']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer.decode_from_list([258, 4, 428, 3, 333])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}