lgb_word2vec.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gensim\n",
    "from gensim.models import Word2Vec\n",
    "import time\n",
    "import os\n",
    "import csv\n",
    "\n",
    "def data_to_sentence():\n",
    "    if not os.path.exists(\"lgb_sentences\"):\n",
    "        f = open(\"lgb_sentences\", \"w\")\n",
    "        f.close()\n",
    "\n",
    "    with open(\"lgb_sentences\", \"w\") as f:\n",
    "        with open(\"/home/kesci/temp/train_tail_50M.csv\") as csv_file1:                    # 训练集\n",
    "            csv_reader = csv.reader(csv_file1)\n",
    "            initial_query_id = 0\n",
    "\n",
    "            for row in csv_reader:\n",
    "                query_id = row[0]\n",
    "                query = row[1].replace(\"\\t\", \"\")\n",
    "                title = row[3].replace(\"\\t\", \"\")\n",
    "\n",
    "                if int(query_id) != initial_query_id:\n",
    "                    f.write(query + \"\\n\")\n",
    "                    f.write(title + \"\\n\")\n",
    "                    initial_query_id = int(query_id)\n",
    "\n",
    "                else:\n",
    "                    f.write(title + \"\\n\")\n",
    "        csv_file1.close()\n",
    "\n",
    "        with open(\"/home/kesci/input/bytedance/bytedance_contest.final_2.csv\") as csv_file2:     # 测试集\n",
    "            csv_reader = csv.reader(csv_file2)\n",
    "            initial_query_id = 0\n",
    "\n",
    "            for row in csv_reader:\n",
    "                query_id = row[0]\n",
    "                query = row[1].replace(\"\\t\", \"\")\n",
    "                title = row[3].replace(\"\\t\", \"\")\n",
    "\n",
    "                if int(query_id) != initial_query_id:\n",
    "                    f.write(query + \"\\n\")\n",
    "                    f.write(title + \"\\n\")\n",
    "                    initial_query_id = int(query_id)\n",
    "\n",
    "                else:\n",
    "                    f.write(title + \"\\n\")\n",
    "        csv_file2.close()\n",
    "    f.close()\n",
    "\n",
    "    print(\"Writing sentence Successful!\")\n",
    "\n",
    "\n",
    "data_to_sentence()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class EpochSaver(gensim.models.callbacks.CallbackAny2Vec):\n",
    "    '''用于保存模型, 打印损失函数等等'''\n",
    "\n",
    "    def __init__(self, save_name=\"lgb_word2vec.model\"):\n",
    "        self.save_path = save_name\n",
    "        self.epoch = 0\n",
    "        self.pre_loss = 0\n",
    "        self.best_loss = 999999999999.9\n",
    "        self.since = time.time()\n",
    "\n",
    "    def on_epoch_end(self, model):\n",
    "        self.epoch += 1\n",
    "        cum_loss = model.get_latest_training_loss()  # 返回的是从第一个epoch累计的\n",
    "        epoch_loss = cum_loss - self.pre_loss\n",
    "        time_taken = time.time() - self.since\n",
    "        print(\"Epoch %d, loss: %.2f, time: %dmin %ds\" % (self.epoch, epoch_loss, time_taken // 60, time_taken % 60))\n",
    "        if self.best_loss > epoch_loss:\n",
    "            self.best_loss = epoch_loss\n",
    "            print(\"Better model. Best loss: %.2f\" % self.best_loss)\n",
    "            model.save(self.save_path)\n",
    "            print(\"Model %s save done!\" % self.save_path)\n",
    "\n",
    "        self.pre_loss = cum_loss\n",
    "        self.since = time.time()\n",
    "\n",
    " \n",
    "model = Word2Vec(min_count=1, window=5, size=150, workers=4)\n",
    "model.build_vocab(corpus_file=\"lgb_sentences\")\n",
    "model.train(corpus_file=\"lgb_sentences\", total_words=model.corpus_total_words,\n",
    "            epochs=5, compute_loss=True, report_delay=60 * 10,\n",
    "            callbacks=[EpochSaver(\"lgb_word2vec.model\")])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}