feature_engineering.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "9065D50168744BD48020FE09FA382A74"
   },
   "outputs": [],
   "source": [
    "# 查看当前挂载的数据集目录\n",
    "!ls /home/kesci/input/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "99A49ACED0FE4059860E79077EEE1623"
   },
   "outputs": [],
   "source": [
    "# 查看个人持久化工作区文件\n",
    "!ls /home/kesci/work/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "22B6EACD2792437C896D5F2AA5F597B1"
   },
   "outputs": [],
   "source": [
    "# 查看当前kernerl下的package\n",
    "!pip list --format=columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "CCB839C6D38E4A228AA42D9857AD803D"
   },
   "outputs": [],
   "source": [
    "# 显示cell运行时长\n",
    "%load_ext klab-autotime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hide_input": false,
    "id": "98E6C8B6D2A54FAB8AF2665FA6744831"
   },
   "outputs": [],
   "source": [
    "# len特征计算\n",
    "# my_feat_calc_func 函数是并行计算特征的函数\n",
    "# 只需要将包含queryid、query、title列的dataframe传入函数my_feat_calc_func()，即可计算出该类特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hide_input": false,
    "id": "03606BA5E1424A13B4CD07443CB7ABFA"
   },
   "outputs": [],
   "source": [
    "# -*- coding: utf-8 -*- \n",
    "'''\n",
    "Created on Jul 2, 2019\n",
    "\n",
    "@author: Greatpan\n",
    "'''\n",
    "\n",
    "import pandas as pd\n",
    "pd.set_option('display.max_rows', 500)\n",
    "pd.set_option('display.max_columns', 500)\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "from tqdm import tqdm\n",
    "tqdm.pandas(desc='Progress')\n",
    "\n",
    "from multiprocessing import Pool\n",
    "\n",
    "def apply_fun(df):\n",
    "    basic_feat=pd.DataFrame()\n",
    "    \n",
    "    df[\"query\"]=df[\"query\"].apply(str.strip)\n",
    "    df[\"title\"]=df[\"title\"].apply(str.strip)\n",
    "    \n",
    "    basic_feat['querykw_num']=df['query'].apply(lambda x:len(x.split(sep=\" \")))\n",
    "    basic_feat['titlekw_num']=df['title'].apply(lambda x:len(x.split(sep=\" \")))\n",
    "    \n",
    "    return basic_feat\n",
    "\n",
    "def my_feat_calc_func(RawData):\n",
    "    '''  关键词长度相关特征  a'''\n",
    "    multiprocessing_nums=16\n",
    "    df_parts = np.array_split(RawData, multiprocessing_nums)\n",
    "    with Pool(processes=multiprocessing_nums) as pool:\n",
    "        result_parts = pool.map(apply_fun, df_parts)\n",
    "    pool.join()\n",
    "    \n",
    "    result_parallel = pd.concat(result_parts)\n",
    "    \n",
    "    return result_parallel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "444C3DEA18FA4CA089120A29150638BA"
   },
   "outputs": [],
   "source": [
    "# query_pos_{1-10}_feat\n",
    "# my_feat_calc_func 函数是并行计算特征的函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hide_input": true,
    "id": "C3DAE0C8E99E4216841CF841B67403AA"
   },
   "outputs": [],
   "source": [
    "# -*- coding: utf-8 -*- \n",
    "'''\n",
    "Created on Jul 4, 2019\n",
    "\n",
    "@author: Greatpan\n",
    "'''\n",
    "\n",
    "import pandas as pd\n",
    "pd.set_option('display.max_rows', 500)\n",
    "pd.set_option('display.max_columns', 500)\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "from tqdm import tqdm\n",
    "tqdm.pandas(desc='Progress')\n",
    "\n",
    "from multiprocessing import Pool\n",
    "\n",
    "def reduce_mem_usage(D,verbose=True):\n",
    "    start_mem = D.memory_usage().sum() / 1024**2\n",
    "    for c, d in zip(D.columns, D.dtypes):\n",
    "        if d.kind == 'f':\n",
    "            D[c] = pd.to_numeric(D[c], downcast='float')\n",
    "        elif d.kind == 'i':\n",
    "            D[c] = pd.to_numeric(D[c], downcast='signed')\n",
    "    end_mem = D.memory_usage().sum() / 1024**2\n",
    "    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n",
    "    return D\n",
    "\n",
    "def keyWordMatchfunc(doc1,doc2,pos_i):\n",
    "    doc1=doc1.split(sep=\" \")\n",
    "    doc2=doc2.split(sep=\" \")\n",
    "    \n",
    "    if pos_i<len(doc1):\n",
    "        title_len=min(len(doc2),25) # title只匹配前25个关键词\n",
    "        for pos_j in range(title_len):\n",
    "            if doc1[pos_i]==doc2[pos_j]:\n",
    "                q_pos=pos_j+1       # 如果匹配上了 记录匹配的位置\n",
    "                break\n",
    "            elif pos_j==title_len-1:\n",
    "                q_pos=0             # 如果没有匹配上 赋值为0\n",
    "    else:\n",
    "        q_pos=-1                    # 如果后续长度不存在 赋值为-1 \n",
    "    \n",
    "    return q_pos\n",
    "\n",
    "def apply_fun(df):\n",
    "    basic_feat=pd.DataFrame()\n",
    "    \n",
    "    df[\"query\"]=df[\"query\"].apply(str.strip)\n",
    "    df[\"title\"]=df[\"title\"].apply(str.strip)\n",
    "    \n",
    "    query_pos_feat=pd.DataFrame()\n",
    "    \n",
    "    for pos_i in range(10):\n",
    "        query_pos_feat['query_pos_'+str(pos_i+1)] = df.apply(\n",
    "                lambda row:keyWordMatchfunc(row[\"query\"],row.title,pos_i),axis=1).astype(np.int8)\n",
    "    \n",
    "    return query_pos_feat\n",
    "\n",
    "def my_feat_calc_func(RawData):\n",
    "    '''  关键词长度相关特征  a'''\n",
    "    multiprocessing_nums=16\n",
    "    df_parts = np.array_split(RawData, multiprocessing_nums)\n",
    "    with Pool(processes=multiprocessing_nums) as pool:\n",
    "        result_parts = pool.map(apply_fun, df_parts)\n",
    "    pool.join()\n",
    "    \n",
    "    result_parallel = pd.concat(result_parts)\n",
    "    \n",
    "    return result_parallel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "3710DACDE9114BB481564C600BF51533"
   },
   "outputs": [],
   "source": [
    "# title_pos_{1-25}_feat\n",
    "# my_feat_calc_func 函数是并行计算特征的函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hide_input": true,
    "id": "048572E550A941E38E15A98ABED70D65"
   },
   "outputs": [],
   "source": [
    "# -*- coding: utf-8 -*- \n",
    "'''\n",
    "Created on Jul 4, 2019\n",
    "\n",
    "@author: Greatpan\n",
    "'''\n",
    "\n",
    "import pandas as pd\n",
    "pd.set_option('display.max_rows', 500)\n",
    "pd.set_option('display.max_columns', 500)\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "from tqdm import tqdm\n",
    "tqdm.pandas(desc='Progress')\n",
    "\n",
    "from multiprocessing import Pool\n",
    "\n",
    "def reduce_mem_usage(D,verbose=True):\n",
    "    start_mem = D.memory_usage().sum() / 1024**2\n",
    "    for c, d in zip(D.columns, D.dtypes):\n",
    "        if d.kind == 'f':\n",
    "            D[c] = pd.to_numeric(D[c], downcast='float')\n",
    "        elif d.kind == 'i':\n",
    "            D[c] = pd.to_numeric(D[c], downcast='signed')\n",
    "    end_mem = D.memory_usage().sum() / 1024**2\n",
    "    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n",
    "    return D\n",
    "\n",
    "def keyWordMatchfunc(doc2,doc1,pos_i):\n",
    "    doc1=doc1.split(sep=\" \")\n",
    "    doc2=doc2.split(sep=\" \")\n",
    "    \n",
    "    if pos_i<len(doc1):\n",
    "        \n",
    "        title_len=len(doc2)         # title只匹配前25个关键词\n",
    "        for pos_j in range(title_len):\n",
    "            if doc1[pos_i]==doc2[pos_j]:\n",
    "                t_pos=pos_j+1       # 如果匹配上了 记录匹配的位置\n",
    "                break\n",
    "            elif pos_j==title_len-1:\n",
    "                t_pos=0             # 如果没有匹配上 赋值为0\n",
    "    else:\n",
    "        t_pos=-1                    # 如果后续长度不存在 赋值为-1 \n",
    "    \n",
    "    return t_pos\n",
    "\n",
    "def apply_fun(df):\n",
    "    basic_feat=pd.DataFrame()\n",
    "    \n",
    "    df[\"query\"]=df[\"query\"].apply(str.strip)\n",
    "    df[\"title\"]=df[\"title\"].apply(str.strip)\n",
    "    \n",
    "    title_pos_feat=pd.DataFrame()\n",
    "    \n",
    "    for pos_i in range(15):\n",
    "        title_pos_feat['title_pos_'+str(pos_i+1)] = df.apply(\n",
    "                lambda row:keyWordMatchfunc(row[\"query\"],row.title,pos_i),axis=1).astype(np.int)\n",
    "    \n",
    "    return title_pos_feat\n",
    "\n",
    "def my_feat_calc_func(RawData):\n",
    "    multiprocessing_nums=16\n",
    "    df_parts = np.array_split(RawData, multiprocessing_nums)\n",
    "    with Pool(processes=multiprocessing_nums) as pool:\n",
    "        result_parts = pool.map(apply_fun, df_parts)\n",
    "    pool.join()\n",
    "    \n",
    "    result_parallel = pd.concat(result_parts)\n",
    "    \n",
    "    return result_parallel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "90A0C58988F2481D9613F306B4BA2F87"
   },
   "outputs": [],
   "source": [
    "# sim_feat 细粒度的特征\n",
    "# my_feat_calc_func 函数是并行计算特征的函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hide_input": true,
    "id": "897E491E42114830857483698C17A5F0"
   },
   "outputs": [],
   "source": [
    "# -*- coding: utf-8 -*- \n",
    "'''\n",
    "Created on Jul 4, 2019\n",
    "\n",
    "@author: Greatpan\n",
    "'''\n",
    "\n",
    "import pandas as pd\n",
    "pd.set_option('display.max_rows', 500)\n",
    "pd.set_option('display.max_columns', 500)\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "from tqdm import tqdm\n",
    "tqdm.pandas(desc='Progress')\n",
    "\n",
    "from multiprocessing import Pool\n",
    "\n",
    "def reduce_mem_usage(D,verbose=True):\n",
    "    start_mem = D.memory_usage().sum() / 1024**2\n",
    "    for c, d in zip(D.columns, D.dtypes):\n",
    "        if d.kind == 'f':\n",
    "            D[c] = pd.to_numeric(D[c], downcast='float')\n",
    "        elif d.kind == 'i':\n",
    "            D[c] = pd.to_numeric(D[c], downcast='signed')\n",
    "    end_mem = D.memory_usage().sum() / 1024**2\n",
    "    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n",
    "    return D\n",
    "\n",
    "import distance\n",
    "def apply_fun(df):\n",
    "    basic_feat=pd.DataFrame()\n",
    "    \n",
    "    df[\"query\"]=df[\"query\"].apply(str.strip)\n",
    "    df[\"title\"]=df[\"title\"].apply(str.strip)\n",
    "    \n",
    "    sim_feat=pd.DataFrame()\n",
    "    # qt=[[10,10],[10,15],[15,15],[15,25]]\n",
    "    # qt=[[3,5],[3,10],[4,5],[4,10],[5,5],[5,10],[10,5],[10,10]]\n",
    "    qt=[[3,3],[3,5],[5,5],[5,10],[10,10],[10,15],[15,15],[15,25]]\n",
    "    \n",
    "    sim_func_dict={\"jaccard\":distance.jaccard,\n",
    "                    # \"sorensen\":distance.sorensen,\n",
    "                    \"levenshtein\":distance.levenshtein,\n",
    "                    # \"ratio\":Levenshtein.ratio\n",
    "    }\n",
    "    \n",
    "    for sim_func in tqdm(sim_func_dict):\n",
    "        sim_feat[sim_func] = df.apply(lambda row:sim_func_dict[sim_func](row[\"query\"].split(sep=\" \"),\n",
    "                                                row[\"title\"].split(sep=\" \")),\n",
    "                                                axis=1)\n",
    "        for qt_len in tqdm(qt):\n",
    "            if qt_len[0]==3 and sim_func==\"levenshtein\":\n",
    "                pass\n",
    "            else:\n",
    "                sim_feat[sim_func+'_q'+str(qt_len[0])+'_t'+str(qt_len[1])] = df.apply(\n",
    "                    lambda row:sim_func_dict[sim_func]( row[\"query\"].split(sep=\" \")[:qt_len[0]],\n",
    "                                                        row[\"title\"].split(sep=\" \")[:qt_len[1]]),\n",
    "                                                        axis=1)\n",
    "    \n",
    "    return sim_feat\n",
    "\n",
    "def my_feat_calc_func(RawData):\n",
    "    multiprocessing_nums=16\n",
    "    df_parts = np.array_split(RawData, multiprocessing_nums)\n",
    "    with Pool(processes=multiprocessing_nums) as pool:\n",
    "        result_parts = pool.map(apply_fun, df_parts)\n",
    "    pool.join()\n",
    "    \n",
    "    result_parallel = pd.concat(result_parts)\n",
    "    \n",
    "    return result_parallel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "CF1F2B38909C48049596E1A37C23184D"
   },
   "outputs": [],
   "source": [
    "# BM25特征\n",
    "# my_feat_calc_func 函数是并行计算特征的函数\n",
    "# 只需要将包含queryid、query、title列的dataframe传入函数my_feat_calc_func()，即可计算出该类特征\n",
    "# 将前一亿、后一亿、test1、test2传入dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hide_input": false,
    "id": "E6B8DB06A8C645CEB41A5D42DAC067A4"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from gensim.summarization.bm25 import BM25\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "# -*- coding: utf-8 -*- \n",
    "'''\n",
    "Created on Jul 4, 2019\n",
    "\n",
    "@author: Greatpan\n",
    "'''\n",
    "\n",
    "import pandas as pd\n",
    "pd.set_option('display.max_rows', 500)\n",
    "pd.set_option('display.max_columns', 500)\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "from tqdm import tqdm\n",
    "tqdm.pandas(desc='Progress')\n",
    "\n",
    "import time\n",
    "\n",
    "from multiprocessing import Pool\n",
    "\n",
    "def reduce_mem_usage(D,verbose=True):\n",
    "    start_mem = D.memory_usage().sum() / 1024**2\n",
    "    for c, d in zip(D.columns, D.dtypes):\n",
    "        if d.kind == 'f':\n",
    "            D[c] = pd.to_numeric(D[c], downcast='float')\n",
    "        elif d.kind == 'i':\n",
    "            D[c] = pd.to_numeric(D[c], downcast='signed')\n",
    "    end_mem = D.memory_usage().sum() / 1024**2\n",
    "    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n",
    "    return D\n",
    "\n",
    "def apply_fun(df):\n",
    "    query_id_group = df.groupby(['query_id'])\n",
    "    bm_list = []\n",
    "    for name, group in query_id_group:\n",
    "        corpus = group['title'].values.tolist()\n",
    "        corpus = [sentence.strip().split() for sentence in corpus]\n",
    "        query = group['query'].values[0].strip().split()\n",
    "        bm = BM25(corpus)\n",
    "        bmscore = bm.get_scores(query)\n",
    "        bm_list.extend(bmscore)\n",
    "    \n",
    "    return pd.DataFrame(bm_list,columns=[\"BM25\"])\n",
    "\n",
    "def my_feat_calc_func(RawData):\n",
    "    '''  关键词长度相关特征  a'''\n",
    "    multiprocessing_nums=13\n",
    "    df_parts = np.array_split(RawData, multiprocessing_nums)\n",
    "    with Pool(processes=multiprocessing_nums) as pool:\n",
    "        result_parts = pool.map(apply_fun, df_parts)\n",
    "    pool.join()\n",
    "    \n",
    "    result_parallel = pd.concat(result_parts)\n",
    "    \n",
    "    return result_parallel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "98BDCE5FA976478F80672920E118A773"
   },
   "outputs": [],
   "source": [
    "# 计算query_match_feat 特征\n",
    "# my_feat_calc_func 函数是并行计算特征的函数\n",
    "# 只需要将包含queryid、query、title列的dataframe传入函数my_feat_calc_func()，即可计算出该类特征\n",
    "# 将前一亿、后一亿、test1、test2传入dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hide_input": true,
    "id": "742DDA518A854632942F22B77C243740"
   },
   "outputs": [],
   "source": [
    "# -*- coding: utf-8 -*- \n",
    "'''\n",
    "Created on Jul 4, 2019\n",
    "\n",
    "@author: Greatpan\n",
    "'''\n",
    "\n",
    "import pandas as pd\n",
    "pd.set_option('display.max_rows', 500)\n",
    "pd.set_option('display.max_columns', 500)\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "from tqdm import tqdm\n",
    "tqdm.pandas(desc='Progress')\n",
    "\n",
    "import time\n",
    "\n",
    "from multiprocessing import Pool\n",
    "\n",
    "def reduce_mem_usage(D,verbose=True):\n",
    "    start_mem = D.memory_usage().sum() / 1024**2\n",
    "    for c, d in zip(D.columns, D.dtypes):\n",
    "        if d.kind == 'f':\n",
    "            D[c] = pd.to_numeric(D[c], downcast='float')\n",
    "        elif d.kind == 'i':\n",
    "            D[c] = pd.to_numeric(D[c], downcast='signed')\n",
    "    end_mem = D.memory_usage().sum() / 1024**2\n",
    "    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n",
    "    return D\n",
    "\n",
    "def wordblockMatchdel(doc1, doc2):\n",
    "    doc1 = doc1.split(sep=\" \")\n",
    "    doc2 = doc2.split(sep=\" \")\n",
    "    q1_start = len(doc1)\n",
    "    q1_end = 0\n",
    "    count = 0\n",
    "    blockcount = 0\n",
    "    i = 0\n",
    "    j = 0\n",
    "    maxMatchBlockLen = 0\n",
    "\n",
    "    while i < len(doc1):\n",
    "        for j in range(len(doc2)):\n",
    "            if doc1[i] == doc2[j]:\n",
    "\n",
    "                q1_start = min(q1_start, j)\n",
    "                count = count+1\n",
    "                blockcount = blockcount+1\n",
    "                maxMatchLen_tmp = 1\n",
    "                while i+1 < len(doc1) and j + \\\n",
    "                        1 < len(doc2) and doc1[i+1] == doc2[j+1]:\n",
    "                    maxMatchLen_tmp = maxMatchLen_tmp+1\n",
    "                    count = count+1\n",
    "                    i = i+1\n",
    "                    j = j+1\n",
    "                maxMatchBlockLen = max(maxMatchBlockLen, maxMatchLen_tmp)\n",
    "                q1_end = max(q1_end, j)\n",
    "                break\n",
    "        i = i+1\n",
    "    if q1_end-q1_start < 0:\n",
    "        q1_start = 0\n",
    "        q1_end = 0\n",
    "    q1_proximity = q1_end-q1_start\n",
    "    return [count, blockcount, q1_proximity,\n",
    "            maxMatchBlockLen, q1_start, q1_end]\n",
    "\n",
    "def apply_fun(df):\n",
    "    match_feat_tmp=pd.DataFrame()\n",
    "    \n",
    "    df[\"query\"]=df[\"query\"].apply(str.strip)\n",
    "    df[\"title\"]=df[\"title\"].apply(str.strip)\n",
    "    \n",
    "    match_feat_tmp[\"match_tmp\"] = df.apply(\n",
    "            lambda row: wordblockMatchdel(\n",
    "                row[\"query\"], row.title), axis=1)\n",
    "    \n",
    "    match_feat=pd.DataFrame()\n",
    "    match_feat[\"count_match\"] = match_feat_tmp.apply(\n",
    "            lambda row: row[\"match_tmp\"][0], axis=1)\n",
    "    \n",
    "    match_feat[\"blockcount_match\"] = match_feat_tmp.apply(\n",
    "            lambda row: row[\"match_tmp\"][1], axis=1)\n",
    "    \n",
    "    match_feat[\"proximity\"] = match_feat_tmp.apply(\n",
    "            lambda row: row[\"match_tmp\"][2], axis=1)\n",
    "    \n",
    "    match_feat[\"maxMatchBlockLen\"] = match_feat_tmp.apply(\n",
    "            lambda row: row[\"match_tmp\"][3], axis=1)\n",
    "    \n",
    "    match_feat[\"q1_match_start\"] = match_feat_tmp.apply(\n",
    "            lambda row: row[\"match_tmp\"][4], axis=1)\n",
    "    \n",
    "    match_feat[\"q1_match_end\"] = match_feat_tmp.apply(\n",
    "            lambda row: row[\"match_tmp\"][5], axis=1)\n",
    "    \n",
    "    return match_feat\n",
    "\n",
    "def my_feat_calc_func(RawData):\n",
    "    multiprocessing_nums=16\n",
    "    df_parts = np.array_split(RawData, multiprocessing_nums)\n",
    "    with Pool(processes=multiprocessing_nums) as pool:\n",
    "        result_parts = pool.map(apply_fun, df_parts)\n",
    "    pool.join()\n",
    "    \n",
    "    result_parallel = pd.concat(result_parts)\n",
    "    \n",
    "    return result_parallel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "C14DA88226D7496C8F3530A8E44B56E3"
   },
   "outputs": [],
   "source": [
    "# fuzz特征\n",
    "# my_feat_calc_func 函数是并行计算特征的函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hide_input": false,
    "id": "E6FF175E2F40421CA45A25CD6E2D5291"
   },
   "outputs": [],
   "source": [
    "# -*- coding: utf-8 -*- \n",
    "'''\n",
    "Created on Jul 4, 2019\n",
    "\n",
    "@author: Greatpan\n",
    "'''\n",
    "\n",
    "import pandas as pd\n",
    "pd.set_option('display.max_rows', 500)\n",
    "pd.set_option('display.max_columns', 500)\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "from tqdm import tqdm\n",
    "tqdm.pandas(desc='Progress')\n",
    "\n",
    "from multiprocessing import Pool\n",
    "\n",
    "def reduce_mem_usage(D,verbose=True):\n",
    "    start_mem = D.memory_usage().sum() / 1024**2\n",
    "    for c, d in zip(D.columns, D.dtypes):\n",
    "        if d.kind == 'f':\n",
    "            D[c] = pd.to_numeric(D[c], downcast='float')\n",
    "        elif d.kind == 'i':\n",
    "            D[c] = pd.to_numeric(D[c], downcast='signed')\n",
    "    end_mem = D.memory_usage().sum() / 1024**2\n",
    "    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n",
    "    return D\n",
    "\n",
    "from fuzzywuzzy import fuzz\n",
    "def WRatio(row):\n",
    "    return fuzz.WRatio(row['query'], row['title'])\n",
    "\n",
    "def QRatio(row):\n",
    "    return fuzz.QRatio(row['query'], row['title'])\n",
    "\n",
    "def partial_token_set_ratio(row):\n",
    "    return fuzz.partial_token_set_ratio(row['query'], row['title'])\n",
    "\n",
    "def partial_token_sort_ratio(row):\n",
    "    return fuzz.partial_token_sort_ratio(row['query'], row['title'])\n",
    "\n",
    "def partial_ratio(row):\n",
    "    return fuzz.partial_ratio(row['query'], row['title'])\n",
    "\n",
    "def token_set_ratio(row):\n",
    "    return fuzz.token_set_ratio(row['query'], row['title'])\n",
    "\n",
    "def token_sort_ratio(row):\n",
    "    return fuzz.token_sort_ratio(row['query'], row['title'])\n",
    "\n",
    "def apply_fun(df):\n",
    "    df[\"query\"]=df[\"query\"].apply(str.strip)\n",
    "    df[\"title\"]=df[\"title\"].apply(str.strip)\n",
    "    \n",
    "    extracted_feature = pd.DataFrame()\n",
    "    extracted_feature['token_sort_ratio'] = df.apply(token_sort_ratio, axis=1)\n",
    "    extracted_feature['token_set_ratio'] = df.apply(token_set_ratio, axis=1)\n",
    "    extracted_feature['partial_ratio'] = df.apply(partial_ratio, axis=1)\n",
    "    extracted_feature['partial_token_sort_ratio'] = df.apply(partial_token_sort_ratio, axis=1)\n",
    "    extracted_feature['partial_token_set_ratio'] = df.apply(partial_token_set_ratio, axis=1)\n",
    "    extracted_feature['QRatio'] = df.apply(QRatio, axis=1)\n",
    "    extracted_feature['WRatio'] = df.apply(WRatio, axis=1)\n",
    "    \n",
    "    return extracted_feature\n",
    "\n",
    "def my_feat_calc_func(RawData):\n",
    "    '''  关键词长度相关特征  a'''\n",
    "    multiprocessing_nums=16\n",
    "    df_parts = np.array_split(RawData, multiprocessing_nums)\n",
    "    with Pool(processes=multiprocessing_nums) as pool:\n",
    "        result_parts = pool.map(apply_fun, df_parts)\n",
    "    pool.join()\n",
    "    \n",
    "    result_parallel = pd.concat(result_parts)\n",
    "    \n",
    "    return result_parallel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "AB8E696AB2B6487D9C2D6D25C95314E8"
   },
   "outputs": [],
   "source": [
    "# textpair 特征\n",
    "# my_feat_calc_func 函数是并行计算特征的函数\n",
    "# 只需要将包含queryid、query、title列的dataframe传入函数my_feat_calc_func()，即可计算出该类特征\n",
    "# 将前一亿、后一亿、test1、test2传入dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hide_input": false,
    "id": "0192FFA60A2644BBA590CDC518938C27"
   },
   "outputs": [],
   "source": [
    "# -*- coding: utf-8 -*- \n",
    "'''\n",
    "Created on Jul 4, 2019\n",
    "\n",
    "@author: Greatpan\n",
    "'''\n",
    "\n",
    "import pandas as pd\n",
    "pd.set_option('display.max_rows', 500)\n",
    "pd.set_option('display.max_columns', 500)\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "from tqdm import tqdm\n",
    "tqdm.pandas(desc='Progress')\n",
    "\n",
    "from multiprocessing import Pool\n",
    "\n",
    "def reduce_mem_usage(D,verbose=True):\n",
    "    start_mem = D.memory_usage().sum() / 1024**2\n",
    "    for c, d in zip(D.columns, D.dtypes):\n",
    "        if d.kind == 'f':\n",
    "            D[c] = pd.to_numeric(D[c], downcast='float')\n",
    "        elif d.kind == 'i':\n",
    "            D[c] = pd.to_numeric(D[c], downcast='signed')\n",
    "    end_mem = D.memory_usage().sum() / 1024**2\n",
    "    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n",
    "    return D\n",
    "\n",
    "from fuzzywuzzy import fuzz\n",
    "def token_set_diff(row):\n",
    "    query = row['query']\n",
    "    title = row['title']\n",
    "    q_set = set(query.split())\n",
    "    t_set = set(title.split())\n",
    "    return abs(len(t_set) - len(q_set))\n",
    "\n",
    "# 词的不同个数之差（有正负）\n",
    "def wc_diff_unique(row):\n",
    "    query = row['query']\n",
    "    title = row['title']\n",
    "    q_set = set(query.split())\n",
    "    t_set = set(title.split())\n",
    "    return len(set(t_set)) - len(set(q_set))\n",
    "\n",
    "# 词的不同个数比值\n",
    "def wc_ratio_unique(row):\n",
    "    query = row['query']\n",
    "    title = row['title']\n",
    "    q_set = set(query.split())\n",
    "    t_set = set(title.split())\n",
    "    l1 = len(q_set) * 1.0\n",
    "    l2 = len(t_set) * 1.0\n",
    "    return l2 / l1\n",
    "\n",
    "# 总的不同的词\n",
    "def total_unique_words(row):\n",
    "    query = row['query']\n",
    "    query = query.split()\n",
    "    title = row['title']\n",
    "    t_set = set(title.split())\n",
    "    return len(t_set.union(query))\n",
    "\n",
    "def same_start(row):\n",
    "    query = row['query']\n",
    "    title = row['title']\n",
    "    l = min(len(query), len(title))\n",
    "    result = 0\n",
    "    for i in range(l):\n",
    "        if query[i] == title[i]:\n",
    "            result += len(title[i])\n",
    "    return result\n",
    "\n",
    "def apply_fun(df):\n",
    "    df[\"query\"]=df[\"query\"].apply(str.strip)\n",
    "    df[\"title\"]=df[\"title\"].apply(str.strip)\n",
    "    \n",
    "    extracted_feature = pd.DataFrame()\n",
    "    \n",
    "    extracted_feature['total_unique_words'] = df.apply(total_unique_words, axis=1)\n",
    "    extracted_feature['wc_ratio_unique'] = df.apply(wc_ratio_unique, axis=1)\n",
    "    extracted_feature['wc_diff_unique'] = df.apply(wc_diff_unique, axis=1)\n",
    "    extracted_feature['token_set_diff'] = df.apply(token_set_diff, axis=1)\n",
    "    extracted_feature['same_start'] = df.apply(same_start, axis=1)\n",
    "    \n",
    "    return extracted_feature\n",
    "\n",
    "def my_feat_calc_func(RawData):\n",
    "    '''  关键词长度相关特征  a'''\n",
    "    multiprocessing_nums=16\n",
    "    df_parts = np.array_split(RawData, multiprocessing_nums)\n",
    "    with Pool(processes=multiprocessing_nums) as pool:\n",
    "        result_parts = pool.map(apply_fun, df_parts)\n",
    "    pool.join()\n",
    "    \n",
    "    result_parallel = pd.concat(result_parts)\n",
    "    \n",
    "    return result_parallel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "85E3903B99444EA18271BF4E88F9600A"
   },
   "outputs": [],
   "source": [
    "# sen句向量相似度/距离 特征\n",
    "# my_feat_calc_func 函数是并行计算特征的函数\n",
    "# 只需要将包含queryid、query、title列的dataframe传入函数my_feat_calc_func()，即可计算出该类特征\n",
    "# 将前一亿、后一亿、test1、test2传入dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hide_input": true,
    "id": "17CD83C2FB3B41E48C70F4580F35AB93"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "pd.set_option('display.max_rows', 500)\n",
    "pd.set_option('display.max_columns', 500)\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "tqdm.pandas(desc='Progress')\n",
    "\n",
    "from multiprocessing import Pool\n",
    "\n",
    "\n",
    "def reduce_mem_usage(D, verbose=True):\n",
    "    start_mem = D.memory_usage().sum() / 1024 ** 2\n",
    "    for c, d in zip(D.columns, D.dtypes):\n",
    "        if d.kind == 'f':\n",
    "            D[c] = pd.to_numeric(D[c], downcast='float')\n",
    "        elif d.kind == 'i':\n",
    "            D[c] = pd.to_numeric(D[c], downcast='signed')\n",
    "    end_mem = D.memory_usage().sum() / 1024 ** 2\n",
    "    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (\n",
    "                start_mem - end_mem) / start_mem))\n",
    "    return D\n",
    "\n",
    "\n",
    "from gensim.corpora import Dictionary\n",
    "from gensim.models import TfidfModel, KeyedVectors\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "from scipy.spatial.distance import cosine, cityblock, jaccard, canberra, euclidean, minkowski, braycurtis\n",
    "\n",
    "tqdm.pandas()\n",
    "\n",
    "dictionary = Dictionary.load('/home/kesci/work/sunrui/tfidf_final/dictionary_final.dic')\n",
    "print(\"dic load finish!\")\n",
    "id2token = {id: token for token, id in dictionary.token2id.items()}\n",
    "print(\"id2token finish!\")\n",
    "tf_idf_model = TfidfModel.load('/home/kesci/work/sunrui/tfidf_final/tf_idf_final.model')\n",
    "print(\"tf_idf_model load finish\")\n",
    "w2v_model = KeyedVectors.load_word2vec_format('/home/kesci/work/word2vec/word2vec_100.bin', binary=True)\n",
    "print(\"w2v_model load finish\")\n",
    "\n",
    "fasttext_model = KeyedVectors.load_word2vec_format('./fasttext/fasttext_100.bin', binary=True)\n",
    "print(\"fasttext_model load finish\")\n",
    "\n",
    "\n",
    "def get_weight_sentence_vec(tf_idf_vec):\n",
    "    vec_list = []\n",
    "    for token_id, tf_idf in tf_idf_vec:\n",
    "        token = id2token[token_id]\n",
    "        w2v = w2v_model[token]\n",
    "        fasttext_vec = fasttext_model[token]\n",
    "        vec = np.concatenate([w2v, fasttext_vec])\n",
    "        weighted_vec = vec * tf_idf\n",
    "        vec_list.append(weighted_vec)\n",
    "    return np.sum(vec_list, axis=0)\n",
    "\n",
    "\n",
    "def get_tfidf_vec(text):\n",
    "    bow = dictionary.doc2bow(text)\n",
    "    tf_idf = tf_idf_model[bow]\n",
    "    return tf_idf\n",
    "\n",
    "from scipy.stats import skew, kurtosis\n",
    "\n",
    "def compute_sen_vec_dis(query, title):\n",
    "    tf_idf_q = get_tfidf_vec(query)\n",
    "    tf_idf_t = get_tfidf_vec(title)\n",
    "    weighted_q_vec = get_weight_sentence_vec(tf_idf_q)\n",
    "    weighted_t_vec = get_weight_sentence_vec(tf_idf_t)\n",
    "\n",
    "    sent_cosine = cosine(weighted_q_vec, weighted_t_vec)\n",
    "    sent_cityblock = cityblock(weighted_q_vec, weighted_t_vec)\n",
    "    sent_jaccard = jaccard(weighted_q_vec, weighted_t_vec)\n",
    "    sent_canberra = canberra(weighted_q_vec, weighted_t_vec)\n",
    "    sent_euclidean = euclidean(weighted_q_vec, weighted_t_vec)\n",
    "    sent_minkowski = minkowski(weighted_q_vec, weighted_t_vec)\n",
    "    sent_braycurtis = braycurtis(weighted_q_vec, weighted_t_vec)\n",
    "\n",
    "    skew_q = skew(weighted_q_vec)\n",
    "    skew_t = skew(weighted_t_vec)\n",
    "    kurtosis_q = kurtosis(weighted_q_vec)\n",
    "    kurtosis_t = kurtosis(weighted_t_vec)\n",
    "\n",
    "    return [sent_cosine,sent_cityblock,sent_jaccard,\n",
    "            sent_canberra,sent_euclidean,sent_minkowski,sent_braycurtis\n",
    "            skew_q,skew_t,kurtosis_q,kurtosis_t]\n",
    "\n",
    "\n",
    "def apply_fun(df):\n",
    "    df[\"query\"] = df[\"query\"].apply(str.strip)\n",
    "    df[\"title\"] = df[\"title\"].apply(str.strip)\n",
    "\n",
    "    sen_dis_feat_tmp = pd.DataFrame()\n",
    "    sen_dis_feat_tmp[\"sen_dis_tmp\"] = df.apply(\n",
    "        lambda row: compute_sen_vec_dis(\n",
    "            row['query'].split(),\n",
    "            row['title'].split()), axis=1)\n",
    "\n",
    "    sen_dis_feat = pd.DataFrame()\n",
    "    \n",
    "    sen_dis_feat_tmp[\"sen_dis_tmp\"] = df.apply(\n",
    "            lambda row: compute_sen_vec_dis(\n",
    "                row['query'].split(), \n",
    "                row['title'].split()), axis=1)\n",
    "    \n",
    "    sen_dis_feat=pd.DataFrame()\n",
    "    sen_dis_feat[\"sent_cosine\"] = sen_dis_feat_tmp.apply(\n",
    "            lambda row: row[\"sen_dis_tmp\"][0], axis=1)\n",
    "    sen_dis_feat[\"sent_cityblock\"] = sen_dis_feat_tmp.apply(\n",
    "            lambda row: row[\"sen_dis_tmp\"][1], axis=1)\n",
    "    sen_dis_feat[\"sent_jaccard\"] = sen_dis_feat_tmp.apply(\n",
    "            lambda row: row[\"sen_dis_tmp\"][2], axis=1)\n",
    "    sen_dis_feat[\"sent_canberra\"] = sen_dis_feat_tmp.apply(\n",
    "            lambda row: row[\"sen_dis_tmp\"][3], axis=1)\n",
    "    sen_dis_feat[\"sent_euclidean\"] = sen_dis_feat_tmp.apply(\n",
    "            lambda row: row[\"sen_dis_tmp\"][4], axis=1)\n",
    "    sen_dis_feat[\"sent_minkowski\"] = sen_dis_feat_tmp.apply(\n",
    "            lambda row: row[\"sen_dis_tmp\"][5], axis=1)\n",
    "    sen_dis_feat[\"sent_braycurtis\"] = sen_dis_feat_tmp.apply(\n",
    "            lambda row: row[\"sen_dis_tmp\"][6], axis=1)\n",
    "    sen_dis_feat[\"skew_q\"] = sen_dis_feat_tmp.apply(\n",
    "        lambda row: row[\"sen_dis_tmp\"][7], axis=1)\n",
    "    sen_dis_feat[\"skew_t\"] = sen_dis_feat_tmp.apply(\n",
    "        lambda row: row[\"sen_dis_tmp\"][8], axis=1)\n",
    "    sen_dis_feat[\"kurtosis_q\"] = sen_dis_feat_tmp.apply(\n",
    "        lambda row: row[\"sen_dis_tmp\"][9], axis=1)\n",
    "    sen_dis_feat[\"kurtosis_t\"] = sen_dis_feat_tmp.apply(\n",
    "        lambda row: row[\"sen_dis_tmp\"][10], axis=1)\n",
    "\n",
    "    return sen_dis_feat\n",
    "\n",
    "\n",
    "def my_feat_calc_func(RawData):\n",
    "    \n",
    "    multiprocessing_nums = 16\n",
    "    df_parts = np.array_split(RawData, multiprocessing_nums)\n",
    "    with Pool(processes=multiprocessing_nums) as pool:\n",
    "        result_parts = pool.map(apply_fun, df_parts)\n",
    "    pool.join()\n",
    "\n",
    "    result_parallel = pd.concat(result_parts)\n",
    "\n",
    "    return result_parallel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "A29441C022BD48778C8AA1664363326A"
   },
   "outputs": [],
   "source": [
    "# 以下是生成title count特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hide_input": true,
    "id": "D487FB78AA5647C58634AE3C03AABDB4"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import gc\n",
    "\n",
    "\n",
    "def reduce_mem_usage(df, verbose=True):\n",
    "    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n",
    "    start_mem = df.memory_usage().sum() / 1024 ** 2\n",
    "    for col in df.columns:\n",
    "        col_type = df[col].dtypes\n",
    "        if col_type in numerics:\n",
    "            c_min = df[col].min()\n",
    "            c_max = df[col].max()\n",
    "            if str(col_type)[:3] == 'int':\n",
    "                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n",
    "                    df[col] = df[col].astype(np.int8)\n",
    "                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n",
    "                    df[col] = df[col].astype(np.int16)\n",
    "                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n",
    "                    df[col] = df[col].astype(np.int32)\n",
    "                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n",
    "                    df[col] = df[col].astype(np.int64)\n",
    "            else:\n",
    "                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n",
    "                    df[col] = df[col].astype(np.float16)\n",
    "                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n",
    "                    df[col] = df[col].astype(np.float32)\n",
    "                else:\n",
    "                    df[col] = df[col].astype(np.float64)\n",
    "    end_mem = df.memory_usage().sum() / 1024 ** 2\n",
    "    if verbose:\n",
    "        print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (\n",
    "                start_mem - end_mem) / start_mem))\n",
    "    return df\n",
    "\n",
    "\n",
    "train_size = 1000000000\n",
    "test1_size = 20000000\n",
    "test_final_size = 100000000\n",
    "\n",
    "# todo 需要修改路径\n",
    "train_df = pd.read_hdf('/home/kesci/work/featureMap/train/title_code_train_feat.h5', key='data')\n",
    "test_1_df = pd.read_hdf('/home/kesci/work/featureMap/test1/title_code_test1_feat.h5', key='data')\n",
    "test_final_df = pd.read_hdf('/home/kesci/work/post_4kw_data/test2/title_code_test2_feat.h5', key='data')\n",
    "\n",
    "all_data = pd.concat([train_df, test_1_df, test_final_df], ignore_index=True)\n",
    "del train_df, test_1_df, test_final_df\n",
    "gc.collect()\n",
    "\n",
    "all_data = reduce_mem_usage(all_data)\n",
    "\n",
    "count_feature = pd.DataFrame()\n",
    "\n",
    "for feature in ['title_code']:\n",
    "    print('计算' + feature + '点击次数')\n",
    "    count_feature[feature + '_count'] = all_data.groupby(feature)[feature].transform('count')\n",
    "    count_feature = reduce_mem_usage(count_feature)\n",
    "\n",
    "train_count_feature = count_feature[:train_size]\n",
    "test1_count_feature = count_feature[train_size:train_size + test1_size]\n",
    "test_final_count_feature = count_feature[train_size + test1_size:]\n",
    "print(f'train shape = {train_count_feature.shape}')\n",
    "print(f'test1 shape = {test1_count_feature.shape}')\n",
    "print(f'test_final shape = {test_final_count_feature.shape}')\n",
    "\n",
    "train_count_feature.to_hdf('/home/kesci/work/featureMap/train/title_count_all.h5', index=None, key='data', complevel=9)\n",
    "test1_count_feature.to_hdf('/home/kesci/work/post_4kw_data/test1/test1_title_count_final.h5', index=None, key='data', complevel=9)\n",
    "test_final_count_feature.to_hdf('/home/kesci/work/post_4kw_data/test2/test2_title_count_final.h5', index=None, key='data', complevel=9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "C039B7178DC34E87BB68D059630796F0"
   },
   "outputs": [],
   "source": [
    "# 以下是生成 query count 特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hide_input": true,
    "id": "A1CE0E0354EB43A081DED54A2A96D21B"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import gc\n",
    "\n",
    "\n",
    "def reduce_mem_usage(df, verbose=True):\n",
    "    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n",
    "    start_mem = df.memory_usage().sum() / 1024 ** 2\n",
    "    for col in df.columns:\n",
    "        col_type = df[col].dtypes\n",
    "        if col_type in numerics:\n",
    "            c_min = df[col].min()\n",
    "            c_max = df[col].max()\n",
    "            if str(col_type)[:3] == 'int':\n",
    "                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n",
    "                    df[col] = df[col].astype(np.int8)\n",
    "                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n",
    "                    df[col] = df[col].astype(np.int16)\n",
    "                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n",
    "                    df[col] = df[col].astype(np.int32)\n",
    "                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n",
    "                    df[col] = df[col].astype(np.int64)\n",
    "            else:\n",
    "                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n",
    "                    df[col] = df[col].astype(np.float16)\n",
    "                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n",
    "                    df[col] = df[col].astype(np.float32)\n",
    "                else:\n",
    "                    df[col] = df[col].astype(np.float64)\n",
    "    end_mem = df.memory_usage().sum() / 1024 ** 2\n",
    "    if verbose:\n",
    "        print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (\n",
    "                start_mem - end_mem) / start_mem))\n",
    "    return df\n",
    "\n",
    "\n",
    "train_size = 1000000000\n",
    "test1_size = 20000000\n",
    "test_final_size = 100000000\n",
    "\n",
    "# todo 需要修改路径\n",
    "train_df = pd.read_hdf('/home/kesci/work/featureMap/train/query_code_train_feat.h5')\n",
    "test_1_df = pd.read_hdf('/home/kesci/work/featureMap/test1/query_code_test1_feat.h5')\n",
    "test_final_df = pd.read_hdf('/home/kesci/work/post_4kw_data/test2/query_code_test2_feat.h5')\n",
    "\n",
    "\n",
    "all_data = pd.concat([train_df, test_1_df, test_final_df], ignore_index=True)\n",
    "del train_df, test_1_df, test_final_df\n",
    "gc.collect()\n",
    "\n",
    "all_data = reduce_mem_usage(all_data)\n",
    "\n",
    "count_feature = pd.DataFrame()\n",
    "\n",
    "for feature in ['query_code']:\n",
    "    print('计算' + feature + '点击次数')\n",
    "    count_feature[feature + '_count'] = all_data.groupby(feature)[feature].transform('count')\n",
    "    count_feature = reduce_mem_usage(count_feature)\n",
    "\n",
    "train_count_feature = count_feature[:train_size]\n",
    "test1_count_feature = count_feature[train_size:train_size + test1_size]\n",
    "test_final_count_feature = count_feature[train_size + test1_size:]\n",
    "print(f'train shape = {train_count_feature.shape}')\n",
    "print(f'test1 shape = {test1_count_feature.shape}')\n",
    "print(f'test_final shape = {test_final_count_feature.shape}')\n",
    "\n",
    "test1_count_feature.to_hdf('/home/kesci/work/post_4kw_data/test1/test1_query_count_sr.h5', index=None, key='data', complevel=9)\n",
    "train_count_feature.to_hdf('/home/kesci/work/featureMap/train/query_count_all_sr.h5', index=None, key='data', complevel=9)\n",
    "test_final_count_feature.to_hdf('/home/kesci/work/post_4kw_data/test2/test2_query_count_sr.h5', index=None, key='data', complevel=9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "7432D3F959D340A8B0844FE11DEF444B"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "4B79117330714BE89413CA67327E04AB"
   },
   "outputs": [],
   "source": [
    "# 以下是生成unique特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hide_input": true,
    "id": "A6CA4ED069504AB488C5116C19470D52"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import gc\n",
    "\n",
    "\n",
    "def reduce_mem_usage(df, verbose=True):\n",
    "    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n",
    "    start_mem = df.memory_usage().sum() / 1024 ** 2\n",
    "    for col in df.columns:\n",
    "        col_type = df[col].dtypes\n",
    "        if col_type in numerics:\n",
    "            c_min = df[col].min()\n",
    "            c_max = df[col].max()\n",
    "            if str(col_type)[:3] == 'int':\n",
    "                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n",
    "                    df[col] = df[col].astype(np.int8)\n",
    "                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n",
    "                    df[col] = df[col].astype(np.int16)\n",
    "                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n",
    "                    df[col] = df[col].astype(np.int32)\n",
    "                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n",
    "                    df[col] = df[col].astype(np.int64)\n",
    "            else:\n",
    "                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n",
    "                    df[col] = df[col].astype(np.float16)\n",
    "                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n",
    "                    df[col] = df[col].astype(np.float32)\n",
    "                else:\n",
    "                    df[col] = df[col].astype(np.float64)\n",
    "    end_mem = df.memory_usage().sum() / 1024 ** 2\n",
    "    if verbose:\n",
    "        print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (\n",
    "                start_mem - end_mem) / start_mem))\n",
    "    return df\n",
    "\n",
    "\n",
    "train_size = 1000000000\n",
    "test1_size = 20000000\n",
    "test_final_size = 100000000\n",
    "\n",
    "# todo 需要修改路径\n",
    "train_title = pd.read_hdf('/home/kesci/work/featureMap/train/title_code_train_feat.h5', key='data')\n",
    "test_1_title = pd.read_hdf('/home/kesci/work/featureMap/test1/title_code_test1_feat.h5', key='data')\n",
    "test_final_title = pd.read_hdf('/home/kesci/work/post_4kw_data/test2/title_code_test2_feat.h5', key='data')\n",
    "\n",
    "train_query = pd.read_hdf('/home/kesci/work/featureMap/train/query_code_train_feat.h5')\n",
    "test_1_query = pd.read_hdf('/home/kesci/work/featureMap/test1/query_code_test1_feat.h5')\n",
    "test_final_query = pd.read_hdf('/home/kesci/work/post_4kw_data/test2/query_code_test2_feat.h5')\n",
    "\n",
    "train_df = pd.concat([train_query, train_title], axis=1)\n",
    "test_1_df = pd.concat([test_1_query, test_1_title], axis=1)\n",
    "test_final_df = pd.concat([test_final_query, test_final_title], axis=1)\n",
    "\n",
    "del train_query, train_title, test_1_query, test_1_title, test_final_query, test_final_title\n",
    "gc.collect()\n",
    "all_data = pd.concat([train_df, test_1_df, test_final_df], ignore_index=True)\n",
    "del train_df, test_1_df, test_final_df\n",
    "gc.collect()\n",
    "\n",
    "all_data = reduce_mem_usage(all_data)\n",
    "\n",
    "nunique_feature = pd.DataFrame()\n",
    "\n",
    "\n",
    "# 下面两步 可能需要注释后分两次运行 一次运行可能会炸内存\n",
    "nunique_feature['query_nunique_title'] = all_data.groupby('query_code').title_code.transform('nunique')\n",
    "nunique_feature = reduce_mem_usage(nunique_feature)\n",
    "nunique_feature['title_nunique_query'] = all_data.groupby('title_code').query_code.transform('nunique')\n",
    "nunique_feature = reduce_mem_usage(nunique_feature)\n",
    "\n",
    "train_nunique_feature = nunique_feature[:train_size]\n",
    "test1_nunique_feature = nunique_feature[train_size:train_size + test1_size]\n",
    "test_final_nunique_feature = nunique_feature[train_size + test1_size:]\n",
    "print(f'train shape = {train_nunique_feature.shape}')\n",
    "print(f'test1 shape = {test1_nunique_feature.shape}')\n",
    "print(f'test_final shape = {test_final_nunique_feature.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "A0FEEF5E1A2B4B1F9E799B0D204B2462"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "2C7AA50C84E34FF78C02627DD36B0E98"
   },
   "outputs": [],
   "source": [
    "# 以下是生成test1和train的title_code的转化率  注意一下需要改一下路径 就是title_code的"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hide_input": true,
    "id": "461DD17955C44A08853F2FF7E56C24CE"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "train_label = pd.read_csv('/home/kesci/input/bytedance/train_final.csv', usecols=[4], names=['label'])\n",
    "\n",
    "train_data = pd.read_csv('/home/kesci/work/featureMap/train/title_code_train_feat.csv.bz2')\n",
    "test_df = pd.read_csv('/home/kesci/work/featureMap/test1/title_code_test1_feat.csv.bz2')\n",
    "\n",
    "train_df = pd.concat([train_data,train_label],axis=1)\n",
    "del train_data,train_label\n",
    "import gc\n",
    "gc.collect()\n",
    "\n",
    "def downcast_data(D):\n",
    "    for c, d in zip(D.columns, D.dtypes):\n",
    "        if d.kind == 'f':\n",
    "            D[c] = pd.to_numeric(D[c], downcast='float')\n",
    "        elif d.kind == 'i':\n",
    "            D[c] = pd.to_numeric(D[c], downcast='signed')\n",
    "    return D\n",
    "train_df = downcast_data(train_df)\n",
    "test_df = downcast_data(test_df)\n",
    "\n",
    "import time\n",
    "\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import KFold, StratifiedKFold\n",
    "import numpy as np\n",
    "import scipy.special as special\n",
    "\n",
    "\n",
    "def log(log: str):\n",
    "    print(log)\n",
    "\n",
    "\n",
    "def time_log(time_elapsed):\n",
    "    print('complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))  # 打印出来时间\n",
    "\n",
    "\n",
    "def log_event(event: str):\n",
    "    log(event)\n",
    "\n",
    "\n",
    "class HyperParam(object):\n",
    "    def __init__(self, alpha, beta):\n",
    "        self.alpha = alpha\n",
    "        self.beta = beta\n",
    "\n",
    "    def sample_from_beta(self, alpha, beta, num, imp_upperbound):\n",
    "        sample = np.random.beta(alpha, beta, num)\n",
    "        I = []\n",
    "        C = []\n",
    "        for click_ratio in sample:\n",
    "            imp = np.random.random() * imp_upperbound\n",
    "            # imp = imp_upperbound\n",
    "            click = imp * click_ratio\n",
    "            I.append(imp)\n",
    "            C.append(click)\n",
    "        return I, C\n",
    "\n",
    "    def update_from_data_by_FPI(self, tries, success, iter_num, epsilon):\n",
    "        '''estimate alpha, beta using fixed point iteration'''\n",
    "        for i in range(iter_num):\n",
    "            new_alpha, new_beta = self.__fixed_point_iteration(tries, success, self.alpha, self.beta)\n",
    "            if abs(new_alpha - self.alpha) < epsilon and abs(new_beta - self.beta) < epsilon:\n",
    "                break\n",
    "            self.alpha = new_alpha\n",
    "            self.beta = new_beta\n",
    "\n",
    "    def __fixed_point_iteration(self, tries, success, alpha, beta):\n",
    "        '''fixed point iteration'''\n",
    "        sumfenzialpha = 0.0\n",
    "        sumfenzibeta = 0.0\n",
    "        sumfenmu = 0.0\n",
    "        for i in range(len(tries)):\n",
    "            sumfenzialpha += (special.digamma(success[i] + alpha) - special.digamma(alpha))\n",
    "            sumfenzibeta += (special.digamma(tries[i] - success[i] + beta) - special.digamma(beta))\n",
    "            sumfenmu += (special.digamma(tries[i] + alpha + beta) - special.digamma(alpha + beta))\n",
    "\n",
    "        return alpha * (sumfenzialpha / sumfenmu), beta * (sumfenzibeta / sumfenmu)\n",
    "\n",
    "    def update_from_data_by_moment(self, tries, success):  # tries尝试了多少次ctr  success 命中了多少次ctr\n",
    "        '''estimate alpha, beta using moment estimation'''\n",
    "        mean, var = self.__compute_moment(tries, success)\n",
    "        # print 'mean and variance: ', mean, var\n",
    "        # self.alpha = mean*(mean*(1-mean)/(var+0.000001)-1)\n",
    "        self.alpha = (mean + 0.000001) * ((mean + 0.000001) * (1.000001 - mean) / (var + 0.000001) - 1)\n",
    "        # self.beta = (1-mean)*(mean*(1-mean)/(var+0.000001)-1)\n",
    "        self.beta = (1.000001 - mean) * ((mean + 0.000001) * (1.000001 - mean) / (var + 0.000001) - 1)\n",
    "\n",
    "    def __compute_moment(self, tries, success):\n",
    "        '''\n",
    "        moment estimation\n",
    "        '''\n",
    "        ctr_list = []\n",
    "        var = 0.0\n",
    "        for i in range(len(tries)):\n",
    "            ctr_list.append(float(success[i]) / (tries[i] + 0.000000001))\n",
    "        mean = sum(ctr_list) / len(ctr_list)\n",
    "        for ctr in ctr_list:\n",
    "            var += pow(ctr - mean, 2)\n",
    "\n",
    "        return mean, var / (len(ctr_list) - 1)\n",
    "\n",
    "\n",
    "def merge_test(train_df, test_df, feature_name, is_fill_na):\n",
    "    temp = train_df.groupby(feature_name, as_index=False)['label'].agg(\n",
    "        {feature_name + '_label_count': 'sum', feature_name + '_all_count': 'count'})\n",
    "    HP = HyperParam(1, 1)\n",
    "    HP.update_from_data_by_moment(temp[feature_name + '_all_count'].values,\n",
    "                                  temp[feature_name + '_label_count'].values)  # 矩估计\n",
    "    temp[feature_name + '_convert'] = (temp[feature_name + '_label_count'] + HP.alpha) / (\n",
    "            temp[feature_name + '_all_count'] + HP.alpha + HP.beta)\n",
    "    temp = temp[[feature_name, feature_name + '_convert', feature_name + '_label_count']].drop_duplicates()\n",
    "    test_df = pd.merge(test_df, temp, on=[feature_name], how='left')\n",
    "    if is_fill_na:\n",
    "        test_df[feature_name + '_convert'].fillna(HP.alpha / (HP.alpha + HP.beta), inplace=True)\n",
    "    return test_df\n",
    "\n",
    "\n",
    "def merge_train(kfold_4_df, kfold_1_df, feature_name, is_fill_na):  # 用其他训练集的四折的数据去merge一折的\n",
    "    temp = kfold_4_df.groupby(feature_name, as_index=False)['label'].agg(\n",
    "        {feature_name + '_label_count': 'sum', feature_name + '_all_count': 'count'})\n",
    "    HP = HyperParam(1, 1)\n",
    "    HP.update_from_data_by_moment(temp[feature_name + '_all_count'].values,\n",
    "                                  temp[feature_name + '_label_count'].values)  # 矩估计\n",
    "    temp[feature_name + '_convert'] = (temp[feature_name + '_label_count'] + HP.alpha) / (\n",
    "            temp[feature_name + '_all_count'] + HP.alpha + HP.beta)\n",
    "    temp = temp[[feature_name, feature_name + '_convert', feature_name + '_label_count']].drop_duplicates()\n",
    "    kfold_1_df = pd.merge(kfold_1_df, temp, on=[feature_name], how='left')\n",
    "    if is_fill_na:\n",
    "        kfold_1_df[feature_name + '_convert'].fillna(HP.alpha / (HP.alpha + HP.beta), inplace=True)\n",
    "    return kfold_1_df\n",
    "    \n",
    "n_splits = 5\n",
    "is_shuffle = False\n",
    "feature_name = 'title_code'\n",
    "is_fill_nan = True\n",
    "\n",
    "log_event('计算 test ' + feature_name + ' 转换率')\n",
    "since = time.time()\n",
    "test_df = merge_test(train_df, test_df, feature_name, is_fill_nan)\n",
    "time_elapsed = time.time() - since\n",
    "time_log(time_elapsed)\n",
    "\n",
    "kf = KFold(n_splits=n_splits, shuffle=is_shuffle, random_state=19951024)\n",
    "\n",
    "if is_shuffle:\n",
    "    train_df['old_index'] = train_df.index\n",
    "\n",
    "temp_list = []\n",
    "for index, (fold_4, fold_1) in enumerate(kf.split(train_df)):\n",
    "    log_event(f'计算 train fold {(index + 1)} ' + feature_name + ' 转换率')\n",
    "    since = time.time()\n",
    "    temp_df = merge_train(train_df.iloc[fold_4], train_df.iloc[fold_1], feature_name, is_fill_nan)\n",
    "    temp_list.append(temp_df)\n",
    "    time_elapsed = time.time() - since\n",
    "    time_log(time_elapsed)\n",
    "\n",
    "result_df = pd.concat(temp_list, ignore_index=True)\n",
    "\n",
    "if is_shuffle:\n",
    "    result_df.sort_values(by='old_index', inplace=True)\n",
    "    result_df = result_df.reset_index(drop=True)\n",
    "    \n",
    "result_df[['title_code_convert','title_code_label_count']].to_hdf('./sunrui/title_convert_train.h5', index=None, key='data',complevel=9)\n",
    "test_df[['title_code_convert','title_code_label_count']].to_hdf('./sunrui/title_convert_test.h5', index=None, key='data',complevel=9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "D2B9AAB5F9484173992E42BE775F74D4"
   },
   "outputs": [],
   "source": [
    "# 一篇文章的质量score计算源码（10折生成前一亿）\n",
    "# calcScore(dataSet,testSet)\n",
    "# 读取queryid、title_code、lable三列特征的dataframe 作为dataSet参数\n",
    "# 读取 title_code 该列特征的dataframe 作为testSet参数\n",
    "# 这里需要计算四次：\n",
    "#    1、第二亿到第十亿的queryid、title_code、lable作为dataSet参数，第一亿的title_code作为testSet参数\n",
    "#    2、第一亿到第九亿的queryid、title_code、lable作为dataSet参数，第十亿的title_code作为testSet参数\n",
    "#    3、第一亿到第十亿的queryid、title_code、lable作为dataSet参数，test1的title_code作为testSet参数\n",
    "#    4、第二亿到第十亿的queryid、title_code、lable作为dataSet参数，test2的title_code作为testSet参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hide_input": true,
    "id": "2618B07D4C764AB18EA7FBD20DED8D84"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import gc\n",
    "\n",
    "def reduce_mem_usage(D,verbose=True):\n",
    "    start_mem = D.memory_usage().sum() / 1024**2\n",
    "    for c, d in zip(D.columns, D.dtypes):\n",
    "        if d.kind == 'f':\n",
    "            D[c] = pd.to_numeric(D[c], downcast='float')\n",
    "        elif d.kind == 'i':\n",
    "            D[c] = pd.to_numeric(D[c], downcast='signed')\n",
    "    end_mem = D.memory_usage().sum() / 1024**2\n",
    "    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n",
    "    return D\n",
    "\n",
    "def calcScore(dataSet,testSet):\n",
    "    dataSet['queryid_num']=dataSet.groupby(by='query_id').label.transform('count')\n",
    "    dataSet['queryid_click_num']=dataSet.groupby(by='query_id').label.transform('sum')\n",
    "    dataSet=dataSet.drop(columns=[\"query_id\"])\n",
    "    print(\"ok1\")\n",
    "    gc.collect()\n",
    "    dataSet = reduce_mem_usage(dataSet,verbose=True)\n",
    "    \n",
    "    dataSet['queryid_click_rate'] = dataSet['queryid_click_num']/dataSet['queryid_num']\n",
    "    print(\"ok2\")\n",
    "    dataSet=dataSet.drop(columns=[\"queryid_num\",\"queryid_click_num\"])\n",
    "    gc.collect()\n",
    "    dataSet = reduce_mem_usage(dataSet,verbose=True)\n",
    "    \n",
    "    dataSet['title_queryid_score']= dataSet['label']- dataSet['queryid_click_rate']\n",
    "    dataSet=dataSet.drop(columns=[\"label\"])\n",
    "    gc.collect()\n",
    "    print(\"ok3\")\n",
    "    dataSet = reduce_mem_usage(dataSet,verbose=True)\n",
    "    \n",
    "    feature='title_code'\n",
    "    dataSet[feature+'_score'] = dataSet.groupby(by=feature)['title_queryid_score'].transform('sum')\n",
    "    dataSet=reduce_mem_usage(dataSet,verbose=True)\n",
    "    dataSet=dataSet[[feature,feature+'_score']].drop_duplicates()\n",
    "    dataSet = reduce_mem_usage(dataSet,verbose=True)\n",
    "    gc.collect()\n",
    "    \n",
    "    testSet = pd.merge(testSet, dataSet, on=[feature], how='left')\n",
    "    \n",
    "    return testSet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "D8152DFA9C6543F3AB43B265EC2CC302"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "58B8FDCAE3F042269C17A8E5AC39C716"
   },
   "outputs": [],
   "source": [
    "# 一篇文章的点击数、出现数目 计算源码（10折生成前一亿）\n",
    "# calcScorecount(dataSet,testSet)\n",
    "# 读取 queryid、title_code、lable三列特征的dataframe 作为dataSet参数\n",
    "# 读取 title_code 该列特征的dataframe 作为testSet参数\n",
    "# 这里需要计算四次：\n",
    "#    1、第二亿到第十亿的queryid、title_code、lable作为dataSet参数，第一亿的title_code作为testSet参数\n",
    "#    2、第一亿到第九亿的queryid、title_code、lable作为dataSet参数，第十亿的title_code作为testSet参数\n",
    "#    3、第一亿到第十亿的queryid、title_code、lable作为dataSet参数，test1的title_code作为testSet参数\n",
    "#    4、第二亿到第十亿的queryid、title_code、lable作为dataSet参数，test2的title_code作为testSet参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hide_input": true,
    "id": "B6B42597A776412E86A8E027368FB063"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import gc\n",
    "\n",
    "def reduce_mem_usage(D,verbose=True):\n",
    "    start_mem = D.memory_usage().sum() / 1024**2\n",
    "    for c, d in zip(D.columns, D.dtypes):\n",
    "        if d.kind == 'f':\n",
    "            D[c] = pd.to_numeric(D[c], downcast='float')\n",
    "        elif d.kind == 'i':\n",
    "            D[c] = pd.to_numeric(D[c], downcast='signed')\n",
    "    end_mem = D.memory_usage().sum() / 1024**2\n",
    "    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n",
    "    return D\n",
    "\n",
    "def calcScorecount(dataSet,testSet):\n",
    "    dataSet[feature+'_count']=dataSet.groupby(by=feature).label.transform('count')\n",
    "    print(\"Ok1\")\n",
    "    dataSet[feature+'_click_num']=dataSet.groupby(by=feature).label.transform('sum')\n",
    "    print(\"ok2\")\n",
    "    dataSet=dataSet[[feature,feature+'_count',feature+'_click_num']].drop_duplicates()\n",
    "    dataSet = reduce_mem_usage(dataSet,verbose=True)\n",
    "    gc.collect()\n",
    "    \n",
    "    testSet = pd.merge(testSet, dataSet, on=[feature], how='left')\n",
    "    return testSet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "31A3EE2DA411400E821A07CFB2FD1CE8"
   },
   "outputs": [],
   "source": [
    "# 以下是生成SIF特征的词频表"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hide_input": false,
    "id": "049A9D6F436746F28585275D1C9FF835"
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import gc\n",
    "from tqdm import tqdm\n",
    "\n",
    "\n",
    "words_dict = {}\n",
    "\n",
    "\n",
    "text_data1 = pd.read_csv('/home/kesci/input/bytedance/train_final.csv', usecols=[1, 3],\n",
    "                         names=['query', 'title'], nrows=100000000)\n",
    "\n",
    "\n",
    "query_list = text_data1['query'].drop_duplicates().values.tolist()\n",
    "\n",
    "for item in tqdm(query_list):\n",
    "    query = item.split()\n",
    "    for word in query:\n",
    "        words_dict[word] = words_dict.get(word, 0) + 1\n",
    "\n",
    "del query_list\n",
    "gc.collect()\n",
    "\n",
    "title_list = text_data1['title'].drop_duplicates().values.tolist()\n",
    "\n",
    "for item in tqdm(title_list):\n",
    "    title = item.split()\n",
    "    for word in title:\n",
    "        words_dict[word] = words_dict.get(word, 0) + 1\n",
    "\n",
    "del title_list\n",
    "\n",
    "del text_data1\n",
    "gc.collect()\n",
    "\n",
    "text_data2 = pd.read_csv('/home/kesci/work/word2vec/post_10kw.csv')\n",
    "\n",
    "\n",
    "query_list = text_data2['query'].drop_duplicates().values.tolist()\n",
    "\n",
    "for item in tqdm(query_list):\n",
    "    query = item.split()\n",
    "    for word in query:\n",
    "        words_dict[word] = words_dict.get(word, 0) + 1\n",
    "\n",
    "del query_list\n",
    "gc.collect()\n",
    "\n",
    "title_list = text_data2['title'].drop_duplicates().values.tolist()\n",
    "\n",
    "for item in tqdm(title_list):\n",
    "    title = item.split()\n",
    "    for word in title:\n",
    "        words_dict[word] = words_dict.get(word, 0) + 1\n",
    "\n",
    "del title_list\n",
    "del text_data2\n",
    "gc.collect()\n",
    "\n",
    "test = pd.read_csv('/home/kesci/input/bytedance/test_final_part1.csv', usecols=[1, 3],\n",
    "                   names=['query', 'title'])\n",
    "\n",
    "\n",
    "query_list = test['query'].drop_duplicates().values.tolist()\n",
    "\n",
    "for item in tqdm(query_list):\n",
    "    query = item.split()\n",
    "    for word in query:\n",
    "        words_dict[word] = words_dict.get(word, 0) + 1\n",
    "\n",
    "del query_list\n",
    "gc.collect()\n",
    "\n",
    "title_list = test['title'].drop_duplicates().values.tolist()\n",
    "\n",
    "for item in tqdm(title_list):\n",
    "    title = item.split()\n",
    "    for word in title:\n",
    "        words_dict[word] = words_dict.get(word, 0) + 1\n",
    "\n",
    "del title_list\n",
    "del test\n",
    "gc.collect()\n",
    "\n",
    "test2 = pd.read_csv('/home/kesci/input/bytedance/bytedance_contest.final_2.csv', usecols=[1, 3],\n",
    "                    names=['query', 'title'])\n",
    "\n",
    "\n",
    "query_list = test2['query'].drop_duplicates().values.tolist()\n",
    "\n",
    "for item in tqdm(query_list):\n",
    "    query = item.split()\n",
    "    for word in query:\n",
    "        words_dict[word] = words_dict.get(word, 0) + 1\n",
    "\n",
    "del query_list\n",
    "gc.collect()\n",
    "\n",
    "title_list = test2['title'].drop_duplicates().values.tolist()\n",
    "\n",
    "for item in tqdm(title_list):\n",
    "    title = item.split()\n",
    "    for word in title:\n",
    "        words_dict[word] = words_dict.get(word, 0) + 1\n",
    "\n",
    "del title_list\n",
    "del test2\n",
    "gc.collect()\n",
    "\n",
    "import operator\n",
    "\n",
    "sorted_words_dict = sorted(words_dict.items(), key=operator.itemgetter(1), reverse=True)\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "with open('/home/kesci/work/sif_embbeding/word_freq_final.txt', 'w') as f: # trianpre10kw post10kw test1 test2\n",
    "    for word, freq in tqdm(sorted_words_dict):\n",
    "        f.write(word + ' ' + str(freq) + '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hide_input": false,
    "id": "99A64717D4754B77BCBC851EE03814F6"
   },
   "outputs": [],
   "source": [
    "# 以下生成SIF句向量的cos_dis l2_dis特征 \n",
    "# sif参考ppt中的内容"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "hide_input": false,
    "id": "C0FB250055434E4189A7A44C3F90E41C"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from gensim.models import KeyedVectors\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "\n",
    "\n",
    "def reduce_mem_usage(df, verbose=True):\n",
    "    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n",
    "    start_mem = df.memory_usage().sum() / 1024 ** 2\n",
    "    for col in df.columns:\n",
    "        col_type = df[col].dtypes\n",
    "        if col_type in numerics:\n",
    "            c_min = df[col].min()\n",
    "            c_max = df[col].max()\n",
    "            if str(col_type)[:3] == 'int':\n",
    "                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n",
    "                    df[col] = df[col].astype(np.int8)\n",
    "                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n",
    "                    df[col] = df[col].astype(np.int16)\n",
    "                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n",
    "                    df[col] = df[col].astype(np.int32)\n",
    "                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n",
    "                    df[col] = df[col].astype(np.int64)\n",
    "            else:\n",
    "                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n",
    "                    df[col] = df[col].astype(np.float16)\n",
    "                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n",
    "                    df[col] = df[col].astype(np.float32)\n",
    "                else:\n",
    "                    df[col] = df[col].astype(np.float64)\n",
    "    end_mem = df.memory_usage().sum() / 1024 ** 2\n",
    "    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (\n",
    "            start_mem - end_mem) / start_mem))\n",
    "    return df\n",
    "\n",
    "\n",
    "class Params(object):\n",
    "    def __init__(self):\n",
    "        self.LW = 1e-5\n",
    "        self.LC = 1e-5\n",
    "        self.eta = 0.05\n",
    "\n",
    "    def __str__(self):\n",
    "        t = \"LW\", self.LW, \", LC\", self.LC, \", eta\", self.eta\n",
    "        t = list(map(str, t))\n",
    "        return ' '.join(t)\n",
    "\n",
    "\n",
    "def getWordmap(w2v_file_name):\n",
    "    w2v_model = KeyedVectors.load_word2vec_format(w2v_file_name, binary=True)\n",
    "    word2ids = w2v_model.wv.vocab\n",
    "    words = {}\n",
    "    for word, vocab in word2ids.items():\n",
    "        words[word] = vocab.index\n",
    "    return (words, w2v_model.vectors)\n",
    "\n",
    "\n",
    "def getWordWeight(weightfile, a=1e-3):\n",
    "    if a <= 0:  # when the parameter makes no sense, use unweighted\n",
    "        a = 1.0\n",
    "\n",
    "    word2weight = {}\n",
    "    with open(weightfile) as f:\n",
    "        lines = f.readlines()\n",
    "    N = 0\n",
    "    for i in lines:\n",
    "        i = i.strip()\n",
    "        if (len(i) > 0):\n",
    "            i = i.split()\n",
    "            if (len(i) == 2):\n",
    "                word2weight[i[0]] = float(i[1])\n",
    "                N += float(i[1])\n",
    "            else:\n",
    "                print(i)\n",
    "    for key, value in word2weight.items():\n",
    "        word2weight[key] = a / (a + value / N)\n",
    "    return word2weight\n",
    "\n",
    "\n",
    "def getWeight(words, word2weight):\n",
    "    weight4ind = {}\n",
    "    for word, ind in words.items():\n",
    "        if word in word2weight:\n",
    "            weight4ind[ind] = word2weight[word]\n",
    "        else:\n",
    "            weight4ind[ind] = 1.0\n",
    "    return weight4ind\n",
    "\n",
    "\n",
    "def prepare_data(list_of_seqs):\n",
    "    lengths = [len(s) for s in list_of_seqs]\n",
    "    n_samples = len(list_of_seqs)\n",
    "    maxlen = np.max(lengths)\n",
    "    x = np.zeros((n_samples, maxlen)).astype('int32')\n",
    "    x_mask = np.zeros((n_samples, maxlen)).astype('float32')\n",
    "    for idx, s in enumerate(list_of_seqs):\n",
    "        x[idx, :lengths[idx]] = s\n",
    "        x_mask[idx, :lengths[idx]] = 1.\n",
    "    x_mask = np.asarray(x_mask, dtype='float32')\n",
    "    return x, x_mask\n",
    "\n",
    "\n",
    "def lookupIDX(words, w):\n",
    "    return words.get(w, len(words) - 1)\n",
    "\n",
    "\n",
    "def getSeq(p1, words):\n",
    "    p1 = p1.split()\n",
    "    X1 = []\n",
    "    for i in p1:\n",
    "        X1.append(lookupIDX(words, i))\n",
    "    return X1\n",
    "\n",
    "\n",
    "def sentences2idx(sentences, words):\n",
    "    \"\"\"\n",
    "    Given a list of sentences, output array of word indices that can be fed into the algorithms.\n",
    "    :param sentences: a list of sentences\n",
    "    :param words: a dictionary, words['str'] is the indices of the word 'str'\n",
    "    :return: x1, m1. x1[i, :] is the word indices in sentence i, m1[i,:] is the mask for sentence i (0 means no word at the location)\n",
    "    \"\"\"\n",
    "    seq1 = []\n",
    "    for i in sentences:\n",
    "        seq1.append(getSeq(i, words))\n",
    "    x1, m1 = prepare_data(seq1)\n",
    "    return x1, m1\n",
    "\n",
    "\n",
    "def seq2weight(seq, mask, weight4ind):\n",
    "    weight4ind_np = np.zeros((len(weight4ind),))\n",
    "    for index, weight in weight4ind.items():\n",
    "        weight4ind_np[index] = weight\n",
    "\n",
    "    weight = np.zeros(seq.shape).astype('float32')\n",
    "    for index, row in enumerate(seq):\n",
    "        weight[index] = weight4ind_np[row]\n",
    "    weight = weight * mask\n",
    "    return weight\n",
    "\n",
    "\n",
    "def get_weighted_average(We, x, w):\n",
    "    \"\"\"\n",
    "    Compute the weighted average vectors\n",
    "    :param We: We[i,:] is the vector for word i\n",
    "    :param x: x[i, :] are the indices of the words in sentence i\n",
    "    :param w: w[i, :] are the weights for the words in sentence i\n",
    "    :return: emb[i, :] are the weighted average vector for sentence i\n",
    "    \"\"\"\n",
    "    n_samples = x.shape[0]\n",
    "    emb = np.zeros((n_samples, We.shape[1]))\n",
    "    for i in range(n_samples):\n",
    "        emb[i, :] = w[i, :].dot(We[x[i, :], :]) / np.count_nonzero(w[i, :])\n",
    "    return emb\n",
    "\n",
    "\n",
    "def compute_pc(X, npc=1):\n",
    "    \"\"\"\n",
    "    Compute the principal components. DO NOT MAKE THE DATA ZERO MEAN!\n",
    "    :param X: X[i,:] is a data point\n",
    "    :param npc: number of principal components to remove\n",
    "    :return: component_[i,:] is the i-th pc\n",
    "    \"\"\"\n",
    "    svd = TruncatedSVD(n_components=npc, n_iter=7, random_state=0)\n",
    "    svd.fit(X)\n",
    "    return svd.components_\n",
    "\n",
    "\n",
    "def remove_pc(X, npc=1):\n",
    "    \"\"\"\n",
    "    Remove the projection on the principal components\n",
    "    :param X: X[i,:] is a data point\n",
    "    :param npc: number of principal components to remove\n",
    "    :return: XX[i, :] is the data point after removing its projection\n",
    "    \"\"\"\n",
    "    pc = compute_pc(X, npc)\n",
    "    if npc == 1:\n",
    "        XX = X - X.dot(pc.transpose()) * pc\n",
    "    else:\n",
    "        XX = X - X.dot(pc.transpose()).dot(pc)\n",
    "    return XX\n",
    "\n",
    "\n",
    "def SIF_embedding(We, x, w, params):\n",
    "    \"\"\"\n",
    "    Compute the scores between pairs of sentences using weighted average + removing the projection on the first principal component\n",
    "    :param We: We[i,:] is the vector for word i\n",
    "    :param x: x[i, :] are the indices of the words in the i-th sentence\n",
    "    :param w: w[i, :] are the weights for the words in the i-th sentence\n",
    "    :param params.rmpc: if >0, remove the projections of the sentence embeddings to their first principal component\n",
    "    :return: emb, emb[i, :] is the embedding for sentence i\n",
    "    \"\"\"\n",
    "    emb = get_weighted_average(We, x, w)\n",
    "    if params.rmpc > 0:\n",
    "        emb = remove_pc(emb, params.rmpc)\n",
    "    return emb\n",
    "\n",
    "\n",
    "wordfile = '/home/kesci/work/word2vec/word2vec_100.bin'  # word vector file, can be downloaded from GloVe website\n",
    "weightfile = '/home/kesci/work/sif_embbeding/word_freq_final.txt'  # each line is a word and its frequency\n",
    "weightpara = 1e-3  # the parameter in the SIF weighting scheme, usually in the range [3e-5, 3e-3]\n",
    "rmpc = 1  # number of principal components to remove in SIF weighting scheme\n",
    "\n",
    "# load word vectors\n",
    "(words, We) = getWordmap(wordfile)\n",
    "print('getWordmap')\n",
    "# load word weights\n",
    "word2weight = getWordWeight(weightfile, weightpara)  # word2weight['str'] is the weight for the word 'str'\n",
    "weight4ind = getWeight(words, word2weight)  # weight4ind[i] is the weight for the i-th word\n",
    "print('weight4ind')\n",
    "\n",
    "import pandas as pd\n",
    "import gc\n",
    "\n",
    "chunk_size = 1000000\n",
    "\n",
    "data_train = pd.read_csv('/home/kesci/input/bytedance/bytedance_contest.final_2.csv', usecols=[1, 3],\n",
    "                    names=['query', 'title'], chunksize=chunk_size)\n",
    "\n",
    "from tqdm import tqdm\n",
    "\n",
    "params = Params()\n",
    "params.rmpc = rmpc\n",
    "\n",
    "import time\n",
    "\n",
    "for index, current_data in enumerate(data_train):\n",
    "    print(index)\n",
    "    train_cos_sim = []\n",
    "    train_l2_sim = []\n",
    "\n",
    "    sentences = current_data['query'].values.tolist() + current_data['title'].values.tolist()  # 前半部分是query 后半部分是title\n",
    "\n",
    "    since = time.time()\n",
    "\n",
    "    x, m = sentences2idx(sentences,\n",
    "                         words)  # x is the array of word indices, m is the binary mask indicating whether there is a word in that location\n",
    "    time_elapsed = time.time() - since\n",
    "    print('complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))\n",
    "\n",
    "    since = time.time()\n",
    "\n",
    "    w = seq2weight(x, m, weight4ind)  # get word weights\n",
    "    # set parameters\n",
    "\n",
    "    time_elapsed = time.time() - since\n",
    "    print('complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))  # 打印出来时间\n",
    "    # get SIF embedding\n",
    "\n",
    "    since = time.time()\n",
    "\n",
    "    embedding = SIF_embedding(We, x, w, params)\n",
    "\n",
    "    time_elapsed = time.time() - since\n",
    "    print('complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))  # 打印出来时间\n",
    "    all_len = embedding.shape[0]\n",
    "    query_emb = embedding[:all_len // 2]\n",
    "    title_emb = embedding[all_len // 2:]\n",
    "\n",
    "    for i in tqdm(range(current_data.shape[0])):\n",
    "        v1 = query_emb[i]\n",
    "        v2 = title_emb[i]\n",
    "        cosine_distance = np.dot(v1, v2) / (np.linalg.norm(v1) * (np.linalg.norm(v2)))\n",
    "        l2_distance = np.linalg.norm(v1 - v2)\n",
    "        train_cos_sim.append(cosine_distance)\n",
    "        train_l2_sim.append(l2_distance)\n",
    "\n",
    "    temp_l2 = pd.DataFrame()\n",
    "    temp_l2['sif_l2'] = train_l2_sim\n",
    "    temp_l2 = reduce_mem_usage(temp_l2)\n",
    "    temp_l2.to_hdf(f'/home/kesci/work/sif_embbeding/sif_100_test2/l2/sif_l2_test2_part{index}.h5', index=None,\n",
    "                   key='data', complevel=9)\n",
    "\n",
    "    temp_cos = pd.DataFrame()\n",
    "    temp_cos['sif_cos'] = train_cos_sim\n",
    "    temp_cos = reduce_mem_usage(temp_cos)\n",
    "    temp_cos.to_hdf(f'/home/kesci/work/sif_embbeding/sif_100_test2/cos/sif_cos_test2_part{index}.h5', index=None,\n",
    "                    key='data', complevel=9)\n",
    "\n",
    "    del sentences, current_data, embedding, query_emb, title_emb, x, m, w, temp_l2, temp_cos\n",
    "    gc.collect()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}