PD.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import random as rd\n",
    "import collections\n",
    "import time\n",
    "import random\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.utils.data import Dataset\n",
    "\n",
    "from used_metric import get_performance\n",
    "\n",
    "r_path = './douban/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda')"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Data(Dataset):\n",
    "    def __init__(self, path=r_path, batch_size = 2048, step = 100):\n",
    "        self.path = path\n",
    "        self.batch_size = batch_size\n",
    "        self.step = step\n",
    "        self.n_users, self.n_items = 0, 0\n",
    "        self.train_user_list = collections.defaultdict(list)\n",
    "        self.train_user_list_time = collections.defaultdict(list)\n",
    "        self.train_item_list = collections.defaultdict(list)\n",
    "        \n",
    "        self.test_user_list = collections.defaultdict(list)\n",
    "        self.test_item_list = collections.defaultdict(list)\n",
    "        \n",
    "        self.load_train_data()\n",
    "        self.load_test_data()\n",
    "        \n",
    "        self.n_users = self.n_users + 1\n",
    "        self.n_items = self.n_items + 1\n",
    "        self.users = list(range(self.n_users))\n",
    "        self.items = list(range(self.n_items))\n",
    "        \n",
    "        self.length = self.batch_size * self.step\n",
    "        \n",
    "    def add_expo_popularity(self,popularity):\n",
    "        self.expo_popularity = popularity\n",
    "        \n",
    "    def load_train_data(self):          \n",
    "        train_file = self.path + 'train_with_time.txt'        \n",
    "        train_data = pd.read_csv(train_file, header=None, sep=' ')\n",
    "        train_data.columns = ['uid','iid','time','stars']\n",
    "        train_data = train_data[['uid','iid','time']]\n",
    "        unique_time = train_data['time'].unique()\n",
    "        print(\"time slot unique in train:\",unique_time)\n",
    "        self.unique_times = list(unique_time)\n",
    "        if train_data['time'].unique().shape[0] < 2:\n",
    "            raise RuntimeWarning(\"there only one time slot for train...., this may cause our method not work\")\n",
    "\n",
    "        for col in train_data.columns:\n",
    "            train_data[col] = train_data[col].astype(int)\n",
    "        user_item_time = train_data.groupby('uid')[['iid','time']].agg(list)\n",
    "        self.train_user_list = dict(zip(user_item_time.index,user_item_time['iid']))\n",
    "        self.train_user_list_time = dict(zip(user_item_time.index,user_item_time['time']))\n",
    "        item_user = train_data.groupby('iid')[['uid','time']].agg(list)\n",
    "        print(item_user.head(2))\n",
    "        self.train_item_list = dict(zip(item_user.index,item_user['uid']))\n",
    "\n",
    "        self.n_users = max(self.n_users,train_data['uid'].max())\n",
    "        self.n_items = max(self.n_items,train_data['iid'].max())\n",
    "        \n",
    "    def load_test_data(self):\n",
    "        test_file = self.path + 'test.txt'\n",
    "        with open(test_file) as f:\n",
    "            for line in f.readlines():\n",
    "                line = line.strip('\\n').split(' ')\n",
    "                if len(line) == 0:\n",
    "                    continue\n",
    "                line = [int(i) for i in line]\n",
    "                user = line[0]\n",
    "                items = line[1:]\n",
    "                if (len(items)==0):\n",
    "                    continue\n",
    "                self.test_user_list[user] = items\n",
    "                for item in items:\n",
    "                    self.test_item_list[item].append(user)\n",
    "                self.n_users = max(self.n_users, user)\n",
    "                self.n_items = max(self.n_items, max(items))\n",
    "\n",
    "    def __len__(self):\n",
    "        return self.length\n",
    "    \n",
    "    def __getitem__(self, idx):\n",
    "        user = random.choice(self.users) \n",
    "\n",
    "        neg_item = None\n",
    "        \n",
    "        pos_items = self.train_user_list[user]\n",
    "        clicked_times = self.train_user_list_time[user]\n",
    "        \n",
    "        random_index = rd.randint(0,len(pos_items) - 1)\n",
    "        pos_item = pos_items[random_index]\n",
    "        pos_time = clicked_times[random_index]\n",
    "    \n",
    "        while neg_item is None or neg_item in pos_items:\n",
    "            neg_item = rd.choice(self.items)\n",
    "      \n",
    "        pos_pop = self.expo_popularity[pos_item,pos_time]\n",
    "        neg_pop = self.expo_popularity[neg_item,pos_time]   \n",
    "    \n",
    "        return user, pos_item, neg_item, pos_pop, neg_pop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class evaluation():\n",
    "    def __init__(self, data, Ks, batch_size):\n",
    "        self.data = data\n",
    "        self.Ks = Ks\n",
    "        self.batch_size = batch_size\n",
    "\n",
    "    def test_one_batch(self, model, batch_user):\n",
    "        batch_user = batch_user.to(device)\n",
    "        batch_rec = model.do_recommendation(batch_user)\n",
    "        batch_rec = torch.sigmoid(batch_rec)\n",
    "        mask = torch.ones_like(batch_rec)\n",
    "        for i in range(len(batch_user)):\n",
    "            mask[i].scatter_(dim = 0, index=torch.tensor(list(self.data.train_user_list[batch_user[i].item()])).to(device), value=torch.tensor(0.0).to(device))\n",
    "        batch_rec = torch.mul(mask, batch_rec)\n",
    "        _, batch_rec = torch.sort(batch_rec, descending=True)\n",
    "        batch_rec = batch_rec.cpu().numpy()\n",
    "        result = {'precision': np.zeros(len(self.Ks)), 'recall': np.zeros(len(self.Ks)), 'ndcg': np.zeros(len(self.Ks)),\n",
    "                  'hit_ratio': np.zeros(len(self.Ks))}\n",
    "        for i in range(len(batch_user)):\n",
    "            u = batch_user[i].item()\n",
    "            r = batch_rec[i]\n",
    "            u_target = self.data.test_user_list[u]\n",
    "            one_user_result = get_performance(u_target, r, self.Ks)\n",
    "            result['precision'] += one_user_result['precision']\n",
    "            result['recall'] += one_user_result['recall']\n",
    "            result['ndcg'] += one_user_result['ndcg']\n",
    "            result['hit_ratio'] += one_user_result['hit_ratio']\n",
    "        return result\n",
    "\n",
    "    def eval(self, model):\n",
    "        result = {'precision': np.zeros(len(self.Ks)), 'recall': np.zeros(len(self.Ks)), 'ndcg': np.zeros(len(self.Ks)),\n",
    "                  'hit_ratio': np.zeros(len(self.Ks))}\n",
    "        all_users = list(self.data.test_user_list.keys())\n",
    "        tot_users = len(all_users)\n",
    "        for i in range(0, tot_users, self.batch_size):\n",
    "            end_idx = min(i + self.batch_size, tot_users)\n",
    "            batch_user = torch.tensor(all_users[i:end_idx], dtype=torch.long)\n",
    "            res = self.test_one_batch(model, batch_user)\n",
    "            \n",
    "            result['precision'] += res['precision']/tot_users\n",
    "            result['recall'] += res['recall']/tot_users\n",
    "            result['ndcg'] += res['ndcg']/tot_users\n",
    "            result['hit_ratio'] += res['hit_ratio']/tot_users\n",
    "        return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_popularity():\n",
    "    pop_save_path = r_path + \"item_pop_seq_ori2.txt\"\n",
    "    print(\"popularity used:\",pop_save_path)\n",
    "    with open(pop_save_path) as f:\n",
    "        print(\"pop save path: \", pop_save_path)\n",
    "        item_list = []\n",
    "        pop_item_all = []\n",
    "        for line in f:\n",
    "            line = line.strip().split()\n",
    "            item, pop_list = int(line[0]), [float(x) for x in line[1:]]\n",
    "            item_list.append(item)\n",
    "            pop_item_all.append(pop_list)\n",
    "    pop_item_all = np.array(pop_item_all)\n",
    "    print(\"pop_item_all shape:\", pop_item_all.shape)\n",
    "    print(\"load pop information:\",pop_item_all.mean(),pop_item_all.max(),pop_item_all.min())\n",
    "    return pop_item_all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "popularity used: ./autodl-tmp/douban/item_pop_seq_ori2.txt\n",
      "pop save path:  ./autodl-tmp/douban/item_pop_seq_ori2.txt\n",
      "pop_item_all shape: (26047, 10)\n",
      "load pop information: 0.0037181866982990796 1.0 0.0\n"
     ]
    }
   ],
   "source": [
    "pop_item_all = load_popularity()\n",
    "last_stage_popualarity = pop_item_all[:,-2]\n",
    "popularity_exp = 0.02\n",
    "popularity_matrix = pop_item_all[:,:-1]\n",
    "popularity_matrix = np.power(popularity_matrix,popularity_exp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConditionalBPRMF(nn.Module):\n",
    "    def __init__(self, n_users, n_items, emb_size, weight_decay):\n",
    "        super(ConditionalBPRMF, self).__init__()\n",
    "        self.n_users = data.n_users\n",
    "        self.n_items = data.n_items\n",
    "\n",
    "        self.decay = weight_decay\n",
    "        self.emb_dim = emb_size\n",
    "\n",
    "        self.user_embedding = nn.Embedding(self.n_users, self.emb_dim)\n",
    "        self.item_embedding = nn.Embedding(self.n_items, self.emb_dim) \n",
    "        \n",
    "        self.user_embedding.weight.data.uniform_(0, 0.005)  # 0-0.005之间均匀分布\n",
    "        self.item_embedding.weight.data.uniform_(0, 0.005)\n",
    "    \n",
    "    def forward(self, users, pos_items, neg_items, pos_pop, neg_pop):\n",
    "        user_embedding = self.user_embedding(users)\n",
    "        pos_item_embedding = self.item_embedding(pos_items)\n",
    "        neg_item_embedding = self.item_embedding(neg_items)\n",
    "\n",
    "        pos_scores = torch.sum(user_embedding * pos_item_embedding, dim=1)\n",
    "        neg_scores = torch.sum(user_embedding * neg_item_embedding, dim=1)\n",
    "\n",
    "        pos_scores = torch.nn.functional.elu(pos_scores) + 1\n",
    "        neg_scores = torch.nn.functional.elu(neg_scores) + 1\n",
    "        pos_scores_with_pop = pos_scores * pos_pop\n",
    "        neg_scores_with_pop = neg_scores * neg_pop\n",
    "\n",
    "        bpr_loss = -torch.mean(torch.log(torch.sigmoid(pos_scores_with_pop - neg_scores_with_pop)))\n",
    "        \n",
    "        regularizer = torch.norm(user_embedding, p=2) + torch.norm(pos_item_embedding, p=2) + torch.norm(neg_item_embedding, p=2)\n",
    "        reg_loss = self.decay * regularizer\n",
    "        return bpr_loss + reg_loss\n",
    "    \n",
    "    def do_recommendation(self, users):\n",
    "        user_emb = self.user_embedding(users)\n",
    "        item_embs = self.item_embedding.weight\n",
    "        scores = torch.mm(user_emb, item_embs.t())\n",
    "        return scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time slot unique in train: [0 1 2 5 3 4 6 7 8]\n",
      "                                                   uid  \\\n",
      "iid                                                      \n",
      "0    [0, 3, 6, 10, 11, 16, 20, 22, 25, 28, 30, 32, ...   \n",
      "1    [0, 1, 40, 60, 80, 83, 91, 106, 108, 114, 124,...   \n",
      "\n",
      "                                                  time  \n",
      "iid                                                     \n",
      "0    [0, 0, 8, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, ...  \n",
      "1    [0, 7, 0, 0, 7, 4, 7, 6, 1, 7, 1, 3, 2, 0, 0, ...  \n"
     ]
    }
   ],
   "source": [
    "data = Data(batch_size = 2048, step = 500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.add_expo_popularity(popularity_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataloader = torch.utils.data.DataLoader(data, batch_size=2048, shuffle=False, num_workers = 6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "evaluator = evaluation(data,[20], 2048)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = ConditionalBPRMF(data.n_users, data.n_items, emb_size = 32, weight_decay = 1e-04).to(device)\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr = 0.001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 10, loss: 81.90510239999207\n",
      "{'precision': array([0.0386284]), 'recall': array([0.03684976]), 'ndcg': array([0.05549095]), 'hit_ratio': array([0.3550144])}\n",
      "Running time: 74.38587455498055 Seconds\n",
      "\n",
      "epoch: 20, loss: 77.72276478553728\n",
      "{'precision': array([0.03767059]), 'recall': array([0.03676469]), 'ndcg': array([0.04959704]), 'hit_ratio': array([0.35895831])}\n",
      "Running time: 75.88603504397906 Seconds\n",
      "\n",
      "epoch: 30, loss: 75.64427268888562\n",
      "{'precision': array([0.03896645]), 'recall': array([0.03930513]), 'ndcg': array([0.05518225]), 'hit_ratio': array([0.37273069])}\n",
      "Running time: 75.07803630596027 Seconds\n",
      "\n",
      "epoch: 40, loss: 74.4729948351049\n",
      "{'precision': array([0.03873482]), 'recall': array([0.03728895]), 'ndcg': array([0.05506855]), 'hit_ratio': array([0.35808188])}\n",
      "Running time: 75.06934122100938 Seconds\n",
      "\n",
      "epoch: 50, loss: 71.1959285593786\n",
      "{'precision': array([0.0386597]), 'recall': array([0.03697963]), 'ndcg': array([0.05123796]), 'hit_ratio': array([0.35282334])}\n",
      "Running time: 74.5764044689713 Seconds\n",
      "\n",
      "epoch: 60, loss: 65.5747715049054\n",
      "{'precision': array([0.04151434]), 'recall': array([0.04085024]), 'ndcg': array([0.05675934]), 'hit_ratio': array([0.3732315])}\n",
      "Running time: 74.82721289998153 Seconds\n",
      "\n",
      "epoch: 70, loss: 59.569487110016325\n",
      "{'precision': array([0.04226243]), 'recall': array([0.0425151]), 'ndcg': array([0.05796762]), 'hit_ratio': array([0.38061851])}\n",
      "Running time: 74.46438274800312 Seconds\n",
      "\n",
      "epoch: 80, loss: 54.932647438799606\n",
      "{'precision': array([0.04423438]), 'recall': array([0.0440527]), 'ndcg': array([0.06260032]), 'hit_ratio': array([0.38894453])}\n",
      "Running time: 74.83747295296052 Seconds\n",
      "\n",
      "epoch: 90, loss: 51.87294874006086\n",
      "{'precision': array([0.04361462]), 'recall': array([0.04235806]), 'ndcg': array([0.05867238]), 'hit_ratio': array([0.37473394])}\n",
      "Running time: 74.76678002701374 Seconds\n",
      "\n",
      "epoch: 100, loss: 49.838852526901924\n",
      "{'precision': array([0.04612182]), 'recall': array([0.04522495]), 'ndcg': array([0.05871682]), 'hit_ratio': array([0.39182421])}\n",
      "Running time: 75.33081630698871 Seconds\n",
      "\n",
      "epoch: 110, loss: 48.21561309606449\n",
      "{'precision': array([0.04591211]), 'recall': array([0.04455508]), 'ndcg': array([0.06154895]), 'hit_ratio': array([0.38875673])}\n",
      "Running time: 74.4486795690027 Seconds\n",
      "\n",
      "epoch: 120, loss: 46.93020079902187\n",
      "{'precision': array([0.04615312]), 'recall': array([0.04521767]), 'ndcg': array([0.06308201]), 'hit_ratio': array([0.3912608])}\n",
      "Running time: 73.69499080302194 Seconds\n",
      "\n",
      "epoch: 130, loss: 45.93982978162854\n",
      "{'precision': array([0.04724552]), 'recall': array([0.04590691]), 'ndcg': array([0.06322009]), 'hit_ratio': array([0.39827219])}\n",
      "Running time: 75.12430015503196 Seconds\n",
      "\n",
      "epoch: 140, loss: 45.375188887288914\n",
      "{'precision': array([0.04754601]), 'recall': array([0.04742425]), 'ndcg': array([0.06354016]), 'hit_ratio': array([0.4023413])}\n",
      "Running time: 73.29786223196425 Seconds\n",
      "\n",
      "epoch: 150, loss: 44.93994036785081\n",
      "{'precision': array([0.04666333]), 'recall': array([0.04526978]), 'ndcg': array([0.0604925]), 'hit_ratio': array([0.3912608])}\n",
      "Running time: 74.25601143500535 Seconds\n",
      "\n"
     ]
    }
   ],
   "source": [
    "start = time.perf_counter()\n",
    "\n",
    "EPOCH = 150\n",
    "for epoch in range(EPOCH):\n",
    "    t_loss = 0.0\n",
    "    for idx, (users, pos_items, neg_items, pos_pop, neg_pop) in enumerate(dataloader):\n",
    "        users = users.to(device)\n",
    "        pos_items = pos_items.to(device)\n",
    "        neg_items = neg_items.to(device)\n",
    "        pos_pop = pos_pop.to(device)\n",
    "        neg_pop = neg_pop.to(device)\n",
    "        optimizer.zero_grad()\n",
    "        loss = model(users, pos_items, neg_items, pos_pop, neg_pop)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        t_loss += loss.item()\n",
    "        \n",
    "    if (1+epoch) % 10 == 0 :\n",
    "        res = evaluator.eval(model)\n",
    "        print(f'epoch: {epoch+1}, loss: {t_loss}')\n",
    "        print(res)\n",
    "        end = time.perf_counter()\n",
    "        print('Running time: %s Seconds\\n' % (end - start))\n",
    "        start = time.perf_counter()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}