generate_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "from typing import List\n",
    "\n",
    "import cv2\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "\n",
    "import amazon_dataset"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Intialize Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [],
   "source": [
    "#DATASET = 'Musical_Instruments'\n",
    "#DATASET = 'Clothing_Shoes_and_Jewelry'\n",
    "#DATASET = 'Home_and_Kitchen'\n",
    "DATASET = 'Movies_and_TV'\n",
    "\n",
    "VALIDATION_SIZE=0.15\n",
    "RANDOM_SEED = 20230219\n",
    "#ALEXNET_IMAGE_FEATURES = Path(f'data/amazon/{DATASET}_alexnet_features.npz')\n",
    "VIT_IMAGE_FEATURES = Path(f'data/amazon/{DATASET}_vit_features.npz')\n",
    "CLIP_IMAGE_FEATURES = Path(f'data/amazon/{DATASET}_clipimage_features.npz')\n",
    "\n",
    "CLIP_TEXT_FEATURES = Path(f'data/amazon/{DATASET}_cliptext_features.npz')\n",
    "BERT_TEXT_FEATURES = Path(f'data/amazon/{DATASET}_bert_features.npz')\n",
    "DEST_FOLDER = Path(f'data/{DATASET}')\n",
    "\n",
    "#assert ALEXNET_IMAGE_FEATURES.exists()\n",
    "assert VIT_IMAGE_FEATURES.exists(), f\"{VIT_IMAGE_FEATURES} does not exist\"\n",
    "assert CLIP_IMAGE_FEATURES.exists(), f\"{CLIP_IMAGE_FEATURES} does not exist\"\n",
    "assert BERT_TEXT_FEATURES.exists(), f\"{BERT_TEXT_FEATURES} does not exist\"\n",
    "assert CLIP_TEXT_FEATURES.exists(), f\"{CLIP_TEXT_FEATURES} does not exist\"\n",
    "\n",
    "image_features = CLIP_IMAGE_FEATURES\n",
    "text_featues = CLIP_TEXT_FEATURES\n",
    "\n",
    "DEST_FOLDER.mkdir(exist_ok=True)\n",
    "\n",
    "with open(DEST_FOLDER / 'signature.txt', mode='w') as f:\n",
    "    f.write(f'Image: {image_features.stem}\\nText: {text_featues.stem}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_unique_ids(series: pd.Series) -> pd.Series:\n",
    "    rng = np.random.default_rng(RANDOM_SEED)\n",
    "    unique_ids = series.unique()\n",
    "    return pd.Series(\n",
    "        index=rng.permutation(unique_ids), \n",
    "        data=range(len(unique_ids))\n",
    "    )\n",
    "\n",
    "def df_stats(df: pd.DataFrame) -> str:\n",
    "    n_items = len(df['asin'].unique())\n",
    "    n_users = len(df['reviewerID'].unique())\n",
    "    sparsity = 1. * len(df) / (n_users * n_items)\n",
    "    return f'{n_users} users {n_items} items ratings: {len(df)}. Sparsity {sparsity * 100:.3f}%'\n",
    "\n",
    "def save_numerized(\n",
    "    reviews: pd.DataFrame, \n",
    "    uids: pd.Series, \n",
    "    product_ids: pd.Series,\n",
    "    user_id_column: str,\n",
    "    product_ids_column: str,\n",
    "    columns: List[str],\n",
    "    dest: Path\n",
    "):\n",
    "    \"\"\"Save a Dataframe following userids and product_ids\"\"\"\n",
    "    joined = reviews.join(uids.to_frame(user_id_column), on='reviewerID')\n",
    "    assert joined[user_id_column].isna().sum() == 0\n",
    "    \n",
    "    joined = joined.join(product_ids.to_frame(product_ids_column), on='asin')\n",
    "    assert joined[product_ids_column].isna().sum() == 0\n",
    "    \n",
    "    res = joined[columns]\n",
    "    \n",
    "    res.to_csv(dest, index=False)\n",
    "\n",
    "    return res\n",
    "\n",
    "def split_train_test_proportion(df: pd.DataFrame, test_prop=0.2):\n",
    "    \"\"\"\n",
    "    Split the dataframe by reviewer and take exactly `test_prop` records\n",
    "    for test and leave the rest for training\n",
    "    \"\"\"\n",
    "    rng = np.random.default_rng(RANDOM_SEED)\n",
    "\n",
    "    res = df.copy()\n",
    "    res['rnd'] = rng.random(size=len(df))\n",
    "    res['rnd_rank'] = res.groupby('reviewerID')['rnd'].rank(pct=True)\n",
    "    condition = res['rnd_rank'] <= test_prop\n",
    "    \n",
    "    (_, train), (_, test) = res.groupby(condition)\n",
    "    \n",
    "    assert(len(set(train.index) & set(test.index)) == 0)\n",
    "\n",
    "    return df.loc[train.index], df.loc[test.index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>asin</th>\n",
       "      <th>reviewerID</th>\n",
       "      <th>reviewerName</th>\n",
       "      <th>overall</th>\n",
       "      <th>text</th>\n",
       "      <th>reviewTime</th>\n",
       "      <th>summary</th>\n",
       "      <th>verified</th>\n",
       "      <th>vote</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8750</th>\n",
       "      <td>329672</td>\n",
       "      <td>B0010CYHW4</td>\n",
       "      <td>A1K6EBWW0BRCBP</td>\n",
       "      <td>enero30</td>\n",
       "      <td>4.0</td>\n",
       "      <td>very nice finish and quality for the price. my...</td>\n",
       "      <td>2017-04-20</td>\n",
       "      <td>Very happy for the price!</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31374</th>\n",
       "      <td>1213860</td>\n",
       "      <td>B003D33W3I</td>\n",
       "      <td>APZB58E43OOS1</td>\n",
       "      <td>Philip Turner</td>\n",
       "      <td>5.0</td>\n",
       "      <td>Finally a ligature with a good tight hold on m...</td>\n",
       "      <td>2017-03-31</td>\n",
       "      <td>A Winner Now</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29147</th>\n",
       "      <td>1122833</td>\n",
       "      <td>B000EELFGU</td>\n",
       "      <td>A1TFH8GU91KIBO</td>\n",
       "      <td>me</td>\n",
       "      <td>5.0</td>\n",
       "      <td>great price, product and delivery</td>\n",
       "      <td>2017-10-24</td>\n",
       "      <td>Five Stars</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            id        asin      reviewerID   reviewerName  overall  \\\n",
       "8750    329672  B0010CYHW4  A1K6EBWW0BRCBP        enero30      4.0   \n",
       "31374  1213860  B003D33W3I   APZB58E43OOS1  Philip Turner      5.0   \n",
       "29147  1122833  B000EELFGU  A1TFH8GU91KIBO             me      5.0   \n",
       "\n",
       "                                                    text  reviewTime  \\\n",
       "8750   very nice finish and quality for the price. my...  2017-04-20   \n",
       "31374  Finally a ligature with a good tight hold on m...  2017-03-31   \n",
       "29147                  great price, product and delivery  2017-10-24   \n",
       "\n",
       "                         summary  verified  vote  \n",
       "8750   Very happy for the price!      True   NaN  \n",
       "31374               A Winner Now      True   NaN  \n",
       "29147                 Five Stars      True   NaN  "
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reviews_df = amazon_dataset.reviews_df(DATASET)\n",
    "reviews_df.sample(n=3)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generate Data for SEM-MacridVAE & DMRL"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Some data stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Musical_Instruments\n",
      "5422 users 8460 items ratings: 40118. Sparsity 0.087%\n"
     ]
    }
   ],
   "source": [
    "print(DATASET)\n",
    "print(df_stats(reviews_df))"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sort some users randomly"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "A18ISN6NS073V2       0\n",
       "A1XFJTXA00NYX1       1\n",
       "A3GITGRRVQ4IP0       2\n",
       "A1W2V8Z9EWTZMM       3\n",
       "AEOQG6GFYR009        4\n",
       "                  ... \n",
       "A2LSUIA9THHK57    5417\n",
       "A31D3XL44W1NBM    5418\n",
       "A3775OP5VTX5ON    5419\n",
       "A1G0HYMR02WM2W    5420\n",
       "A7CN7OR0X715S     5421\n",
       "Length: 5422, dtype: int64"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "user_ids = generate_unique_ids(reviews_df['reviewerID'])\n",
    "user_ids"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Same with products"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "B0002GXZK4       0\n",
       "B00ZUBQY9K       1\n",
       "B00VUU6JL8       2\n",
       "B00PUQK4FU       3\n",
       "B00UTUKFHE       4\n",
       "              ... \n",
       "B000EEJ8VE    8455\n",
       "B00CAL9ZXK    8456\n",
       "B00U1XK5Z6    8457\n",
       "B0002D0CNA    8458\n",
       "B016L4MRMW    8459\n",
       "Length: 8460, dtype: int64"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "item_ids = generate_unique_ids(reviews_df['asin'])\n",
    "item_ids"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Split train test and validation by user"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_reviews, validation_test_reviews = split_train_test_proportion(reviews_df, test_prop=0.4)\n",
    "validation_reviews, test_reviews = split_train_test_proportion(validation_test_reviews, test_prop=0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "25628"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_reviews)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "len(validation_reviews)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6744"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(test_reviews)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sanity checks. All users are in the train validation and test sets!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(train_reviews['reviewerID']) == set(test_reviews['reviewerID']) ==  set(validation_reviews['reviewerID'])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check that the train set and the validation set all have at least one item (same check as above)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>train_size</th>\n",
       "      <th>val_size</th>\n",
       "      <th>test_size</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>5422.000000</td>\n",
       "      <td>5422.000000</td>\n",
       "      <td>5422.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>4.726669</td>\n",
       "      <td>1.428624</td>\n",
       "      <td>1.243821</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>2.451181</td>\n",
       "      <td>0.840090</td>\n",
       "      <td>0.720182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>4.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>5.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>36.000000</td>\n",
       "      <td>12.000000</td>\n",
       "      <td>12.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        train_size     val_size    test_size\n",
       "count  5422.000000  5422.000000  5422.000000\n",
       "mean      4.726669     1.428624     1.243821\n",
       "std       2.451181     0.840090     0.720182\n",
       "min       3.000000     1.000000     1.000000\n",
       "25%       3.000000     1.000000     1.000000\n",
       "50%       4.000000     1.000000     1.000000\n",
       "75%       5.000000     2.000000     1.000000\n",
       "max      36.000000    12.000000    12.000000"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.concat([\n",
    "    train_reviews.groupby('reviewerID').size().to_frame('train_size'),\n",
    "    validation_reviews.groupby('reviewerID').size().to_frame('val_size'),\n",
    "    test_reviews.groupby('reviewerID').size().to_frame('test_size')\n",
    "], axis=1).describe()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Generate users"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_ids.to_csv(DEST_FOLDER / 'users.txt')\n",
    "item_ids.to_csv(DEST_FOLDER / 'items.txt', header=['item_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user</th>\n",
       "      <th>item</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>13260</td>\n",
       "      <td>15762</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>17977</td>\n",
       "      <td>15762</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>14483</td>\n",
       "      <td>15762</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>20167</td>\n",
       "      <td>15762</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>7290</td>\n",
       "      <td>15762</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216099</th>\n",
       "      <td>21143</td>\n",
       "      <td>21474</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216101</th>\n",
       "      <td>14427</td>\n",
       "      <td>17537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216102</th>\n",
       "      <td>3341</td>\n",
       "      <td>10725</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216103</th>\n",
       "      <td>15373</td>\n",
       "      <td>19613</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>216108</th>\n",
       "      <td>4486</td>\n",
       "      <td>9254</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>136645 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         user   item\n",
       "0       13260  15762\n",
       "4       17977  15762\n",
       "5       14483  15762\n",
       "6       20167  15762\n",
       "9        7290  15762\n",
       "...       ...    ...\n",
       "216099  21143  21474\n",
       "216101  14427  17537\n",
       "216102   3341  10725\n",
       "216103  15373  19613\n",
       "216108   4486   9254\n",
       "\n",
       "[136645 rows x 2 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "NUMERIZED_COMMON_PARAMS = {\n",
    "    'uids': user_ids, \n",
    "    'product_ids': item_ids,\n",
    "    'user_id_column': 'user',\n",
    "    'product_ids_column': 'item',\n",
    "    'columns': ['user', 'item']\n",
    "}\n",
    "\n",
    "save_numerized(\n",
    "    train_reviews, \n",
    "    dest=DEST_FOLDER / 'train.txt',\n",
    "    **NUMERIZED_COMMON_PARAMS\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Saved validation data'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "save_numerized(\n",
    "    validation_reviews, \n",
    "    dest=DEST_FOLDER / 'validation.txt',\n",
    "    **NUMERIZED_COMMON_PARAMS\n",
    ")\n",
    "'Saved validation data'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Saved test data'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "save_numerized(\n",
    "    test_reviews, \n",
    "    dest=DEST_FOLDER / 'test.txt',\n",
    "    **NUMERIZED_COMMON_PARAMS\n",
    ")\n",
    "'Saved test data'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Opening file\n",
      "Initializing array (23958, 768)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying features:   0%|          | 101/24.0k [00:00<00:46, 510items/s] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Item B0018CWW96 not found in features file\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying features:   3%|▎         | 602/24.0k [00:01<00:45, 518items/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Item B0013KU93E not found in features file\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying features:   6%|▋         | 1.53k/24.0k [00:04<01:43, 216items/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Item B001C3O6QS not found in features file\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying features:   8%|▊         | 1.85k/24.0k [00:05<01:45, 209items/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Item B0014VPFPO not found in features file\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying features:  15%|█▍        | 3.56k/24.0k [00:12<01:15, 269items/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Item B001ANQY7Y not found in features file\n",
      "Item B001C00AHA not found in features file\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying features:  15%|█▌        | 3.67k/24.0k [00:12<01:34, 216items/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Item B001B3LIOC not found in features file\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying features:  23%|██▎       | 5.43k/24.0k [00:15<00:29, 630items/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Item B0017W22LK not found in features file\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying features:  26%|██▌       | 6.22k/24.0k [00:17<00:27, 654items/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Item B0013LRKRQ not found in features file\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying features:  29%|██▉       | 6.91k/24.0k [00:18<00:30, 551items/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Item B0017ANB08 not found in features file\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying features:  58%|█████▊    | 13.8k/24.0k [00:31<00:18, 563items/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Item B0014CKCCY not found in features file\n",
      "Item B0014F20GM not found in features file\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying features:  61%|██████    | 14.6k/24.0k [00:32<00:15, 587items/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Item B001ADKATC not found in features file\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying features:  68%|██████▊   | 16.2k/24.0k [00:35<00:11, 683items/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Item B0016MOWOG not found in features file\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying features:  71%|███████   | 16.9k/24.0k [00:36<00:11, 608items/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Item B0013TR4GK not found in features file\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying features:  72%|███████▏  | 17.1k/24.0k [00:36<00:11, 612items/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Item B0013LRKV2 not found in features file\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying features:  74%|███████▍  | 17.8k/24.0k [00:37<00:10, 589items/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Item B0019ESNGE not found in features file\n",
      "Item B0015XHR5C not found in features file\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying features:  83%|████████▎ | 19.9k/24.0k [00:41<00:08, 495items/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Item B001AQR39O not found in features file\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying features:  93%|█████████▎| 22.2k/24.0k [00:45<00:02, 690items/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Item B0017RFXRK not found in features file\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying features:  97%|█████████▋| 23.3k/24.0k [00:46<00:00, 792items/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Item B0013K7ZUE not found in features file\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying features: 100%|██████████| 24.0k/24.0k [00:47<00:00, 503items/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(23958, 768)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def copy_features(\n",
    "        item_ids: pd.Series, \n",
    "        features_file: Path\n",
    "):\n",
    "    print('Opening file')\n",
    "    with np.load(features_file) as features:\n",
    "        some_embedding = next(iter(features.values()))\n",
    "        embedding_shape, = some_embedding.shape\n",
    "        array_shape = (len(item_ids),  embedding_shape)\n",
    "        print(f'Initializing array {array_shape}')\n",
    "        res = np.full(array_shape,  fill_value=np.nan)\n",
    "\n",
    "        for asin, idx in tqdm(item_ids.items(), \n",
    "                total=len(item_ids), unit_scale=True, unit='items', \n",
    "                desc='Copying features'):\n",
    "            value = features.get(asin)\n",
    "            if value is None:\n",
    "                print(f'Item {asin} not found in features file')\n",
    "                res[idx, :] = 0    \n",
    "            else:\n",
    "                assert np.isnan(value).sum() == 0, \"Feature has NaN Values\"\n",
    "                res[idx, :] = value\n",
    "\n",
    "        return res\n",
    "\n",
    "\n",
    "image_features_array = copy_features(\n",
    "    item_ids=item_ids, \n",
    "    features_file=image_features\n",
    ")\n",
    "image_features_array.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data/Movies_and_TV/embed_image.npy: 140MiB\n"
     ]
    }
   ],
   "source": [
    "IMAGE_EMBED_DEST = DEST_FOLDER / 'embed_image.npy'\n",
    "np.save(IMAGE_EMBED_DEST, image_features_array)\n",
    "\n",
    "print(f'{str(IMAGE_EMBED_DEST)}: {IMAGE_EMBED_DEST.stat().st_size // 2**20}MiB')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Opening file\n",
      "Initializing array (23958, 768)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying features:   0%|          | 71.0/24.0k [00:00<00:34, 687items/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Item B0018CWW96 not found in features file\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying features:   3%|▎         | 655/24.0k [00:00<00:33, 696items/s] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Item B0013KU93E not found in features file\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying features:   7%|▋         | 1.59k/24.0k [00:02<00:41, 542items/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Item B001C3O6QS not found in features file\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying features:   8%|▊         | 1.92k/24.0k [00:03<00:41, 527items/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Item B0014VPFPO not found in features file\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying features:  15%|█▌        | 3.65k/24.0k [00:06<00:31, 638items/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Item B001ANQY7Y not found in features file\n",
      "Item B001C00AHA not found in features file\n",
      "Item B001B3LIOC not found in features file\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying features:  23%|██▎       | 5.46k/24.0k [00:08<00:29, 622items/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Item B0017W22LK not found in features file\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying features:  26%|██▋       | 6.30k/24.0k [00:09<00:23, 763items/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Item B0013LRKRQ not found in features file\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying features:  29%|██▉       | 6.92k/24.0k [00:10<00:23, 739items/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Item B0017ANB08 not found in features file\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying features:  58%|█████▊    | 13.8k/24.0k [00:21<00:17, 584items/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Item B0014CKCCY not found in features file\n",
      "Item B0014F20GM not found in features file\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying features:  61%|██████    | 14.6k/24.0k [00:22<00:18, 497items/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Item B001ADKATC not found in features file\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying features:  68%|██████▊   | 16.3k/24.0k [00:25<00:12, 616items/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Item B0016MOWOG not found in features file\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying features:  71%|███████   | 17.0k/24.0k [00:26<00:09, 773items/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Item B0013TR4GK not found in features file\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying features:  72%|███████▏  | 17.1k/24.0k [00:26<00:09, 757items/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Item B0013LRKV2 not found in features file\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying features:  74%|███████▍  | 17.8k/24.0k [00:27<00:08, 746items/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Item B0019ESNGE not found in features file\n",
      "Item B0015XHR5C not found in features file\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying features:  83%|████████▎ | 20.0k/24.0k [00:30<00:05, 783items/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Item B001AQR39O not found in features file\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying features:  93%|█████████▎| 22.3k/24.0k [00:33<00:02, 792items/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Item B0017RFXRK not found in features file\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying features:  97%|█████████▋| 23.3k/24.0k [00:34<00:00, 768items/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Item B0013K7ZUE not found in features file\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying features: 100%|██████████| 24.0k/24.0k [00:35<00:00, 676items/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(23958, 768)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_features_array = copy_features(\n",
    "    item_ids=item_ids, \n",
    "    features_file=text_featues\n",
    ")\n",
    "text_features_array.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data/Movies_and_TV/embed_text.npy: 140MiB\n"
     ]
    }
   ],
   "source": [
    "TEXT_EMBED_DEST = DEST_FOLDER / 'embed_text.npy'\n",
    "np.save(TEXT_EMBED_DEST, text_features_array)\n",
    "\n",
    "print(f'{str(TEXT_EMBED_DEST)}: {TEXT_EMBED_DEST.stat().st_size // 2**20}MiB')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generate images"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PosixPath('data/amazon/Musical_Instruments_product_images')"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "products_df = amazon_dataset.items_df(DATASET)\n",
    "products_images_dir = amazon_dataset.product_images_dir(DATASET)\n",
    "products_images_dir"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is necessary only for generating the fancy graph. Puts images on an transparent background"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying images:   0%|          | 40.0/24.0k [00:00<04:27, 89.2items/s, No images: 4]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Not a list\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying images:   2%|▏         | 538/24.0k [00:06<04:52, 80.1items/s, No images: 19]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Not a list\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying images:   6%|▋         | 1.52k/24.0k [00:19<04:59, 74.9items/s, No images: 41]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Not a list\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying images:   8%|▊         | 1.83k/24.0k [00:23<04:24, 83.8items/s, No images: 53]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Not a list\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying images:  15%|█▍        | 3.56k/24.0k [00:43<03:45, 90.4items/s, No images: 87]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Not a list\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying images:  15%|█▌        | 3.60k/24.0k [00:43<03:50, 88.4items/s, No images: 87]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Not a list\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying images:  15%|█▌        | 3.66k/24.0k [00:44<04:37, 73.3items/s, No images: 87]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Not a list\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying images:  22%|██▏       | 5.36k/24.0k [01:04<03:28, 89.4items/s, No images: 135]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Not a list\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying images:  26%|██▌       | 6.16k/24.0k [01:13<03:21, 88.2items/s, No images: 161]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Not a list\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying images:  29%|██▊       | 6.84k/24.0k [01:21<03:16, 87.2items/s, No images: 181]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Not a list\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying images:  57%|█████▋    | 13.8k/24.0k [02:42<02:06, 80.5items/s, No images: 334]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Not a list\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying images:  57%|█████▋    | 13.8k/24.0k [02:42<02:04, 81.7items/s, No images: 334]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Not a list\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying images:  61%|██████    | 14.5k/24.0k [02:51<01:44, 90.4items/s, No images: 354]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Not a list\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying images:  67%|██████▋   | 16.2k/24.0k [03:09<01:28, 88.5items/s, No images: 397]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Not a list\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying images:  70%|███████   | 16.9k/24.0k [03:17<01:16, 92.5items/s, No images: 413]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Not a list\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying images:  71%|███████   | 17.0k/24.0k [03:19<01:17, 89.5items/s, No images: 422]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Not a list\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying images:  74%|███████▍  | 17.7k/24.0k [03:26<01:07, 93.0items/s, No images: 443]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Not a list\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying images:  74%|███████▍  | 17.8k/24.0k [03:27<01:03, 96.2items/s, No images: 445]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Not a list\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying images:  83%|████████▎ | 19.8k/24.0k [03:49<00:41, 99.9items/s, No images: 509]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Not a list\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying images:  93%|█████████▎| 22.2k/24.0k [04:16<00:18, 93.4items/s, No images: 561]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Not a list\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying images:  97%|█████████▋| 23.2k/24.0k [04:28<00:08, 90.0items/s, No images: 588]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Not a list\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Copying images: 100%|██████████| 24.0k/24.0k [04:36<00:00, 86.5items/s, No images: 607]\n"
     ]
    }
   ],
   "source": [
    "def convert_image(src: str, dst: str):\n",
    "    min_white = np.array([235, 235, 235], np.uint8)\n",
    "    white = np.array([255, 255, 255], np.uint8)\n",
    "    img = cv2.imread(src)\n",
    "    \n",
    "    mask = 255 - cv2.inRange(img, min_white, white)\n",
    "    img = cv2.cvtColor(img, cv2.COLOR_BGR2BGRA)\n",
    "    res = cv2.bitwise_and(img, img, mask=mask)\n",
    "    cv2.imwrite(dst, res)\n",
    "\n",
    "def save_images(products_df: pd.DataFrame, item_ids: pd.Series, dest: Path):\n",
    "    products_by_asin = products_df.set_index('asin')\n",
    "    joined  = item_ids.to_frame('item_id').join(\n",
    "        products_by_asin, validate='one_to_one'\n",
    "    )\n",
    "    dest.mkdir(exist_ok=True)\n",
    "    \n",
    "    no_images = 0\n",
    "    \n",
    "    progress = tqdm(\n",
    "        joined.itertuples(), \n",
    "        desc='Copying images', \n",
    "        total=len(joined), \n",
    "        unit_scale=True, unit='items'\n",
    "    )\n",
    "    for row in progress:\n",
    "        if not isinstance(row.image_slug, list):\n",
    "            print('Not a list')\n",
    "            continue\n",
    "        if len(row.image_slug) == 0:\n",
    "            no_images += 1\n",
    "            progress.set_postfix_str(f'No images: {no_images}', refresh=False)\n",
    "            continue\n",
    "        slug = row.image_slug[0]\n",
    "        src = products_images_dir / f\"{slug}.jpg\"\n",
    "        assert src.exists()\n",
    "        dst = dest / f'{row.item_id}.png'\n",
    "        convert_image(str(src), str(dst))\n",
    "\n",
    "save_images(products_df, item_ids,  DEST_FOLDER / 'images')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_titles(products_df: pd.DataFrame, item_ids: pd.Series, dest: Path):\n",
    "    products_by_asin = products_df.set_index('asin')\n",
    "    joined  = item_ids.to_frame('item_id').join(\n",
    "        products_by_asin, validate='one_to_one'\n",
    "    )\n",
    "    dest / 'item_titles.txt'\n",
    "    joined[['item_id', 'title']].to_csv(dest / 'item_titles.txt', index=None)\n",
    "\n",
    "# Not really needed for anything yet\n",
    "#save_titles(products_df, item_ids,  DEST_FOLDER)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generate categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "categories_df = amazon_dataset.product_categories_df(DATASET)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# categories_id = generate_unique_ids(categories_df['name'])\n",
    "# categories_id.to_csv(DEST_FOLDER / 'categories.txt', header=['category_id'])\n",
    "# categories_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_categories_matrix(\n",
    "        categories_df: pd.DataFrame, \n",
    "        products_df: pd.DataFrame, \n",
    "        item_ids: pd.Series, \n",
    "        categories_id: pd.Series\n",
    "    ):\n",
    "    MIN_SUPPORT = 0.01\n",
    "    MAX_SUPPORT = 0.9\n",
    "    \n",
    "    # Categories only with not too many or too few items\n",
    "    categories_support = categories_df['name'].value_counts() \n",
    "\n",
    "    allowed_categories_names = categories_support[1:8].index\n",
    "\n",
    "    print(\"How many allowed categories\")\n",
    "    print(len(allowed_categories_names))\n",
    "    print(allowed_categories_names)\n",
    "    \n",
    "    filtered_categories = categories_df.loc[\n",
    "        categories_df['name'].isin(allowed_categories_names)\n",
    "    ]\n",
    "\n",
    "    products_df = products_df.join(\n",
    "        item_ids.to_frame('item_id'), on='asin', validate='1:1')\n",
    "\n",
    "    res = filtered_categories.set_index('product_id').join(\n",
    "        products_df, validate='m:1')\n",
    "    res = res.reset_index(drop=True)\n",
    "    \n",
    "    # Check all categories are in categories_id\n",
    "    res = res.join(categories_id.to_frame('category_id'), on='name')\n",
    "    assert res['category_id'].isna().sum() == 0\n",
    "\n",
    "    items_with_no_category = set(item_ids) - set(res['item_id'].unique())\n",
    "    print(f\"items with no category: {len(items_with_no_category)}\")\n",
    "    \n",
    "    return products_df.loc[\n",
    "        products_df['asin'].isin(item_ids[items_with_no_category].index)\n",
    "    ]\n",
    "\n",
    "    return res[['item_id', 'category_id']]\n",
    "\n",
    "# categories_matrix = build_categories_matrix(\n",
    "#     categories_df, products_df, item_ids, categories_id\n",
    "# )\n",
    "#categories_matrix\n",
    "#categories_matrix.to_csv(DEST_FOLDER / 'categorical.txt', index=False)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This is a VERY provisional way to analyze categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "def analyse_semantic_categories(products_df: pd.DataFrame, categories_df: pd.DataFrame, item_ids: pd.Series):\n",
    "    items_cate = np.load(DEST_FOLDER / 'item_cate.npy')\n",
    "    item_semantic_category = pd.Series(items_cate).to_frame('semantic_category')\n",
    "    items = item_ids.to_frame('item_id')\n",
    "\n",
    "    item_with_semantic = items.join(\n",
    "        item_semantic_category, \n",
    "        on='item_id', \n",
    "        validate='1:1'\n",
    "    )\n",
    "    products_with_semantic = item_with_semantic.join(\n",
    "        products_df.reset_index().set_index('asin')\n",
    "    )\n",
    "    products_with_semantic.set_index('id', inplace=True)\n",
    "    \n",
    "    categories_and_semantic = categories_df.join(\n",
    "        products_with_semantic, on='product_id', validate='m:1'\n",
    "    )\n",
    "    assert categories_and_semantic['semantic_category'].isna().sum() == 0\n",
    "    \n",
    "    for category_id, df in categories_and_semantic.groupby('semantic_category'):\n",
    "        print(f\"Category {category_id} Size: {len(df)}\")\n",
    "        print(df['name'].value_counts().head(n=10) )\n",
    "\n",
    "#analyse_semantic_categories(products_df, categories_df, item_ids)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.16"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "4c22d1444aa3f68a820bd00264b529de9aca8b813336c5d71bda95dadce90fbf"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}