examples/00_quick_start/fastai_movielens.ipynb

{
    "cells": [
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "<i>Copyright (c) Recommenders contributors.</i>\n",
                "\n",
                "<i>Licensed under the MIT License.</i>"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## FastAI Recommender\n",
                "\n",
                "This notebook shows how to use the [FastAI](https://fast.ai) recommender which is using [Pytorch](https://pytorch.org/) under the hood. "
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 1,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "System version: 3.6.11 | packaged by conda-forge | (default, Aug  5 2020, 20:09:42) \n",
                        "[GCC 7.5.0]\n",
                        "Pandas version: 0.25.3\n",
                        "Fast AI version: 1.0.46\n",
                        "Torch version: 1.4.0\n",
                        "Cuda Available: False\n",
                        "CuDNN Enabled: True\n"
                    ]
                }
            ],
            "source": [
                "# set the environment path to find Recommenders\n",
                "from tempfile import TemporaryDirectory\n",
                "import sys\n",
                "import os\n",
                "import pandas as pd\n",
                "import numpy as np\n",
                "import scrapbook as sb\n",
                "import torch, fastai\n",
                "from fastai.collab import collab_learner, CollabDataBunch, load_learner\n",
                "\n",
                "from recommenders.utils.constants import (\n",
                "    DEFAULT_USER_COL as USER, \n",
                "    DEFAULT_ITEM_COL as ITEM, \n",
                "    DEFAULT_RATING_COL as RATING, \n",
                "    DEFAULT_TIMESTAMP_COL as TIMESTAMP, \n",
                "    DEFAULT_PREDICTION_COL as PREDICTION\n",
                ") \n",
                "from recommenders.utils.timer import Timer\n",
                "from recommenders.datasets import movielens\n",
                "from recommenders.datasets.python_splitters import python_stratified_split\n",
                "from recommenders.models.fastai.fastai_utils import cartesian_product, score\n",
                "from recommenders.evaluation.python_evaluation import map_at_k, ndcg_at_k, precision_at_k, recall_at_k\n",
                "from recommenders.evaluation.python_evaluation import rmse, mae, rsquared, exp_var\n",
                "\n",
                "print(\"System version: {}\".format(sys.version))\n",
                "print(\"Pandas version: {}\".format(pd.__version__))\n",
                "print(\"Fast AI version: {}\".format(fastai.__version__))\n",
                "print(\"Torch version: {}\".format(torch.__version__))\n",
                "print(\"Cuda Available: {}\".format(torch.cuda.is_available()))\n",
                "print(\"CuDNN Enabled: {}\".format(torch.backends.cudnn.enabled))"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "Defining some constants to refer to the different columns of our dataset."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 3,
            "metadata": {
                "tags": [
                    "parameters"
                ]
            },
            "outputs": [],
            "source": [
                "# top k items to recommend\n",
                "TOP_K = 10\n",
                "\n",
                "# Select MovieLens data size: 100k, 1m, 10m, or 20m\n",
                "MOVIELENS_DATA_SIZE = '100k'\n",
                "\n",
                "# Model parameters\n",
                "N_FACTORS = 40\n",
                "EPOCHS = 5"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 4,
            "metadata": {},
            "outputs": [
                {
                    "name": "stderr",
                    "output_type": "stream",
                    "text": [
                        "100%|██████████| 4.81k/4.81k [00:01<00:00, 4.49kKB/s]\n"
                    ]
                },
                {
                    "data": {
                        "text/html": [
                            "<div>\n",
                            "<style scoped>\n",
                            "    .dataframe tbody tr th:only-of-type {\n",
                            "        vertical-align: middle;\n",
                            "    }\n",
                            "\n",
                            "    .dataframe tbody tr th {\n",
                            "        vertical-align: top;\n",
                            "    }\n",
                            "\n",
                            "    .dataframe thead th {\n",
                            "        text-align: right;\n",
                            "    }\n",
                            "</style>\n",
                            "<table border=\"1\" class=\"dataframe\">\n",
                            "  <thead>\n",
                            "    <tr style=\"text-align: right;\">\n",
                            "      <th></th>\n",
                            "      <th>UserId</th>\n",
                            "      <th>MovieId</th>\n",
                            "      <th>Rating</th>\n",
                            "      <th>Timestamp</th>\n",
                            "    </tr>\n",
                            "  </thead>\n",
                            "  <tbody>\n",
                            "    <tr>\n",
                            "      <th>0</th>\n",
                            "      <td>196</td>\n",
                            "      <td>242</td>\n",
                            "      <td>3.0</td>\n",
                            "      <td>881250949</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>1</th>\n",
                            "      <td>186</td>\n",
                            "      <td>302</td>\n",
                            "      <td>3.0</td>\n",
                            "      <td>891717742</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>2</th>\n",
                            "      <td>22</td>\n",
                            "      <td>377</td>\n",
                            "      <td>1.0</td>\n",
                            "      <td>878887116</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>3</th>\n",
                            "      <td>244</td>\n",
                            "      <td>51</td>\n",
                            "      <td>2.0</td>\n",
                            "      <td>880606923</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>4</th>\n",
                            "      <td>166</td>\n",
                            "      <td>346</td>\n",
                            "      <td>1.0</td>\n",
                            "      <td>886397596</td>\n",
                            "    </tr>\n",
                            "  </tbody>\n",
                            "</table>\n",
                            "</div>"
                        ],
                        "text/plain": [
                            "  UserId MovieId  Rating  Timestamp\n",
                            "0    196     242     3.0  881250949\n",
                            "1    186     302     3.0  891717742\n",
                            "2     22     377     1.0  878887116\n",
                            "3    244      51     2.0  880606923\n",
                            "4    166     346     1.0  886397596"
                        ]
                    },
                    "execution_count": 4,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": [
                "ratings_df = movielens.load_pandas_df(\n",
                "    size=MOVIELENS_DATA_SIZE,\n",
                "    header=[USER,ITEM,RATING,TIMESTAMP]\n",
                ")\n",
                "\n",
                "# make sure the IDs are loaded as strings to better prevent confusion with embedding ids\n",
                "ratings_df[USER] = ratings_df[USER].astype('str')\n",
                "ratings_df[ITEM] = ratings_df[ITEM].astype('str')\n",
                "\n",
                "ratings_df.head()"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 5,
            "metadata": {},
            "outputs": [],
            "source": [
                "# Split the dataset\n",
                "train_valid_df, test_df = python_stratified_split(\n",
                "    ratings_df, \n",
                "    ratio=0.75, \n",
                "    min_rating=1, \n",
                "    filter_by=\"item\", \n",
                "    col_user=USER, \n",
                "    col_item=ITEM\n",
                ")"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "# Remove \"cold\" users from test set  \n",
                "test_df = test_df[test_df.userID.isin(train_valid_df.userID)]"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Training"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 6,
            "metadata": {},
            "outputs": [],
            "source": [
                "# fix random seeds to make sure our runs are reproducible\n",
                "np.random.seed(101)\n",
                "torch.manual_seed(101)\n",
                "torch.cuda.manual_seed_all(101)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 7,
            "metadata": {},
            "outputs": [],
            "source": [
                "with Timer() as preprocess_time:\n",
                "    data = CollabDataBunch.from_df(train_valid_df, \n",
                "                                   user_name=USER, \n",
                "                                   item_name=ITEM, \n",
                "                                   rating_name=RATING, \n",
                "                                   valid_pct=0)\n"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 8,
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/html": [
                            "<table border=\"1\" class=\"dataframe\">\n",
                            "  <thead>\n",
                            "    <tr style=\"text-align: right;\">\n",
                            "      <th>UserId</th>\n",
                            "      <th>MovieId</th>\n",
                            "      <th>target</th>\n",
                            "    </tr>\n",
                            "  </thead>\n",
                            "  <tbody>\n",
                            "    <tr>\n",
                            "      <td>543</td>\n",
                            "      <td>1555</td>\n",
                            "      <td>3.0</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <td>90</td>\n",
                            "      <td>945</td>\n",
                            "      <td>5.0</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <td>292</td>\n",
                            "      <td>515</td>\n",
                            "      <td>4.0</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <td>303</td>\n",
                            "      <td>1092</td>\n",
                            "      <td>1.0</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <td>497</td>\n",
                            "      <td>946</td>\n",
                            "      <td>4.0</td>\n",
                            "    </tr>\n",
                            "  </tbody>\n",
                            "</table>"
                        ],
                        "text/plain": [
                            "<IPython.core.display.HTML object>"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                }
            ],
            "source": [
                "data.show_batch()"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "Now we will create a `collab_learner` for the data, which by default uses the [EmbeddingDotBias](https://docs.fast.ai/collab.html#EmbeddingDotBias) model. We will be using 40 latent factors. This will create an embedding for the users and the items that will map each of these to 40 floats as can be seen below. Note that the embedding parameters are not predefined, but are learned by the model.\n",
                "\n",
                "Although ratings can only range from 1-5, we are setting the range of possible ratings to a range from 0 to 5.5 -- that will allow the model to predict values around 1 and 5, which improves accuracy. Lastly, we set a value for weight-decay for regularization."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 9,
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/plain": [
                            "EmbeddingDotBias(\n",
                            "  (u_weight): Embedding(944, 40)\n",
                            "  (i_weight): Embedding(1683, 40)\n",
                            "  (u_bias): Embedding(944, 1)\n",
                            "  (i_bias): Embedding(1683, 1)\n",
                            ")"
                        ]
                    },
                    "execution_count": 9,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": [
                "learn = collab_learner(data, n_factors=N_FACTORS, y_range=[0,5.5], wd=1e-1)\n",
                "learn.model"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "Now train the model for 5 epochs setting the maximal learning rate. The learner will reduce the learning rate with each epoch using cosine annealing."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 10,
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/html": [
                            "<table border=\"1\" class=\"dataframe\">\n",
                            "  <thead>\n",
                            "    <tr style=\"text-align: left;\">\n",
                            "      <th>epoch</th>\n",
                            "      <th>train_loss</th>\n",
                            "      <th>valid_loss</th>\n",
                            "      <th>time</th>\n",
                            "    </tr>\n",
                            "  </thead>\n",
                            "  <tbody>\n",
                            "    <tr>\n",
                            "      <td>1</td>\n",
                            "      <td>0.985993</td>\n",
                            "      <td></td>\n",
                            "      <td>00:05</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <td>2</td>\n",
                            "      <td>0.885496</td>\n",
                            "      <td></td>\n",
                            "      <td>00:05</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <td>3</td>\n",
                            "      <td>0.777637</td>\n",
                            "      <td></td>\n",
                            "      <td>00:05</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <td>4</td>\n",
                            "      <td>0.628971</td>\n",
                            "      <td></td>\n",
                            "      <td>00:05</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <td>5</td>\n",
                            "      <td>0.532328</td>\n",
                            "      <td></td>\n",
                            "      <td>00:06</td>\n",
                            "    </tr>\n",
                            "  </tbody>\n",
                            "</table>"
                        ],
                        "text/plain": [
                            "<IPython.core.display.HTML object>"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                },
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Took 29.5549 seconds for training.\n"
                    ]
                }
            ],
            "source": [
                "with Timer() as train_time:\n",
                "    learn.fit_one_cycle(EPOCHS, max_lr=5e-3)\n",
                "\n",
                "print(\"Took {} seconds for training.\".format(train_time))"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "Save the learner so it can be loaded back later for inferencing / generating recommendations"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "tmp = TemporaryDirectory()\n",
                "model_path = os.path.join(tmp.name, \"movielens_model.pkl\")"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 11,
            "metadata": {},
            "outputs": [],
            "source": [
                "learn.export(model_path)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Generating Recommendations\n",
                "\n",
                "Load the learner from disk."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 12,
            "metadata": {},
            "outputs": [],
            "source": [
                "learner = load_learner(tmp.name, \"movielens_model.pkl\")"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "Get all users and items that the model knows"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 13,
            "metadata": {},
            "outputs": [],
            "source": [
                "total_users, total_items = learner.data.train_ds.x.classes.values()\n",
                "total_items = total_items[1:]\n",
                "total_users = total_users[1:]"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "Get all users from the test set and remove any users that were know in the training set"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 14,
            "metadata": {},
            "outputs": [],
            "source": [
                "test_users = test_df[USER].unique()\n",
                "test_users = np.intersect1d(test_users, total_users)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "Build the cartesian product of test set users and all items known to the model"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 15,
            "metadata": {},
            "outputs": [],
            "source": [
                "users_items = cartesian_product(np.array(test_users),np.array(total_items))\n",
                "users_items = pd.DataFrame(users_items, columns=[USER,ITEM])"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {
                "scrolled": false
            },
            "source": [
                "\n",
                "Lastly, remove the user/items combinations that are in the training set -- we don't want to propose a movie that the user has already watched."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 16,
            "metadata": {
                "scrolled": false
            },
            "outputs": [],
            "source": [
                "training_removed = pd.merge(users_items, train_valid_df.astype(str), on=[USER, ITEM], how='left')\n",
                "training_removed = training_removed[training_removed[RATING].isna()][[USER, ITEM]]"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "### Score the model to find the top K recommendation"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 17,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Took 1.9734 seconds for 1511060 predictions.\n"
                    ]
                }
            ],
            "source": [
                "with Timer() as test_time:\n",
                "    top_k_scores = score(learner, \n",
                "                         test_df=training_removed,\n",
                "                         user_col=USER, \n",
                "                         item_col=ITEM, \n",
                "                         prediction_col=PREDICTION)\n",
                "\n",
                "print(\"Took {} seconds for {} predictions.\".format(test_time, len(training_removed)))"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "Calculate some metrics for our model"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 18,
            "metadata": {},
            "outputs": [],
            "source": [
                "eval_map = map_at_k(test_df, top_k_scores, col_user=USER, col_item=ITEM, \n",
                "                    col_rating=RATING, col_prediction=PREDICTION, \n",
                "                    relevancy_method=\"top_k\", k=TOP_K)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 19,
            "metadata": {},
            "outputs": [],
            "source": [
                "eval_ndcg = ndcg_at_k(test_df, top_k_scores, col_user=USER, col_item=ITEM, \n",
                "                      col_rating=RATING, col_prediction=PREDICTION, \n",
                "                      relevancy_method=\"top_k\", k=TOP_K)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 20,
            "metadata": {},
            "outputs": [],
            "source": [
                "eval_precision = precision_at_k(test_df, top_k_scores, col_user=USER, col_item=ITEM, \n",
                "                                col_rating=RATING, col_prediction=PREDICTION, \n",
                "                                relevancy_method=\"top_k\", k=TOP_K)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 21,
            "metadata": {},
            "outputs": [],
            "source": [
                "eval_recall = recall_at_k(test_df, top_k_scores, col_user=USER, col_item=ITEM, \n",
                "                          col_rating=RATING, col_prediction=PREDICTION, \n",
                "                          relevancy_method=\"top_k\", k=TOP_K)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 22,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Model:\tCollabLearner\n",
                        "Top K:\t10\n",
                        "MAP:\t0.026115\n",
                        "NDCG:\t0.155065\n",
                        "Precision@K:\t0.136691\n",
                        "Recall@K:\t0.054940\n"
                    ]
                }
            ],
            "source": [
                "print(\"Model:\\t\" + learn.__class__.__name__,\n",
                "      \"Top K:\\t%d\" % TOP_K,\n",
                "      \"MAP:\\t%f\" % eval_map,\n",
                "      \"NDCG:\\t%f\" % eval_ndcg,\n",
                "      \"Precision@K:\\t%f\" % eval_precision,\n",
                "      \"Recall@K:\\t%f\" % eval_recall, sep='\\n')"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "The above numbers are lower than [SAR](../sar_single_node_movielens.ipynb), but expected, since the model is explicitly trying to generalize the users and items to the latent factors. Next look at how well the model predicts how the user would rate the movie. Need to score `test_df` user-items only. "
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 23,
            "metadata": {},
            "outputs": [],
            "source": [
                "scores = score(learner, \n",
                "               test_df=test_df.copy(), \n",
                "               user_col=USER, \n",
                "               item_col=ITEM, \n",
                "               prediction_col=PREDICTION)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "Now calculate some regression metrics"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 24,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Model:\tCollabLearner\n",
                        "RMSE:\t0.902379\n",
                        "MAE:\t0.712163\n",
                        "Explained variance:\t0.346523\n",
                        "R squared:\t0.345672\n"
                    ]
                }
            ],
            "source": [
                "eval_r2 = rsquared(test_df, scores, col_user=USER, col_item=ITEM, col_rating=RATING, col_prediction=PREDICTION)\n",
                "eval_rmse = rmse(test_df, scores, col_user=USER, col_item=ITEM, col_rating=RATING, col_prediction=PREDICTION)\n",
                "eval_mae = mae(test_df, scores, col_user=USER, col_item=ITEM, col_rating=RATING, col_prediction=PREDICTION)\n",
                "eval_exp_var = exp_var(test_df, scores, col_user=USER, col_item=ITEM, col_rating=RATING, col_prediction=PREDICTION)\n",
                "\n",
                "print(\"Model:\\t\" + learn.__class__.__name__,\n",
                "      \"RMSE:\\t%f\" % eval_rmse,\n",
                "      \"MAE:\\t%f\" % eval_mae,\n",
                "      \"Explained variance:\\t%f\" % eval_exp_var,\n",
                "      \"R squared:\\t%f\" % eval_r2, sep='\\n')"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "That RMSE is actually quite good when compared to these benchmarks: https://www.librec.net/release/v1.3/example.html"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 25,
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "application/scrapbook.scrap.json+json": {
                            "data": 0.02611475567509659,
                            "encoder": "json",
                            "name": "map",
                            "version": 1
                        }
                    },
                    "metadata": {
                        "scrapbook": {
                            "data": true,
                            "display": false,
                            "name": "map"
                        }
                    },
                    "output_type": "display_data"
                },
                {
                    "data": {
                        "application/scrapbook.scrap.json+json": {
                            "data": 0.15506533130248687,
                            "encoder": "json",
                            "name": "ndcg",
                            "version": 1
                        }
                    },
                    "metadata": {
                        "scrapbook": {
                            "data": true,
                            "display": false,
                            "name": "ndcg"
                        }
                    },
                    "output_type": "display_data"
                },
                {
                    "data": {
                        "application/scrapbook.scrap.json+json": {
                            "data": 0.13669141039236482,
                            "encoder": "json",
                            "name": "precision",
                            "version": 1
                        }
                    },
                    "metadata": {
                        "scrapbook": {
                            "data": true,
                            "display": false,
                            "name": "precision"
                        }
                    },
                    "output_type": "display_data"
                },
                {
                    "data": {
                        "application/scrapbook.scrap.json+json": {
                            "data": 0.05493986799753499,
                            "encoder": "json",
                            "name": "recall",
                            "version": 1
                        }
                    },
                    "metadata": {
                        "scrapbook": {
                            "data": true,
                            "display": false,
                            "name": "recall"
                        }
                    },
                    "output_type": "display_data"
                },
                {
                    "data": {
                        "application/scrapbook.scrap.json+json": {
                            "data": 0.9023793356156464,
                            "encoder": "json",
                            "name": "rmse",
                            "version": 1
                        }
                    },
                    "metadata": {
                        "scrapbook": {
                            "data": true,
                            "display": false,
                            "name": "rmse"
                        }
                    },
                    "output_type": "display_data"
                },
                {
                    "data": {
                        "application/scrapbook.scrap.json+json": {
                            "data": 0.7121634655740025,
                            "encoder": "json",
                            "name": "mae",
                            "version": 1
                        }
                    },
                    "metadata": {
                        "scrapbook": {
                            "data": true,
                            "display": false,
                            "name": "mae"
                        }
                    },
                    "output_type": "display_data"
                },
                {
                    "data": {
                        "application/scrapbook.scrap.json+json": {
                            "data": 0.34652281723228295,
                            "encoder": "json",
                            "name": "exp_var",
                            "version": 1
                        }
                    },
                    "metadata": {
                        "scrapbook": {
                            "data": true,
                            "display": false,
                            "name": "exp_var"
                        }
                    },
                    "output_type": "display_data"
                },
                {
                    "data": {
                        "application/scrapbook.scrap.json+json": {
                            "data": 0.3456716162958503,
                            "encoder": "json",
                            "name": "rsquared",
                            "version": 1
                        }
                    },
                    "metadata": {
                        "scrapbook": {
                            "data": true,
                            "display": false,
                            "name": "rsquared"
                        }
                    },
                    "output_type": "display_data"
                },
                {
                    "data": {
                        "application/scrapbook.scrap.json+json": {
                            "data": 29.554921820759773,
                            "encoder": "json",
                            "name": "train_time",
                            "version": 1
                        }
                    },
                    "metadata": {
                        "scrapbook": {
                            "data": true,
                            "display": false,
                            "name": "train_time"
                        }
                    },
                    "output_type": "display_data"
                },
                {
                    "data": {
                        "application/scrapbook.scrap.json+json": {
                            "data": 1.973397959023714,
                            "encoder": "json",
                            "name": "test_time",
                            "version": 1
                        }
                    },
                    "metadata": {
                        "scrapbook": {
                            "data": true,
                            "display": false,
                            "name": "test_time"
                        }
                    },
                    "output_type": "display_data"
                }
            ],
            "source": [
                "# Record results with papermill for tests\n",
                "sb.glue(\"map\", eval_map)\n",
                "sb.glue(\"ndcg\", eval_ndcg)\n",
                "sb.glue(\"precision\", eval_precision)\n",
                "sb.glue(\"recall\", eval_recall)\n",
                "sb.glue(\"rmse\", eval_rmse)\n",
                "sb.glue(\"mae\", eval_mae)\n",
                "sb.glue(\"exp_var\", eval_exp_var)\n",
                "sb.glue(\"rsquared\", eval_r2)\n",
                "sb.glue(\"train_time\", train_time.interval)\n",
                "sb.glue(\"test_time\", test_time.interval)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "metadata": {},
            "outputs": [],
            "source": [
                "tmp.cleanup()"
            ]
        },
        {
            "cell_type": "raw",
            "metadata": {},
            "source": []
        }
    ],
    "metadata": {
        "celltoolbar": "Tags",
        "kernelspec": {
            "display_name": "Python (reco_gpu)",
            "language": "python",
            "name": "reco_gpu"
        },
        "language_info": {
            "codemirror_mode": {
                "name": "ipython",
                "version": 3
            },
            "file_extension": ".py",
            "mimetype": "text/x-python",
            "name": "python",
            "nbconvert_exporter": "python",
            "pygments_lexer": "ipython3",
            "version": "3.6.11"
        }
    },
    "nbformat": 4,
    "nbformat_minor": 2
}