learning_to_rank.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Learning to Rank or Machine Learned Ranking (MLR)\n",
    "\n",
    "Download the dataset from https://www.microsoft.com/en-us/research/project/mslr/. We are using the smaller version of the dataset MSLR-WEB10K. The dataset is divided into 5 folds. For this excercise we are using only Fold1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "from time import time\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.datasets import load_svmlight_file\n",
    "from sklearn.metrics import r2_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset_path = \"./mslr-web10k/Fold1/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# You need to pass the query_id=True, otherwise you wont get the query ids\n",
    "X_train, y_train, train_query_ids = load_svmlight_file(f\"{dataset_path}train.txt\", query_id=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test, y_test, test_query_ids = load_svmlight_file(f\"{dataset_path}test.txt\", query_id=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(723412, 136) (723412,) (723412,)\n",
      "(241521, 136) (241521,) (241521,)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape, y_train.shape, train_query_ids.shape)\n",
    "print(X_test.shape, y_test.shape, test_query_ids.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reference https://github.com/ogrisel/notebooks/blob/master/Learning%20to%20Rank.ipynb\n",
    "def subsample(X, y, qid, size, seed=None):\n",
    "    rng = np.random.RandomState(seed)\n",
    "    unique_qid = np.unique(qid)\n",
    "    qid_mask = rng.permutation(len(unique_qid))[:size]\n",
    "    subset_mask = np.in1d(qid, unique_qid[qid_mask])\n",
    "    return X[subset_mask], y[subset_mask], qid[subset_mask]\n",
    "\n",
    "X_train_small, y_train_small, qid_train_small = subsample(\n",
    "    X_train, y_train, train_query_ids, 500, seed=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(62244, 136) (62244,) (62244,) 500\n"
     ]
    }
   ],
   "source": [
    "print(X_train_small.shape, y_train_small.shape, qid_train_small.shape, len(np.unique(qid_train_small)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reference https://github.com/ogrisel/notebooks/blob/master/Learning%20to%20Rank.ipynb\n",
    "def dcg(relevances, rank=10):\n",
    "    relevances = np.asarray(relevances)[:rank]\n",
    "    n_relevances = len(relevances)\n",
    "    if n_relevances == 0:\n",
    "        return 0\n",
    "\n",
    "    discounts = np.log2(np.arange(n_relevances) + 2)\n",
    "    return np.sum(relevances/discounts)\n",
    "\n",
    "def ndcg(relevances, rank=10):\n",
    "    best_dcg = dcg(sorted(relevances, reverse=True), rank)\n",
    "    if best_dcg == 0:\n",
    "        return 0\n",
    "    \n",
    "    return dcg(relevances, rank) / best_dcg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.0\n",
      "4.2618595071429155\n",
      "10.654648767857287\n",
      "8.523719014285831\n",
      "7.02371901428583\n"
     ]
    }
   ],
   "source": [
    "print(dcg([0, 0, 2, 2], 3))\n",
    "print(dcg([2, 2, 2, 2], 3))\n",
    "print(dcg([5, 5, 5, 2], 3))\n",
    "print(dcg([5, 4, 2, 2], 3))\n",
    "print(dcg([2, 4, 5, 2], 3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.3065735963827292\n",
      "1.0\n",
      "1.0\n",
      "1.0\n",
      "0.8240204777414663\n"
     ]
    }
   ],
   "source": [
    "print(ndcg([0, 0, 2, 2], 3))\n",
    "print(ndcg([2, 2, 2, 2], 3))\n",
    "print(ndcg([5, 5, 5, 2], 3))\n",
    "print(ndcg([5, 4, 2, 2], 3))\n",
    "print(ndcg([2, 4, 5, 2], 3))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Compare dcg and ndcg. Observe how the normalization affects ndcg. It only cares about the right order. If the order is right then the actual score of the documents is not considered.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Reference https://github.com/ogrisel/notebooks/blob/master/Learning%20to%20Rank.ipynb\n",
    "def mean_ndcg(y_true, y_pred, query_ids, rank=10):\n",
    "    y_true = np.asarray(y_true)\n",
    "    y_pred = np.asarray(y_pred)\n",
    "    query_ids = np.asarray(query_ids)\n",
    "    \n",
    "    ndcg_scores = []\n",
    "    previous_qid = 0\n",
    "    previous_loc = 0\n",
    "    for loc, qid in enumerate(query_ids):\n",
    "        if previous_qid != qid:\n",
    "            chunk = slice(previous_loc, loc)\n",
    "            ranked_relevances = y_true[chunk][np.argsort(y_pred[chunk])[::-1]]\n",
    "            ndcg_scores.append(ndcg(ranked_relevances))\n",
    "            previous_loc = loc\n",
    "        previous_qid = qid\n",
    "        \n",
    "    chunk = slice(previous_loc, loc+1)\n",
    "    ranked_relevances = y_true[chunk][np.argsort(y_pred[chunk])[::-1]]\n",
    "    ndcg_scores.append(ndcg(ranked_relevances))\n",
    "    return np.mean(ndcg_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9795191506818377"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mean_ndcg([4, 3, 1, 4, 3], [4, 0, 1, 4, 2], [0, 0, 0, 2, 2], rank=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def print_evaluation(model, X, y, qid):\n",
    "    tic = time()\n",
    "    y_predicted = model.predict(X)\n",
    "    prediction_time = time() - tic\n",
    "    print(\"Prediction time: {:.3f}s\".format(prediction_time))\n",
    "    print(\"NDCG@5 score: {:.3f}\".format(\n",
    "    mean_ndcg(y, y_predicted, qid, rank=5)))\n",
    "    print(\"NDCG@10 score: {:.3f}\".format(\n",
    "    mean_ndcg(y, y_predicted, qid, rank=10)))\n",
    "    print(\"NDCG score: {:.3f}\".format(\n",
    "    mean_ndcg(y, y_predicted, qid, rank=None)))\n",
    "    print(\"R2 score: {:.3f}\".format(r2_score(y, y_predicted)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 6 µs, sys: 1e+03 ns, total: 7 µs\n",
      "Wall time: 13.4 µs\n"
     ]
    }
   ],
   "source": [
    "%time\n",
    "from sklearn.linear_model import LinearRegression\n",
    "\n",
    "lr = LinearRegression().fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 78.2 ms, sys: 1.39 ms, total: 79.6 ms\n",
      "Wall time: 77.8 ms\n",
      "Prediction time: 0.067s\n",
      "NDCG@5 score: 0.435\n",
      "NDCG@10 score: 0.435\n",
      "NDCG score: 0.435\n",
      "R2 score: 0.111\n"
     ]
    }
   ],
   "source": [
    "%time y_test_lr = lr.predict(X_test)\n",
    "print_evaluation(lr, X_test, y_test, test_query_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/sairahul/.local/share/virtualenvs/fastai_v1-1ZccQiGW/lib/python3.6/site-packages/sklearn/svm/base.py:196: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LibSVM]CPU times: user 2h 3min 40s, sys: 24.8 ms, total: 2h 3min 40s\n",
      "Wall time: 2h 3min 40s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "\n",
    "from sklearn import svm\n",
    "clf = svm.SVC(verbose=True)\n",
    "clf.fit(X_train_small, y_train_small)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prediction time: 3961.430s\n",
      "NDCG@5 score: 0.263\n",
      "NDCG@10 score: 0.263\n",
      "NDCG score: 0.263\n",
      "R2 score: -0.663\n",
      "CPU times: user 1h 6min 1s, sys: 139 ms, total: 1h 6min 2s\n",
      "Wall time: 1h 6min 1s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "print_evaluation(clf, X_test, y_test, test_query_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.base import RegressorMixin\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.base import clone"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def proba_to_relevance(probas):\n",
    "    \"\"\"MCRank-like reduction of classification proba to DCG predictions\"\"\"\n",
    "    rel = np.zeros(probas.shape[0], dtype=np.float32)\n",
    "    for i in range(probas.shape[1]):\n",
    "        rel += i * probas[:, i]\n",
    "    return rel\n",
    "        \n",
    "        \n",
    "class ClassificationRanker(RegressorMixin):\n",
    "    \n",
    "    def __init__(self, base_estimator=None):\n",
    "        self.base_estimator = base_estimator\n",
    "        \n",
    "    def fit(self, X, y):\n",
    "        self.estimator_ = clone(self.base_estimator)\n",
    "        self.scaler_ = StandardScaler(with_mean=False)\n",
    "        X = self.scaler_.fit_transform(X)\n",
    "        self.estimator_.fit(X, y)\n",
    "        \n",
    "    def predict(self, X):\n",
    "        X_scaled = self.scaler_.transform(X)\n",
    "        probas = self.estimator_.predict_proba(X_scaled)\n",
    "        return proba_to_relevance(probas)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "\n",
    "gbc = GradientBoostingClassifier(n_estimators=100, random_state=1)\n",
    "gbr = ClassificationRanker(gbc)\n",
    "gbr.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prediction time: 4.002s\n",
      "NDCG@5 score: 0.522\n",
      "NDCG@10 score: 0.522\n",
      "NDCG score: 0.522\n",
      "R2 score: 0.172\n"
     ]
    }
   ],
   "source": [
    "print_evaluation(gbr, X_test, y_test, test_query_ids)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "723412\n",
      "(723412, 136)\n"
     ]
    }
   ],
   "source": [
    "print(len(train_query_ids))\n",
    "print(X_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xgboost as xgb\n",
    "from xgboost import DMatrix\n",
    "import itertools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 15min 40s, sys: 1.64 s, total: 15min 42s\n",
      "Wall time: 1min 37s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "\n",
    "params = {'objective': 'rank:pairwise', 'learning_rate': 0.1,\n",
    "          'gamma': 0.5, 'min_child_weight': 0.1,\n",
    "          'max_depth': 6, 'n_estimators': 100}\n",
    "\n",
    "train_groups = [len(list(values)) for _, values in itertools.groupby(train_query_ids)]\n",
    "\n",
    "#train_dmatrix = DMatrix(X_train, y_train)\n",
    "#train_dmatrix.set_group(train_groups)\n",
    "#xgb_model = xgb.train(params, train_dmatrix)\n",
    "\n",
    "xgb_model = xgb.sklearn.XGBRanker(**params)\n",
    "xgb_model.fit(X_train, y_train, train_groups)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'rank:pairwise'"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xgb_model.objective"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prediction time: 6.049s\n",
      "NDCG@5 score: 0.523\n",
      "NDCG@10 score: 0.523\n",
      "NDCG score: 0.523\n",
      "R2 score: -0.093\n"
     ]
    }
   ],
   "source": [
    "print_evaluation(xgb_model, X_test, y_test, test_query_ids)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb_y = xgb_model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.48516673 0.43478358 0.4598418  0.5314547  0.33852553 0.53118646\n",
      " 0.4732565  0.46417266 0.5569814  0.50925577 0.54099464 0.43478358\n",
      " 0.55198264 0.66855407 0.5062158  0.54749286 0.5436234  0.5272356\n",
      " 0.5314547  0.44697112 0.33852553 0.55436903 0.53807515 0.5272356\n",
      " 0.49297768 0.4897708  0.38902566 0.53618056 0.54099464 0.54749286\n",
      " 0.51382136 0.523038   0.43478358 0.45770597 0.60123175 0.50147104\n",
      " 0.5272356  0.5526739  0.63090694 0.32597935 0.523038   0.4732565\n",
      " 0.4598418  0.44697112 0.5096137  0.44697112 0.43478358 0.60123175\n",
      " 0.4897708  0.43478358]\n",
      "[2. 1. 3. 1. 0. 0. 1. 0. 0. 2. 1. 1. 2. 2. 1. 2. 1. 2. 0. 1. 0. 1. 0. 0.\n",
      " 1. 0. 0. 2. 2. 2. 1. 1. 0. 0. 1. 0. 0. 1. 3. 0. 1. 1. 1. 0. 1. 1. 1. 0.\n",
      " 2. 0.]\n",
      "241521\n",
      "241521\n"
     ]
    }
   ],
   "source": [
    "print(xgb_y[:50])\n",
    "print(y_test[:50])\n",
    "print(len(xgb_y))\n",
    "print(len(y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "xgb_y_margin = xgb_model.predict(X_test, output_margin=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.48516673 0.43478358 0.4598418  0.5314547  0.33852553 0.53118646\n",
      " 0.4732565  0.46417266 0.5569814  0.50925577]\n",
      "[0.48516673 0.43478358 0.4598418  0.5314547  0.33852553 0.53118646\n",
      " 0.4732565  0.46417266 0.5569814  0.50925577]\n"
     ]
    }
   ],
   "source": [
    "print(xgb_y_margin[:10])\n",
    "print(xgb_y[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_groups = [len(list(values)) for _, values in itertools.groupby(test_query_ids)]\n",
    "\n",
    "test_dmatrix = DMatrix(X_test)\n",
    "test_dmatrix.set_group(test_groups)\n",
    "\n",
    "rank_values = xgb_model.get_booster().predict(test_dmatrix, pred_leaf=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(241521, 4)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rank_values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(241521,)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rank_values.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# References\n",
    "\n",
    "* https://github.com/ogrisel/notebooks/blob/master/Learning%20to%20Rank.ipynb"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}