examples/xfeat_tutorial_notebook.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tutorial\n",
    "\n",
    "xfeat provides transformation classes for feature engineering and exploration."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import xfeat\n",
    "\n",
    "# This tutorial show you how to use xfeat with an example from a synthetic dataset.\n",
    "csv_filepath = \"./example_dataset.csv\"\n",
    "\n",
    "# Serialize the original csv file into the feather file format.\n",
    "# For simplicity, we assume that train and test are concatenated to perform feature engineering.\n",
    "xfeat.utils.compress_df(pd.read_csv(csv_filepath)).to_feather(\"./example_dataset.ftr\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>target</th>\n",
       "      <th>loan_amt</th>\n",
       "      <th>funded_amt</th>\n",
       "      <th>installment</th>\n",
       "      <th>cat1</th>\n",
       "      <th>cat2</th>\n",
       "      <th>cat3</th>\n",
       "      <th>cat4</th>\n",
       "      <th>cat5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>b026324c6904b</td>\n",
       "      <td>2a9cb4b88d6d61c81d1</td>\n",
       "      <td>1</td>\n",
       "      <td>5465.659668</td>\n",
       "      <td>5465.659668</td>\n",
       "      <td>137</td>\n",
       "      <td>AE</td>\n",
       "      <td>Q</td>\n",
       "      <td>DD</td>\n",
       "      <td>Y</td>\n",
       "      <td>O9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>26ab0db90d72e</td>\n",
       "      <td>28ad0ba1e22ee510510</td>\n",
       "      <td>1</td>\n",
       "      <td>9444.073242</td>\n",
       "      <td>9444.073242</td>\n",
       "      <td>100</td>\n",
       "      <td>AE</td>\n",
       "      <td>I</td>\n",
       "      <td>AA</td>\n",
       "      <td>X</td>\n",
       "      <td>O5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6d7fce9fee471</td>\n",
       "      <td>194aa8b5b6e47267f03</td>\n",
       "      <td>0</td>\n",
       "      <td>13622.605469</td>\n",
       "      <td>13622.605469</td>\n",
       "      <td>176</td>\n",
       "      <td>CC</td>\n",
       "      <td>Q</td>\n",
       "      <td>A</td>\n",
       "      <td>X</td>\n",
       "      <td>O2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>48a24b70a0b37</td>\n",
       "      <td>6535542b996af517398</td>\n",
       "      <td>1</td>\n",
       "      <td>1582.478394</td>\n",
       "      <td>1582.478394</td>\n",
       "      <td>179</td>\n",
       "      <td>CC</td>\n",
       "      <td>O</td>\n",
       "      <td>AE</td>\n",
       "      <td>X</td>\n",
       "      <td>O5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1dcca23355272</td>\n",
       "      <td>056f04fe8bf20edfce0</td>\n",
       "      <td>1</td>\n",
       "      <td>17355.650391</td>\n",
       "      <td>20826.779297</td>\n",
       "      <td>133</td>\n",
       "      <td>AE</td>\n",
       "      <td>O</td>\n",
       "      <td>AE</td>\n",
       "      <td>X</td>\n",
       "      <td>O5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              id              user_id  target      loan_amt    funded_amt  \\\n",
       "0  b026324c6904b  2a9cb4b88d6d61c81d1       1   5465.659668   5465.659668   \n",
       "1  26ab0db90d72e  28ad0ba1e22ee510510       1   9444.073242   9444.073242   \n",
       "2  6d7fce9fee471  194aa8b5b6e47267f03       0  13622.605469  13622.605469   \n",
       "3  48a24b70a0b37  6535542b996af517398       1   1582.478394   1582.478394   \n",
       "4  1dcca23355272  056f04fe8bf20edfce0       1  17355.650391  20826.779297   \n",
       "\n",
       "   installment cat1 cat2 cat3 cat4 cat5  \n",
       "0          137   AE    Q   DD    Y   O9  \n",
       "1          100   AE    I   AA    X   O5  \n",
       "2          176   CC    Q    A    X   O2  \n",
       "3          179   CC    O   AE    X   O5  \n",
       "4          133   AE    O   AE    X   O5  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Check the serialized data.\n",
    "df = pd.read_feather(\"./example_dataset.ftr\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "------\n",
    "\n",
    "## Categorical columns\n",
    "\n",
    "xfeat's Data Transformation classes use a DataFrame as an input and a DataFrame as an output.\n",
    "It supports both pandas and cuDF DataFrames.\n",
    "\n",
    "`xfeat.SelectCategorical` extracts only the column of categorical data from the input dataframe.\n",
    "\n",
    "`xfeat.Pipeline` sequentially combines Data Transformation objects."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>cat1</th>\n",
       "      <th>cat2</th>\n",
       "      <th>cat3</th>\n",
       "      <th>cat4</th>\n",
       "      <th>cat5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>b026324c6904b</td>\n",
       "      <td>2a9cb4b88d6d61c81d1</td>\n",
       "      <td>AE</td>\n",
       "      <td>Q</td>\n",
       "      <td>DD</td>\n",
       "      <td>Y</td>\n",
       "      <td>O9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>26ab0db90d72e</td>\n",
       "      <td>28ad0ba1e22ee510510</td>\n",
       "      <td>AE</td>\n",
       "      <td>I</td>\n",
       "      <td>AA</td>\n",
       "      <td>X</td>\n",
       "      <td>O5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>6d7fce9fee471</td>\n",
       "      <td>194aa8b5b6e47267f03</td>\n",
       "      <td>CC</td>\n",
       "      <td>Q</td>\n",
       "      <td>A</td>\n",
       "      <td>X</td>\n",
       "      <td>O2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>48a24b70a0b37</td>\n",
       "      <td>6535542b996af517398</td>\n",
       "      <td>CC</td>\n",
       "      <td>O</td>\n",
       "      <td>AE</td>\n",
       "      <td>X</td>\n",
       "      <td>O5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1dcca23355272</td>\n",
       "      <td>056f04fe8bf20edfce0</td>\n",
       "      <td>AE</td>\n",
       "      <td>O</td>\n",
       "      <td>AE</td>\n",
       "      <td>X</td>\n",
       "      <td>O5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              id              user_id cat1 cat2 cat3 cat4 cat5\n",
       "0  b026324c6904b  2a9cb4b88d6d61c81d1   AE    Q   DD    Y   O9\n",
       "1  26ab0db90d72e  28ad0ba1e22ee510510   AE    I   AA    X   O5\n",
       "2  6d7fce9fee471  194aa8b5b6e47267f03   CC    Q    A    X   O2\n",
       "3  48a24b70a0b37  6535542b996af517398   CC    O   AE    X   O5\n",
       "4  1dcca23355272  056f04fe8bf20edfce0   AE    O   AE    X   O5"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import xfeat\n",
    "\n",
    "xfeat.SelectCategorical().fit_transform(df).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from xfeat import SelectCategorical, LabelEncoder, Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Takes categorical columns from the data frame and performs label encoding on them.\n",
    "# The converted data is stored in the column with suffix defined in `output_suffix`.\n",
    "# By defining `output_suffix=\"\"`, it is possible to store the result in the same column.\n",
    "encoder = Pipeline([\n",
    "    SelectCategorical(exclude_cols=[\"id\", \"user_id\"]),\n",
    "    LabelEncoder(output_suffix=\"\"),\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cat1</th>\n",
       "      <th>cat2</th>\n",
       "      <th>cat3</th>\n",
       "      <th>cat4</th>\n",
       "      <th>cat5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   cat1  cat2  cat3  cat4  cat5\n",
       "0     0     0     0     0     0\n",
       "1     0     1     1     1     1\n",
       "2     1     0     2     1     2\n",
       "3     1     2     3     1     1\n",
       "4     0     2     3     1     1"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoder.fit_transform(df).head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`xfeat.cat_encoder.ConcatCombination` creates new columns by combining the input columns."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cat1cat2</th>\n",
       "      <th>cat1cat3</th>\n",
       "      <th>cat1cat4</th>\n",
       "      <th>cat1cat5</th>\n",
       "      <th>cat2cat3</th>\n",
       "      <th>cat2cat4</th>\n",
       "      <th>cat2cat5</th>\n",
       "      <th>cat3cat4</th>\n",
       "      <th>cat3cat5</th>\n",
       "      <th>cat4cat5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   cat1cat2  cat1cat3  cat1cat4  cat1cat5  cat2cat3  cat2cat4  cat2cat5  \\\n",
       "0         0         0         0         0         0         0         0   \n",
       "1         1         1         1         1         1         1         1   \n",
       "2         2         2         2         2         2         2         2   \n",
       "3         3         3         2         3         3         3         3   \n",
       "4         4         4         1         1         3         3         3   \n",
       "\n",
       "   cat3cat4  cat3cat5  cat4cat5  \n",
       "0         0         0         0  \n",
       "1         1         1         1  \n",
       "2         2         2         2  \n",
       "3         3         3         1  \n",
       "4         3         3         1  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from xfeat import ConcatCombination\n",
    "\n",
    "\n",
    "encoder = Pipeline([\n",
    "    SelectCategorical(exclude_cols=[\"id\", \"user_id\"]),\n",
    "\n",
    "    # If there are many categorical columns,\n",
    "    # users can specify the columns to be combined with `input_cols` kwargs.\n",
    "    # `r=2` specifies the number of columns to combine the columns.\n",
    "    ConcatCombination(drop_origin=True, output_suffix=\"\", r=2),\n",
    "    \n",
    "    LabelEncoder(output_suffix=\"\"),\n",
    "])\n",
    "encoder.fit_transform(df).head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "------\n",
    "\n",
    "## Numerical columns\n",
    "\n",
    "`xfeat.SelectNumerical` extracts only the column of numerical data from the input dataframe.\n",
    "\n",
    "`xfeat.ArithmeticCombinations` creates new columns by applying arithmetic combinations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>loan_amt</th>\n",
       "      <th>funded_amt</th>\n",
       "      <th>installment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5465.659668</td>\n",
       "      <td>5465.659668</td>\n",
       "      <td>137</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9444.073242</td>\n",
       "      <td>9444.073242</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>13622.605469</td>\n",
       "      <td>13622.605469</td>\n",
       "      <td>176</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1582.478394</td>\n",
       "      <td>1582.478394</td>\n",
       "      <td>179</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>17355.650391</td>\n",
       "      <td>20826.779297</td>\n",
       "      <td>133</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       loan_amt    funded_amt  installment\n",
       "0   5465.659668   5465.659668          137\n",
       "1   9444.073242   9444.073242          100\n",
       "2  13622.605469  13622.605469          176\n",
       "3   1582.478394   1582.478394          179\n",
       "4  17355.650391  20826.779297          133"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from xfeat import SelectNumerical\n",
    "\n",
    "SelectNumerical(exclude_cols=[\"target\"]).fit_transform(df).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>loan_amtfunded_amt</th>\n",
       "      <th>loan_amtinstallment</th>\n",
       "      <th>funded_amtinstallment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10931.319336</td>\n",
       "      <td>5602.659668</td>\n",
       "      <td>5602.659668</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>18888.146484</td>\n",
       "      <td>9544.073242</td>\n",
       "      <td>9544.073242</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>27245.210938</td>\n",
       "      <td>13798.605469</td>\n",
       "      <td>13798.605469</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3164.956787</td>\n",
       "      <td>1761.478394</td>\n",
       "      <td>1761.478394</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>38182.429688</td>\n",
       "      <td>17488.650391</td>\n",
       "      <td>20959.779297</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   loan_amtfunded_amt  loan_amtinstallment  funded_amtinstallment\n",
       "0        10931.319336          5602.659668            5602.659668\n",
       "1        18888.146484          9544.073242            9544.073242\n",
       "2        27245.210938         13798.605469           13798.605469\n",
       "3         3164.956787          1761.478394            1761.478394\n",
       "4        38182.429688         17488.650391           20959.779297"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from xfeat import ArithmeticCombinations\n",
    "\n",
    "\n",
    "encoder = Pipeline([\n",
    "    SelectNumerical(exclude_cols=[\"target\"]),\n",
    "    ArithmeticCombinations(\n",
    "        drop_origin=True,\n",
    "        operator=\"+\",\n",
    "        r=2,\n",
    "        output_suffix=\"\",\n",
    "    ),\n",
    "])\n",
    "encoder.fit_transform(df).head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-----\n",
    "\n",
    "## Lambda Encoder\n",
    "\n",
    "`xfeat.LambdaEncoder` takes a lambda function as an argument and transforms the columns of the data frame."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>loan_amtfunded_amt</th>\n",
       "      <th>loan_amtinstallment</th>\n",
       "      <th>funded_amtinstallment</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>10931.0</td>\n",
       "      <td>5602.0</td>\n",
       "      <td>5602.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>18888.0</td>\n",
       "      <td>9544.0</td>\n",
       "      <td>9544.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>27245.0</td>\n",
       "      <td>13798.0</td>\n",
       "      <td>13798.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3164.0</td>\n",
       "      <td>1761.0</td>\n",
       "      <td>1761.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>38182.0</td>\n",
       "      <td>17488.0</td>\n",
       "      <td>20959.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   loan_amtfunded_amt  loan_amtinstallment  funded_amtinstallment\n",
       "0             10931.0               5602.0                 5602.0\n",
       "1             18888.0               9544.0                 9544.0\n",
       "2             27245.0              13798.0                13798.0\n",
       "3              3164.0               1761.0                 1761.0\n",
       "4             38182.0              17488.0                20959.0"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from xfeat import LambdaEncoder\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "encoder = Pipeline([\n",
    "    SelectNumerical(exclude_cols=[\"target\"]),\n",
    "    ArithmeticCombinations(\n",
    "        drop_origin=True,\n",
    "        operator=\"+\",\n",
    "        r=2,\n",
    "        output_suffix=\"\",\n",
    "    ),\n",
    "\n",
    "    LambdaEncoder(\n",
    "        lambda x: float(str(x)[:5]),\n",
    "        output_prefix=\"\",\n",
    "        output_suffix=\"\",\n",
    "        drop_origin=True,\n",
    "    ),\n",
    "])\n",
    "\n",
    "encoder.fit_transform(df).head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Serialize/Deserialize\n",
    "\n",
    "The parameters of the encoder can be serialized/deserialized by pickle.\n",
    "\n",
    "**Serialize:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cat1</th>\n",
       "      <th>cat2</th>\n",
       "      <th>cat3</th>\n",
       "      <th>cat4</th>\n",
       "      <th>cat5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   cat1  cat2  cat3  cat4  cat5\n",
       "0     0     0     0     0     0\n",
       "1     0     1     1     1     1\n",
       "2     1     0     2     1     2\n",
       "3     1     2     3     1     1\n",
       "4     0     2     3     1     1"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pickle\n",
    "\n",
    "df_train = pd.read_feather(\"./example_dataset.ftr\").head(10)\n",
    "df_test = pd.read_feather(\"./example_dataset.ftr\").tail(10)\n",
    "\n",
    "\n",
    "encoder = Pipeline([\n",
    "    SelectCategorical(exclude_cols=[\"id\", \"user_id\"]),\n",
    "    LabelEncoder(output_suffix=\"\"),\n",
    "])\n",
    "df_train_encoded = encoder.fit_transform(df_train)\n",
    "\n",
    "with open(\"label_encoder.pkl\", \"wb\") as f:\n",
    "    pickle.dump(encoder, f)\n",
    "    \n",
    "df_train_encoded.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Deserialize:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cat1</th>\n",
       "      <th>cat2</th>\n",
       "      <th>cat3</th>\n",
       "      <th>cat4</th>\n",
       "      <th>cat5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>-1</td>\n",
       "      <td>2</td>\n",
       "      <td>-1</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    cat1  cat2  cat3  cat4  cat5\n",
       "10    -1     1     6     1     0\n",
       "11     1     1    -1     0     2\n",
       "12    -1     2    -1     1    -1\n",
       "13     1     0     2    -1     0\n",
       "14     0     3    -1     0    -1"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open(\"label_encoder.pkl\", \"rb\") as f:\n",
    "    encoder = pickle.load(f)\n",
    "\n",
    "encoder.transform(df_test).head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The Label encoding mapping is kept in train.csv and test.csv. Unseen values are assigned to -1 in this case."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}