session5_filled_notebook.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyO1WuOyMZDgrhPg4RJMpy1R",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/wjvlno/python_bootcamp/blob/main/session5_filled_notebook.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Setup & Loading the Titanic Dataset"
      ],
      "metadata": {
        "id": "jlYGRhhx5hKV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "from sklearn.model_selection import train_test_split, cross_val_score\n",
        "from sklearn.preprocessing import OneHotEncoder, StandardScaler\n",
        "from sklearn.compose import ColumnTransformer\n",
        "from sklearn.pipeline import Pipeline\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.metrics import accuracy_score, confusion_matrix, classification_report\n",
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt\n",
        "from sklearn.datasets import fetch_openml\n",
        "\n",
        "# Load the Titanic dataset from scikit-learn\n",
        "titanic = fetch_openml('titanic', version=1, as_frame=True)\n",
        "titanic = titanic.frame\n",
        "\n",
        "# view the features for training\n",
        "titanic"
      ],
      "metadata": {
        "id": "XGZqVsBY-0j7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# drop some irrelevant columns\n",
        "titanic.drop(['home.dest', 'body', 'boat', 'embarked'], axis = 1, inplace = True)"
      ],
      "metadata": {
        "id": "W-ZIGbsMCpIE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Getting an Intuition for the Data"
      ],
      "metadata": {
        "id": "pFvzShd25nSB"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "\n",
        "# Handle missing values in the 'age' column\n",
        "# titanic['age'].fillna(titanic['age'].median(), inplace=True)\n",
        "\n",
        "# Create the plot\n",
        "plt.figure(figsize=(15, 8))\n",
        "ax = sns.kdeplot(titanic[\"age\"][titanic.survived == '1'], color=\"darkturquoise\", fill=True, label='Survived')\n",
        "sns.kdeplot(titanic[\"age\"][titanic.survived == '0'], color=\"lightcoral\", fill=True, label='Died')\n",
        "\n",
        "# Add labels and title\n",
        "plt.legend()\n",
        "plt.title('Density Plot of Age for Surviving Population and Deceased Population')\n",
        "ax.set(xlabel='Age')\n",
        "plt.xlim(-10, 85)\n",
        "\n",
        "# Show the plot\n",
        "plt.show()\n"
      ],
      "metadata": {
        "id": "5_rqWdry7JJ9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Fare vs survival\n",
        "\n",
        "plt.figure(figsize=(15,8))\n",
        "ax = sns.kdeplot(titanic[\"fare\"][titanic.survived == '1'], color=\"darkturquoise\", fill=True)\n",
        "sns.kdeplot(titanic[\"fare\"][titanic.survived == '0'], color=\"lightcoral\", fill=True)\n",
        "plt.legend(['Survived', 'Died'])\n",
        "plt.title('Density Plot of Fare for Surviving Population and Deceased Population')\n",
        "ax.set(xlabel='Fare')\n",
        "plt.xlim(-20,200)\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "zwj7Ynh95_HN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Passenger class vs. survival\n",
        "\n",
        "titanic['survived'] = titanic['survived'].astype(int)\n",
        "\n",
        "plt.figure(figsize=(8,8))\n",
        "sns.barplot(x='pclass', y='survived', data=titanic, color=\"darkturquoise\")\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "hAS5jwaM6Vt4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Gender vs. survival\n",
        "\n",
        "plt.figure(figsize=(8,8))\n",
        "sns.barplot(x='sex', y='survived', data=titanic, color=\"aquamarine\")\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "4Wabs_kA62o_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Running a Vanilla ML Model"
      ],
      "metadata": {
        "id": "7cnJyMe4wb3y"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "from sklearn.impute import SimpleImputer\n",
        "from sklearn.pipeline import Pipeline\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.compose import ColumnTransformer\n",
        "from sklearn.preprocessing import OneHotEncoder\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import accuracy_score, confusion_matrix, classification_report\n",
        "\n",
        "\n",
        "# Load the Titanic dataset from scikit-learn\n",
        "titanic = fetch_openml('titanic', version=1, as_frame=True)\n",
        "titanic = titanic.frame\n",
        "\n",
        "titanic.drop(['home.dest', 'body', 'boat', 'embarked'], axis = 1, inplace = True)\n",
        "\n",
        "# Specify numeric and categorical columns\n",
        "numeric_features = titanic.select_dtypes(include=['int64', 'float64']).columns\n",
        "categorical_features = titanic.select_dtypes(include=['object']).columns\n",
        "\n",
        "# Define a preprocessor (more on this later...)\n",
        "preprocessor = ColumnTransformer(\n",
        "    transformers=[\n",
        "        ('num', SimpleImputer(strategy='median'), numeric_features),\n",
        "        ('cat', Pipeline(steps=[\n",
        "            ('imputer', SimpleImputer(strategy='most_frequent')),\n",
        "            ('onehot', OneHotEncoder(handle_unknown='ignore'))\n",
        "        ]), categorical_features)\n",
        "    ])\n",
        "\n",
        "# Build a pipeline that preprocesses the data then fits the logistic classifier\n",
        "pipeline = Pipeline(steps=[\n",
        "    ('preprocessor', preprocessor),\n",
        "    ('classifier', LogisticRegression(max_iter=1000))\n",
        "])\n",
        "\n",
        "# Separate the data into features (X) and target (y)\n",
        "X = titanic.drop(columns='survived')\n",
        "y = titanic['survived'].astype(int)\n",
        "\n",
        "# Split the data into train and test sets\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.5, random_state=42)\n",
        "\n",
        "# Fit the pipeline\n",
        "pipeline.fit(X_train, y_train)\n",
        "\n",
        "# Predict on the test set\n",
        "y_pred = pipeline.predict(X_test)\n",
        "\n",
        "# Evaluate the model\n",
        "print(\"Test Set Accuracy:\", accuracy_score(y_test, y_pred))\n",
        "print(\"Confusion Matrix:\\n\", confusion_matrix(y_test, y_pred))\n",
        "print(\"Classification Report:\\n\", classification_report(y_test, y_pred))\n",
        "\n",
        "# Plot the confusion matrix\n",
        "cm = confusion_matrix(y_test, y_pred)\n",
        "plt.figure(figsize=(6, 6))\n",
        "sns.heatmap(cm, annot=True, fmt='d', cmap='Blues')\n",
        "plt.title('Confusion Matrix')\n",
        "plt.xlabel('Predicted')\n",
        "plt.ylabel('Actual')\n",
        "plt.show()\n",
        "\n",
        "# Plot the ROC curve\n",
        "y_pred_proba = pipeline.predict_proba(X_test)[:, 1]\n",
        "fpr, tpr, _ = roc_curve(y_test, y_pred_proba)\n",
        "roc_auc = roc_auc_score(y_test, y_pred_proba)\n",
        "\n",
        "plt.figure(figsize=(8, 6))\n",
        "plt.plot(fpr, tpr, color='blue', label=f'ROC Curve (area = {roc_auc:.2f})')\n",
        "plt.plot([0, 1], [0, 1], color='red', linestyle='--')\n",
        "plt.xlabel('False Positive Rate')\n",
        "plt.ylabel('True Positive Rate')\n",
        "plt.title('Receiver Operating Characteristic (ROC) Curve')\n",
        "plt.legend(loc='lower right')\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "y_8-zpT_wbSC"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Building a Better Classifier"
      ],
      "metadata": {
        "id": "4P1KpUjvmT55"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Start with the data - preprocessing and handling missing data"
      ],
      "metadata": {
        "id": "-OpzNzcexro5"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Visualize missing data\n",
        "\n",
        "plt.figure(figsize=(12, 8))\n",
        "sns.heatmap(titanic.isnull(), cbar=False, cmap='viridis', yticklabels=False)\n",
        "plt.title('Missing Values Heatmap in Titanic Dataset')\n",
        "plt.xlabel('Columns')\n",
        "plt.ylabel('Rows')\n",
        "plt.show()\n",
        "\n",
        "# quick for loop to print number of missing values in each column\n",
        "for col in titanic.columns:\n",
        "  print(f'{col}: # missing = {titanic[col].isnull().sum()}')\n",
        "\n",
        "\n",
        "\n",
        "# Observation: a lot of missing age and cabin data, plus one missing fare"
      ],
      "metadata": {
        "id": "22N832lT-2BB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Missing Age Values"
      ],
      "metadata": {
        "id": "0u_RE5_vmeYd"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# compute overall median age\n",
        "overall_median_age = titanic['age'].median()\n",
        "\n",
        "# Calculate the median age for each combination of pclass and sex\n",
        "median_ages = titanic.groupby(['sex', 'pclass'])['age'].median().reset_index()\n",
        "\n",
        "# Plotting\n",
        "plt.figure(figsize=(10, 6))\n",
        "g = sns.catplot(x='pclass', y='age', hue='sex', kind='bar', data=median_ages, palette='viridis')\n",
        "\n",
        "# add a dotted line for overall median age\n",
        "plt.axhline(y=overall_median_age, color='red', linestyle='--')\n",
        "\n",
        "\n",
        "# Set plot titles and labels\n",
        "g.set_titles(\"{col_name} Passengers\")\n",
        "g.set_axis_labels(\"Passenger Class (pclass)\", \"Median Age\")\n",
        "g.fig.suptitle('Median Ages for Different \"Pclass\" and \"Sex\"', fontsize=12)\n",
        "\n",
        "plt.show()\n",
        "\n",
        "# print the median ages for these different groups\n",
        "age_by_pclass_sex = titanic.groupby(['sex', 'pclass'])['age'].median()\n",
        "\n",
        "for pclass in range(1, 4):\n",
        "    for sex in ['female', 'male']:\n",
        "        print('Median age of Pclass {} {}s: {}'.format(pclass, sex, age_by_pclass_sex[sex][pclass]))\n",
        "print('Median age of all passengers: {}'.format(titanic['age'].median()))\n",
        "\n",
        "\n",
        "# The overall median might be useful for interpolating the ages of\n",
        "# female passengers in the 2nd passenger class, but would likely undershoot the\n",
        "# ages of the 1st passenger class, and undershoot the 3rd passenger class\n",
        "\n",
        "# A better approach is to interpolate with medians based on the different\n",
        "# levels of \"sex\" and \"pclass\" present in the dataset"
      ],
      "metadata": {
        "id": "xlq3f2A0ADhx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# interpolate age using pclass and sex variables\n",
        "age_by_pclass_sex = titanic.groupby(['sex', 'pclass'])['age'].median()\n",
        "\n",
        "for pclass in range(1, 4):\n",
        "    for sex in ['female', 'male']:\n",
        "        print('Median age of Pclass {} {}s: {}'.format(pclass, sex, age_by_pclass_sex[sex][pclass]))\n",
        "print()\n",
        "print('Median age of all passengers: {}'.format(titanic['age'].median()))\n"
      ],
      "metadata": {
        "id": "FZmwY7m1A7uU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Filling the missing values in Age with the medians of Sex and Pclass groups\n",
        "titanic['age'] = titanic.groupby(['sex', 'pclass'])['age'].apply(lambda x: x.fillna(x.median())).values"
      ],
      "metadata": {
        "id": "jd2RnnJzBBBp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Adding a New Variable (Family Size)"
      ],
      "metadata": {
        "id": "kdEeJB8zmr3I"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# sibsp = number of siblings/spouses aboard\n",
        "# parch = number of parents/children\n",
        "\n",
        "# let's combine these into a family size variable\n",
        "\n",
        "titanic['family_size'] = titanic['sibsp'] + titanic['parch'] + 1"
      ],
      "metadata": {
        "id": "gxZh9usQBsL4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Missing Fare Value"
      ],
      "metadata": {
        "id": "v7lzYY7Sm0fl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Handling the missing fare value\n",
        "\n",
        "# Calculate the median age for each combination of pclass and sex\n",
        "median_fares = titanic.groupby(['pclass', 'family_size'])['fare'].median().reset_index()\n",
        "\n",
        "# Plotting\n",
        "plt.figure(figsize=(10, 6))\n",
        "g = sns.catplot(x='pclass', y='fare', hue='family_size', kind='bar', data=median_fares, palette='viridis')\n",
        "\n",
        "plt.axhline(y = titanic['fare'].median(), color='red', linestyle='--')\n",
        "\n",
        "# Set plot titles and labels\n",
        "g.set_titles(\"{col_name} Passengers\")\n",
        "g.set_axis_labels(\"Passenger Class (pclass)\", \"Median Fare\")\n",
        "g.fig.suptitle('Median Fares for Different \"Pclass\" and \"Family Size\"', fontsize=12)\n",
        "\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "mCVW2iV-BDJm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# check the median fare for different combinations of 'pclass' and 'family_size'\n",
        "\n",
        "fare_by_pclass_fsize = titanic.groupby(['pclass', 'family_size'])['fare'].median()\n",
        "\n",
        "for pclass in range(1, 4):\n",
        "    for fsize in sorted(titanic[titanic['pclass'] == pclass]['family_size'].unique()):\n",
        "        print('Median fare for Pclass {} w/ family size {}: {}'.format(pclass, fsize, fare_by_pclass_fsize[pclass][fsize]))\n",
        "    print()\n",
        "\n",
        "print()\n",
        "print('Median age of all passengers: {}'.format(titanic['fare'].median()))"
      ],
      "metadata": {
        "id": "jf8_y0YGGXW5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# What's the pclass and family size for the passenger with a missing fare?\n",
        "\n",
        "titanic[titanic['fare'].isnull()]\n",
        "\n",
        "# pclass = 3\n",
        "# family_size = 1\n",
        "\n",
        "# so we should interpolate this passenger's fare with 7.8542"
      ],
      "metadata": {
        "id": "sICSdwcvFugU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "titanic['fare'] = titanic['fare'].fillna(titanic.groupby(['pclass', 'family_size'])['fare'].transform('median'))"
      ],
      "metadata": {
        "id": "L8jHlBaDCRaW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Adding Other New Variables"
      ],
      "metadata": {
        "id": "KbzUejmhm5D3"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Let's add some other variables while we're at it.\n",
        "\n",
        "# Since fare tends to increase with family size, let's compute a fare per person\n",
        "titanic['fare_per_person'] = titanic['fare'] / titanic['family_size']\n",
        "\n",
        "# And let's add a variable to specify if a passenger is alone\n",
        "titanic['is_alone'] = np.where(titanic['family_size'] == 1, 1, 0)"
      ],
      "metadata": {
        "id": "O1h2Afj-LwBV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Visualize missing data again\n",
        "\n",
        "plt.figure(figsize=(12, 8))\n",
        "sns.heatmap(titanic.isnull(), cbar=False, cmap='viridis', yticklabels=False)\n",
        "plt.title('Missing Values Heatmap in Titanic Dataset')\n",
        "plt.xlabel('Columns')\n",
        "plt.ylabel('Rows')\n",
        "plt.show()"
      ],
      "metadata": {
        "id": "OwpD_OElCgTD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Handling Missing Cabin Data"
      ],
      "metadata": {
        "id": "jidQXpcEnD1N"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Now for the missing cabin data...\n",
        "\n",
        "# Let's see what the cabin variable looks like\n",
        "\n",
        "for cabin in titanic['cabin'].unique():\n",
        "  print(cabin)"
      ],
      "metadata": {
        "id": "yoR8iKhAI6ro"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# So some values of cabin have multiple entries...\n",
        "\n",
        "# F and G occur together, as do E and F\n",
        "\n",
        "# Let's clean this up by just taking the letter from the first entry in each cell\n",
        "\n",
        "titanic['cabin'].str.split().str[0] # remove anything after a space\n",
        "titanic['cabin'] = titanic['cabin'].fillna('').apply(lambda x: x[0] if x else None) # take only the first value (i.e., letter), if the value isn't 'None'"
      ],
      "metadata": {
        "id": "wlMI5pC2JaLd"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Let's see which cabins different passenger classes occupied\n",
        "plt.figure(figsize=(12, 6))\n",
        "\n",
        "# Create a count plot\n",
        "sns.countplot(data=titanic, x='cabin', hue='pclass', palette='viridis', order=sorted(titanic['cabin'].dropna().unique()))\n",
        "\n",
        "# Set plot titles and labels\n",
        "plt.title('Distribution of Pclass for Different Cabin Categories')\n",
        "plt.xlabel('Cabin Category')\n",
        "plt.ylabel('Count')\n",
        "plt.legend(title='Pclass')\n",
        "plt.show()\n",
        "\n",
        "\n",
        "# Cabins A, B, and C are only occupied by first-class passengers"
      ],
      "metadata": {
        "id": "bOS7q-jfKKOF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# What if we take the fare into account as well?\n",
        "\n",
        "titanic['fare_per_person'] = titanic['fare'] / titanic['family_size']\n",
        "\n",
        "titanic['fpp_bin'] = pd.qcut(titanic['fare_per_person'], 4, labels=[1,2,3,4])\n",
        "\n",
        "# Set up the matplotlib figure\n",
        "plt.figure(figsize=(10, 6))\n",
        "\n",
        "# Create subplots for each fare_per_person_bin (fpp_bin)\n",
        "fare_bins = sorted(titanic['fpp_bin'].unique())\n",
        "for i, bin in enumerate(fare_bins, 1):\n",
        "    plt.subplot(2, 2, i)\n",
        "    sns.countplot(data=titanic[titanic['fpp_bin'] == bin], x='cabin', hue='pclass', palette='viridis', order=sorted(titanic['cabin'].dropna().unique()))\n",
        "    plt.title(f'Pclass vs Cabin (Level {bin} Fare)')\n",
        "    plt.xlabel('Cabin Category')\n",
        "    plt.ylabel('Count')\n",
        "    plt.legend(title='Pclass')\n",
        "\n",
        "# Adjust layout\n",
        "plt.tight_layout()\n",
        "plt.show()\n"
      ],
      "metadata": {
        "id": "Q_mFvitSMO0t"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Probabilistic Interpolation"
      ],
      "metadata": {
        "id": "oKxW_KijnI4x"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Calculate the probabilities of each cabin given fare_per_person_bin and pclass\n",
        "cabin_probs = titanic.dropna(subset=['cabin']).groupby(['fpp_bin', 'pclass', 'cabin']).size().unstack(fill_value=0)\n",
        "cabin_probs = cabin_probs.div(cabin_probs.sum(axis=1), axis=0)\n",
        "cabin_probs = cabin_probs.fillna(0)\n",
        "\n",
        "# Display the cabin probabilities\n",
        "print(cabin_probs)"
      ],
      "metadata": {
        "id": "MgRVq58OOR22"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "null_cabin_indices[0]"
      ],
      "metadata": {
        "id": "SAGpYQik1zqG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "null_cabin_indices = np.where(titanic.cabin.isnull())[0]\n",
        "\n",
        "# Function to assign cabin probabilistically\n",
        "def assign_cabin_probabilistically(row, cabin_probs):\n",
        "    if pd.isna(row['cabin']) or row['cabin'] == '':\n",
        "        fare_bin = row['fpp_bin']\n",
        "        pclass = row['pclass']\n",
        "        if (fare_bin, pclass) in cabin_probs.index:\n",
        "            cabins = cabin_probs.loc[(fare_bin, pclass)]\n",
        "            if cabins.max() == 0:\n",
        "                # Fallback to probabilities for pclass only if all cabin probabilities are zero\n",
        "                cabins = cabin_probs.groupby(level='pclass').sum().loc[pclass]\n",
        "        else:\n",
        "            # Fallback to probabilities for pclass only if the specific combination is not found\n",
        "            cabins = cabin_probs.groupby(level='pclass').sum().loc[pclass]\n",
        "\n",
        "        cabins = cabins[cabins > 0]  # Only consider non-zero probabilities\n",
        "        return np.random.choice(cabins.index, p=cabins.values / cabins.values.sum())\n",
        "    return row['cabin']\n",
        "\n",
        "# Apply the function to assign missing cabin values\n",
        "titanic['cabin_assigned'] = titanic.apply(assign_cabin_probabilistically, axis=1, cabin_probs=cabin_probs)\n",
        "\n",
        "# View the result\n",
        "print(titanic.loc[null_cabin_indices, ['pclass', 'fpp_bin', 'cabin_assigned']].head(30))\n",
        "\n",
        "# fill in missing cabin values\n",
        "titanic['cabin'] = titanic['cabin_assigned']\n",
        "\n",
        "# drop extra columns we created along the way\n",
        "titanic.drop(['cabin_assigned', 'fpp_bin'], axis = 1, inplace = True)"
      ],
      "metadata": {
        "id": "FA7eR_QGOaEc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Adding a Non-Categorical Cabin Variable"
      ],
      "metadata": {
        "id": "oCQ0BGQHnSU1"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Cabins with higher letters (e.g., F and G) were deeper in the ship and possibly\n",
        "# harder to exit, so let's assign integers for cabin level\n",
        "\n",
        "from sklearn.preprocessing import LabelEncoder\n",
        "\n",
        "# Encode cabin alphabetically\n",
        "label_encoder = LabelEncoder()\n",
        "titanic['cabin_int'] = label_encoder.fit_transform(titanic['cabin'])"
      ],
      "metadata": {
        "id": "75XS9heYdNng"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Visualize missing data again (again)\n",
        "\n",
        "plt.figure(figsize=(12, 8))\n",
        "sns.heatmap(titanic.isnull(), cbar=False, cmap='viridis', yticklabels=False)\n",
        "plt.title('Missing Values Heatmap in Titanic Dataset')\n",
        "plt.xlabel('Columns')\n",
        "plt.ylabel('Rows')\n",
        "plt.show()\n",
        "\n",
        "for col in titanic.columns:\n",
        "  print(f'{col}: # missing = {titanic[col].isnull().sum()}')\n",
        "\n",
        "# Much better"
      ],
      "metadata": {
        "id": "5eih90NCRIU-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### More Variables!"
      ],
      "metadata": {
        "id": "g897ugYbnY18"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Adding additional variables - is_married?\n",
        "\n",
        "titanic['title'] = titanic['name'].str.split(', ', expand=True)[1].str.split('.', expand=True)[0]\n",
        "titanic['is_married'] = 0\n",
        "titanic['is_married'].loc[titanic['title'] == 'Mrs'] = 1\n",
        "\n",
        "# drop extra columns\n",
        "titanic.drop(['title', 'name'], axis=1, inplace = True)"
      ],
      "metadata": {
        "id": "VXwQmjWzY-AJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# backup\n",
        "\n",
        "titanic_backup = titanic.copy()\n",
        "# titanic = titanic_backup.copy()"
      ],
      "metadata": {
        "id": "CuPsh31davtJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Final Prep for ML"
      ],
      "metadata": {
        "id": "OvZ3VyxPaeTU"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# ML estimators don't like strings.\n",
        "\n",
        "# For variables like 'sex' ('Male' vs. 'Female'), we need to do some form of dummy coding\n",
        "\n",
        "# Typical dummy codes would assign a number for each level of sex (e.g., 1 for female, 2 for male)\n",
        "# But most ML estimators will treat these dummy coded values linearly (male > female),\n",
        "# which can be problematic.\n",
        "\n",
        "# A common approach is 'One Hot Encoding', where we make a column for Male_yn,\n",
        "# and a different column for Female_yn."
      ],
      "metadata": {
        "id": "DRgTS1_rbG63"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### One Hot Encoding for Categorical Variables"
      ],
      "metadata": {
        "id": "e348H83Anggh"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# One hot encoding for sex variable\n",
        "\n",
        "from sklearn.compose import ColumnTransformer\n",
        "from sklearn.preprocessing import OneHotEncoder\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.pipeline import Pipeline\n",
        "\n",
        "# Define the preprocessor\n",
        "preprocessor = ColumnTransformer(\n",
        "    transformers=[\n",
        "        ('cat', OneHotEncoder(), ['sex'])\n",
        "    ])\n",
        "\n",
        "# Define the pipeline\n",
        "pipeline = Pipeline(steps=[('preprocessor', preprocessor),\n",
        "                           ('classifier', LogisticRegression(max_iter=200))])\n"
      ],
      "metadata": {
        "id": "0sTk-FtEcKdn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Train/Test Split"
      ],
      "metadata": {
        "id": "iapRGRfLnk3X"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Split the data into train and test sets\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "X = titanic.drop(columns='survived')\n",
        "y = titanic['survived'].astype(int)\n",
        "\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)\n"
      ],
      "metadata": {
        "id": "JE5WW8Z-akrH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Refitting the Model"
      ],
      "metadata": {
        "id": "OIlFsZ39nqVL"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YLIvk50p4cgl"
      },
      "outputs": [],
      "source": [
        "# Base classifier\n",
        "\n",
        "# Fit the pipeline\n",
        "pipeline.fit(X_train, y_train)\n",
        "\n",
        "# Evaluate the pipeline\n",
        "from sklearn.metrics import accuracy_score, confusion_matrix, classification_report\n",
        "\n",
        "y_pred = pipeline.predict(X_test)\n",
        "\n",
        "print(\"Base Model Accuracy:\", accuracy_score(y_test, y_pred))\n",
        "print(\"Confusion Matrix:\\n\", confusion_matrix(y_test, y_pred))\n",
        "print(\"Classification Report:\\n\", classification_report(y_test, y_pred))\n",
        "\n",
        "# Plot confusion matrix\n",
        "sns.heatmap(confusion_matrix(y_test, y_pred), annot=True, fmt='d', cmap='Blues')\n",
        "plt.title('Confusion Matrix - Base Model')\n",
        "plt.xlabel('Predicted')\n",
        "plt.ylabel('Actual')\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Further Improvements"
      ],
      "metadata": {
        "id": "cKNIYQF-o5k4"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Feature Engineering"
      ],
      "metadata": {
        "id": "OCZGZLfay1zA"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# What's going on with age?\n",
        "\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt\n",
        "from sklearn.datasets import fetch_openml\n",
        "\n",
        "# Define age bins and labels\n",
        "bins = [0, 10, 20, 30, 40, 50, 60, 70, 80]\n",
        "labels = ['0-10', '11-20', '21-30', '31-40', '41-50', '51-60', '61-70', '71-80']\n",
        "titanic['age_group'] = pd.cut(titanic['age'], bins=bins, labels=labels, right=False)\n",
        "\n",
        "# Convert the 'survived' column to numeric\n",
        "titanic['survived'] = pd.to_numeric(titanic['survived'])\n",
        "\n",
        "# Calculate survival rate by age group\n",
        "age_survival = titanic.groupby('age_group')['survived'].mean().reset_index()\n",
        "\n",
        "# Plot the survival rate by age group\n",
        "plt.figure(figsize=(10, 6))\n",
        "sns.barplot(x='age_group', y='survived', data=age_survival, palette='viridis')\n",
        "plt.title('Survival Rate by Age Group')\n",
        "plt.xlabel('Age Group')\n",
        "plt.ylabel('Survival Rate')\n",
        "plt.ylim(0, 1)\n",
        "plt.show()\n"
      ],
      "metadata": {
        "id": "6RpKHp6Ey4es"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# This doesn't look linear. In fact, we can see a few changes in direction in\n",
        "# the relationship between age and survival rate. This means the relationship is\n",
        "# nonmonotonic. Two changes in direction mean a polynomial of order 3 should explain\n",
        "# the relationship better than a linear relationship, but let's see if this checks out...\n",
        "\n",
        "from sklearn.metrics import mean_squared_error\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "from sklearn.model_selection import train_test_split, cross_val_score\n",
        "from sklearn.preprocessing import PolynomialFeatures, StandardScaler\n",
        "from sklearn.pipeline import Pipeline\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.metrics import accuracy_score, confusion_matrix, classification_report\n",
        "import matplotlib.pyplot as plt\n",
        "from sklearn.datasets import fetch_openml\n",
        "from sklearn.model_selection import KFold\n",
        "\n",
        "# Load the Titanic dataset from scikit-learn\n",
        "titanic_polytest = fetch_openml('titanic', version=1, as_frame=True)\n",
        "titanic_polytest = titanic_polytest.frame\n",
        "\n",
        "# Drop rows with missing target values and relevant features\n",
        "titanic_polytest = titanic_polytest.dropna(subset=['survived', 'age'])\n",
        "\n",
        "# Select relevant features and target\n",
        "X = titanic_polytest[['age']]\n",
        "y = titanic_polytest['survived']\n",
        "\n",
        "# Split the data\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)\n",
        "\n",
        "# Function to perform polynomial regression and return cross-validated RMSE\n",
        "def polynomial_regression(degree, X_train, y_train):\n",
        "    polynomial_features = PolynomialFeatures(degree=degree)\n",
        "    X_train_poly = polynomial_features.fit_transform(X_train)\n",
        "\n",
        "    model = LogisticRegression(max_iter=200)\n",
        "    pipeline = Pipeline([('polynomial_features', polynomial_features),\n",
        "                         ('scaler', StandardScaler()),\n",
        "                         ('logistic_regression', model)])\n",
        "\n",
        "    # Use cross-validation to evaluate the model\n",
        "    kfold = KFold(n_splits=5, shuffle=True, random_state=42)\n",
        "    cv_scores = cross_val_score(pipeline, X_train, y_train, cv=kfold, scoring='accuracy')\n",
        "\n",
        "    # Return the mean of the cross-validation scores\n",
        "    return np.mean(cv_scores)\n",
        "\n",
        "# Evaluate polynomial regression models with different degrees\n",
        "degrees = range(1, 6)\n",
        "cv_scores = [polynomial_regression(degree, X_train, y_train) for degree in degrees]\n",
        "\n",
        "# Plot the cross-validation scores for different polynomial degrees\n",
        "plt.figure(figsize=(10, 6))\n",
        "plt.plot(degrees, cv_scores, marker='o')\n",
        "plt.title('Cross-Validation Accuracy for Polynomial Regression')\n",
        "plt.xlabel('Polynomial Degree')\n",
        "plt.ylabel('Cross-Validation Accuracy')\n",
        "plt.show()\n",
        "\n",
        "# Determine the best degree\n",
        "best_degree = degrees[np.argmax(cv_scores)]\n",
        "print(f'The best polynomial degree is {best_degree}')\n"
      ],
      "metadata": {
        "id": "IK_V8w21zClN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Final Touches"
      ],
      "metadata": {
        "id": "ZCJ9N6k54rsK"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Scaling the Data & Adding Interaction Terms"
      ],
      "metadata": {
        "id": "QeJuHH3T4v1i"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Identify numeric columns to be scaled\n",
        "numeric_features = ['age', 'fare', 'family_size']\n",
        "\n",
        "# Define the preprocessor to scale numeric features\n",
        "preprocessor = ColumnTransformer(\n",
        "    transformers=[\n",
        "        ('num', StandardScaler(), numeric_features)\n",
        "    ])\n",
        "\n",
        "# Apply the preprocessor to scale the numeric features\n",
        "titanic_scaled = preprocessor.fit_transform(titanic[numeric_features])\n",
        "\n",
        "# Convert the scaled features back to a DataFrame\n",
        "titanic_scaled_df = pd.DataFrame(titanic_scaled, columns=numeric_features)\n",
        "\n",
        "# Compute the cubed age after scaling\n",
        "titanic_scaled_df['age^3'] = titanic_scaled_df['age'] ** 3\n",
        "\n",
        "# Create interaction terms using the scaled data\n",
        "titanic_scaled_df['age^3*pclass'] = titanic_scaled_df['age^3'] * titanic['pclass']\n",
        "titanic_scaled_df['fare*pclass'] = titanic_scaled_df['fare'] * titanic['pclass']\n",
        "titanic_scaled_df['age^3*sex'] = titanic_scaled_df['age^3'] * (titanic['sex'] == 'male').astype(int)\n",
        "titanic_scaled_df['sex*pclass'] = (titanic['sex'] == 'male').astype(int) * titanic['pclass']\n",
        "titanic_scaled_df['family_size*pclass'] = titanic_scaled_df['family_size'] * titanic['pclass']\n",
        "\n",
        "# Combine the scaled and interaction terms with other relevant features\n",
        "titanic_prepared = pd.concat([titanic[['pclass', 'sex', 'cabin']], titanic_scaled_df], axis=1)\n"
      ],
      "metadata": {
        "id": "2vdZ99ffr9rV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Feature Selection, Model Tuning, & Cross Validation"
      ],
      "metadata": {
        "id": "J64WY73c5HLH"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Define the preprocessor for categorical and numeric data\n",
        "full_preprocessor = ColumnTransformer(\n",
        "    transformers=[\n",
        "        ('cat', OneHotEncoder(handle_unknown='ignore'), ['pclass', 'sex', 'cabin']),\n",
        "        ('num', 'passthrough', titanic_scaled_df.columns.tolist())\n",
        "    ])\n",
        "\n",
        "# Define the pipeline with RFE and logistic regression\n",
        "pipeline = Pipeline(steps=[\n",
        "    ('preprocessor', full_preprocessor),\n",
        "    ('feature_selection', RFE(estimator=LogisticRegression(max_iter=500))),\n",
        "    ('classifier', LogisticRegression(max_iter=200))\n",
        "])\n",
        "\n",
        "# Split the data into features and target\n",
        "X = titanic_prepared\n",
        "y = titanic['survived'].astype(int)\n",
        "\n",
        "# Define the parameter grid for hyperparameter tuning\n",
        "param_grid = {\n",
        "    'feature_selection__n_features_to_select': [5,6,7,8,9,10],  # Number of top features to select\n",
        "    'classifier__C': [0.01, 0.1, 1, 10, 100],\n",
        "    'classifier__solver': ['liblinear', 'saga']\n",
        "}\n",
        "\n",
        "# Define the GridSearchCV object\n",
        "grid_search = GridSearchCV(pipeline, param_grid, cv=5, n_jobs=-1, scoring='accuracy')\n",
        "\n",
        "# Fit the GridSearchCV\n",
        "grid_search.fit(X, y)\n",
        "\n",
        "# Get the best parameters and best score\n",
        "best_params = grid_search.best_params_\n",
        "best_score = grid_search.best_score_\n",
        "\n",
        "print(\"Best Parameters:\", best_params)\n",
        "print(\"Best Cross-Validation Score:\", best_score)\n",
        "\n",
        "# Evaluate the best model on the test set\n",
        "best_model = grid_search.best_estimator_\n",
        "\n",
        "# Split the data into train and test sets for evaluation\n",
        "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)\n",
        "\n",
        "best_model.fit(X_train, y_train)\n",
        "y_pred = best_model.predict(X_test)\n",
        "\n",
        "print(\"Test Set Accuracy:\", accuracy_score(y_test, y_pred))\n",
        "print(\"Confusion Matrix:\\n\", confusion_matrix(y_test, y_pred))\n",
        "print(\"Classification Report:\\n\", classification_report(y_test, y_pred))\n",
        "\n",
        "# Plot the confusion matrix\n",
        "cm = confusion_matrix(y_test, y_pred)\n",
        "plt.figure(figsize=(6, 6))\n",
        "sns.heatmap(cm, annot=True, fmt='d', cmap='Blues')\n",
        "plt.title('Confusion Matrix')\n",
        "plt.xlabel('Predicted')\n",
        "plt.ylabel('Actual')\n",
        "plt.show()\n",
        "\n",
        "# Plot the ROC curve\n",
        "y_pred_proba = best_model.predict_proba(X_test)[:, 1]\n",
        "fpr, tpr, _ = roc_curve(y_test, y_pred_proba)\n",
        "roc_auc = roc_auc_score(y_test, y_pred_proba)\n",
        "\n",
        "plt.figure(figsize=(8, 6))\n",
        "plt.plot(fpr, tpr, color='blue', label=f'ROC Curve (area = {roc_auc:.2f})')\n",
        "plt.plot([0, 1], [0, 1], color='red', linestyle='--')\n",
        "plt.xlabel('False Positive Rate')\n",
        "plt.ylabel('True Positive Rate')\n",
        "plt.title('Receiver Operating Characteristic (ROC) Curve')\n",
        "plt.legend(loc='lower right')\n",
        "plt.show()\n",
        "\n",
        "# Get the feature selection step\n",
        "feature_selection = best_model.named_steps['feature_selection']\n",
        "\n",
        "# Get the mask of selected features\n",
        "selected_features_mask = feature_selection.support_\n",
        "\n",
        "# Get the original feature names\n",
        "ohe_feature_names = best_model.named_steps['preprocessor'].named_transformers_['cat'].get_feature_names_out(['pclass', 'sex', 'cabin']).tolist()\n",
        "num_feature_names = titanic_scaled_df.columns.tolist()\n",
        "\n",
        "# Combine feature names\n",
        "feature_names = ohe_feature_names + num_feature_names\n",
        "\n",
        "# Get the selected and excluded features\n",
        "selected_features = [feature for feature, selected in zip(feature_names, selected_features_mask) if selected]\n",
        "excluded_features = [feature for feature, selected in zip(feature_names, selected_features_mask) if not selected]\n",
        "\n",
        "print(\"Selected Features:\", selected_features)\n",
        "print(\"Excluded Features:\", excluded_features)\n"
      ],
      "metadata": {
        "id": "qpADZSTIr8Oy"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}