02-create-statsbomb-dataset.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "fa2e2a97-1e2a-4c3c-be51-8481986bee53",
   "metadata": {},
   "source": [
    "# Preparing a StatsBomb dataset\n",
    "\n",
    "This notebook prepares a dataset from the [StatsBomb Open Data](https://github.com/statsbomb/open-data) repository. Specifically, it performs the folowing tasks.\n",
    "\n",
    "1. Download the data.\n",
    "2. Convert the raw event stream data to the [SPADL data format](https://socceraction.readthedocs.io/en/latest/documentation/spadl/spadl.html#spadl).\n",
    "3. Compute a set of basic xG features.\n",
    "4. Store the result in a HDF file.\n",
    "\n",
    "**requirements**\n",
    "- `socceraction==1.5.0`\n",
    "- `soccerxg==1.0.0`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8072432b-e4e6-42a2-829c-fbab28dc55c3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "\n",
    "from socceraction.data.statsbomb import StatsBombLoader\n",
    "from socceraction import spadl\n",
    "from socceraction.spadl.statsbomb import convert_to_actions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e1ff586e-751b-4ccf-ba95-5e799b10cf12",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/cw/dtaijupiter/NoCsBack/dtai/pieterr/Projects/soccer_xg/.venv/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from soccer_xg.data import HDFDataset\n",
    "import soccer_xg.xg as xg\n",
    "import soccer_xg.attributes as fs"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "711ba7d6-ebed-424d-9fd0-a3aabbcac6dc",
   "metadata": {},
   "source": [
    "## Configuration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "53c3e8b8-e104-40ae-8e55-7f88f8a6fe06",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Where the data should be stored\n",
    "DATA_FP = Path(\"data\")\n",
    "\n",
    "# Which dataset to create (uncomment)\n",
    "# DATASET = \"spadl-statsbomb-bigfive-1516\"\n",
    "DATASET = \"spadl-statsbomb-messi-biography\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "7971054c-770e-4112-a985-4c37399259c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "(DATA_FP / DATASET).mkdir(parents=True, exist_ok=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e1fe4b09-1c39-4c6d-8748-f5df4e182843",
   "metadata": {},
   "source": [
    "\n",
    "## Set up the StatsBombLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "c0fccc49-bc9d-4ff9-ab73-15e7952397fd",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "SBL = StatsBombLoader(getter=\"remote\", creds={\"user\": None, \"passwd\": None})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "8a9a815d-5406-4c78-bd09-450abf97fb36",
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "# suppress warning about missing authentication while downloading public StatsBomb data\n",
    "from statsbombpy.api_client import NoAuthWarning\n",
    "warnings.simplefilter('ignore', NoAuthWarning)\n",
    "# surpress warnings regarding data version\n",
    "warnings.filterwarnings(\"ignore\", message=\".*fidelity.*\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "d7375f62-fc60-4178-8e5f-fedba1690973",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'1. Bundesliga',\n",
       " 'Champions League',\n",
       " 'Copa del Rey',\n",
       " \"FA Women's Super League\",\n",
       " 'FIFA U20 World Cup',\n",
       " 'FIFA World Cup',\n",
       " 'Indian Super league',\n",
       " 'La Liga',\n",
       " 'Liga Profesional',\n",
       " 'Ligue 1',\n",
       " 'NWSL',\n",
       " 'North American League',\n",
       " 'Premier League',\n",
       " 'Serie A',\n",
       " 'UEFA Euro',\n",
       " 'UEFA Europa League',\n",
       " \"UEFA Women's Euro\",\n",
       " \"Women's World Cup\"}"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# View all available competitions\n",
    "competitions = SBL.competitions()\n",
    "set(competitions.competition_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "0bed7459-bc71-4b46-8a56-4084b72744db",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>season_id</th>\n",
       "      <th>competition_id</th>\n",
       "      <th>competition_name</th>\n",
       "      <th>country_name</th>\n",
       "      <th>competition_gender</th>\n",
       "      <th>season_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>90</td>\n",
       "      <td>11</td>\n",
       "      <td>La Liga</td>\n",
       "      <td>Spain</td>\n",
       "      <td>male</td>\n",
       "      <td>2020/2021</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>42</td>\n",
       "      <td>11</td>\n",
       "      <td>La Liga</td>\n",
       "      <td>Spain</td>\n",
       "      <td>male</td>\n",
       "      <td>2019/2020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>4</td>\n",
       "      <td>11</td>\n",
       "      <td>La Liga</td>\n",
       "      <td>Spain</td>\n",
       "      <td>male</td>\n",
       "      <td>2018/2019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>La Liga</td>\n",
       "      <td>Spain</td>\n",
       "      <td>male</td>\n",
       "      <td>2017/2018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>La Liga</td>\n",
       "      <td>Spain</td>\n",
       "      <td>male</td>\n",
       "      <td>2016/2017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>27</td>\n",
       "      <td>11</td>\n",
       "      <td>La Liga</td>\n",
       "      <td>Spain</td>\n",
       "      <td>male</td>\n",
       "      <td>2015/2016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>26</td>\n",
       "      <td>11</td>\n",
       "      <td>La Liga</td>\n",
       "      <td>Spain</td>\n",
       "      <td>male</td>\n",
       "      <td>2014/2015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>25</td>\n",
       "      <td>11</td>\n",
       "      <td>La Liga</td>\n",
       "      <td>Spain</td>\n",
       "      <td>male</td>\n",
       "      <td>2013/2014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>24</td>\n",
       "      <td>11</td>\n",
       "      <td>La Liga</td>\n",
       "      <td>Spain</td>\n",
       "      <td>male</td>\n",
       "      <td>2012/2013</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>23</td>\n",
       "      <td>11</td>\n",
       "      <td>La Liga</td>\n",
       "      <td>Spain</td>\n",
       "      <td>male</td>\n",
       "      <td>2011/2012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>22</td>\n",
       "      <td>11</td>\n",
       "      <td>La Liga</td>\n",
       "      <td>Spain</td>\n",
       "      <td>male</td>\n",
       "      <td>2010/2011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>21</td>\n",
       "      <td>11</td>\n",
       "      <td>La Liga</td>\n",
       "      <td>Spain</td>\n",
       "      <td>male</td>\n",
       "      <td>2009/2010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>41</td>\n",
       "      <td>11</td>\n",
       "      <td>La Liga</td>\n",
       "      <td>Spain</td>\n",
       "      <td>male</td>\n",
       "      <td>2008/2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>40</td>\n",
       "      <td>11</td>\n",
       "      <td>La Liga</td>\n",
       "      <td>Spain</td>\n",
       "      <td>male</td>\n",
       "      <td>2007/2008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>39</td>\n",
       "      <td>11</td>\n",
       "      <td>La Liga</td>\n",
       "      <td>Spain</td>\n",
       "      <td>male</td>\n",
       "      <td>2006/2007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>38</td>\n",
       "      <td>11</td>\n",
       "      <td>La Liga</td>\n",
       "      <td>Spain</td>\n",
       "      <td>male</td>\n",
       "      <td>2005/2006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>37</td>\n",
       "      <td>11</td>\n",
       "      <td>La Liga</td>\n",
       "      <td>Spain</td>\n",
       "      <td>male</td>\n",
       "      <td>2004/2005</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    season_id  competition_id competition_name country_name  \\\n",
       "35         90              11          La Liga        Spain   \n",
       "36         42              11          La Liga        Spain   \n",
       "37          4              11          La Liga        Spain   \n",
       "38          1              11          La Liga        Spain   \n",
       "39          2              11          La Liga        Spain   \n",
       "40         27              11          La Liga        Spain   \n",
       "41         26              11          La Liga        Spain   \n",
       "42         25              11          La Liga        Spain   \n",
       "43         24              11          La Liga        Spain   \n",
       "44         23              11          La Liga        Spain   \n",
       "45         22              11          La Liga        Spain   \n",
       "46         21              11          La Liga        Spain   \n",
       "47         41              11          La Liga        Spain   \n",
       "48         40              11          La Liga        Spain   \n",
       "49         39              11          La Liga        Spain   \n",
       "50         38              11          La Liga        Spain   \n",
       "51         37              11          La Liga        Spain   \n",
       "\n",
       "   competition_gender season_name  \n",
       "35               male   2020/2021  \n",
       "36               male   2019/2020  \n",
       "37               male   2018/2019  \n",
       "38               male   2017/2018  \n",
       "39               male   2016/2017  \n",
       "40               male   2015/2016  \n",
       "41               male   2014/2015  \n",
       "42               male   2013/2014  \n",
       "43               male   2012/2013  \n",
       "44               male   2011/2012  \n",
       "45               male   2010/2011  \n",
       "46               male   2009/2010  \n",
       "47               male   2008/2009  \n",
       "48               male   2007/2008  \n",
       "49               male   2006/2007  \n",
       "50               male   2005/2006  \n",
       "51               male   2004/2005  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "if DATASET == \"spadl-statsbomb-bigfive-1516\":\n",
    "    # Bigfive 15-16\n",
    "    selected_competitions = competitions[\n",
    "        competitions.competition_name.isin(['1. Bundesliga', 'La Liga', 'Premier League', 'Serie A', 'Ligue 1'])\n",
    "        & (competitions.season_name == \"2015/2016\")\n",
    "    ]\n",
    "elif DATASET == \"spadl-statsbomb-messi-biography\":\n",
    "    # Messi data\n",
    "    selected_competitions = competitions[\n",
    "        (competitions.competition_name == \"La Liga\")\n",
    "        & (competitions.season_name.between(\"2004/2005\", \"2020/2021\"))\n",
    "    ]\n",
    "else:\n",
    "    raise ValueError(f\"The dataset {DATASET} is not recognized.\")\n",
    "    \n",
    "selected_competitions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "39ed7fbc-3a7a-4a9b-b045-0061c299321d",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>home_team_id</th>\n",
       "      <th>away_team_id</th>\n",
       "      <th>game_date</th>\n",
       "      <th>home_score</th>\n",
       "      <th>away_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>206</td>\n",
       "      <td>217</td>\n",
       "      <td>2020-10-31 21:00:00</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1049</td>\n",
       "      <td>217</td>\n",
       "      <td>2021-01-09 18:30:00</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>217</td>\n",
       "      <td>209</td>\n",
       "      <td>2021-05-16 18:30:00</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>218</td>\n",
       "      <td>217</td>\n",
       "      <td>2021-02-07 21:00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>422</td>\n",
       "      <td>217</td>\n",
       "      <td>2021-03-06 21:00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>217</td>\n",
       "      <td>608</td>\n",
       "      <td>2005-05-01 19:00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>217</td>\n",
       "      <td>221</td>\n",
       "      <td>2004-12-21 20:00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>608</td>\n",
       "      <td>217</td>\n",
       "      <td>2004-12-11 20:00:00</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>217</td>\n",
       "      <td>216</td>\n",
       "      <td>2005-04-17 21:00:00</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>214</td>\n",
       "      <td>217</td>\n",
       "      <td>2004-10-16 22:00:00</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>867 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    home_team_id  away_team_id           game_date  home_score  away_score\n",
       "0            206           217 2020-10-31 21:00:00           1           1\n",
       "1           1049           217 2021-01-09 18:30:00           0           4\n",
       "2            217           209 2021-05-16 18:30:00           1           2\n",
       "3            218           217 2021-02-07 21:00:00           2           3\n",
       "4            422           217 2021-03-06 21:00:00           0           2\n",
       "..           ...           ...                 ...         ...         ...\n",
       "2            217           608 2005-05-01 19:00:00           2           0\n",
       "3            217           221 2004-12-21 20:00:00           2           1\n",
       "4            608           217 2004-12-11 20:00:00           1           2\n",
       "5            217           216 2005-04-17 21:00:00           2           0\n",
       "6            214           217 2004-10-16 22:00:00           0           1\n",
       "\n",
       "[867 rows x 5 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get games from all selected competitions\n",
    "games = pd.concat([\n",
    "    SBL.games(row.competition_id, row.season_id)\n",
    "    for row in selected_competitions.itertuples()\n",
    "])\n",
    "games[[\"home_team_id\", \"away_team_id\", \"game_date\", \"home_score\", \"away_score\"]]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b046f705-341a-4bdb-96fc-278a730ab65d",
   "metadata": {},
   "source": [
    "## Load and convert match data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "e739042e-1c63-42e0-9fcc-ee8bbdb4be58",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Importing La Liga 2020/2021 ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading game data...: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 35/35 [01:35<00:00,  2.74s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Importing La Liga 2019/2020 ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading game data...: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 33/33 [01:27<00:00,  2.66s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Importing La Liga 2018/2019 ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading game data...: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 34/34 [01:36<00:00,  2.85s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Importing La Liga 2017/2018 ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading game data...: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 36/36 [01:36<00:00,  2.69s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Importing La Liga 2016/2017 ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading game data...: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 34/34 [01:30<00:00,  2.66s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Importing La Liga 2015/2016 ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading game data...: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 380/380 [15:29<00:00,  2.45s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Importing La Liga 2014/2015 ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading game data...: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 38/38 [01:35<00:00,  2.50s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Importing La Liga 2013/2014 ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading game data...: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 31/31 [01:21<00:00,  2.61s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Importing La Liga 2012/2013 ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading game data...: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 32/32 [01:23<00:00,  2.61s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Importing La Liga 2011/2012 ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading game data...: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 37/37 [01:38<00:00,  2.66s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Importing La Liga 2010/2011 ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading game data...: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 33/33 [01:27<00:00,  2.66s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Importing La Liga 2009/2010 ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading game data...: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 35/35 [01:32<00:00,  2.63s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Importing La Liga 2008/2009 ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading game data...: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 31/31 [01:17<00:00,  2.50s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Importing La Liga 2007/2008 ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading game data...: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 28/28 [01:12<00:00,  2.58s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Importing La Liga 2006/2007 ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading game data...: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 26/26 [01:06<00:00,  2.57s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Importing La Liga 2005/2006 ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading game data...: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 17/17 [00:44<00:00,  2.60s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Importing La Liga 2004/2005 ...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading game data...: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 7/7 [00:18<00:00,  2.61s/it]\n"
     ]
    }
   ],
   "source": [
    "# create a HDF dataset\n",
    "dataset = HDFDataset(\n",
    "    path=DATA_FP / DATASET / \"dataset.h5\", \n",
    "    mode=\"w\"\n",
    ")\n",
    "for _, comp in selected_competitions.iterrows():\n",
    "    # get name and id of competition\n",
    "    competition_name, competition_id = comp.competition_name, comp.competition_id\n",
    "    season_name, season_id = comp.season_name, comp.season_id\n",
    "    print(f\"Importing {competition_name} {season_name} ...\")\n",
    "    # import data\n",
    "    dataset.import_data(\n",
    "        SBL, \n",
    "        convert_to_actions, \n",
    "        competition_id, \n",
    "        season_id\n",
    "    )\n",
    "dataset.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "79545267-e799-4057-95cf-7b9deaf45fcc",
   "metadata": {},
   "source": [
    "## xG features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "fb430a00-4234-4f91-97e0-5260f0f32ca1",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = HDFDataset(\n",
    "    path=DATA_FP / DATASET / \"dataset.h5\", \n",
    "    mode=\"a\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "2d71d4c6-6687-4ceb-8825-89dd1eebcc25",
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_generators = [\n",
    "    fs.startlocation,\n",
    "    fs.shot_angle,\n",
    "    fs.shot_dist,\n",
    "    fs.shot_bodypart_onehot\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "e9a40d31-59b6-430d-8984-85d67354710a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>start_x_shot</th>\n",
       "      <th>start_y_shot</th>\n",
       "      <th>start_x_a1</th>\n",
       "      <th>start_y_a1</th>\n",
       "      <th>start_x_a2</th>\n",
       "      <th>start_y_a2</th>\n",
       "      <th>angle_shot</th>\n",
       "      <th>dist_shot</th>\n",
       "      <th>bodypart_foot_shot</th>\n",
       "      <th>bodypart_head_shot</th>\n",
       "      <th>bodypart_other_shot</th>\n",
       "      <th>bodypart_head/other_shot</th>\n",
       "      <th>bodypart_foot_left_shot</th>\n",
       "      <th>bodypart_foot_right_shot</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>action_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>151</th>\n",
       "      <td>94.54375</td>\n",
       "      <td>41.5225</td>\n",
       "      <td>102.94375</td>\n",
       "      <td>16.5325</td>\n",
       "      <td>102.94375</td>\n",
       "      <td>16.5325</td>\n",
       "      <td>0.623645</td>\n",
       "      <td>12.881039</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>207</th>\n",
       "      <td>96.99375</td>\n",
       "      <td>31.8325</td>\n",
       "      <td>79.75625</td>\n",
       "      <td>51.4675</td>\n",
       "      <td>79.75625</td>\n",
       "      <td>59.5425</td>\n",
       "      <td>0.264388</td>\n",
       "      <td>8.294462</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>240</th>\n",
       "      <td>103.11875</td>\n",
       "      <td>43.3075</td>\n",
       "      <td>101.63125</td>\n",
       "      <td>33.0225</td>\n",
       "      <td>92.18125</td>\n",
       "      <td>22.0575</td>\n",
       "      <td>1.371361</td>\n",
       "      <td>9.495718</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>360</th>\n",
       "      <td>89.03125</td>\n",
       "      <td>44.5825</td>\n",
       "      <td>81.06875</td>\n",
       "      <td>44.0725</td>\n",
       "      <td>81.06875</td>\n",
       "      <td>44.0725</td>\n",
       "      <td>0.585252</td>\n",
       "      <td>19.156990</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>431</th>\n",
       "      <td>95.59375</td>\n",
       "      <td>45.5175</td>\n",
       "      <td>91.21875</td>\n",
       "      <td>50.6175</td>\n",
       "      <td>77.65625</td>\n",
       "      <td>41.0975</td>\n",
       "      <td>0.885960</td>\n",
       "      <td>14.870452</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           start_x_shot  start_y_shot  start_x_a1  start_y_a1  start_x_a2  \\\n",
       "action_id                                                                   \n",
       "151            94.54375       41.5225   102.94375     16.5325   102.94375   \n",
       "207            96.99375       31.8325    79.75625     51.4675    79.75625   \n",
       "240           103.11875       43.3075   101.63125     33.0225    92.18125   \n",
       "360            89.03125       44.5825    81.06875     44.0725    81.06875   \n",
       "431            95.59375       45.5175    91.21875     50.6175    77.65625   \n",
       "\n",
       "           start_y_a2  angle_shot  dist_shot  bodypart_foot_shot  \\\n",
       "action_id                                                          \n",
       "151           16.5325    0.623645  12.881039                True   \n",
       "207           59.5425    0.264388   8.294462                True   \n",
       "240           22.0575    1.371361   9.495718                True   \n",
       "360           44.0725    0.585252  19.156990                True   \n",
       "431           41.0975    0.885960  14.870452                True   \n",
       "\n",
       "           bodypart_head_shot  bodypart_other_shot  bodypart_head/other_shot  \\\n",
       "action_id                                                                      \n",
       "151                     False                False                     False   \n",
       "207                     False                False                     False   \n",
       "240                     False                False                     False   \n",
       "360                     False                False                     False   \n",
       "431                     False                False                     False   \n",
       "\n",
       "           bodypart_foot_left_shot  bodypart_foot_right_shot  \n",
       "action_id                                                     \n",
       "151                           True                     False  \n",
       "207                          False                      True  \n",
       "240                           True                     False  \n",
       "360                           True                     False  \n",
       "431                          False                      True  "
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Generate the features and labels for a single game as a test\n",
    "GID = dataset.games().index[0]\n",
    "X, y = fs.compute_attributes(\n",
    "    game=dataset.games().loc[GID], \n",
    "    actions=dataset.actions(game_id=GID), \n",
    "    events=dataset.events(game_id=GID), \n",
    "    xfns=feature_generators\n",
    ")\n",
    "X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "6833845c-1603-4067-b901-f8eca38efa4f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Preparing dataset: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1823/1823 [09:10<00:00,  3.31it/s]\n"
     ]
    }
   ],
   "source": [
    "# Now do it for all games in the dataset and store them\n",
    "dataset[\"xg/features\"], dataset[\"xg/labels\"] = xg.prepare(dataset, xfns=feature_generators)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "8c40fc88-bb79-4fd2-8141-3c0b9494e7ee",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "87a74edb-04f2-4316-83cd-b6ec018b1ef3",
   "metadata": {},
   "source": [
    "## Extract shots\n",
    "\n",
    "We extract all shots (with their xG features and labels) from the dataset and store them in a Parquet file such that we can access them quickly."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "0c8b7535-5d2d-4c74-80d6-a949a2b09b9d",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = HDFDataset(\n",
    "    path=DATA_FP / DATASET / \"dataset.h5\", \n",
    "    mode=\"r\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "42057e1d-132d-4a04-af15-7d8378d41da6",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Loading shots: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 867/867 [00:45<00:00, 19.03it/s]\n"
     ]
    }
   ],
   "source": [
    "# Load all shots with xG features and labels.\n",
    "players = dataset.players().reset_index()[[\"player_id\", \"player_name\"]].drop_duplicates(subset=\"player_id\")\n",
    "teams = dataset.teams().reset_index().drop_duplicates(subset=\"team_id\")\n",
    "games = (\n",
    "    dataset.games().reset_index()\n",
    "    .merge(teams.add_prefix('home_'), how='left')\n",
    "    .merge(teams.add_prefix('away_'), how='left'))\n",
    "shots = []\n",
    "for game in tqdm(list(games.itertuples()), desc=\"Loading shots\"):\n",
    "    actions = dataset.actions(game.game_id).reset_index()\n",
    "    actions = spadl.utils.add_names(actions)\n",
    "    actions = spadl.utils.play_left_to_right(actions, game.home_team_id)\n",
    "    shots.append(\n",
    "        actions[actions.type_name.isin(['shot', 'shot_freekick', 'shot_penalty'])]\n",
    "        .merge(players, how=\"left\", on=\"player_id\")\n",
    "        .merge(teams, how=\"left\", on=\"team_id\")\n",
    "        .assign(season=game.season_id)\n",
    "        .assign(league=game.competition_id)\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "ee5eb045-deba-4b3c-8560-187a2de971a0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total shots: 21182\n",
      "Filtered shots: 19614\n"
     ]
    }
   ],
   "source": [
    "df_shots = pd.concat(shots).set_index([\"game_id\", \"action_id\"])\n",
    "print(f\"Total shots: {len(df_shots)}\")\n",
    "# simplify bodypart\n",
    "df_shots[\"bodypart_name_simple\"] = df_shots.bodypart_name.copy()\n",
    "df_shots.loc[df_shots.bodypart_name == \"foot_left\", \"bodypart_name_simple\"] = \"foot\"\n",
    "df_shots.loc[df_shots.bodypart_name == \"foot_right\", \"bodypart_name_simple\"] = \"foot\"\n",
    "# define filters\n",
    "owngoal = df_shots.result_name == 'owngoal'\n",
    "openplay = df_shots.type_name == 'shot'\n",
    "# apply filters\n",
    "df_shots_filt = df_shots.loc[~owngoal & openplay]\n",
    "print(f\"Filtered shots: {len(df_shots_filt)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "282fe9a1-9612-4f34-8166-ca668aaf4f35",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_shots_features = dataset[\"xg/features\"].loc[df_shots_filt.index]\n",
    "df_shots_labels = dataset[\"xg/labels\"].loc[df_shots_filt.index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "45ea8310-b89a-4097-ab9b-26fefd5945ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save datasets\n",
    "df_shots_filt.to_parquet(DATA_FP / DATASET / 'df_shots.parquet')\n",
    "df_shots_features.to_parquet(DATA_FP / DATASET / 'df_shots_features.parquet')\n",
    "df_shots_labels.to_parquet(DATA_FP / DATASET / 'df_shots_labels.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "98f34738-e077-4d90-8a15-86aabd63e306",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "/home/pieterr/Jupiter/Projects/soccer-xmskill",
   "language": "python",
   "name": "soccer-xmskill"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}