{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "8f597771",
   "metadata": {},
   "source": [
    "Player/match dataset creation\n",
    "\n",
    "For each serie A (already happened) match and player, a database entry is generated, with this form:\n",
    "    \n",
    "[y = player votes data, player data, team data, vs team data, opp team data, vs opp team data]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "3ecf3676",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a7d3a9d6",
   "metadata": {},
   "source": [
    "Load team data from FBref"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "8a6867a5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\nicol\\AppData\\Local\\Temp\\ipykernel_17612\\552672520.py:19: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  team_data['team_idx'][i] = team_data['team_idx'][i].split(' ')[-1]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>team</th>\n",
       "      <th>team_players_used</th>\n",
       "      <th>team_possession</th>\n",
       "      <th>team_games</th>\n",
       "      <th>team_games_starts</th>\n",
       "      <th>team_minutes</th>\n",
       "      <th>team_goals</th>\n",
       "      <th>team_assists</th>\n",
       "      <th>team_pens_made</th>\n",
       "      <th>team_pens_att</th>\n",
       "      <th>...</th>\n",
       "      <th>vs_team_fouls</th>\n",
       "      <th>vs_team_fouled</th>\n",
       "      <th>vs_team_offsides</th>\n",
       "      <th>vs_team_pens_won</th>\n",
       "      <th>vs_team_pens_conceded</th>\n",
       "      <th>vs_team_own_goals</th>\n",
       "      <th>vs_team_ball_recoveries</th>\n",
       "      <th>vs_team_aerials_won</th>\n",
       "      <th>vs_team_aerials_lost</th>\n",
       "      <th>vs_team_aerials_won_pct</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>team_idx</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Atalanta</th>\n",
       "      <td>Atalanta</td>\n",
       "      <td>23.0</td>\n",
       "      <td>46.4</td>\n",
       "      <td>14.0</td>\n",
       "      <td>154.0</td>\n",
       "      <td>1260.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>...</td>\n",
       "      <td>150.0</td>\n",
       "      <td>168.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>834.0</td>\n",
       "      <td>181.0</td>\n",
       "      <td>211.0</td>\n",
       "      <td>46.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Bologna</th>\n",
       "      <td>Bologna</td>\n",
       "      <td>23.0</td>\n",
       "      <td>51.1</td>\n",
       "      <td>14.0</td>\n",
       "      <td>154.0</td>\n",
       "      <td>1260.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>182.0</td>\n",
       "      <td>158.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>731.0</td>\n",
       "      <td>165.0</td>\n",
       "      <td>123.0</td>\n",
       "      <td>57.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Cremonese</th>\n",
       "      <td>Cremonese</td>\n",
       "      <td>27.0</td>\n",
       "      <td>43.2</td>\n",
       "      <td>14.0</td>\n",
       "      <td>154.0</td>\n",
       "      <td>1260.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>145.0</td>\n",
       "      <td>188.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>744.0</td>\n",
       "      <td>242.0</td>\n",
       "      <td>184.0</td>\n",
       "      <td>56.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Empoli</th>\n",
       "      <td>Empoli</td>\n",
       "      <td>25.0</td>\n",
       "      <td>46.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>154.0</td>\n",
       "      <td>1260.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>176.0</td>\n",
       "      <td>151.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>723.0</td>\n",
       "      <td>163.0</td>\n",
       "      <td>135.0</td>\n",
       "      <td>54.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Fiorentina</th>\n",
       "      <td>Fiorentina</td>\n",
       "      <td>25.0</td>\n",
       "      <td>58.7</td>\n",
       "      <td>14.0</td>\n",
       "      <td>154.0</td>\n",
       "      <td>1260.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>186.0</td>\n",
       "      <td>176.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>694.0</td>\n",
       "      <td>193.0</td>\n",
       "      <td>224.0</td>\n",
       "      <td>46.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Verona</th>\n",
       "      <td>Hellas Verona</td>\n",
       "      <td>29.0</td>\n",
       "      <td>43.9</td>\n",
       "      <td>14.0</td>\n",
       "      <td>154.0</td>\n",
       "      <td>1260.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>130.0</td>\n",
       "      <td>205.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>758.0</td>\n",
       "      <td>240.0</td>\n",
       "      <td>229.0</td>\n",
       "      <td>51.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Inter</th>\n",
       "      <td>Inter</td>\n",
       "      <td>23.0</td>\n",
       "      <td>54.6</td>\n",
       "      <td>14.0</td>\n",
       "      <td>154.0</td>\n",
       "      <td>1260.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>22.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>...</td>\n",
       "      <td>182.0</td>\n",
       "      <td>154.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>596.0</td>\n",
       "      <td>126.0</td>\n",
       "      <td>172.0</td>\n",
       "      <td>42.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Juventus</th>\n",
       "      <td>Juventus</td>\n",
       "      <td>26.0</td>\n",
       "      <td>50.5</td>\n",
       "      <td>14.0</td>\n",
       "      <td>154.0</td>\n",
       "      <td>1260.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>...</td>\n",
       "      <td>161.0</td>\n",
       "      <td>156.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>674.0</td>\n",
       "      <td>184.0</td>\n",
       "      <td>166.0</td>\n",
       "      <td>52.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Lazio</th>\n",
       "      <td>Lazio</td>\n",
       "      <td>21.0</td>\n",
       "      <td>50.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>154.0</td>\n",
       "      <td>1260.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>...</td>\n",
       "      <td>197.0</td>\n",
       "      <td>124.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>746.0</td>\n",
       "      <td>151.0</td>\n",
       "      <td>126.0</td>\n",
       "      <td>54.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Lecce</th>\n",
       "      <td>Lecce</td>\n",
       "      <td>24.0</td>\n",
       "      <td>40.7</td>\n",
       "      <td>14.0</td>\n",
       "      <td>154.0</td>\n",
       "      <td>1260.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>...</td>\n",
       "      <td>184.0</td>\n",
       "      <td>184.0</td>\n",
       "      <td>27.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>703.0</td>\n",
       "      <td>216.0</td>\n",
       "      <td>188.0</td>\n",
       "      <td>53.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Milan</th>\n",
       "      <td>Milan</td>\n",
       "      <td>27.0</td>\n",
       "      <td>53.9</td>\n",
       "      <td>14.0</td>\n",
       "      <td>154.0</td>\n",
       "      <td>1260.0</td>\n",
       "      <td>26.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>...</td>\n",
       "      <td>164.0</td>\n",
       "      <td>161.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>690.0</td>\n",
       "      <td>168.0</td>\n",
       "      <td>205.0</td>\n",
       "      <td>45.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Monza</th>\n",
       "      <td>Monza</td>\n",
       "      <td>29.0</td>\n",
       "      <td>55.5</td>\n",
       "      <td>14.0</td>\n",
       "      <td>154.0</td>\n",
       "      <td>1260.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>207.0</td>\n",
       "      <td>178.0</td>\n",
       "      <td>24.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>692.0</td>\n",
       "      <td>141.0</td>\n",
       "      <td>160.0</td>\n",
       "      <td>46.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Napoli</th>\n",
       "      <td>Napoli</td>\n",
       "      <td>24.0</td>\n",
       "      <td>59.8</td>\n",
       "      <td>14.0</td>\n",
       "      <td>154.0</td>\n",
       "      <td>1260.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>28.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>195.0</td>\n",
       "      <td>116.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>687.0</td>\n",
       "      <td>142.0</td>\n",
       "      <td>181.0</td>\n",
       "      <td>44.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Roma</th>\n",
       "      <td>Roma</td>\n",
       "      <td>23.0</td>\n",
       "      <td>50.8</td>\n",
       "      <td>14.0</td>\n",
       "      <td>154.0</td>\n",
       "      <td>1260.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>11.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>198.0</td>\n",
       "      <td>157.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>709.0</td>\n",
       "      <td>142.0</td>\n",
       "      <td>159.0</td>\n",
       "      <td>47.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Salernitana</th>\n",
       "      <td>Salernitana</td>\n",
       "      <td>24.0</td>\n",
       "      <td>47.1</td>\n",
       "      <td>14.0</td>\n",
       "      <td>154.0</td>\n",
       "      <td>1260.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>150.0</td>\n",
       "      <td>157.0</td>\n",
       "      <td>34.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>750.0</td>\n",
       "      <td>169.0</td>\n",
       "      <td>175.0</td>\n",
       "      <td>49.1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sampdoria</th>\n",
       "      <td>Sampdoria</td>\n",
       "      <td>24.0</td>\n",
       "      <td>50.8</td>\n",
       "      <td>14.0</td>\n",
       "      <td>154.0</td>\n",
       "      <td>1260.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>220.0</td>\n",
       "      <td>192.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>742.0</td>\n",
       "      <td>215.0</td>\n",
       "      <td>217.0</td>\n",
       "      <td>49.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sassuolo</th>\n",
       "      <td>Sassuolo</td>\n",
       "      <td>26.0</td>\n",
       "      <td>48.1</td>\n",
       "      <td>14.0</td>\n",
       "      <td>154.0</td>\n",
       "      <td>1260.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>...</td>\n",
       "      <td>177.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>47.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>710.0</td>\n",
       "      <td>146.0</td>\n",
       "      <td>130.0</td>\n",
       "      <td>52.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Spezia</th>\n",
       "      <td>Spezia</td>\n",
       "      <td>24.0</td>\n",
       "      <td>46.9</td>\n",
       "      <td>14.0</td>\n",
       "      <td>154.0</td>\n",
       "      <td>1260.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>136.0</td>\n",
       "      <td>177.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>776.0</td>\n",
       "      <td>212.0</td>\n",
       "      <td>179.0</td>\n",
       "      <td>54.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Torino</th>\n",
       "      <td>Torino</td>\n",
       "      <td>23.0</td>\n",
       "      <td>51.3</td>\n",
       "      <td>14.0</td>\n",
       "      <td>154.0</td>\n",
       "      <td>1260.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>148.0</td>\n",
       "      <td>199.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>710.0</td>\n",
       "      <td>228.0</td>\n",
       "      <td>215.0</td>\n",
       "      <td>51.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Udinese</th>\n",
       "      <td>Udinese</td>\n",
       "      <td>22.0</td>\n",
       "      <td>50.6</td>\n",
       "      <td>14.0</td>\n",
       "      <td>154.0</td>\n",
       "      <td>1260.0</td>\n",
       "      <td>21.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>176.0</td>\n",
       "      <td>162.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>707.0</td>\n",
       "      <td>142.0</td>\n",
       "      <td>187.0</td>\n",
       "      <td>43.2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20 rows × 313 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                      team  team_players_used  team_possession  team_games  \\\n",
       "team_idx                                                                     \n",
       "Atalanta          Atalanta               23.0             46.4        14.0   \n",
       "Bologna            Bologna               23.0             51.1        14.0   \n",
       "Cremonese        Cremonese               27.0             43.2        14.0   \n",
       "Empoli              Empoli               25.0             46.0        14.0   \n",
       "Fiorentina      Fiorentina               25.0             58.7        14.0   \n",
       "Verona       Hellas Verona               29.0             43.9        14.0   \n",
       "Inter                Inter               23.0             54.6        14.0   \n",
       "Juventus          Juventus               26.0             50.5        14.0   \n",
       "Lazio                Lazio               21.0             50.0        14.0   \n",
       "Lecce                Lecce               24.0             40.7        14.0   \n",
       "Milan                Milan               27.0             53.9        14.0   \n",
       "Monza                Monza               29.0             55.5        14.0   \n",
       "Napoli              Napoli               24.0             59.8        14.0   \n",
       "Roma                  Roma               23.0             50.8        14.0   \n",
       "Salernitana    Salernitana               24.0             47.1        14.0   \n",
       "Sampdoria        Sampdoria               24.0             50.8        14.0   \n",
       "Sassuolo          Sassuolo               26.0             48.1        14.0   \n",
       "Spezia              Spezia               24.0             46.9        14.0   \n",
       "Torino              Torino               23.0             51.3        14.0   \n",
       "Udinese            Udinese               22.0             50.6        14.0   \n",
       "\n",
       "             team_games_starts  team_minutes  team_goals  team_assists  \\\n",
       "team_idx                                                                 \n",
       "Atalanta                 154.0        1260.0        19.0          14.0   \n",
       "Bologna                  154.0        1260.0        17.0          12.0   \n",
       "Cremonese                154.0        1260.0        11.0           5.0   \n",
       "Empoli                   154.0        1260.0        10.0           5.0   \n",
       "Fiorentina               154.0        1260.0        17.0          15.0   \n",
       "Verona                   154.0        1260.0        10.0           7.0   \n",
       "Inter                    154.0        1260.0        31.0          22.0   \n",
       "Juventus                 154.0        1260.0        21.0          17.0   \n",
       "Lazio                    154.0        1260.0        25.0          20.0   \n",
       "Lecce                    154.0        1260.0        12.0           8.0   \n",
       "Milan                    154.0        1260.0        26.0          21.0   \n",
       "Monza                    154.0        1260.0        13.0           8.0   \n",
       "Napoli                   154.0        1260.0        34.0          28.0   \n",
       "Roma                     154.0        1260.0        17.0          11.0   \n",
       "Salernitana              154.0        1260.0        18.0          12.0   \n",
       "Sampdoria                154.0        1260.0         6.0           6.0   \n",
       "Sassuolo                 154.0        1260.0        15.0          12.0   \n",
       "Spezia                   154.0        1260.0        10.0           6.0   \n",
       "Torino                   154.0        1260.0        15.0          10.0   \n",
       "Udinese                  154.0        1260.0        21.0          19.0   \n",
       "\n",
       "             team_pens_made  team_pens_att  ...  vs_team_fouls  \\\n",
       "team_idx                                    ...                  \n",
       "Atalanta                4.0            5.0  ...          150.0   \n",
       "Bologna                 3.0            3.0  ...          182.0   \n",
       "Cremonese               1.0            3.0  ...          145.0   \n",
       "Empoli                  0.0            0.0  ...          176.0   \n",
       "Fiorentina              1.0            3.0  ...          186.0   \n",
       "Verona                  0.0            0.0  ...          130.0   \n",
       "Inter                   2.0            2.0  ...          182.0   \n",
       "Juventus                1.0            2.0  ...          161.0   \n",
       "Lazio                   1.0            2.0  ...          197.0   \n",
       "Lecce                   1.0            2.0  ...          184.0   \n",
       "Milan                   2.0            2.0  ...          164.0   \n",
       "Monza                   1.0            1.0  ...          207.0   \n",
       "Napoli                  3.0            3.0  ...          195.0   \n",
       "Roma                    2.0            3.0  ...          198.0   \n",
       "Salernitana             1.0            1.0  ...          150.0   \n",
       "Sampdoria               0.0            0.0  ...          220.0   \n",
       "Sassuolo                1.0            2.0  ...          177.0   \n",
       "Spezia                  1.0            1.0  ...          136.0   \n",
       "Torino                  1.0            1.0  ...          148.0   \n",
       "Udinese                 0.0            0.0  ...          176.0   \n",
       "\n",
       "             vs_team_fouled  vs_team_offsides  vs_team_pens_won  \\\n",
       "team_idx                                                          \n",
       "Atalanta              168.0              19.0               0.0   \n",
       "Bologna               158.0              24.0               2.0   \n",
       "Cremonese             188.0              19.0               2.0   \n",
       "Empoli                151.0              28.0               2.0   \n",
       "Fiorentina            176.0              35.0               1.0   \n",
       "Verona                205.0              18.0               1.0   \n",
       "Inter                 154.0              14.0               1.0   \n",
       "Juventus              156.0              21.0               0.0   \n",
       "Lazio                 124.0              35.0               1.0   \n",
       "Lecce                 184.0              27.0               3.0   \n",
       "Milan                 161.0              15.0               2.0   \n",
       "Monza                 178.0              24.0               0.0   \n",
       "Napoli                116.0              18.0               1.0   \n",
       "Roma                  157.0               7.0               0.0   \n",
       "Salernitana           157.0              34.0               4.0   \n",
       "Sampdoria             192.0              42.0               1.0   \n",
       "Sassuolo              120.0              47.0               2.0   \n",
       "Spezia                177.0              32.0               1.0   \n",
       "Torino                199.0              17.0               2.0   \n",
       "Udinese               162.0              17.0               2.0   \n",
       "\n",
       "             vs_team_pens_conceded  vs_team_own_goals  \\\n",
       "team_idx                                                \n",
       "Atalanta                       5.0                1.0   \n",
       "Bologna                        3.0                0.0   \n",
       "Cremonese                      3.0                0.0   \n",
       "Empoli                         0.0                0.0   \n",
       "Fiorentina                     3.0                0.0   \n",
       "Verona                         0.0                1.0   \n",
       "Inter                          2.0                0.0   \n",
       "Juventus                       2.0                0.0   \n",
       "Lazio                          2.0                1.0   \n",
       "Lecce                          2.0                0.0   \n",
       "Milan                          2.0                1.0   \n",
       "Monza                          1.0                0.0   \n",
       "Napoli                         3.0                0.0   \n",
       "Roma                           3.0                0.0   \n",
       "Salernitana                    1.0                1.0   \n",
       "Sampdoria                      0.0                0.0   \n",
       "Sassuolo                       2.0                0.0   \n",
       "Spezia                         1.0                2.0   \n",
       "Torino                         1.0                0.0   \n",
       "Udinese                        0.0                1.0   \n",
       "\n",
       "             vs_team_ball_recoveries  vs_team_aerials_won  \\\n",
       "team_idx                                                    \n",
       "Atalanta                       834.0                181.0   \n",
       "Bologna                        731.0                165.0   \n",
       "Cremonese                      744.0                242.0   \n",
       "Empoli                         723.0                163.0   \n",
       "Fiorentina                     694.0                193.0   \n",
       "Verona                         758.0                240.0   \n",
       "Inter                          596.0                126.0   \n",
       "Juventus                       674.0                184.0   \n",
       "Lazio                          746.0                151.0   \n",
       "Lecce                          703.0                216.0   \n",
       "Milan                          690.0                168.0   \n",
       "Monza                          692.0                141.0   \n",
       "Napoli                         687.0                142.0   \n",
       "Roma                           709.0                142.0   \n",
       "Salernitana                    750.0                169.0   \n",
       "Sampdoria                      742.0                215.0   \n",
       "Sassuolo                       710.0                146.0   \n",
       "Spezia                         776.0                212.0   \n",
       "Torino                         710.0                228.0   \n",
       "Udinese                        707.0                142.0   \n",
       "\n",
       "             vs_team_aerials_lost  vs_team_aerials_won_pct  \n",
       "team_idx                                                    \n",
       "Atalanta                    211.0                     46.2  \n",
       "Bologna                     123.0                     57.3  \n",
       "Cremonese                   184.0                     56.8  \n",
       "Empoli                      135.0                     54.7  \n",
       "Fiorentina                  224.0                     46.3  \n",
       "Verona                      229.0                     51.2  \n",
       "Inter                       172.0                     42.3  \n",
       "Juventus                    166.0                     52.6  \n",
       "Lazio                       126.0                     54.5  \n",
       "Lecce                       188.0                     53.5  \n",
       "Milan                       205.0                     45.0  \n",
       "Monza                       160.0                     46.8  \n",
       "Napoli                      181.0                     44.0  \n",
       "Roma                        159.0                     47.2  \n",
       "Salernitana                 175.0                     49.1  \n",
       "Sampdoria                   217.0                     49.8  \n",
       "Sassuolo                    130.0                     52.9  \n",
       "Spezia                      179.0                     54.2  \n",
       "Torino                      215.0                     51.5  \n",
       "Udinese                     187.0                     43.2  \n",
       "\n",
       "[20 rows x 313 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rcsv = pd.read_csv('fbref_data/teams.csv')   \n",
    "team = pd.DataFrame(rcsv)\n",
    "\n",
    "rcsv = pd.read_csv('fbref_data/teams_vs.csv')   \n",
    "vs_team = pd.DataFrame(rcsv)\n",
    "\n",
    "for i in range(1, team.shape[1]):\n",
    "    team = team.rename(columns = {team.columns[i] : 'team_' + team.columns[i]})\n",
    "    \n",
    "for i in range(1, vs_team.shape[1]):\n",
    "    vs_team = vs_team.rename(columns = {vs_team.columns[i] : 'vs_team_' + vs_team.columns[i]})\n",
    "    \n",
    "vs_team.pop('team')\n",
    "\n",
    "team_data = pd.concat([team, vs_team], axis = 1)\n",
    "\n",
    "team_data['team_idx'] = team_data['team']\n",
    "for i in range(team_data.shape[0]):\n",
    "    team_data['team_idx'][i] = team_data['team_idx'][i].split(' ')[-1]\n",
    "\n",
    "team_data = team_data.set_index('team_idx')\n",
    "\n",
    "team_data\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "b6a3db98",
   "metadata": {},
   "outputs": [],
   "source": [
    "team_data.to_excel('mid_outputs/team_data.xlsx')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cc50dde9",
   "metadata": {},
   "source": [
    "Load data generated from players_dataset_creation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "c34eb398",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>id</th>\n",
       "      <th>r</th>\n",
       "      <th>team</th>\n",
       "      <th>surname</th>\n",
       "      <th>initial</th>\n",
       "      <th>fb_ID</th>\n",
       "      <th>age</th>\n",
       "      <th>birth_year</th>\n",
       "      <th>games</th>\n",
       "      <th>...</th>\n",
       "      <th>gk_pct_goal_kicks_launched</th>\n",
       "      <th>gk_goal_kick_length_avg</th>\n",
       "      <th>gk_crosses</th>\n",
       "      <th>gk_crosses_stopped</th>\n",
       "      <th>gk_crosses_stopped_pct</th>\n",
       "      <th>gk_def_actions_outside_pen_area</th>\n",
       "      <th>gk_def_actions_outside_pen_area_per90</th>\n",
       "      <th>gk_avg_distance_def_actions</th>\n",
       "      <th>vote_avg</th>\n",
       "      <th>vote_std</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>name</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Meret</th>\n",
       "      <td>0</td>\n",
       "      <td>572</td>\n",
       "      <td>P</td>\n",
       "      <td>Napoli</td>\n",
       "      <td>Meret</td>\n",
       "      <td>NaN</td>\n",
       "      <td>502</td>\n",
       "      <td>25-234</td>\n",
       "      <td>1997</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>19.6</td>\n",
       "      <td>26.6</td>\n",
       "      <td>142.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>4.2</td>\n",
       "      <td>10.0</td>\n",
       "      <td>0.71</td>\n",
       "      <td>15.1</td>\n",
       "      <td>6.214286</td>\n",
       "      <td>0.557875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Provedel</th>\n",
       "      <td>1</td>\n",
       "      <td>2814</td>\n",
       "      <td>P</td>\n",
       "      <td>Lazio</td>\n",
       "      <td>Provedel</td>\n",
       "      <td>NaN</td>\n",
       "      <td>509</td>\n",
       "      <td>28-239</td>\n",
       "      <td>1994</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>37.2</td>\n",
       "      <td>36.6</td>\n",
       "      <td>201.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2.5</td>\n",
       "      <td>29.0</td>\n",
       "      <td>2.08</td>\n",
       "      <td>17.6</td>\n",
       "      <td>6.321429</td>\n",
       "      <td>0.305143</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Maignan</th>\n",
       "      <td>2</td>\n",
       "      <td>4312</td>\n",
       "      <td>P</td>\n",
       "      <td>Milan</td>\n",
       "      <td>Maignan</td>\n",
       "      <td>NaN</td>\n",
       "      <td>500</td>\n",
       "      <td>27-131</td>\n",
       "      <td>1995</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>38.9</td>\n",
       "      <td>40.4</td>\n",
       "      <td>65.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>7.7</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.86</td>\n",
       "      <td>12.9</td>\n",
       "      <td>6.357143</td>\n",
       "      <td>0.440315</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Silvestri</th>\n",
       "      <td>3</td>\n",
       "      <td>2211</td>\n",
       "      <td>P</td>\n",
       "      <td>Udinese</td>\n",
       "      <td>Silvestri</td>\n",
       "      <td>NaN</td>\n",
       "      <td>512</td>\n",
       "      <td>31-254</td>\n",
       "      <td>1991</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>40.9</td>\n",
       "      <td>38.2</td>\n",
       "      <td>194.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2.1</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.36</td>\n",
       "      <td>11.3</td>\n",
       "      <td>6.321429</td>\n",
       "      <td>0.485767</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sepe</th>\n",
       "      <td>4</td>\n",
       "      <td>159</td>\n",
       "      <td>P</td>\n",
       "      <td>Salernitana</td>\n",
       "      <td>Sepe</td>\n",
       "      <td>NaN</td>\n",
       "      <td>511</td>\n",
       "      <td>31-187</td>\n",
       "      <td>1991</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>48.4</td>\n",
       "      <td>41.7</td>\n",
       "      <td>187.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>7.5</td>\n",
       "      <td>9.0</td>\n",
       "      <td>0.64</td>\n",
       "      <td>13.7</td>\n",
       "      <td>6.428571</td>\n",
       "      <td>0.371154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>De Luca</th>\n",
       "      <td>549</td>\n",
       "      <td>5512</td>\n",
       "      <td>A</td>\n",
       "      <td>Sampdoria</td>\n",
       "      <td>Luca</td>\n",
       "      <td>NaN</td>\n",
       "      <td>122</td>\n",
       "      <td>24-117</td>\n",
       "      <td>1998</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.877688</td>\n",
       "      <td>0.485814</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Soule'</th>\n",
       "      <td>550</td>\n",
       "      <td>5734</td>\n",
       "      <td>A</td>\n",
       "      <td>Juventus</td>\n",
       "      <td>Soule</td>\n",
       "      <td>NaN</td>\n",
       "      <td>422</td>\n",
       "      <td>19-210</td>\n",
       "      <td>2003</td>\n",
       "      <td>5</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.067846</td>\n",
       "      <td>0.581002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Lazetic</th>\n",
       "      <td>551</td>\n",
       "      <td>5785</td>\n",
       "      <td>A</td>\n",
       "      <td>Milan</td>\n",
       "      <td>Lazetic</td>\n",
       "      <td>NaN</td>\n",
       "      <td>248</td>\n",
       "      <td>18-293</td>\n",
       "      <td>2004</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.170097</td>\n",
       "      <td>0.204814</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Voelkerling Persson</th>\n",
       "      <td>552</td>\n",
       "      <td>5837</td>\n",
       "      <td>A</td>\n",
       "      <td>Lecce</td>\n",
       "      <td>Persson</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.041983</td>\n",
       "      <td>0.689071</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Sanca</th>\n",
       "      <td>553</td>\n",
       "      <td>6063</td>\n",
       "      <td>A</td>\n",
       "      <td>Spezia</td>\n",
       "      <td>Sanca</td>\n",
       "      <td>NaN</td>\n",
       "      <td>395</td>\n",
       "      <td>22-311</td>\n",
       "      <td>2000</td>\n",
       "      <td>3</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.991021</td>\n",
       "      <td>0.262247</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>554 rows × 165 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                     Unnamed: 0    id  r         team    surname initial  \\\n",
       "name                                                                       \n",
       "Meret                         0   572  P       Napoli      Meret     NaN   \n",
       "Provedel                      1  2814  P        Lazio   Provedel     NaN   \n",
       "Maignan                       2  4312  P        Milan    Maignan     NaN   \n",
       "Silvestri                     3  2211  P      Udinese  Silvestri     NaN   \n",
       "Sepe                          4   159  P  Salernitana       Sepe     NaN   \n",
       "...                         ...   ... ..          ...        ...     ...   \n",
       "De Luca                     549  5512  A    Sampdoria       Luca     NaN   \n",
       "Soule'                      550  5734  A     Juventus      Soule     NaN   \n",
       "Lazetic                     551  5785  A        Milan    Lazetic     NaN   \n",
       "Voelkerling Persson         552  5837  A        Lecce    Persson     NaN   \n",
       "Sanca                       553  6063  A       Spezia      Sanca     NaN   \n",
       "\n",
       "                     fb_ID     age  birth_year  games  ...  \\\n",
       "name                                                   ...   \n",
       "Meret                  502  25-234        1997      0  ...   \n",
       "Provedel               509  28-239        1994      0  ...   \n",
       "Maignan                500  27-131        1995      0  ...   \n",
       "Silvestri              512  31-254        1991      0  ...   \n",
       "Sepe                   511  31-187        1991      0  ...   \n",
       "...                    ...     ...         ...    ...  ...   \n",
       "De Luca                122  24-117        1998      1  ...   \n",
       "Soule'                 422  19-210        2003      5  ...   \n",
       "Lazetic                248  18-293        2004      1  ...   \n",
       "Voelkerling Persson     -1       0           0      0  ...   \n",
       "Sanca                  395  22-311        2000      3  ...   \n",
       "\n",
       "                     gk_pct_goal_kicks_launched  gk_goal_kick_length_avg  \\\n",
       "name                                                                       \n",
       "Meret                                      19.6                     26.6   \n",
       "Provedel                                   37.2                     36.6   \n",
       "Maignan                                    38.9                     40.4   \n",
       "Silvestri                                  40.9                     38.2   \n",
       "Sepe                                       48.4                     41.7   \n",
       "...                                         ...                      ...   \n",
       "De Luca                                     0.0                      0.0   \n",
       "Soule'                                      0.0                      0.0   \n",
       "Lazetic                                     0.0                      0.0   \n",
       "Voelkerling Persson                         0.0                      0.0   \n",
       "Sanca                                       0.0                      0.0   \n",
       "\n",
       "                     gk_crosses  gk_crosses_stopped  gk_crosses_stopped_pct  \\\n",
       "name                                                                          \n",
       "Meret                     142.0                 6.0                     4.2   \n",
       "Provedel                  201.0                 5.0                     2.5   \n",
       "Maignan                    65.0                 5.0                     7.7   \n",
       "Silvestri                 194.0                 4.0                     2.1   \n",
       "Sepe                      187.0                14.0                     7.5   \n",
       "...                         ...                 ...                     ...   \n",
       "De Luca                     0.0                 0.0                     0.0   \n",
       "Soule'                      0.0                 0.0                     0.0   \n",
       "Lazetic                     0.0                 0.0                     0.0   \n",
       "Voelkerling Persson         0.0                 0.0                     0.0   \n",
       "Sanca                       0.0                 0.0                     0.0   \n",
       "\n",
       "                     gk_def_actions_outside_pen_area  \\\n",
       "name                                                   \n",
       "Meret                                           10.0   \n",
       "Provedel                                        29.0   \n",
       "Maignan                                          6.0   \n",
       "Silvestri                                        5.0   \n",
       "Sepe                                             9.0   \n",
       "...                                              ...   \n",
       "De Luca                                          0.0   \n",
       "Soule'                                           0.0   \n",
       "Lazetic                                          0.0   \n",
       "Voelkerling Persson                              0.0   \n",
       "Sanca                                            0.0   \n",
       "\n",
       "                     gk_def_actions_outside_pen_area_per90  \\\n",
       "name                                                         \n",
       "Meret                                                 0.71   \n",
       "Provedel                                              2.08   \n",
       "Maignan                                               0.86   \n",
       "Silvestri                                             0.36   \n",
       "Sepe                                                  0.64   \n",
       "...                                                    ...   \n",
       "De Luca                                               0.00   \n",
       "Soule'                                                0.00   \n",
       "Lazetic                                               0.00   \n",
       "Voelkerling Persson                                   0.00   \n",
       "Sanca                                                 0.00   \n",
       "\n",
       "                     gk_avg_distance_def_actions  vote_avg  vote_std  \n",
       "name                                                                  \n",
       "Meret                                       15.1  6.214286  0.557875  \n",
       "Provedel                                    17.6  6.321429  0.305143  \n",
       "Maignan                                     12.9  6.357143  0.440315  \n",
       "Silvestri                                   11.3  6.321429  0.485767  \n",
       "Sepe                                        13.7  6.428571  0.371154  \n",
       "...                                          ...       ...       ...  \n",
       "De Luca                                      0.0  5.877688  0.485814  \n",
       "Soule'                                       0.0  6.067846  0.581002  \n",
       "Lazetic                                      0.0  6.170097  0.204814  \n",
       "Voelkerling Persson                          0.0  6.041983  0.689071  \n",
       "Sanca                                        0.0  5.991021  0.262247  \n",
       "\n",
       "[554 rows x 165 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rx = pd.read_excel('mid_outputs/players_stats.xlsx', index_col = 3)   \n",
    "players = pd.DataFrame(rx)\n",
    "\n",
    "players\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ae37dfe3",
   "metadata": {},
   "source": [
    "Function for merging match data (player, team, opponent team)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "71c8804e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def player_match_data(player, pteam, oppteam):\n",
    "    if(not(player in players.index)):\n",
    "        return None\n",
    "    \n",
    "    pdata = players.loc[[player]]\n",
    "    \n",
    "    pteam_stats = team_data.loc[[pteam]].rename(index = {pteam : player})\n",
    "    \n",
    "    oppteam_stats = team_data.loc[[oppteam]].rename(index = {oppteam : player})\n",
    "    \n",
    "    oppteam_stats = oppteam_stats.rename(lambda x: 'opp_' + x, axis='columns')\n",
    "    \n",
    "    out = pd.concat([pdata, pteam_stats, oppteam_stats], axis = 1)\n",
    "    \n",
    "    return(out)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a6fdd15b",
   "metadata": {},
   "source": [
    "Output to excel file data for a player. \n",
    "\n",
    "The file is copied and there the desired features are manually selected and printed."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "7eb4e666",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>id</th>\n",
       "      <th>r</th>\n",
       "      <th>team</th>\n",
       "      <th>surname</th>\n",
       "      <th>initial</th>\n",
       "      <th>fb_ID</th>\n",
       "      <th>age</th>\n",
       "      <th>birth_year</th>\n",
       "      <th>games</th>\n",
       "      <th>...</th>\n",
       "      <th>opp_vs_team_fouls</th>\n",
       "      <th>opp_vs_team_fouled</th>\n",
       "      <th>opp_vs_team_offsides</th>\n",
       "      <th>opp_vs_team_pens_won</th>\n",
       "      <th>opp_vs_team_pens_conceded</th>\n",
       "      <th>opp_vs_team_own_goals</th>\n",
       "      <th>opp_vs_team_ball_recoveries</th>\n",
       "      <th>opp_vs_team_aerials_won</th>\n",
       "      <th>opp_vs_team_aerials_lost</th>\n",
       "      <th>opp_vs_team_aerials_won_pct</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Frattesi</th>\n",
       "      <td>270</td>\n",
       "      <td>2848</td>\n",
       "      <td>C</td>\n",
       "      <td>Sassuolo</td>\n",
       "      <td>Frattesi</td>\n",
       "      <td>NaN</td>\n",
       "      <td>175</td>\n",
       "      <td>23-020</td>\n",
       "      <td>1999</td>\n",
       "      <td>14</td>\n",
       "      <td>...</td>\n",
       "      <td>150.0</td>\n",
       "      <td>168.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>834.0</td>\n",
       "      <td>181.0</td>\n",
       "      <td>211.0</td>\n",
       "      <td>46.2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1 rows × 791 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          Unnamed: 0    id  r      team   surname initial  fb_ID     age  \\\n",
       "Frattesi         270  2848  C  Sassuolo  Frattesi     NaN    175  23-020   \n",
       "\n",
       "          birth_year  games  ...  opp_vs_team_fouls  opp_vs_team_fouled  \\\n",
       "Frattesi        1999     14  ...              150.0               168.0   \n",
       "\n",
       "          opp_vs_team_offsides  opp_vs_team_pens_won  \\\n",
       "Frattesi                  19.0                   0.0   \n",
       "\n",
       "          opp_vs_team_pens_conceded  opp_vs_team_own_goals  \\\n",
       "Frattesi                        5.0                    1.0   \n",
       "\n",
       "          opp_vs_team_ball_recoveries  opp_vs_team_aerials_won  \\\n",
       "Frattesi                        834.0                    181.0   \n",
       "\n",
       "          opp_vs_team_aerials_lost  opp_vs_team_aerials_won_pct  \n",
       "Frattesi                     211.0                         46.2  \n",
       "\n",
       "[1 rows x 791 columns]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test = player_match_data('Frattesi', 'Sassuolo', 'Atalanta')\n",
    "test.to_excel('tmp/playermatchdata.xlsx')\n",
    "\n",
    "test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "6b4c1b00",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Sassuolo'"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pteam = 'Sassuolo'\n",
    "pteam_stats = team_data.loc[[pteam]]\n",
    "pteam_stats.index[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "6ea6df1f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['r',\n",
       " 'fb_ID',\n",
       " 'games',\n",
       " 'games_starts',\n",
       " 'minutes',\n",
       " 'goals',\n",
       " 'assists',\n",
       " 'cards_yellow',\n",
       " 'cards_red',\n",
       " 'xg',\n",
       " 'npxg',\n",
       " 'shots_on_target',\n",
       " 'shots_on_target_pct',\n",
       " 'goals_per_shot',\n",
       " 'goals_per_shot_on_target',\n",
       " 'passes_completed',\n",
       " 'passes_pct',\n",
       " 'passes_into_final_third',\n",
       " 'passes_into_penalty_area',\n",
       " 'progressive_passes',\n",
       " 'passes_live',\n",
       " 'passes_dead',\n",
       " 'through_balls',\n",
       " 'passes_switches',\n",
       " 'crosses',\n",
       " 'corner_kicks',\n",
       " 'dribble_tackles',\n",
       " 'dribbles_vs',\n",
       " 'dribble_tackles_pct',\n",
       " 'dribbled_past',\n",
       " 'blocks',\n",
       " 'blocked_shots',\n",
       " 'blocked_passes',\n",
       " 'interceptions',\n",
       " 'clearances',\n",
       " 'errors',\n",
       " 'touches',\n",
       " 'touches_def_pen_area',\n",
       " 'touches_def_3rd',\n",
       " 'touches_mid_3rd',\n",
       " 'touches_att_3rd',\n",
       " 'touches_att_pen_area',\n",
       " 'touches_live_ball',\n",
       " 'dribbles_completed',\n",
       " 'dribbles',\n",
       " 'dribbles_completed_pct',\n",
       " 'passes_received',\n",
       " 'miscontrols',\n",
       " 'dispossessed',\n",
       " 'fouls',\n",
       " 'fouled',\n",
       " 'aerials_won',\n",
       " 'aerials_lost',\n",
       " 'aerials_won_pct',\n",
       " 'team_possession',\n",
       " 'team_goals_assists_per90',\n",
       " 'team_goals_pens_per90',\n",
       " 'team_goals_assists_pens_per90',\n",
       " 'team_xg_per90',\n",
       " 'team_gk_goals_against_per90',\n",
       " 'team_gk_save_pct',\n",
       " 'team_gk_clean_sheets_pct',\n",
       " 'team_passes_pct',\n",
       " 'team_passes_pct_medium',\n",
       " 'team_passes_pct_long',\n",
       " 'team_sca_per90',\n",
       " 'team_gca_per90',\n",
       " 'team_dribble_tackles_pct',\n",
       " 'team_aerials_won_pct',\n",
       " 'vs_team_possession',\n",
       " 'vs_team_goals_per90',\n",
       " 'vs_team_assists_per90',\n",
       " 'vs_team_xg_per90',\n",
       " 'vs_team_gk_save_pct',\n",
       " 'vs_team_gk_clean_sheets_pct',\n",
       " 'vs_team_gk_pct_passes_launched',\n",
       " 'vs_team_gk_crosses_stopped_pct',\n",
       " 'vs_team_shots_on_target_per90',\n",
       " 'vs_team_passes_pct',\n",
       " 'vs_team_passes_pct_short',\n",
       " 'vs_team_passes_pct_medium',\n",
       " 'vs_team_passes_pct_long',\n",
       " 'vs_team_sca_per90',\n",
       " 'vs_team_gca_per90',\n",
       " 'vs_team_dribble_tackles_pct',\n",
       " 'vs_team_dribbles_completed_pct',\n",
       " 'vs_team_aerials_won_pct']"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rx = pd.read_excel('tmp/playermatchdata_sel.xlsx', index_col = 0, header = None)   \n",
    "\n",
    "out = list()\n",
    "\n",
    "for i in range(1, rx.shape[1]):\n",
    "        if(rx[i][2] == \"x\"):\n",
    "            out.append(rx[i][0])\n",
    "            \n",
    "out"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d4097382",
   "metadata": {},
   "source": [
    "Define and select the dataset features: abs features and rel features.\n",
    "    \n",
    "Rel features are divided by the player minutes played.\n",
    "\n",
    "Rel features (game corrected) are divided by the player number of games instead of minutes. \n",
    "For example, a player who plays 20 minutes and scores, would be to match for stats averaging. It is better to consider the goals made per match."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "47bc651f",
   "metadata": {},
   "outputs": [],
   "source": [
    "features_abs = ['r',\n",
    " 'games',\n",
    " 'games_starts',                \n",
    " 'minutes',\n",
    " 'shots_on_target_pct',\n",
    " 'goals_per_shot',\n",
    " 'goals_per_shot_on_target',\n",
    " 'passes_pct',\n",
    " 'dribble_tackles_pct',\n",
    " 'dribbles_completed_pct',\n",
    " 'aerials_won_pct',\n",
    " 'team_possession',\n",
    " 'team_goals_assists_per90',\n",
    " 'team_goals_pens_per90',\n",
    " 'team_goals_assists_pens_per90',\n",
    " 'team_xg_per90',\n",
    " 'team_gk_goals_against_per90',\n",
    " 'team_gk_save_pct',\n",
    " 'team_gk_clean_sheets_pct',\n",
    " 'team_passes_pct',\n",
    " 'team_passes_pct_medium',\n",
    " 'team_passes_pct_long',\n",
    " 'team_sca_per90',\n",
    " 'team_gca_per90',\n",
    " 'team_dribble_tackles_pct',\n",
    " 'team_aerials_won_pct',\n",
    " 'vs_team_possession',\n",
    " 'vs_team_goals_per90',\n",
    " 'vs_team_assists_per90',\n",
    " 'vs_team_xg_per90',\n",
    " 'vs_team_gk_save_pct',\n",
    " 'vs_team_gk_clean_sheets_pct',\n",
    " 'vs_team_gk_pct_passes_launched',\n",
    " 'vs_team_gk_crosses_stopped_pct',\n",
    " 'vs_team_shots_on_target_per90',\n",
    " 'vs_team_passes_pct',\n",
    " 'vs_team_passes_pct_short',\n",
    " 'vs_team_passes_pct_medium',\n",
    " 'vs_team_passes_pct_long',\n",
    " 'vs_team_sca_per90',\n",
    " 'vs_team_gca_per90',\n",
    " 'vs_team_dribble_tackles_pct',\n",
    " 'vs_team_dribbles_completed_pct',\n",
    " 'vs_team_aerials_won_pct',\n",
    " 'opp_team_possession',\n",
    " 'opp_team_goals_assists_per90',\n",
    " 'opp_team_goals_pens_per90',\n",
    " 'opp_team_goals_assists_pens_per90',\n",
    " 'opp_team_xg_per90',\n",
    " 'opp_team_gk_goals_against_per90',\n",
    " 'opp_team_gk_save_pct',\n",
    " 'opp_team_gk_clean_sheets_pct',\n",
    " 'opp_team_passes_pct',\n",
    " 'opp_team_passes_pct_medium',\n",
    " 'opp_team_passes_pct_long',\n",
    " 'opp_team_sca_per90',\n",
    " 'opp_team_gca_per90',\n",
    " 'opp_team_dribble_tackles_pct',\n",
    " 'opp_team_aerials_won_pct',\n",
    " 'opp_vs_team_possession',\n",
    " 'opp_vs_team_goals_per90',\n",
    " 'opp_vs_team_assists_per90',\n",
    " 'opp_vs_team_xg_per90',\n",
    " 'opp_vs_team_gk_save_pct',\n",
    " 'opp_vs_team_gk_clean_sheets_pct',\n",
    " 'opp_vs_team_gk_pct_passes_launched',\n",
    " 'opp_vs_team_gk_crosses_stopped_pct',\n",
    " 'opp_vs_team_shots_on_target_per90',\n",
    " 'opp_vs_team_passes_pct',\n",
    " 'opp_vs_team_passes_pct_short',\n",
    " 'opp_vs_team_passes_pct_medium',\n",
    " 'opp_vs_team_passes_pct_long',\n",
    " 'opp_vs_team_sca_per90',\n",
    " 'opp_vs_team_gca_per90',\n",
    " 'opp_vs_team_dribble_tackles_pct',\n",
    " 'opp_vs_team_dribbles_completed_pct',\n",
    " 'opp_vs_team_aerials_won_pct',\n",
    "                \n",
    " 'vote_avg',\n",
    " 'vote_std']\n",
    "\n",
    "features_rel = [\n",
    " 'goals',\n",
    " 'assists',\n",
    " 'cards_yellow',\n",
    " 'cards_red',\n",
    " 'xg',\n",
    " 'npxg',\n",
    " 'shots_on_target',\n",
    " 'passes_completed',\n",
    " 'passes_into_final_third',\n",
    " 'passes_into_penalty_area',\n",
    " 'progressive_passes',\n",
    " 'passes_live',\n",
    " 'passes_dead',\n",
    " 'through_balls',\n",
    " 'passes_switches',\n",
    " 'crosses',\n",
    " 'corner_kicks',\n",
    " 'dribble_tackles',\n",
    " 'dribbles_vs',\n",
    " 'dribbled_past',\n",
    " 'blocks',\n",
    " 'blocked_shots',\n",
    " 'blocked_passes',\n",
    " 'interceptions',\n",
    " 'clearances',\n",
    " 'errors',\n",
    " 'touches',\n",
    " 'touches_def_pen_area',\n",
    " 'touches_def_3rd',\n",
    " 'touches_mid_3rd',\n",
    " 'touches_att_3rd',\n",
    " 'touches_att_pen_area',\n",
    " 'touches_live_ball',\n",
    " 'dribbles_completed',\n",
    " 'dribbles',\n",
    " 'passes_received',\n",
    " 'miscontrols',\n",
    " 'dispossessed',\n",
    " 'fouls',\n",
    " 'fouled',\n",
    " 'aerials_won',\n",
    " 'aerials_lost']\n",
    "\n",
    "features_rel_gamecorr = [\n",
    "    'goals',\n",
    "    'assists',\n",
    "    'xg',\n",
    "    'npxg',\n",
    "    'cards_yellow',\n",
    "    'cards_red'\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ced70285",
   "metadata": {},
   "source": [
    "Function for merging match data (player, team, opponent team) and select only the desired features, applying the desired corrections."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "3f43d509",
   "metadata": {},
   "outputs": [],
   "source": [
    "def player_match_data_ext(player, pteam, oppteam):\n",
    "    pdata = player_match_data(player, pteam, oppteam)\n",
    "    \n",
    "    if(not isinstance(pdata, pd.DataFrame)):\n",
    "        return None\n",
    "    \n",
    "    out = pd.concat([pdata[features_abs], pdata[features_rel]], axis = 1)\n",
    "    \n",
    "    out[features_rel] = out[features_rel] / max(pdata['minutes'][0], 1)\n",
    "    \n",
    "    out[features_rel_gamecorr] = out[features_rel_gamecorr] * (pdata['minutes'][0] / max(pdata['games'][0], 1) / 90)\n",
    "    \n",
    "    return out\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "959bf152",
   "metadata": {},
   "source": [
    "Columns variable for the match data, to be then added to votes data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "930d00c7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>r</th>\n",
       "      <th>games</th>\n",
       "      <th>games_starts</th>\n",
       "      <th>minutes</th>\n",
       "      <th>shots_on_target_pct</th>\n",
       "      <th>goals_per_shot</th>\n",
       "      <th>goals_per_shot_on_target</th>\n",
       "      <th>passes_pct</th>\n",
       "      <th>dribble_tackles_pct</th>\n",
       "      <th>dribbles_completed_pct</th>\n",
       "      <th>...</th>\n",
       "      <th>touches_live_ball</th>\n",
       "      <th>dribbles_completed</th>\n",
       "      <th>dribbles</th>\n",
       "      <th>passes_received</th>\n",
       "      <th>miscontrols</th>\n",
       "      <th>dispossessed</th>\n",
       "      <th>fouls</th>\n",
       "      <th>fouled</th>\n",
       "      <th>aerials_won</th>\n",
       "      <th>aerials_lost</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Frattesi</th>\n",
       "      <td>C</td>\n",
       "      <td>14</td>\n",
       "      <td>14</td>\n",
       "      <td>1129</td>\n",
       "      <td>50.0</td>\n",
       "      <td>0.13</td>\n",
       "      <td>0.27</td>\n",
       "      <td>75.6</td>\n",
       "      <td>52.6</td>\n",
       "      <td>57.6</td>\n",
       "      <td>...</td>\n",
       "      <td>0.476528</td>\n",
       "      <td>0.016829</td>\n",
       "      <td>0.029229</td>\n",
       "      <td>0.330381</td>\n",
       "      <td>0.019486</td>\n",
       "      <td>0.015943</td>\n",
       "      <td>0.011515</td>\n",
       "      <td>0.022143</td>\n",
       "      <td>0.009743</td>\n",
       "      <td>0.0062</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1 rows × 121 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          r  games  games_starts  minutes  shots_on_target_pct  \\\n",
       "Frattesi  C     14            14     1129                 50.0   \n",
       "\n",
       "          goals_per_shot  goals_per_shot_on_target  passes_pct  \\\n",
       "Frattesi            0.13                      0.27        75.6   \n",
       "\n",
       "          dribble_tackles_pct  dribbles_completed_pct  ...  touches_live_ball  \\\n",
       "Frattesi                 52.6                    57.6  ...           0.476528   \n",
       "\n",
       "          dribbles_completed  dribbles  passes_received  miscontrols  \\\n",
       "Frattesi            0.016829  0.029229         0.330381     0.019486   \n",
       "\n",
       "          dispossessed     fouls    fouled  aerials_won  aerials_lost  \n",
       "Frattesi      0.015943  0.011515  0.022143     0.009743        0.0062  \n",
       "\n",
       "[1 rows x 121 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test = player_match_data_ext('Frattesi', 'Sassuolo', 'Atalanta')\n",
    "\n",
    "add_columns = test.columns\n",
    "\n",
    "test"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8d95c9ef",
   "metadata": {},
   "source": [
    "Load Fantacalcio votes data for each match of the season."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "3ae718d2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>matchday</th>\n",
       "      <th>player</th>\n",
       "      <th>team</th>\n",
       "      <th>oppteam</th>\n",
       "      <th>home</th>\n",
       "      <th>vote</th>\n",
       "      <th>goals</th>\n",
       "      <th>assists</th>\n",
       "      <th>cards_malus</th>\n",
       "      <th>fantavote</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Musso</td>\n",
       "      <td>Atalanta</td>\n",
       "      <td>Sampdoria</td>\n",
       "      <td>0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>5.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Toloi</td>\n",
       "      <td>Atalanta</td>\n",
       "      <td>Sampdoria</td>\n",
       "      <td>0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>Djimsiti</td>\n",
       "      <td>Atalanta</td>\n",
       "      <td>Sampdoria</td>\n",
       "      <td>0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>Hateboer</td>\n",
       "      <td>Atalanta</td>\n",
       "      <td>Sampdoria</td>\n",
       "      <td>0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>5.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>Okoli</td>\n",
       "      <td>Atalanta</td>\n",
       "      <td>Sampdoria</td>\n",
       "      <td>0</td>\n",
       "      <td>5.5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4010</th>\n",
       "      <td>14</td>\n",
       "      <td>Sulemana I.</td>\n",
       "      <td>Verona</td>\n",
       "      <td>Juventus</td>\n",
       "      <td>1</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4011</th>\n",
       "      <td>14</td>\n",
       "      <td>Lasagna</td>\n",
       "      <td>Verona</td>\n",
       "      <td>Juventus</td>\n",
       "      <td>1</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4012</th>\n",
       "      <td>14</td>\n",
       "      <td>Kallon</td>\n",
       "      <td>Verona</td>\n",
       "      <td>Juventus</td>\n",
       "      <td>1</td>\n",
       "      <td>5.5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4013</th>\n",
       "      <td>14</td>\n",
       "      <td>Djuric</td>\n",
       "      <td>Verona</td>\n",
       "      <td>Juventus</td>\n",
       "      <td>1</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>5.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4014</th>\n",
       "      <td>14</td>\n",
       "      <td>Henry</td>\n",
       "      <td>Verona</td>\n",
       "      <td>Juventus</td>\n",
       "      <td>1</td>\n",
       "      <td>6.5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4015 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      matchday       player      team    oppteam  home  vote  goals  assists  \\\n",
       "0            1        Musso  Atalanta  Sampdoria     0   6.0      0        0   \n",
       "1            1        Toloi  Atalanta  Sampdoria     0   7.0      1        0   \n",
       "2            1     Djimsiti  Atalanta  Sampdoria     0   6.0      0        0   \n",
       "3            1     Hateboer  Atalanta  Sampdoria     0   6.0      0        0   \n",
       "4            1        Okoli  Atalanta  Sampdoria     0   5.5      0        0   \n",
       "...        ...          ...       ...        ...   ...   ...    ...      ...   \n",
       "4010        14  Sulemana I.    Verona   Juventus     1   6.0      0        0   \n",
       "4011        14      Lasagna    Verona   Juventus     1   6.0      0        0   \n",
       "4012        14       Kallon    Verona   Juventus     1   5.5      0        0   \n",
       "4013        14       Djuric    Verona   Juventus     1   6.0      0        0   \n",
       "4014        14        Henry    Verona   Juventus     1   6.5      0        0   \n",
       "\n",
       "      cards_malus  fantavote  \n",
       "0             0.5        5.5  \n",
       "1             0.0       10.0  \n",
       "2             0.0        6.0  \n",
       "3             0.5        5.5  \n",
       "4             0.5        5.0  \n",
       "...           ...        ...  \n",
       "4010          0.0        6.0  \n",
       "4011          0.0        6.0  \n",
       "4012          0.0        5.5  \n",
       "4013          0.5        5.5  \n",
       "4014          0.0        6.5  \n",
       "\n",
       "[4015 rows x 10 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "votes = pd.read_excel('mid_outputs/players_votes.xlsx', index_col = 0)\n",
    "votes"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8cf677e4",
   "metadata": {},
   "source": [
    "Merge votes data with match data for the player."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "ec0bc56b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "1100\n",
      "1300\n",
      "1400\n",
      "1500\n",
      "1600\n",
      "1700\n",
      "1800\n",
      "1900\n",
      "2000\n",
      "2100\n",
      "2200\n",
      "2300\n",
      "2400\n",
      "2500\n",
      "2600\n",
      "2700\n",
      "2800\n",
      "2900\n",
      "3000\n",
      "3100\n",
      "3200\n",
      "3300\n",
      "3400\n",
      "3500\n",
      "3600\n",
      "3700\n",
      "3800\n",
      "3900\n",
      "4000\n"
     ]
    }
   ],
   "source": [
    "db = votes.copy()\n",
    "\n",
    "delta = pd.DataFrame(columns = add_columns)\n",
    "\n",
    "for i in range(db.shape[0]):\n",
    "    ext_data = player_match_data_ext(db['player'][i], db['team'][i], db['oppteam'][i])\n",
    "    \n",
    "    if(not isinstance(ext_data, pd.DataFrame)):\n",
    "        continue\n",
    "    \n",
    "    ext_data = ext_data.rename(index = {db['player'][i] : i})\n",
    "    \n",
    "    delta = pd.concat([delta, ext_data], axis = 0)\n",
    "    \n",
    "    if(i % 100 == 0):\n",
    "        print(i)\n",
    "    \n",
    "db = pd.concat([db, delta], axis = 1)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "b292f9f9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>matchday</th>\n",
       "      <th>player</th>\n",
       "      <th>team</th>\n",
       "      <th>oppteam</th>\n",
       "      <th>home</th>\n",
       "      <th>vote</th>\n",
       "      <th>goals</th>\n",
       "      <th>assists</th>\n",
       "      <th>cards_malus</th>\n",
       "      <th>fantavote</th>\n",
       "      <th>...</th>\n",
       "      <th>touches_live_ball</th>\n",
       "      <th>dribbles_completed</th>\n",
       "      <th>dribbles</th>\n",
       "      <th>passes_received</th>\n",
       "      <th>miscontrols</th>\n",
       "      <th>dispossessed</th>\n",
       "      <th>fouls</th>\n",
       "      <th>fouled</th>\n",
       "      <th>aerials_won</th>\n",
       "      <th>aerials_lost</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Musso</td>\n",
       "      <td>Atalanta</td>\n",
       "      <td>Sampdoria</td>\n",
       "      <td>0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>5.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Toloi</td>\n",
       "      <td>Atalanta</td>\n",
       "      <td>Sampdoria</td>\n",
       "      <td>0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.617096</td>\n",
       "      <td>0.001171</td>\n",
       "      <td>0.004684</td>\n",
       "      <td>0.357143</td>\n",
       "      <td>0.007026</td>\n",
       "      <td>0.001171</td>\n",
       "      <td>0.009368</td>\n",
       "      <td>0.005855</td>\n",
       "      <td>0.018735</td>\n",
       "      <td>0.008197</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>Djimsiti</td>\n",
       "      <td>Atalanta</td>\n",
       "      <td>Sampdoria</td>\n",
       "      <td>0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.736842</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.002924</td>\n",
       "      <td>0.511696</td>\n",
       "      <td>0.002924</td>\n",
       "      <td>0.008772</td>\n",
       "      <td>0.002924</td>\n",
       "      <td>0.002924</td>\n",
       "      <td>0.005848</td>\n",
       "      <td>0.008772</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>Hateboer</td>\n",
       "      <td>Atalanta</td>\n",
       "      <td>Sampdoria</td>\n",
       "      <td>0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>5.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0.574423</td>\n",
       "      <td>0.002096</td>\n",
       "      <td>0.005241</td>\n",
       "      <td>0.339623</td>\n",
       "      <td>0.006289</td>\n",
       "      <td>0.005241</td>\n",
       "      <td>0.013627</td>\n",
       "      <td>0.002096</td>\n",
       "      <td>0.015723</td>\n",
       "      <td>0.012579</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>Okoli</td>\n",
       "      <td>Atalanta</td>\n",
       "      <td>Sampdoria</td>\n",
       "      <td>0</td>\n",
       "      <td>5.5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>5.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.598540</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001217</td>\n",
       "      <td>0.278589</td>\n",
       "      <td>0.013382</td>\n",
       "      <td>0.003650</td>\n",
       "      <td>0.015815</td>\n",
       "      <td>0.008516</td>\n",
       "      <td>0.048662</td>\n",
       "      <td>0.026764</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4010</th>\n",
       "      <td>14</td>\n",
       "      <td>Sulemana I.</td>\n",
       "      <td>Verona</td>\n",
       "      <td>Juventus</td>\n",
       "      <td>1</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4011</th>\n",
       "      <td>14</td>\n",
       "      <td>Lasagna</td>\n",
       "      <td>Verona</td>\n",
       "      <td>Juventus</td>\n",
       "      <td>1</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.279255</td>\n",
       "      <td>0.006649</td>\n",
       "      <td>0.018617</td>\n",
       "      <td>0.186170</td>\n",
       "      <td>0.038564</td>\n",
       "      <td>0.023936</td>\n",
       "      <td>0.021277</td>\n",
       "      <td>0.018617</td>\n",
       "      <td>0.005319</td>\n",
       "      <td>0.029255</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4012</th>\n",
       "      <td>14</td>\n",
       "      <td>Kallon</td>\n",
       "      <td>Verona</td>\n",
       "      <td>Juventus</td>\n",
       "      <td>1</td>\n",
       "      <td>5.5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0.417949</td>\n",
       "      <td>0.007692</td>\n",
       "      <td>0.038462</td>\n",
       "      <td>0.289744</td>\n",
       "      <td>0.038462</td>\n",
       "      <td>0.025641</td>\n",
       "      <td>0.017949</td>\n",
       "      <td>0.023077</td>\n",
       "      <td>0.010256</td>\n",
       "      <td>0.028205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4013</th>\n",
       "      <td>14</td>\n",
       "      <td>Djuric</td>\n",
       "      <td>Verona</td>\n",
       "      <td>Juventus</td>\n",
       "      <td>1</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>5.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0.473214</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.401786</td>\n",
       "      <td>0.022321</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.022321</td>\n",
       "      <td>0.022321</td>\n",
       "      <td>0.178571</td>\n",
       "      <td>0.066964</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4014</th>\n",
       "      <td>14</td>\n",
       "      <td>Henry</td>\n",
       "      <td>Verona</td>\n",
       "      <td>Juventus</td>\n",
       "      <td>1</td>\n",
       "      <td>6.5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0.342424</td>\n",
       "      <td>0.003030</td>\n",
       "      <td>0.014141</td>\n",
       "      <td>0.282828</td>\n",
       "      <td>0.049495</td>\n",
       "      <td>0.025253</td>\n",
       "      <td>0.024242</td>\n",
       "      <td>0.015152</td>\n",
       "      <td>0.039394</td>\n",
       "      <td>0.039394</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4015 rows × 131 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      matchday       player      team    oppteam  home  vote  goals  assists  \\\n",
       "0            1        Musso  Atalanta  Sampdoria     0   6.0      0        0   \n",
       "1            1        Toloi  Atalanta  Sampdoria     0   7.0      1        0   \n",
       "2            1     Djimsiti  Atalanta  Sampdoria     0   6.0      0        0   \n",
       "3            1     Hateboer  Atalanta  Sampdoria     0   6.0      0        0   \n",
       "4            1        Okoli  Atalanta  Sampdoria     0   5.5      0        0   \n",
       "...        ...          ...       ...        ...   ...   ...    ...      ...   \n",
       "4010        14  Sulemana I.    Verona   Juventus     1   6.0      0        0   \n",
       "4011        14      Lasagna    Verona   Juventus     1   6.0      0        0   \n",
       "4012        14       Kallon    Verona   Juventus     1   5.5      0        0   \n",
       "4013        14       Djuric    Verona   Juventus     1   6.0      0        0   \n",
       "4014        14        Henry    Verona   Juventus     1   6.5      0        0   \n",
       "\n",
       "      cards_malus  fantavote  ... touches_live_ball dribbles_completed  \\\n",
       "0             0.5        5.5  ...          0.000000           0.000000   \n",
       "1             0.0       10.0  ...          0.617096           0.001171   \n",
       "2             0.0        6.0  ...          0.736842           0.000000   \n",
       "3             0.5        5.5  ...          0.574423           0.002096   \n",
       "4             0.5        5.0  ...          0.598540           0.000000   \n",
       "...           ...        ...  ...               ...                ...   \n",
       "4010          0.0        6.0  ...               NaN                NaN   \n",
       "4011          0.0        6.0  ...          0.279255           0.006649   \n",
       "4012          0.0        5.5  ...          0.417949           0.007692   \n",
       "4013          0.5        5.5  ...          0.473214           0.000000   \n",
       "4014          0.0        6.5  ...          0.342424           0.003030   \n",
       "\n",
       "      dribbles passes_received  miscontrols  dispossessed     fouls    fouled  \\\n",
       "0     0.000000        0.000000     0.000000      0.000000  0.000000  0.000000   \n",
       "1     0.004684        0.357143     0.007026      0.001171  0.009368  0.005855   \n",
       "2     0.002924        0.511696     0.002924      0.008772  0.002924  0.002924   \n",
       "3     0.005241        0.339623     0.006289      0.005241  0.013627  0.002096   \n",
       "4     0.001217        0.278589     0.013382      0.003650  0.015815  0.008516   \n",
       "...        ...             ...          ...           ...       ...       ...   \n",
       "4010       NaN             NaN          NaN           NaN       NaN       NaN   \n",
       "4011  0.018617        0.186170     0.038564      0.023936  0.021277  0.018617   \n",
       "4012  0.038462        0.289744     0.038462      0.025641  0.017949  0.023077   \n",
       "4013  0.000000        0.401786     0.022321      0.000000  0.022321  0.022321   \n",
       "4014  0.014141        0.282828     0.049495      0.025253  0.024242  0.015152   \n",
       "\n",
       "      aerials_won  aerials_lost  \n",
       "0        0.000000      0.000000  \n",
       "1        0.018735      0.008197  \n",
       "2        0.005848      0.008772  \n",
       "3        0.015723      0.012579  \n",
       "4        0.048662      0.026764  \n",
       "...           ...           ...  \n",
       "4010          NaN           NaN  \n",
       "4011     0.005319      0.029255  \n",
       "4012     0.010256      0.028205  \n",
       "4013     0.178571      0.066964  \n",
       "4014     0.039394      0.039394  \n",
       "\n",
       "[4015 rows x 131 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "db"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d237f87b",
   "metadata": {},
   "source": [
    "Remove undesired database entries."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "1db9f77f",
   "metadata": {},
   "outputs": [],
   "source": [
    "db = db.drop(db[db.r == 'P'].index) # remove goalkeepers\n",
    "db = db.drop(db[db.r != db.r].index) # remove non existing players (by searching NaN values)\n",
    "db = db.drop(db[db.xg != db.xg].index)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "f56df3ca",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>matchday</th>\n",
       "      <th>player</th>\n",
       "      <th>team</th>\n",
       "      <th>oppteam</th>\n",
       "      <th>home</th>\n",
       "      <th>vote</th>\n",
       "      <th>goals</th>\n",
       "      <th>assists</th>\n",
       "      <th>cards_malus</th>\n",
       "      <th>fantavote</th>\n",
       "      <th>...</th>\n",
       "      <th>touches_live_ball</th>\n",
       "      <th>dribbles_completed</th>\n",
       "      <th>dribbles</th>\n",
       "      <th>passes_received</th>\n",
       "      <th>miscontrols</th>\n",
       "      <th>dispossessed</th>\n",
       "      <th>fouls</th>\n",
       "      <th>fouled</th>\n",
       "      <th>aerials_won</th>\n",
       "      <th>aerials_lost</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Toloi</td>\n",
       "      <td>Atalanta</td>\n",
       "      <td>Sampdoria</td>\n",
       "      <td>0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.617096</td>\n",
       "      <td>0.001171</td>\n",
       "      <td>0.004684</td>\n",
       "      <td>0.357143</td>\n",
       "      <td>0.007026</td>\n",
       "      <td>0.001171</td>\n",
       "      <td>0.009368</td>\n",
       "      <td>0.005855</td>\n",
       "      <td>0.018735</td>\n",
       "      <td>0.008197</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>Djimsiti</td>\n",
       "      <td>Atalanta</td>\n",
       "      <td>Sampdoria</td>\n",
       "      <td>0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.736842</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.002924</td>\n",
       "      <td>0.511696</td>\n",
       "      <td>0.002924</td>\n",
       "      <td>0.008772</td>\n",
       "      <td>0.002924</td>\n",
       "      <td>0.002924</td>\n",
       "      <td>0.005848</td>\n",
       "      <td>0.008772</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>Hateboer</td>\n",
       "      <td>Atalanta</td>\n",
       "      <td>Sampdoria</td>\n",
       "      <td>0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>5.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0.574423</td>\n",
       "      <td>0.002096</td>\n",
       "      <td>0.005241</td>\n",
       "      <td>0.339623</td>\n",
       "      <td>0.006289</td>\n",
       "      <td>0.005241</td>\n",
       "      <td>0.013627</td>\n",
       "      <td>0.002096</td>\n",
       "      <td>0.015723</td>\n",
       "      <td>0.012579</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>Okoli</td>\n",
       "      <td>Atalanta</td>\n",
       "      <td>Sampdoria</td>\n",
       "      <td>0</td>\n",
       "      <td>5.5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>5.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.598540</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001217</td>\n",
       "      <td>0.278589</td>\n",
       "      <td>0.013382</td>\n",
       "      <td>0.003650</td>\n",
       "      <td>0.015815</td>\n",
       "      <td>0.008516</td>\n",
       "      <td>0.048662</td>\n",
       "      <td>0.026764</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1</td>\n",
       "      <td>Zortea</td>\n",
       "      <td>Atalanta</td>\n",
       "      <td>Sampdoria</td>\n",
       "      <td>0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>5.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0.634921</td>\n",
       "      <td>0.052910</td>\n",
       "      <td>0.111111</td>\n",
       "      <td>0.328042</td>\n",
       "      <td>0.026455</td>\n",
       "      <td>0.005291</td>\n",
       "      <td>0.021164</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.026455</td>\n",
       "      <td>0.005291</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4009</th>\n",
       "      <td>14</td>\n",
       "      <td>Terracciano F.</td>\n",
       "      <td>Verona</td>\n",
       "      <td>Juventus</td>\n",
       "      <td>1</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.477778</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.008889</td>\n",
       "      <td>0.271111</td>\n",
       "      <td>0.015556</td>\n",
       "      <td>0.006667</td>\n",
       "      <td>0.008889</td>\n",
       "      <td>0.022222</td>\n",
       "      <td>0.020000</td>\n",
       "      <td>0.024444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4011</th>\n",
       "      <td>14</td>\n",
       "      <td>Lasagna</td>\n",
       "      <td>Verona</td>\n",
       "      <td>Juventus</td>\n",
       "      <td>1</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.279255</td>\n",
       "      <td>0.006649</td>\n",
       "      <td>0.018617</td>\n",
       "      <td>0.186170</td>\n",
       "      <td>0.038564</td>\n",
       "      <td>0.023936</td>\n",
       "      <td>0.021277</td>\n",
       "      <td>0.018617</td>\n",
       "      <td>0.005319</td>\n",
       "      <td>0.029255</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4012</th>\n",
       "      <td>14</td>\n",
       "      <td>Kallon</td>\n",
       "      <td>Verona</td>\n",
       "      <td>Juventus</td>\n",
       "      <td>1</td>\n",
       "      <td>5.5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0.417949</td>\n",
       "      <td>0.007692</td>\n",
       "      <td>0.038462</td>\n",
       "      <td>0.289744</td>\n",
       "      <td>0.038462</td>\n",
       "      <td>0.025641</td>\n",
       "      <td>0.017949</td>\n",
       "      <td>0.023077</td>\n",
       "      <td>0.010256</td>\n",
       "      <td>0.028205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4013</th>\n",
       "      <td>14</td>\n",
       "      <td>Djuric</td>\n",
       "      <td>Verona</td>\n",
       "      <td>Juventus</td>\n",
       "      <td>1</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.5</td>\n",
       "      <td>5.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0.473214</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.401786</td>\n",
       "      <td>0.022321</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.022321</td>\n",
       "      <td>0.022321</td>\n",
       "      <td>0.178571</td>\n",
       "      <td>0.066964</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4014</th>\n",
       "      <td>14</td>\n",
       "      <td>Henry</td>\n",
       "      <td>Verona</td>\n",
       "      <td>Juventus</td>\n",
       "      <td>1</td>\n",
       "      <td>6.5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0.342424</td>\n",
       "      <td>0.003030</td>\n",
       "      <td>0.014141</td>\n",
       "      <td>0.282828</td>\n",
       "      <td>0.049495</td>\n",
       "      <td>0.025253</td>\n",
       "      <td>0.024242</td>\n",
       "      <td>0.015152</td>\n",
       "      <td>0.039394</td>\n",
       "      <td>0.039394</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3717 rows × 131 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      matchday          player      team    oppteam  home  vote  goals  \\\n",
       "1            1           Toloi  Atalanta  Sampdoria     0   7.0      1   \n",
       "2            1        Djimsiti  Atalanta  Sampdoria     0   6.0      0   \n",
       "3            1        Hateboer  Atalanta  Sampdoria     0   6.0      0   \n",
       "4            1           Okoli  Atalanta  Sampdoria     0   5.5      0   \n",
       "5            1          Zortea  Atalanta  Sampdoria     0   6.0      0   \n",
       "...        ...             ...       ...        ...   ...   ...    ...   \n",
       "4009        14  Terracciano F.    Verona   Juventus     1   6.0      0   \n",
       "4011        14         Lasagna    Verona   Juventus     1   6.0      0   \n",
       "4012        14          Kallon    Verona   Juventus     1   5.5      0   \n",
       "4013        14          Djuric    Verona   Juventus     1   6.0      0   \n",
       "4014        14           Henry    Verona   Juventus     1   6.5      0   \n",
       "\n",
       "      assists  cards_malus  fantavote  ... touches_live_ball  \\\n",
       "1           0          0.0       10.0  ...          0.617096   \n",
       "2           0          0.0        6.0  ...          0.736842   \n",
       "3           0          0.5        5.5  ...          0.574423   \n",
       "4           0          0.5        5.0  ...          0.598540   \n",
       "5           0          0.5        5.5  ...          0.634921   \n",
       "...       ...          ...        ...  ...               ...   \n",
       "4009        0          0.0        6.0  ...          0.477778   \n",
       "4011        0          0.0        6.0  ...          0.279255   \n",
       "4012        0          0.0        5.5  ...          0.417949   \n",
       "4013        0          0.5        5.5  ...          0.473214   \n",
       "4014        0          0.0        6.5  ...          0.342424   \n",
       "\n",
       "     dribbles_completed  dribbles passes_received  miscontrols  dispossessed  \\\n",
       "1              0.001171  0.004684        0.357143     0.007026      0.001171   \n",
       "2              0.000000  0.002924        0.511696     0.002924      0.008772   \n",
       "3              0.002096  0.005241        0.339623     0.006289      0.005241   \n",
       "4              0.000000  0.001217        0.278589     0.013382      0.003650   \n",
       "5              0.052910  0.111111        0.328042     0.026455      0.005291   \n",
       "...                 ...       ...             ...          ...           ...   \n",
       "4009           0.000000  0.008889        0.271111     0.015556      0.006667   \n",
       "4011           0.006649  0.018617        0.186170     0.038564      0.023936   \n",
       "4012           0.007692  0.038462        0.289744     0.038462      0.025641   \n",
       "4013           0.000000  0.000000        0.401786     0.022321      0.000000   \n",
       "4014           0.003030  0.014141        0.282828     0.049495      0.025253   \n",
       "\n",
       "         fouls    fouled  aerials_won  aerials_lost  \n",
       "1     0.009368  0.005855     0.018735      0.008197  \n",
       "2     0.002924  0.002924     0.005848      0.008772  \n",
       "3     0.013627  0.002096     0.015723      0.012579  \n",
       "4     0.015815  0.008516     0.048662      0.026764  \n",
       "5     0.021164  0.000000     0.026455      0.005291  \n",
       "...        ...       ...          ...           ...  \n",
       "4009  0.008889  0.022222     0.020000      0.024444  \n",
       "4011  0.021277  0.018617     0.005319      0.029255  \n",
       "4012  0.017949  0.023077     0.010256      0.028205  \n",
       "4013  0.022321  0.022321     0.178571      0.066964  \n",
       "4014  0.024242  0.015152     0.039394      0.039394  \n",
       "\n",
       "[3717 rows x 131 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "db"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab69d09e",
   "metadata": {},
   "source": [
    "Save outfield players database to file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "df26c8e0",
   "metadata": {},
   "outputs": [],
   "source": [
    "db.to_excel('mid_outputs/database_entries.xlsx')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "38b564d2",
   "metadata": {},
   "source": [
    "Do the same process for goalkeepers, generating a different database."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "4d06576a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 - matchday\n",
      "1 - player\n",
      "2 - team\n",
      "3 - oppteam\n",
      "4 - home\n",
      "5 - vote\n",
      "6 - goals\n",
      "7 - assists\n",
      "8 - cards_malus\n",
      "9 - fantavote\n",
      "10 - r\n",
      "11 - games\n",
      "12 - games_starts\n",
      "13 - minutes\n",
      "14 - shots_on_target_pct\n",
      "15 - goals_per_shot\n",
      "16 - goals_per_shot_on_target\n",
      "17 - passes_pct\n",
      "18 - dribble_tackles_pct\n",
      "19 - dribbles_completed_pct\n",
      "20 - aerials_won_pct\n",
      "21 - team_possession\n",
      "22 - team_goals_assists_per90\n",
      "23 - team_goals_pens_per90\n",
      "24 - team_goals_assists_pens_per90\n",
      "25 - team_xg_per90\n",
      "26 - team_gk_goals_against_per90\n",
      "27 - team_gk_save_pct\n",
      "28 - team_gk_clean_sheets_pct\n",
      "29 - team_passes_pct\n",
      "30 - team_passes_pct_medium\n",
      "31 - team_passes_pct_long\n",
      "32 - team_sca_per90\n",
      "33 - team_gca_per90\n",
      "34 - team_dribble_tackles_pct\n",
      "35 - team_aerials_won_pct\n",
      "36 - vs_team_possession\n",
      "37 - vs_team_goals_per90\n",
      "38 - vs_team_assists_per90\n",
      "39 - vs_team_xg_per90\n",
      "40 - vs_team_gk_save_pct\n",
      "41 - vs_team_gk_clean_sheets_pct\n",
      "42 - vs_team_gk_pct_passes_launched\n",
      "43 - vs_team_gk_crosses_stopped_pct\n",
      "44 - vs_team_shots_on_target_per90\n",
      "45 - vs_team_passes_pct\n",
      "46 - vs_team_passes_pct_short\n",
      "47 - vs_team_passes_pct_medium\n",
      "48 - vs_team_passes_pct_long\n",
      "49 - vs_team_sca_per90\n",
      "50 - vs_team_gca_per90\n",
      "51 - vs_team_dribble_tackles_pct\n",
      "52 - vs_team_dribbles_completed_pct\n",
      "53 - vs_team_aerials_won_pct\n",
      "54 - opp_team_possession\n",
      "55 - opp_team_goals_assists_per90\n",
      "56 - opp_team_goals_pens_per90\n",
      "57 - opp_team_goals_assists_pens_per90\n",
      "58 - opp_team_xg_per90\n",
      "59 - opp_team_gk_goals_against_per90\n",
      "60 - opp_team_gk_save_pct\n",
      "61 - opp_team_gk_clean_sheets_pct\n",
      "62 - opp_team_passes_pct\n",
      "63 - opp_team_passes_pct_medium\n",
      "64 - opp_team_passes_pct_long\n",
      "65 - opp_team_sca_per90\n",
      "66 - opp_team_gca_per90\n",
      "67 - opp_team_dribble_tackles_pct\n",
      "68 - opp_team_aerials_won_pct\n",
      "69 - opp_vs_team_possession\n",
      "70 - opp_vs_team_goals_per90\n",
      "71 - opp_vs_team_assists_per90\n",
      "72 - opp_vs_team_xg_per90\n",
      "73 - opp_vs_team_gk_save_pct\n",
      "74 - opp_vs_team_gk_clean_sheets_pct\n",
      "75 - opp_vs_team_gk_pct_passes_launched\n",
      "76 - opp_vs_team_gk_crosses_stopped_pct\n",
      "77 - opp_vs_team_shots_on_target_per90\n",
      "78 - opp_vs_team_passes_pct\n",
      "79 - opp_vs_team_passes_pct_short\n",
      "80 - opp_vs_team_passes_pct_medium\n",
      "81 - opp_vs_team_passes_pct_long\n",
      "82 - opp_vs_team_sca_per90\n",
      "83 - opp_vs_team_gca_per90\n",
      "84 - opp_vs_team_dribble_tackles_pct\n",
      "85 - opp_vs_team_dribbles_completed_pct\n",
      "86 - opp_vs_team_aerials_won_pct\n",
      "87 - vote_avg\n",
      "88 - vote_std\n",
      "89 - goals\n",
      "90 - assists\n",
      "91 - cards_yellow\n",
      "92 - cards_red\n",
      "93 - xg\n",
      "94 - npxg\n",
      "95 - shots_on_target\n",
      "96 - passes_completed\n",
      "97 - passes_into_final_third\n",
      "98 - passes_into_penalty_area\n",
      "99 - progressive_passes\n",
      "100 - passes_live\n",
      "101 - passes_dead\n",
      "102 - through_balls\n",
      "103 - passes_switches\n",
      "104 - crosses\n",
      "105 - corner_kicks\n",
      "106 - dribble_tackles\n",
      "107 - dribbles_vs\n",
      "108 - dribbled_past\n",
      "109 - blocks\n",
      "110 - blocked_shots\n",
      "111 - blocked_passes\n",
      "112 - interceptions\n",
      "113 - clearances\n",
      "114 - errors\n",
      "115 - touches\n",
      "116 - touches_def_pen_area\n",
      "117 - touches_def_3rd\n",
      "118 - touches_mid_3rd\n",
      "119 - touches_att_3rd\n",
      "120 - touches_att_pen_area\n",
      "121 - touches_live_ball\n",
      "122 - dribbles_completed\n",
      "123 - dribbles\n",
      "124 - passes_received\n",
      "125 - miscontrols\n",
      "126 - dispossessed\n",
      "127 - fouls\n",
      "128 - fouled\n",
      "129 - aerials_won\n",
      "130 - aerials_lost\n"
     ]
    }
   ],
   "source": [
    "for i in range(db.columns.shape[0]):\n",
    "    print(str(i) + \" - \" + str(db.columns[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "01bb7413",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['r',\n",
       " 'fb_ID',\n",
       " 'gk_games',\n",
       " 'gk_games_starts',\n",
       " 'gk_minutes',\n",
       " 'gk_goals_against_per90',\n",
       " 'gk_shots_on_target_against',\n",
       " 'gk_saves',\n",
       " 'gk_save_pct',\n",
       " 'gk_clean_sheets_pct',\n",
       " 'gk_free_kick_goals_against',\n",
       " 'gk_corner_kick_goals_against',\n",
       " 'gk_own_goals_against',\n",
       " 'gk_psxg',\n",
       " 'gk_psnpxg_per_shot_on_target_against',\n",
       " 'gk_psxg_net',\n",
       " 'gk_psxg_net_per90',\n",
       " 'gk_passes_completed_launched',\n",
       " 'gk_passes_launched',\n",
       " 'gk_passes_pct_launched',\n",
       " 'gk_passes',\n",
       " 'gk_passes_throws',\n",
       " 'gk_pct_passes_launched',\n",
       " 'gk_passes_length_avg',\n",
       " 'gk_goal_kicks',\n",
       " 'gk_pct_goal_kicks_launched',\n",
       " 'gk_goal_kick_length_avg',\n",
       " 'gk_crosses',\n",
       " 'gk_crosses_stopped',\n",
       " 'gk_crosses_stopped_pct',\n",
       " 'gk_def_actions_outside_pen_area',\n",
       " 'gk_def_actions_outside_pen_area_per90',\n",
       " 'gk_avg_distance_def_actions',\n",
       " 'team_possession',\n",
       " 'team_goals_assists_per90',\n",
       " 'team_goals_pens_per90',\n",
       " 'team_goals_assists_pens_per90',\n",
       " 'team_xg_per90',\n",
       " 'team_gk_goals_against_per90',\n",
       " 'team_gk_save_pct',\n",
       " 'team_gk_clean_sheets_pct',\n",
       " 'team_passes_pct',\n",
       " 'team_passes_pct_medium',\n",
       " 'team_passes_pct_long',\n",
       " 'team_sca_per90',\n",
       " 'team_gca_per90',\n",
       " 'team_dribble_tackles_pct',\n",
       " 'team_aerials_won_pct',\n",
       " 'vs_team_possession',\n",
       " 'vs_team_goals_per90',\n",
       " 'vs_team_assists_per90',\n",
       " 'vs_team_xg_per90',\n",
       " 'vs_team_gk_save_pct',\n",
       " 'vs_team_gk_clean_sheets_pct',\n",
       " 'vs_team_gk_pct_passes_launched',\n",
       " 'vs_team_gk_crosses_stopped_pct',\n",
       " 'vs_team_shots_on_target_per90',\n",
       " 'vs_team_passes_pct',\n",
       " 'vs_team_passes_pct_short',\n",
       " 'vs_team_passes_pct_medium',\n",
       " 'vs_team_passes_pct_long',\n",
       " 'vs_team_sca_per90',\n",
       " 'vs_team_gca_per90',\n",
       " 'vs_team_dribble_tackles_pct',\n",
       " 'vs_team_dribbles_completed_pct',\n",
       " 'vs_team_aerials_won_pct']"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rx = pd.read_excel('tmp/playermatchdata_sel_gk.xlsx', index_col = 0, header = None)   \n",
    "\n",
    "out = list()\n",
    "\n",
    "for i in range(1, rx.shape[1]):\n",
    "        if(rx[i][2] == \"x\"):\n",
    "            out.append(rx[i][0])\n",
    "            \n",
    "out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "02f961d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "features_abs_gk = [\n",
    " 'gk_games',\n",
    " 'gk_games_starts',\n",
    " 'gk_minutes',\n",
    " 'gk_goals_against_per90', \n",
    " 'gk_save_pct',\n",
    " 'gk_clean_sheets_pct',\n",
    " 'gk_psxg_net_per90',\n",
    " 'gk_passes_pct_launched',\n",
    " 'gk_pct_passes_launched',\n",
    " 'gk_passes_length_avg',\n",
    " 'gk_pct_goal_kicks_launched',\n",
    " 'gk_goal_kick_length_avg',\n",
    " 'gk_crosses_stopped_pct',\n",
    " 'gk_def_actions_outside_pen_area_per90',\n",
    " 'gk_avg_distance_def_actions',\n",
    "    \n",
    " 'team_possession',\n",
    " 'team_goals_assists_per90',\n",
    " 'team_goals_pens_per90',\n",
    " 'team_goals_assists_pens_per90',\n",
    " 'team_xg_per90',\n",
    " 'team_gk_goals_against_per90',\n",
    " 'team_gk_save_pct',\n",
    " 'team_gk_clean_sheets_pct',\n",
    " 'team_passes_pct',\n",
    " 'team_passes_pct_medium',\n",
    " 'team_passes_pct_long',\n",
    " 'team_sca_per90',\n",
    " 'team_gca_per90',\n",
    " 'team_dribble_tackles_pct',\n",
    " 'team_aerials_won_pct',\n",
    " 'vs_team_possession',\n",
    " 'vs_team_goals_per90',\n",
    " 'vs_team_assists_per90',\n",
    " 'vs_team_xg_per90',\n",
    " 'vs_team_gk_save_pct',\n",
    " 'vs_team_gk_clean_sheets_pct',\n",
    " 'vs_team_gk_pct_passes_launched',\n",
    " 'vs_team_gk_crosses_stopped_pct',\n",
    " 'vs_team_shots_on_target_per90',\n",
    " 'vs_team_passes_pct',\n",
    " 'vs_team_passes_pct_short',\n",
    " 'vs_team_passes_pct_medium',\n",
    " 'vs_team_passes_pct_long',\n",
    " 'vs_team_sca_per90',\n",
    " 'vs_team_gca_per90',\n",
    " 'vs_team_dribble_tackles_pct',\n",
    " 'vs_team_dribbles_completed_pct',\n",
    " 'vs_team_aerials_won_pct',\n",
    " 'opp_team_possession',\n",
    " 'opp_team_goals_assists_per90',\n",
    " 'opp_team_goals_pens_per90',\n",
    " 'opp_team_goals_assists_pens_per90',\n",
    " 'opp_team_xg_per90',\n",
    " 'opp_team_gk_goals_against_per90',\n",
    " 'opp_team_gk_save_pct',\n",
    " 'opp_team_gk_clean_sheets_pct',\n",
    " 'opp_team_passes_pct',\n",
    " 'opp_team_passes_pct_medium',\n",
    " 'opp_team_passes_pct_long',\n",
    " 'opp_team_sca_per90',\n",
    " 'opp_team_gca_per90',\n",
    " 'opp_team_dribble_tackles_pct',\n",
    " 'opp_team_aerials_won_pct',\n",
    " 'opp_vs_team_possession',\n",
    " 'opp_vs_team_goals_per90',\n",
    " 'opp_vs_team_assists_per90',\n",
    " 'opp_vs_team_xg_per90',\n",
    " 'opp_vs_team_gk_save_pct',\n",
    " 'opp_vs_team_gk_clean_sheets_pct',\n",
    " 'opp_vs_team_gk_pct_passes_launched',\n",
    " 'opp_vs_team_gk_crosses_stopped_pct',\n",
    " 'opp_vs_team_shots_on_target_per90',\n",
    " 'opp_vs_team_passes_pct',\n",
    " 'opp_vs_team_passes_pct_short',\n",
    " 'opp_vs_team_passes_pct_medium',\n",
    " 'opp_vs_team_passes_pct_long',\n",
    " 'opp_vs_team_sca_per90',\n",
    " 'opp_vs_team_gca_per90',\n",
    " 'opp_vs_team_dribble_tackles_pct',\n",
    " 'opp_vs_team_dribbles_completed_pct',\n",
    " 'opp_vs_team_aerials_won_pct',\n",
    "                \n",
    " 'vote_avg',\n",
    " 'vote_std']\n",
    "\n",
    "features_rel_gk = [\n",
    " 'gk_shots_on_target_against',\n",
    " 'gk_saves',\n",
    " 'gk_free_kick_goals_against',\n",
    " 'gk_corner_kick_goals_against',\n",
    " 'gk_own_goals_against',\n",
    " 'gk_psxg',\n",
    " 'gk_psnpxg_per_shot_on_target_against',\n",
    " 'gk_psxg_net',\n",
    " 'gk_passes_completed_launched',\n",
    " 'gk_passes_launched',\n",
    " 'gk_passes',\n",
    " 'gk_passes_throws',\n",
    " 'gk_goal_kicks',\n",
    " 'gk_crosses',\n",
    " 'gk_crosses_stopped',\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "5e3694a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def player_match_data_ext_gk(player, pteam, oppteam):\n",
    "    pdata = player_match_data(player, pteam, oppteam)\n",
    "    \n",
    "    if(not isinstance(pdata, pd.DataFrame)):\n",
    "        return None\n",
    "    \n",
    "    if(pdata['gk_games'][0] <= 0):\n",
    "        return None\n",
    "    \n",
    "    out = pd.concat([pdata[features_abs_gk], pdata[features_rel_gk]], axis = 1)\n",
    "    \n",
    "    out[features_rel_gk] = out[features_rel_gk] / max(pdata['minutes'][0], 1)\n",
    "\n",
    "    return out\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "9b671646",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gk_games</th>\n",
       "      <th>gk_games_starts</th>\n",
       "      <th>gk_minutes</th>\n",
       "      <th>gk_goals_against_per90</th>\n",
       "      <th>gk_save_pct</th>\n",
       "      <th>gk_clean_sheets_pct</th>\n",
       "      <th>gk_psxg_net_per90</th>\n",
       "      <th>gk_passes_pct_launched</th>\n",
       "      <th>gk_pct_passes_launched</th>\n",
       "      <th>gk_passes_length_avg</th>\n",
       "      <th>...</th>\n",
       "      <th>gk_psxg</th>\n",
       "      <th>gk_psnpxg_per_shot_on_target_against</th>\n",
       "      <th>gk_psxg_net</th>\n",
       "      <th>gk_passes_completed_launched</th>\n",
       "      <th>gk_passes_launched</th>\n",
       "      <th>gk_passes</th>\n",
       "      <th>gk_passes_throws</th>\n",
       "      <th>gk_goal_kicks</th>\n",
       "      <th>gk_crosses</th>\n",
       "      <th>gk_crosses_stopped</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Consigli</th>\n",
       "      <td>14.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>1260</td>\n",
       "      <td>1.36</td>\n",
       "      <td>58.5</td>\n",
       "      <td>35.7</td>\n",
       "      <td>-0.17</td>\n",
       "      <td>41.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>34.7</td>\n",
       "      <td>...</td>\n",
       "      <td>16.6</td>\n",
       "      <td>0.37</td>\n",
       "      <td>-2.4</td>\n",
       "      <td>84.0</td>\n",
       "      <td>205.0</td>\n",
       "      <td>523.0</td>\n",
       "      <td>77.0</td>\n",
       "      <td>134.0</td>\n",
       "      <td>189.0</td>\n",
       "      <td>13.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1 rows × 98 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          gk_games  gk_games_starts  gk_minutes  gk_goals_against_per90  \\\n",
       "Consigli      14.0             14.0        1260                    1.36   \n",
       "\n",
       "          gk_save_pct  gk_clean_sheets_pct  gk_psxg_net_per90  \\\n",
       "Consigli         58.5                 35.7              -0.17   \n",
       "\n",
       "          gk_passes_pct_launched  gk_pct_passes_launched  \\\n",
       "Consigli                    41.0                    31.0   \n",
       "\n",
       "          gk_passes_length_avg  ...  gk_psxg  \\\n",
       "Consigli                  34.7  ...     16.6   \n",
       "\n",
       "          gk_psnpxg_per_shot_on_target_against  gk_psxg_net  \\\n",
       "Consigli                                  0.37         -2.4   \n",
       "\n",
       "          gk_passes_completed_launched  gk_passes_launched  gk_passes  \\\n",
       "Consigli                          84.0               205.0      523.0   \n",
       "\n",
       "          gk_passes_throws  gk_goal_kicks  gk_crosses  gk_crosses_stopped  \n",
       "Consigli              77.0          134.0       189.0                13.0  \n",
       "\n",
       "[1 rows x 98 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test = player_match_data_ext_gk('Consigli', 'Sassuolo', 'Atalanta')\n",
    "\n",
    "add_columns_gk = test.columns\n",
    "\n",
    "test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "33f805a9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "100\n",
      "200\n",
      "300\n",
      "400\n",
      "500\n",
      "600\n",
      "700\n",
      "800\n",
      "900\n",
      "1000\n",
      "1100\n"
     ]
    }
   ],
   "source": [
    "db_gk = votes.copy()\n",
    "\n",
    "delta = pd.DataFrame(columns = add_columns_gk)\n",
    "\n",
    "for i in range(db_gk.shape[0]):\n",
    "    ext_data = player_match_data_ext_gk(db_gk['player'][i], db_gk['team'][i], db_gk['oppteam'][i])\n",
    "    \n",
    "    if(i % 100 == 0):\n",
    "        print(i)\n",
    "        \n",
    "    if(not isinstance(ext_data, pd.DataFrame)):\n",
    "        continue\n",
    "    \n",
    "    ext_data = ext_data.rename(index = {db_gk['player'][i] : i})\n",
    "    \n",
    "    delta = pd.concat([delta, ext_data], axis = 0)\n",
    "    \n",
    "    \n",
    "    \n",
    "db_gk = pd.concat([db_gk, delta], axis = 1)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4d484846",
   "metadata": {},
   "outputs": [],
   "source": [
    "db_gk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "adab3577",
   "metadata": {},
   "outputs": [],
   "source": [
    "db_gk = db_gk.drop(db_gk[db_gk.gk_games != db_gk.gk_games].index) # remove non extisting players"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e7ae792d",
   "metadata": {},
   "outputs": [],
   "source": [
    "db_gk"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8dbb7476",
   "metadata": {},
   "source": [
    "Save goalkeepers players database to file."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d8b8b6da",
   "metadata": {},
   "outputs": [],
   "source": [
    "db_gk.to_excel('mid_outputs/database_entries_gk.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "54519b24",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 - matchday\n",
      "1 - player\n",
      "2 - team\n",
      "3 - oppteam\n",
      "4 - home\n",
      "5 - vote\n",
      "6 - goals\n",
      "7 - assists\n",
      "8 - cards_malus\n",
      "9 - fantavote\n",
      "10 - gk_games\n",
      "11 - gk_games_starts\n",
      "12 - gk_minutes\n",
      "13 - gk_goals_against_per90\n",
      "14 - gk_save_pct\n",
      "15 - gk_clean_sheets_pct\n",
      "16 - gk_psxg_net_per90\n",
      "17 - gk_passes_pct_launched\n",
      "18 - gk_pct_passes_launched\n",
      "19 - gk_passes_length_avg\n",
      "20 - gk_pct_goal_kicks_launched\n",
      "21 - gk_goal_kick_length_avg\n",
      "22 - gk_crosses_stopped_pct\n",
      "23 - gk_def_actions_outside_pen_area_per90\n",
      "24 - gk_avg_distance_def_actions\n",
      "25 - team_possession\n",
      "26 - team_goals_assists_per90\n",
      "27 - team_goals_pens_per90\n",
      "28 - team_goals_assists_pens_per90\n",
      "29 - team_xg_per90\n",
      "30 - team_gk_goals_against_per90\n",
      "31 - team_gk_save_pct\n",
      "32 - team_gk_clean_sheets_pct\n",
      "33 - team_passes_pct\n",
      "34 - team_passes_pct_medium\n",
      "35 - team_passes_pct_long\n",
      "36 - team_sca_per90\n",
      "37 - team_gca_per90\n",
      "38 - team_dribble_tackles_pct\n",
      "39 - team_aerials_won_pct\n",
      "40 - vs_team_possession\n",
      "41 - vs_team_goals_per90\n",
      "42 - vs_team_assists_per90\n",
      "43 - vs_team_xg_per90\n",
      "44 - vs_team_gk_save_pct\n",
      "45 - vs_team_gk_clean_sheets_pct\n",
      "46 - vs_team_gk_pct_passes_launched\n",
      "47 - vs_team_gk_crosses_stopped_pct\n",
      "48 - vs_team_shots_on_target_per90\n",
      "49 - vs_team_passes_pct\n",
      "50 - vs_team_passes_pct_short\n",
      "51 - vs_team_passes_pct_medium\n",
      "52 - vs_team_passes_pct_long\n",
      "53 - vs_team_sca_per90\n",
      "54 - vs_team_gca_per90\n",
      "55 - vs_team_dribble_tackles_pct\n",
      "56 - vs_team_dribbles_completed_pct\n",
      "57 - vs_team_aerials_won_pct\n",
      "58 - opp_team_possession\n",
      "59 - opp_team_goals_assists_per90\n",
      "60 - opp_team_goals_pens_per90\n",
      "61 - opp_team_goals_assists_pens_per90\n",
      "62 - opp_team_xg_per90\n",
      "63 - opp_team_gk_goals_against_per90\n",
      "64 - opp_team_gk_save_pct\n",
      "65 - opp_team_gk_clean_sheets_pct\n",
      "66 - opp_team_passes_pct\n",
      "67 - opp_team_passes_pct_medium\n",
      "68 - opp_team_passes_pct_long\n",
      "69 - opp_team_sca_per90\n",
      "70 - opp_team_gca_per90\n",
      "71 - opp_team_dribble_tackles_pct\n",
      "72 - opp_team_aerials_won_pct\n",
      "73 - opp_vs_team_possession\n",
      "74 - opp_vs_team_goals_per90\n",
      "75 - opp_vs_team_assists_per90\n",
      "76 - opp_vs_team_xg_per90\n",
      "77 - opp_vs_team_gk_save_pct\n",
      "78 - opp_vs_team_gk_clean_sheets_pct\n",
      "79 - opp_vs_team_gk_pct_passes_launched\n",
      "80 - opp_vs_team_gk_crosses_stopped_pct\n",
      "81 - opp_vs_team_shots_on_target_per90\n",
      "82 - opp_vs_team_passes_pct\n",
      "83 - opp_vs_team_passes_pct_short\n",
      "84 - opp_vs_team_passes_pct_medium\n",
      "85 - opp_vs_team_passes_pct_long\n",
      "86 - opp_vs_team_sca_per90\n",
      "87 - opp_vs_team_gca_per90\n",
      "88 - opp_vs_team_dribble_tackles_pct\n",
      "89 - opp_vs_team_dribbles_completed_pct\n",
      "90 - opp_vs_team_aerials_won_pct\n",
      "91 - vote_avg\n",
      "92 - vote_std\n",
      "93 - gk_shots_on_target_against\n",
      "94 - gk_saves\n",
      "95 - gk_free_kick_goals_against\n",
      "96 - gk_corner_kick_goals_against\n",
      "97 - gk_own_goals_against\n",
      "98 - gk_psxg\n",
      "99 - gk_psnpxg_per_shot_on_target_against\n",
      "100 - gk_psxg_net\n",
      "101 - gk_passes_completed_launched\n",
      "102 - gk_passes_launched\n",
      "103 - gk_passes\n",
      "104 - gk_passes_throws\n",
      "105 - gk_goal_kicks\n",
      "106 - gk_crosses\n",
      "107 - gk_crosses_stopped\n"
     ]
    }
   ],
   "source": [
    "for i in range(db_gk.columns.shape[0]):\n",
    "    print(str(i) + \" - \" + str(db_gk.columns[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b0209868",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}