rams/fantabeto
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
14923289583410043a6671d9b622ae2f19f6b029
fantabeto/.ipynb_checkpoints/5_scraping_match_probable_players-checkpoint.ipynb
T
Giuseppe Musicco 1492328958 Code updated for Serie A 2023/2024 
- Reorganization of data folders
- Code adapted for the new Serie A season, also considering only a small number of league games has been played
- Stats from other leagues are considered for players at their first Serie A season (rookies), for now without any adaptation based on league difficulty
- Minor fixes
2023-09-09 19:31:52 +02:00

714 lines
21 KiB
Plaintext
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
{
"cells": [
{
"cell_type": "markdown",
"id": "fd8a6cb9",
"metadata": {},
"source": [
"Scraping for playing probability for the current Serie A matchday.\n",
"\n",
"Data from http://fantacalcio.it"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "4c7b7d26",
"metadata": {},
"outputs": [],
"source": [
"import requests\n",
"from bs4 import BeautifulSoup\n",
"import pandas as pd\n",
"import numpy as np\n",
"import re\n",
"import sys, getopt\n",
"import csv\n"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "91897dbb",
"metadata": {},
"outputs": [],
"source": [
"res = requests.get('https://www.fantacalcio.it/probabili-formazioni-serie-a')"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "7ba89027",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>player</th>\n",
" <th>starter</th>\n",
" <th>percentage</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Szczesny</td>\n",
" <td>1</td>\n",
" <td>80.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>Danilo</td>\n",
" <td>1</td>\n",
" <td>90.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>Bremer</td>\n",
" <td>1</td>\n",
" <td>90.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>Gatti</td>\n",
" <td>1</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>Weah</td>\n",
" <td>1</td>\n",
" <td>80.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>474</th>\n",
" <td>Fabbian</td>\n",
" <td>0</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>475</th>\n",
" <td>Aebischer</td>\n",
" <td>0</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>476</th>\n",
" <td>El Azzouzi</td>\n",
" <td>0</td>\n",
" <td>50.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>477</th>\n",
" <td>Urbanski</td>\n",
" <td>0</td>\n",
" <td>15.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>478</th>\n",
" <td>Ndoye</td>\n",
" <td>0</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>479 rows × 3 columns</p>\n",
"</div>"
],
"text/plain": [
" player starter percentage\n",
"0 Szczesny 1 80.0\n",
"1 Danilo 1 90.0\n",
"2 Bremer 1 90.0\n",
"3 Gatti 1 60.0\n",
"4 Weah 1 80.0\n",
".. ... ... ...\n",
"474 Fabbian 0 55.0\n",
"475 Aebischer 0 60.0\n",
"476 El Azzouzi 0 50.0\n",
"477 Urbanski 0 15.0\n",
"478 Ndoye 0 60.0\n",
"\n",
"[479 rows x 3 columns]"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"soup = BeautifulSoup(res.text)\n",
"\n",
"groups = soup.findAll('ul', attrs = {'class' : 'player-list'})\n",
"\n",
"probables = pd.DataFrame(columns = ['player', 'starter', 'percentage'])\n",
"\n",
"for i in range (len(groups)):\n",
" players = groups[i].findAll('a', attrs = {'class' : 'player-name'})\n",
" bars = groups[i].findAll('div', attrs = {'class' : 'progress-bar'})\n",
" \n",
" starter_string = groups[i]['class'][1]\n",
" if(starter_string == 'starters'):\n",
" starter = 1\n",
" else:\n",
" starter = 0\n",
" \n",
" for j in range(len(players)):\n",
" player = players[j].contents[1].contents[0]\n",
" perc = float(bars[j]['aria-valuenow'])\n",
" \n",
" row_df = pd.DataFrame([[player, starter, perc]], columns = probables.columns)\n",
" \n",
" probables = pd.concat([probables, row_df], ignore_index = True)\n",
" \n",
"probables "
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "29e5b389",
"metadata": {},
"outputs": [],
"source": [
"probables = probables.set_index('player')"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "d711726e",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>player1</th>\n",
" <th>player2</th>\n",
" <th>percentage</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Fagioli</td>\n",
" <td>Miretti</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>Kostic</td>\n",
" <td>Cambiaso</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>Gatti</td>\n",
" <td>Alex Sandro</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>Kamada</td>\n",
" <td>Guendouzi</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>De Vrij</td>\n",
" <td>Acerbi</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>Mkhitaryan</td>\n",
" <td>Frattesi</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>Dumfries</td>\n",
" <td>Cuadrado</td>\n",
" <td>65.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>Giroud</td>\n",
" <td>Okafor</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>Krunic</td>\n",
" <td>Musah</td>\n",
" <td>65.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>Badelj</td>\n",
" <td>Thorsby</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>10</th>\n",
" <td>Sabelli</td>\n",
" <td>Martin</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>11</th>\n",
" <td>Politano</td>\n",
" <td>Raspadori</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>12</th>\n",
" <td>Olivera</td>\n",
" <td>Mario Rui</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>13</th>\n",
" <td>Luvumbo</td>\n",
" <td>Oristanio</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>14</th>\n",
" <td>Petagna</td>\n",
" <td>Shomurodov</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>15</th>\n",
" <td>Lucca</td>\n",
" <td>Success</td>\n",
" <td>65.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16</th>\n",
" <td>Ferreira J.</td>\n",
" <td>Ebosele</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>17</th>\n",
" <td>Kamara H.</td>\n",
" <td>Zemura</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>18</th>\n",
" <td>Soule'</td>\n",
" <td>Baez</td>\n",
" <td>65.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>19</th>\n",
" <td>Gelli</td>\n",
" <td>Caso</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>20</th>\n",
" <td>Thorstvedt</td>\n",
" <td>Bajrami</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>21</th>\n",
" <td>Colombo</td>\n",
" <td>Mota</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>22</th>\n",
" <td>Birindelli</td>\n",
" <td>Kyriakopoulos</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>23</th>\n",
" <td>Kaba</td>\n",
" <td>Gonzalez J.</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>24</th>\n",
" <td>Djimsiti</td>\n",
" <td>Toloi</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>25</th>\n",
" <td>De Ketelaere</td>\n",
" <td>Lookman</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>26</th>\n",
" <td>Ederson D.s.</td>\n",
" <td>Pasalic</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>27</th>\n",
" <td>Dybala</td>\n",
" <td>El Shaarawy</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>28</th>\n",
" <td>Kristensen</td>\n",
" <td>Celik</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>29</th>\n",
" <td>Spinazzola</td>\n",
" <td>Zalewski</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>30</th>\n",
" <td>Fazzini</td>\n",
" <td>Grassi</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>31</th>\n",
" <td>Ebuehi</td>\n",
" <td>Bereszynski</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>32</th>\n",
" <td>Pezzella Giu.</td>\n",
" <td>Cacace</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>33</th>\n",
" <td>Botheim</td>\n",
" <td>Dia</td>\n",
" <td>65.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>34</th>\n",
" <td>Ilic</td>\n",
" <td>Linetty</td>\n",
" <td>65.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>35</th>\n",
" <td>Bellanova</td>\n",
" <td>Lazaro</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>36</th>\n",
" <td>Lazovic</td>\n",
" <td>Folorunsho</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>37</th>\n",
" <td>Mboula</td>\n",
" <td>Bonazzoli</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>38</th>\n",
" <td>Karlsson</td>\n",
" <td>Ndoye</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>39</th>\n",
" <td>Orsolini</td>\n",
" <td>Ndoye</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" player1 player2 percentage\n",
"0 Fagioli Miretti 60.0\n",
"1 Kostic Cambiaso 55.0\n",
"2 Gatti Alex Sandro 60.0\n",
"3 Kamada Guendouzi 55.0\n",
"4 De Vrij Acerbi 60.0\n",
"5 Mkhitaryan Frattesi 55.0\n",
"6 Dumfries Cuadrado 65.0\n",
"7 Giroud Okafor 55.0\n",
"8 Krunic Musah 65.0\n",
"9 Badelj Thorsby 60.0\n",
"10 Sabelli Martin 60.0\n",
"11 Politano Raspadori 60.0\n",
"12 Olivera Mario Rui 55.0\n",
"13 Luvumbo Oristanio 60.0\n",
"14 Petagna Shomurodov 55.0\n",
"15 Lucca Success 65.0\n",
"16 Ferreira J. Ebosele 55.0\n",
"17 Kamara H. Zemura 60.0\n",
"18 Soule' Baez 65.0\n",
"19 Gelli Caso 55.0\n",
"20 Thorstvedt Bajrami 55.0\n",
"21 Colombo Mota 60.0\n",
"22 Birindelli Kyriakopoulos 60.0\n",
"23 Kaba Gonzalez J. 60.0\n",
"24 Djimsiti Toloi 55.0\n",
"25 De Ketelaere Lookman 60.0\n",
"26 Ederson D.s. Pasalic 55.0\n",
"27 Dybala El Shaarawy 60.0\n",
"28 Kristensen Celik 55.0\n",
"29 Spinazzola Zalewski 55.0\n",
"30 Fazzini Grassi 60.0\n",
"31 Ebuehi Bereszynski 60.0\n",
"32 Pezzella Giu. Cacace 55.0\n",
"33 Botheim Dia 65.0\n",
"34 Ilic Linetty 65.0\n",
"35 Bellanova Lazaro 55.0\n",
"36 Lazovic Folorunsho 55.0\n",
"37 Mboula Bonazzoli 60.0\n",
"38 Karlsson Ndoye 60.0\n",
"39 Orsolini Ndoye 60.0"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"groups = soup.findAll('ul', attrs = {'class' : 'ballot-list'})\n",
"\n",
"ballots = pd.DataFrame(columns = ['player1', 'player2', 'percentage'])\n",
"for i in range (len(groups)):\n",
" players = groups[i].findAll('a', attrs = {'class' : 'player-name'})\n",
" percs = groups[i].findAll('strong', attrs = {'class' : 'percentage'})\n",
" \n",
" player1 = players[0].contents[1].contents[0]\n",
" player2 = players[1].contents[1].contents[0]\n",
" perc = float(percs[0].contents[0].replace('%', ''))\n",
" \n",
" row_df = pd.DataFrame([[player1, player2, perc]], columns = ballots.columns)\n",
" \n",
" ballots = pd.concat([ballots, row_df], ignore_index = True)\n",
" \n",
"ballots"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "0171c61d",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>starter</th>\n",
" <th>percentage</th>\n",
" </tr>\n",
" <tr>\n",
" <th>player</th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>Szczesny</th>\n",
" <td>1</td>\n",
" <td>80.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>Danilo</th>\n",
" <td>1</td>\n",
" <td>90.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>Bremer</th>\n",
" <td>1</td>\n",
" <td>90.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>Gatti</th>\n",
" <td>0.6</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>Weah</th>\n",
" <td>1</td>\n",
" <td>80.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>Fabbian</th>\n",
" <td>0</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>Aebischer</th>\n",
" <td>0</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>El Azzouzi</th>\n",
" <td>0</td>\n",
" <td>50.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>Urbanski</th>\n",
" <td>0</td>\n",
" <td>15.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>Ndoye</th>\n",
" <td>0.4</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>479 rows × 2 columns</p>\n",
"</div>"
],
"text/plain": [
" starter percentage\n",
"player \n",
"Szczesny 1 80.0\n",
"Danilo 1 90.0\n",
"Bremer 1 90.0\n",
"Gatti 0.6 60.0\n",
"Weah 1 80.0\n",
"... ... ...\n",
"Fabbian 0 55.0\n",
"Aebischer 0 60.0\n",
"El Azzouzi 0 50.0\n",
"Urbanski 0 15.0\n",
"Ndoye 0.4 60.0\n",
"\n",
"[479 rows x 2 columns]"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"for i in range(ballots.shape[0]):\n",
" p1 = ballots['player1'][i]\n",
" p2 = ballots['player2'][i]\n",
" \n",
" perc = float(ballots['percentage'][i])\n",
" \n",
" probables.at[p1, 'starter'] = perc/100\n",
" probables.at[p2, 'starter'] = 1 - perc/100\n",
" \n",
"probables"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "8b5468aa",
"metadata": {},
"outputs": [],
"source": [
"probables.to_excel('mid_outputs/match_probable_players.xlsx') "
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.13"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Reference in New Issue View Git Blame Copy Permalink