rams/fantabeto
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
c4a50cecd212bf78614bbb0ab4cace77977736a9
fantabeto/.ipynb_checkpoints/5_scraping_match_probable_players-checkpoint.ipynb
T
Giuseppe Musicco c4a50cecd2 Update Jan before matchday 20
2023-01-28 11:06:11 +01:00

763 lines
23 KiB
Plaintext
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
{
"cells": [
{
"cell_type": "markdown",
"id": "fd8a6cb9",
"metadata": {},
"source": [
"Scraping for playing probability for the current Serie A matchday.\n",
"\n",
"Data from http://fantacalcio.it"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "4c7b7d26",
"metadata": {},
"outputs": [],
"source": [
"import requests\n",
"from bs4 import BeautifulSoup\n",
"import pandas as pd\n",
"import numpy as np\n",
"import re\n",
"import sys, getopt\n",
"import csv\n"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "91897dbb",
"metadata": {},
"outputs": [],
"source": [
"res = requests.get('https://www.fantacalcio.it/probabili-formazioni-serie-a')"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "7ba89027",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>player</th>\n",
" <th>starter</th>\n",
" <th>percentage</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Skorupski</td>\n",
" <td>1</td>\n",
" <td>100.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>Posch</td>\n",
" <td>1</td>\n",
" <td>100.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>Soumaoro</td>\n",
" <td>1</td>\n",
" <td>100.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>Lucumi'</td>\n",
" <td>1</td>\n",
" <td>100.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>Cambiaso</td>\n",
" <td>1</td>\n",
" <td>100.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>435</th>\n",
" <td>Verdi</td>\n",
" <td>0</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>436</th>\n",
" <td>Kallon</td>\n",
" <td>0</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>437</th>\n",
" <td>Piccoli</td>\n",
" <td>0</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>438</th>\n",
" <td>Ngonge</td>\n",
" <td>0</td>\n",
" <td>25.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>439</th>\n",
" <td>Braaf</td>\n",
" <td>0</td>\n",
" <td>40.0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>440 rows × 3 columns</p>\n",
"</div>"
],
"text/plain": [
" player starter percentage\n",
"0 Skorupski 1 100.0\n",
"1 Posch 1 100.0\n",
"2 Soumaoro 1 100.0\n",
"3 Lucumi' 1 100.0\n",
"4 Cambiaso 1 100.0\n",
".. ... ... ...\n",
"435 Verdi 0 55.0\n",
"436 Kallon 0 55.0\n",
"437 Piccoli 0 55.0\n",
"438 Ngonge 0 25.0\n",
"439 Braaf 0 40.0\n",
"\n",
"[440 rows x 3 columns]"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"soup = BeautifulSoup(res.text)\n",
"\n",
"groups = soup.findAll('ul', attrs = {'class' : 'player-list'})\n",
"\n",
"probables = pd.DataFrame(columns = ['player', 'starter', 'percentage'])\n",
"\n",
"for i in range (len(groups)):\n",
" players = groups[i].findAll('a', attrs = {'class' : 'player-name'})\n",
" bars = groups[i].findAll('div', attrs = {'class' : 'progress-bar'})\n",
" \n",
" starter_string = groups[i]['class'][1]\n",
" if(starter_string == 'starters'):\n",
" starter = 1\n",
" else:\n",
" starter = 0\n",
" \n",
" for j in range(len(players)):\n",
" player = players[j].contents[0].replace('\\n \\n ', '')\n",
" perc = float(bars[j]['aria-valuenow'])\n",
" \n",
" row_df = pd.DataFrame([[player, starter, perc]], columns = probables.columns)\n",
" \n",
" probables = pd.concat([probables, row_df], ignore_index = True)\n",
" \n",
"probables "
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "29e5b389",
"metadata": {},
"outputs": [],
"source": [
"probables = probables.set_index('player')"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "d711726e",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>player1</th>\n",
" <th>player2</th>\n",
" <th>percentage</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Colombo</td>\n",
" <td>Ceesay</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>Pezzella Giu.</td>\n",
" <td>Gallo</td>\n",
" <td>65.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>Banda</td>\n",
" <td>Di Francesco F.</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>Lovato</td>\n",
" <td>Bronn</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>Nicolussi Caviglia</td>\n",
" <td>Bohinen</td>\n",
" <td>65.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>Dia</td>\n",
" <td>Bonazzoli</td>\n",
" <td>65.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>Haas</td>\n",
" <td>Akpa Akpro</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>Satriano</td>\n",
" <td>Bajrami</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>Ebuehi</td>\n",
" <td>Stojanovic</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>Buongiorno</td>\n",
" <td>Adopo</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>10</th>\n",
" <td>Linetty</td>\n",
" <td>Lukic</td>\n",
" <td>65.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>11</th>\n",
" <td>Sanabria</td>\n",
" <td>Seck</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>12</th>\n",
" <td>Sanabria</td>\n",
" <td>Radonjic</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>13</th>\n",
" <td>Dessers</td>\n",
" <td>Ciofani D.</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>14</th>\n",
" <td>Chiriches</td>\n",
" <td>Bianchetti</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>15</th>\n",
" <td>Asllani</td>\n",
" <td>Gagliardini</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16</th>\n",
" <td>Darmian</td>\n",
" <td>D'ambrosio</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>17</th>\n",
" <td>Dumfries</td>\n",
" <td>D'ambrosio</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>18</th>\n",
" <td>Maehle</td>\n",
" <td>Ruggeri</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>19</th>\n",
" <td>Boga</td>\n",
" <td>Pasalic</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>20</th>\n",
" <td>Rincon</td>\n",
" <td>Vieira</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>21</th>\n",
" <td>Gabbiadini</td>\n",
" <td>Quagliarella</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>22</th>\n",
" <td>Giroud</td>\n",
" <td>Origi</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>23</th>\n",
" <td>De Ketelaere</td>\n",
" <td>Diaz B.</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>24</th>\n",
" <td>Saelemaekers</td>\n",
" <td>Messias</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>25</th>\n",
" <td>Lopez M.</td>\n",
" <td>Obiang</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>26</th>\n",
" <td>Defrel</td>\n",
" <td>Alvarez A.</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>27</th>\n",
" <td>Traore' Hj.</td>\n",
" <td>Thorstvedt</td>\n",
" <td>65.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>28</th>\n",
" <td>Mckennie</td>\n",
" <td>Chiesa</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>29</th>\n",
" <td>Kean</td>\n",
" <td>Milik</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>30</th>\n",
" <td>Fagioli</td>\n",
" <td>Miretti</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>31</th>\n",
" <td>Rovella</td>\n",
" <td>Machin</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>32</th>\n",
" <td>Mari'</td>\n",
" <td>Caldirola</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>33</th>\n",
" <td>Petagna</td>\n",
" <td>Mota</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>34</th>\n",
" <td>Marusic</td>\n",
" <td>Hysaj</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>35</th>\n",
" <td>Cataldi</td>\n",
" <td>Marcos Antonio</td>\n",
" <td>65.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>36</th>\n",
" <td>Igor</td>\n",
" <td>Martinez Quarta</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>37</th>\n",
" <td>Kouame'</td>\n",
" <td>Jovic</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>38</th>\n",
" <td>Gonzalez N.</td>\n",
" <td>Saponara</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>39</th>\n",
" <td>Elmas</td>\n",
" <td>Lozano</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>40</th>\n",
" <td>Elmas</td>\n",
" <td>Politano</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>41</th>\n",
" <td>Mario Rui</td>\n",
" <td>Olivera</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>42</th>\n",
" <td>Spinazzola</td>\n",
" <td>El Shaarawy</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>43</th>\n",
" <td>Matic</td>\n",
" <td>Bove</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>44</th>\n",
" <td>Arslan</td>\n",
" <td>Makengo</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>45</th>\n",
" <td>Samardzic</td>\n",
" <td>Lovric</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>46</th>\n",
" <td>Lasagna</td>\n",
" <td>Kallon</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" player1 player2 percentage\n",
"0 Colombo Ceesay 55.0\n",
"1 Pezzella Giu. Gallo 65.0\n",
"2 Banda Di Francesco F. 60.0\n",
"3 Lovato Bronn 55.0\n",
"4 Nicolussi Caviglia Bohinen 65.0\n",
"5 Dia Bonazzoli 65.0\n",
"6 Haas Akpa Akpro 60.0\n",
"7 Satriano Bajrami 55.0\n",
"8 Ebuehi Stojanovic 60.0\n",
"9 Buongiorno Adopo 60.0\n",
"10 Linetty Lukic 65.0\n",
"11 Sanabria Seck 55.0\n",
"12 Sanabria Radonjic 55.0\n",
"13 Dessers Ciofani D. 55.0\n",
"14 Chiriches Bianchetti 60.0\n",
"15 Asllani Gagliardini 55.0\n",
"16 Darmian D'ambrosio 60.0\n",
"17 Dumfries D'ambrosio 55.0\n",
"18 Maehle Ruggeri 60.0\n",
"19 Boga Pasalic 55.0\n",
"20 Rincon Vieira 55.0\n",
"21 Gabbiadini Quagliarella 55.0\n",
"22 Giroud Origi 60.0\n",
"23 De Ketelaere Diaz B. 55.0\n",
"24 Saelemaekers Messias 60.0\n",
"25 Lopez M. Obiang 55.0\n",
"26 Defrel Alvarez A. 60.0\n",
"27 Traore' Hj. Thorstvedt 65.0\n",
"28 Mckennie Chiesa 60.0\n",
"29 Kean Milik 60.0\n",
"30 Fagioli Miretti 55.0\n",
"31 Rovella Machin 55.0\n",
"32 Mari' Caldirola 60.0\n",
"33 Petagna Mota 60.0\n",
"34 Marusic Hysaj 60.0\n",
"35 Cataldi Marcos Antonio 65.0\n",
"36 Igor Martinez Quarta 60.0\n",
"37 Kouame' Jovic 55.0\n",
"38 Gonzalez N. Saponara 55.0\n",
"39 Elmas Lozano 55.0\n",
"40 Elmas Politano 55.0\n",
"41 Mario Rui Olivera 55.0\n",
"42 Spinazzola El Shaarawy 60.0\n",
"43 Matic Bove 55.0\n",
"44 Arslan Makengo 60.0\n",
"45 Samardzic Lovric 55.0\n",
"46 Lasagna Kallon 60.0"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"groups = soup.findAll('ul', attrs = {'class' : 'ballot-list'})\n",
"\n",
"ballots = pd.DataFrame(columns = ['player1', 'player2', 'percentage'])\n",
"for i in range (len(groups)):\n",
" players = groups[i].findAll('a', attrs = {'class' : 'player-name'})\n",
" percs = groups[i].findAll('strong', attrs = {'class' : 'percentage'})\n",
" \n",
" player1 = players[0].contents[0].replace('\\n \\n ', '')\n",
" player2 = players[1].contents[0].replace('\\n \\n ', '')\n",
" perc = float(percs[0].contents[0].replace('%', ''))\n",
" \n",
" row_df = pd.DataFrame([[player1, player2, perc]], columns = ballots.columns)\n",
" \n",
" ballots = pd.concat([ballots, row_df], ignore_index = True)\n",
" \n",
"ballots"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "0171c61d",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>starter</th>\n",
" <th>percentage</th>\n",
" </tr>\n",
" <tr>\n",
" <th>player</th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>Skorupski</th>\n",
" <td>1</td>\n",
" <td>100.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>Posch</th>\n",
" <td>1</td>\n",
" <td>100.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>Soumaoro</th>\n",
" <td>1</td>\n",
" <td>100.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>Lucumi'</th>\n",
" <td>1</td>\n",
" <td>100.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>Cambiaso</th>\n",
" <td>1</td>\n",
" <td>100.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>Verdi</th>\n",
" <td>0</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>Kallon</th>\n",
" <td>0.4</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>Piccoli</th>\n",
" <td>0</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>Ngonge</th>\n",
" <td>0</td>\n",
" <td>25.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>Braaf</th>\n",
" <td>0</td>\n",
" <td>40.0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>440 rows × 2 columns</p>\n",
"</div>"
],
"text/plain": [
" starter percentage\n",
"player \n",
"Skorupski 1 100.0\n",
"Posch 1 100.0\n",
"Soumaoro 1 100.0\n",
"Lucumi' 1 100.0\n",
"Cambiaso 1 100.0\n",
"... ... ...\n",
"Verdi 0 55.0\n",
"Kallon 0.4 55.0\n",
"Piccoli 0 55.0\n",
"Ngonge 0 25.0\n",
"Braaf 0 40.0\n",
"\n",
"[440 rows x 2 columns]"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"for i in range(ballots.shape[0]):\n",
" p1 = ballots['player1'][i]\n",
" p2 = ballots['player2'][i]\n",
" \n",
" perc = float(ballots['percentage'][i])\n",
" \n",
" probables.at[p1, 'starter'] = perc/100\n",
" probables.at[p2, 'starter'] = 1 - perc/100\n",
" \n",
"probables"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "8b5468aa",
"metadata": {},
"outputs": [],
"source": [
"probables.to_excel('mid_outputs/match_probable_players.xlsx') "
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.13"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Reference in New Issue View Git Blame Copy Permalink