{ "cells": [ { "cell_type": "markdown", "id": "fd8a6cb9", "metadata": {}, "source": [ "Scraping for playing probability for the current Serie A matchday.\n", "\n", "Data from http://fantacalcio.it" ] }, { "cell_type": "code", "execution_count": 1, "id": "4c7b7d26", "metadata": {}, "outputs": [], "source": [ "import requests\n", "from bs4 import BeautifulSoup\n", "import pandas as pd\n", "import numpy as np\n", "import re\n", "import sys, getopt\n", "import csv\n" ] }, { "cell_type": "code", "execution_count": 2, "id": "91897dbb", "metadata": {}, "outputs": [], "source": [ "res = requests.get('https://www.fantacalcio.it/probabili-formazioni-serie-a')" ] }, { "cell_type": "code", "execution_count": 3, "id": "7ba89027", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
playerstarterpercentage
0Szczesny180.0
1Danilo190.0
2Bremer190.0
3Gatti160.0
4Weah180.0
............
474Fabbian055.0
475Aebischer060.0
476El Azzouzi050.0
477Urbanski015.0
478Ndoye060.0
\n", "

479 rows × 3 columns

\n", "
" ], "text/plain": [ " player starter percentage\n", "0 Szczesny 1 80.0\n", "1 Danilo 1 90.0\n", "2 Bremer 1 90.0\n", "3 Gatti 1 60.0\n", "4 Weah 1 80.0\n", ".. ... ... ...\n", "474 Fabbian 0 55.0\n", "475 Aebischer 0 60.0\n", "476 El Azzouzi 0 50.0\n", "477 Urbanski 0 15.0\n", "478 Ndoye 0 60.0\n", "\n", "[479 rows x 3 columns]" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "soup = BeautifulSoup(res.text)\n", "\n", "groups = soup.findAll('ul', attrs = {'class' : 'player-list'})\n", "\n", "probables = pd.DataFrame(columns = ['player', 'starter', 'percentage'])\n", "\n", "for i in range (len(groups)):\n", " players = groups[i].findAll('a', attrs = {'class' : 'player-name'})\n", " bars = groups[i].findAll('div', attrs = {'class' : 'progress-bar'})\n", " \n", " starter_string = groups[i]['class'][1]\n", " if(starter_string == 'starters'):\n", " starter = 1\n", " else:\n", " starter = 0\n", " \n", " for j in range(len(players)):\n", " player = players[j].contents[1].contents[0]\n", " perc = float(bars[j]['aria-valuenow'])\n", " \n", " row_df = pd.DataFrame([[player, starter, perc]], columns = probables.columns)\n", " \n", " probables = pd.concat([probables, row_df], ignore_index = True)\n", " \n", "probables " ] }, { "cell_type": "code", "execution_count": 4, "id": "29e5b389", "metadata": {}, "outputs": [], "source": [ "probables = probables.set_index('player')" ] }, { "cell_type": "code", "execution_count": 5, "id": "d711726e", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
player1player2percentage
0FagioliMiretti60.0
1KosticCambiaso55.0
2GattiAlex Sandro60.0
3KamadaGuendouzi55.0
4De VrijAcerbi60.0
5MkhitaryanFrattesi55.0
6DumfriesCuadrado65.0
7GiroudOkafor55.0
8KrunicMusah65.0
9BadeljThorsby60.0
10SabelliMartin60.0
11PolitanoRaspadori60.0
12OliveraMario Rui55.0
13LuvumboOristanio60.0
14PetagnaShomurodov55.0
15LuccaSuccess65.0
16Ferreira J.Ebosele55.0
17Kamara H.Zemura60.0
18Soule'Baez65.0
19GelliCaso55.0
20ThorstvedtBajrami55.0
21ColomboMota60.0
22BirindelliKyriakopoulos60.0
23KabaGonzalez J.60.0
24DjimsitiToloi55.0
25De KetelaereLookman60.0
26Ederson D.s.Pasalic55.0
27DybalaEl Shaarawy60.0
28KristensenCelik55.0
29SpinazzolaZalewski55.0
30FazziniGrassi60.0
31EbuehiBereszynski60.0
32Pezzella Giu.Cacace55.0
33BotheimDia65.0
34IlicLinetty65.0
35BellanovaLazaro55.0
36LazovicFolorunsho55.0
37MboulaBonazzoli60.0
38KarlssonNdoye60.0
39OrsoliniNdoye60.0
\n", "
" ], "text/plain": [ " player1 player2 percentage\n", "0 Fagioli Miretti 60.0\n", "1 Kostic Cambiaso 55.0\n", "2 Gatti Alex Sandro 60.0\n", "3 Kamada Guendouzi 55.0\n", "4 De Vrij Acerbi 60.0\n", "5 Mkhitaryan Frattesi 55.0\n", "6 Dumfries Cuadrado 65.0\n", "7 Giroud Okafor 55.0\n", "8 Krunic Musah 65.0\n", "9 Badelj Thorsby 60.0\n", "10 Sabelli Martin 60.0\n", "11 Politano Raspadori 60.0\n", "12 Olivera Mario Rui 55.0\n", "13 Luvumbo Oristanio 60.0\n", "14 Petagna Shomurodov 55.0\n", "15 Lucca Success 65.0\n", "16 Ferreira J. Ebosele 55.0\n", "17 Kamara H. Zemura 60.0\n", "18 Soule' Baez 65.0\n", "19 Gelli Caso 55.0\n", "20 Thorstvedt Bajrami 55.0\n", "21 Colombo Mota 60.0\n", "22 Birindelli Kyriakopoulos 60.0\n", "23 Kaba Gonzalez J. 60.0\n", "24 Djimsiti Toloi 55.0\n", "25 De Ketelaere Lookman 60.0\n", "26 Ederson D.s. Pasalic 55.0\n", "27 Dybala El Shaarawy 60.0\n", "28 Kristensen Celik 55.0\n", "29 Spinazzola Zalewski 55.0\n", "30 Fazzini Grassi 60.0\n", "31 Ebuehi Bereszynski 60.0\n", "32 Pezzella Giu. Cacace 55.0\n", "33 Botheim Dia 65.0\n", "34 Ilic Linetty 65.0\n", "35 Bellanova Lazaro 55.0\n", "36 Lazovic Folorunsho 55.0\n", "37 Mboula Bonazzoli 60.0\n", "38 Karlsson Ndoye 60.0\n", "39 Orsolini Ndoye 60.0" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "groups = soup.findAll('ul', attrs = {'class' : 'ballot-list'})\n", "\n", "ballots = pd.DataFrame(columns = ['player1', 'player2', 'percentage'])\n", "for i in range (len(groups)):\n", " players = groups[i].findAll('a', attrs = {'class' : 'player-name'})\n", " percs = groups[i].findAll('strong', attrs = {'class' : 'percentage'})\n", " \n", " player1 = players[0].contents[1].contents[0]\n", " player2 = players[1].contents[1].contents[0]\n", " perc = float(percs[0].contents[0].replace('%', ''))\n", " \n", " row_df = pd.DataFrame([[player1, player2, perc]], columns = ballots.columns)\n", " \n", " ballots = pd.concat([ballots, row_df], ignore_index = True)\n", " \n", "ballots" ] }, { "cell_type": "code", "execution_count": 6, "id": "0171c61d", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
starterpercentage
player
Szczesny180.0
Danilo190.0
Bremer190.0
Gatti0.660.0
Weah180.0
.........
Fabbian055.0
Aebischer060.0
El Azzouzi050.0
Urbanski015.0
Ndoye0.460.0
\n", "

479 rows × 2 columns

\n", "
" ], "text/plain": [ " starter percentage\n", "player \n", "Szczesny 1 80.0\n", "Danilo 1 90.0\n", "Bremer 1 90.0\n", "Gatti 0.6 60.0\n", "Weah 1 80.0\n", "... ... ...\n", "Fabbian 0 55.0\n", "Aebischer 0 60.0\n", "El Azzouzi 0 50.0\n", "Urbanski 0 15.0\n", "Ndoye 0.4 60.0\n", "\n", "[479 rows x 2 columns]" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "for i in range(ballots.shape[0]):\n", " p1 = ballots['player1'][i]\n", " p2 = ballots['player2'][i]\n", " \n", " perc = float(ballots['percentage'][i])\n", " \n", " probables.at[p1, 'starter'] = perc/100\n", " probables.at[p2, 'starter'] = 1 - perc/100\n", " \n", "probables" ] }, { "cell_type": "code", "execution_count": 7, "id": "8b5468aa", "metadata": {}, "outputs": [], "source": [ "probables.to_excel('mid_outputs/match_probable_players.xlsx') " ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.13" } }, "nbformat": 4, "nbformat_minor": 5 }