{ "cells": [ { "cell_type": "markdown", "id": "fd8a6cb9", "metadata": {}, "source": [ "Scraping for playing probability for the current Serie A matchday.\n", "\n", "Data from http://fantacalcio.it" ] }, { "cell_type": "code", "execution_count": 1, "id": "4c7b7d26", "metadata": {}, "outputs": [], "source": [ "import requests\n", "from bs4 import BeautifulSoup\n", "import pandas as pd\n", "import numpy as np\n", "import re\n", "import sys, getopt\n", "import csv\n" ] }, { "cell_type": "code", "execution_count": 2, "id": "91897dbb", "metadata": {}, "outputs": [], "source": [ "res = requests.get('https://www.fantacalcio.it/probabili-formazioni-serie-a')" ] }, { "cell_type": "code", "execution_count": 3, "id": "7ba89027", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
playerstarterpercentage
0Skorupski1100.0
1Posch1100.0
2Soumaoro1100.0
3Lucumi'1100.0
4Cambiaso1100.0
............
435Verdi055.0
436Kallon055.0
437Piccoli055.0
438Ngonge025.0
439Braaf040.0
\n", "

440 rows × 3 columns

\n", "
" ], "text/plain": [ " player starter percentage\n", "0 Skorupski 1 100.0\n", "1 Posch 1 100.0\n", "2 Soumaoro 1 100.0\n", "3 Lucumi' 1 100.0\n", "4 Cambiaso 1 100.0\n", ".. ... ... ...\n", "435 Verdi 0 55.0\n", "436 Kallon 0 55.0\n", "437 Piccoli 0 55.0\n", "438 Ngonge 0 25.0\n", "439 Braaf 0 40.0\n", "\n", "[440 rows x 3 columns]" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "soup = BeautifulSoup(res.text)\n", "\n", "groups = soup.findAll('ul', attrs = {'class' : 'player-list'})\n", "\n", "probables = pd.DataFrame(columns = ['player', 'starter', 'percentage'])\n", "\n", "for i in range (len(groups)):\n", " players = groups[i].findAll('a', attrs = {'class' : 'player-name'})\n", " bars = groups[i].findAll('div', attrs = {'class' : 'progress-bar'})\n", " \n", " starter_string = groups[i]['class'][1]\n", " if(starter_string == 'starters'):\n", " starter = 1\n", " else:\n", " starter = 0\n", " \n", " for j in range(len(players)):\n", " player = players[j].contents[0].replace('\\n \\n ', '')\n", " perc = float(bars[j]['aria-valuenow'])\n", " \n", " row_df = pd.DataFrame([[player, starter, perc]], columns = probables.columns)\n", " \n", " probables = pd.concat([probables, row_df], ignore_index = True)\n", " \n", "probables " ] }, { "cell_type": "code", "execution_count": 4, "id": "29e5b389", "metadata": {}, "outputs": [], "source": [ "probables = probables.set_index('player')" ] }, { "cell_type": "code", "execution_count": 5, "id": "d711726e", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
player1player2percentage
0ColomboCeesay55.0
1Pezzella Giu.Gallo65.0
2BandaDi Francesco F.60.0
3LovatoBronn55.0
4Nicolussi CavigliaBohinen65.0
5DiaBonazzoli65.0
6HaasAkpa Akpro60.0
7SatrianoBajrami55.0
8EbuehiStojanovic60.0
9BuongiornoAdopo60.0
10LinettyLukic65.0
11SanabriaSeck55.0
12SanabriaRadonjic55.0
13DessersCiofani D.55.0
14ChirichesBianchetti60.0
15AsllaniGagliardini55.0
16DarmianD'ambrosio60.0
17DumfriesD'ambrosio55.0
18MaehleRuggeri60.0
19BogaPasalic55.0
20RinconVieira55.0
21GabbiadiniQuagliarella55.0
22GiroudOrigi60.0
23De KetelaereDiaz B.55.0
24SaelemaekersMessias60.0
25Lopez M.Obiang55.0
26DefrelAlvarez A.60.0
27Traore' Hj.Thorstvedt65.0
28MckennieChiesa60.0
29KeanMilik60.0
30FagioliMiretti55.0
31RovellaMachin55.0
32Mari'Caldirola60.0
33PetagnaMota60.0
34MarusicHysaj60.0
35CataldiMarcos Antonio65.0
36IgorMartinez Quarta60.0
37Kouame'Jovic55.0
38Gonzalez N.Saponara55.0
39ElmasLozano55.0
40ElmasPolitano55.0
41Mario RuiOlivera55.0
42SpinazzolaEl Shaarawy60.0
43MaticBove55.0
44ArslanMakengo60.0
45SamardzicLovric55.0
46LasagnaKallon60.0
\n", "
" ], "text/plain": [ " player1 player2 percentage\n", "0 Colombo Ceesay 55.0\n", "1 Pezzella Giu. Gallo 65.0\n", "2 Banda Di Francesco F. 60.0\n", "3 Lovato Bronn 55.0\n", "4 Nicolussi Caviglia Bohinen 65.0\n", "5 Dia Bonazzoli 65.0\n", "6 Haas Akpa Akpro 60.0\n", "7 Satriano Bajrami 55.0\n", "8 Ebuehi Stojanovic 60.0\n", "9 Buongiorno Adopo 60.0\n", "10 Linetty Lukic 65.0\n", "11 Sanabria Seck 55.0\n", "12 Sanabria Radonjic 55.0\n", "13 Dessers Ciofani D. 55.0\n", "14 Chiriches Bianchetti 60.0\n", "15 Asllani Gagliardini 55.0\n", "16 Darmian D'ambrosio 60.0\n", "17 Dumfries D'ambrosio 55.0\n", "18 Maehle Ruggeri 60.0\n", "19 Boga Pasalic 55.0\n", "20 Rincon Vieira 55.0\n", "21 Gabbiadini Quagliarella 55.0\n", "22 Giroud Origi 60.0\n", "23 De Ketelaere Diaz B. 55.0\n", "24 Saelemaekers Messias 60.0\n", "25 Lopez M. Obiang 55.0\n", "26 Defrel Alvarez A. 60.0\n", "27 Traore' Hj. Thorstvedt 65.0\n", "28 Mckennie Chiesa 60.0\n", "29 Kean Milik 60.0\n", "30 Fagioli Miretti 55.0\n", "31 Rovella Machin 55.0\n", "32 Mari' Caldirola 60.0\n", "33 Petagna Mota 60.0\n", "34 Marusic Hysaj 60.0\n", "35 Cataldi Marcos Antonio 65.0\n", "36 Igor Martinez Quarta 60.0\n", "37 Kouame' Jovic 55.0\n", "38 Gonzalez N. Saponara 55.0\n", "39 Elmas Lozano 55.0\n", "40 Elmas Politano 55.0\n", "41 Mario Rui Olivera 55.0\n", "42 Spinazzola El Shaarawy 60.0\n", "43 Matic Bove 55.0\n", "44 Arslan Makengo 60.0\n", "45 Samardzic Lovric 55.0\n", "46 Lasagna Kallon 60.0" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "groups = soup.findAll('ul', attrs = {'class' : 'ballot-list'})\n", "\n", "ballots = pd.DataFrame(columns = ['player1', 'player2', 'percentage'])\n", "for i in range (len(groups)):\n", " players = groups[i].findAll('a', attrs = {'class' : 'player-name'})\n", " percs = groups[i].findAll('strong', attrs = {'class' : 'percentage'})\n", " \n", " player1 = players[0].contents[0].replace('\\n \\n ', '')\n", " player2 = players[1].contents[0].replace('\\n \\n ', '')\n", " perc = float(percs[0].contents[0].replace('%', ''))\n", " \n", " row_df = pd.DataFrame([[player1, player2, perc]], columns = ballots.columns)\n", " \n", " ballots = pd.concat([ballots, row_df], ignore_index = True)\n", " \n", "ballots" ] }, { "cell_type": "code", "execution_count": 6, "id": "0171c61d", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
starterpercentage
player
Skorupski1100.0
Posch1100.0
Soumaoro1100.0
Lucumi'1100.0
Cambiaso1100.0
.........
Verdi055.0
Kallon0.455.0
Piccoli055.0
Ngonge025.0
Braaf040.0
\n", "

440 rows × 2 columns

\n", "
" ], "text/plain": [ " starter percentage\n", "player \n", "Skorupski 1 100.0\n", "Posch 1 100.0\n", "Soumaoro 1 100.0\n", "Lucumi' 1 100.0\n", "Cambiaso 1 100.0\n", "... ... ...\n", "Verdi 0 55.0\n", "Kallon 0.4 55.0\n", "Piccoli 0 55.0\n", "Ngonge 0 25.0\n", "Braaf 0 40.0\n", "\n", "[440 rows x 2 columns]" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "for i in range(ballots.shape[0]):\n", " p1 = ballots['player1'][i]\n", " p2 = ballots['player2'][i]\n", " \n", " perc = float(ballots['percentage'][i])\n", " \n", " probables.at[p1, 'starter'] = perc/100\n", " probables.at[p2, 'starter'] = 1 - perc/100\n", " \n", "probables" ] }, { "cell_type": "code", "execution_count": 7, "id": "8b5468aa", "metadata": {}, "outputs": [], "source": [ "probables.to_excel('mid_outputs/match_probable_players.xlsx') " ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.13" } }, "nbformat": 4, "nbformat_minor": 5 }