{ "cells": [ { "cell_type": "markdown", "id": "fd8a6cb9", "metadata": {}, "source": [ "Scraping for playing probability for the current Serie A matchday.\n", "\n", "Data from http://fantacalcio.it" ] }, { "cell_type": "code", "execution_count": 1, "id": "4c7b7d26", "metadata": {}, "outputs": [], "source": [ "import requests\n", "from bs4 import BeautifulSoup\n", "import pandas as pd\n", "import numpy as np\n", "import re\n", "import sys, getopt\n", "import csv\n" ] }, { "cell_type": "code", "execution_count": 2, "id": "91897dbb", "metadata": {}, "outputs": [], "source": [ "res = requests.get('https://www.fantacalcio.it/probabili-formazioni-serie-a')" ] }, { "cell_type": "code", "execution_count": 3, "id": "7ba89027", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
playerstarterpercentage
0Ochoa190.0
1Lovato180.0
2Gyomber180.0
3Pirola180.0
4Mazzocchi180.0
............
471Pisilli010.0
472Aouar055.0
473El Shaarawy055.0
474Belotti060.0
475Azmoun035.0
\n", "

476 rows × 3 columns

\n", "
" ], "text/plain": [ " player starter percentage\n", "0 Ochoa 1 90.0\n", "1 Lovato 1 80.0\n", "2 Gyomber 1 80.0\n", "3 Pirola 1 80.0\n", "4 Mazzocchi 1 80.0\n", ".. ... ... ...\n", "471 Pisilli 0 10.0\n", "472 Aouar 0 55.0\n", "473 El Shaarawy 0 55.0\n", "474 Belotti 0 60.0\n", "475 Azmoun 0 35.0\n", "\n", "[476 rows x 3 columns]" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "soup = BeautifulSoup(res.text)\n", "\n", "groups = soup.findAll('ul', attrs = {'class' : 'player-list'})\n", "\n", "probables = pd.DataFrame(columns = ['player', 'starter', 'percentage'])\n", "\n", "for i in range (len(groups)):\n", " players = groups[i].findAll('a', attrs = {'class' : 'player-name'})\n", " bars = groups[i].findAll('div', attrs = {'class' : 'progress-bar'})\n", " \n", " starter_string = groups[i]['class'][1]\n", " if(starter_string == 'starters'):\n", " starter = 1\n", " else:\n", " starter = 0\n", " \n", " for j in range(len(players)):\n", " player = players[j].contents[1].contents[0]\n", " perc = float(bars[j]['aria-valuenow'])\n", " \n", " row_df = pd.DataFrame([[player, starter, perc]], columns = probables.columns)\n", " \n", " probables = pd.concat([probables, row_df], ignore_index = True)\n", " \n", "probables " ] }, { "cell_type": "code", "execution_count": 4, "id": "29e5b389", "metadata": {}, "outputs": [], "source": [ "probables = probables.set_index('player')" ] }, { "cell_type": "code", "execution_count": 5, "id": "d711726e", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
player1player2percentage
0IkwuemesiBotheim55.0
1JovaneKastanos55.0
2OkoliMonterisi55.0
3CasoBaez55.0
4VasquezMartin60.0
5MalinovskyiDe Winter60.0
6KjaerThiaw60.0
7Rafael LeaoChukwueze60.0
8GiroudJovic60.0
9BonazzoliDjuric60.0
10MboulaFolorunsho55.0
11TressoldiViti60.0
12VinaPedersen65.0
13BajramiThorstvedt60.0
14FagioliMiretti60.0
15MckennieWeah55.0
16ZaccagniPedro60.0
17RomagnoliPatric60.0
18GuendouziKamada55.0
19BirindelliKyriakopoulos65.0
20MalehBastoni S.55.0
21FazziniGrassi60.0
22BereszynskiEbuehi60.0
23ThuramArnautovic60.0
24DimarcoCarlos Augusto55.0
25PavardDarmian60.0
26De VrijAcerbi60.0
27FrattesiMkhitaryan65.0
28ZappacostaZortea60.0
29RuggeriBakker60.0
30ToloiDjimsiti55.0
31JanktoDeiola55.0
32ObertPrati55.0
33EboseleFerreira J.60.0
34Kamara H.Zemura60.0
35NzolaBeltran L.60.0
36MandragoraDuncan55.0
37Ranieri L.Martinez Quarta55.0
38Kouame'Sottil55.0
39Moro N.Aebischer55.0
40NdoyeOrsolini55.0
41ZielinskiElmas65.0
42Juan JesusNatan55.0
43PolitanoRaspadori60.0
44Mario RuiOlivera55.0
45VlasicSeck65.0
46IlicTameze55.0
47ParedesAouar60.0
48KristensenCelik60.0
49SpinazzolaZalewski60.0
\n", "
" ], "text/plain": [ " player1 player2 percentage\n", "0 Ikwuemesi Botheim 55.0\n", "1 Jovane Kastanos 55.0\n", "2 Okoli Monterisi 55.0\n", "3 Caso Baez 55.0\n", "4 Vasquez Martin 60.0\n", "5 Malinovskyi De Winter 60.0\n", "6 Kjaer Thiaw 60.0\n", "7 Rafael Leao Chukwueze 60.0\n", "8 Giroud Jovic 60.0\n", "9 Bonazzoli Djuric 60.0\n", "10 Mboula Folorunsho 55.0\n", "11 Tressoldi Viti 60.0\n", "12 Vina Pedersen 65.0\n", "13 Bajrami Thorstvedt 60.0\n", "14 Fagioli Miretti 60.0\n", "15 Mckennie Weah 55.0\n", "16 Zaccagni Pedro 60.0\n", "17 Romagnoli Patric 60.0\n", "18 Guendouzi Kamada 55.0\n", "19 Birindelli Kyriakopoulos 65.0\n", "20 Maleh Bastoni S. 55.0\n", "21 Fazzini Grassi 60.0\n", "22 Bereszynski Ebuehi 60.0\n", "23 Thuram Arnautovic 60.0\n", "24 Dimarco Carlos Augusto 55.0\n", "25 Pavard Darmian 60.0\n", "26 De Vrij Acerbi 60.0\n", "27 Frattesi Mkhitaryan 65.0\n", "28 Zappacosta Zortea 60.0\n", "29 Ruggeri Bakker 60.0\n", "30 Toloi Djimsiti 55.0\n", "31 Jankto Deiola 55.0\n", "32 Obert Prati 55.0\n", "33 Ebosele Ferreira J. 60.0\n", "34 Kamara H. Zemura 60.0\n", "35 Nzola Beltran L. 60.0\n", "36 Mandragora Duncan 55.0\n", "37 Ranieri L. Martinez Quarta 55.0\n", "38 Kouame' Sottil 55.0\n", "39 Moro N. Aebischer 55.0\n", "40 Ndoye Orsolini 55.0\n", "41 Zielinski Elmas 65.0\n", "42 Juan Jesus Natan 55.0\n", "43 Politano Raspadori 60.0\n", "44 Mario Rui Olivera 55.0\n", "45 Vlasic Seck 65.0\n", "46 Ilic Tameze 55.0\n", "47 Paredes Aouar 60.0\n", "48 Kristensen Celik 60.0\n", "49 Spinazzola Zalewski 60.0" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "groups = soup.findAll('ul', attrs = {'class' : 'ballot-list'})\n", "\n", "ballots = pd.DataFrame(columns = ['player1', 'player2', 'percentage'])\n", "for i in range (len(groups)):\n", " players = groups[i].findAll('a', attrs = {'class' : 'player-name'})\n", " percs = groups[i].findAll('strong', attrs = {'class' : 'percentage'})\n", " \n", " player1 = players[0].contents[1].contents[0]\n", " player2 = players[1].contents[1].contents[0]\n", " perc = float(percs[0].contents[0].replace('%', ''))\n", " \n", " row_df = pd.DataFrame([[player1, player2, perc]], columns = ballots.columns)\n", " \n", " ballots = pd.concat([ballots, row_df], ignore_index = True)\n", " \n", "ballots" ] }, { "cell_type": "code", "execution_count": 6, "id": "0171c61d", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
starterpercentage
player
Ochoa190.0
Lovato180.0
Gyomber180.0
Pirola180.0
Mazzocchi180.0
.........
Pisilli010.0
Aouar0.455.0
El Shaarawy055.0
Belotti060.0
Azmoun035.0
\n", "

476 rows × 2 columns

\n", "
" ], "text/plain": [ " starter percentage\n", "player \n", "Ochoa 1 90.0\n", "Lovato 1 80.0\n", "Gyomber 1 80.0\n", "Pirola 1 80.0\n", "Mazzocchi 1 80.0\n", "... ... ...\n", "Pisilli 0 10.0\n", "Aouar 0.4 55.0\n", "El Shaarawy 0 55.0\n", "Belotti 0 60.0\n", "Azmoun 0 35.0\n", "\n", "[476 rows x 2 columns]" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "for i in range(ballots.shape[0]):\n", " p1 = ballots['player1'][i]\n", " p2 = ballots['player2'][i]\n", " \n", " perc = float(ballots['percentage'][i])\n", " \n", " probables.at[p1, 'starter'] = perc/100\n", " probables.at[p2, 'starter'] = 1 - perc/100\n", " \n", "probables" ] }, { "cell_type": "code", "execution_count": 7, "id": "8b5468aa", "metadata": {}, "outputs": [], "source": [ "probables.to_excel('mid_outputs/match_probable_players.xlsx') " ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.13" } }, "nbformat": 4, "nbformat_minor": 5 }