{ "cells": [ { "cell_type": "markdown", "id": "fd8a6cb9", "metadata": {}, "source": [ "Scraping for playing probability for the current Serie A matchday.\n", "\n", "Data from http://fantacalcio.it" ] }, { "cell_type": "code", "execution_count": 1, "id": "4c7b7d26", "metadata": {}, "outputs": [], "source": [ "import requests\n", "from bs4 import BeautifulSoup\n", "import pandas as pd\n", "import numpy as np\n", "import re\n", "import sys, getopt\n", "import csv\n" ] }, { "cell_type": "code", "execution_count": 2, "id": "91897dbb", "metadata": {}, "outputs": [], "source": [ "res = requests.get('https://www.fantacalcio.it/probabili-formazioni-serie-a')" ] }, { "cell_type": "code", "execution_count": 3, "id": "7ba89027", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
playerstarterpercentage
0Falcone190.0
1Gendrey190.0
2Baschirotto190.0
3Pongracic180.0
4Gallo160.0
............
462Oristanio050.0
463Jankto015.0
464Mancosu010.0
465Pavoletti055.0
466Shomurodov060.0
\n", "

467 rows × 3 columns

\n", "
" ], "text/plain": [ " player starter percentage\n", "0 Falcone 1 90.0\n", "1 Gendrey 1 90.0\n", "2 Baschirotto 1 90.0\n", "3 Pongracic 1 80.0\n", "4 Gallo 1 60.0\n", ".. ... ... ...\n", "462 Oristanio 0 50.0\n", "463 Jankto 0 15.0\n", "464 Mancosu 0 10.0\n", "465 Pavoletti 0 55.0\n", "466 Shomurodov 0 60.0\n", "\n", "[467 rows x 3 columns]" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "soup = BeautifulSoup(res.text)\n", "\n", "groups = soup.findAll('ul', attrs = {'class' : 'player-list'})\n", "\n", "probables = pd.DataFrame(columns = ['player', 'starter', 'percentage'])\n", "\n", "for i in range (len(groups)):\n", " players = groups[i].findAll('a', attrs = {'class' : 'player-name'})\n", " bars = groups[i].findAll('div', attrs = {'class' : 'progress-bar'})\n", " \n", " starter_string = groups[i]['class'][1]\n", " if(starter_string == 'starters'):\n", " starter = 1\n", " else:\n", " starter = 0\n", " \n", " for j in range(len(players)):\n", " player = players[j].contents[1].contents[0]\n", " perc = float(bars[j]['aria-valuenow'])\n", " \n", " row_df = pd.DataFrame([[player, starter, perc]], columns = probables.columns)\n", " \n", " probables = pd.concat([probables, row_df], ignore_index = True)\n", " \n", "probables " ] }, { "cell_type": "code", "execution_count": 4, "id": "29e5b389", "metadata": {}, "outputs": [], "source": [ "probables = probables.set_index('player')" ] }, { "cell_type": "code", "execution_count": 5, "id": "d711726e", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
player1player2percentage
0GalloDorgu60.0
1BlinGonzalez J.60.0
2PolitanoLindstrom60.0
3Zambo AnguissaCajuste60.0
4ZielinskiRaspadori55.0
5KvaratskheliaElmas65.0
6Mario RuiOlivera55.0
7CalabriaFlorenzi55.0
8PulisicChukwueze55.0
9ReijndersMusah65.0
10RomagnoliPatric65.0
11Pellegrini Lu.Marusic65.0
12ImmobileCastellanos60.0
13Luis AlbertoGuendouzi60.0
14DaniliucLovato60.0
15LegowskiBohinen60.0
16MarteganiDia60.0
17MkhitaryanKlaassen55.0
18SanchezMartinez L.55.0
19AcerbiBastoni55.0
20Moro N.Fabbian55.0
21FergusonAebischer60.0
22NdoyeOrsolini60.0
23CancellieriShpendi S.60.0
24FazziniRanocchia F.60.0
25GrassiMarin60.0
26SuccessLucca55.0
27EboseleFerreira J.60.0
28Kamara H.Zemura60.0
29MartinMatturro55.0
30PasalicMuriel60.0
31RuggeriHolm55.0
32MckennieWeah60.0
33FagioliMiretti60.0
34ChiesaMilik55.0
35DybalaEl Shaarawy65.0
36ZalewskiSpinazzola55.0
37ParedesCelik60.0
38CasoBaez60.0
39BrescianiniGarritano60.0
40Romagnoli S.Monterisi60.0
41VinaPedersen55.0
42ErlicViti60.0
43BajramiCastillejo60.0
44ColomboMaric65.0
45IzzoD'ambrosio60.0
46BirindelliKyriakopoulos60.0
47BellanovaSoppy60.0
48LazaroVojvoda60.0
49Zapata D.Sanabria55.0
50BonazzoliHenry60.0
51MagnaniCoppola D.60.0
52FolorunshoSaponara55.0
53BonaventuraBarak60.0
54Lopez M.Arthur Melo55.0
55NzolaBeltran L.55.0
56NandezDeiola60.0
57PetagnaShomurodov60.0
58HatzidiakosObert60.0
\n", "
" ], "text/plain": [ " player1 player2 percentage\n", "0 Gallo Dorgu 60.0\n", "1 Blin Gonzalez J. 60.0\n", "2 Politano Lindstrom 60.0\n", "3 Zambo Anguissa Cajuste 60.0\n", "4 Zielinski Raspadori 55.0\n", "5 Kvaratskhelia Elmas 65.0\n", "6 Mario Rui Olivera 55.0\n", "7 Calabria Florenzi 55.0\n", "8 Pulisic Chukwueze 55.0\n", "9 Reijnders Musah 65.0\n", "10 Romagnoli Patric 65.0\n", "11 Pellegrini Lu. Marusic 65.0\n", "12 Immobile Castellanos 60.0\n", "13 Luis Alberto Guendouzi 60.0\n", "14 Daniliuc Lovato 60.0\n", "15 Legowski Bohinen 60.0\n", "16 Martegani Dia 60.0\n", "17 Mkhitaryan Klaassen 55.0\n", "18 Sanchez Martinez L. 55.0\n", "19 Acerbi Bastoni 55.0\n", "20 Moro N. Fabbian 55.0\n", "21 Ferguson Aebischer 60.0\n", "22 Ndoye Orsolini 60.0\n", "23 Cancellieri Shpendi S. 60.0\n", "24 Fazzini Ranocchia F. 60.0\n", "25 Grassi Marin 60.0\n", "26 Success Lucca 55.0\n", "27 Ebosele Ferreira J. 60.0\n", "28 Kamara H. Zemura 60.0\n", "29 Martin Matturro 55.0\n", "30 Pasalic Muriel 60.0\n", "31 Ruggeri Holm 55.0\n", "32 Mckennie Weah 60.0\n", "33 Fagioli Miretti 60.0\n", "34 Chiesa Milik 55.0\n", "35 Dybala El Shaarawy 65.0\n", "36 Zalewski Spinazzola 55.0\n", "37 Paredes Celik 60.0\n", "38 Caso Baez 60.0\n", "39 Brescianini Garritano 60.0\n", "40 Romagnoli S. Monterisi 60.0\n", "41 Vina Pedersen 55.0\n", "42 Erlic Viti 60.0\n", "43 Bajrami Castillejo 60.0\n", "44 Colombo Maric 65.0\n", "45 Izzo D'ambrosio 60.0\n", "46 Birindelli Kyriakopoulos 60.0\n", "47 Bellanova Soppy 60.0\n", "48 Lazaro Vojvoda 60.0\n", "49 Zapata D. Sanabria 55.0\n", "50 Bonazzoli Henry 60.0\n", "51 Magnani Coppola D. 60.0\n", "52 Folorunsho Saponara 55.0\n", "53 Bonaventura Barak 60.0\n", "54 Lopez M. Arthur Melo 55.0\n", "55 Nzola Beltran L. 55.0\n", "56 Nandez Deiola 60.0\n", "57 Petagna Shomurodov 60.0\n", "58 Hatzidiakos Obert 60.0" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "groups = soup.findAll('ul', attrs = {'class' : 'ballot-list'})\n", "\n", "ballots = pd.DataFrame(columns = ['player1', 'player2', 'percentage'])\n", "for i in range (len(groups)):\n", " players = groups[i].findAll('a', attrs = {'class' : 'player-name'})\n", " percs = groups[i].findAll('strong', attrs = {'class' : 'percentage'})\n", " \n", " player1 = players[0].contents[1].contents[0]\n", " player2 = players[1].contents[1].contents[0]\n", " perc = float(percs[0].contents[0].replace('%', ''))\n", " \n", " row_df = pd.DataFrame([[player1, player2, perc]], columns = ballots.columns)\n", " \n", " ballots = pd.concat([ballots, row_df], ignore_index = True)\n", " \n", "ballots" ] }, { "cell_type": "code", "execution_count": 6, "id": "0171c61d", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
starterpercentage
player
Falcone190.0
Gendrey190.0
Baschirotto190.0
Pongracic180.0
Gallo0.660.0
.........
Oristanio050.0
Jankto015.0
Mancosu010.0
Pavoletti055.0
Shomurodov0.460.0
\n", "

467 rows × 2 columns

\n", "
" ], "text/plain": [ " starter percentage\n", "player \n", "Falcone 1 90.0\n", "Gendrey 1 90.0\n", "Baschirotto 1 90.0\n", "Pongracic 1 80.0\n", "Gallo 0.6 60.0\n", "... ... ...\n", "Oristanio 0 50.0\n", "Jankto 0 15.0\n", "Mancosu 0 10.0\n", "Pavoletti 0 55.0\n", "Shomurodov 0.4 60.0\n", "\n", "[467 rows x 2 columns]" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "for i in range(ballots.shape[0]):\n", " p1 = ballots['player1'][i]\n", " p2 = ballots['player2'][i]\n", " \n", " perc = float(ballots['percentage'][i])\n", " \n", " probables.at[p1, 'starter'] = perc/100\n", " probables.at[p2, 'starter'] = 1 - perc/100\n", " \n", "probables" ] }, { "cell_type": "code", "execution_count": 7, "id": "8b5468aa", "metadata": {}, "outputs": [], "source": [ "probables.to_excel('mid_outputs/match_probable_players.xlsx') " ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.13" } }, "nbformat": 4, "nbformat_minor": 5 }