{ "cells": [ { "cell_type": "markdown", "id": "fd8a6cb9", "metadata": {}, "source": [ "Scraping for playing probability for the current Serie A matchday.\n", "\n", "Data from http://fantacalcio.it" ] }, { "cell_type": "code", "execution_count": 1, "id": "4c7b7d26", "metadata": {}, "outputs": [], "source": [ "import requests\n", "from bs4 import BeautifulSoup\n", "import pandas as pd\n", "import numpy as np\n", "import re\n", "import sys, getopt\n", "import csv\n" ] }, { "cell_type": "code", "execution_count": 2, "id": "91897dbb", "metadata": {}, "outputs": [], "source": [ "res = requests.get('https://www.fantacalcio.it/probabili-formazioni-serie-a')" ] }, { "cell_type": "code", "execution_count": 3, "id": "7ba89027", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
playerstarterpercentage
0Vicario1100.0
1Ebuehi1100.0
2Ismajli1100.0
3Walukiewicz1100.0
4Parisi1100.0
............
444Basic055.0
445Bertini010.0
446Marcos Antonio050.0
447Romero L.050.0
448Cancellieri055.0
\n", "

449 rows × 3 columns

\n", "
" ], "text/plain": [ " player starter percentage\n", "0 Vicario 1 100.0\n", "1 Ebuehi 1 100.0\n", "2 Ismajli 1 100.0\n", "3 Walukiewicz 1 100.0\n", "4 Parisi 1 100.0\n", ".. ... ... ...\n", "444 Basic 0 55.0\n", "445 Bertini 0 10.0\n", "446 Marcos Antonio 0 50.0\n", "447 Romero L. 0 50.0\n", "448 Cancellieri 0 55.0\n", "\n", "[449 rows x 3 columns]" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "soup = BeautifulSoup(res.text)\n", "\n", "groups = soup.findAll('ul', attrs = {'class' : 'player-list'})\n", "\n", "probables = pd.DataFrame(columns = ['player', 'starter', 'percentage'])\n", "\n", "for i in range (len(groups)):\n", " players = groups[i].findAll('a', attrs = {'class' : 'player-name'})\n", " bars = groups[i].findAll('div', attrs = {'class' : 'progress-bar'})\n", " \n", " starter_string = groups[i]['class'][1]\n", " if(starter_string == 'starters'):\n", " starter = 1\n", " else:\n", " starter = 0\n", " \n", " for j in range(len(players)):\n", " player = players[j].contents[0].replace('\\n \\n ', '')\n", " perc = float(bars[j]['aria-valuenow'])\n", " \n", " row_df = pd.DataFrame([[player, starter, perc]], columns = probables.columns)\n", " \n", " probables = pd.concat([probables, row_df], ignore_index = True)\n", " \n", "probables " ] }, { "cell_type": "code", "execution_count": 4, "id": "29e5b389", "metadata": {}, "outputs": [], "source": [ "probables = probables.set_index('player')" ] }, { "cell_type": "code", "execution_count": 5, "id": "d711726e", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
player1player2percentage
0LozanoPolitano60.0
1Zambo AnguissaNdombele'60.0
2Mario RuiOlivera55.0
3ElmasRaspadori55.0
4BetoSuccess60.0
5LovricSamardzic55.0
6EhizibueNuytinck60.0
7RinconVieira60.0
8AugelloMurru55.0
9DjuricicLeris65.0
10CaputoMontevago60.0
11Di Francesco F.Banda55.0
12ColomboCeesay55.0
13Gonzalez J.Bistrovic60.0
14LykogiannisCambiaso55.0
15BarrowOrsolini55.0
16FergusonSchouten55.0
17Traore' Hj.Berardi55.0
18RogerioKyriakopoulos60.0
19DjimsitiScalvini60.0
20Zapata D.Hojlund60.0
21PasalicMalinovskyi55.0
22BastoniAcerbi55.0
23BrozovicMkhitaryan55.0
24SkriniarD'ambrosio65.0
25LazovicDepaoli55.0
26VerdiHongla60.0
27DawidowiczHien60.0
28VerdeGyasi55.0
29Bastoni S.Amian55.0
30Carlos AugustoD'alessandro60.0
31BirindelliCiurria60.0
32PetagnaGytkjaer55.0
33CaldirolaCarboni65.0
34LovatoBronn55.0
35PiatekBonazzoli55.0
36CandrevaBradaric55.0
37ZalewskiEl Shaarawy60.0
38VolpatoDybala55.0
39MaticCamara Ma.55.0
40VojvodaSingo60.0
41SanabriaPellegri55.0
42MiranchukRadonjic55.0
43KjaerGabbia55.0
44De KetelaereDiaz B.55.0
45BennacerPobega60.0
46Martinez QuartaIgor55.0
47BonaventuraMandragora55.0
48Dodo'Venuti55.0
49Ikone'Saponara55.0
50LocatelliParedes60.0
51MilikVlahovic65.0
52MirettiDi Maria55.0
53ImmobileCancellieri60.0
54VecinoLuis Alberto55.0
\n", "
" ], "text/plain": [ " player1 player2 percentage\n", "0 Lozano Politano 60.0\n", "1 Zambo Anguissa Ndombele' 60.0\n", "2 Mario Rui Olivera 55.0\n", "3 Elmas Raspadori 55.0\n", "4 Beto Success 60.0\n", "5 Lovric Samardzic 55.0\n", "6 Ehizibue Nuytinck 60.0\n", "7 Rincon Vieira 60.0\n", "8 Augello Murru 55.0\n", "9 Djuricic Leris 65.0\n", "10 Caputo Montevago 60.0\n", "11 Di Francesco F. Banda 55.0\n", "12 Colombo Ceesay 55.0\n", "13 Gonzalez J. Bistrovic 60.0\n", "14 Lykogiannis Cambiaso 55.0\n", "15 Barrow Orsolini 55.0\n", "16 Ferguson Schouten 55.0\n", "17 Traore' Hj. Berardi 55.0\n", "18 Rogerio Kyriakopoulos 60.0\n", "19 Djimsiti Scalvini 60.0\n", "20 Zapata D. Hojlund 60.0\n", "21 Pasalic Malinovskyi 55.0\n", "22 Bastoni Acerbi 55.0\n", "23 Brozovic Mkhitaryan 55.0\n", "24 Skriniar D'ambrosio 65.0\n", "25 Lazovic Depaoli 55.0\n", "26 Verdi Hongla 60.0\n", "27 Dawidowicz Hien 60.0\n", "28 Verde Gyasi 55.0\n", "29 Bastoni S. Amian 55.0\n", "30 Carlos Augusto D'alessandro 60.0\n", "31 Birindelli Ciurria 60.0\n", "32 Petagna Gytkjaer 55.0\n", "33 Caldirola Carboni 65.0\n", "34 Lovato Bronn 55.0\n", "35 Piatek Bonazzoli 55.0\n", "36 Candreva Bradaric 55.0\n", "37 Zalewski El Shaarawy 60.0\n", "38 Volpato Dybala 55.0\n", "39 Matic Camara Ma. 55.0\n", "40 Vojvoda Singo 60.0\n", "41 Sanabria Pellegri 55.0\n", "42 Miranchuk Radonjic 55.0\n", "43 Kjaer Gabbia 55.0\n", "44 De Ketelaere Diaz B. 55.0\n", "45 Bennacer Pobega 60.0\n", "46 Martinez Quarta Igor 55.0\n", "47 Bonaventura Mandragora 55.0\n", "48 Dodo' Venuti 55.0\n", "49 Ikone' Saponara 55.0\n", "50 Locatelli Paredes 60.0\n", "51 Milik Vlahovic 65.0\n", "52 Miretti Di Maria 55.0\n", "53 Immobile Cancellieri 60.0\n", "54 Vecino Luis Alberto 55.0" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "groups = soup.findAll('ul', attrs = {'class' : 'ballot-list'})\n", "\n", "ballots = pd.DataFrame(columns = ['player1', 'player2', 'percentage'])\n", "for i in range (len(groups)):\n", " players = groups[i].findAll('a', attrs = {'class' : 'player-name'})\n", " percs = groups[i].findAll('strong', attrs = {'class' : 'percentage'})\n", " \n", " player1 = players[0].contents[0].replace('\\n \\n ', '')\n", " player2 = players[1].contents[0].replace('\\n \\n ', '')\n", " perc = float(percs[0].contents[0].replace('%', ''))\n", " \n", " row_df = pd.DataFrame([[player1, player2, perc]], columns = ballots.columns)\n", " \n", " ballots = pd.concat([ballots, row_df], ignore_index = True)\n", " \n", "ballots" ] }, { "cell_type": "code", "execution_count": 6, "id": "0171c61d", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
starterpercentage
player
Vicario1100.0
Ebuehi1100.0
Ismajli1100.0
Walukiewicz1100.0
Parisi1100.0
.........
Basic055.0
Bertini010.0
Marcos Antonio050.0
Romero L.050.0
Cancellieri0.455.0
\n", "

449 rows × 2 columns

\n", "
" ], "text/plain": [ " starter percentage\n", "player \n", "Vicario 1 100.0\n", "Ebuehi 1 100.0\n", "Ismajli 1 100.0\n", "Walukiewicz 1 100.0\n", "Parisi 1 100.0\n", "... ... ...\n", "Basic 0 55.0\n", "Bertini 0 10.0\n", "Marcos Antonio 0 50.0\n", "Romero L. 0 50.0\n", "Cancellieri 0.4 55.0\n", "\n", "[449 rows x 2 columns]" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "for i in range(ballots.shape[0]):\n", " p1 = ballots['player1'][i]\n", " p2 = ballots['player2'][i]\n", " \n", " perc = float(ballots['percentage'][i])\n", " \n", " probables.at[p1, 'starter'] = perc/100\n", " probables.at[p2, 'starter'] = 1 - perc/100\n", " \n", "probables" ] }, { "cell_type": "code", "execution_count": 7, "id": "8b5468aa", "metadata": {}, "outputs": [], "source": [ "probables.to_excel('mid_outputs/match_probable_players.xlsx') " ] }, { "cell_type": "code", "execution_count": null, "id": "cf992051", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.13" } }, "nbformat": 4, "nbformat_minor": 5 }