{ "cells": [ { "cell_type": "markdown", "id": "fd8a6cb9", "metadata": {}, "source": [ "Scraping for playing probability for the current Serie A matchday.\n", "\n", "Data from http://fantacalcio.it" ] }, { "cell_type": "code", "execution_count": 1, "id": "4c7b7d26", "metadata": {}, "outputs": [], "source": [ "import requests\n", "from bs4 import BeautifulSoup\n", "import pandas as pd\n", "import numpy as np\n", "import re\n", "import sys, getopt\n", "import csv\n" ] }, { "cell_type": "code", "execution_count": 2, "id": "91897dbb", "metadata": {}, "outputs": [], "source": [ "res = requests.get('https://www.fantacalcio.it/probabili-formazioni-serie-a')" ] }, { "cell_type": "code", "execution_count": 3, "id": "7ba89027", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
playerstarterpercentage
0Tatarusanu190.0
1Thiaw165.0
2Kjaer185.0
3Kalulu190.0
4Saelemaekers160.0
............
433Dumfries060.0
434Mkhitaryan055.0
435Asllani030.0
436Gagliardini050.0
437Lukaku060.0
\n", "

438 rows × 3 columns

\n", "
" ], "text/plain": [ " player starter percentage\n", "0 Tatarusanu 1 90.0\n", "1 Thiaw 1 65.0\n", "2 Kjaer 1 85.0\n", "3 Kalulu 1 90.0\n", "4 Saelemaekers 1 60.0\n", ".. ... ... ...\n", "433 Dumfries 0 60.0\n", "434 Mkhitaryan 0 55.0\n", "435 Asllani 0 30.0\n", "436 Gagliardini 0 50.0\n", "437 Lukaku 0 60.0\n", "\n", "[438 rows x 3 columns]" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "soup = BeautifulSoup(res.text)\n", "\n", "groups = soup.findAll('ul', attrs = {'class' : 'player-list'})\n", "\n", "probables = pd.DataFrame(columns = ['player', 'starter', 'percentage'])\n", "\n", "for i in range (len(groups)):\n", " players = groups[i].findAll('a', attrs = {'class' : 'player-name'})\n", " bars = groups[i].findAll('div', attrs = {'class' : 'progress-bar'})\n", " \n", " starter_string = groups[i]['class'][1]\n", " if(starter_string == 'starters'):\n", " starter = 1\n", " else:\n", " starter = 0\n", " \n", " for j in range(len(players)):\n", " player = players[j].contents[1].contents[0]\n", " perc = float(bars[j]['aria-valuenow'])\n", " \n", " row_df = pd.DataFrame([[player, starter, perc]], columns = probables.columns)\n", " \n", " probables = pd.concat([probables, row_df], ignore_index = True)\n", " \n", "probables " ] }, { "cell_type": "code", "execution_count": 4, "id": "29e5b389", "metadata": {}, "outputs": [], "source": [ "probables = probables.set_index('player')" ] }, { "cell_type": "code", "execution_count": 5, "id": "d711726e", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
player1player2percentage
0SaelemaekersCalabria60.0
1Diaz B.De Ketelaere60.0
2ThiawMessias65.0
3SingoAina55.0
4Rodriguez R.Buongiorno55.0
5Akpa AkproHaas60.0
6SatrianoPiccoli60.0
7EbuehiStojanovic60.0
8HolmFerrer55.0
9NzolaShomurodov55.0
10AgudeloCipot60.0
11Di Francesco F.Banda65.0
12GalloPezzella Giu.60.0
13El ShaarawyCelik60.0
14MaticBove60.0
15CataldiVecino65.0
16LazzariHysaj55.0
17BogaEderson D.s.55.0
18HojlundZapata D.65.0
19DjimsitiDemiral60.0
20ThauvinSuccess55.0
21ArslanLovric60.0
22BajramiThorstvedt55.0
23RuanFerrari G.60.0
24Lopez M.Obiang55.0
25ZirkzeeBarrow65.0
26CambiasoLykogiannis55.0
27SorianoAebischer65.0
28MarlonCaldirola55.0
29PetagnaMota60.0
30SensiMachin55.0
31De SciglioChiesa60.0
32BrekaloSaponara55.0
33BrekaloIkone'55.0
34JovicCabral60.0
35BarakDuncan60.0
36LozanoElmas55.0
37LozanoPolitano55.0
38Mario RuiOlivera60.0
39BuonaiutoAfena-Gyan60.0
40BenassiCastagnetti60.0
41CeccheriniCoppola D.60.0
42MagnaniDawidowicz55.0
43NgongeGaich60.0
44BronnDaniliuc60.0
45LovatoSambia55.0
46BohinenNicolussi Caviglia55.0
47DjuricicSabiri65.0
48MurruMurillo60.0
49RinconCuisance60.0
50BrozovicMkhitaryan55.0
51DzekoLukaku60.0
52DarmianDumfries55.0
\n", "
" ], "text/plain": [ " player1 player2 percentage\n", "0 Saelemaekers Calabria 60.0\n", "1 Diaz B. De Ketelaere 60.0\n", "2 Thiaw Messias 65.0\n", "3 Singo Aina 55.0\n", "4 Rodriguez R. Buongiorno 55.0\n", "5 Akpa Akpro Haas 60.0\n", "6 Satriano Piccoli 60.0\n", "7 Ebuehi Stojanovic 60.0\n", "8 Holm Ferrer 55.0\n", "9 Nzola Shomurodov 55.0\n", "10 Agudelo Cipot 60.0\n", "11 Di Francesco F. Banda 65.0\n", "12 Gallo Pezzella Giu. 60.0\n", "13 El Shaarawy Celik 60.0\n", "14 Matic Bove 60.0\n", "15 Cataldi Vecino 65.0\n", "16 Lazzari Hysaj 55.0\n", "17 Boga Ederson D.s. 55.0\n", "18 Hojlund Zapata D. 65.0\n", "19 Djimsiti Demiral 60.0\n", "20 Thauvin Success 55.0\n", "21 Arslan Lovric 60.0\n", "22 Bajrami Thorstvedt 55.0\n", "23 Ruan Ferrari G. 60.0\n", "24 Lopez M. Obiang 55.0\n", "25 Zirkzee Barrow 65.0\n", "26 Cambiaso Lykogiannis 55.0\n", "27 Soriano Aebischer 65.0\n", "28 Marlon Caldirola 55.0\n", "29 Petagna Mota 60.0\n", "30 Sensi Machin 55.0\n", "31 De Sciglio Chiesa 60.0\n", "32 Brekalo Saponara 55.0\n", "33 Brekalo Ikone' 55.0\n", "34 Jovic Cabral 60.0\n", "35 Barak Duncan 60.0\n", "36 Lozano Elmas 55.0\n", "37 Lozano Politano 55.0\n", "38 Mario Rui Olivera 60.0\n", "39 Buonaiuto Afena-Gyan 60.0\n", "40 Benassi Castagnetti 60.0\n", "41 Ceccherini Coppola D. 60.0\n", "42 Magnani Dawidowicz 55.0\n", "43 Ngonge Gaich 60.0\n", "44 Bronn Daniliuc 60.0\n", "45 Lovato Sambia 55.0\n", "46 Bohinen Nicolussi Caviglia 55.0\n", "47 Djuricic Sabiri 65.0\n", "48 Murru Murillo 60.0\n", "49 Rincon Cuisance 60.0\n", "50 Brozovic Mkhitaryan 55.0\n", "51 Dzeko Lukaku 60.0\n", "52 Darmian Dumfries 55.0" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "groups = soup.findAll('ul', attrs = {'class' : 'ballot-list'})\n", "\n", "ballots = pd.DataFrame(columns = ['player1', 'player2', 'percentage'])\n", "for i in range (len(groups)):\n", " players = groups[i].findAll('a', attrs = {'class' : 'player-name'})\n", " percs = groups[i].findAll('strong', attrs = {'class' : 'percentage'})\n", " \n", " player1 = players[0].contents[1].contents[0]\n", " player2 = players[1].contents[1].contents[0]\n", " perc = float(percs[0].contents[0].replace('%', ''))\n", " \n", " row_df = pd.DataFrame([[player1, player2, perc]], columns = ballots.columns)\n", " \n", " ballots = pd.concat([ballots, row_df], ignore_index = True)\n", " \n", "ballots" ] }, { "cell_type": "code", "execution_count": 6, "id": "0171c61d", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
starterpercentage
player
Tatarusanu190.0
Thiaw0.6565.0
Kjaer185.0
Kalulu190.0
Saelemaekers0.660.0
.........
Dumfries0.4560.0
Mkhitaryan0.4555.0
Asllani030.0
Gagliardini050.0
Lukaku0.460.0
\n", "

438 rows × 2 columns

\n", "
" ], "text/plain": [ " starter percentage\n", "player \n", "Tatarusanu 1 90.0\n", "Thiaw 0.65 65.0\n", "Kjaer 1 85.0\n", "Kalulu 1 90.0\n", "Saelemaekers 0.6 60.0\n", "... ... ...\n", "Dumfries 0.45 60.0\n", "Mkhitaryan 0.45 55.0\n", "Asllani 0 30.0\n", "Gagliardini 0 50.0\n", "Lukaku 0.4 60.0\n", "\n", "[438 rows x 2 columns]" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "for i in range(ballots.shape[0]):\n", " p1 = ballots['player1'][i]\n", " p2 = ballots['player2'][i]\n", " \n", " perc = float(ballots['percentage'][i])\n", " \n", " probables.at[p1, 'starter'] = perc/100\n", " probables.at[p2, 'starter'] = 1 - perc/100\n", " \n", "probables" ] }, { "cell_type": "code", "execution_count": 7, "id": "8b5468aa", "metadata": {}, "outputs": [], "source": [ "probables.to_excel('mid_outputs/match_probable_players.xlsx') " ] }, { "cell_type": "code", "execution_count": null, "id": "62164549", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.13" } }, "nbformat": 4, "nbformat_minor": 5 }