{ "cells": [ { "cell_type": "markdown", "id": "fd8a6cb9", "metadata": {}, "source": [ "Scraping for playing probability for the current Serie A matchday.\n", "\n", "Data from http://fantacalcio.it" ] }, { "cell_type": "code", "execution_count": 1, "id": "4c7b7d26", "metadata": {}, "outputs": [], "source": [ "import requests\n", "from bs4 import BeautifulSoup\n", "import pandas as pd\n", "import numpy as np\n", "import re\n", "import sys, getopt\n", "import csv\n" ] }, { "cell_type": "code", "execution_count": 2, "id": "91897dbb", "metadata": {}, "outputs": [], "source": [ "res = requests.get('https://www.fantacalcio.it/probabili-formazioni-serie-a')" ] }, { "cell_type": "code", "execution_count": 3, "id": "7ba89027", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
playerstarterpercentage
0Falcone190.0
1Gendrey190.0
2Baschirotto190.0
3Umtiti190.0
4Gallo165.0
............
469Barrenechea020.0
470Pogba050.0
471Chiesa055.0
472Soule'020.0
473Milik060.0
\n", "

474 rows × 3 columns

\n", "
" ], "text/plain": [ " player starter percentage\n", "0 Falcone 1 90.0\n", "1 Gendrey 1 90.0\n", "2 Baschirotto 1 90.0\n", "3 Umtiti 1 90.0\n", "4 Gallo 1 65.0\n", ".. ... ... ...\n", "469 Barrenechea 0 20.0\n", "470 Pogba 0 50.0\n", "471 Chiesa 0 55.0\n", "472 Soule' 0 20.0\n", "473 Milik 0 60.0\n", "\n", "[474 rows x 3 columns]" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "soup = BeautifulSoup(res.text)\n", "\n", "groups = soup.findAll('ul', attrs = {'class' : 'player-list'})\n", "\n", "probables = pd.DataFrame(columns = ['player', 'starter', 'percentage'])\n", "\n", "for i in range (len(groups)):\n", " players = groups[i].findAll('a', attrs = {'class' : 'player-name'})\n", " bars = groups[i].findAll('div', attrs = {'class' : 'progress-bar'})\n", " \n", " starter_string = groups[i]['class'][1]\n", " if(starter_string == 'starters'):\n", " starter = 1\n", " else:\n", " starter = 0\n", " \n", " for j in range(len(players)):\n", " player = players[j].contents[1].contents[0]\n", " perc = float(bars[j]['aria-valuenow'])\n", " \n", " row_df = pd.DataFrame([[player, starter, perc]], columns = probables.columns)\n", " \n", " probables = pd.concat([probables, row_df], ignore_index = True)\n", " \n", "probables " ] }, { "cell_type": "code", "execution_count": 4, "id": "29e5b389", "metadata": {}, "outputs": [], "source": [ "probables = probables.set_index('player')" ] }, { "cell_type": "code", "execution_count": 5, "id": "d711726e", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
player1player2percentage
0GalloPezzella Giu.65.0
1CeesayColombo60.0
2Gonzalez J.Oudin55.0
3SamardzicSuccess55.0
4Esposito Sa.Agudelo55.0
5Bastoni S.Reca60.0
6VerdeShomurodov60.0
7CaldirolaMarlon60.0
8ColpaniMachin55.0
9PetagnaMota55.0
10LozanoPolitano65.0
11ZielinskiElmas60.0
12PiatekBotheim55.0
13KastanosMazzocchi55.0
14LovatoTroost-Ekong55.0
15El ShaarawyDybala65.0
16BelottiAbraham55.0
17KaluluKjaer60.0
18Diaz B.Saelemaekers55.0
19GiroudOrigi60.0
20MiranchukVlasic60.0
21GravillonDjidji60.0
22Ricci S.Linetty60.0
23PalominoDjimsiti60.0
24PasalicEderson D.s.60.0
25HojlundBoga60.0
26LukakuDzeko60.0
27DimarcoGosens60.0
28ImmobilePedro60.0
29HysajLazzari65.0
30CastagnettiGaldames60.0
31VasquezLochoshvili60.0
32BuonaiutoGhiglione55.0
33DudaAbildgaard55.0
34GaichDjuric55.0
35LazovicDoig60.0
36ErlicFerrari G.60.0
37BajramiDefrel60.0
38Matheus HenriqueHarroui60.0
39VicarioPerisan55.0
40MarinGrassi65.0
41CambiaghiSatriano55.0
42Gonzalez N.Ikone'55.0
43SaponaraSottil60.0
44Martinez QuartaIgor60.0
45DjuricicCuisance55.0
46LammersJese'60.0
47AebischerOrsolini60.0
48ZirkzeeSansone55.0
49DominguezMoro N.60.0
50Di MariaChiesa60.0
51CuadradoDe Sciglio60.0
52VlahovicMilik60.0
\n", "
" ], "text/plain": [ " player1 player2 percentage\n", "0 Gallo Pezzella Giu. 65.0\n", "1 Ceesay Colombo 60.0\n", "2 Gonzalez J. Oudin 55.0\n", "3 Samardzic Success 55.0\n", "4 Esposito Sa. Agudelo 55.0\n", "5 Bastoni S. Reca 60.0\n", "6 Verde Shomurodov 60.0\n", "7 Caldirola Marlon 60.0\n", "8 Colpani Machin 55.0\n", "9 Petagna Mota 55.0\n", "10 Lozano Politano 65.0\n", "11 Zielinski Elmas 60.0\n", "12 Piatek Botheim 55.0\n", "13 Kastanos Mazzocchi 55.0\n", "14 Lovato Troost-Ekong 55.0\n", "15 El Shaarawy Dybala 65.0\n", "16 Belotti Abraham 55.0\n", "17 Kalulu Kjaer 60.0\n", "18 Diaz B. Saelemaekers 55.0\n", "19 Giroud Origi 60.0\n", "20 Miranchuk Vlasic 60.0\n", "21 Gravillon Djidji 60.0\n", "22 Ricci S. Linetty 60.0\n", "23 Palomino Djimsiti 60.0\n", "24 Pasalic Ederson D.s. 60.0\n", "25 Hojlund Boga 60.0\n", "26 Lukaku Dzeko 60.0\n", "27 Dimarco Gosens 60.0\n", "28 Immobile Pedro 60.0\n", "29 Hysaj Lazzari 65.0\n", "30 Castagnetti Galdames 60.0\n", "31 Vasquez Lochoshvili 60.0\n", "32 Buonaiuto Ghiglione 55.0\n", "33 Duda Abildgaard 55.0\n", "34 Gaich Djuric 55.0\n", "35 Lazovic Doig 60.0\n", "36 Erlic Ferrari G. 60.0\n", "37 Bajrami Defrel 60.0\n", "38 Matheus Henrique Harroui 60.0\n", "39 Vicario Perisan 55.0\n", "40 Marin Grassi 65.0\n", "41 Cambiaghi Satriano 55.0\n", "42 Gonzalez N. Ikone' 55.0\n", "43 Saponara Sottil 60.0\n", "44 Martinez Quarta Igor 60.0\n", "45 Djuricic Cuisance 55.0\n", "46 Lammers Jese' 60.0\n", "47 Aebischer Orsolini 60.0\n", "48 Zirkzee Sansone 55.0\n", "49 Dominguez Moro N. 60.0\n", "50 Di Maria Chiesa 60.0\n", "51 Cuadrado De Sciglio 60.0\n", "52 Vlahovic Milik 60.0" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "groups = soup.findAll('ul', attrs = {'class' : 'ballot-list'})\n", "\n", "ballots = pd.DataFrame(columns = ['player1', 'player2', 'percentage'])\n", "for i in range (len(groups)):\n", " players = groups[i].findAll('a', attrs = {'class' : 'player-name'})\n", " percs = groups[i].findAll('strong', attrs = {'class' : 'percentage'})\n", " \n", " player1 = players[0].contents[1].contents[0]\n", " player2 = players[1].contents[1].contents[0]\n", " perc = float(percs[0].contents[0].replace('%', ''))\n", " \n", " row_df = pd.DataFrame([[player1, player2, perc]], columns = ballots.columns)\n", " \n", " ballots = pd.concat([ballots, row_df], ignore_index = True)\n", " \n", "ballots" ] }, { "cell_type": "code", "execution_count": 6, "id": "0171c61d", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
starterpercentage
player
Falcone190.0
Gendrey190.0
Baschirotto190.0
Umtiti190.0
Gallo0.6565.0
.........
Barrenechea020.0
Pogba050.0
Chiesa0.455.0
Soule'020.0
Milik0.460.0
\n", "

474 rows × 2 columns

\n", "
" ], "text/plain": [ " starter percentage\n", "player \n", "Falcone 1 90.0\n", "Gendrey 1 90.0\n", "Baschirotto 1 90.0\n", "Umtiti 1 90.0\n", "Gallo 0.65 65.0\n", "... ... ...\n", "Barrenechea 0 20.0\n", "Pogba 0 50.0\n", "Chiesa 0.4 55.0\n", "Soule' 0 20.0\n", "Milik 0.4 60.0\n", "\n", "[474 rows x 2 columns]" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "for i in range(ballots.shape[0]):\n", " p1 = ballots['player1'][i]\n", " p2 = ballots['player2'][i]\n", " \n", " perc = float(ballots['percentage'][i])\n", " \n", " probables.at[p1, 'starter'] = perc/100\n", " probables.at[p2, 'starter'] = 1 - perc/100\n", " \n", "probables" ] }, { "cell_type": "code", "execution_count": 7, "id": "8b5468aa", "metadata": {}, "outputs": [], "source": [ "probables.to_excel('mid_outputs/match_probable_players.xlsx') " ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.13" } }, "nbformat": 4, "nbformat_minor": 5 }