{ "cells": [ { "cell_type": "markdown", "id": "fd8a6cb9", "metadata": {}, "source": [ "Scraping for playing probability for the current Serie A matchday.\n", "\n", "Data from http://fantacalcio.it" ] }, { "cell_type": "code", "execution_count": 1, "id": "4c7b7d26", "metadata": {}, "outputs": [], "source": [ "import requests\n", "from bs4 import BeautifulSoup\n", "import pandas as pd\n", "import numpy as np\n", "import re\n", "import sys, getopt\n", "import csv\n" ] }, { "cell_type": "code", "execution_count": 2, "id": "91897dbb", "metadata": {}, "outputs": [], "source": [ "res = requests.get('https://www.fantacalcio.it/probabili-formazioni-serie-a')" ] }, { "cell_type": "code", "execution_count": 3, "id": "7ba89027", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
playerstarterpercentage
0Carnesecchi190.0
1Bianchetti155.0
2Chiriches180.0
3Vasquez190.0
4Sernicola190.0
............
444Cuadrado060.0
445Pogba030.0
446Miretti050.0
447Soule'030.0
448Kean060.0
\n", "

449 rows × 3 columns

\n", "
" ], "text/plain": [ " player starter percentage\n", "0 Carnesecchi 1 90.0\n", "1 Bianchetti 1 55.0\n", "2 Chiriches 1 80.0\n", "3 Vasquez 1 90.0\n", "4 Sernicola 1 90.0\n", ".. ... ... ...\n", "444 Cuadrado 0 60.0\n", "445 Pogba 0 30.0\n", "446 Miretti 0 50.0\n", "447 Soule' 0 30.0\n", "448 Kean 0 60.0\n", "\n", "[449 rows x 3 columns]" ] }, "execution_count": 3, "metadata": {}, "output_type": "execute_result" } ], "source": [ "soup = BeautifulSoup(res.text)\n", "\n", "groups = soup.findAll('ul', attrs = {'class' : 'player-list'})\n", "\n", "probables = pd.DataFrame(columns = ['player', 'starter', 'percentage'])\n", "\n", "for i in range (len(groups)):\n", " players = groups[i].findAll('a', attrs = {'class' : 'player-name'})\n", " bars = groups[i].findAll('div', attrs = {'class' : 'progress-bar'})\n", " \n", " starter_string = groups[i]['class'][1]\n", " if(starter_string == 'starters'):\n", " starter = 1\n", " else:\n", " starter = 0\n", " \n", " for j in range(len(players)):\n", " player = players[j].contents[0].replace('\\n \\n ', '')\n", " perc = float(bars[j]['aria-valuenow'])\n", " \n", " row_df = pd.DataFrame([[player, starter, perc]], columns = probables.columns)\n", " \n", " probables = pd.concat([probables, row_df], ignore_index = True)\n", " \n", "probables " ] }, { "cell_type": "code", "execution_count": 4, "id": "29e5b389", "metadata": {}, "outputs": [], "source": [ "probables = probables.set_index('player')" ] }, { "cell_type": "code", "execution_count": 5, "id": "d711726e", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
player1player2percentage
0DessersCiofani D.65.0
1BianchettiFerrari A.55.0
2Pezzella Giu.Gallo60.0
3Di Francesco F.Banda55.0
4AbrahamBelotti65.0
5MaticBove60.0
6Akpa AkproHaas60.0
7SatrianoCambiaghi60.0
8EbuehiStojanovic60.0
9BajramiThorstvedt55.0
10ZorteaRuan60.0
11ObiangLopez M.60.0
12DemiralDjimsiti55.0
13MaehleRuggeri60.0
14BogaPasalic60.0
15AgudeloCipot60.0
16VerdeMaldini60.0
17HolmFerrer65.0
18LozanoElmas55.0
19LozanoPolitano55.0
20Mario RuiOlivera60.0
21VojvodaAina60.0
22BuongiornoAdopo60.0
23SanabriaSeck55.0
24SuccessThauvin65.0
25ArslanLovric60.0
26AmrabatMandragora65.0
27BonaventuraBarak55.0
28Gonzalez N.Ikone'55.0
29Dodo'Venuti55.0
30MilenkovicMartinez Quarta60.0
31CambiasoLykogiannis55.0
32DominguezMoro N.60.0
33SorianoAebischer60.0
34BrozovicMkhitaryan55.0
35DzekoLukaku65.0
36DumfriesDarmian55.0
37GiroudOrigi60.0
38SaelemaekersMessias60.0
39De KetelaereDiaz B.55.0
40DawidowiczMagnani65.0
41LasagnaBraaf55.0
42DoigVerdi60.0
43LazzariHysaj60.0
44Luis AlbertoVecino60.0
45CataldiMarcos Antonio65.0
46MarlonCaldirola60.0
47PetagnaMota55.0
48BirindelliMachin55.0
49ColleyAmione60.0
50DjuricicCuisance55.0
51BronnDaniliuc60.0
52SambiaLovato55.0
53Nicolussi CavigliaBohinen60.0
54ChiesaDe Sciglio55.0
55Alex SandroGatti60.0
56VlahovicKean60.0
\n", "
" ], "text/plain": [ " player1 player2 percentage\n", "0 Dessers Ciofani D. 65.0\n", "1 Bianchetti Ferrari A. 55.0\n", "2 Pezzella Giu. Gallo 60.0\n", "3 Di Francesco F. Banda 55.0\n", "4 Abraham Belotti 65.0\n", "5 Matic Bove 60.0\n", "6 Akpa Akpro Haas 60.0\n", "7 Satriano Cambiaghi 60.0\n", "8 Ebuehi Stojanovic 60.0\n", "9 Bajrami Thorstvedt 55.0\n", "10 Zortea Ruan 60.0\n", "11 Obiang Lopez M. 60.0\n", "12 Demiral Djimsiti 55.0\n", "13 Maehle Ruggeri 60.0\n", "14 Boga Pasalic 60.0\n", "15 Agudelo Cipot 60.0\n", "16 Verde Maldini 60.0\n", "17 Holm Ferrer 65.0\n", "18 Lozano Elmas 55.0\n", "19 Lozano Politano 55.0\n", "20 Mario Rui Olivera 60.0\n", "21 Vojvoda Aina 60.0\n", "22 Buongiorno Adopo 60.0\n", "23 Sanabria Seck 55.0\n", "24 Success Thauvin 65.0\n", "25 Arslan Lovric 60.0\n", "26 Amrabat Mandragora 65.0\n", "27 Bonaventura Barak 55.0\n", "28 Gonzalez N. Ikone' 55.0\n", "29 Dodo' Venuti 55.0\n", "30 Milenkovic Martinez Quarta 60.0\n", "31 Cambiaso Lykogiannis 55.0\n", "32 Dominguez Moro N. 60.0\n", "33 Soriano Aebischer 60.0\n", "34 Brozovic Mkhitaryan 55.0\n", "35 Dzeko Lukaku 65.0\n", "36 Dumfries Darmian 55.0\n", "37 Giroud Origi 60.0\n", "38 Saelemaekers Messias 60.0\n", "39 De Ketelaere Diaz B. 55.0\n", "40 Dawidowicz Magnani 65.0\n", "41 Lasagna Braaf 55.0\n", "42 Doig Verdi 60.0\n", "43 Lazzari Hysaj 60.0\n", "44 Luis Alberto Vecino 60.0\n", "45 Cataldi Marcos Antonio 65.0\n", "46 Marlon Caldirola 60.0\n", "47 Petagna Mota 55.0\n", "48 Birindelli Machin 55.0\n", "49 Colley Amione 60.0\n", "50 Djuricic Cuisance 55.0\n", "51 Bronn Daniliuc 60.0\n", "52 Sambia Lovato 55.0\n", "53 Nicolussi Caviglia Bohinen 60.0\n", "54 Chiesa De Sciglio 55.0\n", "55 Alex Sandro Gatti 60.0\n", "56 Vlahovic Kean 60.0" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "groups = soup.findAll('ul', attrs = {'class' : 'ballot-list'})\n", "\n", "ballots = pd.DataFrame(columns = ['player1', 'player2', 'percentage'])\n", "for i in range (len(groups)):\n", " players = groups[i].findAll('a', attrs = {'class' : 'player-name'})\n", " percs = groups[i].findAll('strong', attrs = {'class' : 'percentage'})\n", " \n", " player1 = players[0].contents[0].replace('\\n \\n ', '')\n", " player2 = players[1].contents[0].replace('\\n \\n ', '')\n", " perc = float(percs[0].contents[0].replace('%', ''))\n", " \n", " row_df = pd.DataFrame([[player1, player2, perc]], columns = ballots.columns)\n", " \n", " ballots = pd.concat([ballots, row_df], ignore_index = True)\n", " \n", "ballots" ] }, { "cell_type": "code", "execution_count": 6, "id": "0171c61d", "metadata": {}, "outputs": [ { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
starterpercentage
player
Carnesecchi190.0
Bianchetti0.5555.0
Chiriches180.0
Vasquez190.0
Sernicola190.0
.........
Cuadrado060.0
Pogba030.0
Miretti050.0
Soule'030.0
Kean0.460.0
\n", "

449 rows × 2 columns

\n", "
" ], "text/plain": [ " starter percentage\n", "player \n", "Carnesecchi 1 90.0\n", "Bianchetti 0.55 55.0\n", "Chiriches 1 80.0\n", "Vasquez 1 90.0\n", "Sernicola 1 90.0\n", "... ... ...\n", "Cuadrado 0 60.0\n", "Pogba 0 30.0\n", "Miretti 0 50.0\n", "Soule' 0 30.0\n", "Kean 0.4 60.0\n", "\n", "[449 rows x 2 columns]" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "for i in range(ballots.shape[0]):\n", " p1 = ballots['player1'][i]\n", " p2 = ballots['player2'][i]\n", " \n", " perc = float(ballots['percentage'][i])\n", " \n", " probables.at[p1, 'starter'] = perc/100\n", " probables.at[p2, 'starter'] = 1 - perc/100\n", " \n", "probables" ] }, { "cell_type": "code", "execution_count": 7, "id": "8b5468aa", "metadata": {}, "outputs": [], "source": [ "probables.to_excel('mid_outputs/match_probable_players.xlsx') " ] }, { "cell_type": "code", "execution_count": null, "id": "62164549", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.13" } }, "nbformat": 4, "nbformat_minor": 5 }