Files
fantabeto/5_scraping_match_probable_players.ipynb
T

813 lines
24 KiB
Plaintext
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
{
"cells": [
{
"cell_type": "markdown",
"id": "fd8a6cb9",
"metadata": {},
"source": [
"Scraping for playing probability for the current Serie A matchday.\n",
"\n",
"Data from http://fantacalcio.it"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "4c7b7d26",
"metadata": {},
"outputs": [],
"source": [
"import requests\n",
"from bs4 import BeautifulSoup\n",
"import pandas as pd\n",
"import numpy as np\n",
"import re\n",
"import sys, getopt\n",
"import csv\n"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "91897dbb",
"metadata": {},
"outputs": [],
"source": [
"res = requests.get('https://www.fantacalcio.it/probabili-formazioni-serie-a')"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "7ba89027",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>player</th>\n",
" <th>starter</th>\n",
" <th>percentage</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Tatarusanu</td>\n",
" <td>1</td>\n",
" <td>90.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>Thiaw</td>\n",
" <td>1</td>\n",
" <td>65.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>Kjaer</td>\n",
" <td>1</td>\n",
" <td>85.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>Kalulu</td>\n",
" <td>1</td>\n",
" <td>90.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>Saelemaekers</td>\n",
" <td>1</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>433</th>\n",
" <td>Dumfries</td>\n",
" <td>0</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>434</th>\n",
" <td>Mkhitaryan</td>\n",
" <td>0</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>435</th>\n",
" <td>Asllani</td>\n",
" <td>0</td>\n",
" <td>30.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>436</th>\n",
" <td>Gagliardini</td>\n",
" <td>0</td>\n",
" <td>50.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>437</th>\n",
" <td>Lukaku</td>\n",
" <td>0</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>438 rows × 3 columns</p>\n",
"</div>"
],
"text/plain": [
" player starter percentage\n",
"0 Tatarusanu 1 90.0\n",
"1 Thiaw 1 65.0\n",
"2 Kjaer 1 85.0\n",
"3 Kalulu 1 90.0\n",
"4 Saelemaekers 1 60.0\n",
".. ... ... ...\n",
"433 Dumfries 0 60.0\n",
"434 Mkhitaryan 0 55.0\n",
"435 Asllani 0 30.0\n",
"436 Gagliardini 0 50.0\n",
"437 Lukaku 0 60.0\n",
"\n",
"[438 rows x 3 columns]"
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"soup = BeautifulSoup(res.text)\n",
"\n",
"groups = soup.findAll('ul', attrs = {'class' : 'player-list'})\n",
"\n",
"probables = pd.DataFrame(columns = ['player', 'starter', 'percentage'])\n",
"\n",
"for i in range (len(groups)):\n",
" players = groups[i].findAll('a', attrs = {'class' : 'player-name'})\n",
" bars = groups[i].findAll('div', attrs = {'class' : 'progress-bar'})\n",
" \n",
" starter_string = groups[i]['class'][1]\n",
" if(starter_string == 'starters'):\n",
" starter = 1\n",
" else:\n",
" starter = 0\n",
" \n",
" for j in range(len(players)):\n",
" player = players[j].contents[1].contents[0]\n",
" perc = float(bars[j]['aria-valuenow'])\n",
" \n",
" row_df = pd.DataFrame([[player, starter, perc]], columns = probables.columns)\n",
" \n",
" probables = pd.concat([probables, row_df], ignore_index = True)\n",
" \n",
"probables "
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "29e5b389",
"metadata": {},
"outputs": [],
"source": [
"probables = probables.set_index('player')"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "d711726e",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>player1</th>\n",
" <th>player2</th>\n",
" <th>percentage</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>Saelemaekers</td>\n",
" <td>Calabria</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>Diaz B.</td>\n",
" <td>De Ketelaere</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>Thiaw</td>\n",
" <td>Messias</td>\n",
" <td>65.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>Singo</td>\n",
" <td>Aina</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>Rodriguez R.</td>\n",
" <td>Buongiorno</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>Akpa Akpro</td>\n",
" <td>Haas</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>Satriano</td>\n",
" <td>Piccoli</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>Ebuehi</td>\n",
" <td>Stojanovic</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>Holm</td>\n",
" <td>Ferrer</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>Nzola</td>\n",
" <td>Shomurodov</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>10</th>\n",
" <td>Agudelo</td>\n",
" <td>Cipot</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>11</th>\n",
" <td>Di Francesco F.</td>\n",
" <td>Banda</td>\n",
" <td>65.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>12</th>\n",
" <td>Gallo</td>\n",
" <td>Pezzella Giu.</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>13</th>\n",
" <td>El Shaarawy</td>\n",
" <td>Celik</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>14</th>\n",
" <td>Matic</td>\n",
" <td>Bove</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>15</th>\n",
" <td>Cataldi</td>\n",
" <td>Vecino</td>\n",
" <td>65.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16</th>\n",
" <td>Lazzari</td>\n",
" <td>Hysaj</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>17</th>\n",
" <td>Boga</td>\n",
" <td>Ederson D.s.</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>18</th>\n",
" <td>Hojlund</td>\n",
" <td>Zapata D.</td>\n",
" <td>65.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>19</th>\n",
" <td>Djimsiti</td>\n",
" <td>Demiral</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>20</th>\n",
" <td>Thauvin</td>\n",
" <td>Success</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>21</th>\n",
" <td>Arslan</td>\n",
" <td>Lovric</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>22</th>\n",
" <td>Bajrami</td>\n",
" <td>Thorstvedt</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>23</th>\n",
" <td>Ruan</td>\n",
" <td>Ferrari G.</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>24</th>\n",
" <td>Lopez M.</td>\n",
" <td>Obiang</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>25</th>\n",
" <td>Zirkzee</td>\n",
" <td>Barrow</td>\n",
" <td>65.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>26</th>\n",
" <td>Cambiaso</td>\n",
" <td>Lykogiannis</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>27</th>\n",
" <td>Soriano</td>\n",
" <td>Aebischer</td>\n",
" <td>65.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>28</th>\n",
" <td>Marlon</td>\n",
" <td>Caldirola</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>29</th>\n",
" <td>Petagna</td>\n",
" <td>Mota</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>30</th>\n",
" <td>Sensi</td>\n",
" <td>Machin</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>31</th>\n",
" <td>De Sciglio</td>\n",
" <td>Chiesa</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>32</th>\n",
" <td>Brekalo</td>\n",
" <td>Saponara</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>33</th>\n",
" <td>Brekalo</td>\n",
" <td>Ikone'</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>34</th>\n",
" <td>Jovic</td>\n",
" <td>Cabral</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>35</th>\n",
" <td>Barak</td>\n",
" <td>Duncan</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>36</th>\n",
" <td>Lozano</td>\n",
" <td>Elmas</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>37</th>\n",
" <td>Lozano</td>\n",
" <td>Politano</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>38</th>\n",
" <td>Mario Rui</td>\n",
" <td>Olivera</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>39</th>\n",
" <td>Buonaiuto</td>\n",
" <td>Afena-Gyan</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>40</th>\n",
" <td>Benassi</td>\n",
" <td>Castagnetti</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>41</th>\n",
" <td>Ceccherini</td>\n",
" <td>Coppola D.</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>42</th>\n",
" <td>Magnani</td>\n",
" <td>Dawidowicz</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>43</th>\n",
" <td>Ngonge</td>\n",
" <td>Gaich</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>44</th>\n",
" <td>Bronn</td>\n",
" <td>Daniliuc</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>45</th>\n",
" <td>Lovato</td>\n",
" <td>Sambia</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>46</th>\n",
" <td>Bohinen</td>\n",
" <td>Nicolussi Caviglia</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>47</th>\n",
" <td>Djuricic</td>\n",
" <td>Sabiri</td>\n",
" <td>65.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>48</th>\n",
" <td>Murru</td>\n",
" <td>Murillo</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>49</th>\n",
" <td>Rincon</td>\n",
" <td>Cuisance</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>50</th>\n",
" <td>Brozovic</td>\n",
" <td>Mkhitaryan</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>51</th>\n",
" <td>Dzeko</td>\n",
" <td>Lukaku</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>52</th>\n",
" <td>Darmian</td>\n",
" <td>Dumfries</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" player1 player2 percentage\n",
"0 Saelemaekers Calabria 60.0\n",
"1 Diaz B. De Ketelaere 60.0\n",
"2 Thiaw Messias 65.0\n",
"3 Singo Aina 55.0\n",
"4 Rodriguez R. Buongiorno 55.0\n",
"5 Akpa Akpro Haas 60.0\n",
"6 Satriano Piccoli 60.0\n",
"7 Ebuehi Stojanovic 60.0\n",
"8 Holm Ferrer 55.0\n",
"9 Nzola Shomurodov 55.0\n",
"10 Agudelo Cipot 60.0\n",
"11 Di Francesco F. Banda 65.0\n",
"12 Gallo Pezzella Giu. 60.0\n",
"13 El Shaarawy Celik 60.0\n",
"14 Matic Bove 60.0\n",
"15 Cataldi Vecino 65.0\n",
"16 Lazzari Hysaj 55.0\n",
"17 Boga Ederson D.s. 55.0\n",
"18 Hojlund Zapata D. 65.0\n",
"19 Djimsiti Demiral 60.0\n",
"20 Thauvin Success 55.0\n",
"21 Arslan Lovric 60.0\n",
"22 Bajrami Thorstvedt 55.0\n",
"23 Ruan Ferrari G. 60.0\n",
"24 Lopez M. Obiang 55.0\n",
"25 Zirkzee Barrow 65.0\n",
"26 Cambiaso Lykogiannis 55.0\n",
"27 Soriano Aebischer 65.0\n",
"28 Marlon Caldirola 55.0\n",
"29 Petagna Mota 60.0\n",
"30 Sensi Machin 55.0\n",
"31 De Sciglio Chiesa 60.0\n",
"32 Brekalo Saponara 55.0\n",
"33 Brekalo Ikone' 55.0\n",
"34 Jovic Cabral 60.0\n",
"35 Barak Duncan 60.0\n",
"36 Lozano Elmas 55.0\n",
"37 Lozano Politano 55.0\n",
"38 Mario Rui Olivera 60.0\n",
"39 Buonaiuto Afena-Gyan 60.0\n",
"40 Benassi Castagnetti 60.0\n",
"41 Ceccherini Coppola D. 60.0\n",
"42 Magnani Dawidowicz 55.0\n",
"43 Ngonge Gaich 60.0\n",
"44 Bronn Daniliuc 60.0\n",
"45 Lovato Sambia 55.0\n",
"46 Bohinen Nicolussi Caviglia 55.0\n",
"47 Djuricic Sabiri 65.0\n",
"48 Murru Murillo 60.0\n",
"49 Rincon Cuisance 60.0\n",
"50 Brozovic Mkhitaryan 55.0\n",
"51 Dzeko Lukaku 60.0\n",
"52 Darmian Dumfries 55.0"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"groups = soup.findAll('ul', attrs = {'class' : 'ballot-list'})\n",
"\n",
"ballots = pd.DataFrame(columns = ['player1', 'player2', 'percentage'])\n",
"for i in range (len(groups)):\n",
" players = groups[i].findAll('a', attrs = {'class' : 'player-name'})\n",
" percs = groups[i].findAll('strong', attrs = {'class' : 'percentage'})\n",
" \n",
" player1 = players[0].contents[1].contents[0]\n",
" player2 = players[1].contents[1].contents[0]\n",
" perc = float(percs[0].contents[0].replace('%', ''))\n",
" \n",
" row_df = pd.DataFrame([[player1, player2, perc]], columns = ballots.columns)\n",
" \n",
" ballots = pd.concat([ballots, row_df], ignore_index = True)\n",
" \n",
"ballots"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "0171c61d",
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>starter</th>\n",
" <th>percentage</th>\n",
" </tr>\n",
" <tr>\n",
" <th>player</th>\n",
" <th></th>\n",
" <th></th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>Tatarusanu</th>\n",
" <td>1</td>\n",
" <td>90.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>Thiaw</th>\n",
" <td>0.65</td>\n",
" <td>65.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>Kjaer</th>\n",
" <td>1</td>\n",
" <td>85.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>Kalulu</th>\n",
" <td>1</td>\n",
" <td>90.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>Saelemaekers</th>\n",
" <td>0.6</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>Dumfries</th>\n",
" <td>0.45</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>Mkhitaryan</th>\n",
" <td>0.45</td>\n",
" <td>55.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>Asllani</th>\n",
" <td>0</td>\n",
" <td>30.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>Gagliardini</th>\n",
" <td>0</td>\n",
" <td>50.0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>Lukaku</th>\n",
" <td>0.4</td>\n",
" <td>60.0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>438 rows × 2 columns</p>\n",
"</div>"
],
"text/plain": [
" starter percentage\n",
"player \n",
"Tatarusanu 1 90.0\n",
"Thiaw 0.65 65.0\n",
"Kjaer 1 85.0\n",
"Kalulu 1 90.0\n",
"Saelemaekers 0.6 60.0\n",
"... ... ...\n",
"Dumfries 0.45 60.0\n",
"Mkhitaryan 0.45 55.0\n",
"Asllani 0 30.0\n",
"Gagliardini 0 50.0\n",
"Lukaku 0.4 60.0\n",
"\n",
"[438 rows x 2 columns]"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"for i in range(ballots.shape[0]):\n",
" p1 = ballots['player1'][i]\n",
" p2 = ballots['player2'][i]\n",
" \n",
" perc = float(ballots['percentage'][i])\n",
" \n",
" probables.at[p1, 'starter'] = perc/100\n",
" probables.at[p2, 'starter'] = 1 - perc/100\n",
" \n",
"probables"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "8b5468aa",
"metadata": {},
"outputs": [],
"source": [
"probables.to_excel('mid_outputs/match_probable_players.xlsx') "
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "62164549",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.13"
}
},
"nbformat": 4,
"nbformat_minor": 5
}