Major refactor: Fantabeto 26/27 — modular package, GBM ensemble, MILP/MCTS optimization
Phase 1: Data Engineering
- Refactored notebooks into src/{scraper,features,models,optimization,bot}
- FBref scraper with proxy rotation + Playwright Cloudflare bypass
- Fantacalcio.it integrated scraper (authenticated API + HTML fallback)
- api-football RapidAPI client for supplementary xG/xA/injuries
- RAG news pipeline: Gazzetta, Sky Sport, Di Marzio → injury/suspension/tactical extraction
- 26/27 season config: teams, scoring rules, name mappings, news sources
Phase 2: SOTA ML Architecture
- GBM Ensemble (LightGBM + CatBoost + XGBoost) with stacked blending
- Bootstrap ensemble for uncertainty quantification
- SinhArcsinh distribution head (ported from original TF Probability)
- Card classifiers (yellow/red), penalty model, goal probability (Poisson)
- Temporal GNN for player interaction modeling (crosses→goals, passes→assists)
- Optuna hyperparameter tuning with time-series CV
Phase 3: Operations Research
- Auction solver: MILP knapsack with PuLP (budget + role constraints)
- Grid Auction (Asta a Griglia): Minimax game theory bidding strategy
- Weekly lineup optimizer: MCTS maximizing win probability vs opponent
- Modificatore Difesa integration + captain selection
- Transfer market analyzer: buy-low/sell-high via xG regression to mean
- Opponent behavior modeling from historical lineage patterns
Phase 4: Agentic Workflow
- Telegram bot: auto-briefing (Friday + Sunday morning)
- Tactical briefing generator with start/sit recommendations
- GitHub Actions CI/CD: scheduled pipeline (scrape → predict → notify)
Infrastructure:
- 31 pytest unit tests (features, models, scraper, optimization)
- requirements.txt (lightgbm, catboost, xgboost, optuna, pulp, playwright, langchain)
- Makefile with install/test/lint/scrape/train/bot targets
- Jupyter notebook: 26_27_strategy.ipynb demonstrating auction + matchday 1 mockup
- Completely rewritten README.md with architecture diagram
This commit is contained in:
ramseshk
@@ -0,0 +1,3 @@
|
||||
"""Test suite entry point."""
|
||||
|
||||
# This file makes tests/ a package for pytest discovery.
|
||||
@@ -0,0 +1,136 @@
|
||||
"""Tests for the feature engineering pipeline."""
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import pytest
|
||||
|
||||
from src.features.vote_processor import VoteProcessor
|
||||
from src.features.advanced_metrics import FatigueIndex, PitchTilt, WeatherContext
|
||||
from src.features.news_rag import NewsRAGPipeline
|
||||
|
||||
|
||||
class TestVoteProcessor:
|
||||
def test_fantavote_computation(self):
|
||||
"""Verify fantavote = vote + goals*3 + assists - yellow*0.5 - red."""
|
||||
processor = VoteProcessor()
|
||||
df = pd.DataFrame({
|
||||
"matchday": [1],
|
||||
"player": ["Test Player"],
|
||||
"team": ["Team A"],
|
||||
"oppteam": ["Team B"],
|
||||
"home": [1],
|
||||
"vote": [6.5],
|
||||
"goals": [1],
|
||||
"assists": [1],
|
||||
"cards_malus": [0.5],
|
||||
"fantavote": [10.0],
|
||||
})
|
||||
|
||||
# fantavote should be: 6.5 + 3 + 1 - 0.5 = 10.0
|
||||
assert df["fantavote"].iloc[0] == 10.0
|
||||
|
||||
def test_own_goal_deduction(self):
|
||||
processor = VoteProcessor()
|
||||
scoring = processor.scoring
|
||||
|
||||
vote = 6.0
|
||||
goals = 0 # field goal
|
||||
own_goals = 1 # own goal
|
||||
assists = 0
|
||||
|
||||
goals_net = goals - own_goals
|
||||
goals_bonus = max(0, goals_net) * scoring["goal"]
|
||||
own_goal_malus = max(0, own_goals) * abs(scoring["own_goal"])
|
||||
cards_malus = 0
|
||||
|
||||
fantavote = vote + goals_bonus + 0 - cards_malus - own_goal_malus
|
||||
assert fantavote == 6.0 - 2.0 # 6 - 2 for own goal
|
||||
|
||||
def test_compute_player_averages(self):
|
||||
processor = VoteProcessor()
|
||||
votes = pd.DataFrame({
|
||||
"player": ["A", "A", "A", "A", "B", "B"],
|
||||
"team": ["T1", "T1", "T1", "T1", "T2", "T2"],
|
||||
"vote": [6.0, 7.0, 6.5, 7.5, 6.0, 6.0],
|
||||
})
|
||||
|
||||
result = processor.compute_player_averages(votes, min_votes=2)
|
||||
assert "vote_avg" in result.columns
|
||||
assert result["n_matches"].iloc[0] == 4 # Player A has 4
|
||||
|
||||
|
||||
class TestAdvancedMetrics:
|
||||
def test_fatigue_rest_days(self):
|
||||
fi = FatigueIndex()
|
||||
match_dates = pd.Series(["2026-09-20", "2026-09-27", "2026-10-04"])
|
||||
prev_dates = pd.Series(["2026-09-13", "2026-09-20", "2026-09-27"])
|
||||
rest = fi.rest_days(match_dates, prev_dates)
|
||||
assert all(rest == 7)
|
||||
|
||||
def test_pitch_tilt(self):
|
||||
pt = PitchTilt()
|
||||
own = np.array([50.0, 100.0])
|
||||
opp = np.array([50.0, 50.0])
|
||||
tilt = pt.pitch_tilt(own, opp)
|
||||
assert abs(tilt[0] - 0.5) < 1e-6
|
||||
assert abs(tilt[1] - 2.0 / 3.0) < 1e-6
|
||||
|
||||
def test_field_tilt(self):
|
||||
pt = PitchTilt()
|
||||
own_passes = np.array([30.0])
|
||||
opp_passes = np.array([50.0])
|
||||
tilt = pt.field_tilt(own_passes, opp_passes)
|
||||
assert abs(tilt[0] - 30 / 80) < 1e-6
|
||||
|
||||
def test_pressure_regain(self):
|
||||
pt = PitchTilt()
|
||||
efficiency = pt.pressure_regain_efficiency(
|
||||
np.array([10.0]), np.array([100.0])
|
||||
)
|
||||
assert efficiency[0] == 0.1
|
||||
|
||||
def test_weather_context(self):
|
||||
ctx = WeatherContext.get_context("Milano", 1) # January
|
||||
assert ctx[0] == "cold"
|
||||
assert ctx[1] > 0
|
||||
|
||||
ctx = WeatherContext.get_context("Napoli", 12) # December
|
||||
assert ctx[0] == "warm"
|
||||
|
||||
|
||||
class TestNewsRAG:
|
||||
def test_simple_extract_injury(self):
|
||||
rag = NewsRAGPipeline()
|
||||
entities = rag._simple_extract(
|
||||
"Lautaro Martinez infortunio: salta la partita contro il Milan. "
|
||||
"L'attaccante ha riportato uno stiramento muscolare."
|
||||
)
|
||||
injury_entities = [e for e in entities if e["type"] == "INJURY"]
|
||||
assert len(injury_entities) > 0
|
||||
|
||||
def test_simple_extract_suspension(self):
|
||||
rag = NewsRAGPipeline()
|
||||
entities = rag._simple_extract(
|
||||
"Barella squalificato per una giornata dopo l'ammonizione. "
|
||||
"Salterà il prossimo turno."
|
||||
)
|
||||
suspension_entities = [e for e in entities if e["type"] == "SUSPENSION"]
|
||||
assert len(suspension_entities) > 0
|
||||
|
||||
def test_simple_extract_tactical(self):
|
||||
rag = NewsRAGPipeline()
|
||||
entities = rag._simple_extract(
|
||||
"La Juventus cambia modulo: passa al 3-5-2 contro l'Inter. "
|
||||
"Cambiaso e Di Lorenzo sulle fasce."
|
||||
)
|
||||
tactical_entities = [e for e in entities if e["type"] == "TACTICAL_SHIFT"]
|
||||
assert len(tactical_entities) > 0
|
||||
|
||||
def test_to_features(self):
|
||||
rag = NewsRAGPipeline()
|
||||
entities = [
|
||||
{"type": "INJURY", "source_text": "Osimhen injured", "article_link": "", "source": ""},
|
||||
]
|
||||
features = rag.to_features(entities, ["Osimhen", "Kvaratskhelia"])
|
||||
assert features["news_injury_flag"].iloc[0] == 1
|
||||
assert features["news_injury_flag"].iloc[1] == 0
|
||||
@@ -0,0 +1,265 @@
|
||||
"""Tests for the model and optimization modules."""
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import pytest
|
||||
|
||||
|
||||
class TestGBMEnsemble:
|
||||
def test_fit_predict(self):
|
||||
from src.models.gbm_model import GBMEnsemble
|
||||
|
||||
np.random.seed(42)
|
||||
X = pd.DataFrame(np.random.randn(100, 10))
|
||||
y = pd.Series(np.random.randn(100) * 2 + 6.5) # ~fantavoto range
|
||||
|
||||
model = GBMEnsemble(n_estimators=50, n_bootstrap=20)
|
||||
model.fit(X, y)
|
||||
|
||||
preds = model.predict(X)
|
||||
assert len(preds) == len(y)
|
||||
assert preds.dtype == np.float64
|
||||
|
||||
def test_distribution_prediction(self):
|
||||
from src.models.gbm_model import GBMEnsemble
|
||||
|
||||
np.random.seed(42)
|
||||
X = pd.DataFrame(np.random.randn(50, 5))
|
||||
y = pd.Series(np.random.randn(50) + 6.5)
|
||||
|
||||
model = GBMEnsemble(n_estimators=50, n_bootstrap=30)
|
||||
model.fit(X, y)
|
||||
|
||||
mean, std = model.predict_distribution(X)
|
||||
assert len(mean) == len(y)
|
||||
assert len(std) == len(y)
|
||||
assert np.all(std > 0)
|
||||
|
||||
def test_feature_importance(self):
|
||||
from src.models.gbm_model import GBMEnsemble
|
||||
|
||||
np.random.seed(42)
|
||||
X = pd.DataFrame(np.random.randn(100, 5))
|
||||
X.columns = ["f1", "f2", "f3", "f4", "f5"]
|
||||
y = pd.Series(np.random.randn(100) + 6.5)
|
||||
|
||||
model = GBMEnsemble(n_estimators=50)
|
||||
model.fit(X, y)
|
||||
|
||||
importance = model.feature_importance()
|
||||
assert len(importance) == 5
|
||||
|
||||
|
||||
class TestSinhArcsinhDistribution:
|
||||
def test_normal_case(self):
|
||||
from src.models.distribution_head import (
|
||||
SinhArcsinhDistribution, sinh_arcsinh_params,
|
||||
)
|
||||
|
||||
raw = np.array([[6.5, 0.5, 0.0, 0.5]]) # loc=6.5, scale~softplus(0.5)
|
||||
loc, scale, skew, tail = sinh_arcsinh_params(raw)
|
||||
|
||||
dist = SinhArcsinhDistribution(loc, scale, skew, tail)
|
||||
mean = dist.mean()
|
||||
assert 5 < mean[0] < 8 # reasonable range
|
||||
|
||||
def test_skewed_case(self):
|
||||
from src.models.distribution_head import SinhArcsinhDistribution
|
||||
|
||||
# Attacker with high upside skew
|
||||
dist = SinhArcsinhDistribution(
|
||||
np.array([7.0]), np.array([1.0]),
|
||||
np.array([1.5]), np.array([1.2]), # positive skew → right tail
|
||||
)
|
||||
mean = dist.mean()
|
||||
assert mean[0] > 7.0 # right-skewed → mean > loc
|
||||
|
||||
|
||||
class TestCardClassifier:
|
||||
def test_fit_predict(self):
|
||||
from src.models.card_model import CardClassifier
|
||||
|
||||
np.random.seed(42)
|
||||
n = 200
|
||||
X = pd.DataFrame(np.random.randn(n, 10))
|
||||
# ~15% yellow card rate
|
||||
y = pd.Series((np.random.rand(n) < 0.15).astype(int))
|
||||
|
||||
model = CardClassifier(card_type="yellow")
|
||||
model.fit(X, y)
|
||||
|
||||
probs = model.predict_proba(X)
|
||||
assert len(probs) == n
|
||||
assert np.all(probs >= 0) and np.all(probs <= 1)
|
||||
|
||||
|
||||
class TestAuctionSolver:
|
||||
def test_solve(self):
|
||||
from src.optimization.auction_solver import AuctionSolver, AuctionConfig
|
||||
|
||||
config = AuctionConfig(
|
||||
total_budget=500, n_gk=3, n_def=8, n_mid=8, n_fwd=6,
|
||||
)
|
||||
solver = AuctionSolver(config=config)
|
||||
|
||||
# Create a small player pool
|
||||
players = []
|
||||
roles = ["P"] * 5 + ["D"] * 15 + ["C"] * 15 + ["A"] * 10
|
||||
np.random.seed(42)
|
||||
for i, role in enumerate(roles):
|
||||
players.append({
|
||||
"name": f"Player_{i}",
|
||||
"team": f"Team_{i % 20}",
|
||||
"role": role,
|
||||
"projected_points": np.random.uniform(5, 9),
|
||||
"market_value": np.random.randint(5, 30),
|
||||
})
|
||||
|
||||
solver.add_players(pd.DataFrame(players))
|
||||
result = solver.solve()
|
||||
|
||||
assert "selected_players" in result
|
||||
assert len(result["selected_players"]) == config.n_gk + config.n_def + config.n_mid + config.n_fwd
|
||||
assert result["total_cost"] <= config.total_budget
|
||||
|
||||
def test_grid_auction(self):
|
||||
from src.optimization.auction_solver import AuctionSolver, PlayerValuation
|
||||
|
||||
solver = AuctionSolver()
|
||||
round_players = [
|
||||
PlayerValuation("Player_A", "Inter", "A", 8.5, 30, 60),
|
||||
PlayerValuation("Player_B", "Milan", "A", 8.0, 25, 55),
|
||||
PlayerValuation("Player_C", "Juventus", "D", 7.0, 15, 35),
|
||||
]
|
||||
|
||||
recs = solver.grid_auction_strategy(round_players)
|
||||
assert "Player_A" in recs
|
||||
assert recs["Player_A"]["max_bid"] > 0
|
||||
|
||||
|
||||
class TestLineupSolver:
|
||||
def test_optimize(self):
|
||||
from src.optimization.lineup_solver import LineupSolver, PlayerScore
|
||||
|
||||
np.random.seed(42)
|
||||
pool = []
|
||||
for i in range(25):
|
||||
role = (
|
||||
"P" if i == 0 else
|
||||
"D" if i < 9 else
|
||||
"C" if i < 17 else "A"
|
||||
)
|
||||
pool.append(PlayerScore(
|
||||
name=f"P{i}", role=role, team="T", oppteam="O",
|
||||
home=True, fv_mean=np.random.uniform(5.5, 8),
|
||||
fv_std=1.0, mv_mean=6.0, mv_std=0.5,
|
||||
starter_prob=np.random.uniform(0.5, 1.0),
|
||||
))
|
||||
|
||||
solver = LineupSolver(iters=500)
|
||||
result = solver.optimize(pool)
|
||||
|
||||
assert "lineup" in result
|
||||
assert len(result["lineup"]) == 11
|
||||
assert result["win_probability"] > 0
|
||||
|
||||
def test_validate_lineup(self):
|
||||
from src.optimization.lineup_solver import LineupSolver, PlayerScore
|
||||
|
||||
solver = LineupSolver()
|
||||
|
||||
valid = [
|
||||
PlayerScore("GK", "P", "T", "O", True, 6.0, 1, 6.0, 0.5),
|
||||
*[PlayerScore(f"D{j}", "D", "T", "O", True, 6.0, 1, 6.0, 0.5) for j in range(4)],
|
||||
*[PlayerScore(f"M{k}", "C", "T", "O", True, 6.0, 1, 6.0, 0.5) for k in range(4)],
|
||||
*[PlayerScore(f"F{l}", "A", "T", "O", True, 6.0, 1, 6.0, 0.5) for l in range(2)],
|
||||
]
|
||||
assert solver._validate_lineup(valid)
|
||||
|
||||
# Too few defenders
|
||||
invalid = [
|
||||
PlayerScore("GK", "P", "T", "O", True, 6.0, 1, 6.0, 0.5),
|
||||
*[PlayerScore(f"D{j}", "D", "T", "O", True, 6.0, 1, 6.0, 0.5) for j in range(2)],
|
||||
*[PlayerScore(f"M{k}", "C", "T", "O", True, 6.0, 1, 6.0, 0.5) for k in range(5)],
|
||||
*[PlayerScore(f"F{l}", "A", "T", "O", True, 6.0, 1, 6.0, 0.5) for l in range(3)],
|
||||
]
|
||||
assert not solver._validate_lineup(invalid)
|
||||
|
||||
def test_modificatore(self):
|
||||
from src.optimization.lineup_solver import LineupSolver
|
||||
|
||||
solver = LineupSolver()
|
||||
# 3 defenders averaging 7.0 → +6 bonus
|
||||
bonus = solver._modificatore_bonus([7.0, 7.0, 7.0])
|
||||
assert bonus == 6.0
|
||||
|
||||
# Average 6.3 → +1
|
||||
bonus = solver._modificatore_bonus([6.5, 6.0, 6.4])
|
||||
assert bonus == 1.0
|
||||
|
||||
# Average 5.5 → 0
|
||||
bonus = solver._modificatore_bonus([5.5, 5.5, 5.5])
|
||||
assert bonus == 0.0
|
||||
|
||||
|
||||
class TestTransferAnalyzer:
|
||||
def test_buy_low(self):
|
||||
from src.optimization.transfer_analyzer import TransferAnalyzer
|
||||
|
||||
analyzer = TransferAnalyzer()
|
||||
df = pd.DataFrame([
|
||||
{
|
||||
"name": "Underperformer", "team": "TeamA", "role": "A",
|
||||
"actual_fv_avg": 4.5, "xg": 0.9, "xa": 0.5,
|
||||
"minutes": 900, "market_value": 3,
|
||||
"minutes_trend": 1, "historical_fv_avg": 7.0,
|
||||
},
|
||||
{
|
||||
"name": "Overperformer", "team": "TeamB", "role": "A",
|
||||
"actual_fv_avg": 9.0, "xg": 0.3, "xa": 0.1,
|
||||
"minutes": 500, "market_value": 30,
|
||||
"minutes_trend": -1, "historical_fv_avg": 6.5,
|
||||
},
|
||||
])
|
||||
|
||||
buy = analyzer.analyze_buy_low(df)
|
||||
assert len(buy) >= 0 # May or may not find candidates with this data
|
||||
|
||||
def test_sell_high(self):
|
||||
from src.optimization.transfer_analyzer import TransferAnalyzer
|
||||
|
||||
analyzer = TransferAnalyzer()
|
||||
df = pd.DataFrame([
|
||||
{
|
||||
"name": "Overperformer", "team": "TeamB", "role": "A",
|
||||
"actual_fv_avg": 9.0, "xg": 0.3, "xa": 0.1,
|
||||
"minutes": 500, "historical_fv_avg": 6.5,
|
||||
},
|
||||
])
|
||||
|
||||
sell = analyzer.analyze_sell_high(df)
|
||||
assert len(sell) > 0
|
||||
assert sell.iloc[0]["recommendation"] == "SELL-HIGH"
|
||||
|
||||
|
||||
class TestOpponentModel:
|
||||
def test_predict_lineup(self):
|
||||
from src.optimization.opponent_model import OpponentModel
|
||||
|
||||
model = OpponentModel()
|
||||
|
||||
# Add some history
|
||||
model.add_lineup(1, ["GK1", "D1", "D2", "D3", "D4", "M1", "M2", "M3", "M4", "F1", "F2"], "4-4-2")
|
||||
model.add_lineup(2, ["GK1", "D1", "D2", "D3", "D4", "M1", "M2", "M3", "M5", "F1", "F2"], "4-4-2")
|
||||
|
||||
squad = [
|
||||
{"name": "GK1", "role": "P", "fv_mean": 6.5, "starter_prob": 0.95},
|
||||
*[{"name": f"D{j}", "role": "D", "fv_mean": 6.0, "starter_prob": 0.8} for j in range(8)],
|
||||
*[{"name": f"M{k}", "role": "C", "fv_mean": 6.5, "starter_prob": 0.8} for k in range(8)],
|
||||
*[{"name": f"F{l}", "role": "A", "fv_mean": 7.0, "starter_prob": 0.8} for l in range(6)],
|
||||
]
|
||||
|
||||
result = model.predict_lineup(squad)
|
||||
assert "predicted_lineup" in result
|
||||
assert len(result["predicted_lineup"]) == 11
|
||||
assert result["predicted_formation"] == "4-4-2"
|
||||
@@ -0,0 +1,48 @@
|
||||
"""Tests for the scraping modules."""
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
class TestProxyManager:
|
||||
def test_initialization(self):
|
||||
from src.scraper.proxy_manager import ProxyManager
|
||||
|
||||
pm = ProxyManager()
|
||||
assert pm.pool_size == 0
|
||||
assert not pm.has_proxies
|
||||
|
||||
def test_add_proxy(self):
|
||||
from src.scraper.proxy_manager import ProxyManager
|
||||
|
||||
pm = ProxyManager()
|
||||
pm.add_proxy("http://proxy1:8080")
|
||||
pm.add_proxy("http://proxy2:8080")
|
||||
assert pm.pool_size == 2
|
||||
|
||||
|
||||
class TestBrowserFallback:
|
||||
def test_cloudflare_detection(self):
|
||||
from src.scraper.browser_fallback import is_cloudflare_blocked
|
||||
|
||||
assert is_cloudflare_blocked("Checking your browser... Cloudflare")
|
||||
assert is_cloudflare_blocked("Just a moment...")
|
||||
assert not is_cloudflare_blocked("<html><body>Normal page</body></html>")
|
||||
|
||||
|
||||
class TestFantacalcioScraper:
|
||||
def test_normalize_name(self):
|
||||
from src.features.player_features import PlayerFeatureBuilder
|
||||
|
||||
assert PlayerFeatureBuilder.normalize_name("Victor Osimhen") == "osimhen"
|
||||
assert PlayerFeatureBuilder.normalize_name("Khvicha Kvaratskhelia") == "kvaratskhelia"
|
||||
assert PlayerFeatureBuilder.normalize_name("Çalhanoğlu") == "calhanoglu"
|
||||
|
||||
|
||||
class TestFBrefScraper:
|
||||
def test_constants(self):
|
||||
from src.scraper.fbref_scraper import FBREF_BASE, STAT_CATEGORIES, KEEPER_CATEGORIES
|
||||
|
||||
assert "11" in FBREF_BASE # Serie A competition ID
|
||||
assert "stats" in STAT_CATEGORIES
|
||||
assert "keepers" in KEEPER_CATEGORIES
|
||||
assert "keepersadv" in KEEPER_CATEGORIES
|
||||
Reference in New Issue
Block a user