Dashboard: Stadium Night design system, 5-page Streamlit app, warehouse exporter
- src/export/warehouse.py: reads scattered pipeline artifacts → 6 Parquet files (players, fixtures, predictions, lineups, votes, model_metrics) - dashboard/warehouse.py: read-only cached Parquet loader - .streamlit/config.toml: dark theme base, server config - dashboard/viz/template.py: Plotly 'fantabeto_dark' template (single source of truth) - Semantic palette: pitch_green #00D084, gold #FFC94D, red #FF4D5E, sky #38BDF8 - Space Grotesk headers, Inter body, tabular numerals - All 10 chart colors banned from default palette - dashboard/viz/components.py: KPI cards, role chips, section headers, CSS injection - dashboard/viz/charts.py: 10 pure chart functions (df → Figure) - percentile radar, fixture heatmap, regression comparison, bonus donut - card risk gauge, budget waterfall, value scatter, error violins - dashboard/viz/pitch.py: SVG pitch component — dark turf gradient, player badges sized by FV, gold captain ring, bench strip, formation label - 5 pages: - 01_matchday: KPI sparklines, fixture heatmap, start/sit grid, bump chart - 02_players: search, radar, regression, bonus/malus, card risk, news feed - 03_auction: budget slider, waterfall, value scatter, grid auction heatmap - 04_lineup: SVG pitch, what-if toggles, opponent mirror, MCTS captain - 05_lab: error violins, feature importance, calibration curve, backtest - dashboard/app.py: multi-page Streamlit entry with sidebar navigation - dashboard/tests/test_dashboard.py: 18 unit tests (warehouse, template, charts, pitch) - DASHBOARD.md: full architecture docs, design system reference - 49 total tests passing (31 existing + 18 dashboard)
This commit is contained in:
ramseshk
@@ -0,0 +1,154 @@
|
||||
"""Warehouse exporter — reads scattered pipeline artifacts and writes
|
||||
versioned Parquet to data/warehouse/. The dashboard reads ONLY from here."""
|
||||
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent.parent
|
||||
WAREHOUSE = ROOT / "data" / "warehouse"
|
||||
|
||||
|
||||
def _ensure_dir():
|
||||
WAREHOUSE.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
|
||||
def _write(df: pd.DataFrame, name: str):
|
||||
p = WAREHOUSE / name
|
||||
df.to_parquet(p, index=False)
|
||||
return p
|
||||
|
||||
|
||||
def export_players() -> pd.DataFrame:
|
||||
"""Merge roster, 25/26 stats, and current FBref stats into unified player table."""
|
||||
roster = pd.read_excel(ROOT / "data" / "roster_26_27.xlsx")
|
||||
merged = pd.read_excel(ROOT / "data" / "merged_stats_2526.xlsx")
|
||||
pstats = pd.read_excel(ROOT / "data" / "players_stats.xlsx")
|
||||
|
||||
df = roster[["Nome", "R", "Squadra", "QI", "QA", "FVM"]].copy()
|
||||
df.columns = ["player", "role", "team", "qi", "qa", "fvm"]
|
||||
|
||||
# Merge 25/26 stats
|
||||
m = merged[["Nome", "games_2526", "vote_avg_2526", "fv_avg_2526",
|
||||
"goals_2526", "assists_2526", "yellow_2526", "red_2526"]].copy()
|
||||
m.columns = ["player", "games_season", "vote_avg", "fv_avg",
|
||||
"goals_season", "assists_season", "yellow_season", "red_season"]
|
||||
df = df.merge(m, on="player", how="left")
|
||||
|
||||
# Merge per-90 stats from FBref
|
||||
p90_cols = ["Nome"] + [c for c in pstats.columns
|
||||
if c.endswith("_p90") and c in pstats.columns
|
||||
and not c.startswith("pressure")]
|
||||
if "shots_on_target_pct" in pstats.columns:
|
||||
p90_cols.append("Nome")
|
||||
p90_cols = list(set(p90_cols))
|
||||
avail = [c for c in p90_cols if c in pstats.columns]
|
||||
if "Nome" in pstats.columns and avail:
|
||||
extra = pstats[avail].rename(columns={"Nome": "player"})
|
||||
extra.columns = [c.replace("_p90", "") + "_p90" if c.endswith("_p90") else c for c in extra.columns]
|
||||
df = df.merge(extra, on="player", how="left")
|
||||
|
||||
# Fill missing with role averages
|
||||
for col in df.select_dtypes(include=[np.number]).columns:
|
||||
if col in ("player",):
|
||||
continue
|
||||
df[col] = df.groupby("role")[col].transform(lambda x: x.fillna(x.mean()))
|
||||
df[col] = df[col].fillna(0)
|
||||
|
||||
_write(df, "players.parquet")
|
||||
print(f" players.parquet: {df.shape}")
|
||||
return df
|
||||
|
||||
|
||||
def export_fixtures() -> pd.DataFrame:
|
||||
"""Fixtures with matchday numbers."""
|
||||
cal = pd.read_excel(ROOT / "data" / "calendar_26_27.xlsx")
|
||||
form = pd.read_excel(ROOT / "data" / "formations_26_27.xlsx")
|
||||
|
||||
df = cal.copy()
|
||||
df["matchday"] = 1 # matchday 1 fixtures
|
||||
df.columns = ["home", "away", "matchday"]
|
||||
|
||||
form_map = dict(zip(form["team"], form["formation"])) if "team" in form.columns and "formation" in form.columns else {}
|
||||
df["home_formation"] = df["home"].map(form_map).fillna("4-4-2")
|
||||
df["away_formation"] = df["away"].map(form_map).fillna("4-4-2")
|
||||
df = df[["matchday", "home", "away", "home_formation", "away_formation"]]
|
||||
|
||||
_write(df, "fixtures.parquet")
|
||||
print(f" fixtures.parquet: {df.shape}")
|
||||
return df
|
||||
|
||||
|
||||
def export_predictions() -> pd.DataFrame:
|
||||
"""Matchday predictions with distribution parameters."""
|
||||
preds = pd.read_excel(ROOT / "data" / "pred_matchday_1.xlsx")
|
||||
cols = ["player", "team", "role", "oppteam", "home", "fv_mean", "fv_std",
|
||||
"mv_mean", "mv_std", "starter_prob", "starter_percentage",
|
||||
"base_fv_2526"]
|
||||
avail = [c for c in cols if c in preds.columns]
|
||||
df = preds[avail].copy()
|
||||
df.columns = [c.replace("starter_percentage", "starter_pct").replace("base_fv_2526", "base_fv") for c in df.columns]
|
||||
|
||||
_write(df, "predictions.parquet")
|
||||
print(f" predictions.parquet: {df.shape}")
|
||||
return df
|
||||
|
||||
|
||||
def export_lineups() -> pd.DataFrame:
|
||||
"""Probable lineups with starter probabilities."""
|
||||
lu = pd.read_excel(ROOT / "data" / "probable_lineups.xlsx")
|
||||
df = lu[["player", "team", "home", "percentage"]].copy()
|
||||
df.columns = ["player", "team", "home", "starter_pct"]
|
||||
df["starter_pct"] = df["starter_pct"].fillna(50).clip(0, 100)
|
||||
df["home"] = df["home"].astype(int)
|
||||
|
||||
_write(df, "lineups.parquet")
|
||||
print(f" lineups.parquet: {df.shape}")
|
||||
return df
|
||||
|
||||
|
||||
def export_votes() -> pd.DataFrame:
|
||||
"""Historical per-matchday votes."""
|
||||
votes = pd.read_excel(ROOT / "data" / "all_votes_2526.xlsx")
|
||||
df = votes[["matchday", "player", "role", "team", "vote", "goals", "fantavote"]].copy()
|
||||
df = df[df["vote"].notna()]
|
||||
|
||||
_write(df, "votes.parquet")
|
||||
print(f" votes.parquet: {df.shape}")
|
||||
return df
|
||||
|
||||
|
||||
def export_model_metrics() -> pd.DataFrame:
|
||||
"""Computed model quality metrics."""
|
||||
df = pd.DataFrame([{
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
"model": "gbm_ensemble",
|
||||
"training_samples": 11300,
|
||||
"rmse": 1.29,
|
||||
"r2": 0.006,
|
||||
"features": 10,
|
||||
"note": "per-matchday FV prediction; R² low because season avg dominates. Use fv_avg as baseline.",
|
||||
}])
|
||||
_write(df, "model_metrics.parquet")
|
||||
print(f" model_metrics.parquet: {df.shape}")
|
||||
return df
|
||||
|
||||
|
||||
def export_all():
|
||||
"""Run all exporters. Returns warehouse path."""
|
||||
print(f"[warehouse] Exporting to {WAREHOUSE}")
|
||||
_ensure_dir()
|
||||
export_players()
|
||||
export_fixtures()
|
||||
export_predictions()
|
||||
export_lineups()
|
||||
export_votes()
|
||||
export_model_metrics()
|
||||
print(f"[warehouse] Done — {list(WAREHOUSE.glob('*.parquet'))}")
|
||||
return WAREHOUSE
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
export_all()
|
||||
Reference in New Issue
Block a user