""" Walk-forward validation framework. Splits market data into sequential IS/OOS windows, optimizes strategy parameters on in-sample data, and tests on out-of-sample data. This is the minimum bar for any strategy before live deployment. Computes: - OOS Sharpe per window - Walk-forward consistency (% positive OOS windows) - Performance decay (IS → OOS degradation) - Concatenated OOS equity curve """ from __future__ import annotations import logging from dataclasses import dataclass, field from datetime import datetime, timezone from typing import Optional import numpy as np from quant.significance import QuantVerdict logger = logging.getLogger(__name__) @dataclass class WFWindow: """Single walk-forward window result.""" window_idx: int is_start: str is_end: str oos_start: str oos_end: str is_sharpe: float oos_sharpe: float is_return_pct: float oos_return_pct: float oos_trades: int best_params: dict = field(default_factory=dict) @dataclass class WFReport: """Complete walk-forward analysis report.""" strategy: str interval: str n_windows: int windows: list[WFWindow] = field(default_factory=list) oos_equity_curve: list[dict] = field(default_factory=list) @property def oos_sharpe(self) -> float: if not self.oos_equity_curve: return 0.0 vals = [p["v"] for p in self.oos_equity_curve if p.get("v")] if len(vals) < 2: return 0.0 rets = [(vals[i] - vals[i - 1]) / vals[i - 1] for i in range(1, len(vals)) if vals[i - 1] > 0] if not rets: return 0.0 mean = sum(rets) / len(rets) std = (sum((r - mean) ** 2 for r in rets) / max(len(rets) - 1, 1)) ** 0.5 return mean / std * np.sqrt(365 * 24) if std > 0 else 0.0 @property def consistency(self) -> float: """Fraction of windows with positive OOS Sharpe.""" if not self.windows: return 0.0 positive = sum(1 for w in self.windows if w.oos_sharpe > 0) return positive / len(self.windows) @property def avg_oos_sharpe(self) -> float: if not self.windows: return 0.0 return sum(w.oos_sharpe for w in self.windows) / len(self.windows) @property def performance_decay(self) -> float: """IS Sharpe → OOS decay ratio. <1 = decay, >1 = improvement (rare).""" avg_is = sum(w.is_sharpe for w in self.windows) / max(len(self.windows), 1) avg_oos = self.avg_oos_sharpe return avg_oos / avg_is if avg_is != 0 else 0.0 @property def total_oos_trades(self) -> int: return sum(w.oos_trades for w in self.windows) def significance_report(self, n_trials: int = 639) -> dict: return QuantVerdict( observed_sharpe=self.oos_sharpe, wf_consistency=self.consistency, n_trials=n_trials, n_periods=max(self.total_oos_trades, 1), positive_regimes=0, ).evaluate() def summary(self) -> dict: return { "strategy": self.strategy, "interval": self.interval, "n_windows": self.n_windows, "consistency": round(self.consistency, 3), "oos_sharpe": round(self.oos_sharpe, 3), "avg_oos_sharpe": round(self.avg_oos_sharpe, 3), "performance_decay": round(self.performance_decay, 3), "total_oos_trades": self.total_oos_trades, "verdict": self.significance_report()["verdict"], } class WalkForwardRunner: """Run walk-forward validation using VBT runner on historical data.""" def __init__( self, n_windows: int = 5, bar_limits: list[int] | None = None, fee_tier: int = 0, staking_tier: str = "none", ): self._n_windows = n_windows self._bar_limits = bar_limits or [100, 200, 500, 1000, 2000] self._fee_tier = fee_tier self._staking_tier = staking_tier def run( self, strategy: str, interval: str = "1h", coin: str = "BTC", testnet: bool = False, ) -> WFReport: """Execute walk-forward validation on real Hyperliquid data. Uses HyperliquidDataProvider to fetch candle data, then splits into sequential IS/OOS windows. For each window: 1. Optimize bar limit on IS data (pick best by Sharpe) 2. Test the optimal bar limit on OOS data 3. Record IS/OOS Sharpe, returns, trades """ from framework.data import HyperliquidDataProvider from backtests.vbt_runner import VBTBacktestRunner provider = HyperliquidDataProvider(testnet=testnet) # Fetch maximum data needed max_bars = max(self._bar_limits) * (self._n_windows + 1) df = provider.fetch_candles(coin, interval=interval, limit=max_bars) if df.empty or len(df) < 100: return WFReport(strategy=strategy, interval=interval, n_windows=self._n_windows) total_bars = len(df) window_size = total_bars // (self._n_windows + 1) if window_size < 50: return WFReport(strategy=strategy, interval=interval, n_windows=self._n_windows) report = WFReport(strategy=strategy, interval=interval, n_windows=self._n_windows) cumulative_oos_equity = 10000.0 report.oos_equity_curve.append({"t": 0, "v": cumulative_oos_equity}) for w in range(self._n_windows): is_start_idx = w * window_size is_end_idx = is_start_idx + window_size oos_start_idx = is_end_idx oos_end_idx = min(oos_start_idx + window_size, total_bars) is_start_ts = str(df.index[is_start_idx])[:10] is_end_ts = str(df.index[min(is_end_idx - 1, total_bars - 1)])[:10] oos_start_ts = str(df.index[min(oos_start_idx, total_bars - 1)])[:10] oos_end_ts = str(df.index[min(oos_end_idx - 1, total_bars - 1)])[:10] # Convert dates to ms for HL API is_start_ms = int(df.index[is_start_idx].timestamp() * 1000) is_end_ms = int(df.index[min(is_end_idx - 1, total_bars - 1)].timestamp() * 1000) oos_start_ms = int(df.index[min(oos_start_idx, total_bars - 1)].timestamp() * 1000) oos_end_ms = int(df.index[min(oos_end_idx - 1, total_bars - 1)].timestamp() * 1000) # IN-SAMPLE: optimize bar limit best_limit = self._bar_limits[0] best_is_sharpe = -999.0 best_is_return = 0.0 for limit in self._bar_limits: is_bars = min(limit, window_size) try: runner = VBTBacktestRunner( vip_tier=self._fee_tier, staking_tier=self._staking_tier ) result = runner.run_strategy( strategy=strategy, interval=interval, testnet=testnet, limit=is_bars, start_ms=is_start_ms, end_ms=is_end_ms, ) if result and result.get("sharpe", -999) > best_is_sharpe: best_is_sharpe = result.get("sharpe", -999) best_is_return = result.get("total_return_pct", 0) best_limit = limit except Exception: pass # OUT-OF-SAMPLE: test the best bar limit if best_is_sharpe <= -998: continue oos_bars = min(best_limit, oos_end_idx - oos_start_idx) try: runner = VBTBacktestRunner( vip_tier=self._fee_tier, staking_tier=self._staking_tier ) oos_result = runner.run_strategy( strategy=strategy, interval=interval, testnet=testnet, limit=oos_bars, start_ms=oos_start_ms, end_ms=oos_end_ms, ) if oos_result: oos_sharpe = oos_result.get("sharpe", 0) oos_return = oos_result.get("total_return_pct", 0) oos_trades = len(oos_result.get("trades", [])) cumulative_oos_equity += cumulative_oos_equity * oos_return / 100.0 report.oos_equity_curve.append({ "t": w + 1, "v": round(cumulative_oos_equity, 2), }) report.windows.append(WFWindow( window_idx=w, is_start=is_start_ts, is_end=is_end_ts, oos_start=oos_start_ts, oos_end=oos_end_ts, is_sharpe=round(best_is_sharpe, 3), oos_sharpe=round(oos_sharpe, 3), is_return_pct=round(best_is_return, 2), oos_return_pct=round(oos_return, 2), oos_trades=oos_trades, best_params={"limit": best_limit}, )) except Exception: pass return report # ── Quick validation ──────────────────────────────────────── def quick_validate(strategy: str, interval: str = "1h", **kwargs) -> dict: """Run walk-forward and return significance report in one call.""" wfr = WalkForwardRunner(**kwargs) report = wfr.run(strategy=strategy, interval=interval) return report.summary()