rams/ftdt-quant-lab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: HFT infrastructure — tick backtest runner, VPIN-gated A-S maker, WQI predictor, queue-aware fills

Browse Source 
- backtests/tick_runner.py: TickBacktestRunner replays stored Parquet L2/trade events
  through sim/engine.py with queue position modeling, producing PnL breakdowns,
  equity curves, VPIN curves, and QuantVerdict significance reports
- VPINGatedASMaker: VPIN-toxicity-gated A-S market maker with inventory skew
  and dynamic spread widening; blocks quoting when VPIN >= alarm threshold
- sim/engine.py: Added SimConfig.from_fee_tier() factory — constructs sim
  config from Hyperliquid fee tier (VIP + staking)
- sim/fills.py: Added QueueAwareFillModel — realistic queue-priority fill
  simulation replacing random fills in paper trading
- strategies/wqi_predictor.py: WQI z-score directional strategy with
  adverse selection gating, timeout exit, stop-loss, and take-profit
- cli.py: Added 'tick', 'markout' analysis, and 'discover' signal-discovery
  commands for end-to-end tick-level HFT research pipeline

301 tests passing (23 new).
This commit is contained in:
ramseshk
2026-08-11 10:43:51 +08:00
parent cdc7a01986
commit 50d63e1ecc
6 changed files with 1517 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cli.py
+290 -1
View File
@@ -105,6 +105,71 @@ def cmd_analyze(args):
regime = funding_regime(rates, window_hours=24, n_samples_per_hour=1)
print(f"Funding regime: {json.dumps(regime, indent=2, default=str)}")
elif args.channel == "markouts":
from microstructure.trades import compute_markouts, markout_summary
l2_messages = read_range(
args.data_dir,
channel="l2book",
coin=args.coin.upper(),
start_date=args.start_date,
end_date=args.end_date,
)
trade_messages = read_range(
args.data_dir,
channel="trades",
coin=args.coin.upper(),
start_date=args.start_date,
end_date=args.end_date,
)
trades = []
mids = []
times = []
book_state = {}
for msg in sorted(l2_messages + trade_messages, key=lambda m: m.get("exchange_ts", 0) or 0):
ch = msg.get("channel", "")
payload = msg.get("payload", {})
ts = msg.get("exchange_ts", 0) or 0
if ch == "l2book":
levels = payload.get("levels", [])
if isinstance(levels, list) and len(levels) >= 2:
bids = [float(l["px"]) for l in levels[0] if float(l.get("sz", 0)) > 0]
asks = [float(l["px"]) for l in levels[1] if float(l.get("sz", 0)) > 0]
if bids and asks:
book_state["mid"] = (bids[0] + asks[0]) / 2
elif ch == "trades":
px = float(payload.get("px", 0))
if px > 0:
trades.append(payload)
mid = book_state.get("mid", px)
mids.append(mid)
times.append(ts)
if not trades:
print("No trade data with L2 context available for markout analysis.")
return
print(f"Analyzing {len(trades)} trades with L2 context...")
markouts = compute_markouts(trades, mids, times)
summary = markout_summary(markouts)
print(f"\n{'─' * 70}")
print(f"{'Horizon':>10s} {'Buy Mean':>10s} {'Buy T-Stat':>10s} {'Buy N':>7s} "
f"{'Sell Mean':>10s} {'Sell T-Stat':>10s} {'Sell N':>7s}")
print(f"{'─' * 70}")
horizons = [100, 500, 1000, 5000, 10000, 30000, 60000]
for h in horizons:
b = summary.get("buy", {}).get(h, {})
s = summary.get("sell", {}).get(h, {})
print(f"{f'{h}ms':>10s} "
f"{b.get('mean_bps', 0):>10.2f} {b.get('t_stat', 0):>10.3f} {b.get('count', 0):>7d} "
f"{s.get('mean_bps', 0):>10.2f} {s.get('t_stat', 0):>10.3f} {s.get('count', 0):>7d}")
print(f"\nBuy markout: + = price rises after buy (good for seller, bad for buyer)")
print(f"Sell markout: + = price falls after sell (good for buyer, bad for seller)")
print(f"t-stat > 2.0 = statistically significant predictive power")
else:
print(f"Channel '{args.channel}' — raw dump:")
for msg in messages[:5]:
@@ -243,6 +308,197 @@ def cmd_backtest(args):
print(f"\n{len(result['trades'])} trades")
def cmd_discover(args):
"""Signal discovery — test microstructure signals against forward returns.
For each L2 snapshot, computes WQI, OBI, VPIN, depth imbalance, and microprice.
Then measures how well each signal predicts mid-price movement at multiple horizons.
"""
from data.store import read_range
from microstructure.book import (
mid_price, order_book_imbalance, depth_imbalance, microprice, spread_stats, depth_resiliency,
)
from microstructure.trades import classify_lee_ready
from microstructure.toxicity import compute_vpin
from strategies.queue_imbalance import QueueImbalance
import numpy as np
horizons = [int(h) for h in args.horizons.split(",")]
l2_msgs = read_range(
args.data_dir,
channel="l2book",
coin=args.coin.upper(),
start_date=args.start_date,
end_date=args.end_date,
)
trade_msgs = read_range(
args.data_dir,
channel="trades",
coin=args.coin.upper(),
start_date=args.start_date,
end_date=args.end_date,
)
if not l2_msgs or not trade_msgs:
print("No data available for signal discovery.")
return
print(f"Loading {len(l2_msgs)} L2 messages and {len(trade_msgs)} trades for {args.coin}...")
book_state = {"bids": {}, "asks": {}, "mid": 0.0, "ts": 0.0}
buy_vol = []
sell_vol = []
prev_bids = {}
prev_asks = {}
qi = QueueImbalance(depth_levels=10)
prev_mid = 0.0
events = []
for msg in sorted(l2_msgs + trade_msgs, key=lambda m: m.get("exchange_ts", 0) or 0):
ch = msg.get("channel", "")
payload = msg.get("payload", {})
ts = float(msg.get("exchange_ts", 0) or 0) / 1000.0
events.append((ts, ch, payload))
events.sort(key=lambda e: e[0])
signals = []
mids_series = []
times_series = []
for ts, ch, payload in events:
if ch == "l2book":
bids = {}
asks = {}
levels = payload.get("levels", [])
if isinstance(levels, list) and len(levels) >= 2:
bid_list = [(float(l["px"]), float(l["sz"])) for l in levels[0] if float(l.get("sz", 0)) > 0]
ask_list = [(float(l["px"]), float(l["sz"])) for l in levels[1] if float(l.get("sz", 0)) > 0]
bids = dict(bid_list)
asks = dict(ask_list)
if bids and asks:
book_state["bids"] = bids
book_state["asks"] = asks
mid = mid_price(bids, asks)
book_state["mid"] = mid
book_state["ts"] = ts
obi = order_book_imbalance(bids, asks)
di = depth_imbalance(bids, asks)
mp = microprice(bids, asks)
ss = spread_stats(bids, asks)
dr = depth_resiliency(bids, asks)
wqi = qi.compute_wqi(bid_list, ask_list)
vpin_val = 0.0
if buy_vol and sell_vol:
v = compute_vpin(buy_vol, sell_vol, n_buckets=50)
vpin_val = v.get("vpin_value", 0.0)
bid_depth = sum(sz for _, sz in bid_list[:10])
ask_depth = sum(sz for _, sz in ask_list[:10])
total_depth = bid_depth + ask_depth
signals.append({
"ts": ts,
"mid": mid,
"obi": obi,
"wqi": wqi,
"vpin": vpin_val,
"depth_imbalance": di,
"microprice_ratio": mp / mid if mid > 0 else 1.0,
"spread_bps": ss["spread_bps"],
"bid_depth": bid_depth,
"ask_depth": ask_depth,
"depth_total": total_depth,
"resiliency": dr.get("resiliency", 0),
})
mids_series.append(mid)
times_series.append(ts)
prev_bids = bid_list
prev_asks = ask_list
elif ch == "trades":
px = float(payload.get("px", 0))
sz = float(payload.get("sz", 0))
if px > 0 and sz > 0:
side = classify_lee_ready(px, book_state["mid"])
if side == "buy":
buy_vol.append(sz)
else:
sell_vol.append(sz)
n = len(signals)
if n < 50:
print("Too few L2 snapshots for signal discovery. Need more data.")
return
print(f"Signal discovery on {n} L2 snapshots across {horizons}ms horizons...")
signal_names = ["obi", "wqi", "vpin", "depth_imbalance", "spread_bps", "resiliency"]
horizons = sorted(horizons)
print(f"\n{'─' * 80}")
print(f"{'Signal':>18s} ", end="")
for h in horizons:
print(f" {'t-{h}ms':>10s}", end="")
print(f" {'r²':>8s}")
print(f"{'─' * 80}")
for sig_name in signal_names:
sig_vals = [s.get(sig_name, 0) for s in signals]
print(f"{sig_name:>18s} ", end="")
for horizon in horizons:
t_stats = []
for i in range(n - 1):
future_idx = i
for j in range(i + 1, min(n, len(times_series))):
if times_series[j] - times_series[i] >= horizon / 1000.0:
future_idx = j
break
if future_idx > i and mids_series[i] > 0:
forward_return = (mids_series[future_idx] - mids_series[i]) / mids_series[i] * 10000
if abs(forward_return) < 500:
zipped = list(zip(sig_vals, [forward_return] * len(sig_vals)))
t_tuple = zipped[i] if i < len(zipped) else None
t_stats = []
for i in range(n - 1):
future_idx = i
for j in range(i + 1, n):
if times_series[j] - times_series[i] >= horizon / 1000.0:
future_idx = j
break
if future_idx > i and mids_series[i] > 0:
forward_return = (mids_series[future_idx] - mids_series[i]) / mids_series[i] * 10000
signal_val = sig_vals[i]
if abs(forward_return) < 500 and abs(signal_val) < 100:
t_stats.append((signal_val, forward_return))
if len(t_stats) >= 10:
xs = np.array([t[0] for t in t_stats])
ys = np.array([t[1] for t in t_stats])
r = np.corrcoef(xs, ys)[0, 1] if len(xs) > 1 else 0
t_stat = r * np.sqrt(len(t_stats) - 2) / np.sqrt(1 - r * r) if abs(r) < 1 else 0
print(f" {t_stat:>10.3f}", end="")
else:
print(f" {'N/A':>10s}", end="")
if len(t_stats) >= 10:
xs = np.array([t[0] for t in t_stats])
ys = np.array([t[1] for t in t_stats])
r_sq = np.corrcoef(xs, ys)[0, 1] ** 2 if len(xs) > 1 else 0
print(f" {r_sq:>8.4f}")
else:
print()
print(f"\nPipeline ready. Run 'python -m cli tick' to backtest strategies on this data.")
def main():
import argparse
p = argparse.ArgumentParser(description="FTDT Quant Lab CLI")
@@ -259,7 +515,7 @@ def main():
# analyze
pa = sp.add_parser("analyze", help="Run microstructure analytics")
pa.add_argument("--data-dir", default="data/raw")
pa.add_argument("--channel", default="l2book", choices=["l2book", "trades", "funding", "mark", "open_interest", "liquidation"])
pa.add_argument("--channel", default="l2book", choices=["l2book", "trades", "funding", "mark", "open_interest", "liquidation", "markouts"])
pa.add_argument("--coin", default="BTC")
pa.add_argument("--start-date", default="2026-08-01")
pa.add_argument("--end-date", default="2026-08-07")
@@ -294,6 +550,34 @@ def main():
pb.add_argument("--interval", default="1h")
pb.add_argument("--limit", type=int, default=500)
# tick
pt = sp.add_parser("tick", help="Tick-level backtest (Parquet L2+trade replay)")
pt.add_argument("--coin", default="BTC")
pt.add_argument("--data-dir", default="data/raw")
pt.add_argument("--start-date", default="2026-08-01")
pt.add_argument("--end-date", default="2026-08-07")
pt.add_argument("--maker", default="as_mm", choices=["as_mm", "vpin_as_mm"])
pt.add_argument("--gamma", type=float, default=0.1)
pt.add_argument("--base-size", type=float, default=0.001)
pt.add_argument("--max-inventory", type=float, default=0.005)
pt.add_argument("--skew-factor", type=float, default=0.5)
pt.add_argument("--vpin-threshold", type=float, default=0.30)
pt.add_argument("--vpin-alarm", type=float, default=0.50)
pt.add_argument("--maker-fee", type=float, default=0.02, help="Maker fee in %")
pt.add_argument("--taker-fee", type=float, default=0.05, help="Taker fee in %")
pt.add_argument("--adverse-prob", type=float, default=0.15)
pt.add_argument("--cancel-after-ms", type=float, default=5000.0)
pt.add_argument("--quote-refresh-ms", type=float, default=2000.0)
pt.add_argument("--seed", type=int, default=42)
# discover
pd = sp.add_parser("discover", help="Signal discovery — test microstructure signals against forward returns")
pd.add_argument("--data-dir", default="data/raw")
pd.add_argument("--coin", default="BTC")
pd.add_argument("--start-date", default="2026-08-01")
pd.add_argument("--end-date", default="2026-08-07")
pd.add_argument("--horizons", default="100,500,1000,5000,10000", help="Comma-separated ms horizons")
args = p.parse_args()
import json as _json
@@ -309,6 +593,11 @@ def main():
cmd_run(args)
elif args.command == "backtest":
cmd_backtest(args)
elif args.command == "tick":
from backtests.tick_runner import cmd_tick_backtest
cmd_tick_backtest(args)
elif args.command == "discover":
cmd_discover(args)
if __name__ == "__main__":