feat: Phase 5 — integration layer (analytics pipeline, production node v2, CLI) + 12 tests
live/integrator.py (AnalyticsPipeline):
Real-time pipeline: data → microstructure → signals.
Accumulates book snapshots + trades, computes OBI, VPIN, microprice,
spread, depth, trade imbalance, HFT regime, and emits composite
signal with confidence and breakdown. Per-coin isolation.
live/node_v2.py (ProductionNode):
Rebuilt production node integrating ALL Phase 1-4 modules:
- REST data fetching (order book, mark prices, funding rates)
- AnalyticsPipeline per coin for real-time microstructure signals
- Treasury for position/capital/PnL/breaker management
- ToxicityFilter integration via HlMakerPool makers
- HlMakerPool for per-coin A-S quoting
- CrossVenueMonitor, FundingBasisMonitor, LiquidationRiskOverlay
- Paper trading with probabilistic fill simulation
- Dashboard metrics JSON output (equity, treasury, analytics, maker)
- Periodic status logging
cli.py (unified CLI):
Subcommands integrating all modules:
collect — Run Hyperliquid data collector to Parquet
analyze — Run microstructure analytics on stored data
simulate — Run market-making simulator on stored data
run — Start production trading node (paper or live)
backtest — Run VectorBT backtest
12 integration tests (all pass):
- AnalyticsPipeline: empty, book, trade, VPIN, emit, regime, isolation
- ProductionNode: creation, tick cycle (3 ticks), metrics JSON output
- CLI: import verification
Total test suite: 184 tests, all passing.
This commit is contained in:
ramseshk
@@ -0,0 +1,314 @@
|
||||
"""
|
||||
FTDT Quant Lab — unified CLI.
|
||||
|
||||
Subcommands:
|
||||
collect — Run data collector (streams to Parquet)
|
||||
analyze — Run analytics on stored data (Phase 2)
|
||||
simulate — Run market-making simulator on stored data (Phase 3)
|
||||
run — Start production trading node (Phase 4)
|
||||
backtest — Run VectorBT backtest (existing)
|
||||
|
||||
Usage:
|
||||
python -m cli collect --coins BTC,ETH --data-dir data/raw
|
||||
python -m cli analyze --data-dir data/raw --start 2026-08-01 --end 2026-08-07
|
||||
python -m cli simulate --data-dir data/raw --coin BTC --hours 24
|
||||
python -m cli run --coins BTC,ETH --mode paper
|
||||
python -m cli backtest --strategy pairs --interval 1h
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
|
||||
|
||||
def cmd_collect(args):
|
||||
"""Run the Hyperliquid data collector."""
|
||||
from data.collectors.hyperliquid import HyperliquidCollector
|
||||
from data.store import RawMessageStore
|
||||
|
||||
store = RawMessageStore(
|
||||
data_dir=args.data_dir,
|
||||
flush_interval_sec=args.flush_interval,
|
||||
)
|
||||
|
||||
collector = HyperliquidCollector(
|
||||
store=store,
|
||||
coins=args.coins,
|
||||
testnet=not args.mainnet,
|
||||
poll_interval_sec=args.poll_interval,
|
||||
)
|
||||
|
||||
asyncio.run(collector.run())
|
||||
|
||||
|
||||
def cmd_analyze(args):
|
||||
"""Run microstructure analytics on stored data."""
|
||||
from data.store import read_range
|
||||
|
||||
print(f"Reading {args.channel}/{args.coin} from {args.start_date} to {args.end_date}...")
|
||||
messages = read_range(
|
||||
args.data_dir,
|
||||
channel=args.channel,
|
||||
coin=args.coin.upper(),
|
||||
start_date=args.start_date,
|
||||
end_date=args.end_date,
|
||||
)
|
||||
print(f"Loaded {len(messages)} messages")
|
||||
|
||||
if args.channel == "l2book":
|
||||
from microstructure.book import batch_book_stats
|
||||
snapshots = []
|
||||
for msg in messages:
|
||||
payload = msg["payload"]
|
||||
levels = payload.get("levels", [])
|
||||
if levels and isinstance(levels, list) and len(levels) >= 2:
|
||||
bids = {}
|
||||
asks = {}
|
||||
for bid in levels[0]:
|
||||
if float(bid.get("sz", 0)) > 0:
|
||||
bids[float(bid["px"])] = float(bid["sz"])
|
||||
for ask in levels[1]:
|
||||
if float(ask.get("sz", 0)) > 0:
|
||||
asks[float(ask["px"])] = float(ask["sz"])
|
||||
snapshots.append({"bids": bids, "asks": asks})
|
||||
|
||||
stats = batch_book_stats(snapshots)
|
||||
print(json.dumps(stats, indent=2, default=str))
|
||||
|
||||
elif args.channel == "trades":
|
||||
from microstructure.trades import classify_bulk_lee_ready, trade_arrival_rate, trade_volume_profile
|
||||
|
||||
trades = [msg["payload"] for msg in messages]
|
||||
mids = [float(msg["payload"].get("px", 0)) for msg in messages]
|
||||
times = [msg["exchange_ts"] for msg in messages]
|
||||
sides = classify_bulk_lee_ready(trades, mids)
|
||||
buys = sum(1 for s in sides if s == "buy")
|
||||
sells = sum(1 for s in sides if s == "sell")
|
||||
arrival = trade_arrival_rate(times)
|
||||
vol = trade_volume_profile(trades)
|
||||
|
||||
print(f"Trades: {len(trades)} total ({buys} buy, {sells} sell)")
|
||||
print(f"Arrival rate: {json.dumps(arrival, indent=2, default=str)}")
|
||||
print(f"Volume profile: {json.dumps(vol, indent=2, default=str)}")
|
||||
|
||||
elif args.channel == "funding":
|
||||
from microstructure.funding import funding_regime, basis_spread
|
||||
|
||||
rates = [float(msg["payload"].get("funding", 0)) for msg in messages]
|
||||
marks = [float(msg["payload"].get("mark_px", 0)) for msg in messages]
|
||||
regime = funding_regime(rates, window_hours=24, n_samples_per_hour=1)
|
||||
print(f"Funding regime: {json.dumps(regime, indent=2, default=str)}")
|
||||
|
||||
else:
|
||||
print(f"Channel '{args.channel}' — raw dump:")
|
||||
for msg in messages[:5]:
|
||||
print(json.dumps(msg, indent=2, default=str))
|
||||
if len(messages) > 5:
|
||||
print(f"... and {len(messages) - 5} more")
|
||||
|
||||
|
||||
def cmd_simulate(args):
|
||||
"""Run market-making simulator on stored data with L2 events."""
|
||||
from data.store import read_range
|
||||
from sim.engine import SimulationEngine, SimConfig
|
||||
from sim.maker import MakerConfig
|
||||
|
||||
print(f"Loading L2 book data for {args.coin} from {args.start_date} to {args.end_date}...")
|
||||
l2_messages = read_range(
|
||||
args.data_dir,
|
||||
channel="l2book",
|
||||
coin=args.coin.upper(),
|
||||
start_date=args.start_date,
|
||||
end_date=args.end_date,
|
||||
)
|
||||
print(f"Loaded {len(l2_messages)} L2 updates")
|
||||
|
||||
trade_messages = read_range(
|
||||
args.data_dir,
|
||||
channel="trades",
|
||||
coin=args.coin.upper(),
|
||||
start_date=args.start_date,
|
||||
end_date=args.end_date,
|
||||
)
|
||||
print(f"Loaded {len(trade_messages)} trades")
|
||||
|
||||
events = []
|
||||
for msg in l2_messages:
|
||||
payload = msg["payload"]
|
||||
levels = payload.get("levels", [])
|
||||
bids = {}
|
||||
asks = {}
|
||||
if levels and isinstance(levels, list) and len(levels) >= 2:
|
||||
for bid in levels[0]:
|
||||
if float(bid.get("sz", 0)) > 0:
|
||||
bids[float(bid["px"])] = float(bid["sz"])
|
||||
for ask in levels[1]:
|
||||
if float(ask.get("sz", 0)) > 0:
|
||||
asks[float(ask["px"])] = float(ask["sz"])
|
||||
events.append({
|
||||
"type": "l2",
|
||||
"data": {"bids": bids, "asks": asks},
|
||||
"time": msg["local_ts"],
|
||||
"coin": args.coin.upper(),
|
||||
})
|
||||
|
||||
for msg in trade_messages:
|
||||
payload = msg["payload"]
|
||||
events.append({
|
||||
"type": "trade",
|
||||
"data": payload,
|
||||
"time": msg["local_ts"],
|
||||
"coin": args.coin.upper(),
|
||||
})
|
||||
|
||||
events.sort(key=lambda e: e["time"])
|
||||
print(f"Total events: {len(events)}")
|
||||
|
||||
config = SimConfig(
|
||||
maker=MakerConfig(
|
||||
base_size=args.base_size,
|
||||
max_inventory=args.max_inventory,
|
||||
gamma=args.gamma,
|
||||
),
|
||||
max_inventory=args.max_inventory,
|
||||
cancel_after_ms=args.cancel_after_ms,
|
||||
quote_refresh_ms=args.quote_refresh_ms,
|
||||
seed=args.seed,
|
||||
)
|
||||
|
||||
engine = SimulationEngine(config=config, seed=args.seed)
|
||||
engine.run(events)
|
||||
|
||||
stats = engine.stats()
|
||||
breakdown = engine.breakdown()
|
||||
|
||||
print("\n=== Simulation Results ===")
|
||||
print(f"Duration: {events[-1]['time'] - events[0]['time']:.0f}s" if events else "0s")
|
||||
print(f"Trades: {stats.total_trades} ({stats.bid_fills} bid, {stats.ask_fills} ask)")
|
||||
print(f"Toxic fills: {stats.toxic_fills} ({stats.adverse_rate:.1%})")
|
||||
print(f"Cancels: {stats.cancels}")
|
||||
print(f"Avg spread: {stats.avg_spread_bps} bps")
|
||||
print(f"Max inventory: {stats.max_inventory}")
|
||||
print(f"Max drawdown: {stats.max_drawdown}%")
|
||||
print(f"Sharpe: {stats.sharpe} Sortino: {stats.sortino}")
|
||||
print(f"Uptime: {stats.uptime_pct}%")
|
||||
print(f"\nPnL Breakdown:")
|
||||
print(f" Spread capture: ${breakdown.spread_capture:.4f}")
|
||||
print(f" Inventory PnL: ${breakdown.inventory_pnl:.4f}")
|
||||
print(f" Maker fees: ${breakdown.maker_fees:.4f}")
|
||||
print(f" Taker fees: ${breakdown.taker_fees:.4f}")
|
||||
print(f" Funding PnL: ${breakdown.funding_pnl:.4f}")
|
||||
print(f" Adverse selection: ${breakdown.adverse_selection_cost:.4f}")
|
||||
print(f" ─────────────────────────────")
|
||||
print(f" Gross PnL: ${breakdown.gross_pnl:.4f}")
|
||||
print(f" Net PnL: ${breakdown.net_pnl:.4f}")
|
||||
|
||||
|
||||
def cmd_run(args):
|
||||
"""Start the production trading node."""
|
||||
import asyncio
|
||||
from live.node_v2 import ProductionNode
|
||||
|
||||
node = ProductionNode(
|
||||
coins=args.coins,
|
||||
testnet=not args.mainnet,
|
||||
mode=args.mode,
|
||||
max_position_per_coin=args.max_position,
|
||||
base_quote_size=args.base_size,
|
||||
initial_equity=args.equity,
|
||||
tick_interval_sec=args.tick_interval,
|
||||
metrics_file=args.metrics_file,
|
||||
)
|
||||
|
||||
asyncio.run(node.run())
|
||||
|
||||
|
||||
def cmd_backtest(args):
|
||||
"""Run a VBT backtest (existing functionality)."""
|
||||
from backtests.vbt_runner import VBTBacktestRunner
|
||||
runner = VBTBacktestRunner()
|
||||
result = runner.run_strategy(strategy=args.strategy, interval=args.interval, limit=args.limit)
|
||||
import json as _json
|
||||
print(_json.dumps({k: v for k, v in (result or {}).items()
|
||||
if k not in ("trades", "equity_curve")}, indent=2, default=str))
|
||||
if result and result.get("trades"):
|
||||
print(f"\n{len(result['trades'])} trades")
|
||||
|
||||
|
||||
def main():
|
||||
import argparse
|
||||
p = argparse.ArgumentParser(description="FTDT Quant Lab CLI")
|
||||
sp = p.add_subparsers(dest="command", required=True)
|
||||
|
||||
# collect
|
||||
pc = sp.add_parser("collect", help="Run data collector")
|
||||
pc.add_argument("--coins", nargs="+", default=["BTC", "ETH"])
|
||||
pc.add_argument("--mainnet", action="store_true")
|
||||
pc.add_argument("--data-dir", default="data/raw")
|
||||
pc.add_argument("--poll-interval", type=float, default=60.0)
|
||||
pc.add_argument("--flush-interval", type=float, default=5.0)
|
||||
|
||||
# analyze
|
||||
pa = sp.add_parser("analyze", help="Run microstructure analytics")
|
||||
pa.add_argument("--data-dir", default="data/raw")
|
||||
pa.add_argument("--channel", default="l2book", choices=["l2book", "trades", "funding", "mark", "open_interest", "liquidation"])
|
||||
pa.add_argument("--coin", default="BTC")
|
||||
pa.add_argument("--start-date", default="2026-08-01")
|
||||
pa.add_argument("--end-date", default="2026-08-07")
|
||||
|
||||
# simulate
|
||||
ps = sp.add_parser("simulate", help="Run market-making simulator")
|
||||
ps.add_argument("--data-dir", default="data/raw")
|
||||
ps.add_argument("--coin", default="BTC")
|
||||
ps.add_argument("--start-date", default="2026-08-01")
|
||||
ps.add_argument("--end-date", default="2026-08-07")
|
||||
ps.add_argument("--gamma", type=float, default=0.1)
|
||||
ps.add_argument("--base-size", type=float, default=0.001)
|
||||
ps.add_argument("--max-inventory", type=float, default=0.005)
|
||||
ps.add_argument("--cancel-after-ms", type=float, default=5000.0)
|
||||
ps.add_argument("--quote-refresh-ms", type=float, default=2000.0)
|
||||
ps.add_argument("--seed", type=int, default=42)
|
||||
|
||||
# run
|
||||
pr = sp.add_parser("run", help="Start production node")
|
||||
pr.add_argument("--coins", nargs="+", default=["BTC", "ETH"])
|
||||
pr.add_argument("--mainnet", action="store_true")
|
||||
pr.add_argument("--mode", default="paper", choices=["paper", "live"])
|
||||
pr.add_argument("--max-position", type=float, default=0.003)
|
||||
pr.add_argument("--base-size", type=float, default=0.0002)
|
||||
pr.add_argument("--equity", type=float, default=10000.0)
|
||||
pr.add_argument("--tick-interval", type=float, default=2.0)
|
||||
pr.add_argument("--metrics-file", default="/tmp/ftdt-metrics-v2.json")
|
||||
|
||||
# backtest
|
||||
pb = sp.add_parser("backtest", help="Run VBT backtest")
|
||||
pb.add_argument("--strategy", default="pairs")
|
||||
pb.add_argument("--interval", default="1h")
|
||||
pb.add_argument("--limit", type=int, default=500)
|
||||
|
||||
args = p.parse_args()
|
||||
|
||||
import json as _json
|
||||
import json
|
||||
|
||||
if args.command == "collect":
|
||||
cmd_collect(args)
|
||||
elif args.command == "analyze":
|
||||
cmd_analyze(args)
|
||||
elif args.command == "simulate":
|
||||
cmd_simulate(args)
|
||||
elif args.command == "run":
|
||||
cmd_run(args)
|
||||
elif args.command == "backtest":
|
||||
cmd_backtest(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s", datefmt="%H:%M:%S")
|
||||
main()
|
||||
Reference in New Issue
Block a user