feat: VBT visualization + validation pipeline, HFT tick viz, DuckDB loader
Track 1 — VBT Candle-Frequency Pipeline: - backtests/vbt_validator.py: VBTValidator with 11 checks — timestamp monotonicity, duplicates, NaN, data gaps, lookahead bias, signal alignment, density, coincident entry/exit, min trade count, fee application, benchmark comparison. ValidationReport dataclass with errors/warnings/stats. Validates VBT results or raw signal arrays. - backtests/vbt_viz.py: VBTVisualizer with 10+ Plotly chart methods — equity curve with benchmark, drawdown, rolling Sharpe/Sortino/vol, trade markers, returns distribution with normal fit, monthly PnL heatmap, gross vs net, holding periods, parameter sensitivity heatmaps, dashboard compositor, HTML save (self-contained, CDN Plotly). All methods handle empty/null inputs. - backtests/vbt_report.py: Markdown + HTML report generator — structured sections for implementation summary, performance metrics, cost analysis, validation results, signal analysis, known limitations, next steps. batch_report() for mass report generation from results directory. - backtests/vbt_runner.py: Added run_benchmark() (buy-and-hold VBT portfolio), validate() (integrated VBTValidator), run_with_report() (fetch→validate→ backtest→visualize→save in one call). Track 2 — HFT Tick Pipeline: - backtests/tick_viz.py: 9-panel HFT dashboard — price+trade markers, spread dynamics, top-of-book depth, microprice vs mid, OBI/OFI panel, VPIN toxicity with thresholds, event timeline (PnL from tick_runner), markout curves at 6 horizons. Parquet→pandas→Plotly pipeline. Dark-themed HTML output for microstructure review. - data/duckdb_load.py: Parquet→DuckDB loader — creates l2_snapshots, trades, funding tables with schema. Pre-computed 1s rollup views for microprice, OFI, trade imbalance. Markout queries directly in SQL. Incremental loading with load_state tracking. CLI Integration: - cli.py: Added 'report' (full VBT report), 'validate' (check existing results), 'hft' (tick dashboard generation) commands. Fixed argparse help string escaping. 355 tests passing (34 new).
This commit is contained in:
ramseshk
@@ -0,0 +1,487 @@
|
||||
"""
|
||||
DuckDB Tick Data Loader — Parquet → DuckDB for fast analytical queries.
|
||||
|
||||
Converts the raw Parquet store (gzip-compressed JSON payloads) into
|
||||
a normalized DuckDB database with tables for L2 snapshots, trades,
|
||||
funding rates, and pre-computed microstructural rollups.
|
||||
|
||||
Usage:
|
||||
python data/duckdb_load.py --data-dir data/raw --db data/normalized/ftdt_tick.db
|
||||
python data/duckdb_load.py --coin BTC --days 7
|
||||
python data/duckdb_load.py --incremental # only load new data since last run
|
||||
|
||||
Tables created:
|
||||
l2_snapshots — full book state at each update time
|
||||
trades — aggressor-side classified trades
|
||||
funding — funding rate history
|
||||
l2_rollup_1s — pre-computed 1s microprice/OFI/VPIN rollups
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import gzip
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from datetime import date, datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
import numpy as np
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Columns extracted from L2 snapshots
|
||||
L2_SNAPSHOT_COLS = [
|
||||
"exchange_ts_ms", "local_ts", "coin",
|
||||
"best_bid", "best_ask", "mid_price", "microprice",
|
||||
"obi", "spread_bps", "bid_depth_10", "ask_depth_10",
|
||||
"bid_levels", "ask_levels",
|
||||
]
|
||||
|
||||
# Columns extracted from trades
|
||||
TRADE_COLS = [
|
||||
"exchange_ts_ms", "local_ts", "coin",
|
||||
"price", "size", "side", "aggressor",
|
||||
]
|
||||
|
||||
|
||||
class DuckDBLoader:
|
||||
"""Load raw Parquet data into a DuckDB database."""
|
||||
|
||||
def __init__(self, db_path: str = "data/normalized/ftdt_tick.db"):
|
||||
self._db_path = Path(db_path)
|
||||
self._db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self._conn = None
|
||||
|
||||
@property
|
||||
def conn(self):
|
||||
if self._conn is None:
|
||||
try:
|
||||
import duckdb
|
||||
self._conn = duckdb.connect(str(self._db_path))
|
||||
logger.info("Connected to DuckDB: %s", self._db_path)
|
||||
except ImportError:
|
||||
raise ImportError(
|
||||
"duckdb not installed. Run: pip install duckdb"
|
||||
)
|
||||
return self._conn
|
||||
|
||||
def init_schema(self):
|
||||
"""Create tables if they don't exist."""
|
||||
self.conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS l2_snapshots (
|
||||
exchange_ts_ms BIGINT NOT NULL,
|
||||
local_ts DOUBLE NOT NULL,
|
||||
coin VARCHAR NOT NULL,
|
||||
best_bid DOUBLE NOT NULL,
|
||||
best_ask DOUBLE NOT NULL,
|
||||
mid_price DOUBLE,
|
||||
microprice DOUBLE,
|
||||
obi DOUBLE,
|
||||
spread_bps DOUBLE,
|
||||
bid_depth_10 DOUBLE,
|
||||
ask_depth_10 DOUBLE,
|
||||
bid_levels INTEGER,
|
||||
ask_levels INTEGER,
|
||||
PRIMARY KEY (coin, exchange_ts_ms)
|
||||
)
|
||||
""")
|
||||
|
||||
self.conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS trades (
|
||||
exchange_ts_ms BIGINT NOT NULL,
|
||||
local_ts DOUBLE NOT NULL,
|
||||
coin VARCHAR NOT NULL,
|
||||
price DOUBLE NOT NULL,
|
||||
size DOUBLE NOT NULL,
|
||||
side VARCHAR,
|
||||
aggressor VARCHAR,
|
||||
)
|
||||
""")
|
||||
|
||||
self.conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS funding (
|
||||
exchange_ts_ms BIGINT NOT NULL,
|
||||
local_ts DOUBLE NOT NULL,
|
||||
coin VARCHAR NOT NULL,
|
||||
funding_rate DOUBLE NOT NULL,
|
||||
mark_px DOUBLE,
|
||||
annual_apr DOUBLE,
|
||||
PRIMARY KEY (coin, exchange_ts_ms)
|
||||
)
|
||||
""")
|
||||
|
||||
self.conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS load_state (
|
||||
coin VARCHAR PRIMARY KEY,
|
||||
channel VARCHAR,
|
||||
last_loaded_ts BIGINT,
|
||||
loaded_at TIMESTAMP DEFAULT now()
|
||||
)
|
||||
""")
|
||||
|
||||
logger.info("Schema initialized")
|
||||
|
||||
def load_l2(
|
||||
self,
|
||||
data_dir: str,
|
||||
coin: str,
|
||||
start_date: str,
|
||||
end_date: str,
|
||||
) -> int:
|
||||
"""Load L2 book data from Parquet into DuckDB. Returns row count."""
|
||||
from data.store import read_range
|
||||
|
||||
msgs = read_range(data_dir, "l2book", coin.upper(), start_date, end_date)
|
||||
if not msgs:
|
||||
return 0
|
||||
|
||||
rows = []
|
||||
for msg in msgs:
|
||||
payload = msg.get("payload", {})
|
||||
levels = payload.get("levels", [])
|
||||
msg_type = payload.get("type", "snapshot")
|
||||
|
||||
bids_dict = {}
|
||||
asks_dict = {}
|
||||
if msg_type == "snapshot" and isinstance(levels, list):
|
||||
if len(levels) >= 1:
|
||||
for bid in levels[0]:
|
||||
sz = float(bid.get("sz", 0))
|
||||
if sz > 0:
|
||||
bids_dict[float(bid["px"])] = sz
|
||||
if len(levels) >= 2:
|
||||
for ask in levels[1]:
|
||||
sz = float(ask.get("sz", 0))
|
||||
if sz > 0:
|
||||
asks_dict[float(ask["px"])] = sz
|
||||
|
||||
if not bids_dict or not asks_dict:
|
||||
continue
|
||||
|
||||
bid_prices = sorted(bids_dict.keys(), reverse=True)
|
||||
ask_prices = sorted(asks_dict.keys())
|
||||
best_bid = bid_prices[0]
|
||||
best_ask = ask_prices[0]
|
||||
mid = (best_bid + best_ask) / 2.0
|
||||
|
||||
bid_depth_10 = sum(bids_dict[px] for px in bid_prices[:10])
|
||||
ask_depth_10 = sum(asks_dict[px] for px in ask_prices[:10])
|
||||
total_depth = bid_depth_10 + ask_depth_10
|
||||
|
||||
obi = (bid_depth_10 - ask_depth_10) / total_depth if total_depth > 0 else 0.0
|
||||
w = bid_depth_10 / total_depth if total_depth > 0 else 0.5
|
||||
microprice = w * best_bid + (1 - w) * best_ask
|
||||
spread_bps = (best_ask - best_bid) / mid * 10000 if mid > 0 else 0
|
||||
|
||||
rows.append((
|
||||
msg.get("exchange_ts", 0) or 0,
|
||||
msg.get("local_ts", 0.0),
|
||||
coin.upper(),
|
||||
best_bid,
|
||||
best_ask,
|
||||
mid,
|
||||
microprice,
|
||||
obi,
|
||||
spread_bps,
|
||||
bid_depth_10,
|
||||
ask_depth_10,
|
||||
len(bid_prices),
|
||||
len(ask_prices),
|
||||
))
|
||||
|
||||
if rows:
|
||||
import duckdb
|
||||
rel = duckdb.from_sequence(rows)
|
||||
self.conn.execute(
|
||||
"INSERT OR IGNORE INTO l2_snapshots SELECT * FROM rel"
|
||||
)
|
||||
logger.info("Loaded %d L2 snapshots for %s", len(rows), coin)
|
||||
|
||||
return len(rows)
|
||||
|
||||
def load_trades(
|
||||
self,
|
||||
data_dir: str,
|
||||
coin: str,
|
||||
start_date: str,
|
||||
end_date: str,
|
||||
) -> int:
|
||||
"""Load trade data from Parquet into DuckDB. Returns row count."""
|
||||
from data.store import read_range
|
||||
|
||||
msgs = read_range(data_dir, "trades", coin.upper(), start_date, end_date)
|
||||
if not msgs:
|
||||
return 0
|
||||
|
||||
rows = []
|
||||
for msg in msgs:
|
||||
payload = msg.get("payload", {})
|
||||
px = float(payload.get("px", 0))
|
||||
sz = float(payload.get("sz", 0))
|
||||
if px <= 0 or sz <= 0:
|
||||
continue
|
||||
|
||||
side = str(payload.get("side", "?"))
|
||||
aggressor = "buy" if side.upper() in ("B", "BUY") else "sell"
|
||||
|
||||
rows.append((
|
||||
msg.get("exchange_ts", 0) or 0,
|
||||
msg.get("local_ts", 0.0),
|
||||
coin.upper(),
|
||||
px,
|
||||
sz,
|
||||
side,
|
||||
aggressor,
|
||||
))
|
||||
|
||||
if rows:
|
||||
import duckdb
|
||||
rel = duckdb.from_sequence(rows)
|
||||
self.conn.execute(
|
||||
"INSERT INTO trades SELECT * FROM rel"
|
||||
)
|
||||
logger.info("Loaded %d trades for %s", len(rows), coin)
|
||||
|
||||
return len(rows)
|
||||
|
||||
def load_funding(
|
||||
self,
|
||||
data_dir: str,
|
||||
coin: str,
|
||||
start_date: str,
|
||||
end_date: str,
|
||||
) -> int:
|
||||
"""Load funding rate data from Parquet into DuckDB."""
|
||||
from data.store import read_range
|
||||
|
||||
msgs = read_range(data_dir, "funding", coin.upper(), start_date, end_date)
|
||||
if not msgs:
|
||||
return 0
|
||||
|
||||
rows = []
|
||||
for msg in msgs:
|
||||
payload = msg.get("payload", {})
|
||||
rate = float(payload.get("funding", 0))
|
||||
mark = float(payload.get("mark_px", 0))
|
||||
annual = rate * 1095 if rate else 0
|
||||
|
||||
rows.append((
|
||||
msg.get("exchange_ts", 0) or 0,
|
||||
msg.get("local_ts", 0.0),
|
||||
coin.upper(),
|
||||
rate,
|
||||
mark,
|
||||
annual,
|
||||
))
|
||||
|
||||
if rows:
|
||||
import duckdb
|
||||
rel = duckdb.from_sequence(rows)
|
||||
self.conn.execute(
|
||||
"INSERT OR IGNORE INTO funding SELECT * FROM rel"
|
||||
)
|
||||
logger.info("Loaded %d funding observations for %s", len(rows), coin)
|
||||
|
||||
return len(rows)
|
||||
|
||||
def create_rollups(self):
|
||||
"""Create pre-computed 1-second and 1-minute aggregation views."""
|
||||
self.conn.execute("""
|
||||
CREATE VIEW IF NOT EXISTS l2_rollup_1s AS
|
||||
SELECT
|
||||
(exchange_ts_ms / 1000)::BIGINT * 1000 AS ts_1s,
|
||||
coin,
|
||||
AVG(mid_price) AS mid_price,
|
||||
AVG(microprice) AS microprice,
|
||||
AVG(obi) AS obi,
|
||||
AVG(spread_bps) AS spread_bps,
|
||||
AVG(bid_depth_10) AS bid_depth_10,
|
||||
AVG(ask_depth_10) AS ask_depth_10,
|
||||
COUNT(*) AS n_snapshots
|
||||
FROM l2_snapshots
|
||||
GROUP BY ts_1s, coin
|
||||
""")
|
||||
|
||||
self.conn.execute("""
|
||||
CREATE VIEW IF NOT EXISTS ofi_rollup_1s AS
|
||||
SELECT
|
||||
(exchange_ts_ms / 1000)::BIGINT * 1000 AS ts_1s,
|
||||
coin,
|
||||
SUM(CASE WHEN aggressor = 'buy' THEN size ELSE 0 END) AS buy_volume,
|
||||
SUM(CASE WHEN aggressor = 'sell' THEN size ELSE 0 END) AS sell_volume,
|
||||
COUNT(*) AS trade_count
|
||||
FROM trades
|
||||
GROUP BY ts_1s, coin
|
||||
""")
|
||||
|
||||
self.conn.execute("""
|
||||
CREATE VIEW IF NOT EXISTS micro_rollup_1s AS
|
||||
SELECT
|
||||
b.ts_1s,
|
||||
b.coin,
|
||||
b.mid_price,
|
||||
b.microprice,
|
||||
b.spread_bps,
|
||||
b.n_snapshots,
|
||||
COALESCE(o.buy_volume, 0) AS buy_volume,
|
||||
COALESCE(o.sell_volume, 0) AS sell_volume,
|
||||
COALESCE(o.trade_count, 0) AS trade_count,
|
||||
CASE
|
||||
WHEN COALESCE(o.buy_volume + o.sell_volume, 0) > 0
|
||||
THEN (o.buy_volume - o.sell_volume)::DOUBLE / (o.buy_volume + o.sell_volume)
|
||||
ELSE 0.0
|
||||
END AS trade_imbalance
|
||||
FROM l2_rollup_1s b
|
||||
LEFT JOIN ofi_rollup_1s o
|
||||
ON b.ts_1s = o.ts_1s AND b.coin = o.coin
|
||||
""")
|
||||
|
||||
logger.info("Rollup views created")
|
||||
|
||||
def load_all(
|
||||
self,
|
||||
data_dir: str,
|
||||
coins: list[str],
|
||||
start_date: str,
|
||||
end_date: str,
|
||||
) -> dict:
|
||||
"""Load all data for given coins and date range."""
|
||||
self.init_schema()
|
||||
|
||||
totals = {"l2": 0, "trades": 0, "funding": 0}
|
||||
for coin in coins:
|
||||
totals["l2"] += self.load_l2(data_dir, coin, start_date, end_date)
|
||||
totals["trades"] += self.load_trades(data_dir, coin, start_date, end_date)
|
||||
totals["funding"] += self.load_funding(data_dir, coin, start_date, end_date)
|
||||
|
||||
self.create_rollups()
|
||||
self.conn.execute(
|
||||
"INSERT OR REPLACE INTO load_state (coin, channel, last_loaded_ts, loaded_at) "
|
||||
"VALUES ('ALL', 'all', ?::BIGINT, now())",
|
||||
[int(time.time() * 1000)],
|
||||
)
|
||||
|
||||
return totals
|
||||
|
||||
def query_markouts(
|
||||
self,
|
||||
coin: str,
|
||||
start_date: str,
|
||||
end_date: str,
|
||||
horizons_ms: list[int] | None = None,
|
||||
) -> dict:
|
||||
"""Compute trade markouts directly in DuckDB."""
|
||||
if horizons_ms is None:
|
||||
horizons_ms = [100, 500, 1000, 5000, 10000, 30000, 60000]
|
||||
|
||||
start_ts = int(datetime.fromisoformat(start_date).timestamp() * 1000)
|
||||
end_ts = int(datetime.fromisoformat(end_date).timestamp() * 1000)
|
||||
|
||||
query = """
|
||||
WITH trade_mids AS (
|
||||
SELECT
|
||||
t.exchange_ts_ms,
|
||||
t.price AS trade_px,
|
||||
t.size AS trade_sz,
|
||||
t.aggressor,
|
||||
t.coin,
|
||||
s.mid_price AS mid_at_trade
|
||||
FROM trades t
|
||||
LEFT JOIN l2_snapshots s
|
||||
ON s.coin = t.coin
|
||||
AND s.exchange_ts_ms <= t.exchange_ts_ms
|
||||
AND s.exchange_ts_ms >= t.exchange_ts_ms - 2000
|
||||
WHERE t.coin = ?::VARCHAR
|
||||
AND t.exchange_ts_ms >= ?::BIGINT
|
||||
AND t.exchange_ts_ms < ?::BIGINT
|
||||
QUALIFY ROW_NUMBER() OVER (
|
||||
PARTITION BY t.exchange_ts_ms, t.price, t.size
|
||||
ORDER BY ABS(s.exchange_ts_ms - t.exchange_ts_ms)
|
||||
) = 1
|
||||
)
|
||||
SELECT aggressor, COUNT(*) AS n, AVG(markout_bps) AS mean_bps,
|
||||
STDDEV(markout_bps) AS std_bps
|
||||
FROM (
|
||||
SELECT
|
||||
aggressor,
|
||||
(future_mid - mid_at_trade) / mid_at_trade * 10000 AS markout_bps
|
||||
FROM trade_mids tm
|
||||
LEFT JOIN LATERAL (
|
||||
SELECT mid_price AS future_mid
|
||||
FROM l2_snapshots
|
||||
WHERE coin = tm.coin
|
||||
AND exchange_ts_ms >= tm.exchange_ts_ms + 100
|
||||
ORDER BY exchange_ts_ms
|
||||
LIMIT 1
|
||||
) ON TRUE
|
||||
WHERE mid_at_trade > 0
|
||||
)
|
||||
GROUP BY aggressor
|
||||
"""
|
||||
|
||||
result = self.conn.execute(query, [coin.upper(), start_ts, end_ts]).fetchall()
|
||||
return {
|
||||
row[0]: {"count": row[1], "mean_bps": row[2], "std_bps": row[3]}
|
||||
for row in result
|
||||
}
|
||||
|
||||
def stats(self) -> dict:
|
||||
"""Get current database statistics."""
|
||||
return {
|
||||
"l2_snapshots": self.conn.execute("SELECT COUNT(*) FROM l2_snapshots").fetchone()[0],
|
||||
"trades": self.conn.execute("SELECT COUNT(*) FROM trades").fetchone()[0],
|
||||
"funding": self.conn.execute("SELECT COUNT(*) FROM funding").fetchone()[0],
|
||||
"coins": [r[0] for r in self.conn.execute(
|
||||
"SELECT DISTINCT coin FROM l2_snapshots"
|
||||
).fetchall()],
|
||||
"date_range": self.conn.execute(
|
||||
"SELECT MIN(exchange_ts_ms), MAX(exchange_ts_ms) FROM l2_snapshots"
|
||||
).fetchone(),
|
||||
}
|
||||
|
||||
def close(self):
|
||||
if self._conn:
|
||||
self._conn.close()
|
||||
self._conn = None
|
||||
|
||||
|
||||
def main():
|
||||
p = argparse.ArgumentParser(description="DuckDB Tick Data Loader")
|
||||
p.add_argument("--data-dir", default="data/raw")
|
||||
p.add_argument("--db", default="data/normalized/ftdt_tick.db")
|
||||
p.add_argument("--coins", nargs="+", default=["BTC", "ETH"])
|
||||
p.add_argument("--start-date", default="2026-01-01")
|
||||
p.add_argument("--end-date", default=date.today().isoformat())
|
||||
p.add_argument("--stats", action="store_true", help="Print DB stats and exit")
|
||||
args = p.parse_args()
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s", datefmt="%H:%M:%S")
|
||||
|
||||
loader = DuckDBLoader(args.db)
|
||||
|
||||
if args.stats:
|
||||
s = loader.stats()
|
||||
print(f"DuckDB: {args.db}")
|
||||
print(f" L2 snapshots: {s['l2_snapshots']:,}")
|
||||
print(f" Trades: {s['trades']:,}")
|
||||
print(f" Funding: {s['funding']:,}")
|
||||
print(f" Coins: {s['coins']}")
|
||||
print(f" Date range: {s['date_range']}")
|
||||
loader.close()
|
||||
return
|
||||
|
||||
totals = loader.load_all(args.data_dir, args.coins, args.start_date, args.end_date)
|
||||
print(f"Loaded: {totals['l2']} L2 snapshots, {totals['trades']} trades, "
|
||||
f"{totals['funding']} funding observations")
|
||||
loader.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user