feat: Phase 1 — real-time & historical data system
New data/ module with: - data/store.py: Parquet-based raw message storage with background writer thread. Messages partitioned by channel/coin/date. Thread-safe queue. Supports pyarrow Parquet with zstd compression. Includes read_range() helper for replay. - data/collectors/hyperliquid.py: HL WebSocket + REST collector - WebSocket: l2Book (full book reconstruction), trades, allMids (mark prices) - REST pollers: funding rates, predicted funding, open interest, liquidations - Per-coin OrderBook class with snapshot/update reconstruction - Sequence gap detection with per-coin re-snapshot on gap - Latency tracking (exchange transport, signal, order, roundtrip) - Periodic stats reporter (book stats + latency summary every 60s) - CLI entrypoint: python -m data.collectors.hyperliquid --coins BTC ETH - data/normalizer.py: Timestamp normalization (ms, s, ISO strings from HL/Binance/Bybit/OKX/Coinbase/Deribit) + SequenceTracker with gap detection - data/latency.py: Rolling-window latency metrics (p50/p90/p95/p99) for transport, signal computation, order submission, and roundtrip - Added pyarrow + aiohttp to requirements.txt
This commit is contained in:
@@ -0,0 +1,124 @@
|
||||
"""
|
||||
Timestamp normalization and sequence gap detection for market data.
|
||||
|
||||
Exchange timestamps come in various formats (ms since epoch, ISO strings,
|
||||
exchange-specific formats). This module normalizes them to a consistent
|
||||
int64 milliseconds-since-epoch.
|
||||
|
||||
Gap detection tracks per-channel per-coin sequence numbers and flags
|
||||
missing messages so order books can be re-snapshotted.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
|
||||
# ── Timestamp normalization ───────────────────────────────────
|
||||
|
||||
def normalize_timestamp(ts, source: str = "hl") -> int:
|
||||
"""Normalize a timestamp to int64 milliseconds since epoch.
|
||||
|
||||
Args:
|
||||
ts: raw timestamp — can be int (ms), float (seconds), str (ISO 8601)
|
||||
source: 'hl' (Hyperliquid), 'binance', 'bybit', 'okx', 'coinbase', 'deribit'
|
||||
|
||||
Returns int64 milliseconds since epoch.
|
||||
"""
|
||||
if ts is None:
|
||||
return int(time.time() * 1000)
|
||||
|
||||
if isinstance(ts, (int, float)):
|
||||
if ts > 1_000_000_000_000:
|
||||
return int(ts) # already ms
|
||||
if ts > 1_000_000_000:
|
||||
return int(ts * 1000) # seconds → ms
|
||||
return int(ts * 1000) # fractional seconds → ms
|
||||
|
||||
if isinstance(ts, str):
|
||||
return _parse_iso_ms(ts)
|
||||
|
||||
if isinstance(ts, datetime):
|
||||
return int(ts.timestamp() * 1000)
|
||||
|
||||
return int(time.time() * 1000)
|
||||
|
||||
|
||||
def _parse_iso_ms(s: str) -> int:
|
||||
for fmt in [
|
||||
"%Y-%m-%dT%H:%M:%S.%fZ",
|
||||
"%Y-%m-%dT%H:%M:%S.%f",
|
||||
"%Y-%m-%dT%H:%M:%SZ",
|
||||
"%Y-%m-%dT%H:%M:%S",
|
||||
"%Y-%m-%d %H:%M:%S.%f",
|
||||
"%Y-%m-%d %H:%M:%S",
|
||||
]:
|
||||
try:
|
||||
dt = datetime.strptime(s.replace("+00:00", "").rstrip("Z"), fmt)
|
||||
if dt.tzinfo is None:
|
||||
dt = dt.replace(tzinfo=timezone.utc)
|
||||
return int(dt.timestamp() * 1000)
|
||||
except ValueError:
|
||||
continue
|
||||
return int(time.time() * 1000)
|
||||
|
||||
|
||||
# ── Sequence gap detection ────────────────────────────────────
|
||||
|
||||
class SequenceTracker:
|
||||
"""Track per-channel per-coin sequence numbers and detect gaps.
|
||||
|
||||
Usage:
|
||||
tracker = SequenceTracker()
|
||||
gap = tracker.check("l2book", "BTC", seq_num=1042)
|
||||
if gap:
|
||||
print(f"Gap detected: expected {gap['expected']}, got {gap['got']}")
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self._state: dict[str, int] = {} # key = "channel:coin", value = last_seq
|
||||
self._gap_count: dict[str, int] = {}
|
||||
|
||||
def check(self, channel: str, coin: str, seq_num: int) -> dict | None:
|
||||
"""Check for sequence gap. Returns None if ok, dict if gap."""
|
||||
key = f"{channel}:{coin}"
|
||||
last = self._state.get(key)
|
||||
|
||||
if last is None:
|
||||
self._state[key] = seq_num
|
||||
return None
|
||||
|
||||
expected = last + 1
|
||||
if seq_num == expected or seq_num > expected:
|
||||
self._state[key] = seq_num
|
||||
if seq_num > expected:
|
||||
gap_size = seq_num - expected
|
||||
self._gap_count[key] = self._gap_count.get(key, 0) + gap_size
|
||||
return {"key": key, "expected": expected, "got": seq_num, "gap_size": gap_size}
|
||||
return None
|
||||
|
||||
return None
|
||||
|
||||
def reset(self, channel: str, coin: str):
|
||||
"""Reset tracker (call after re-snapshot)."""
|
||||
self._state.pop(f"{channel}:{coin}", None)
|
||||
|
||||
@property
|
||||
def gap_counts(self) -> dict[str, int]:
|
||||
return dict(self._gap_count)
|
||||
|
||||
|
||||
def detect_sequence_gap(
|
||||
current_seq: int,
|
||||
last_seq: int | None,
|
||||
max_gap: int = 10,
|
||||
) -> int:
|
||||
"""Return gap size. 0 = ok, >0 = gap count, -1 = negative gap (dupe/reset)."""
|
||||
if last_seq is None:
|
||||
return 0
|
||||
diff = current_seq - last_seq
|
||||
if diff == 1:
|
||||
return 0
|
||||
if diff > 1:
|
||||
return min(diff, max_gap * 100) # cap reporting size
|
||||
return -1 # duplicate or reset
|
||||
Reference in New Issue
Block a user