ftdt-quant-lab/live/monitors/sequencer_latency.py

"""
Sequencer latency & stale-state arbitrage detector.

Hyperliquid uses a centralized sequencer for off-chain order matching.
There is a microsecond-to-millisecond delta between:
  1. WebSocket trade print (fastest)
  2. REST API state (slower)
  3. Cross-margin collateral recalculation (slowest)

When a liquidation hits one asset, cross-margin collateral drops for ALL
assets in that portfolio — but the sequencer may not have updated margin
limits on OTHER order books yet. This creates a stale-state window.

Strategy: detect liquidations on BTC, then hit ETH book before margins update.
"""

from __future__ import annotations

import time
from collections import deque
from typing import Optional


class SequencerLatencyDetector:
    """Detect and exploit sequencer state-update latency.

    Tracks WebSocket vs REST timestamps to measure the gap,
    and identifies stale-state windows after liquidation events.
    """

    def __init__(
        self,
        window_seconds: float = 60.0,
        stale_threshold_ms: float = 50.0,      # >50ms REST lag = stale
        max_history: int = 1000,
    ):
        self._window = window_seconds
        self._stale_threshold = stale_threshold_ms
        self._max_history = max_history

        # Latency tracking per channel per coin
        self._ws_timestamps: dict[str, dict[str, deque[float]]] = {}
        self._rest_timestamps: dict[str, dict[str, deque[float]]] = {}
        self._latency_measurements: deque = deque(maxlen=max_history)

        # Liquidation event log
        self._liquidations: deque = deque(maxlen=500)

        # Sequencer state
        self._stale_window_active: bool = False
        self._stale_assets: list[str] = []
        self._stale_start: float = 0.0

    # ── Data feed ────────────────────────────────────────────

    def record_ws_event(self, coin: str, channel: str, exchange_ts_ms: int):
        """Record WebSocket event timestamp (fastest)."""
        local_ts = time.time() * 1000
        c = coin.upper()
        ch = channel

        if c not in self._ws_timestamps:
            self._ws_timestamps[c] = {}
        self._ws_timestamps[c].setdefault(ch, deque(maxlen=self._max_history)).append(exchange_ts_ms)

        # Measure latency: how much time between exchange and local receipt
        lat = local_ts - exchange_ts_ms
        self._latency_measurements.append({
            "time": time.time(),
            "coin": c,
            "channel": ch,
            "latency_ms": round(lat, 2),
            "direction": "ws",
        })

    def record_rest_event(self, coin: str, endpoint: str, exchange_ts_ms: int):
        """Record REST API timestamp (slower)."""
        c = coin.upper()
        if c not in self._rest_timestamps:
            self._rest_timestamps[c] = {}
        self._rest_timestamps[c].setdefault(endpoint, deque(maxlen=self._max_history)).append(exchange_ts_ms)

    def record_liquidation(self, coin: str, size: float, price: float, timestamp_ms: int):
        """Record a liquidation event — triggers stale-state analysis."""
        self._liquidations.append({
            "time": time.time(),
            "coin": coin.upper(),
            "size": size,
            "price": price,
            "exchange_ts": timestamp_ms,
        })

    # ── Analysis ─────────────────────────────────────────────

    def ws_rest_latency(self, coin: str, channel: str = "l2book") -> dict | None:
        """Measure latency between WebSocket and REST for a coin/channel.

        Returns median latency (ms), count of measurements, and stale flag.
        """
        ws = list(self._ws_timestamps.get(coin.upper(), {}).get(channel, []))
        rest = list(self._rest_timestamps.get(coin.upper(), {}).get(channel, []))
        if not ws or not rest:
            return None

        # Compare latest timestamps — if WS is newer than REST by > threshold
        ws_latest = ws[-1]
        rest_latest = rest[-1]
        delta = rest_latest - ws_latest

        return {
            "coin": coin.upper(),
            "channel": channel,
            "ws_latest_ms": ws_latest,
            "rest_latest_ms": rest_latest,
            "delta_ms": round(delta, 2),
            "is_stale": delta > self._stale_threshold,
            "measurements": min(len(ws), len(rest)),
        }

    def avg_transport_latency(self) -> dict:
        """Average WebSocket transport latency (exchange → local).

        Returns p50, p99, max, and sample count.
        """
        lats = [m["latency_ms"] for m in self._latency_measurements]
        if not lats:
            return {"p50_ms": 0, "p99_ms": 0, "max_ms": 0, "count": 0}

        sorted_lats = sorted(lats)
        n = len(sorted_lats)
        return {
            "p50_ms": round(sorted_lats[int(n * 0.50)], 2),
            "p99_ms": round(sorted_lats[int(n * 0.99)], 2),
            "max_ms": round(max(lats), 2),
            "count": n,
        }

    def recent_liquidations(self) -> list[dict]:
        """Liquidation events in the last window_seconds."""
        cutoff = time.time() - self._window
        return [l for l in self._liquidations if l["time"] >= cutoff]

    def stale_state_signal(self) -> dict:
        """Detect if a stale-state window is active after liquidation.

        If a liquidation just occurred and REST lag is > threshold:
          - Mark affected assets as stale
          - Signal which other assets may have outdated margin limits
        """
        recent_liqs = self.recent_liquidations()
        if not recent_liqs:
            return {"stale_window_active": False, "signal": "none"}

        latest_liq = recent_liqs[-1]
        age_ms = (time.time() - latest_liq["time"]) * 1000

        # Check if any asset has REST lag exceeding threshold
        stale_assets = []
        for coin, channels in self._ws_timestamps.items():
            for channel in channels:
                lat = self.ws_rest_latency(coin, channel)
                if lat and lat["is_stale"]:
                    stale_assets.append(lat)

        is_stale = len(stale_assets) > 0 and age_ms < 1000  # within 1s of liquidation

        return {
            "stale_window_active": is_stale,
            "signal": "stale_state_detected" if is_stale else "none",
            "liquidation_coin": latest_liq["coin"],
            "liquidation_age_ms": round(age_ms, 2),
            "stale_assets": [s["coin"] for s in stale_assets],
            "recommended_action": "check_margin_limits_on_STALE_ASSETS" if is_stale else "none",
        }

    def summary(self) -> dict:
        return {
            "avg_latency": self.avg_transport_latency(),
            "recent_liquidations": len(self.recent_liquidations()),
            "stale_state": self.stale_state_signal(),
        }