ramseshk
fcfc136384
New microstructure/ module with pure-function analytics: microstructure/book.py: microprice() — depth-weighted mid price mid_price() — simple bid/ask midpoint order_book_imbalance() — ranged [-1, 1] volume skew depth_imbalance() — imbalance at fixed price distance spread_stats() — spread, spread_bps, mid, bid, ask depth_resiliency() — bid/ask volume within impact radius queue_depletion_prob() — Poisson fill probability at level batch_book_stats() — aggregate stats across snapshots microstructure/trades.py: classify_lee_ready() — Lee-Ready aggressor classification classify_bulk_lee_ready() — batch classification with mids/bids/asks compute_markouts() — forward mid-price change at configurable horizons markout_summary() — mean/std/t-stat per side per horizon trade_volume_profile() — size bucket distribution trade_arrival_rate() — rolling trades/sec with burst detection microstructure/toxicity.py: compute_vpin() — volume-synchronized informed trading probability compute_vpin_time_series() — rolling VPIN with alarm threshold fill_toxicity() — adverse price movement post-trade adverse_selection_ratio() — per-side adverse selection liquidation_clustering() — cluster detection in liquidation events microstructure/funding.py: funding_regime() — classify regime (neutral/positive/negative/high) funding_predictability() — AR(1) autocorrelation analysis funding_carry_pnl() — cumulative carry PnL estimation basis_spread() — perp premium over spot (bps) basis_convergence_speed() — mean-reversion half-life via AR(1) microstructure/signals.py: composite_signal() — weighted OBI + trade + VPIN + funding signal SignalPipeline — stateful pipeline accumulating book/trade updates detect_hft_regime() — regime classifier for HFT strategy selection Bug fixes in Phase 1: - data/latency.py: proper linear-interpolation percentiles - data/normalizer.py: UTC timezone for naive datetimes - data/normalizer.py: detect_sequence_gap returns gap-1 (missing count) - microstructure/toxicity.py: consistent vpin_value key in compute_vpin 81 tests across 4 test files (store, normalizer, latency, microstructure)
111 lines
3.4 KiB
Python
111 lines
3.4 KiB
Python
"""
|
|
Tests for data/store.py — Parquet-based raw message storage.
|
|
"""
|
|
import json
|
|
import os
|
|
import tempfile
|
|
import time
|
|
|
|
from data.store import RawMessageStore, read_range
|
|
|
|
|
|
def _make_parquet_dir():
|
|
"""Create a temporary directory for parquet storage."""
|
|
return tempfile.mkdtemp()
|
|
|
|
|
|
def test_store_basic_write_read():
|
|
"""Write messages, flush, read them back."""
|
|
tmpdir = tempfile.mkdtemp()
|
|
store = RawMessageStore(data_dir=tmpdir, flush_interval_sec=0.5)
|
|
store.start()
|
|
|
|
for i in range(10):
|
|
store.push(
|
|
channel="l2book",
|
|
coin="BTC",
|
|
exchange_ts=1718000000000 + i * 1000,
|
|
payload={"type": "snapshot", "levels": [[{"px": "50000", "sz": "1.0"}], [{"px": "50001", "sz": "0.5"}]]},
|
|
)
|
|
|
|
time.sleep(1.5)
|
|
store.stop()
|
|
assert store.total_written == 10
|
|
|
|
from datetime import date
|
|
today = date.today().isoformat()
|
|
rows = read_range(tmpdir, "l2book", "BTC", today, today)
|
|
assert len(rows) == 10
|
|
assert rows[0]["payload"]["type"] == "snapshot"
|
|
assert rows[0]["exchange_ts"] == 1718000000000
|
|
assert rows[-1]["exchange_ts"] == 1718000009000
|
|
|
|
|
|
def test_store_multiple_channels():
|
|
"""Write to different channels and verify partitioning."""
|
|
tmpdir = tempfile.mkdtemp()
|
|
store = RawMessageStore(data_dir=tmpdir, flush_interval_sec=0.5)
|
|
store.start()
|
|
|
|
channels = ["l2book", "trades", "funding", "mark", "open_interest"]
|
|
for ch in channels:
|
|
for i in range(3):
|
|
store.push(channel=ch, coin="BTC", exchange_ts=1718000000000 + i * 1000, payload={"ch": ch, "i": i})
|
|
|
|
time.sleep(1.5)
|
|
store.stop()
|
|
assert store.total_written == 15
|
|
|
|
from datetime import date
|
|
today = date.today().isoformat()
|
|
for ch in channels:
|
|
rows = read_range(tmpdir, ch, "BTC", today, today)
|
|
assert len(rows) == 3, f"Expected 3 rows for {ch}, got {len(rows)}"
|
|
|
|
|
|
def test_store_append_to_existing():
|
|
"""Write in two batches to same file — should append."""
|
|
tmpdir = tempfile.mkdtemp()
|
|
store = RawMessageStore(data_dir=tmpdir, flush_interval_sec=0.3)
|
|
store.start()
|
|
|
|
for i in range(5):
|
|
store.push(channel="trades", coin="ETH", exchange_ts=1718000000000 + i * 1000, payload={"batch": 1, "i": i})
|
|
|
|
time.sleep(1)
|
|
store.stop()
|
|
|
|
store2 = RawMessageStore(data_dir=tmpdir, flush_interval_sec=0.3)
|
|
store2.start()
|
|
for i in range(5):
|
|
store2.push(channel="trades", coin="ETH", exchange_ts=1718000005000 + i * 1000, payload={"batch": 2, "i": i})
|
|
time.sleep(1)
|
|
store2.stop()
|
|
|
|
from datetime import date
|
|
today = date.today().isoformat()
|
|
rows = read_range(tmpdir, "trades", "ETH", today, today)
|
|
assert len(rows) == 10
|
|
batches = [r["payload"]["batch"] for r in rows]
|
|
assert batches == [1] * 5 + [2] * 5
|
|
|
|
|
|
def test_store_queue_full_does_not_crash():
|
|
"""Small queue — push many, ensure no crash."""
|
|
tmpdir = tempfile.mkdtemp()
|
|
store = RawMessageStore(data_dir=tmpdir, flush_interval_sec=60.0, max_queue_size=10)
|
|
store.start()
|
|
|
|
for i in range(1000):
|
|
store.push(channel="trades", coin="BTC", exchange_ts=1718000000000 + i, payload={"i": i})
|
|
|
|
store.stop()
|
|
assert store.total_written >= 0 # some may be lost, but no crash
|
|
|
|
|
|
def test_read_range_empty():
|
|
"""Read a date range with no data returns empty list."""
|
|
tmpdir = tempfile.mkdtemp()
|
|
rows = read_range(tmpdir, "nonexistent", "BTC", "2020-01-01", "2020-01-02")
|
|
assert rows == []
|