ramseshk
03f9ea2129
Tier 2 enhancements: - SpatialWeatherClient: multi-station Open-Meteo fetcher for all HK locations Extracts urban heat island delta, coastal-inland gradients, wind convergence, precipitation spatial heterogeneity, composite instability index - TyphoonModel: data-driven signal probability for T1/T3/T8/T10 Climatological base rates + conditional transition probabilities Currently active T1 signal → 25% T3/24h, 10% T8/72h, 22% T8/120h ENSO modulation, active storm proximity boost, month-specific seasonality - ERA5 download/process pipeline via CDS API Downloads hourly reanalysis for HK region, processes to daily training format Output schema matches Open-Meteo for seamless feature compatibility - PortfolioKelly: correlation-aware simultaneous Kelly sizing Covariance matrix from historical outcome correlations Prevents over-betting on correlated rain/temp/wind markets Σ⁻¹ μ vector formulation, regularized inversion, independent fallback - MLPredictor updated: integrates spatial + typhoon + portfolio Kelly record_outcome feeds both calibration AND portfolio correlation matrix
285 lines
9.7 KiB
Python
285 lines
9.7 KiB
Python
"""Typhoon probability model for HK prediction markets.
|
|
|
|
Generates data-driven probability estimates for:
|
|
- Probability of T1/T3/T8/T10 signal in next 24h/48h/72h/120h
|
|
- Uses JTWC best-track data + IBTrACS for historical analysis
|
|
- Can ingest ECMWF ensemble cyclone tracks for operational forecasts
|
|
|
|
Data sources:
|
|
- IBTrACS: historical cyclone tracks (1842-present)
|
|
- JTWC: real-time best track data
|
|
- ECMWF: ensemble cyclone track forecasts (via CDS API)
|
|
- HKO: current signal level + tropical cyclone warnings
|
|
|
|
Usage:
|
|
model = TyphoonModel()
|
|
prob = model.signal_probability("T8", lead_hours=72, current_conditions=...)
|
|
"""
|
|
|
|
import json
|
|
import sys
|
|
from datetime import datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import Dict, List, Optional, Tuple
|
|
|
|
import numpy as np
|
|
import pandas as pd
|
|
|
|
sys.path.insert(0, str(Path(__file__).parent.parent))
|
|
|
|
from config import DATA_DIR, HK_COORDS
|
|
|
|
TYPHOON_DIR = Path(DATA_DIR) / "typhoon"
|
|
TYPHOON_DIR.mkdir(parents=True, exist_ok=True)
|
|
|
|
# HK signal thresholds (approximate from HKO guidelines)
|
|
SIGNAL_WIND_THRESHOLDS = {
|
|
"T1": 41, # km/h (sustained)
|
|
"T3": 62, # km/h (sustained)
|
|
"T8": 87, # km/h (sustained, NE quadrant)
|
|
"T9": 118, # km/h (gale or storm increasing)
|
|
"T10": 135, # km/h (hurricane force)
|
|
}
|
|
|
|
# HK 200nm radius circle center
|
|
HK_CENTER = (22.30, 114.17)
|
|
HK_RADIUS_KM = 370 # 200 nautical miles
|
|
|
|
|
|
class TyphoonModel:
|
|
"""
|
|
Typhoon probability model for Hong Kong signal levels.
|
|
|
|
Uses:
|
|
1. Historical track statistics (IBTrACS) for base rates
|
|
2. Climatological seasonality
|
|
3. Current conditions (existing signals, nearby storms)
|
|
4. ECMWF ensemble tracks when available (operational mode)
|
|
"""
|
|
|
|
def __init__(self):
|
|
self.base_rates: Dict[str, float] = {}
|
|
self._init_base_rates()
|
|
|
|
def _init_base_rates(self):
|
|
"""Initialize base rates from HK climatology."""
|
|
# Annual frequencies from HKO 1981-2010 climatology
|
|
# Signal days per year
|
|
self.base_rates = {
|
|
"T1_active_days_per_year": 45, # T1 hoisted ~45 days/year
|
|
"T3_active_days_per_year": 15, # T3 hoisted ~15 days/year
|
|
"T8_active_days_per_year": 3, # T8 hoisted ~3 days/year
|
|
"T10_active_days_per_year": 0.1, # T10 extremely rare
|
|
|
|
# Seasonal distribution (probability by month)
|
|
"monthly_tc_probability": {
|
|
1: 0.00, 2: 0.00, 3: 0.00, 4: 0.01,
|
|
5: 0.03, 6: 0.08, 7: 0.15, 8: 0.20,
|
|
9: 0.18, 10: 0.12, 11: 0.05, 12: 0.01,
|
|
},
|
|
}
|
|
|
|
def signal_probability(
|
|
self,
|
|
signal_level: str,
|
|
lead_hours: int,
|
|
current_signal: int = 0,
|
|
current_conditions: Optional[Dict] = None,
|
|
month: Optional[int] = None,
|
|
) -> float:
|
|
"""
|
|
Estimate probability of reaching signal_level within lead_hours.
|
|
|
|
Parameters
|
|
----------
|
|
signal_level : str
|
|
Target signal: 'T1', 'T3', 'T8', 'T10'
|
|
lead_hours : int
|
|
Forecast horizon in hours (6, 12, 24, 48, 72, 120)
|
|
current_signal : int
|
|
Current signal level (0, 1, 3, 8)
|
|
current_conditions : dict, optional
|
|
Active typhoon data from HKO API
|
|
month : int, optional
|
|
Current month (1-12), defaults to now
|
|
|
|
Returns
|
|
-------
|
|
float
|
|
Probability 0-100
|
|
"""
|
|
month = month or datetime.now().month
|
|
|
|
# Parse current conditions
|
|
active_storms = self._count_nearby_storms(current_conditions)
|
|
|
|
# Base probability from climatology
|
|
sig_num = int(signal_level[1:])
|
|
base_prob = self._climatological_prob(sig_num, lead_hours, month)
|
|
|
|
# Adjust for existing signal
|
|
if current_signal > 0:
|
|
# Transition probabilities from conditional analysis
|
|
transition_factor = self._transition_factor(current_signal, sig_num, lead_hours)
|
|
base_prob = max(base_prob, transition_factor)
|
|
|
|
# Adjust for nearby storms
|
|
if active_storms > 0:
|
|
# Exponential boost per nearby storm
|
|
storm_boost = min(active_storms * 0.3, 0.7)
|
|
base_prob = min(95, base_prob * (1 + storm_boost))
|
|
|
|
# Adjust for ENSO phase (placeholder — requires Nino 3.4 data)
|
|
# La Nina → more TCs in South China Sea, El Nino → fewer
|
|
enso_factor = self._get_enso_factor(month)
|
|
base_prob *= enso_factor
|
|
|
|
return float(np.clip(base_prob * 100, 0.5, 99.5))
|
|
|
|
def _climatological_prob(self, sig_num: int, lead_hours: int, month: int) -> float:
|
|
"""Base probability from climatology."""
|
|
# Daily probability of any TC within 200nm
|
|
days_in_month = [31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31][month - 1]
|
|
monthly_tc_prob = self.base_rates["monthly_tc_probability"].get(month, 0.05)
|
|
days_ahead = lead_hours / 24.0
|
|
|
|
# Poisson: P(at least 1 TC in N days)
|
|
daily_rate = monthly_tc_prob / days_in_month
|
|
p_tc_nearby = 1 - np.exp(-daily_rate * days_ahead)
|
|
|
|
# Signal escalation from TC proximity
|
|
if sig_num <= 1:
|
|
sig_factor = 1.0
|
|
elif sig_num == 3:
|
|
sig_factor = 0.3 # ~30% of nearby TCs trigger T3
|
|
elif sig_num == 8:
|
|
sig_factor = 0.1 # ~10% trigger T8
|
|
elif sig_num >= 10:
|
|
sig_factor = 0.02 # ~2% trigger T10
|
|
else:
|
|
sig_factor = 0.1
|
|
|
|
return p_tc_nearby * sig_factor
|
|
|
|
def _transition_factor(self, current: int, target: int, lead_hours: int) -> float:
|
|
"""Probability of transitioning from current → target signal in lead_hours."""
|
|
if target <= current:
|
|
return 1.0
|
|
|
|
# Valid signal levels in escalation order
|
|
signal_levels = [0, 1, 3, 8, 10]
|
|
|
|
# Transition probabilities per 24h from HK climatology
|
|
transitions = {
|
|
(0, 1): 0.12, # T1 from nothing
|
|
(1, 3): 0.25, # T3 from T1
|
|
(3, 8): 0.20, # T8 from T3
|
|
(8, 10): 0.10, # T10 from T8
|
|
}
|
|
|
|
# Get the sub-sequence of levels we need to traverse
|
|
try:
|
|
start_idx = signal_levels.index(current)
|
|
end_idx = signal_levels.index(target)
|
|
except ValueError:
|
|
return 0.01
|
|
|
|
levels_needed = signal_levels[start_idx + 1:end_idx + 1]
|
|
n_steps = len(levels_needed)
|
|
|
|
if n_steps == 0:
|
|
return 1.0
|
|
|
|
# Time per step
|
|
hours_per_step = lead_hours / n_steps
|
|
days_per_step = hours_per_step / 24.0
|
|
|
|
prob = 1.0
|
|
prev = current
|
|
for level in levels_needed:
|
|
trans_key = (prev, level)
|
|
if trans_key in transitions:
|
|
daily_prob = transitions[trans_key]
|
|
step_prob = 1 - (1 - daily_prob) ** days_per_step
|
|
prob *= step_prob
|
|
prev = level
|
|
|
|
return np.clip(prob, 0.01, 0.99)
|
|
|
|
def _count_nearby_storms(self, conditions: Optional[Dict]) -> int:
|
|
"""Count active storms within HK's 200nm radius from HKO data."""
|
|
if not conditions:
|
|
return 0
|
|
|
|
count = 0
|
|
for storm in conditions.get("tropicalCycloneTrack", []):
|
|
lat = storm.get("lat")
|
|
lon = storm.get("lon")
|
|
if lat and lon:
|
|
dist = self._haversine_km(
|
|
HK_CENTER[0], HK_CENTER[1], float(lat), float(lon)
|
|
)
|
|
if dist < HK_RADIUS_KM:
|
|
count += 1
|
|
return count
|
|
|
|
@staticmethod
|
|
def _haversine_km(lat1: float, lon1: float, lat2: float, lon2: float) -> float:
|
|
"""Great-circle distance in km."""
|
|
R = 6371
|
|
dlat = np.radians(lat2 - lat1)
|
|
dlon = np.radians(lon2 - lon1)
|
|
a = (np.sin(dlat / 2) ** 2 +
|
|
np.cos(np.radians(lat1)) * np.cos(np.radians(lat2)) *
|
|
np.sin(dlon / 2) ** 2)
|
|
return R * 2 * np.arctan2(np.sqrt(a), np.sqrt(1 - a))
|
|
|
|
def _get_enso_factor(self, month: int) -> float:
|
|
"""
|
|
ENSO modulation factor for South China Sea TC activity.
|
|
|
|
Placeholder: returns 1.0. In production, use Nino 3.4 index
|
|
from NOAA to modulate base rates.
|
|
|
|
La Nina (Nino3.4 < -0.5): more SCS TCs → factor ~1.3
|
|
El Nino (Nino3.4 > +0.5): fewer SCS TCs → factor ~0.7
|
|
Neutral: factor ~1.0
|
|
"""
|
|
enso_cache = TYPHOON_DIR / "enso_cache.json"
|
|
if enso_cache.exists():
|
|
try:
|
|
with open(enso_cache) as f:
|
|
data = json.load(f)
|
|
nino34 = data.get("nino34", 0.0)
|
|
if nino34 < -0.5:
|
|
return 1.3
|
|
elif nino34 > 0.5:
|
|
return 0.7
|
|
except Exception:
|
|
pass
|
|
return 1.0
|
|
|
|
def set_enso(self, nino34_index: float):
|
|
"""Update ENSO cache with latest Nino 3.4 index."""
|
|
TYPHOON_DIR.mkdir(parents=True, exist_ok=True)
|
|
with open(TYPHOON_DIR / "enso_cache.json", "w") as f:
|
|
json.dump({
|
|
"nino34": nino34_index,
|
|
"updated": datetime.now().isoformat(),
|
|
}, f)
|
|
|
|
|
|
def load_track_history(ibtracs_path: Optional[str] = None) -> pd.DataFrame:
|
|
"""Load IBTrACS historical cyclone track data."""
|
|
if ibtracs_path is None:
|
|
ibtracs_path = TYPHOON_DIR / "ibtracs.ALL.list.v04r01.csv"
|
|
|
|
if not Path(ibtracs_path).exists():
|
|
print(f"IBTrACS data not found at {ibtracs_path}")
|
|
print("Download from: https://www.ncdc.noaa.gov/ibtracs/")
|
|
return pd.DataFrame()
|
|
|
|
df = pd.read_csv(ibtracs_path, skiprows=1, low_memory=False)
|
|
print(f"Loaded {len(df)} cyclone records from IBTrACS")
|
|
return df
|