HK Weather Prediction Market Pipeline: WeatherNext + HKO + Polymarket
- Open-Meteo WeatherNext API client for HK forecasts - HKO public data client (current conditions, 9-day forecast, typhoon warnings) - HK-specific weather extraction and calibration - Polymarket market scanning, price discovery, and market creation proposals - Trading strategy engine: edge detection, Kelly criterion sizing, probability calibration - End-to-end pipeline with dry-run mode and scheduled runner - Interactive dashboard with live HK weather + forecasts + trading signals Dependencies: Python 3.10+, openmeteo-requests, pandas No API keys needed for dry-run mode. Polymarket trading requires private key in .env.
This commit is contained in:
ramseshk
@@ -0,0 +1,7 @@
|
||||
"""Weather data for Hong Kong weather prediction markets."""
|
||||
|
||||
from .openmeteo_client import OpenMeteoClient
|
||||
from .hko_client import HKOClient
|
||||
from .hk_extractor import HKExtractor
|
||||
|
||||
__all__ = ["OpenMeteoClient", "HKOClient", "HKExtractor"]
|
||||
@@ -0,0 +1,197 @@
|
||||
"""Extract Hong Kong specific forecasts from global model outputs.
|
||||
|
||||
Handles regional extraction, downscaling hints, and local calibration
|
||||
based on HKO station data for the Hong Kong region.
|
||||
"""
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Optional, Dict, List
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from config import HK_COORDS, HK_BBOX
|
||||
from .openmeteo_client import OpenMeteoClient
|
||||
from .hko_client import HKOClient
|
||||
|
||||
|
||||
class HKExtractor:
|
||||
"""Extract and calibrate HK-specific weather forecasts from global models."""
|
||||
|
||||
# Known stations for calibration (HKO stations with good historical data)
|
||||
CALIBRATION_STATIONS = [
|
||||
"Hong Kong Observatory", # Tsim Sha Tsui
|
||||
"Chek Lap Kok", # Airport
|
||||
"Sha Tin",
|
||||
"Tuen Mun",
|
||||
"Sai Kung",
|
||||
"Ta Kwu Ling",
|
||||
"Sheung Shui",
|
||||
"Stanley",
|
||||
]
|
||||
|
||||
# Calibration offsets - will be learned over time
|
||||
# (model_bias, model_std) for key variables
|
||||
DEFAULT_BIAS = {
|
||||
"temperature_2m_max": 0.0,
|
||||
"temperature_2m_min": 0.0,
|
||||
"precipitation_probability_max": 0.0,
|
||||
"wind_speed_10m_max": 0.0,
|
||||
}
|
||||
|
||||
def __init__(self, calibrate: bool = True):
|
||||
self.openmeteo = OpenMeteoClient()
|
||||
self.hko = HKOClient()
|
||||
self.calibrate = calibrate
|
||||
self.bias_model = self.DEFAULT_BIAS.copy()
|
||||
self._load_calibration()
|
||||
|
||||
def _load_calibration(self):
|
||||
"""Load calibration params from stored file if available."""
|
||||
import os
|
||||
import json
|
||||
path = os.path.join(os.path.dirname(__file__), "..", "data", "calibration.json")
|
||||
if os.path.exists(path):
|
||||
try:
|
||||
with open(path) as f:
|
||||
stored = json.load(f)
|
||||
self.bias_model.update(stored.get("bias", {}))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def save_calibration(self):
|
||||
"""Save calibration params for future runs."""
|
||||
import os
|
||||
import json
|
||||
path = os.path.join(os.path.dirname(__file__), "..", "data", "calibration.json")
|
||||
with open(path, "w") as f:
|
||||
json.dump({"bias": self.bias_model, "updated": datetime.now().isoformat()}, f, indent=2)
|
||||
|
||||
def get_hk_forecast(self, lead_days: int = 7) -> Dict:
|
||||
"""Get calibrated HK-specific forecast combining multiple sources."""
|
||||
forecast = {
|
||||
"fetch_time": datetime.now().isoformat(),
|
||||
"sources": {},
|
||||
}
|
||||
|
||||
wnext = self.openmeteo.get_forecast(lead_days=lead_days)
|
||||
if wnext is not None:
|
||||
forecast["sources"]["weathernext"] = self._calibrate_forecast(wnext)
|
||||
|
||||
hko_fc = self.hko.get_forecast()
|
||||
if hko_fc:
|
||||
forecast["sources"]["hko"] = hko_fc
|
||||
|
||||
current = self.hko.get_current_weather()
|
||||
if current:
|
||||
forecast["current_observations"] = current
|
||||
|
||||
typhoon = self.hko.get_typhoon_info()
|
||||
if typhoon:
|
||||
forecast["typhoon_info"] = typhoon
|
||||
|
||||
forecast["consensus"] = self._build_consensus(forecast)
|
||||
|
||||
return forecast
|
||||
|
||||
def _calibrate_forecast(self, df: pd.DataFrame) -> List[Dict]:
|
||||
"""Apply calibration to model forecast and return structured data."""
|
||||
results = []
|
||||
for idx, row in df.iterrows():
|
||||
day = {
|
||||
"date": idx.strftime("%Y-%m-%d"),
|
||||
"temp_max_calibrated": float(row.get("temperature_2m_max", np.nan)) + self.bias_model.get("temperature_2m_max", 0),
|
||||
"temp_min_calibrated": float(row.get("temperature_2m_min", np.nan)) + self.bias_model.get("temperature_2m_min", 0),
|
||||
"temp_max_raw": float(row.get("temperature_2m_max", np.nan)),
|
||||
"temp_min_raw": float(row.get("temperature_2m_min", np.nan)),
|
||||
"precipitation_probability_calibrated": min(100, max(0, float(row.get("precipitation_probability_max", 0)) + self.bias_model.get("precipitation_probability_max", 0))),
|
||||
"precipitation_probability_raw": float(row.get("precipitation_probability_max", 0)),
|
||||
"precipitation_sum": float(row.get("precipitation_sum", 0)),
|
||||
"wind_speed_max_calibrated": float(row.get("wind_speed_10m_max", np.nan)) + self.bias_model.get("wind_speed_10m_max", 0),
|
||||
"wind_speed_max_raw": float(row.get("wind_speed_10m_max", np.nan)),
|
||||
"wind_gusts_max": float(row.get("wind_gusts_10m_max", np.nan)),
|
||||
}
|
||||
results.append(day)
|
||||
return results
|
||||
|
||||
def _build_consensus(self, forecast: Dict) -> Dict:
|
||||
"""Build a consensus forecast from all available sources."""
|
||||
consensus = {}
|
||||
|
||||
if "weathernext" in forecast.get("sources", {}):
|
||||
w = forecast["sources"]["weathernext"]
|
||||
if w:
|
||||
d0 = w[0]
|
||||
consensus["tomorrow"] = d0
|
||||
|
||||
if "hko" in forecast.get("sources", {}):
|
||||
h = forecast["sources"]["hko"]
|
||||
if h and len(h) > 0:
|
||||
consensus["hko_tomorrow"] = h[0]
|
||||
|
||||
current = forecast.get("current_observations", {})
|
||||
if current:
|
||||
consensus["current_temp"] = (
|
||||
current.get("temperature", [{}])[0].get("value") if current.get("temperature") else None
|
||||
)
|
||||
|
||||
return consensus
|
||||
|
||||
def get_combined_tomorrow_forecast(self) -> Dict:
|
||||
"""Get a single combined forecast for 'tomorrow' from all sources."""
|
||||
fc = self.get_hk_forecast()
|
||||
return fc.get("consensus", {})
|
||||
|
||||
def should_bet_rain_tomorrow(self) -> Optional[float]:
|
||||
"""Returns model-implied probability of rain tomorrow (0-100)."""
|
||||
fc = self.get_hk_forecast()
|
||||
consensus = fc.get("consensus", {})
|
||||
tomorrow = consensus.get("tomorrow", {})
|
||||
hko = consensus.get("hko_tomorrow", {})
|
||||
|
||||
probs = []
|
||||
if "precipitation_probability_calibrated" in tomorrow:
|
||||
probs.append(tomorrow["precipitation_probability_calibrated"])
|
||||
|
||||
hko_prob_str = hko.get("forecast_rain_probability", "")
|
||||
if hko_prob_str:
|
||||
try:
|
||||
nums = [int(x.replace("%", "")) for x in hko_prob_str.split("/")]
|
||||
probs.append(max(nums))
|
||||
except (ValueError, AttributeError):
|
||||
pass
|
||||
|
||||
if not probs:
|
||||
return None
|
||||
return float(np.mean(probs))
|
||||
|
||||
def should_bet_temp_above(self, threshold: float = 30.0) -> Optional[float]:
|
||||
"""Returns model-implied probability that temp exceeds threshold tomorrow."""
|
||||
fc = self.get_hk_forecast()
|
||||
consensus = fc.get("consensus", {})
|
||||
tomorrow = consensus.get("tomorrow", {})
|
||||
hko = consensus.get("hko_tomorrow", {})
|
||||
|
||||
temp_max_raw = tomorrow.get("temp_max_raw", np.nan)
|
||||
temp_max_cal = tomorrow.get("temp_max_calibrated", np.nan)
|
||||
|
||||
# Simple: if calibrated max is above threshold, probability from how far above
|
||||
if not np.isnan(temp_max_cal):
|
||||
excess = temp_max_cal - threshold
|
||||
prob = min(100, max(0, 50 + excess * 20)) # Simple sigmoid-like
|
||||
return prob
|
||||
|
||||
return None
|
||||
|
||||
def update_calibration(self, forecast_date: str, observed: Dict):
|
||||
"""Update calibration based on observed vs predicted."""
|
||||
# This would be called after the scoring window closes
|
||||
# Simple exponential moving average of errors
|
||||
alpha = 0.1
|
||||
for var in self.bias_model:
|
||||
if var in observed and var.replace("_calibrated", "_raw") in observed:
|
||||
# We'd need to store the forecast that was made for this date
|
||||
# This is a placeholder for the calibration loop
|
||||
pass
|
||||
|
||||
self.save_calibration()
|
||||
@@ -0,0 +1,177 @@
|
||||
"""Hong Kong Observatory data client.
|
||||
|
||||
Fetches real-time weather observations, warnings, and forecasts from HKO.
|
||||
Uses public RSS/JSON feeds where available.
|
||||
"""
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from typing import Optional, List
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
class HKOClient:
|
||||
"""Fetch data from Hong Kong Observatory's public data feeds."""
|
||||
|
||||
BASE_URL = "https://data.weather.gov.hk"
|
||||
|
||||
ENDPOINTS = {
|
||||
"current_weather": f"{BASE_URL}/weatherAPI/opendata/weather.php?dataType=rhrread&lang=en",
|
||||
"9day_forecast": f"{BASE_URL}/weatherAPI/opendata/weather.php?dataType=fnd&lang=en",
|
||||
"warnings": f"{BASE_URL}/weatherAPI/opendata/weather.php?dataType=warnsum&lang=en",
|
||||
"local_forecast": f"{BASE_URL}/weatherAPI/opendata/weather.php?dataType=flw&lang=en",
|
||||
"swt": f"{BASE_URL}/weatherAPI/opendata/weather.php?dataType=swt&lang=en",
|
||||
"rainfall": f"{BASE_URL}/weatherAPI/opendata/weather.php?dataType=rfmap",
|
||||
"lightning": f"{BASE_URL}/weatherAPI/opendata/weather.php?dataType=lmap",
|
||||
"uv_index": f"{BASE_URL}/weatherAPI/opendata/weather.php?dataType=uvi",
|
||||
"earthquake": f"{BASE_URL}/weatherAPI/opendata/weather.php?dataType=eq",
|
||||
}
|
||||
|
||||
def __init__(self):
|
||||
self.session = requests.Session()
|
||||
self.session.headers.update({
|
||||
"User-Agent": "HK-Weather-Market/1.0",
|
||||
"Accept": "application/json",
|
||||
})
|
||||
|
||||
def get_current_weather(self) -> Optional[dict]:
|
||||
"""Get current weather observations for Hong Kong."""
|
||||
try:
|
||||
resp = self.session.get(self.ENDPOINTS["current_weather"], timeout=15)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
result = {
|
||||
"update_time": data.get("updateTime", ""),
|
||||
"temperature": [],
|
||||
"humidity": [],
|
||||
"rainfall": [],
|
||||
"warning_message": data.get("warningMessage", ""),
|
||||
"rainstorm_reminder": data.get("rainstormReminder", ""),
|
||||
"mintemp_from00to09": data.get("mintempFrom00To09", ""),
|
||||
}
|
||||
|
||||
# Parse temperature data
|
||||
for record in data.get("temperature", {}).get("data", []):
|
||||
result["temperature"].append({
|
||||
"place": record.get("place"),
|
||||
"value": record.get("value"),
|
||||
"unit": record.get("unit", "C"),
|
||||
})
|
||||
|
||||
for record in data.get("humidity", {}).get("data", []):
|
||||
result["humidity"].append({
|
||||
"place": record.get("place"),
|
||||
"value": record.get("value"),
|
||||
"unit": record.get("unit", "%"),
|
||||
})
|
||||
|
||||
# Parse rainfall
|
||||
for record in data.get("rainfall", {}).get("data", []):
|
||||
result["rainfall"].append({
|
||||
"place": record.get("place"),
|
||||
"max": record.get("max"),
|
||||
"min": record.get("min"),
|
||||
"unit": record.get("unit", "mm"),
|
||||
})
|
||||
|
||||
return result
|
||||
|
||||
except Exception as e:
|
||||
print(f"HKO current weather error: {e}")
|
||||
return None
|
||||
|
||||
def get_forecast(self) -> Optional[List[dict]]:
|
||||
"""Get 9-day weather forecast from HKO."""
|
||||
try:
|
||||
resp = self.session.get(self.ENDPOINTS["9day_forecast"], timeout=15)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
|
||||
forecasts = []
|
||||
for day in data.get("weatherForecast", []):
|
||||
forecasts.append({
|
||||
"date": day.get("forecastDate"),
|
||||
"week": day.get("week"),
|
||||
"forecast_wind": day.get("forecastWind"),
|
||||
"forecast_weather": day.get("forecastWeather"),
|
||||
"forecast_temp_max": day.get("forecastMaxtemp", {}).get("value"),
|
||||
"forecast_temp_min": day.get("forecastMintemp", {}).get("value"),
|
||||
"forecast_humidity_max": day.get("forecastMaxrh", {}).get("value"),
|
||||
"forecast_humidity_min": day.get("forecastMinrh", {}).get("value"),
|
||||
"forecast_rain_probability": self._parse_probability(
|
||||
day.get("forecastRainProbability", {})
|
||||
),
|
||||
"psr": day.get("PSR"),
|
||||
})
|
||||
|
||||
return forecasts
|
||||
|
||||
except Exception as e:
|
||||
print(f"HKO forecast error: {e}")
|
||||
return None
|
||||
|
||||
def get_warnings(self) -> Optional[dict]:
|
||||
"""Get current weather warnings and signals."""
|
||||
try:
|
||||
resp = self.session.get(self.ENDPOINTS["warnings"], timeout=15)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
except Exception as e:
|
||||
print(f"HKO warnings error: {e}")
|
||||
return None
|
||||
|
||||
def get_typhoon_info(self) -> Optional[dict]:
|
||||
"""Get tropical cyclone warnings and tracks from HKO."""
|
||||
try:
|
||||
resp = self.session.get(self.ENDPOINTS["swt"], timeout=15)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
except Exception as e:
|
||||
print(f"HKO typhoon info error: {e}")
|
||||
return None
|
||||
|
||||
def get_rainfall_map(self) -> Optional[dict]:
|
||||
"""Get recent rainfall distribution."""
|
||||
try:
|
||||
resp = self.session.get(self.ENDPOINTS["rainfall"], timeout=15)
|
||||
resp.raise_for_status()
|
||||
return resp.json()
|
||||
except Exception as e:
|
||||
print(f"HKO rainfall map error: {e}")
|
||||
return None
|
||||
|
||||
def is_t8_active(self) -> bool:
|
||||
"""Check if T8 or higher typhoon signal is active."""
|
||||
warnings = self.get_warnings()
|
||||
if not warnings:
|
||||
return False
|
||||
|
||||
for detail in warnings.get("details", []):
|
||||
name = detail.get("name", "")
|
||||
if any(signal in name for signal in ["8", "9", "10"]):
|
||||
if "tropical cyclone" in name.lower() or "typhoon" in name.lower():
|
||||
return True
|
||||
return False
|
||||
|
||||
def get_current_signal_level(self) -> int:
|
||||
"""Get current typhoon signal level (0, 1, 3, 8, 9, 10)."""
|
||||
typhoon = self.get_typhoon_info()
|
||||
if not typhoon:
|
||||
return 0
|
||||
|
||||
for signal_text in ["T1", "T3", "T8", "T9", "T10"]:
|
||||
if signal_text in str(typhoon):
|
||||
return int(signal_text[1:])
|
||||
return 0
|
||||
|
||||
@staticmethod
|
||||
def _parse_probability(prob_data: dict) -> Optional[str]:
|
||||
"""Parse HKO rainfall probability data."""
|
||||
values = []
|
||||
for period in prob_data.get("probability", []):
|
||||
perc = period.get("value")
|
||||
if perc:
|
||||
values.append(perc)
|
||||
return " / ".join(values) if values else None
|
||||
@@ -0,0 +1,216 @@
|
||||
"""Local WeatherNext model runner using JAX/GPU.
|
||||
|
||||
Runs WeatherNext Cyclones Mini on RTX 4070 SUPER (12GB VRAM).
|
||||
"""
|
||||
|
||||
import os
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Optional, Dict
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from config import WEIGHTS_DIR, HK_BBOX, HK_COORDS
|
||||
|
||||
|
||||
class WeatherNextRunner:
|
||||
"""Run WeatherNext model locally for Hong Kong region forecasts."""
|
||||
|
||||
def __init__(self, model_name: str = "WeatherNextCyclones_Mini_<2024"):
|
||||
self.model_name = model_name
|
||||
self.weights_path = WEIGHTS_DIR / f"{model_name}.npz"
|
||||
self.model = None
|
||||
self.params = None
|
||||
self.state = None
|
||||
self.initialized = False
|
||||
|
||||
def check_ready(self) -> bool:
|
||||
"""Check if model weights exist and JAX is working."""
|
||||
if not self.weights_path.exists():
|
||||
print(f"Weights not found: {self.weights_path}")
|
||||
print("Run: python scripts/download_weights.py")
|
||||
return False
|
||||
|
||||
try:
|
||||
import jax
|
||||
devices = jax.devices()
|
||||
if not devices:
|
||||
print("No JAX devices found")
|
||||
return False
|
||||
print(f"JAX devices: {devices}")
|
||||
return True
|
||||
except ImportError:
|
||||
print("JAX not installed")
|
||||
return False
|
||||
|
||||
def load_model(self) -> bool:
|
||||
"""Load WeatherNext model and weights."""
|
||||
if not self.check_ready():
|
||||
return False
|
||||
|
||||
try:
|
||||
import jax
|
||||
import jax.numpy as jnp
|
||||
from weathernext.models.fgn import FGN, FGNConfig
|
||||
|
||||
print(f"Loading model weights from {self.weights_path}...")
|
||||
params = dict(np.load(self.weights_path, allow_pickle=True))
|
||||
|
||||
config = FGNConfig(
|
||||
resolution=60, # 1° for Mini
|
||||
num_layers=12,
|
||||
model_dim=384,
|
||||
num_heads=6,
|
||||
mlp_ratio=4.0,
|
||||
patch_size=4,
|
||||
max_path_length=240,
|
||||
)
|
||||
|
||||
model = FGN(config)
|
||||
self.model = model
|
||||
self.params = params
|
||||
self.initialized = True
|
||||
print("Model loaded successfully")
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
print(f"Failed to load model: {e}")
|
||||
print("Using Open-Meteo WeatherNext API as fallback.")
|
||||
return False
|
||||
|
||||
def load_initial_state(self, source: str = "era5") -> Optional[Dict]:
|
||||
"""Load initial atmospheric state for model input.
|
||||
|
||||
source: 'era5' (reanalysis), 'hres' (operational), or 'gfs'
|
||||
"""
|
||||
import xarray as xr
|
||||
|
||||
try:
|
||||
if source == "era5":
|
||||
ds = self._load_era5_latest()
|
||||
elif source == "gfs":
|
||||
ds = self._load_gfs_latest()
|
||||
else:
|
||||
print(f"Unknown source: {source}")
|
||||
return None
|
||||
|
||||
return self._preprocess_for_model(ds)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Failed to load initial state: {e}")
|
||||
return None
|
||||
|
||||
def run_forecast(self, lead_hours: int = 120, steps: int = 20) -> Optional[pd.DataFrame]:
|
||||
"""Run an autoregressive forecast for Hong Kong region.
|
||||
|
||||
lead_hours: Total forecast hours
|
||||
steps: Number of model steps (at 6h per step for Mini)
|
||||
"""
|
||||
if not self.initialized:
|
||||
if not self.load_model():
|
||||
return None
|
||||
|
||||
import jax
|
||||
import jax.numpy as jnp
|
||||
|
||||
initial_state = self.load_initial_state("era5")
|
||||
if initial_state is None:
|
||||
return None
|
||||
|
||||
try:
|
||||
input_tensor = jnp.array(initial_state["fields"])
|
||||
|
||||
@jax.jit
|
||||
def step_fn(params, state):
|
||||
return self.model.apply({"params": params}, state)
|
||||
|
||||
results = []
|
||||
current = input_tensor
|
||||
for i in range(steps):
|
||||
current = step_fn(self.params, current)
|
||||
if (i + 1) * 6 <= lead_hours:
|
||||
results.append(self._extract_hk_region(np.array(current), i * 6 + 6))
|
||||
|
||||
return self._format_forecast_df(results)
|
||||
|
||||
except Exception as e:
|
||||
print(f"Model inference failed: {e}")
|
||||
print("Falling back to API-based forecasts.")
|
||||
return None
|
||||
|
||||
def _extract_hk_region(self, field: np.ndarray, lead_hour: int) -> Dict:
|
||||
"""Extract Hong Kong region data from global field.
|
||||
|
||||
With 1° resolution, HK is roughly a single grid cell.
|
||||
"""
|
||||
lat_idx = slice(
|
||||
int((HK_BBOX["lat_min"] + 90) / 1.0),
|
||||
int((HK_BBOX["lat_max"] + 90) / 1.0) + 1,
|
||||
)
|
||||
lon_idx = slice(
|
||||
int((HK_BBOX["lon_min"] + 180) / 1.0),
|
||||
int((HK_BBOX["lon_max"] + 180) / 1.0) + 1,
|
||||
)
|
||||
|
||||
# Placeholder - actual variable mapping depends on WeatherNext output channels
|
||||
hk_slice = field[..., lat_idx, lon_idx]
|
||||
return {
|
||||
"lead_hour": lead_hour,
|
||||
"temperature_2m_mean": float(np.mean(hk_slice[0]) if hk_slice.size > 0 else np.nan),
|
||||
"precipitation_mean": float(np.mean(hk_slice[-2]) if hk_slice.size > 1 else np.nan),
|
||||
}
|
||||
|
||||
def _format_forecast_df(self, results: list) -> pd.DataFrame:
|
||||
"""Format forecast results as DataFrame."""
|
||||
if not results:
|
||||
return pd.DataFrame()
|
||||
|
||||
df = pd.DataFrame(results)
|
||||
now = datetime.now()
|
||||
df["valid_time"] = [now + pd.Timedelta(hours=r["lead_hour"]) for r in results]
|
||||
return df.set_index("valid_time")
|
||||
|
||||
def _load_era5_latest(self):
|
||||
"""Load latest ERA5 data. Requires CDS API setup."""
|
||||
import xarray as xr
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
cds_ds = xr.open_dataset(
|
||||
"https://storage.googleapis.com/dm_graphcast/dataset/dataset_test.nc",
|
||||
engine="h5netcdf",
|
||||
)
|
||||
return cds_ds
|
||||
|
||||
def _load_gfs_latest(self):
|
||||
"""Load latest GFS analysis."""
|
||||
import xarray as xr
|
||||
from datetime import datetime
|
||||
|
||||
now = datetime.utcnow()
|
||||
url = f"https://nomads.ncep.noaa.gov/dods/gfs_0p25/gfs{now:%Y%m%d}/gfs_0p25_00z"
|
||||
try:
|
||||
return xr.open_dataset(url)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _preprocess_for_model(self, ds) -> Dict:
|
||||
"""Convert raw dataset to model input format."""
|
||||
required_vars = [
|
||||
"2m_temperature", "10m_u_component_of_wind", "10m_v_component_of_wind",
|
||||
"mean_sea_level_pressure", "geopotential", "specific_humidity",
|
||||
"temperature", "u_component_of_wind", "v_component_of_wind",
|
||||
]
|
||||
|
||||
fields = []
|
||||
for var in required_vars:
|
||||
if var in ds:
|
||||
fields.append(np.array(ds[var].isel(time=-1)))
|
||||
else:
|
||||
fields.append(np.zeros((721, 1440)))
|
||||
|
||||
return {
|
||||
"fields": np.stack(fields),
|
||||
"timestamp": str(ds.time.isel(time=-1).values),
|
||||
}
|
||||
@@ -0,0 +1,250 @@
|
||||
"""Open-Meteo client for WeatherNext and other weather model data."""
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Optional
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
try:
|
||||
import openmeteo_requests
|
||||
import requests_cache
|
||||
from retry_requests import retry
|
||||
except ImportError:
|
||||
import subprocess, sys
|
||||
subprocess.check_call([sys.executable, "-m", "pip", "install", "openmeteo-requests", "requests-cache", "retry-requests"])
|
||||
import openmeteo_requests
|
||||
import requests_cache
|
||||
from retry_requests import retry
|
||||
|
||||
from config import HK_COORDS, HK_BBOX, OPENMETEO_API_KEY
|
||||
|
||||
|
||||
class OpenMeteoClient:
|
||||
"""Fetch weather data from Open-Meteo including WeatherNext model outputs."""
|
||||
|
||||
BASE_URL = "https://api.open-meteo.com/v1/"
|
||||
|
||||
# Available weather models via Open-Meteo
|
||||
MODELS = {
|
||||
"weathernext": "google_weathernext",
|
||||
"ecmwf": "ecmwf_ifs04",
|
||||
"gfs": "gfs_seamless",
|
||||
}
|
||||
|
||||
def __init__(self, model: str = "weathernext", cache_ttl: int = 3600):
|
||||
cache = requests_cache.CachedSession('.openmeteo_cache', expire_after=cache_ttl)
|
||||
retry_session = retry(cache, retries=3, backoff_factor=0.2)
|
||||
self.client = openmeteo_requests.Client(session=retry_session)
|
||||
self.model = model
|
||||
self.params = {
|
||||
"latitude": HK_COORDS["hko_headquarters"][0],
|
||||
"longitude": HK_COORDS["hko_headquarters"][1],
|
||||
"timezone": "Asia/Hong_Kong",
|
||||
}
|
||||
|
||||
def get_forecast(self, lead_days: int = 7) -> Optional[pd.DataFrame]:
|
||||
"""Fetch forecast for Hong Kong from selected model."""
|
||||
params = {
|
||||
**self.params,
|
||||
"daily": [
|
||||
"temperature_2m_max",
|
||||
"temperature_2m_min",
|
||||
"temperature_2m_mean",
|
||||
"precipitation_sum",
|
||||
"precipitation_probability_max",
|
||||
"rain_sum",
|
||||
"wind_speed_10m_max",
|
||||
"wind_gusts_10m_max",
|
||||
"wind_direction_10m_dominant",
|
||||
"shortwave_radiation_sum",
|
||||
"et0_fao_evapotranspiration",
|
||||
"weather_code",
|
||||
],
|
||||
"hourly": [
|
||||
"temperature_2m",
|
||||
"relative_humidity_2m",
|
||||
"dew_point_2m",
|
||||
"apparent_temperature",
|
||||
"precipitation_probability",
|
||||
"precipitation",
|
||||
"rain",
|
||||
"cloud_cover",
|
||||
"cloud_cover_low",
|
||||
"cloud_cover_mid",
|
||||
"cloud_cover_high",
|
||||
"wind_speed_10m",
|
||||
"wind_speed_100m",
|
||||
"wind_gusts_10m",
|
||||
"wind_direction_10m",
|
||||
"wind_direction_100m",
|
||||
"surface_pressure",
|
||||
"visibility",
|
||||
],
|
||||
"past_days": 0,
|
||||
"forecast_days": lead_days,
|
||||
}
|
||||
|
||||
if OPENMETEO_API_KEY:
|
||||
params["apikey"] = OPENMETEO_API_KEY
|
||||
|
||||
try:
|
||||
responses = self.client.weather_api(self.BASE_URL + "forecast", params=params)
|
||||
return self._parse_response(responses[0])
|
||||
except Exception as e:
|
||||
print(f"Open-Meteo API error: {e}")
|
||||
return None
|
||||
|
||||
def _parse_response(self, response) -> pd.DataFrame:
|
||||
"""Parse Open-Meteo response into a DataFrame."""
|
||||
hourly = response.Hourly()
|
||||
daily = response.Daily()
|
||||
|
||||
self._hourly_vars = [
|
||||
"temperature_2m", "relative_humidity_2m", "dew_point_2m",
|
||||
"apparent_temperature", "precipitation_probability", "precipitation",
|
||||
"rain", "cloud_cover", "cloud_cover_low", "cloud_cover_mid",
|
||||
"cloud_cover_high", "wind_speed_10m", "wind_speed_100m",
|
||||
"wind_gusts_10m", "wind_direction_10m", "wind_direction_100m",
|
||||
"surface_pressure", "visibility",
|
||||
]
|
||||
self._daily_vars = [
|
||||
"temperature_2m_max", "temperature_2m_min", "temperature_2m_mean",
|
||||
"precipitation_sum", "precipitation_probability_max", "rain_sum",
|
||||
"wind_speed_10m_max", "wind_gusts_10m_max",
|
||||
"wind_direction_10m_dominant", "shortwave_radiation_sum",
|
||||
"et0_fao_evapotranspiration", "weather_code",
|
||||
]
|
||||
|
||||
hourly_data = {
|
||||
"date": pd.date_range(
|
||||
start=pd.Timestamp(hourly.Time(), unit="s", tz="UTC"),
|
||||
end=pd.Timestamp(hourly.TimeEnd(), unit="s", tz="UTC"),
|
||||
freq=pd.Timedelta(seconds=hourly.Interval()),
|
||||
inclusive="left",
|
||||
)
|
||||
}
|
||||
for i in range(hourly.VariablesLength()):
|
||||
var = hourly.Variables(i)
|
||||
if i < len(self._hourly_vars):
|
||||
hourly_data[self._hourly_vars[i]] = var.ValuesAsNumpy()
|
||||
|
||||
hourly_df = pd.DataFrame(hourly_data).set_index("date")
|
||||
hourly_df.index = hourly_df.index.tz_convert("Asia/Hong_Kong")
|
||||
|
||||
daily_data = {
|
||||
"date": pd.date_range(
|
||||
start=pd.Timestamp(daily.Time(), unit="s", tz="UTC"),
|
||||
end=pd.Timestamp(daily.TimeEnd(), unit="s", tz="UTC"),
|
||||
freq=pd.Timedelta(seconds=daily.Interval()),
|
||||
inclusive="left",
|
||||
)
|
||||
}
|
||||
for i in range(daily.VariablesLength()):
|
||||
var = daily.Variables(i)
|
||||
if i < len(self._daily_vars):
|
||||
daily_data[self._daily_vars[i]] = var.ValuesAsNumpy()
|
||||
|
||||
daily_df = pd.DataFrame(daily_data).set_index("date")
|
||||
daily_df.index = daily_df.index.tz_convert("Asia/Hong_Kong")
|
||||
|
||||
daily_df.attrs["model"] = self.model
|
||||
daily_df.attrs["hourly"] = hourly_df
|
||||
daily_df.attrs["fetch_time"] = datetime.now()
|
||||
|
||||
return daily_df
|
||||
|
||||
def get_current_conditions(self) -> dict:
|
||||
"""Get current weather conditions at HKO headquarters."""
|
||||
self._current_vars = [
|
||||
"temperature_2m", "relative_humidity_2m", "apparent_temperature",
|
||||
"precipitation", "rain", "cloud_cover", "wind_speed_10m",
|
||||
"wind_direction_10m", "wind_gusts_10m", "surface_pressure",
|
||||
"weather_code",
|
||||
]
|
||||
params = {
|
||||
**self.params,
|
||||
"current": self._current_vars,
|
||||
"forecast_days": 1,
|
||||
}
|
||||
|
||||
try:
|
||||
responses = self.client.weather_api(self.BASE_URL + "forecast", params=params)
|
||||
current = responses[0].Current()
|
||||
result = {"timestamp": datetime.now().isoformat()}
|
||||
for i in range(current.VariablesLength()):
|
||||
if i < len(self._current_vars):
|
||||
result[self._current_vars[i]] = current.Variables(i).Value()
|
||||
return {
|
||||
"temperature": result.get("temperature_2m"),
|
||||
"humidity": result.get("relative_humidity_2m"),
|
||||
"apparent_temp": result.get("apparent_temperature"),
|
||||
"precipitation": result.get("precipitation"),
|
||||
"rain": result.get("rain"),
|
||||
"cloud_cover": result.get("cloud_cover"),
|
||||
"wind_speed": result.get("wind_speed_10m"),
|
||||
"wind_direction": result.get("wind_direction_10m"),
|
||||
"wind_gusts": result.get("wind_gusts_10m"),
|
||||
"surface_pressure": result.get("surface_pressure"),
|
||||
"weather_code": result.get("weather_code"),
|
||||
"timestamp": datetime.now().isoformat(),
|
||||
}
|
||||
except Exception as e:
|
||||
print(f"Open-Meteo current conditions error: {e}")
|
||||
return {}
|
||||
|
||||
def get_precipitation_probability(self, hours_ahead: int = 24) -> float:
|
||||
"""Get precipitation probability for next N hours."""
|
||||
df = self.get_forecast(lead_days=2)
|
||||
if df is not None and hasattr(df, 'attrs') and 'hourly' in df.attrs:
|
||||
hourly = df.attrs['hourly']
|
||||
now = pd.Timestamp.now(tz="Asia/Hong_Kong")
|
||||
future = hourly[hourly.index <= now + pd.Timedelta(hours=hours_ahead)]
|
||||
if 'precipitation_probability' in future.columns:
|
||||
return float(future['precipitation_probability'].max())
|
||||
return 0.0
|
||||
|
||||
def get_scoring_window_summary(self, target_date: str) -> dict:
|
||||
"""
|
||||
Get a forecast summary for a specific scoring window (target date).
|
||||
Used by the signal generator to create trading signals.
|
||||
|
||||
Returns a dict with all relevant forecast variables for market comparison.
|
||||
"""
|
||||
df = self.get_forecast(lead_days=7)
|
||||
if df is None:
|
||||
return {}
|
||||
|
||||
target = pd.Timestamp(target_date).tz_localize("Asia/Hong_Kong")
|
||||
if target not in df.index:
|
||||
return {}
|
||||
|
||||
row = df.loc[target]
|
||||
hourly = df.attrs.get("hourly", pd.DataFrame())
|
||||
|
||||
if not hourly.empty:
|
||||
day_hourly = hourly[
|
||||
(hourly.index >= target) &
|
||||
(hourly.index < target + pd.Timedelta(days=1))
|
||||
]
|
||||
else:
|
||||
day_hourly = pd.DataFrame()
|
||||
|
||||
summary = {
|
||||
"date": target_date,
|
||||
"model": self.model,
|
||||
"fetch_time": df.attrs.get("fetch_time", datetime.now()).isoformat(),
|
||||
"temperature_2m_max": float(row.get("temperature_2m_max", np.nan)),
|
||||
"temperature_2m_min": float(row.get("temperature_2m_min", np.nan)),
|
||||
"precipitation_sum": float(row.get("precipitation_sum", 0)),
|
||||
"precipitation_probability_max": float(row.get("precipitation_probability_max", 0)),
|
||||
"wind_speed_10m_max": float(row.get("wind_speed_10m_max", np.nan)),
|
||||
"wind_gusts_10m_max": float(row.get("wind_gusts_10m_max", np.nan)),
|
||||
}
|
||||
|
||||
if not day_hourly.empty:
|
||||
summary["temperature_2m_max_hourly"] = float(day_hourly["temperature_2m"].max()) if "temperature_2m" in day_hourly else np.nan
|
||||
summary["precipitation_probability_max_hourly"] = float(day_hourly["precipitation_probability"].max()) if "precipitation_probability" in day_hourly else 0
|
||||
summary["precipitation_sum_hourly"] = float(day_hourly["precipitation"].sum()) if "precipitation" in day_hourly else 0
|
||||
|
||||
return summary
|
||||
Reference in New Issue
Block a user