ramseshk
7d7a67bd20
Tier 1 ML enhancements: - Feature engineering (37 features across 5 groups: thermal, dynamic, moisture, temporal, interaction) from NWP model output - 7 LightGBM probability models for rain/temp/wind thresholds - Temperature-scaled probabilities to prevent overconfidence on bootstrap data - MLPredictor: unified inference pipeline replacing heuristic sigmoids - Ensemble disagreement signals (composite spread → edge amplification) - Fixed calibration loop: update_calibration() now functional (EMA of errors) - record_outcome() wired for post-resolution feedback - Nautilus strategy updated: ML predictions take priority, heuristics as fallback - Historical backtest engine with Sharpe/ROI/max-DD simulation - Bootstrap training data generator from HK climate normals Run: python ml/train.py && python ml/backtest.py --edge 50
323 lines
15 KiB
Python
323 lines
15 KiB
Python
"""Feature engineering for HK weather prediction from NWP model output.
|
||
|
||
Transforms raw Open-Meteo daily/hourly forecast data into ML features.
|
||
Features are designed to capture the physical processes driving HK weather:
|
||
- Thermal: temperature, humidity, heat index
|
||
- Dynamic: wind patterns, pressure gradients, shear
|
||
- Moisture: precipitation, cloud cover, convergence
|
||
- Temporal: day-over-day changes, seasonal cycles
|
||
- Ensemble: multi-model disagreement
|
||
|
||
Input: Open-Meteo daily + hourly DataFrames for HK region
|
||
Output: numpy feature matrix with named columns
|
||
"""
|
||
|
||
import numpy as np
|
||
import pandas as pd
|
||
from datetime import datetime
|
||
from typing import Dict, List, Optional, Tuple
|
||
|
||
|
||
class FeatureEngine:
|
||
"""
|
||
Engineer features from NWP model output for ML training and inference.
|
||
|
||
Usage:
|
||
engine = FeatureEngine()
|
||
X = engine.transform(om_daily_df, om_hourly_df)
|
||
# X.shape = (n_days, n_features)
|
||
"""
|
||
|
||
# Feature groups for documentation and validation
|
||
FEATURE_GROUPS = {
|
||
"thermal": [
|
||
"t2m_max", "t2m_min", "t2m_mean", "t2m_range",
|
||
"rh2m_mean", "rh2m_min", "apparent_t_max",
|
||
"heat_index", "dewpoint_depression",
|
||
],
|
||
"dynamic": [
|
||
"wind_speed_10m_max", "wind_gusts_10m_max",
|
||
"wind_speed_100m_max", "wind_dir_10m_zonal",
|
||
"wind_dir_10m_merid", "surface_pressure_mean",
|
||
"pressure_tendency_24h",
|
||
],
|
||
"moisture": [
|
||
"precip_sum", "precip_prob_max", "rain_sum",
|
||
"cloud_cover_mean", "cloud_cover_low_mean",
|
||
"cloud_cover_mid_mean", "cloud_cover_high_mean",
|
||
"visibility_min",
|
||
],
|
||
"temporal": [
|
||
"day_of_year_sin", "day_of_year_cos",
|
||
"month_sin", "month_cos",
|
||
"t2m_max_delta_24h", "t2m_min_delta_24h",
|
||
"precip_prob_delta_24h", "pressure_delta_24h",
|
||
],
|
||
"interaction": [
|
||
"temp_wind_interaction", "heat_humidity_index",
|
||
"precip_wind_interaction", "storm_proxy",
|
||
"convection_potential",
|
||
],
|
||
}
|
||
|
||
def __init__(self):
|
||
self.feature_names: List[str] = []
|
||
|
||
def _build_feature_names(self):
|
||
"""Compile ordered feature name list."""
|
||
names = []
|
||
for group in self.FEATURE_GROUPS.values():
|
||
names.extend(group)
|
||
self.feature_names = names
|
||
|
||
def transform(
|
||
self,
|
||
daily: pd.DataFrame,
|
||
hourly: Optional[pd.DataFrame] = None,
|
||
) -> np.ndarray:
|
||
"""
|
||
Transform NWP output into ML feature matrix.
|
||
|
||
Parameters
|
||
----------
|
||
daily : pd.DataFrame
|
||
Daily forecast with columns: temperature_2m_max, temperature_2m_min,
|
||
temperature_2m_mean, precipitation_sum, precipitation_probability_max,
|
||
rain_sum, wind_speed_10m_max, wind_gusts_10m_max,
|
||
wind_direction_10m_dominant, shortwave_radiation_sum,
|
||
et0_fao_evapotranspiration, weather_code
|
||
hourly : pd.DataFrame, optional
|
||
Hourly forecast with columns: temperature_2m, relative_humidity_2m,
|
||
precipitation_probability, precipitation, rain, cloud_cover,
|
||
cloud_cover_low, cloud_cover_mid, cloud_cover_high, wind_speed_10m,
|
||
wind_speed_100m, wind_gusts_10m, wind_direction_10m, surface_pressure,
|
||
visibility
|
||
|
||
Returns
|
||
-------
|
||
np.ndarray of shape (n_days, n_features)
|
||
"""
|
||
self._build_feature_names()
|
||
features_list = []
|
||
|
||
for day_idx, (_, day_row) in enumerate(daily.iterrows()):
|
||
day_features = {}
|
||
|
||
if hourly is not None and not hourly.empty:
|
||
# Get hourly slice for this day
|
||
day_start = day_row.name.normalize()
|
||
if hasattr(day_start, 'tz_localize'):
|
||
day_start = day_start.tz_localize(None)
|
||
day_end = day_start + pd.Timedelta(days=1)
|
||
|
||
# Check if hourly index is tz-aware
|
||
if hasattr(hourly.index, 'tz') and hourly.index.tz is not None:
|
||
hourly_local = hourly.copy()
|
||
hourly_local.index = hourly_local.index.tz_localize(None)
|
||
else:
|
||
hourly_local = hourly.copy()
|
||
|
||
day_hourly = hourly_local[
|
||
(hourly_local.index >= day_start) &
|
||
(hourly_local.index < day_end)
|
||
]
|
||
else:
|
||
day_hourly = pd.DataFrame()
|
||
|
||
# === THERMAL FEATURES ===
|
||
day_features["t2m_max"] = float(day_row.get("temperature_2m_max", np.nan))
|
||
day_features["t2m_min"] = float(day_row.get("temperature_2m_min", np.nan))
|
||
day_features["t2m_mean"] = float(day_row.get("temperature_2m_mean", np.nan))
|
||
day_features["t2m_range"] = day_features["t2m_max"] - day_features["t2m_min"]
|
||
|
||
if not day_hourly.empty:
|
||
day_features["rh2m_mean"] = float(day_hourly["relative_humidity_2m"].mean()) if "relative_humidity_2m" in day_hourly else np.nan
|
||
day_features["rh2m_min"] = float(day_hourly["relative_humidity_2m"].min()) if "relative_humidity_2m" in day_hourly else np.nan
|
||
day_features["apparent_t_max"] = float(day_hourly["apparent_temperature"].max()) if "apparent_temperature" in day_hourly else np.nan
|
||
else:
|
||
day_features["rh2m_mean"] = np.nan
|
||
day_features["rh2m_min"] = np.nan
|
||
day_features["apparent_t_max"] = np.nan
|
||
|
||
# Heat index (Steadman approximation, simplified)
|
||
if not np.isnan(day_features.get("t2m_max", np.nan)) and not np.isnan(day_features.get("rh2m_mean", np.nan)):
|
||
T = day_features["t2m_max"]
|
||
RH = day_features["rh2m_mean"]
|
||
day_features["heat_index"] = self._heat_index(T, RH)
|
||
else:
|
||
day_features["heat_index"] = np.nan
|
||
|
||
# Dewpoint depression (T - Td, proxy for convection potential)
|
||
day_features["dewpoint_depression"] = np.nan
|
||
if not day_hourly.empty and "dew_point_2m" in day_hourly:
|
||
dp = float(day_hourly["dew_point_2m"].mean())
|
||
T = float(day_hourly["temperature_2m"].mean()) if "temperature_2m" in day_hourly else np.nan
|
||
if not np.isnan(T) and not np.isnan(dp):
|
||
day_features["dewpoint_depression"] = T - dp
|
||
|
||
# === DYNAMIC FEATURES ===
|
||
day_features["wind_speed_10m_max"] = float(day_row.get("wind_speed_10m_max", np.nan))
|
||
day_features["wind_gusts_10m_max"] = float(day_row.get("wind_gusts_10m_max", np.nan))
|
||
day_features["wind_speed_100m_max"] = np.nan
|
||
|
||
if not day_hourly.empty:
|
||
if "wind_speed_100m" in day_hourly:
|
||
day_features["wind_speed_100m_max"] = float(day_hourly["wind_speed_100m"].max())
|
||
|
||
# Wind direction → zonal/meridional decomposition
|
||
if "wind_direction_10m" in day_hourly:
|
||
wd_mean = float(day_hourly["wind_direction_10m"].mean())
|
||
day_features["wind_dir_10m_zonal"] = -np.sin(np.radians(wd_mean))
|
||
day_features["wind_dir_10m_merid"] = -np.cos(np.radians(wd_mean))
|
||
else:
|
||
day_features["wind_dir_10m_zonal"] = np.nan
|
||
day_features["wind_dir_10m_merid"] = np.nan
|
||
|
||
if "surface_pressure" in day_hourly:
|
||
day_features["surface_pressure_mean"] = float(day_hourly["surface_pressure"].mean())
|
||
else:
|
||
day_features["surface_pressure_mean"] = np.nan
|
||
else:
|
||
wd = float(day_row.get("wind_direction_10m_dominant", np.nan))
|
||
day_features["wind_dir_10m_zonal"] = -np.sin(np.radians(wd)) if not np.isnan(wd) else np.nan
|
||
day_features["wind_dir_10m_merid"] = -np.cos(np.radians(wd)) if not np.isnan(wd) else np.nan
|
||
day_features["surface_pressure_mean"] = np.nan
|
||
|
||
day_features["pressure_tendency_24h"] = np.nan # Computed in post-processing
|
||
|
||
# === MOISTURE FEATURES ===
|
||
day_features["precip_sum"] = float(day_row.get("precipitation_sum", 0))
|
||
day_features["precip_prob_max"] = float(day_row.get("precipitation_probability_max", 0))
|
||
day_features["rain_sum"] = float(day_row.get("rain_sum", 0))
|
||
|
||
if not day_hourly.empty:
|
||
day_features["cloud_cover_mean"] = float(day_hourly["cloud_cover"].mean()) if "cloud_cover" in day_hourly else np.nan
|
||
day_features["cloud_cover_low_mean"] = float(day_hourly["cloud_cover_low"].mean()) if "cloud_cover_low" in day_hourly else np.nan
|
||
day_features["cloud_cover_mid_mean"] = float(day_hourly["cloud_cover_mid"].mean()) if "cloud_cover_mid" in day_hourly else np.nan
|
||
day_features["cloud_cover_high_mean"] = float(day_hourly["cloud_cover_high"].mean()) if "cloud_cover_high" in day_hourly else np.nan
|
||
day_features["visibility_min"] = float(day_hourly["visibility"].min()) if "visibility" in day_hourly else np.nan
|
||
else:
|
||
for c in ["cloud_cover_mean", "cloud_cover_low_mean", "cloud_cover_mid_mean", "cloud_cover_high_mean", "visibility_min"]:
|
||
day_features[c] = np.nan
|
||
|
||
# === TEMPORAL FEATURES ===
|
||
date = day_row.name
|
||
if hasattr(date, 'to_pydatetime'):
|
||
date = date.to_pydatetime()
|
||
doy = date.timetuple().tm_yday
|
||
day_features["day_of_year_sin"] = np.sin(2 * np.pi * doy / 365.25)
|
||
day_features["day_of_year_cos"] = np.cos(2 * np.pi * doy / 365.25)
|
||
day_features["month_sin"] = np.sin(2 * np.pi * date.month / 12)
|
||
day_features["month_cos"] = np.cos(2 * np.pi * date.month / 12)
|
||
|
||
# Deltas compute in post-processing
|
||
day_features["t2m_max_delta_24h"] = np.nan
|
||
day_features["t2m_min_delta_24h"] = np.nan
|
||
day_features["precip_prob_delta_24h"] = np.nan
|
||
day_features["pressure_delta_24h"] = np.nan
|
||
|
||
# === INTERACTION FEATURES ===
|
||
if not np.isnan(day_features.get("t2m_max", np.nan)) and not np.isnan(day_features.get("wind_speed_10m_max", np.nan)):
|
||
day_features["temp_wind_interaction"] = day_features["t2m_max"] * day_features["wind_speed_10m_max"]
|
||
else:
|
||
day_features["temp_wind_interaction"] = np.nan
|
||
|
||
if not np.isnan(day_features.get("heat_index", np.nan)) and not np.isnan(day_features.get("rh2m_mean", np.nan)):
|
||
day_features["heat_humidity_index"] = day_features["heat_index"] * day_features["rh2m_mean"]
|
||
else:
|
||
day_features["heat_humidity_index"] = np.nan
|
||
|
||
if not np.isnan(day_features.get("precip_sum", np.nan)) and not np.isnan(day_features.get("wind_gusts_10m_max", np.nan)):
|
||
day_features["precip_wind_interaction"] = day_features["precip_sum"] * day_features["wind_gusts_10m_max"]
|
||
else:
|
||
day_features["precip_wind_interaction"] = np.nan
|
||
|
||
# Storm proxy: high wind + high precip + low pressure
|
||
if not any(np.isnan(day_features[k]) for k in ["wind_gusts_10m_max", "precip_sum", "surface_pressure_mean"] if k in day_features):
|
||
day_features["storm_proxy"] = (
|
||
day_features["wind_gusts_10m_max"] / 40.0 +
|
||
day_features["precip_sum"] / 50.0 +
|
||
(1013.0 - day_features["surface_pressure_mean"]) / 20.0
|
||
)
|
||
else:
|
||
day_features["storm_proxy"] = np.nan
|
||
|
||
# Convection potential: wind shear × instability proxy
|
||
if not day_hourly.empty and "wind_speed_100m" in day_hourly and "wind_speed_10m" in day_hourly:
|
||
shear = float(day_hourly["wind_speed_100m"].mean() - day_hourly["wind_speed_10m"].mean())
|
||
conv = day_features["precip_prob_max"] * (day_features["t2m_max"] - 20) / 20 if not np.isnan(day_features.get("t2m_max", np.nan)) else 0
|
||
day_features["convection_potential"] = shear * conv / 10.0
|
||
else:
|
||
day_features["convection_potential"] = np.nan
|
||
|
||
features_list.append(day_features)
|
||
|
||
df_features = pd.DataFrame(features_list, columns=self.feature_names)
|
||
|
||
# Post-processing: compute deltas
|
||
if len(df_features) > 1:
|
||
df_features["t2m_max_delta_24h"] = df_features["t2m_max"].diff()
|
||
df_features["t2m_min_delta_24h"] = df_features["t2m_min"].diff()
|
||
df_features["precip_prob_delta_24h"] = df_features["precip_prob_max"].diff()
|
||
df_features["pressure_tendency_24h"] = df_features["surface_pressure_mean"].diff()
|
||
|
||
# Fill remaining NaNs with column means (or 0)
|
||
X = df_features.fillna(df_features.mean()).fillna(0).values
|
||
|
||
return X.astype(np.float32)
|
||
|
||
def transform_single(
|
||
self,
|
||
daily: pd.DataFrame,
|
||
hourly: Optional[pd.DataFrame] = None,
|
||
day_index: int = 0,
|
||
) -> np.ndarray:
|
||
"""Transform a single day's forecast into feature vector for inference."""
|
||
self._build_feature_names()
|
||
|
||
if daily is None or len(daily) <= day_index:
|
||
raise ValueError(f"Daily data has {len(daily)} rows, need day_index {day_index}")
|
||
|
||
# Select single row + context
|
||
start = max(0, day_index - 1)
|
||
end = min(len(daily), day_index + 2)
|
||
subset = daily.iloc[start:end]
|
||
|
||
if hourly is not None:
|
||
day_start = daily.index[day_index]
|
||
day_end = day_start + pd.Timedelta(days=1)
|
||
if hasattr(hourly.index, 'tz') and hourly.index.tz is not None:
|
||
hourly_local = hourly.copy()
|
||
hourly_local.index = hourly_local.index.tz_localize(None)
|
||
else:
|
||
hourly_local = hourly.copy()
|
||
h_subset = hourly_local[
|
||
(hourly_local.index >= day_start) & (hourly_local.index < day_end)
|
||
]
|
||
else:
|
||
h_subset = None
|
||
|
||
X = self.transform(subset, h_subset)
|
||
return X[max(0, min(day_index, len(subset) - 1))].reshape(1, -1)
|
||
|
||
@staticmethod
|
||
def _heat_index(T: float, RH: float) -> float:
|
||
"""
|
||
Simplified heat index (Steadman, 1979).
|
||
Valid for T > 27°C and RH > 40%.
|
||
"""
|
||
if T < 27 or RH < 40:
|
||
return T
|
||
c1, c2, c3 = -8.784695, 1.61139411, 2.338549
|
||
c4, c5, c6 = -0.14611605, -1.2308094e-2, -1.6424828e-2
|
||
c7, c8, c9 = 2.211732e-3, 7.2546e-4, -3.582e-6
|
||
HI = (c1 + c2 * T + c3 * RH + c4 * T * RH +
|
||
c5 * T**2 + c6 * RH**2 + c7 * T**2 * RH +
|
||
c8 * T * RH**2 + c9 * T**2 * RH**2)
|
||
return HI
|
||
|
||
@property
|
||
def n_features(self) -> int:
|
||
self._build_feature_names()
|
||
return len(self.feature_names)
|