"""Feature engineering for HK weather prediction from NWP model output. Transforms raw Open-Meteo daily/hourly forecast data into ML features. Features are designed to capture the physical processes driving HK weather: - Thermal: temperature, humidity, heat index - Dynamic: wind patterns, pressure gradients, shear - Moisture: precipitation, cloud cover, convergence - Temporal: day-over-day changes, seasonal cycles - Ensemble: multi-model disagreement Input: Open-Meteo daily + hourly DataFrames for HK region Output: numpy feature matrix with named columns """ import numpy as np import pandas as pd from datetime import datetime from typing import Dict, List, Optional, Tuple class FeatureEngine: """ Engineer features from NWP model output for ML training and inference. Usage: engine = FeatureEngine() X = engine.transform(om_daily_df, om_hourly_df) # X.shape = (n_days, n_features) """ # Feature groups for documentation and validation FEATURE_GROUPS = { "thermal": [ "t2m_max", "t2m_min", "t2m_mean", "t2m_range", "rh2m_mean", "rh2m_min", "apparent_t_max", "heat_index", "dewpoint_depression", ], "dynamic": [ "wind_speed_10m_max", "wind_gusts_10m_max", "wind_speed_100m_max", "wind_dir_10m_zonal", "wind_dir_10m_merid", "surface_pressure_mean", "pressure_tendency_24h", ], "moisture": [ "precip_sum", "precip_prob_max", "rain_sum", "cloud_cover_mean", "cloud_cover_low_mean", "cloud_cover_mid_mean", "cloud_cover_high_mean", "visibility_min", ], "temporal": [ "day_of_year_sin", "day_of_year_cos", "month_sin", "month_cos", "t2m_max_delta_24h", "t2m_min_delta_24h", "precip_prob_delta_24h", "pressure_delta_24h", ], "interaction": [ "temp_wind_interaction", "heat_humidity_index", "precip_wind_interaction", "storm_proxy", "convection_potential", ], } def __init__(self): self.feature_names: List[str] = [] def _build_feature_names(self): """Compile ordered feature name list.""" names = [] for group in self.FEATURE_GROUPS.values(): names.extend(group) self.feature_names = names def transform( self, daily: pd.DataFrame, hourly: Optional[pd.DataFrame] = None, ) -> np.ndarray: """ Transform NWP output into ML feature matrix. Parameters ---------- daily : pd.DataFrame Daily forecast with columns: temperature_2m_max, temperature_2m_min, temperature_2m_mean, precipitation_sum, precipitation_probability_max, rain_sum, wind_speed_10m_max, wind_gusts_10m_max, wind_direction_10m_dominant, shortwave_radiation_sum, et0_fao_evapotranspiration, weather_code hourly : pd.DataFrame, optional Hourly forecast with columns: temperature_2m, relative_humidity_2m, precipitation_probability, precipitation, rain, cloud_cover, cloud_cover_low, cloud_cover_mid, cloud_cover_high, wind_speed_10m, wind_speed_100m, wind_gusts_10m, wind_direction_10m, surface_pressure, visibility Returns ------- np.ndarray of shape (n_days, n_features) """ self._build_feature_names() features_list = [] for day_idx, (_, day_row) in enumerate(daily.iterrows()): day_features = {} if hourly is not None and not hourly.empty: # Get hourly slice for this day day_start = day_row.name.normalize() if hasattr(day_start, 'tz_localize'): day_start = day_start.tz_localize(None) day_end = day_start + pd.Timedelta(days=1) # Check if hourly index is tz-aware if hasattr(hourly.index, 'tz') and hourly.index.tz is not None: hourly_local = hourly.copy() hourly_local.index = hourly_local.index.tz_localize(None) else: hourly_local = hourly.copy() day_hourly = hourly_local[ (hourly_local.index >= day_start) & (hourly_local.index < day_end) ] else: day_hourly = pd.DataFrame() # === THERMAL FEATURES === day_features["t2m_max"] = float(day_row.get("temperature_2m_max", np.nan)) day_features["t2m_min"] = float(day_row.get("temperature_2m_min", np.nan)) day_features["t2m_mean"] = float(day_row.get("temperature_2m_mean", np.nan)) day_features["t2m_range"] = day_features["t2m_max"] - day_features["t2m_min"] if not day_hourly.empty: day_features["rh2m_mean"] = float(day_hourly["relative_humidity_2m"].mean()) if "relative_humidity_2m" in day_hourly else np.nan day_features["rh2m_min"] = float(day_hourly["relative_humidity_2m"].min()) if "relative_humidity_2m" in day_hourly else np.nan day_features["apparent_t_max"] = float(day_hourly["apparent_temperature"].max()) if "apparent_temperature" in day_hourly else np.nan else: day_features["rh2m_mean"] = np.nan day_features["rh2m_min"] = np.nan day_features["apparent_t_max"] = np.nan # Heat index (Steadman approximation, simplified) if not np.isnan(day_features.get("t2m_max", np.nan)) and not np.isnan(day_features.get("rh2m_mean", np.nan)): T = day_features["t2m_max"] RH = day_features["rh2m_mean"] day_features["heat_index"] = self._heat_index(T, RH) else: day_features["heat_index"] = np.nan # Dewpoint depression (T - Td, proxy for convection potential) day_features["dewpoint_depression"] = np.nan if not day_hourly.empty and "dew_point_2m" in day_hourly: dp = float(day_hourly["dew_point_2m"].mean()) T = float(day_hourly["temperature_2m"].mean()) if "temperature_2m" in day_hourly else np.nan if not np.isnan(T) and not np.isnan(dp): day_features["dewpoint_depression"] = T - dp # === DYNAMIC FEATURES === day_features["wind_speed_10m_max"] = float(day_row.get("wind_speed_10m_max", np.nan)) day_features["wind_gusts_10m_max"] = float(day_row.get("wind_gusts_10m_max", np.nan)) day_features["wind_speed_100m_max"] = np.nan if not day_hourly.empty: if "wind_speed_100m" in day_hourly: day_features["wind_speed_100m_max"] = float(day_hourly["wind_speed_100m"].max()) # Wind direction → zonal/meridional decomposition if "wind_direction_10m" in day_hourly: wd_mean = float(day_hourly["wind_direction_10m"].mean()) day_features["wind_dir_10m_zonal"] = -np.sin(np.radians(wd_mean)) day_features["wind_dir_10m_merid"] = -np.cos(np.radians(wd_mean)) else: day_features["wind_dir_10m_zonal"] = np.nan day_features["wind_dir_10m_merid"] = np.nan if "surface_pressure" in day_hourly: day_features["surface_pressure_mean"] = float(day_hourly["surface_pressure"].mean()) else: day_features["surface_pressure_mean"] = np.nan else: wd = float(day_row.get("wind_direction_10m_dominant", np.nan)) day_features["wind_dir_10m_zonal"] = -np.sin(np.radians(wd)) if not np.isnan(wd) else np.nan day_features["wind_dir_10m_merid"] = -np.cos(np.radians(wd)) if not np.isnan(wd) else np.nan day_features["surface_pressure_mean"] = np.nan day_features["pressure_tendency_24h"] = np.nan # Computed in post-processing # === MOISTURE FEATURES === day_features["precip_sum"] = float(day_row.get("precipitation_sum", 0)) day_features["precip_prob_max"] = float(day_row.get("precipitation_probability_max", 0)) day_features["rain_sum"] = float(day_row.get("rain_sum", 0)) if not day_hourly.empty: day_features["cloud_cover_mean"] = float(day_hourly["cloud_cover"].mean()) if "cloud_cover" in day_hourly else np.nan day_features["cloud_cover_low_mean"] = float(day_hourly["cloud_cover_low"].mean()) if "cloud_cover_low" in day_hourly else np.nan day_features["cloud_cover_mid_mean"] = float(day_hourly["cloud_cover_mid"].mean()) if "cloud_cover_mid" in day_hourly else np.nan day_features["cloud_cover_high_mean"] = float(day_hourly["cloud_cover_high"].mean()) if "cloud_cover_high" in day_hourly else np.nan day_features["visibility_min"] = float(day_hourly["visibility"].min()) if "visibility" in day_hourly else np.nan else: for c in ["cloud_cover_mean", "cloud_cover_low_mean", "cloud_cover_mid_mean", "cloud_cover_high_mean", "visibility_min"]: day_features[c] = np.nan # === TEMPORAL FEATURES === date = day_row.name if hasattr(date, 'to_pydatetime'): date = date.to_pydatetime() doy = date.timetuple().tm_yday day_features["day_of_year_sin"] = np.sin(2 * np.pi * doy / 365.25) day_features["day_of_year_cos"] = np.cos(2 * np.pi * doy / 365.25) day_features["month_sin"] = np.sin(2 * np.pi * date.month / 12) day_features["month_cos"] = np.cos(2 * np.pi * date.month / 12) # Deltas compute in post-processing day_features["t2m_max_delta_24h"] = np.nan day_features["t2m_min_delta_24h"] = np.nan day_features["precip_prob_delta_24h"] = np.nan day_features["pressure_delta_24h"] = np.nan # === INTERACTION FEATURES === if not np.isnan(day_features.get("t2m_max", np.nan)) and not np.isnan(day_features.get("wind_speed_10m_max", np.nan)): day_features["temp_wind_interaction"] = day_features["t2m_max"] * day_features["wind_speed_10m_max"] else: day_features["temp_wind_interaction"] = np.nan if not np.isnan(day_features.get("heat_index", np.nan)) and not np.isnan(day_features.get("rh2m_mean", np.nan)): day_features["heat_humidity_index"] = day_features["heat_index"] * day_features["rh2m_mean"] else: day_features["heat_humidity_index"] = np.nan if not np.isnan(day_features.get("precip_sum", np.nan)) and not np.isnan(day_features.get("wind_gusts_10m_max", np.nan)): day_features["precip_wind_interaction"] = day_features["precip_sum"] * day_features["wind_gusts_10m_max"] else: day_features["precip_wind_interaction"] = np.nan # Storm proxy: high wind + high precip + low pressure if not any(np.isnan(day_features[k]) for k in ["wind_gusts_10m_max", "precip_sum", "surface_pressure_mean"] if k in day_features): day_features["storm_proxy"] = ( day_features["wind_gusts_10m_max"] / 40.0 + day_features["precip_sum"] / 50.0 + (1013.0 - day_features["surface_pressure_mean"]) / 20.0 ) else: day_features["storm_proxy"] = np.nan # Convection potential: wind shear × instability proxy if not day_hourly.empty and "wind_speed_100m" in day_hourly and "wind_speed_10m" in day_hourly: shear = float(day_hourly["wind_speed_100m"].mean() - day_hourly["wind_speed_10m"].mean()) conv = day_features["precip_prob_max"] * (day_features["t2m_max"] - 20) / 20 if not np.isnan(day_features.get("t2m_max", np.nan)) else 0 day_features["convection_potential"] = shear * conv / 10.0 else: day_features["convection_potential"] = np.nan features_list.append(day_features) df_features = pd.DataFrame(features_list, columns=self.feature_names) # Post-processing: compute deltas if len(df_features) > 1: df_features["t2m_max_delta_24h"] = df_features["t2m_max"].diff() df_features["t2m_min_delta_24h"] = df_features["t2m_min"].diff() df_features["precip_prob_delta_24h"] = df_features["precip_prob_max"].diff() df_features["pressure_tendency_24h"] = df_features["surface_pressure_mean"].diff() # Fill remaining NaNs with column means (or 0) X = df_features.fillna(df_features.mean()).fillna(0).values return X.astype(np.float32) def transform_single( self, daily: pd.DataFrame, hourly: Optional[pd.DataFrame] = None, day_index: int = 0, ) -> np.ndarray: """Transform a single day's forecast into feature vector for inference.""" self._build_feature_names() if daily is None or len(daily) <= day_index: raise ValueError(f"Daily data has {len(daily)} rows, need day_index {day_index}") # Select single row + context start = max(0, day_index - 1) end = min(len(daily), day_index + 2) subset = daily.iloc[start:end] if hourly is not None: day_start = daily.index[day_index] day_end = day_start + pd.Timedelta(days=1) if hasattr(hourly.index, 'tz') and hourly.index.tz is not None: hourly_local = hourly.copy() hourly_local.index = hourly_local.index.tz_localize(None) else: hourly_local = hourly.copy() h_subset = hourly_local[ (hourly_local.index >= day_start) & (hourly_local.index < day_end) ] else: h_subset = None X = self.transform(subset, h_subset) return X[max(0, min(day_index, len(subset) - 1))].reshape(1, -1) @staticmethod def _heat_index(T: float, RH: float) -> float: """ Simplified heat index (Steadman, 1979). Valid for T > 27°C and RH > 40%. """ if T < 27 or RH < 40: return T c1, c2, c3 = -8.784695, 1.61139411, 2.338549 c4, c5, c6 = -0.14611605, -1.2308094e-2, -1.6424828e-2 c7, c8, c9 = 2.211732e-3, 7.2546e-4, -3.582e-6 HI = (c1 + c2 * T + c3 * RH + c4 * T * RH + c5 * T**2 + c6 * RH**2 + c7 * T**2 * RH + c8 * T * RH**2 + c9 * T**2 * RH**2) return HI @property def n_features(self) -> int: self._build_feature_names() return len(self.feature_names)