Fix ML calibration: logistic regression + Platt/isotonic + realistic NWP errors
Calibration overhaul: - Logistic regression mode for synthetic/bootstrap data (prevents LightGBM overfit) - 3-layer calibration stack: raw LR → Platt scaling → isotonic regression - Extreme probability smoothing: blend toward 0.5 when raw>0.95 or raw<0.05 - Platt preferred over isotonic (isotonic produces step functions with few points) - Continuous precipitation probability in bootstrap (beta distribution, not just 0/100) - Realistic NWP forecast errors: temp σ=2.0°C, rain calibration bias, diurnal-aware noise - Outlier injection: 10% of days have 2-3x larger errors (typhoon/low-pressure days) - LR model + StandardScaler saved as _lr.pkl alongside .lgb marker Results: - temp_gt_30c: AUC=0.987, Brier=0.049, predictions vary 20-85% per day - rain_gt_0mm: AUC=0.979, Brier=0.042, predictions vary 15-85% per day - temp_gt_35c: AUC=0.713 (realistic — extreme heat is hard to predict)
This commit is contained in:
ramseshk
+211
-203
@@ -1,19 +1,14 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Train LightGBM models for HK weather prediction targets.
|
||||
"""
|
||||
Train LightGBM models for HK weather prediction targets.
|
||||
|
||||
Uses ERA5 reanalysis data or Open-Meteo historical data to train
|
||||
probability models for rain, temperature, and wind thresholds.
|
||||
The training data simulates the relationship between NWP model forecasts
|
||||
and actual observations. NWP models have systematic errors:
|
||||
- Temperature: RMSE ~1.5°C at 24h lead
|
||||
- Precipitation probability: poor calibration, often overconfident
|
||||
- Wind: RMSE 3-5 km/h at 24h lead
|
||||
|
||||
Data preparation:
|
||||
Option 1 (ERA5): Requires CDS API setup. Downloads daily + hourly data.
|
||||
Option 2 (Synthetic bootstrap): Generate plausible training data from
|
||||
historical HK climate normals + Open-Meteo forecast structure.
|
||||
Option 3 (Open-Meteo archive): Use Open-Meteo historical weather API.
|
||||
|
||||
Usage:
|
||||
python ml/train.py # Train all models
|
||||
python ml/train.py --target temp_gt_30c_24h # Single target
|
||||
python ml/train.py --bootstrap # Bootstrap from climate normals
|
||||
The model learns to MAP noisy forecast features → binary outcome truth.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
@@ -33,253 +28,275 @@ from ml.model import WeatherModel, TARGET_DEFINITIONS, MODEL_DIR
|
||||
from config import HK_COORDS
|
||||
|
||||
|
||||
def bootstrap_training_data(n_samples: int = 5000) -> Tuple[pd.DataFrame, pd.DataFrame]:
|
||||
def _add_nwp_forecast_error(
|
||||
daily_truth: pd.DataFrame,
|
||||
hourly_truth: pd.DataFrame,
|
||||
rng: np.random.RandomState,
|
||||
) -> Tuple[pd.DataFrame, pd.DataFrame]:
|
||||
"""
|
||||
Generate synthetic training data from HK climate normals + variability.
|
||||
Add realistic NWP forecast errors to truth data.
|
||||
|
||||
This is a bootstrap approach when ERA5/Open-Meteo historical data isn't
|
||||
available. It samples from known HK climate distributions with realistic
|
||||
seasonal cycles, correlations, and day-to-day persistence.
|
||||
Returns (daily_forecast, hourly_forecast) simulating:
|
||||
- Temperature: RMSE 1.5-2.5°C, warm bias in summer anticyclones
|
||||
- Precipitation: continuous calibrated probabilities (not just 0/100),
|
||||
systematic overforecasting of light rain, underforecasting of heavy
|
||||
- Wind: multiplicative errors 0.7-1.5x
|
||||
- Cloud cover: RMSE 15-20%
|
||||
"""
|
||||
n_days = len(daily_truth)
|
||||
|
||||
While not as good as real reanalysis data, it:
|
||||
- Captures correct seasonal patterns (hot+wet summer, cool+dry winter)
|
||||
- Maintains realistic correlations (rain↔cloud↔temperature)
|
||||
- Includes meaningful day-to-day autocorrelation
|
||||
- Trains a model that can be replaced with real data later
|
||||
# === TEMPERATURE: larger noise for wider training distribution ===
|
||||
temp_error_max = rng.normal(0.3, 2.0, n_days) # μ=0.3 bias, σ=2.0 RMSE
|
||||
temp_error_min = rng.normal(0.2, 1.8, n_days)
|
||||
# Random injection of larger errors (10% of days have outlier errors)
|
||||
outlier_mask = rng.random(n_days) < 0.10
|
||||
temp_error_max[outlier_mask] += rng.normal(0, 3.0, outlier_mask.sum())
|
||||
temp_error_min[outlier_mask] += rng.normal(0, 2.5, outlier_mask.sum())
|
||||
|
||||
daily_fc = daily_truth.copy()
|
||||
if "temperature_2m_max" in daily_fc.columns:
|
||||
daily_fc["temperature_2m_max"] = np.clip(daily_truth["temperature_2m_max"] + temp_error_max, 5, 42)
|
||||
if "temperature_2m_min" in daily_fc.columns:
|
||||
daily_fc["temperature_2m_min"] = np.clip(daily_truth["temperature_2m_min"] + temp_error_min, 0, 33)
|
||||
|
||||
# === PRECIPITATION: continuous calibrated probabilities + multiplicative rain error ===
|
||||
# NWP models output continuous probabilities, not 0/100
|
||||
# Base prob from truth, then add calibration noise
|
||||
true_prob = daily_truth["precipitation_probability_max"].values / 100.0
|
||||
|
||||
# Systematic miscalibration: NWP overestimates low prob, underestimates high prob
|
||||
calibration_bias = 0.15 * (0.5 - true_prob) # +7.5% at prob=0, -7.5% at prob=1
|
||||
calibration_noise = rng.normal(0, 0.15, n_days)
|
||||
fc_prob = np.clip(true_prob + calibration_bias + calibration_noise, 0.01, 0.99)
|
||||
|
||||
if "precipitation_probability_max" in daily_fc.columns:
|
||||
daily_fc["precipitation_probability_max"] = fc_prob * 100.0
|
||||
|
||||
# Rain amount: multiplicative error, more noise on heavy rain
|
||||
rain_mult_error = np.where(
|
||||
daily_truth["precipitation_sum"] > 5,
|
||||
rng.lognormal(0, 0.4, n_days), # High variance for heavy rain
|
||||
rng.lognormal(0, 0.25, n_days), # Lower variance for light rain
|
||||
)
|
||||
if "precipitation_sum" in daily_fc.columns:
|
||||
daily_fc["precipitation_sum"] = daily_truth["precipitation_sum"] * rain_mult_error
|
||||
|
||||
# === WIND: multiplicative with 10% outlier days ===
|
||||
wind_mult = rng.lognormal(0, 0.20, n_days)
|
||||
gust_mult = rng.lognormal(0, 0.30, n_days)
|
||||
outlier_wind = rng.random(n_days) < 0.10
|
||||
wind_mult[outlier_wind] *= rng.uniform(1.3, 2.0, outlier_wind.sum())
|
||||
gust_mult[outlier_wind] *= rng.uniform(1.3, 2.5, outlier_wind.sum())
|
||||
|
||||
for col, mult in [("wind_speed_10m_max", wind_mult), ("wind_gusts_10m_max", gust_mult)]:
|
||||
if col in daily_fc.columns:
|
||||
daily_fc[col] = np.clip(daily_truth[col] * mult, 0, 200)
|
||||
|
||||
# === CLOUD COVER: systematic bias (underestimate in convective conditions) ===
|
||||
if "weather_code" in daily_fc.columns:
|
||||
daily_fc["weather_code"] = daily_truth["weather_code"]
|
||||
|
||||
# === HOURLY: larger noise ranges ===
|
||||
hourly_fc = hourly_truth.copy()
|
||||
n_hours = len(hourly_fc)
|
||||
|
||||
# Temperature: diurnal-cycle-aware errors (larger at night)
|
||||
hour_of_day = np.array([i % 24 for i in range(n_hours)])
|
||||
t_noise_scale = 1.2 + 0.8 * np.sin(2 * np.pi * (hour_of_day - 14) / 24) # Peak error at night
|
||||
if "temperature_2m" in hourly_fc.columns:
|
||||
hourly_fc["temperature_2m"] = np.clip(
|
||||
hourly_truth["temperature_2m"] + rng.normal(0, 2.0, n_hours) * t_noise_scale,
|
||||
-5, 45,
|
||||
)
|
||||
|
||||
# Humidity: large errors (NWP struggles with boundary layer moisture)
|
||||
if "relative_humidity_2m" in hourly_fc.columns:
|
||||
rh_err = rng.normal(-3, 12, n_hours)
|
||||
# More error during convective hours
|
||||
convective_mask = (hour_of_day > 11) & (hour_of_day < 19)
|
||||
rh_err[convective_mask] *= 1.5
|
||||
hourly_fc["relative_humidity_2m"] = np.clip(hourly_truth["relative_humidity_2m"] + rh_err, 15, 100)
|
||||
|
||||
# Cloud cover: large RMSE
|
||||
if "cloud_cover" in hourly_fc.columns:
|
||||
hourly_fc["cloud_cover"] = np.clip(hourly_truth["cloud_cover"] + rng.normal(0, 20, n_hours), 0, 100)
|
||||
for level in ["cloud_cover_low", "cloud_cover_mid", "cloud_cover_high"]:
|
||||
if level in hourly_fc.columns:
|
||||
hourly_fc[level] = np.clip(hourly_truth[level] + rng.normal(0, 15, n_hours), 0, 100)
|
||||
|
||||
# Pressure: typical errors
|
||||
if "surface_pressure" in hourly_fc.columns:
|
||||
hourly_fc["surface_pressure"] = hourly_truth["surface_pressure"] + rng.normal(0, 3.0, n_hours)
|
||||
|
||||
# Wind: multiplicative
|
||||
for col in ["wind_speed_10m", "wind_speed_100m", "wind_gusts_10m"]:
|
||||
if col in hourly_fc.columns:
|
||||
mult = rng.lognormal(0, 0.25, n_hours)
|
||||
hourly_fc[col] = hourly_truth[col] * mult
|
||||
|
||||
return daily_fc, hourly_fc
|
||||
|
||||
|
||||
def bootstrap_training_data(n_samples: int = 8000) -> Tuple[pd.DataFrame, pd.DataFrame, pd.DataFrame, pd.DataFrame]:
|
||||
"""
|
||||
Generate NWP forecast + observation training pairs.
|
||||
|
||||
Returns (daily_forecast, hourly_forecast, daily_truth, hourly_truth)
|
||||
where forecast has realistic NWP errors and truth is the actual observation.
|
||||
"""
|
||||
rng = np.random.RandomState(42)
|
||||
rng_noise = np.random.RandomState(99)
|
||||
|
||||
# Generate dates covering 10 years
|
||||
start_date = datetime(2015, 1, 1)
|
||||
dates = [start_date + timedelta(days=i) for i in range(n_samples)]
|
||||
|
||||
# HK seasonal cycles (sinusoidal with harmonics)
|
||||
doy = np.array([d.timetuple().tm_yday for d in dates])
|
||||
doy_sin = np.sin(2 * np.pi * doy / 365.25)
|
||||
doy_cos = np.cos(2 * np.pi * doy / 365.25)
|
||||
|
||||
# === TEMPERATURE ===
|
||||
# HK: mean Tmax 26°C, range 18-35°C, seasonal amplitude ~7°C
|
||||
tmax_base = 26.0 + 7.0 * np.sin(2 * np.pi * (doy - 200) / 365.25) # Peak Aug
|
||||
tmax = tmax_base + rng.normal(0, 2.0, n_samples)
|
||||
tmax = np.clip(tmax, 8, 38)
|
||||
|
||||
tmin = tmax - (7.0 + rng.exponential(2.0, n_samples)) # Diurnal range
|
||||
tmin = np.clip(tmin, 4, 30)
|
||||
|
||||
# === Generate OBSERVATION TRUTH (clean, no NWP error) ===
|
||||
tmax_base = 26.0 + 7.0 * np.sin(2 * np.pi * (doy - 200) / 365.25)
|
||||
tmax = np.clip(tmax_base + rng_noise.normal(0, 2.0, n_samples), 8, 38)
|
||||
tmin = np.clip(tmax - (7.0 + rng_noise.exponential(2.0, n_samples)), 4, 30)
|
||||
tmean = (tmax + tmin) / 2
|
||||
|
||||
# Apparent temperature (feels-like, always >= temp in HK humidity)
|
||||
apparent_t_max = tmax + rng.exponential(2.0, n_samples)
|
||||
apparent_t_max = np.clip(apparent_t_max, tmax, tmax + 12)
|
||||
rain_seasonal = 4.0 + 10.0 * np.maximum(0, np.sin(2 * np.pi * (doy - 172) / 365.25))
|
||||
rain_day_mask_prob = 0.3 + 0.4 * np.maximum(0, np.sin(2 * np.pi * (doy - 172) / 365.25))
|
||||
rain_day = rng_noise.random(n_samples) < rain_day_mask_prob
|
||||
rain_sum = np.where(rain_day, rng_noise.exponential(rain_seasonal, n_samples), 0)
|
||||
rain_sum = np.where(rain_sum < 0.1, 0, rain_sum)
|
||||
|
||||
# === HUMIDITY ===
|
||||
# HK: mean RH 78%, range 55-98%, lower in winter, higher in summer
|
||||
rh_base = 78 + 12 * doy_sin # Higher in summer
|
||||
rh_mean = rh_base + rng.normal(0, 6, n_samples)
|
||||
rh_mean = np.clip(rh_mean, 45, 98)
|
||||
# Continuous precipitation probability: beta distribution centered on actual prob
|
||||
from scipy.stats import beta as beta_dist
|
||||
precip_prob = np.zeros(n_samples)
|
||||
for i in range(n_samples):
|
||||
# Center the beta around the climatological rain prob
|
||||
p = rain_day_mask_prob[i]
|
||||
a = max(0.5, p * 8)
|
||||
b = max(0.5, (1 - p) * 8)
|
||||
precip_prob[i] = rng_noise.beta(a, b) * 100.0
|
||||
precip_prob = np.clip(precip_prob, 0.5, 99.5)
|
||||
|
||||
rh_min = rh_mean - rng.exponential(5, n_samples)
|
||||
rh_min = np.clip(rh_min, rh_mean - 30, rh_mean)
|
||||
|
||||
# Dewpoint (from temp and RH)
|
||||
dewpoint = tmean - ((100 - rh_mean) / 5.0) + rng.normal(0, 0.5, n_samples)
|
||||
dewpoint = np.clip(dewpoint, -5, 28)
|
||||
|
||||
# === PRECIPITATION ===
|
||||
# Rain: Poisson-like, strongly seasonal, zero-inflated
|
||||
rain_seasonal = 4.0 + 10.0 * np.maximum(0, doy_sin) # Peak summer
|
||||
rain_day_mask = rng.random(n_samples) < (0.3 + 0.4 * np.maximum(0, doy_sin))
|
||||
rain_sum = np.where(rain_day_mask, rng.exponential(rain_seasonal, n_samples), 0)
|
||||
rain_sum[rain_sum < 0.1] = 0 # Trace → 0
|
||||
|
||||
precip_prob = 100.0 * rain_day_mask + rng.normal(0, 5, n_samples)
|
||||
precip_prob = np.clip(precip_prob, 0, 100)
|
||||
|
||||
rain_minor_threshold = np.where(rain_sum > 1.0, rng.binomial(1, 0.6, n_samples), 0) # Heavy vs light
|
||||
|
||||
# === WIND ===
|
||||
# Wind: seasonal, typhoon-season peaks
|
||||
wind_base = 15 + 8 * np.maximum(0, np.sin(2 * np.pi * (doy - 180) / 365.25))
|
||||
wind_speed_max = wind_base + rng.exponential(5, n_samples)
|
||||
wind_speed_max = np.clip(wind_speed_max, 3, 120)
|
||||
wind_max = np.clip(wind_base + rng_noise.exponential(5, n_samples), 3, 120)
|
||||
gusts_max = np.clip(wind_max * (1.0 + rng_noise.exponential(0.5, n_samples)), wind_max, 200)
|
||||
|
||||
wind_gusts_max = wind_speed_max * (1.0 + rng.exponential(0.5, n_samples))
|
||||
wind_gusts_max = np.clip(wind_gusts_max, wind_speed_max, 200)
|
||||
wind_dir = rng_noise.uniform(0, 360, n_samples)
|
||||
cloud = np.clip(30 + rng_noise.beta(2, 3, n_samples) * 70 * (0.5 + 0.5 * (rain_sum > 0)), 0, 100)
|
||||
pressure = np.clip(1013 - 5 * np.sin(2 * np.pi * (doy - 172) / 365.25) + rng_noise.normal(0, 3, n_samples), 980, 1035)
|
||||
sw_rad = np.clip(5.0 + 10.0 * np.sin(2 * np.pi * (doy - 172) / 365.25) * (1 - cloud / 100) + rng_noise.normal(0, 2, n_samples), 0, 30)
|
||||
|
||||
wind_speed_100m_max = wind_speed_max * 1.3 + rng.normal(0, 2, n_samples)
|
||||
wind_speed_100m_max = np.clip(wind_speed_100m_max, wind_speed_max, wind_speed_max * 2.5)
|
||||
|
||||
wind_dir = rng.uniform(0, 360, n_samples)
|
||||
|
||||
# === CLOUD COVER ===
|
||||
cloud_cover = 30 + rng.beta(2, 3, n_samples) * 70
|
||||
cloud_cover = np.clip(cloud_cover, 0, 100)
|
||||
cloud_cover *= (0.5 + 0.5 * (rain_sum > 0)) # More clouds when raining
|
||||
|
||||
cloud_low = cloud_cover * rng.beta(2, 5, n_samples)
|
||||
cloud_mid = cloud_cover * rng.beta(2, 5, n_samples) * 0.5
|
||||
cloud_high = cloud_cover * rng.beta(2, 5, n_samples) * 0.3
|
||||
|
||||
# === PRESSURE ===
|
||||
# Mean sea level pressure: 1013 hPa ± seasonal
|
||||
pressure = 1013 - 5 * doy_sin + rng.normal(0, 3, n_samples)
|
||||
pressure = np.clip(pressure, 980, 1035)
|
||||
|
||||
# === VISIBILITY ===
|
||||
visibility = 15000 - rain_sum * 500 + rng.normal(0, 2000, n_samples)
|
||||
visibility = np.clip(visibility, 500, 25000)
|
||||
|
||||
# === SW RADIATION ===
|
||||
sw_rad = 5.0 + 10.0 * doy_sin * (1 - cloud_cover / 100) + rng.normal(0, 2, n_samples)
|
||||
sw_rad = np.clip(sw_rad, 0, 30)
|
||||
|
||||
# Build daily DataFrame
|
||||
daily_data = {
|
||||
"date": dates,
|
||||
daily_truth = pd.DataFrame({
|
||||
"temperature_2m_max": tmax,
|
||||
"temperature_2m_min": tmin,
|
||||
"temperature_2m_mean": tmean,
|
||||
"precipitation_sum": rain_sum,
|
||||
"precipitation_probability_max": precip_prob,
|
||||
"rain_sum": rain_sum,
|
||||
"wind_speed_10m_max": wind_speed_max,
|
||||
"wind_gusts_10m_max": wind_gusts_max,
|
||||
"wind_speed_10m_max": wind_max,
|
||||
"wind_gusts_10m_max": gusts_max,
|
||||
"wind_direction_10m_dominant": wind_dir,
|
||||
"shortwave_radiation_sum": sw_rad,
|
||||
"et0_fao_evapotranspiration": sw_rad * 0.4,
|
||||
"weather_code": np.where(rain_sum > 0, np.where(rain_sum > 10, 63, 61), 0),
|
||||
}
|
||||
daily = pd.DataFrame(daily_data).set_index("date")
|
||||
daily.index = pd.to_datetime(daily.index)
|
||||
}, index=pd.to_datetime(dates))
|
||||
|
||||
# Generate hourly data with diurnal cycles
|
||||
# Hourly truth
|
||||
hours_per_day = 24
|
||||
total_hours = n_samples * hours_per_day
|
||||
hour_timestamps = [start_date + timedelta(hours=i) for i in range(total_hours)]
|
||||
hour_of_day = np.tile(np.arange(24), n_samples)
|
||||
|
||||
# Diurnal temperature: sinusoid between tmin and tmax, peaking at 14:00
|
||||
day_indices = np.repeat(np.arange(n_samples), hours_per_day)
|
||||
t_range = np.repeat(tmax - tmin, hours_per_day)
|
||||
t_phase = 2 * np.pi * (hour_of_day - 14) / 24
|
||||
t_hourly = np.repeat(tmin, hours_per_day) + t_range * (0.5 + 0.5 * np.cos(t_phase)) + rng.normal(0, 0.5, total_hours)
|
||||
|
||||
# RH: inverse of temperature cycle
|
||||
rh_hourly = np.repeat(rh_mean, hours_per_day) - 5 * np.cos(t_phase) + rng.normal(0, 3, total_hours)
|
||||
rh_hourly = np.clip(rh_hourly, 20, 100)
|
||||
daily_rh = np.clip(78 + 12 * np.sin(2 * np.pi * (doy - 172) / 365.25) + rng_noise.normal(0, 6, n_samples), 45, 98)
|
||||
dewpoint = tmean - ((100 - daily_rh) / 5.0) + rng_noise.normal(0, 0.5, n_samples)
|
||||
|
||||
hourly_data = {
|
||||
"date": hour_timestamps,
|
||||
t_hourly = np.repeat(tmin, hours_per_day) + t_range * (0.5 + 0.5 * np.cos(t_phase)) + rng_noise.normal(0, 0.5, total_hours)
|
||||
rh_hourly = np.clip(np.repeat(daily_rh, hours_per_day) - 5 * np.cos(t_phase) + rng_noise.normal(0, 3, total_hours), 20, 100)
|
||||
|
||||
hour_timestamps = [start_date + timedelta(hours=i) for i in range(total_hours)]
|
||||
hourly_truth = pd.DataFrame({
|
||||
"temperature_2m": t_hourly,
|
||||
"relative_humidity_2m": rh_hourly,
|
||||
"dew_point_2m": np.repeat(dewpoint, hours_per_day) + rng.normal(0, 0.5, total_hours),
|
||||
"apparent_temperature": t_hourly + rng.exponential(2.0, total_hours),
|
||||
"precipitation_probability": np.repeat(precip_prob, hours_per_day) / 24 + rng.normal(0, 1, total_hours),
|
||||
"precipitation": np.repeat(rain_sum, hours_per_day) / 24 * rng.uniform(0.5, 1.5, total_hours),
|
||||
"dew_point_2m": np.repeat(dewpoint, hours_per_day) + rng_noise.normal(0, 0.5, total_hours),
|
||||
"apparent_temperature": t_hourly + rng_noise.exponential(2.0, total_hours),
|
||||
"precipitation_probability": np.clip(np.repeat(precip_prob, hours_per_day) / 24 + rng_noise.normal(0, 1, total_hours), 0, 100),
|
||||
"precipitation": np.repeat(rain_sum, hours_per_day) / 24 * rng_noise.uniform(0.5, 1.5, total_hours),
|
||||
"rain": np.repeat(rain_sum, hours_per_day) / 24,
|
||||
"cloud_cover": np.repeat(cloud_cover, hours_per_day) + rng.normal(0, 5, total_hours),
|
||||
"cloud_cover_low": np.repeat(cloud_low, hours_per_day),
|
||||
"cloud_cover_mid": np.repeat(cloud_mid, hours_per_day),
|
||||
"cloud_cover_high": np.repeat(cloud_high, hours_per_day),
|
||||
"wind_speed_10m": np.repeat(wind_speed_max, hours_per_day) * 0.5 * (0.5 + 0.5 * np.cos(t_phase)),
|
||||
"wind_speed_100m": np.repeat(wind_speed_100m_max, hours_per_day) * 0.6,
|
||||
"wind_gusts_10m": np.repeat(wind_gusts_max, hours_per_day) * (0.3 + 0.7 * rng.beta(2, 5, total_hours)),
|
||||
"wind_direction_10m": np.repeat(wind_dir, hours_per_day) + rng.normal(0, 10, total_hours),
|
||||
"surface_pressure": np.repeat(pressure, hours_per_day) + rng.normal(0, 0.5, total_hours),
|
||||
"visibility": np.repeat(visibility, hours_per_day) + rng.normal(0, 500, total_hours),
|
||||
}
|
||||
hourly = pd.DataFrame(hourly_data).set_index("date")
|
||||
hourly.index = pd.to_datetime(hourly.index)
|
||||
"cloud_cover": np.clip(np.repeat(cloud, hours_per_day) + rng_noise.normal(0, 5, total_hours), 0, 100),
|
||||
"cloud_cover_low": np.clip(np.repeat(cloud * 0.6, hours_per_day), 0, 100),
|
||||
"cloud_cover_mid": np.clip(np.repeat(cloud * 0.3, hours_per_day), 0, 100),
|
||||
"cloud_cover_high": np.clip(np.repeat(cloud * 0.2, hours_per_day), 0, 100),
|
||||
"wind_speed_10m": np.repeat(wind_max, hours_per_day) * 0.5 * (0.5 + 0.5 * np.cos(t_phase)),
|
||||
"wind_speed_100m": np.repeat(wind_max, hours_per_day) * 1.3 * 0.6,
|
||||
"wind_gusts_10m": np.repeat(gusts_max, hours_per_day) * (0.3 + 0.7 * rng_noise.beta(2, 5, total_hours)),
|
||||
"wind_direction_10m": np.repeat(wind_dir, hours_per_day) + rng_noise.normal(0, 10, total_hours),
|
||||
"surface_pressure": np.repeat(pressure, hours_per_day) + rng_noise.normal(0, 0.5, total_hours),
|
||||
"visibility": np.clip(15000 - np.repeat(rain_sum, hours_per_day) * 500 + rng_noise.normal(0, 2000, total_hours), 500, 25000),
|
||||
}, index=pd.to_datetime(hour_timestamps))
|
||||
|
||||
# Clip all values to realistic ranges
|
||||
hourly["cloud_cover"] = np.clip(hourly["cloud_cover"], 0, 100)
|
||||
hourly["cloud_cover_low"] = np.clip(hourly["cloud_cover_low"], 0, 100)
|
||||
hourly["cloud_cover_mid"] = np.clip(hourly["cloud_cover_mid"], 0, 100)
|
||||
hourly["cloud_cover_high"] = np.clip(hourly["cloud_cover_high"], 0, 100)
|
||||
hourly["visibility"] = np.clip(hourly["visibility"], 100, 30000)
|
||||
hourly["precipitation_probability"] = np.clip(hourly["precipitation_probability"], 0, 100)
|
||||
# === Add NWP forecast errors ===
|
||||
daily_fc, hourly_fc = _add_nwp_forecast_error(daily_truth.copy(), hourly_truth.copy(), rng)
|
||||
|
||||
return daily, hourly
|
||||
return daily_fc, hourly_fc, daily_truth, hourly_truth
|
||||
|
||||
|
||||
def train_targets(
|
||||
target_names: Optional[list] = None,
|
||||
n_bootstrap: int = 5000,
|
||||
test_split: float = 0.2,
|
||||
):
|
||||
"""Train all or selected target models."""
|
||||
def train_targets(target_names=None, n_bootstrap=8000, test_split=0.2):
|
||||
"""Train models with proper NWP forecast → observation mapping."""
|
||||
if target_names is None:
|
||||
target_names = list(TARGET_DEFINITIONS.keys())
|
||||
|
||||
print(f"Training {len(target_names)} models...")
|
||||
print(f"Bootstrap samples: {n_bootstrap} (test split: {test_split:.0%})")
|
||||
print(f"Training {len(target_names)} models with realistic NWP errors")
|
||||
print(f" Samples: {n_bootstrap} (test: {test_split:.0%})")
|
||||
print()
|
||||
|
||||
daily, hourly = bootstrap_training_data(n_bootstrap)
|
||||
daily_fc, hourly_fc, daily_truth, hourly_truth = bootstrap_training_data(n_bootstrap)
|
||||
|
||||
engine = FeatureEngine()
|
||||
X = engine.transform(daily, hourly)
|
||||
print(f"Features: {X.shape[1]} from {len(engine.FEATURE_GROUPS)} groups")
|
||||
print(f" Thermal: {len(engine.FEATURE_GROUPS['thermal'])}")
|
||||
print(f" Dynamic: {len(engine.FEATURE_GROUPS['dynamic'])}")
|
||||
print(f" Moisture: {len(engine.FEATURE_GROUPS['moisture'])}")
|
||||
print(f" Temporal: {len(engine.FEATURE_GROUPS['temporal'])}")
|
||||
print(f" Interaction: {len(engine.FEATURE_GROUPS['interaction'])}")
|
||||
print()
|
||||
# Features from FORECAST (noisy NWP output)
|
||||
X = engine.transform(daily_fc, hourly_fc)
|
||||
print(f"Features: {X.shape[1]} from NWP forecast output")
|
||||
|
||||
# Train/test split (temporal order, no shuffle)
|
||||
split_idx = int(len(daily) * (1 - test_split))
|
||||
split_idx = int(len(daily_fc) * (1 - test_split))
|
||||
X_train, X_test = X[:split_idx], X[split_idx:]
|
||||
daily_train, daily_test = daily.iloc[:split_idx], daily.iloc[split_idx:]
|
||||
daily_truth_train, daily_truth_test = daily_truth.iloc[:split_idx], daily_truth.iloc[split_idx:]
|
||||
|
||||
results = {}
|
||||
|
||||
# Feature augmentation: add Gaussian noise to prevent overfitting on synthetic data
|
||||
X_train_noisy = X_train + np.random.RandomState(42).normal(0, 0.1, X_train.shape).astype(np.float32)
|
||||
X_test_noisy = X_test + np.random.RandomState(43).normal(0, 0.05, X_test.shape).astype(np.float32)
|
||||
|
||||
for target_name in target_names:
|
||||
print(f"{'='*60}")
|
||||
print(f"Training: {target_name}")
|
||||
print(f" {TARGET_DEFINITIONS[target_name]['description']}")
|
||||
tdef = TARGET_DEFINITIONS[target_name]
|
||||
print(f"\n{'='*60}")
|
||||
print(f" {target_name} — {tdef['description']}")
|
||||
print(f"{'='*60}")
|
||||
|
||||
tdef = TARGET_DEFINITIONS[target_name]
|
||||
y_train = WeatherModel.build_target(
|
||||
daily_train, tdef["variable"], tdef["threshold"], tdef["op"]
|
||||
)
|
||||
y_test = WeatherModel.build_target(
|
||||
daily_test, tdef["variable"], tdef["threshold"], tdef["op"]
|
||||
)
|
||||
# Targets from TRUTH (actual observation)
|
||||
y_train = WeatherModel.build_target(daily_truth_train, tdef["variable"], tdef["threshold"], tdef["op"])
|
||||
y_test = WeatherModel.build_target(daily_truth_test, tdef["variable"], tdef["threshold"], tdef["op"])
|
||||
|
||||
p_yes = y_train.mean() * 100
|
||||
print(f" Class balance: {p_yes:.1f}% YES / {100-p_yes:.1f}% NO")
|
||||
|
||||
model = WeatherModel(target_name)
|
||||
model.train(X_train_noisy, y_train, X_test_noisy, y_test)
|
||||
model = WeatherModel(target_name, mode="lr")
|
||||
model.train(X_train, y_train, X_test, y_test)
|
||||
metrics = model.evaluate(X_test, y_test)
|
||||
model.save()
|
||||
|
||||
results[target_name] = metrics
|
||||
|
||||
print(f" Brier score: {metrics['brier_score']:.4f}")
|
||||
print(f" ROC AUC: {metrics['roc_auc']:.3f}")
|
||||
print(f" Predicted mean: {metrics['p_yes_predicted']:.1f}% (actual: {metrics['p_yes_actual']:.1f}%)")
|
||||
print(f" Top 10 features:")
|
||||
for feat, imp in list(model.top_features(10).items()):
|
||||
print(f" Pre-calibration Brier: — ")
|
||||
print(f" Post-calibration:")
|
||||
print(f" Brier: {metrics['brier_score']:.4f} AUC: {metrics['roc_auc']:.3f}")
|
||||
print(f" Predicted mean: {metrics['p_yes_predicted']:.1f}% Actual: {metrics['p_yes_actual']:.1f}%")
|
||||
print(f" Calibration: {metrics['calibration_method']}")
|
||||
print(f" Top 8 features:")
|
||||
for feat, imp in list(model.top_features(8).items()):
|
||||
print(f" {feat:30s} {imp:>10.1f}")
|
||||
print()
|
||||
|
||||
# Summary
|
||||
print(f"\n{'='*60}")
|
||||
print("TRAINING SUMMARY")
|
||||
print(f"{'='*60}")
|
||||
print(f"{'Target':<25s} {'Brier':>8s} {'ROC AUC':>8s} {'Cal Err %':>10s} {'Samples':>8s}")
|
||||
print("-" * 62)
|
||||
print(f"{'Target':<25s} {'Brier':>8s} {'AUC':>8s} {'Cal Err%':>9s} {'Cal':>10s}")
|
||||
print("-" * 65)
|
||||
for name, m in results.items():
|
||||
cal_err = abs(m["p_yes_predicted"] - m["p_yes_actual"])
|
||||
print(f"{name:<25s} {m['brier_score']:>8.4f} {m['roc_auc']:>8.3f} {cal_err:>10.1f} {m['n_samples']:>8d}")
|
||||
print(f"{name:<25s} {m['brier_score']:>8.4f} {m['roc_auc']:>8.3f} {cal_err:>9.1f} {m['calibration_method']:>10s}")
|
||||
|
||||
print(f"\nModels saved to: {MODEL_DIR}")
|
||||
return results
|
||||
@@ -287,30 +304,21 @@ def train_targets(
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Train HK weather prediction models")
|
||||
parser.add_argument("--target", type=str, default=None, help="Train single target (e.g., temp_gt_30c_24h)")
|
||||
parser.add_argument("--bootstrap", action="store_true", default=True, help="Use bootstrap training data")
|
||||
parser.add_argument("--samples", type=int, default=5000, help="Bootstrap sample count")
|
||||
parser.add_argument("--all", action="store_true", default=False, help="Train all targets")
|
||||
parser.add_argument("--target", type=str, default=None)
|
||||
parser.add_argument("--samples", type=int, default=8000)
|
||||
parser.add_argument("--all", action="store_true", default=False)
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.target:
|
||||
targets = [args.target]
|
||||
elif args.all:
|
||||
targets = list(TARGET_DEFINITIONS.keys())
|
||||
else:
|
||||
targets = list(TARGET_DEFINITIONS.keys())
|
||||
|
||||
targets = [args.target] if args.target else list(TARGET_DEFINITIONS.keys())
|
||||
results = train_targets(targets, n_bootstrap=args.samples)
|
||||
|
||||
# Save summary
|
||||
MODEL_DIR.mkdir(parents=True, exist_ok=True)
|
||||
summary_path = MODEL_DIR / "training_summary.json"
|
||||
with open(summary_path, "w") as f:
|
||||
with open(MODEL_DIR / "training_summary.json", "w") as f:
|
||||
json.dump({
|
||||
"training_date": datetime.now().isoformat(),
|
||||
"n_bootstrap_samples": args.samples,
|
||||
"n_samples": args.samples,
|
||||
"results": results,
|
||||
}, f, indent=2)
|
||||
}, f, indent=2, default=str)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user