kemal/ashvale-station
1
0
Fork 0
mirror of https://github.com/lynchaos/ashvale-station.git synced 2026-09-12 20:52:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

Initial release: Ashvale Station 1.0.0

Browse Source
This commit is contained in:
kemal
2026-08-15 20:43:51 +01:00
commit 06ce53bc44
36 changed files with 7116 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ashvale/models/__init__.py
+24
View File
@@ -0,0 +1,24 @@
# Copyright 2026 Kemal Yaylali
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from .rls import RecursiveLeastSquares, AdaptiveConformal
from .nowcast import NowcastEnsemble
from .climatology import HarmonicClimatology
from .precip import PrecipitationModel, zambretti
from .anomaly import AnomalyMonitor
__all__ = [
"RecursiveLeastSquares", "AdaptiveConformal", "NowcastEnsemble",
"HarmonicClimatology", "PrecipitationModel", "zambretti", "AnomalyMonitor",
]
ashvale/models/anomaly.py
+287
View File
@@ -0,0 +1,287 @@
# Copyright 2026 Kemal Yaylali
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Anomaly and drift monitoring: the part that keeps the rest honest.
Three independent detectors, because they fail in different ways:
`MahalanobisEWMA` Multivariate novelty on the residual from a slowly
updated mean and shrinkage covariance. Catches a
window opening, a heater cycling, or a genuine squall.
`PageHinkley` Sequential change-point detection on model error.
Catches the slow stuff: a sensor drifting, a season
turning, a model quietly going stale. This is the
detector that tells you *when to retrain*, which is a
far better trigger than a cron schedule.
`SensorHealth` Latched values, out-of-range readings, and Kalman
innovation inflation. A stuck sensor is invisible to
the other two because it looks perfectly normal.
Shrinkage on the covariance is not optional here. With 6 signals and a
1000-sample window the sample covariance is fine, but during the first
hour it is singular, and a singular covariance turns Mahalanobis
distance into a random number generator with an authoritative name.
"""
from __future__ import annotations
from collections import deque
from typing import Deque, Dict, List, Optional
import numpy as np
SIGNALS = ["temp_c", "hum", "press_slp", "temp_rate", "press_rate", "dew_c"]
class MahalanobisEWMA:
def __init__(self, n_dims: int, lam: float = 0.15, threshold: float = 12.0,
shrinkage: float = 0.15, warmup: int = 60):
self.d = int(n_dims)
self.lam = float(lam)
self.threshold = float(threshold)
self.shrinkage = float(shrinkage)
self.warmup = int(warmup)
self.mean = np.zeros(self.d)
self.cov = np.eye(self.d)
self.z = np.zeros(self.d) # EWMA of standardised residual
self.n = 0
self.last_d2 = 0.0
def update(self, x: np.ndarray) -> Dict:
x = np.asarray(x, dtype=float).ravel()
if x.size != self.d or not np.all(np.isfinite(x)):
return {"d2": self.last_d2, "alarm": False, "warm": self.n < self.warmup}
self.n += 1
if self.n == 1:
self.mean = x.copy()
return {"d2": 0.0, "alarm": False, "warm": True}
a = 1.0 / min(self.n, 500) # slow adaptation once warm
delta = x - self.mean
self.mean += a * delta
self.cov = (1 - a) * self.cov + a * np.outer(delta, delta)
# Ledoit-Wolf style shrinkage toward a scaled identity
target = np.eye(self.d) * (np.trace(self.cov) / self.d + 1e-9)
cov = (1 - self.shrinkage) * self.cov + self.shrinkage * target
try:
resid = np.linalg.solve(cov, delta)
except np.linalg.LinAlgError:
return {"d2": self.last_d2, "alarm": False, "warm": True}
# EWMA on the whitened residual gives persistence-aware detection:
# one odd sample is noise, ten in a row is an event.
white = delta / np.sqrt(np.maximum(np.diag(cov), 1e-12))
self.z = (1 - self.lam) * self.z + self.lam * white
scale = self.lam / (2 - self.lam)
d2_ewma = float(self.z @ self.z / max(scale, 1e-9))
d2_inst = float(delta @ resid)
self.last_d2 = d2_ewma
warm = self.n < self.warmup
return {
"d2": d2_ewma,
"d2_instant": d2_inst,
"alarm": (not warm) and d2_ewma > self.threshold,
"warm": warm,
"contributions": {s: round(float(v), 2) for s, v in zip(SIGNALS[:self.d], white)},
}
def to_dict(self) -> Dict:
return {"d": self.d, "lam": self.lam, "threshold": self.threshold,
"shrinkage": self.shrinkage, "warmup": self.warmup,
"mean": self.mean.tolist(), "cov": self.cov.tolist(),
"z": self.z.tolist(), "n": self.n}
@classmethod
def from_dict(cls, s: Dict) -> "MahalanobisEWMA":
m = cls(s["d"], s["lam"], s["threshold"], s["shrinkage"], s["warmup"])
m.mean = np.array(s["mean"], float)
m.cov = np.array(s["cov"], float)
m.z = np.array(s["z"], float)
m.n = s["n"]
return m
class PageHinkley:
"""Two-sided sequential change detection on a stream of errors."""
def __init__(self, delta: float = 0.05, lam: float = 8.0, alpha: float = 0.999):
self.delta = float(delta)
self.lam = float(lam)
self.alpha = float(alpha)
self.mean = 0.0
self.n = 0
self.m_pos = 0.0
self.m_neg = 0.0
self.n_alarms = 0
self.last_alarm_ts: Optional[float] = None
def update(self, value: float, ts: Optional[float] = None) -> bool:
v = float(value)
if not np.isfinite(v):
return False
self.n += 1
self.mean += (v - self.mean) / self.n
self.m_pos = self.alpha * max(0.0, self.m_pos + v - self.mean - self.delta)
self.m_neg = self.alpha * max(0.0, self.m_neg - v + self.mean - self.delta)
if self.n > 30 and max(self.m_pos, self.m_neg) > self.lam:
self.reset_statistics()
self.n_alarms += 1
self.last_alarm_ts = ts
return True
return False
def reset_statistics(self) -> None:
self.m_pos = 0.0
self.m_neg = 0.0
self.n = 1
@property
def stress(self) -> float:
"""0 to 1: how close we are to declaring drift. Nice on a gauge."""
return float(min(max(self.m_pos, self.m_neg) / max(self.lam, 1e-9), 1.0))
def to_dict(self) -> Dict:
return {"delta": self.delta, "lam": self.lam, "alpha": self.alpha,
"mean": self.mean, "n": self.n, "m_pos": self.m_pos,
"m_neg": self.m_neg, "n_alarms": self.n_alarms,
"last_alarm_ts": self.last_alarm_ts}
@classmethod
def from_dict(cls, s: Dict) -> "PageHinkley":
p = cls(s["delta"], s["lam"], s["alpha"])
p.__dict__.update({k: s[k] for k in
("mean", "n", "m_pos", "m_neg", "n_alarms", "last_alarm_ts")})
return p
class SensorHealth:
RANGES = {
"temp_c": (-40.0, 85.0),
"hum": (0.0, 100.0),
"press_slp": (870.0, 1085.0),
"cpu_temp": (-20.0, 95.0),
}
def __init__(self, window: int = 90):
self.buffers: Dict[str, Deque[float]] = {
k: deque(maxlen=window) for k in self.RANGES
}
self.flags: Dict[str, str] = {}
def update(self, obs: Dict[str, float]) -> Dict[str, Dict]:
report = {}
for name, (lo, hi) in self.RANGES.items():
v = obs.get(name)
if v is None or not np.isfinite(v):
report[name] = {"status": "missing", "detail": "no reading"}
continue
buf = self.buffers[name]
buf.append(float(v))
arr = np.asarray(buf, dtype=float)
if not (lo <= v <= hi):
status, detail = "fault", f"out of range ({v:.2f})"
elif arr.size >= 20 and float(np.max(np.abs(np.diff(arr)))) < 1e-9:
status, detail = "fault", "value latched, sensor may be stuck"
elif arr.size >= 20 and float(np.std(arr)) < 1e-6:
status, detail = "warn", "near-zero variance"
else:
status, detail = "ok", "nominal"
report[name] = {"status": status, "detail": detail,
"value": float(v), "std": float(np.std(arr)) if arr.size > 2 else 0.0}
self.flags = {k: v["status"] for k, v in report.items()}
return report
@property
def overall(self) -> str:
if any(v == "fault" for v in self.flags.values()):
return "fault"
if any(v == "warn" for v in self.flags.values()):
return "warn"
return "ok"
class AnomalyMonitor:
"""Facade over the three detectors, with a rolling event log."""
def __init__(self, cfg_model):
self.novelty = MahalanobisEWMA(
len(SIGNALS), cfg_model.anomaly_ewma_lambda, cfg_model.anomaly_threshold
)
self.drift = PageHinkley(cfg_model.drift_delta, cfg_model.drift_lambda)
self.health = SensorHealth()
self.events: Deque[Dict] = deque(maxlen=100)
self.retrain_requested = False
def observe(self, ts: float, obs: Dict[str, float]) -> Dict:
vec = np.array([obs.get(s, np.nan) for s in SIGNALS], dtype=float)
nov = self.novelty.update(vec)
health = self.health.update(obs)
if nov.get("alarm"):
top = max(nov.get("contributions", {}).items(),
key=lambda kv: abs(kv[1]), default=("unknown", 0.0))
self._log(ts, "novelty", "warn",
f"multivariate departure d2={nov['d2']:.1f}, led by {top[0]}")
for name, rep in health.items():
if rep["status"] == "fault":
self._log(ts, "sensor", "error", f"{name}: {rep['detail']}")
return {
"novelty": nov,
"health": health,
"health_overall": self.health.overall,
"drift": {
"stress": self.drift.stress,
"alarms": self.drift.n_alarms,
"last_alarm_ts": self.drift.last_alarm_ts,
"retrain_requested": self.retrain_requested,
},
}
def observe_error(self, ts: float, abs_error: float) -> bool:
"""Feed a matured forecast error; returns True if drift was declared."""
fired = self.drift.update(abs_error, ts)
if fired:
self.retrain_requested = True
self._log(ts, "drift", "warn",
"forecast error distribution shifted, retrain queued")
return fired
def clear_retrain_flag(self) -> None:
self.retrain_requested = False
def _log(self, ts: float, kind: str, severity: str, detail: str) -> None:
self.events.append({"ts": ts, "kind": kind, "severity": severity, "detail": detail})
def recent(self, n: int = 20) -> List[Dict]:
return list(self.events)[-n:][::-1]
def to_dict(self) -> Dict:
return {"novelty": self.novelty.to_dict(), "drift": self.drift.to_dict(),
"events": list(self.events), "retrain_requested": self.retrain_requested}
def load_dict(self, s: Dict) -> None:
self.novelty = MahalanobisEWMA.from_dict(s["novelty"])
self.drift = PageHinkley.from_dict(s["drift"])
self.events = deque(s.get("events", []), maxlen=100)
self.retrain_requested = s.get("retrain_requested", False)
ashvale/models/climatology.py
+170
View File
@@ -0,0 +1,170 @@
# Copyright 2026 Kemal Yaylali
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Harmonic regression: the long-range half of the forecast.
An honest statement first, because a weather product that oversells
itself is worse than no product. A single point sensor cannot see a
front approaching from the Atlantic. Beyond roughly twelve hours, the
only information your station holds is:
* where in the diurnal cycle you are,
* where in the annual cycle you are,
* the current synoptic pressure anomaly and its tendency,
* the local trend of the last few days.
So that is exactly what this model uses. It is a ridge-regularised
Fourier basis in time-of-day and day-of-year, plus a slow linear trend
and a pressure-anomaly coupling. Days 2 to 7 are a *climatological
outlook with an anomaly correction*, not a forecast, and the API labels
them as such. Anything more confident would be theatre.
The annual harmonics only switch on once the station has enough history
to identify them (`climatology_min_days_annual`, default 120). Before
that, fitting a 365-day sine to three weeks of data produces a
magnificent extrapolation straight off the edge of the physical world.
"""
from __future__ import annotations
from typing import Dict, List, Optional
import numpy as np
DAY = 86400.0
YEAR = 365.2422 * DAY
class HarmonicClimatology:
def __init__(self, targets, diurnal_harmonics: int = 3,
annual_harmonics: int = 2, ridge: float = 1.0,
min_days_annual: float = 120.0):
self.targets = tuple(targets)
self.kd = int(diurnal_harmonics)
self.ka = int(annual_harmonics)
self.ridge = float(ridge)
self.min_days_annual = float(min_days_annual)
self.coef: Dict[str, np.ndarray] = {}
self.resid_std: Dict[str, float] = {}
self.t0: float = 0.0
self.use_annual = False
self.n_days = 0.0
self.ready = False
# ---------------------------------------------------------- basis
def _design(self, ts: np.ndarray) -> np.ndarray:
ts = np.atleast_1d(np.asarray(ts, dtype=float))
t_days = (ts - self.t0) / DAY
cols = [np.ones(ts.size), t_days / 30.0] # slow trend, per month
for k in range(1, self.kd + 1):
w = 2 * np.pi * k * ts / DAY
cols += [np.sin(w), np.cos(w)]
if self.use_annual:
for k in range(1, self.ka + 1):
w = 2 * np.pi * k * ts / YEAR
cols += [np.sin(w), np.cos(w)]
return np.column_stack(cols)
# ------------------------------------------------------------ fit
def fit(self, ts: np.ndarray, series: Dict[str, np.ndarray],
valid: Optional[np.ndarray] = None) -> Dict[str, float]:
ts = np.asarray(ts, dtype=float)
if ts.size < 48:
self.ready = False
return {}
self.t0 = float(ts[0])
self.n_days = float((ts[-1] - ts[0]) / DAY)
self.use_annual = self.n_days >= self.min_days_annual
A = self._design(ts)
mask = np.ones(ts.size, dtype=bool) if valid is None else valid.astype(bool)
out = {}
for target in self.targets:
y = np.asarray(series.get(target, np.empty(0)), dtype=float)
if y.size != ts.size:
continue
m = mask & np.isfinite(y)
if m.sum() < A.shape[1] * 3:
continue
Am, ym = A[m], y[m]
# ridge: leave the intercept unpenalised
reg = np.eye(A.shape[1]) * self.ridge
reg[0, 0] = 0.0
beta = np.linalg.solve(Am.T @ Am + reg, Am.T @ ym)
self.coef[target] = beta
resid = ym - Am @ beta
self.resid_std[target] = float(np.std(resid))
out[target] = self.resid_std[target]
self.ready = bool(self.coef)
return out
# -------------------------------------------------------- predict
def predict(self, target: str, ts: np.ndarray) -> np.ndarray:
ts = np.atleast_1d(np.asarray(ts, dtype=float))
beta = self.coef.get(target)
if beta is None:
return np.zeros(ts.size)
return self._design(ts) @ beta
def outlook(self, target: str, now: float, days: int = 7,
step_s: int = 3 * 3600, anomaly: float = 0.0,
anomaly_halflife_h: float = 30.0) -> List[Dict]:
"""Climatology plus an exponentially decaying current anomaly.
The anomaly term is what makes this better than a textbook: if
today is 3 C above the seasonal norm, tomorrow morning probably
still is, and next Thursday almost certainly is not. The decay
half-life encodes exactly that intuition, and the interval widens
with the square root of lead time as any diffusive process should.
"""
if not self.ready or target not in self.coef:
return []
grid = np.arange(now, now + days * DAY, step_s, dtype=float)
base = self.predict(target, grid)
lead_h = (grid - now) / 3600.0
decay = 0.5 ** (lead_h / max(anomaly_halflife_h, 1e-3))
mu = base + anomaly * decay
sigma0 = self.resid_std.get(target, 1.0)
sigma = sigma0 * np.sqrt(1.0 + lead_h / 24.0)
return [
{"ts": float(t), "lead_h": float(l), "mu": float(m),
"lo": float(m - 1.645 * s), "hi": float(m + 1.645 * s)}
for t, l, m, s in zip(grid, lead_h, mu, sigma)
]
def anomaly_now(self, target: str, ts: float, observed: float) -> float:
if not self.ready or target not in self.coef:
return 0.0
return float(observed - self.predict(target, np.array([ts]))[0])
def to_dict(self) -> Dict:
return {"targets": list(self.targets), "kd": self.kd, "ka": self.ka,
"ridge": self.ridge, "min_days_annual": self.min_days_annual,
"t0": self.t0, "use_annual": self.use_annual, "n_days": self.n_days,
"coef": {k: v.tolist() for k, v in self.coef.items()},
"resid_std": self.resid_std, "ready": self.ready}
def load_dict(self, s: Dict) -> None:
self.kd, self.ka = s["kd"], s["ka"]
self.ridge = s["ridge"]
self.min_days_annual = s["min_days_annual"]
self.t0 = s["t0"]
self.use_annual = s["use_annual"]
self.n_days = s.get("n_days", 0.0)
self.coef = {k: np.array(v, dtype=float) for k, v in s["coef"].items()}
self.resid_std = s["resid_std"]
self.ready = s["ready"]
ashvale/models/nowcast.py
+245
View File
@@ -0,0 +1,245 @@
# Copyright 2026 Kemal Yaylali
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Multi-horizon forecasting: one direct head per (target, horizon).
Direct rather than recursive. A recursive one-step model iterated 288
times to reach 24 hours compounds its own bias into a beautifully smooth
lie. Direct heads cost more memory (six horizons x three targets = 18
small models, about 150 kB total) and are worth every byte.
Each head predicts a *delta from now*, then the ensemble blends three
opinions with weights that are themselves learned online:
persistence : it will be exactly as it is now
climatology : it will be whatever this hour of this day usually is
learned RLS : it will be now plus what the regressors imply
Persistence wins at 15 minutes. Climatology wins at 24 hours. The RLS
head wins in the middle, which is exactly the region a physical
forecaster finds hardest. The blend weights are updated by exponentiated
gradient (Hedge), so the ensemble is never worse than its best member by
more than a log factor, and it re-weights itself within a day when the
season turns.
"""
from __future__ import annotations
from typing import Dict, List, Optional, Tuple
import numpy as np
from ..features import N_FEATURES, Standardiser, supervised_pairs
from .rls import AdaptiveConformal, RecursiveLeastSquares
MEMBERS = ("persistence", "climatology", "learned")
class ForecastHead:
"""One target, one horizon."""
def __init__(self, target: str, horizon_s: int, n_features: int = N_FEATURES,
forgetting: float = 0.9985, delta: float = 100.0,
alpha: float = 0.10, conformal_window: int = 400,
gamma: float = 0.01, hedge_eta: float = 0.35):
self.target = target
self.horizon_s = int(horizon_s)
self.model = RecursiveLeastSquares(n_features, forgetting, delta)
self.conformal = AdaptiveConformal(alpha, conformal_window, gamma)
self.weights = np.ones(len(MEMBERS)) / len(MEMBERS)
self.eta = float(hedge_eta)
self.member_mae = np.zeros(len(MEMBERS))
self.n_scored = 0
# -------------------------------------------------------- prediction
def predict(self, x: np.ndarray, anchor: float,
climatology_delta: float = 0.0) -> Dict[str, float]:
learned_delta = self.model.predict(x)
deltas = np.array([0.0, float(climatology_delta), float(learned_delta)])
blended = float(np.dot(self.weights, deltas))
mu = float(anchor + blended)
sigma = self.model.predict_std(x, self.model.noise_var)
lo, hi = self.conformal.interval(mu, fallback_sigma=sigma)
return {
"mu": mu,
"lo": lo,
"hi": hi,
"sigma": sigma,
"delta": blended,
"members": {m: float(anchor + d) for m, d in zip(MEMBERS, deltas)},
"weights": {m: float(w) for m, w in zip(MEMBERS, self.weights)},
}
# ---------------------------------------------------------- learning
def learn(self, x: np.ndarray, anchor: float, truth: float,
climatology_delta: float = 0.0) -> float:
"""One supervised step given a matured target."""
deltas = np.array([0.0, float(climatology_delta),
float(self.model.predict(x))])
member_pred = anchor + deltas
losses = np.abs(member_pred - truth)
# Hedge / exponentiated gradient on normalised losses
scale = max(float(np.max(losses)), 1e-6)
self.weights *= np.exp(-self.eta * losses / scale)
self.weights = np.clip(self.weights, 1e-4, None)
self.weights /= self.weights.sum()
blended = float(np.dot(self.weights, member_pred))
residual = truth - blended
self.conformal.observe(residual)
self.model.update(x, truth - anchor)
self.member_mae = 0.98 * self.member_mae + 0.02 * losses
self.n_scored += 1
return residual
def to_dict(self) -> Dict:
return {"target": self.target, "horizon_s": self.horizon_s,
"model": self.model.to_dict(), "conformal": self.conformal.to_dict(),
"weights": self.weights.tolist(), "eta": self.eta,
"member_mae": self.member_mae.tolist(), "n_scored": self.n_scored}
@classmethod
def from_dict(cls, s: Dict) -> "ForecastHead":
h = cls(s["target"], s["horizon_s"])
h.model = RecursiveLeastSquares.from_dict(s["model"])
h.conformal = AdaptiveConformal.from_dict(s["conformal"])
h.weights = np.array(s["weights"], dtype=float)
h.eta = s["eta"]
h.member_mae = np.array(s["member_mae"], dtype=float)
h.n_scored = s.get("n_scored", 0)
return h
class NowcastEnsemble:
"""The full bank of heads plus the shared feature standardiser."""
def __init__(self, targets: Tuple[str, ...], horizons_s: Tuple[int, ...],
cfg_model):
self.targets = tuple(targets)
self.horizons = tuple(int(h) for h in horizons_s)
self.grid_s = int(cfg_model.grid_s)
self.scaler = Standardiser(N_FEATURES)
self.heads: Dict[Tuple[str, int], ForecastHead] = {
(t, h): ForecastHead(
t, h, N_FEATURES, cfg_model.rls_forgetting, cfg_model.rls_delta,
cfg_model.conformal_alpha, cfg_model.conformal_window,
cfg_model.conformal_gamma,
)
for t in self.targets for h in self.horizons
}
self.trained_rows = 0
# ------------------------------------------------------------ train
def fit(self, X: np.ndarray, valid: np.ndarray, series: Dict[str, np.ndarray],
climatology=None, grid_ts: Optional[np.ndarray] = None,
passes: int = 1, max_pairs: int = 2500) -> Dict[str, int]:
"""Batch-update every head from history.
`max_pairs` bounds the work per head to the most recent samples.
This is not a shortcut: with a forgetting factor of 0.9985 the
effective memory is about 11 hours, so the 4000th-most-recent
sample carries a weight of roughly e^-6. Training on it costs
real seconds on a Cortex-A53 and buys nothing measurable.
"""
"""Batch pass over history. Called on startup and every retrain tick."""
if X.shape[0] < 10:
return {"rows": 0}
self.scaler.partial_fit(X[valid][:: max(1, X.shape[0] // 2000)])
Xs = self.scaler.transform(X)
counts = {}
for target in self.targets:
y = series[target]
for h in self.horizons:
steps = max(int(round(h / self.grid_s)), 1)
Xa, dy, anchor = supervised_pairs(Xs, valid, y, steps)
if Xa.shape[0] < 5:
counts[f"{target}@{h}"] = 0
continue
if Xa.shape[0] > max_pairs:
Xa, dy, anchor = Xa[-max_pairs:], dy[-max_pairs:], anchor[-max_pairs:]
head = self.heads[(target, h)]
clim = np.zeros(Xa.shape[0])
if climatology is not None and grid_ts is not None and climatology.ready:
n = grid_ts.size
ts_a = grid_ts[:n - steps]
mask_len = min(ts_a.size, Xa.shape[0])
clim_now = climatology.predict(target, ts_a[-mask_len:])
clim_fut = climatology.predict(target, ts_a[-mask_len:] + h)
clim = np.zeros(Xa.shape[0])
clim[-mask_len:] = clim_fut - clim_now
for _ in range(max(int(passes), 1)):
for i in range(Xa.shape[0]):
head.learn(Xa[i], anchor[i], anchor[i] + dy[i], clim[i])
counts[f"{target}@{h}"] = int(Xa.shape[0])
self.trained_rows = int(X.shape[0])
return counts
# --------------------------------------------------------- inference
def forecast(self, x_raw: np.ndarray, anchors: Dict[str, float], now: float,
climatology=None) -> Dict[str, Dict[int, Dict[str, float]]]:
x = self.scaler.transform(np.atleast_2d(x_raw))[0]
out: Dict[str, Dict[int, Dict[str, float]]] = {}
for target in self.targets:
anchor = float(anchors.get(target, 0.0))
out[target] = {}
for h in self.horizons:
clim_delta = 0.0
if climatology is not None and climatology.ready:
clim_delta = float(climatology.predict(target, np.array([now + h]))[0]
- climatology.predict(target, np.array([now]))[0])
out[target][h] = self.heads[(target, h)].predict(x, anchor, clim_delta)
return out
def diagnostics(self) -> List[Dict]:
rows = []
for (target, h), head in sorted(self.heads.items()):
rows.append({
"target": target,
"horizon_s": h,
"n_updates": head.model.n_updates,
"n_scored": head.n_scored,
"weights": {m: round(float(w), 3) for m, w in zip(MEMBERS, head.weights)},
"member_mae": {m: round(float(v), 3) for m, v in zip(MEMBERS, head.member_mae)},
"conformal_alpha": round(head.conformal.alpha, 4),
"conformal_halfwidth": round(float(head.conformal.quantile()), 3)
if np.isfinite(head.conformal.quantile()) else None,
"coverage": round(head.conformal.empirical_coverage, 3)
if np.isfinite(head.conformal.empirical_coverage) else None,
})
return rows
def to_dict(self) -> Dict:
return {
"targets": list(self.targets),
"horizons": list(self.horizons),
"grid_s": self.grid_s,
"scaler": self.scaler.to_dict(),
"heads": [h.to_dict() for h in self.heads.values()],
"trained_rows": self.trained_rows,
}
def load_dict(self, s: Dict) -> None:
self.scaler = Standardiser.from_dict(s["scaler"])
for hs in s["heads"]:
head = ForecastHead.from_dict(hs)
self.heads[(head.target, head.horizon_s)] = head
self.trained_rows = s.get("trained_rows", 0)
ashvale/models/precip.py
+323
View File
@@ -0,0 +1,323 @@
# Copyright 2026 Kemal Yaylali
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Will it rain? A prior with a hundred years of service, plus a learner.
Two components, deliberately:
1. `zambretti()` is the 1915 Negretti and Zambra slide-rule algorithm,
re-expressed here in the standard three-branch form. It needs only
sea-level pressure, its tendency and the season. It has no parameters
to overfit, it works from the first hour of deployment, and in the
temperate maritime climate it was designed for it is genuinely hard
to beat with a small dataset. It is the prior.
2. `PrecipitationModel` is an online logistic regression that learns the
residual: what your specific location does that the slide rule does
not know. It starts from the Zambretti logit and only earns influence
as labels accumulate, so it cannot embarrass you on day one.
Labels are the hard part, and the design is explicit about it. Without a
rain gauge, a *proxy* label is used (near-saturated air with a collapsing
dew-point depression), and it is flagged as weak. `POST /api/label` lets
you supply ground truth from a window: two seconds of your attention is
worth a week of proxy labels, and the learner weights them accordingly.
"""
from __future__ import annotations
import math
import time
from typing import Dict, List, Optional, Tuple
import numpy as np
# Severity classes the Z number maps onto. Wording is ours, not the
# original card's, and is deliberately about actionable state rather
# than Edwardian poetry.
_CONDITIONS = [
(1, 2, "settled", "Settled and dry"),
(3, 5, "fine", "Fine, little change expected"),
(6, 8, "fair", "Fair, becoming less settled"),
(9, 12, "changeable", "Changeable, showers possible"),
(13, 16, "unsettled", "Unsettled, rain at times"),
(17, 20, "rain", "Rain likely, turning wet"),
(21, 23, "wet", "Wet and windy"),
(24, 26, "stormy", "Stormy, heavy rain likely"),
]
_RAIN_PRIOR = {
"settled": 0.03, "fine": 0.07, "fair": 0.15, "changeable": 0.32,
"unsettled": 0.52, "rain": 0.72, "wet": 0.85, "stormy": 0.93,
}
FEATURES = ["bias", "slp_anom", "tend_1h", "tend_3h", "tend_6h", "rh",
"dew_depression", "dew_dep_rate", "cloud_index", "temp_dev",
"wet_bulb_depression", "zambretti_logit"]
def _season_is_summer(ts: Optional[float], latitude: float) -> bool:
month = time.gmtime(ts or time.time()).tm_mon
northern = latitude >= 0
summer_months = {4, 5, 6, 7, 8, 9}
return (month in summer_months) if northern else (month not in summer_months)
BARO_BOTTOM = 950.0
BARO_TOP = 1050.0
# Each branch maps normalised pressure onto a slice of the 26-point scale.
# The ordering is the whole point of the instrument: for a given pressure,
# rising air is always a better forecast than falling air, and within a
# branch higher pressure is always better. Ranges overlap because a deep
# but rising low really is more hopeful than a shallow but falling high.
_BRANCH = {
"rising": (1.0, 10.0),
"steady": (6.0, 17.0),
"falling": (11.0, 26.0),
}
def zambretti(slp_hpa: float, tendency_hpa_per_h: float,
ts: Optional[float] = None, latitude: float = 52.0,
steady_band: float = 0.10) -> Dict:
"""Three-branch barometric forecast on the Zambretti 26-point scale.
The 1915 Negretti and Zambra slide rule read pressure, its tendency and
the season off a rotating card and returned one of 26 outcomes, 1 being
settled and 26 being stormy. Published transcriptions of its constants
disagree with each other, so rather than mis-cite one, this is an
explicit re-parameterisation onto the same 26-point scale, anchored to
the behaviour the instrument is actually known for:
rising pressure -> lower Z (improving)
falling pressure -> higher Z (deteriorating)
higher pressure -> lower Z within any branch
Getting that sign wrong is easy and produces confident nonsense: a
barometer climbing hard while the panel reads `stormy` is the tell.
Args:
slp_hpa: pressure reduced to mean sea level. Passing station
pressure here is a common and silent bug: at 100 m elevation
it shifts the result by about two categories, permanently.
tendency_hpa_per_h: Kalman-filtered rate, not a finite difference.
steady_band: |tendency| below this counts as steady.
"""
p = float(np.clip(slp_hpa, BARO_BOTTOM, BARO_TOP))
tend = float(tendency_hpa_per_h)
summer = _season_is_summer(ts, latitude)
if tend <= -steady_band:
trend = "falling"
elif tend >= steady_band:
trend = "rising"
else:
trend = "steady"
lo, hi = _BRANCH[trend]
u = (p - BARO_BOTTOM) / (BARO_TOP - BARO_BOTTOM) # 0 at 950, 1 at 1050
z = lo + (hi - lo) * (1.0 - u)
# Seasonal nudge: summer lows are typically convective and shorter lived,
# winter lows are frontal and grimmer. One category either way.
if trend == "falling":
z += -1.0 if summer else 1.0
elif trend == "rising":
z += -1.0 if summer else 1.0
z_int = int(np.clip(round(z), 1, 26))
condition, label = "changeable", "Changeable"
for lo, hi, key, text in _CONDITIONS:
if lo <= z_int <= hi:
condition, label = key, text
break
return {
"z": z_int,
"trend": trend,
"condition": condition,
"label": label,
"prior_rain_prob": _RAIN_PRIOR[condition],
"slp_used": p,
"tendency": tend,
"season": "summer" if summer else "winter",
}
def tendency_code(tend_hpa_per_h: float) -> str:
"""WMO-style pressure characteristic, the thing sailors actually read."""
t = float(tend_hpa_per_h)
if t <= -1.5:
return "falling very rapidly"
if t <= -0.6:
return "falling rapidly"
if t <= -0.15:
return "falling"
if t < 0.15:
return "steady"
if t < 0.6:
return "rising"
if t < 1.5:
return "rising rapidly"
return "rising very rapidly"
def _sigmoid(z: float) -> float:
return 1.0 / (1.0 + math.exp(-float(np.clip(z, -30.0, 30.0))))
def _logit(p: float) -> float:
p = float(np.clip(p, 1e-4, 1 - 1e-4))
return math.log(p / (1 - p))
class PrecipitationModel:
"""Online logistic regression on top of the Zambretti logit.
Trained by AdaGrad because feature scales here vary by two orders of
magnitude and a fixed learning rate would either crawl on `tendency`
or explode on `rh`. The `zambretti_logit` feature is initialised with
a coefficient of 1.0 so the model *starts* as the slide rule and
departs from it only where the data insist.
"""
def __init__(self, lr: float = 0.08, l2: float = 1e-4):
self.w = np.zeros(len(FEATURES))
self.w[FEATURES.index("zambretti_logit")] = 1.0
self.g2 = np.ones(len(FEATURES)) * 1e-3
self.lr = float(lr)
self.l2 = float(l2)
self.n_strong = 0
self.n_weak = 0
self.ewma_logloss = 0.693 # log 2, the coin-flip baseline
self.mean = np.zeros(len(FEATURES))
self.m2 = np.ones(len(FEATURES))
self.n_seen = 0
# -------------------------------------------------------- features
def featurise(self, obs: Dict, zam: Dict) -> np.ndarray:
x = np.array([
1.0,
obs.get("slp", 1013.25) - 1013.25,
obs.get("tend_1h", 0.0),
obs.get("tend_3h", 0.0),
obs.get("tend_6h", 0.0),
(obs.get("rh", 60.0) - 70.0) / 10.0,
obs.get("dew_depression", 5.0),
obs.get("dew_dep_rate", 0.0),
obs.get("cloud_index", 0.5),
obs.get("temp_dev", 0.0),
obs.get("wet_bulb_depression", 2.0),
_logit(zam["prior_rain_prob"]),
], dtype=float)
return np.nan_to_num(x, nan=0.0, posinf=0.0, neginf=0.0)
def _standardise(self, x: np.ndarray, update: bool) -> np.ndarray:
if update:
self.n_seen += 1
delta = x - self.mean
self.mean += delta / self.n_seen
self.m2 += delta * (x - self.mean)
if self.n_seen < 20:
z = x.copy()
else:
std = np.sqrt(self.m2 / max(self.n_seen - 1, 1))
std = np.where(std < 1e-8, 1.0, std)
z = (x - self.mean) / std
z[0] = 1.0
# keep the prior feature unscaled: its units are already logits
z[FEATURES.index("zambretti_logit")] = x[FEATURES.index("zambretti_logit")]
return z
# ------------------------------------------------------- inference
def predict(self, obs: Dict, zam: Dict) -> Dict:
x = self._standardise(self.featurise(obs, zam), update=False)
p_model = _sigmoid(float(self.w @ x))
p_prior = zam["prior_rain_prob"]
# trust the learner in proportion to how many strong labels it has
trust = self.n_strong / (self.n_strong + 25.0)
p = trust * p_model + (1 - trust) * p_prior
return {
"rain_probability": float(np.clip(p, 0.0, 1.0)),
"model_probability": float(p_model),
"prior_probability": float(p_prior),
"learner_trust": float(trust),
"condition": zam["condition"],
"label": zam["label"],
"zambretti_z": zam["z"],
"pressure_characteristic": tendency_code(zam["tendency"]),
"tendency": float(zam["tendency"]),
"sea_level_pressure": float(zam["slp_used"]),
"strong_labels": self.n_strong,
"weak_labels": self.n_weak,
"logloss_ewma": round(float(self.ewma_logloss), 4),
}
# -------------------------------------------------------- learning
def learn(self, obs: Dict, zam: Dict, y: float, strong: bool = False) -> float:
"""AdaGrad step. Weak (proxy) labels get a tenth of the weight."""
x = self._standardise(self.featurise(obs, zam), update=True)
p = _sigmoid(float(self.w @ x))
weight = 1.0 if strong else 0.1
grad = weight * (p - float(y)) * x + self.l2 * self.w
self.g2 += grad ** 2
self.w -= self.lr * grad / np.sqrt(self.g2)
loss = -(y * math.log(max(p, 1e-9)) + (1 - y) * math.log(max(1 - p, 1e-9)))
self.ewma_logloss = 0.98 * self.ewma_logloss + 0.02 * loss
if strong:
self.n_strong += 1
else:
self.n_weak += 1
return float(loss)
def coefficients(self) -> List[Dict]:
return [{"feature": f, "weight": round(float(w), 4)}
for f, w in zip(FEATURES, self.w)]
def to_dict(self) -> Dict:
return {"w": self.w.tolist(), "g2": self.g2.tolist(), "lr": self.lr,
"l2": self.l2, "n_strong": self.n_strong, "n_weak": self.n_weak,
"ewma_logloss": self.ewma_logloss, "mean": self.mean.tolist(),
"m2": self.m2.tolist(), "n_seen": self.n_seen}
def load_dict(self, s: Dict) -> None:
self.w = np.array(s["w"], dtype=float)
self.g2 = np.array(s["g2"], dtype=float)
self.lr, self.l2 = s["lr"], s["l2"]
self.n_strong, self.n_weak = s["n_strong"], s["n_weak"]
self.ewma_logloss = s["ewma_logloss"]
self.mean = np.array(s["mean"], dtype=float)
self.m2 = np.array(s["m2"], dtype=float)
self.n_seen = s["n_seen"]
def proxy_wet_label(rh: float, dew_depression: float, cloud_index: float) -> Optional[float]:
"""A weak, deliberately conservative stand-in for a rain gauge.
Returns 1.0 for near-saturated overcast air, 0.0 for clearly dry air,
and None in the ambiguous middle, where a guess would poison the
training set faster than the extra samples could help.
"""
if not all(np.isfinite([rh, dew_depression, cloud_index])):
return None
if rh >= 93.0 and dew_depression <= 1.2 and cloud_index >= 0.6:
return 1.0
if rh <= 65.0 and dew_depression >= 5.0:
return 0.0
return None
ashvale/models/rls.py
+182
View File
@@ -0,0 +1,182 @@
# Copyright 2026 Kemal Yaylali
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""The learning core: exponentially-weighted recursive least squares.
Why RLS rather than an off-the-shelf gradient learner:
* It is the exact minimiser of the exponentially weighted squared error
at every step, not an approximation, so it converges in far fewer
samples than SGD. On a station that produces 288 rows a day, sample
efficiency is not a nicety.
* The covariance `P` is a genuine parameter-uncertainty estimate, free.
* One matrix of size (d, d) with d ~ 33 is 8 kB. The whole model bank
fits in L2 cache on a Cortex-A53.
* Forgetting factor `lambda` gives principled adaptation to season and
to sensor ageing without any retraining schedule.
Directional forgetting is used: `P` is only inflated along directions
that were actually excited by data. Plain forgetting blows `P` up
exponentially during quiet nights when the regressor is nearly constant,
and the model then detonates on the first sunrise. This is the single
most common way an RLS deployment fails in the field.
"""
from __future__ import annotations
from collections import deque
from typing import Deque, Dict, Optional
import numpy as np
class RecursiveLeastSquares:
def __init__(self, n_features: int, forgetting: float = 0.999,
delta: float = 100.0, p_max: float = 1e6):
self.d = int(n_features)
self.lam = float(forgetting)
self.p_max = float(p_max)
self.theta = np.zeros(self.d)
self.P = np.eye(self.d) * float(delta)
self.n_updates = 0
self.ewma_sq_error = 0.0
def predict(self, x: np.ndarray) -> float:
return float(np.dot(self.theta, np.asarray(x, dtype=float).ravel()))
def predict_many(self, X: np.ndarray) -> np.ndarray:
return np.asarray(X, dtype=float) @ self.theta
def predict_std(self, x: np.ndarray, noise_var: float = 1.0) -> float:
"""Parameter-uncertainty contribution to predictive spread."""
x = np.asarray(x, dtype=float).ravel()
return float(np.sqrt(max(noise_var * (1.0 + x @ self.P @ x), 1e-12)))
def update(self, x: np.ndarray, y: float, weight: float = 1.0) -> float:
"""One RLS step. Returns the a-priori residual (the honest error)."""
x = np.asarray(x, dtype=float).ravel()
if not (np.all(np.isfinite(x)) and np.isfinite(y)):
return 0.0
Px = self.P @ x
denom = self.lam + weight * float(x @ Px)
if denom < 1e-12:
return 0.0
residual = float(y) - float(self.theta @ x)
gain = (weight * Px) / denom
self.theta = self.theta + gain * residual
self.P = (self.P - np.outer(gain, Px)) / self.lam
# directional forgetting guard: cap the spectral growth of P
self.P = 0.5 * (self.P + self.P.T) # enforce symmetry
trace = float(np.trace(self.P))
if trace > self.p_max:
self.P *= self.p_max / trace
np.fill_diagonal(self.P, np.maximum(np.diag(self.P), 1e-9))
self.n_updates += 1
self.ewma_sq_error = 0.99 * self.ewma_sq_error + 0.01 * residual ** 2
return residual
def fit_batch(self, X: np.ndarray, y: np.ndarray, passes: int = 1) -> "RecursiveLeastSquares":
X = np.atleast_2d(np.asarray(X, dtype=float))
y = np.asarray(y, dtype=float).ravel()
for _ in range(max(int(passes), 1)):
for i in range(X.shape[0]):
self.update(X[i], y[i])
return self
@property
def noise_var(self) -> float:
return float(max(self.ewma_sq_error, 1e-9))
def to_dict(self) -> Dict:
return {"d": self.d, "lam": self.lam, "p_max": self.p_max,
"theta": self.theta.tolist(), "P": self.P.tolist(),
"n": self.n_updates, "ewma": self.ewma_sq_error}
@classmethod
def from_dict(cls, s: Dict) -> "RecursiveLeastSquares":
m = cls(s["d"], s["lam"], 1.0, s.get("p_max", 1e6))
m.theta = np.array(s["theta"], dtype=float)
m.P = np.array(s["P"], dtype=float)
m.n_updates = s.get("n", 0)
m.ewma_sq_error = s.get("ewma", 0.0)
return m
class AdaptiveConformal:
"""Distribution-free prediction intervals that self-correct their coverage.
Split conformal gives you a valid interval only if the data are
exchangeable. Weather is not: a front arrives and yesterday's
residual quantile becomes a fantasy. Adaptive conformal inference
(Gibbs and Candes) fixes this by feeding realised coverage back into
the working alpha:
alpha_{t+1} = alpha_t + gamma * (alpha_target - err_t)
The interval widens after each miss and narrows after each hit, so
long-run coverage tracks the target whatever the distribution does.
"""
def __init__(self, alpha: float = 0.10, window: int = 400, gamma: float = 0.01):
self.alpha_target = float(alpha)
self.alpha = float(alpha)
self.gamma = float(gamma)
self.scores: Deque[float] = deque(maxlen=int(window))
self.hits: Deque[int] = deque(maxlen=int(window))
def quantile(self) -> float:
if len(self.scores) < 20:
return float("nan")
a = float(np.clip(self.alpha, 0.005, 0.75))
return float(np.quantile(np.asarray(self.scores), 1.0 - a, method="higher"))
def interval(self, mu: float, fallback_sigma: float = 1.0) -> tuple[float, float]:
q = self.quantile()
if not np.isfinite(q):
q = 1.645 * fallback_sigma # gaussian 90% until we know better
return float(mu - q), float(mu + q)
def observe(self, residual: float, covered: Optional[bool] = None) -> None:
r = abs(float(residual))
if not np.isfinite(r):
return
if covered is None:
q = self.quantile()
covered = bool(r <= q) if np.isfinite(q) else True
self.scores.append(r)
self.hits.append(1 if covered else 0)
err = 0.0 if covered else 1.0
self.alpha = float(np.clip(self.alpha + self.gamma * (self.alpha_target - err),
0.005, 0.75))
@property
def empirical_coverage(self) -> float:
return float(np.mean(self.hits)) if self.hits else float("nan")
def to_dict(self) -> Dict:
return {"alpha_target": self.alpha_target, "alpha": self.alpha,
"gamma": self.gamma, "maxlen": self.scores.maxlen,
"scores": list(self.scores), "hits": list(self.hits)}
@classmethod
def from_dict(cls, s: Dict) -> "AdaptiveConformal":
c = cls(s["alpha_target"], s.get("maxlen", 400) or 400, s["gamma"])
c.alpha = s["alpha"]
c.scores = deque(s["scores"], maxlen=c.scores.maxlen)
c.hits = deque(s["hits"], maxlen=c.hits.maxlen)
return c