From e27a4b41c8bf7e3f7cbdba6cf54ea1e841e55d99 Mon Sep 17 00:00:00 2001 From: Kemal Yaylali Date: Sat, 15 Aug 2026 21:36:10 +0100 Subject: [PATCH] Outlook to Live, humidity calibration, Models tab rebuilt without scrollers Seven day outlook moves from History to Live, which now runs four rows. Conditions ahead tightened so the Live column no longer needs a scroller. Adds HumidityCompensator: an additive RH offset estimated by one-step RLS from a trusted hygrometer, clamped to +/-35%, persisted, exposed at POST /api/calibrate/humidity and on the renamed Models and calibration tab. It also implements the psychrometric term (RH moved from element temperature onto air temperature via conserved vapour pressure) but leaves it OFF by default. The thermal argument predicts a hot element reads low; measured against a reference hygrometer this board read 75.4% where the truth was 50.4%, so it reads HIGH and that correction would push it the wrong way. When the flag is enabled, simulate.py applies the exact inverse, per the simulator/compensator trap in DESIGN.md section 2. Models pane rebuilt: the scorecard is one column per target so all 18 heads are visible, and no panel on the tab uses an internal scroller. Verified in Chromium at 1600x900: Live, History and Models all report zero scrollbars, zero clipping, no page scroll, zero console errors. Backtest is numerically identical to the previous commit, confirming the humidity work is a no-op while the flag is off. --- .github/ISSUE_TEMPLATE/forecast_quality.md | 2 +- README.md | 4 +- ashvale/api.py | 22 +++ ashvale/config.py | 15 ++ ashvale/dashboard.py | 215 ++++++++++++--------- ashvale/estimation.py | 105 +++++++++- ashvale/station.py | 28 +++ docs/DESIGN.md | 24 ++- scripts/simulate.py | 17 +- 9 files changed, 336 insertions(+), 96 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/forecast_quality.md b/.github/ISSUE_TEMPLATE/forecast_quality.md index b26a65f..fa50b83 100644 --- a/.github/ISSUE_TEMPLATE/forecast_quality.md +++ b/.github/ISSUE_TEMPLATE/forecast_quality.md @@ -11,7 +11,7 @@ labels: forecasting ``` ``` -**Scorecard from the Models tab** (or `GET /api/scorecard`) +**Scorecard from the Models and calibration tab** (or `GET /api/scorecard`) ```json ``` diff --git a/README.md b/README.md index 1270f54..c727526 100644 --- a/README.md +++ b/README.md @@ -166,9 +166,9 @@ eight-pixel type. | Tab | Answers | | --- | --- | -| **Live** | What is it doing now, what it expects next, and how sure it is | +| **Live** | What is it doing now, what it expects next, how sure it is, and the week ahead | | **History** | What did it do, over any timeframe you ask for | -| **Models** | Has the model earned its confidence | +| **Models and calibration** | Has the model earned its confidence, and the calibration inputs | | **Methods** | How the whole thing is wired, and how each stage fails | Live carries the current readings, the observed-and-forecast chart with its 90% diff --git a/ashvale/api.py b/ashvale/api.py index 78c3cbc..b3ec24c 100644 --- a/ashvale/api.py +++ b/ashvale/api.py @@ -108,6 +108,13 @@ class CalibrationIn(BaseModel): "covariance, returning to the configured prior") +class HumidityCalibrationIn(BaseModel): + reference_pct: Optional[float] = Field(None, ge=0, le=100, + description="Trusted relative humidity in %") + reset: bool = Field(False, description="Discard the learned offset, returning to " + "the configured prior") + + # ------------------------------------------------------------ endpoints @app.get("/api/telemetry") @@ -140,6 +147,8 @@ def telemetry() -> Dict: "cpu_temp": live.get("cpu_temp"), "cpu_offset": live.get("cpu_offset"), "compensator_k": live.get("compensator_k"), + "hum_offset": live.get("hum_offset"), + "hum_psychrometric": live.get("hum_psychrometric"), "rates": { "temperature_c_per_h": live.get("temp_rate"), "humidity_pct_per_h": live.get("hum_rate"), @@ -382,6 +391,19 @@ def calibrate(body: CalibrationIn) -> Dict: return _clean(result) +@app.post("/api/calibrate/humidity") +def calibrate_humidity(body: HumidityCalibrationIn) -> Dict: + st = _st() + if body.reset: + return _clean(st.reset_humidity_calibration()) + if body.reference_pct is None: + raise HTTPException(422, "provide reference_pct, or reset=true") + result = st.calibrate_humidity(body.reference_pct) + if "error" in result: + raise HTTPException(409, result["error"]) + return _clean(result) + + @app.get("/api/status") def status() -> Dict: st = _st() diff --git a/ashvale/config.py b/ashvale/config.py index ca6f91f..e65b254 100644 --- a/ashvale/config.py +++ b/ashvale/config.py @@ -55,6 +55,21 @@ class SensorConfig: cpu_heat_k: float = 0.55 cpu_heat_k_min: float = 0.15 cpu_heat_k_max: float = 1.20 + # Additive RH bias of the element. The datasheet claims about +/-3.5%, but + # measured against a reference hygrometer this board read 75.4% where the + # truth was 50.4%, so the clamp has to allow far more than spec. Kept finite + # so one mistyped reference still cannot run away. + # Move RH from the element's temperature onto the compensated air temperature + # via conserved vapour pressure. Physically correct IF the humidity element + # really sits at temp_raw. Measured on this board it does not: against a + # reference hygrometer reading 50.4%, the HTS221 reported 75.4%, so it reads + # HIGH and this correction would push it higher still. The error is an + # additive element bias, not a thermal gradient. Leave off unless your own + # reference says otherwise. + hum_psychrometric: bool = False + hum_offset: float = 0.0 + hum_offset_min: float = -35.0 + hum_offset_max: float = 35.0 # Kalman process/measurement noise (per-signal) kalman_q_temp: float = 2.0e-6 kalman_r_temp: float = 0.02 diff --git a/ashvale/dashboard.py b/ashvale/dashboard.py index f81b2ab..c0faf40 100644 --- a/ashvale/dashboard.py +++ b/ashvale/dashboard.py @@ -120,14 +120,14 @@ DASHBOARD_HTML = r"""
-
+
@@ -216,9 +216,9 @@ DASHBOARD_HTML = r"""

Conditions ahead

Zambretti prior + online logistic

-
-
-
--
+
+
+
--
Z=- · -
@@ -226,7 +226,7 @@ DASHBOARD_HTML = r"""
prior -learner -trust -
-
+
Was it wet in the last hour?
@@ -239,12 +239,23 @@ DASHBOARD_HTML = r""" pressure, last 24 h --
-
-

The only signal here that sees past your walls. Its slope, not its level, is what drives the forecast above.

+
+

The only signal that sees past your walls. Its slope drives the forecast.

+
+
+
+

Seven day outlook

+

climatology plus decaying anomaly, not a synoptic forecast

+
+ warming up +
+
+
+
wet bulb
--
vpd
--
@@ -260,7 +271,7 @@ DASHBOARD_HTML = r"""
-
+
@@ -300,85 +311,86 @@ DASHBOARD_HTML = r"""
-
-
-
-

Seven day outlook

-

climatology plus decaying anomaly, not a synoptic forecast

-
- warming up -
-
-
- -
-
+ + +
+ +

Verification scorecard

-

skill above zero means it beats persistence

+

skill above zero means it beats persistence · p/c/l is the ensemble weight

-
- - - - - - - - - -
targetleadMAEpersistskillcovernp/c/l
-

No matured forecasts yet. Rows appear as each horizon reaches its validity time: 15 minutes first, 24 hours tomorrow. The p/c/l column is the ensemble weight on persistence, climatology and the learned model.

+
+

No matured forecasts yet. Rows appear as each horizon reaches its validity time: 15 minutes first, 24 hours tomorrow.

+
+ +
+
+

Temperature

+ self-heating +
+
+
coefficient k--
+
cpu offset--
+
+ + + +
+
Enter a trusted thermometer reading in °C. One good reading is enough.
-
-

Calibration and state

-
-
-
self-heating k--
-
cpu offset--
-
Removes the SoC bias. Set it from the reading below.
+
+
+

Humidity

+ element bias +
+
+
offset--
+
psychrometric--
+
+ + +
-
-
Trusted thermometer reading
-
- - - -
-
Recursive least squares on the self-heating coefficient. One good reading is enough.
-
-
-
Storage tiers
-
-
-
-
Precipitation coefficients
-
-
-
-
novelty d²--
-
-
drift pressure--
-
-
Novelty is a multivariate departure from the recent norm. Drift reaching 100% queues a retrain.
-
-
+
Enter a trusted hygrometer reading in %. The psychrometric term is derived, not fitted.
-
-

Station log

-
+
+

Estimator

+
+
novelty d²--
+
+
drift pressure--
+
+
+
+
+ +
+

Storage and weights

+
+
+
+
+
+ +
+

Station log

+
@@ -453,7 +465,7 @@ el('live-span').addEventListener('click', e => { (x===b ? 'bg-indigo-600/20 text-indigo-300' : 'text-slate-400 hover:text-slate-200')); loadForecast(); }); -loaders.live = loadForecast; +loaders.live = () => { loadForecast(); loadOutlook(); }; function setFlash(id,val) { const node = el(id); if (!node || node.innerText===val) return; @@ -491,6 +503,8 @@ function applyTelemetry(d) { el('d-az').innerText = fmt(d.accel && d.accel.z,2); el('e-k').innerText = fmt(d.compensator_k,4); + el('e-hoff').innerText = (d.hum_offset===undefined||d.hum_offset===null) ? '--' : (d.hum_offset>=0?'+':'')+fmt(d.hum_offset,2)+'%'; + el('e-hpsy').innerText = (d.hum_psychrometric===undefined||d.hum_psychrometric===null) ? '--' : (d.hum_psychrometric>=0?'+':'')+fmt(d.hum_psychrometric,2)+'%'; el('e-off').innerText = fmt(d.cpu_offset,1)+' K'; el('e-nov').innerText = fmt(d.novelty_d2,1); el('e-nov-bar').style.width = Math.min((d.novelty_d2||0)/24,1)*100+'%'; @@ -754,7 +768,7 @@ function toggleRec(daily) { } el('rec-tab-daily').addEventListener('click', ()=>toggleRec(true)); el('rec-tab-all').addEventListener('click', ()=>toggleRec(false)); -loaders.history = () => { if (!histData) loadHistory(); loadOutlook(); }; +loaders.history = () => { if (!histData) loadHistory(); }; /* ---------------- MODELS ---------------- */ async function loadModels() { @@ -768,19 +782,31 @@ async function loadModels() { (md.nowcast||[]).forEach(h => { wmap[h.target+'@'+h.horizon_s] = h.weights; }); const rows = sc.rows||[]; el('m-empty').style.display = rows.length ? 'none' : 'block'; - el('m-score').innerHTML = rows.map(r=>{ - const sk = r.skill||0; - const cls = sk>0.05?'text-emerald-300':sk<-0.05?'text-rose-300':'text-slate-400'; - const lead = r.horizon_s<3600 ? (r.horizon_s/60)+'m' : r.horizon_s<86400 ? (r.horizon_s/3600)+'h' : (r.horizon_s/86400)+'d'; - const w = wmap[r.target+'@'+r.horizon_s]; - const ws = w ? Math.round(w.persistence*100)+'/'+Math.round(w.climatology*100)+'/'+Math.round(w.learned*100) : '-'; - return ''+r.target+''+ - ''+lead+''+fmt(r.mae,3)+''+ - ''+fmt(r.mae_persistence,3)+''+ - ''+Math.round(sk*100)+'%'+ - ''+Math.round((r.coverage||0)*100)+'%'+ - ''+r.n+''+ - ''+ws+''; + // One column per target rather than one 18-row table: every head stays + // visible instead of hiding behind a scrollbar. + const byTarget = {}; + rows.forEach(r => { (byTarget[r.target] = byTarget[r.target] || []).push(r); }); + el('m-score').innerHTML = Object.keys(byTarget).map(target => { + const body = byTarget[target].map(r => { + const sk = r.skill||0; + const cls = sk>0.05?'text-emerald-300':sk<-0.05?'text-rose-300':'text-slate-400'; + const lead = r.horizon_s<3600 ? (r.horizon_s/60)+'m' : r.horizon_s<86400 ? (r.horizon_s/3600)+'h' : (r.horizon_s/86400)+'d'; + const w = wmap[r.target+'@'+r.horizon_s]; + const ws = w ? Math.round(w.persistence*100)+'/'+Math.round(w.climatology*100)+'/'+Math.round(w.learned*100) : '-'; + return ''+ + ''+lead+''+ + ''+fmt(r.mae,2)+''+ + ''+fmt(r.mae_persistence,2)+''+ + ''+Math.round(sk*100)+'%'+ + ''+Math.round((r.coverage||0)*100)+'%'+ + ''+ws+''; + }).join(''); + return '
'+ + '
'+target+'
'+ + ''+ + ''+ + ''+ + ''+body+'
leadMAEpersskillcovp/c/l
'; }).join(''); el('m-storage').innerHTML = (sg.tiers||[]).map(t=> @@ -837,6 +863,20 @@ el('m-calrst').addEventListener('click', async () => { body: JSON.stringify({reset:true})}).then(r=>r.json()); el('m-calstat').innerHTML = 'Reset to prior k = '+r.k+''; }); +el('m-hcalgo').addEventListener('click', async () => { + const v = parseFloat(el('m-hcalin').value); + if (Number.isNaN(v) || v < 0 || v > 100) { el('m-hcalstat').innerText = 'Enter a relative humidity between 0 and 100%.'; return; } + const r = await fetch('/api/calibrate/humidity', {method:'POST', headers:{'Content-Type':'application/json'}, + body: JSON.stringify({reference_pct:v})}).then(r=>r.json()); + el('m-hcalstat').innerHTML = r.offset!==undefined + ? 'offset is now '+r.offset.toFixed(2)+'%, residual '+r.residual.toFixed(2)+'%' + : 'Rejected: no live reading yet.'; +}); +el('m-hcalrst').addEventListener('click', async () => { + const r = await fetch('/api/calibrate/humidity', {method:'POST', headers:{'Content-Type':'application/json'}, + body: JSON.stringify({reset:true})}).then(r=>r.json()); + el('m-hcalstat').innerHTML = 'Reset to prior offset = '+r.offset+'%'; +}); /* ---------------- METHODS ---------------- */ let methodsDoc = null, methodSel = 'acquire'; @@ -917,10 +957,11 @@ loaders.methods = loadMethods; /* ---------------- BOOT ---------------- */ connectStream(); loadForecast(); +loadOutlook(); fetch('/api/status').then(r=>r.json()).then(s => { el('hd-days').innerText = fmt(s.history_days,2); }); -setInterval(() => { if (activeTab==='live') loadForecast(); }, 60000); +setInterval(() => { if (activeTab==='live') { loadForecast(); loadOutlook(); } }, 60000); setInterval(() => { if (activeTab==='models') loadModels(); }, 60000); -setInterval(() => { if (activeTab==='history') { loadHistory(); loadOutlook(); } }, 300000); +setInterval(() => { if (activeTab==='history') loadHistory(); }, 300000); diff --git a/ashvale/estimation.py b/ashvale/estimation.py index 2a10019..5ba7e93 100644 --- a/ashvale/estimation.py +++ b/ashvale/estimation.py @@ -35,6 +35,8 @@ from typing import Dict, Optional import numpy as np +from .physics import saturation_vapour_pressure + @dataclass class KalmanCV: @@ -158,14 +160,105 @@ class ThermalCompensator: return tc +class HumidityCompensator: + """Corrects relative humidity for a sensor sitting hotter than the air. + + The HTS221 reports RH at its own temperature, but the air you care about is + at the compensated temperature. Vapour pressure is what is conserved between + the two, so + + RH_true = RH_sensor * es(T_sensor) / es(T_true) + + Because the element runs hot, es(T_sensor) > es(T_true) and an uncorrected + reading is biased low, by several points on a warm board. That term needs no + calibration constant at all: it falls out of the thermal compensation that is + already running. + + Measured on real hardware the psychrometric term is the wrong model: against + a reference hygrometer reading 50.4%, this board's HTS221 reported 75.4%, so + it reads HIGH where the thermal argument predicts LOW. The dominant error is + an additive element bias, which is what `offset` corrects, estimated from a + trusted reference by the same one-step RLS used for `k` with the regressor + fixed at 1, so repeated calibrations converge to a weighted mean. The + psychrometric term is therefore off by default and gated on config. + + How it fails: calibrate against a reference while the board is cool, then let + CPU load rise, and an offset-only correction drifts because the psychrometric + error grows with the gradient. Applying the vapour-pressure term first is + exactly what keeps `offset` a constant rather than a function of CPU load. + Clamped for the same reason `k` is: one mistyped reference otherwise biases + every reading until you notice. + """ + + def __init__(self, offset: float = 0.0, off_min: float = -35.0, + off_max: float = 35.0, forgetting: float = 0.98, + psychrometric: bool = False): + self.psychrometric = bool(psychrometric) + self.offset = float(offset) + self.off_min, self.off_max = float(off_min), float(off_max) + self.P = 10.0 + self.lam = float(forgetting) + self.n_calibrations = 0 + self.last_residual = 0.0 + + def _psychrometric(self, rh_sensor: float, t_sensor: float, t_true: float) -> float: + if not self.psychrometric: + return float(rh_sensor) + es_s = float(saturation_vapour_pressure(t_sensor)) + es_t = float(saturation_vapour_pressure(t_true)) + if not np.isfinite(es_s) or not np.isfinite(es_t) or es_t <= 1e-9: + return float(rh_sensor) + return float(rh_sensor * es_s / es_t) + + def compensate(self, rh_sensor: float, t_sensor: float, t_true: float) -> float: + if not (np.isfinite(rh_sensor) and np.isfinite(t_sensor) and np.isfinite(t_true)): + return float(rh_sensor) + base = self._psychrometric(rh_sensor, t_sensor, t_true) + return float(np.clip(base + self.offset, 0.0, 100.0)) + + def calibrate(self, rh_sensor: float, t_sensor: float, t_true: float, + rh_reference: float) -> Dict: + """One RLS step on the residual offset. Regressor is 1.""" + base = self._psychrometric(rh_sensor, t_sensor, t_true) + target = float(rh_reference) - base + denom = self.lam + self.P + gain = self.P / denom if denom > 1e-12 else 0.0 + residual = target - self.offset + self.offset = float(np.clip(self.offset + gain * residual, + self.off_min, self.off_max)) + self.P = float(np.clip((self.P - gain * self.P) / self.lam, 1e-6, 1e4)) + self.n_calibrations += 1 + self.last_residual = float(residual) + return {"offset": self.offset, "residual": self.last_residual, + "n": self.n_calibrations, "psychrometric": base - float(rh_sensor)} + + def to_dict(self) -> Dict: + return {"offset": self.offset, "P": self.P, "lam": self.lam, + "off_min": self.off_min, "off_max": self.off_max, + "n": self.n_calibrations, "psychrometric": self.psychrometric} + + @classmethod + def from_dict(cls, d: Dict) -> "HumidityCompensator": + hc = cls(d["offset"], d["off_min"], d["off_max"], d["lam"], + d.get("psychrometric", False)) + hc.P = d["P"] + hc.n_calibrations = d.get("n", 0) + return hc + + class SignalTracker: - """Bank of Kalman filters plus the compensator, driven at sample rate.""" + """Bank of Kalman filters plus the compensators, driven at sample rate.""" def __init__(self, cfg): self.compensator = ThermalCompensator( cfg.sensor.cpu_heat_k, cfg.sensor.cpu_heat_k_min, cfg.sensor.cpu_heat_k_max, ) + self.hum_compensator = HumidityCompensator( + cfg.sensor.hum_offset, cfg.sensor.hum_offset_min, + cfg.sensor.hum_offset_max, + psychrometric=cfg.sensor.hum_psychrometric, + ) self.filters = { "temperature": KalmanCV(cfg.sensor.kalman_q_temp, cfg.sensor.kalman_r_temp), "humidity": KalmanCV(cfg.sensor.kalman_q_hum, cfg.sensor.kalman_r_hum), @@ -179,12 +272,16 @@ class SignalTracker: self.last_ts = ts temp_c = self.compensator.compensate(temp_raw, cpu_temp) + # RH is reported at the element's temperature, not the air's, so it must + # be moved onto the compensated temperature before it is filtered. + hum_c = self.hum_compensator.compensate(hum, temp_raw, temp_c) t_lvl, t_rate = self.filters["temperature"].update(temp_c, dt) - h_lvl, h_rate = self.filters["humidity"].update(hum, dt) + h_lvl, h_rate = self.filters["humidity"].update(hum_c, dt) p_lvl, p_rate = self.filters["pressure"].update(press, dt) return { "temp_c": temp_c, + "hum_c": hum_c, "temp_smooth": t_lvl, "temp_rate": t_rate * 3600.0, # C per hour "hum_smooth": h_lvl, @@ -198,12 +295,16 @@ class SignalTracker: def to_dict(self) -> Dict: return { "compensator": self.compensator.to_dict(), + "hum_compensator": self.hum_compensator.to_dict(), "filters": {k: v.to_dict() for k, v in self.filters.items()}, "last_ts": self.last_ts, } def load_dict(self, d: Dict) -> None: self.compensator = ThermalCompensator.from_dict(d["compensator"]) + # Absent from state files written before humidity compensation existed. + if d.get("hum_compensator"): + self.hum_compensator = HumidityCompensator.from_dict(d["hum_compensator"]) self.filters = {k: KalmanCV.from_dict(v) for k, v in d["filters"].items()} self.last_ts = d.get("last_ts") diff --git a/ashvale/station.py b/ashvale/station.py index 1431026..dcee58f 100644 --- a/ashvale/station.py +++ b/ashvale/station.py @@ -190,6 +190,8 @@ class Station: "cloud_index": cloud, "cpu_offset": (raw.get("cpu_temp") or float("nan")) - (raw.get("temp_raw") or float("nan")), "compensator_k": self.tracker.compensator.k, + "hum_offset": self.tracker.hum_compensator.offset, + "hum_psychrometric": float(est["hum_c"]) - float(raw.get("hum") or float("nan")), "health": anomaly["health_overall"], "novelty_d2": anomaly["novelty"].get("d2", 0.0), } @@ -265,6 +267,32 @@ class Station: f"k -> {result['k']:.3f} (residual {result['residual']:+.2f} C)") return result + def calibrate_humidity(self, reference_pct: float) -> Dict: + raw_h = self.live.get("hum") + raw_t = self.live.get("temp_raw") + temp_c = self.live.get("temp_c") + if raw_h is None or raw_t is None or temp_c is None: + return {"error": "no live reading yet"} + result = self.tracker.hum_compensator.calibrate( + float(raw_h), float(raw_t), float(temp_c), float(reference_pct)) + self.save_state() + self.store.log_event("calibration", "info", + f"rh offset -> {result['offset']:+.2f}% " + f"(residual {result['residual']:+.2f}%)") + return result + + def reset_humidity_calibration(self) -> Dict: + from .estimation import HumidityCompensator + self.tracker.hum_compensator = HumidityCompensator( + self.cfg.sensor.hum_offset, self.cfg.sensor.hum_offset_min, + self.cfg.sensor.hum_offset_max, + psychrometric=self.cfg.sensor.hum_psychrometric, + ) + self.save_state() + self.store.log_event("calibration", "info", + f"rh offset reset to prior {self.cfg.sensor.hum_offset}") + return {"offset": self.tracker.hum_compensator.offset, "reset": True, "n": 0} + def reset_calibration(self) -> Dict: """Return the self-heating coefficient to its configured prior. diff --git a/docs/DESIGN.md b/docs/DESIGN.md index d3ef1f5..895a271 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -99,6 +99,27 @@ exact inverse:** `T_raw = (T + k·T_cpu)/(1 + k)`. Generating the bias as 1.2 °C of phantom noise floor that caps every skill score. This has already happened once. +### Humidity compensation + +`HumidityCompensator` carries an additive `offset` on relative humidity, +estimated from a trusted hygrometer by the same one-step RLS used for `k`, with +the regressor fixed at 1 so repeated calibrations converge to a weighted mean. +Clamped to +/-35% for the same reason `k` is clamped. + +It also implements a psychrometric term, moving RH from the element's +temperature onto the compensated air temperature through conserved vapour +pressure, `RH_true = RH_sensor * es(T_sensor) / es(T_true)`. That term is +**off by default**, and the reason is worth recording. The thermal argument +predicts a hot element reads LOW. Measured against a reference hygrometer this +board read 75.4% where the truth was 50.4%, so it reads HIGH by 25 points, and +the correction would have pushed it further the wrong way. The dominant error on +this hardware is additive element bias, not a thermal gradient. + +If you enable `sensor.hum_psychrometric`, `scripts/simulate.py` applies the exact +inverse when generating synthetic humidity. It has to: the same +simulator/compensator algebra trap described above for temperature applies here, +and getting it wrong bakes in a bias no calibration can remove. + ### The Kalman bank One constant-velocity filter per signal. State `x = [level, rate]`, standard @@ -302,7 +323,8 @@ consecutive readings are the only tell. | Symptom | Knob | Direction | |---|---|---| -| Temperature reads consistently high | Calibrate from the Models tab, or `sensor.cpu_heat_k` | Raise | +| Temperature reads consistently high | Calibrate from the Models and calibration tab, or `sensor.cpu_heat_k` | Raise | +| Humidity reads consistently off | Calibrate against a reference hygrometer, or `sensor.hum_offset` | Either | | Readings over-smoothed, lag real change | `sensor.kalman_q_temp` | Raise | | Rates look noisy | `sensor.kalman_q_*` down, or `kalman_r_*` up | | | NIS persistently much above 1 | Filter too confident, raise `q` | Raise | diff --git a/scripts/simulate.py b/scripts/simulate.py index ec50e17..b8ca00b 100644 --- a/scripts/simulate.py +++ b/scripts/simulate.py @@ -62,7 +62,8 @@ from ashvale.storage import Store # noqa: E402 def generate(days: float, step_s: int, lat: float, lon: float, - seed: int = 11, end: float | None = None) -> dict: + seed: int = 11, end: float | None = None, + psychrometric: bool = False) -> dict: rng = np.random.default_rng(seed) n = int(days * 86400 / step_s) # Anchoring to wall clock makes a fixed seed insufficient for reproducibility: @@ -131,11 +132,21 @@ def generate(days: float, step_s: int, lat: float, lon: float, # no amount of calibration can remove, and quietly caps your skill score. k_true = 0.55 temp_raw = (temp + k_true * cpu) / (1.0 + k_true) + 0.05 * rng.normal(size=n) + # If the compensator will move RH from the element temperature onto the air + # temperature, the forward model here must be its exact inverse, or the + # synthetic data bakes in a bias no calibration can remove. Same trap as the + # thermal algebra above. Off by default, matching sensor.hum_psychrometric. + if psychrometric: + es_raw = 6.112 * np.exp(17.625 * temp_raw / (243.04 + temp_raw)) + rh_sensor = np.clip(rh * es_t / es_raw, 0.0, 100.0) + else: + rh_sensor = rh + press_station = press_slp / (1.0 + 0.0) - 1.8 # nominal 15 m offset press_station += 0.05 * rng.normal(size=n) return { - "ts": ts, "temp": temp, "temp_raw": temp_raw, "rh": rh + 0.4 * rng.normal(size=n), + "ts": ts, "temp": temp, "temp_raw": temp_raw, "rh": rh_sensor + 0.4 * rng.normal(size=n), "press": press_station, "press_slp": press_slp, "cpu": cpu, "lux": lux * (0.85 + 0.3 * rng.random(n)), "dew": dew, "cloud": cloud, } @@ -176,7 +187,7 @@ def main() -> None: print("cleared existing telemetry, forecasts and scores") data = generate(args.days, args.step, cfg.site.latitude, cfg.site.longitude, - args.seed, args.end) + args.seed, args.end, cfg.sensor.hum_psychrometric) tracker = SignalTracker(cfg) n = data["ts"].size