Log both Sense HAT thermometers, and migrate schemas that predate them

The board carries two independent thermometers and the code averaged them
into temp_raw without ever recording either. Measured over 12 samples on a
real station: HTS221 30.973 C at sd 0.060, LPS25HB 29.810 C at sd 0.443, a
standing gradient of 1.163 C with the SoC at 44.55 C.

Two things follow from that and neither is possible without the raw channels.
A plain average of a quiet sensor and one seven times noisier lands at sd
0.223 where inverse-variance weighting reaches 0.060, and the gradient between
two chips at different distances from the SoC is a second observation of
self-heating that could identify the compensator's k with no reference
thermometer. Both need history, and history cannot be backfilled, so the
columns land on their own ahead of the work that consumes them.

CREATE TABLE IF NOT EXISTS is a no-op against a table that already exists, so
adding to COLUMNS would have reached a fresh install and silently missed every
station already running, then surfaced as an OperationalError inside
insert_telemetry. That sits on the sample loop, so it takes a station down
rather than leaving a gap. Store now reconciles the table against COLUMNS on
open, which makes every future column addition safe rather than just this one.

The simulator gains the same two channels, with couplings solved so their
forward models average to exactly the k = 0.55 the compensator is tuned
against. Aggregate behaviour is unchanged; only the per-channel detail is new.
Simulated temp_raw noise does rise from 0.05 to 0.223, which is not a
regression but the end of an over-optimistic figure: it was modelling the
quiet sensor and calling it the average.

Co-Authored-By: Claude Opus 5 <[email protected]>

ashvale/sensors.py

@@ -53,6 +53,21 @@ def read_cpu_temperature() -> float:
         return float("nan")
 
 
+# Per-chip thermal coupling to the SoC, and per-chip noise.
+#
+# The Sense HAT carries two independent thermometers at different distances
+# from the SoC, and they are not equally good. Measured over 12 samples on a
+# real board: HTS221 30.973 C at sd 0.060, LPS25HB 29.810 C at sd 0.443, a
+# standing gradient of 1.163 C with the SoC at 44.55 C.
+#
+# These two couplings are chosen so their forward models average to exactly the
+# k = 0.55 the compensator is tuned against. The aggregate behaviour is
+# therefore unchanged and only the per-channel detail is new, which matters
+# because that gradient is a second observation of self-heating.
+K_HTS221, K_LPS25HB = 0.6164, 0.4889
+SD_HTS221, SD_LPS25HB = 0.060, 0.443
+
+
 class SimulatedBoard:
     """Ornstein-Uhlenbeck weather with a diurnal driver. Good enough to
     exercise every code path and to sanity-check a model's skill score."""
@@ -91,9 +106,12 @@ class SimulatedBoard:
         lux = max(0.0, 60000.0 * max(math.sin(math.radians(max(elev, 0.0))), 0.0)) + 8.0
         cpu = temp + 22.0 + 1.5 * self.rng.normal()
         # forward model must invert the compensator exactly, see scripts/simulate.py
-        k_true = 0.55
+        t_h = (temp + K_HTS221 * cpu) / (1.0 + K_HTS221) + SD_HTS221 * self.rng.normal()
+        t_p = (temp + K_LPS25HB * cpu) / (1.0 + K_LPS25HB) + SD_LPS25HB * self.rng.normal()
         return {
-            "temp_raw": (temp + k_true * cpu) / (1.0 + k_true) + 0.05 * self.rng.normal(),
+            "temp_raw": (t_h + t_p) / 2.0,
+            "temp_h": t_h,
+            "temp_p": t_p,
             "hum": rh + 0.4 * self.rng.normal(),
             "press": press + 0.05 * self.rng.normal(),
             "cpu_temp": cpu,

ashvale/station.py

@@ -152,6 +152,8 @@ class Station:
         row = {
             "ts": ts,
             "temp_raw": raw.get("temp_raw"),
+            "temp_h": raw.get("temp_h"),
+            "temp_p": raw.get("temp_p"),
             "temp_c": est["temp_c"],
             "temp_smooth": temp_c,
             "temp_rate": est["temp_rate"],

ashvale/storage.py

@@ -35,7 +35,8 @@ TIER_5MIN = 1
 TIER_HOUR = 2
 
 COLUMNS = [
-    "ts", "temp_raw", "temp_c", "temp_smooth", "temp_rate", "hum", "hum_smooth",
+    "ts", "temp_raw", "temp_h", "temp_p", "temp_c", "temp_smooth", "temp_rate",
+    "hum", "hum_smooth",
     "press", "press_slp", "press_smooth", "press_rate", "cpu_temp", "dew_c",
     "lux", "r", "g", "b", "pitch", "roll", "yaw", "compass",
     "ax", "ay", "az", "gx", "gy", "gz",
@@ -97,6 +98,28 @@ class Store:
         self._local = threading.local()
         with self._conn() as conn:
             conn.executescript(SCHEMA)
+            self._migrate(conn)
+
+    @staticmethod
+    def _migrate(conn: sqlite3.Connection) -> None:
+        """Add any column COLUMNS has gained since this database was created.
+
+        CREATE TABLE IF NOT EXISTS is a no-op against a table that already
+        exists, so a new entry in COLUMNS reaches a fresh install and silently
+        misses every station that has been running. The failure then surfaces
+        as an OperationalError inside insert_telemetry, which is on the sample
+        loop, so a column addition would take a live station down rather than
+        merely leaving a gap in its record.
+        """
+        have = {row[1] for row in conn.execute("PRAGMA table_info(telemetry)")}
+        if not have:
+            return
+        for col in COLUMNS:
+            if col in have:
+                continue
+            if not col.isidentifier():
+                raise ValueError(f"refusing to splice a non-identifier column: {col!r}")
+            conn.execute(f"ALTER TABLE telemetry ADD COLUMN {col} REAL")
 
     def _conn(self) -> sqlite3.Connection:
         conn = getattr(self._local, "conn", None)

scripts/simulate.py

@@ -58,6 +58,12 @@ from ashvale.physics import (  # noqa: E402
     sea_level_pressure,
     solar_position,
 )
+from ashvale.sensors import (  # noqa: E402
+    K_HTS221,
+    K_LPS25HB,
+    SD_HTS221,
+    SD_LPS25HB,
+)
 from ashvale.storage import Store  # noqa: E402
 
 
@@ -130,8 +136,15 @@ def generate(days: float, step_s: int, lat: float, lon: float,
     # model must be its exact inverse:  T_raw = (T + k T_cpu) / (1 + k).
     # Generating it any other way bakes a bias into the synthetic data that
     # no amount of calibration can remove, and quietly caps your skill score.
-    k_true = 0.55
-    temp_raw = (temp + k_true * cpu) / (1.0 + k_true) + 0.05 * rng.normal(size=n)
+    # Two thermometers, not one, because the board has two. Their forward
+    # models average to the k = 0.55 case this used to generate directly, so
+    # temp_raw is unchanged in expectation. Its noise is not: a real board
+    # averages sd 0.060 with sd 0.443 and lands at 0.223, where this used to
+    # claim 0.05. Simulating the quiet sensor and calling it the average is
+    # what let an over-optimistic measurement noise go unnoticed.
+    temp_h = (temp + K_HTS221 * cpu) / (1.0 + K_HTS221) + SD_HTS221 * rng.normal(size=n)
+    temp_p = (temp + K_LPS25HB * cpu) / (1.0 + K_LPS25HB) + SD_LPS25HB * rng.normal(size=n)
+    temp_raw = (temp_h + temp_p) / 2.0
     # If the compensator will move RH from the element temperature onto the air
     # temperature, the forward model here must be its exact inverse, or the
     # synthetic data bakes in a bias no calibration can remove. Same trap as the
@@ -146,7 +159,9 @@ def generate(days: float, step_s: int, lat: float, lon: float,
     press_station += 0.05 * rng.normal(size=n)
 
     return {
-        "ts": ts, "temp": temp, "temp_raw": temp_raw, "rh": rh_sensor + 0.4 * rng.normal(size=n),
+        "ts": ts, "temp": temp, "temp_raw": temp_raw,
+        "temp_h": temp_h, "temp_p": temp_p,
+        "rh": rh_sensor + 0.4 * rng.normal(size=n),
         "press": press_station, "press_slp": press_slp, "cpu": cpu,
         "lux": lux * (0.85 + 0.3 * rng.random(n)), "dew": dew, "cloud": cloud,
     }
@@ -201,6 +216,8 @@ def main() -> None:
         store.insert_telemetry({
             "ts": ts,
             "temp_raw": data["temp_raw"][i],
+            "temp_h": data["temp_h"][i],
+            "temp_p": data["temp_p"][i],
             "temp_c": est["temp_c"],
             "temp_smooth": est["temp_smooth"],
             "temp_rate": est["temp_rate"],

tests/test_storage.py

@@ -0,0 +1,93 @@
+# Copyright 2026 Kemal Yaylali
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Schema migration.
+
+The column-addition test is the important one. CREATE TABLE IF NOT EXISTS is a
+no-op against a table that already exists, so every new entry in COLUMNS
+reaches a fresh install and silently misses every station already running. It
+then surfaces as an OperationalError inside insert_telemetry, which sits on the
+sample loop, so a column addition takes a live station down rather than merely
+leaving a gap in its record.
+"""
+
+from __future__ import annotations
+
+import sqlite3
+import time
+
+import pytest
+
+from ashvale.storage import COLUMNS, Store
+
+
+def _cols(path: str) -> set[str]:
+    with sqlite3.connect(path) as c:
+        return {r[1] for r in c.execute("PRAGMA table_info(telemetry)")}
+
+
+def test_fresh_database_has_every_declared_column(tmp_path):
+    p = str(tmp_path / "fresh.db")
+    Store(p)
+    assert not [c for c in COLUMNS if c not in _cols(p)]
+
+
+def test_migration_adds_a_new_column_without_touching_the_rows(tmp_path):
+    """Simulates a station that has been running since before a column existed."""
+    p = str(tmp_path / "old.db")
+    legacy = [c for c in COLUMNS if c not in ("temp_h", "temp_p")]
+    with sqlite3.connect(p) as c:
+        c.execute(f"CREATE TABLE telemetry (ts REAL PRIMARY KEY, "
+                  f"{', '.join(f'{x} REAL' for x in legacy if x != 'ts')}, "
+                  f"tier INTEGER NOT NULL DEFAULT 0)")
+        c.executemany("INSERT INTO telemetry (ts, temp_raw) VALUES (?, ?)",
+                      [(float(i), 20.0 + i) for i in range(50)])
+
+    assert "temp_h" not in _cols(p)
+    store = Store(p)
+    assert "temp_h" in _cols(p) and "temp_p" in _cols(p)
+
+    with sqlite3.connect(p) as c:
+        n = c.execute("SELECT COUNT(*) FROM telemetry").fetchone()[0]
+        old = c.execute("SELECT temp_raw FROM telemetry WHERE ts = 7.0").fetchone()[0]
+    assert n == 50, "migration must not lose rows"
+    assert old == 27.0, "migration must not disturb existing values"
+
+    # The point of the exercise: a write using the new columns must now work.
+    store.insert_telemetry({"ts": 999.0, "temp_raw": 20.0, "temp_h": 20.6, "temp_p": 19.4})
+    with sqlite3.connect(p) as c:
+        row = c.execute("SELECT temp_h, temp_p FROM telemetry WHERE ts = 999.0").fetchone()
+    assert row == (20.6, 19.4)
+
+
+def test_migration_is_idempotent(tmp_path):
+    p = str(tmp_path / "twice.db")
+    Store(p)
+    Store(p)
+    Store(p)
+    assert not [c for c in COLUMNS if c not in _cols(p)]
+
+
+def test_both_thermometers_survive_a_round_trip(tmp_path):
+    """temp_h and temp_p are logged so the self-heating gradient can be
+    recovered later. They cannot be backfilled, so a silent drop is permanent."""
+    p = str(tmp_path / "rt.db")
+    store = Store(p)
+    now = time.time()
+    store.insert_telemetry({"ts": now, "temp_raw": 30.39, "temp_h": 30.973,
+                            "temp_p": 29.810, "cpu_temp": 44.55})
+    got = store.window(24.0, ["ts", "temp_h", "temp_p", "cpu_temp"])
+    assert got["temp_h"][0] == 30.973
+    assert got["temp_p"][0] == 29.810
+    assert got["temp_h"][0] - got["temp_p"][0] == pytest.approx(1.163)