diff --git a/ashvale/sensors.py b/ashvale/sensors.py index dda3aac..e26e2d6 100644 --- a/ashvale/sensors.py +++ b/ashvale/sensors.py @@ -53,6 +53,21 @@ def read_cpu_temperature() -> float: return float("nan") +# Per-chip thermal coupling to the SoC, and per-chip noise. +# +# The Sense HAT carries two independent thermometers at different distances +# from the SoC, and they are not equally good. Measured over 12 samples on a +# real board: HTS221 30.973 C at sd 0.060, LPS25HB 29.810 C at sd 0.443, a +# standing gradient of 1.163 C with the SoC at 44.55 C. +# +# These two couplings are chosen so their forward models average to exactly the +# k = 0.55 the compensator is tuned against. The aggregate behaviour is +# therefore unchanged and only the per-channel detail is new, which matters +# because that gradient is a second observation of self-heating. +K_HTS221, K_LPS25HB = 0.6164, 0.4889 +SD_HTS221, SD_LPS25HB = 0.060, 0.443 + + class SimulatedBoard: """Ornstein-Uhlenbeck weather with a diurnal driver. Good enough to exercise every code path and to sanity-check a model's skill score.""" @@ -91,9 +106,12 @@ class SimulatedBoard: lux = max(0.0, 60000.0 * max(math.sin(math.radians(max(elev, 0.0))), 0.0)) + 8.0 cpu = temp + 22.0 + 1.5 * self.rng.normal() # forward model must invert the compensator exactly, see scripts/simulate.py - k_true = 0.55 + t_h = (temp + K_HTS221 * cpu) / (1.0 + K_HTS221) + SD_HTS221 * self.rng.normal() + t_p = (temp + K_LPS25HB * cpu) / (1.0 + K_LPS25HB) + SD_LPS25HB * self.rng.normal() return { - "temp_raw": (temp + k_true * cpu) / (1.0 + k_true) + 0.05 * self.rng.normal(), + "temp_raw": (t_h + t_p) / 2.0, + "temp_h": t_h, + "temp_p": t_p, "hum": rh + 0.4 * self.rng.normal(), "press": press + 0.05 * self.rng.normal(), "cpu_temp": cpu, diff --git a/ashvale/station.py b/ashvale/station.py index 50c46cd..d221fc9 100644 --- a/ashvale/station.py +++ b/ashvale/station.py @@ -152,6 +152,8 @@ class Station: row = { "ts": ts, "temp_raw": raw.get("temp_raw"), + "temp_h": raw.get("temp_h"), + "temp_p": raw.get("temp_p"), "temp_c": est["temp_c"], "temp_smooth": temp_c, "temp_rate": est["temp_rate"], diff --git a/ashvale/storage.py b/ashvale/storage.py index 0c20501..1a972f2 100644 --- a/ashvale/storage.py +++ b/ashvale/storage.py @@ -35,7 +35,8 @@ TIER_5MIN = 1 TIER_HOUR = 2 COLUMNS = [ - "ts", "temp_raw", "temp_c", "temp_smooth", "temp_rate", "hum", "hum_smooth", + "ts", "temp_raw", "temp_h", "temp_p", "temp_c", "temp_smooth", "temp_rate", + "hum", "hum_smooth", "press", "press_slp", "press_smooth", "press_rate", "cpu_temp", "dew_c", "lux", "r", "g", "b", "pitch", "roll", "yaw", "compass", "ax", "ay", "az", "gx", "gy", "gz", @@ -97,6 +98,28 @@ class Store: self._local = threading.local() with self._conn() as conn: conn.executescript(SCHEMA) + self._migrate(conn) + + @staticmethod + def _migrate(conn: sqlite3.Connection) -> None: + """Add any column COLUMNS has gained since this database was created. + + CREATE TABLE IF NOT EXISTS is a no-op against a table that already + exists, so a new entry in COLUMNS reaches a fresh install and silently + misses every station that has been running. The failure then surfaces + as an OperationalError inside insert_telemetry, which is on the sample + loop, so a column addition would take a live station down rather than + merely leaving a gap in its record. + """ + have = {row[1] for row in conn.execute("PRAGMA table_info(telemetry)")} + if not have: + return + for col in COLUMNS: + if col in have: + continue + if not col.isidentifier(): + raise ValueError(f"refusing to splice a non-identifier column: {col!r}") + conn.execute(f"ALTER TABLE telemetry ADD COLUMN {col} REAL") def _conn(self) -> sqlite3.Connection: conn = getattr(self._local, "conn", None) diff --git a/scripts/simulate.py b/scripts/simulate.py index b8ca00b..2106356 100644 --- a/scripts/simulate.py +++ b/scripts/simulate.py @@ -58,6 +58,12 @@ from ashvale.physics import ( # noqa: E402 sea_level_pressure, solar_position, ) +from ashvale.sensors import ( # noqa: E402 + K_HTS221, + K_LPS25HB, + SD_HTS221, + SD_LPS25HB, +) from ashvale.storage import Store # noqa: E402 @@ -130,8 +136,15 @@ def generate(days: float, step_s: int, lat: float, lon: float, # model must be its exact inverse: T_raw = (T + k T_cpu) / (1 + k). # Generating it any other way bakes a bias into the synthetic data that # no amount of calibration can remove, and quietly caps your skill score. - k_true = 0.55 - temp_raw = (temp + k_true * cpu) / (1.0 + k_true) + 0.05 * rng.normal(size=n) + # Two thermometers, not one, because the board has two. Their forward + # models average to the k = 0.55 case this used to generate directly, so + # temp_raw is unchanged in expectation. Its noise is not: a real board + # averages sd 0.060 with sd 0.443 and lands at 0.223, where this used to + # claim 0.05. Simulating the quiet sensor and calling it the average is + # what let an over-optimistic measurement noise go unnoticed. + temp_h = (temp + K_HTS221 * cpu) / (1.0 + K_HTS221) + SD_HTS221 * rng.normal(size=n) + temp_p = (temp + K_LPS25HB * cpu) / (1.0 + K_LPS25HB) + SD_LPS25HB * rng.normal(size=n) + temp_raw = (temp_h + temp_p) / 2.0 # If the compensator will move RH from the element temperature onto the air # temperature, the forward model here must be its exact inverse, or the # synthetic data bakes in a bias no calibration can remove. Same trap as the @@ -146,7 +159,9 @@ def generate(days: float, step_s: int, lat: float, lon: float, press_station += 0.05 * rng.normal(size=n) return { - "ts": ts, "temp": temp, "temp_raw": temp_raw, "rh": rh_sensor + 0.4 * rng.normal(size=n), + "ts": ts, "temp": temp, "temp_raw": temp_raw, + "temp_h": temp_h, "temp_p": temp_p, + "rh": rh_sensor + 0.4 * rng.normal(size=n), "press": press_station, "press_slp": press_slp, "cpu": cpu, "lux": lux * (0.85 + 0.3 * rng.random(n)), "dew": dew, "cloud": cloud, } @@ -201,6 +216,8 @@ def main() -> None: store.insert_telemetry({ "ts": ts, "temp_raw": data["temp_raw"][i], + "temp_h": data["temp_h"][i], + "temp_p": data["temp_p"][i], "temp_c": est["temp_c"], "temp_smooth": est["temp_smooth"], "temp_rate": est["temp_rate"], diff --git a/tests/test_storage.py b/tests/test_storage.py new file mode 100644 index 0000000..ef47302 --- /dev/null +++ b/tests/test_storage.py @@ -0,0 +1,93 @@ +# Copyright 2026 Kemal Yaylali +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Schema migration. + +The column-addition test is the important one. CREATE TABLE IF NOT EXISTS is a +no-op against a table that already exists, so every new entry in COLUMNS +reaches a fresh install and silently misses every station already running. It +then surfaces as an OperationalError inside insert_telemetry, which sits on the +sample loop, so a column addition takes a live station down rather than merely +leaving a gap in its record. +""" + +from __future__ import annotations + +import sqlite3 +import time + +import pytest + +from ashvale.storage import COLUMNS, Store + + +def _cols(path: str) -> set[str]: + with sqlite3.connect(path) as c: + return {r[1] for r in c.execute("PRAGMA table_info(telemetry)")} + + +def test_fresh_database_has_every_declared_column(tmp_path): + p = str(tmp_path / "fresh.db") + Store(p) + assert not [c for c in COLUMNS if c not in _cols(p)] + + +def test_migration_adds_a_new_column_without_touching_the_rows(tmp_path): + """Simulates a station that has been running since before a column existed.""" + p = str(tmp_path / "old.db") + legacy = [c for c in COLUMNS if c not in ("temp_h", "temp_p")] + with sqlite3.connect(p) as c: + c.execute(f"CREATE TABLE telemetry (ts REAL PRIMARY KEY, " + f"{', '.join(f'{x} REAL' for x in legacy if x != 'ts')}, " + f"tier INTEGER NOT NULL DEFAULT 0)") + c.executemany("INSERT INTO telemetry (ts, temp_raw) VALUES (?, ?)", + [(float(i), 20.0 + i) for i in range(50)]) + + assert "temp_h" not in _cols(p) + store = Store(p) + assert "temp_h" in _cols(p) and "temp_p" in _cols(p) + + with sqlite3.connect(p) as c: + n = c.execute("SELECT COUNT(*) FROM telemetry").fetchone()[0] + old = c.execute("SELECT temp_raw FROM telemetry WHERE ts = 7.0").fetchone()[0] + assert n == 50, "migration must not lose rows" + assert old == 27.0, "migration must not disturb existing values" + + # The point of the exercise: a write using the new columns must now work. + store.insert_telemetry({"ts": 999.0, "temp_raw": 20.0, "temp_h": 20.6, "temp_p": 19.4}) + with sqlite3.connect(p) as c: + row = c.execute("SELECT temp_h, temp_p FROM telemetry WHERE ts = 999.0").fetchone() + assert row == (20.6, 19.4) + + +def test_migration_is_idempotent(tmp_path): + p = str(tmp_path / "twice.db") + Store(p) + Store(p) + Store(p) + assert not [c for c in COLUMNS if c not in _cols(p)] + + +def test_both_thermometers_survive_a_round_trip(tmp_path): + """temp_h and temp_p are logged so the self-heating gradient can be + recovered later. They cannot be backfilled, so a silent drop is permanent.""" + p = str(tmp_path / "rt.db") + store = Store(p) + now = time.time() + store.insert_telemetry({"ts": now, "temp_raw": 30.39, "temp_h": 30.973, + "temp_p": 29.810, "cpu_temp": 44.55}) + got = store.window(24.0, ["ts", "temp_h", "temp_p", "cpu_temp"]) + assert got["temp_h"][0] == 30.973 + assert got["temp_p"][0] == 29.810 + assert got["temp_h"][0] - got["temp_p"][0] == pytest.approx(1.163)