Joystick labelling, a clock guard, and throttle logging

Three things the hardware offers that the code ignored.

The joystick has never had a line of code. Left records a dry label, right a
wet one, middle cycles the LED scene, and a full-panel flash acknowledges the
press because a headless box gives no other sign and a button you cannot tell
worked gets pressed twice. Precipitation is the weakest head in the bank and
strong labels are its binding constraint: this station has 80 of them against
thousands of proxy ones, entirely because the only label control lives in a web
page, and a web page is not where anyone is standing when it starts raining.

The board has no RTC, so a power cut without a network gives a clock somewhere
in 1970 on the next boot. Solar elevation, the diurnal harmonics and a sample's
position on the 5-minute grid then all lie with complete confidence, and unlike
a gap in the record the damage cannot be identified afterwards. train() now
refuses a clock below 2025 or one that has stepped behind the newest stored
row, and logs the refusal rather than training on fiction.

Undervoltage and thermal capping both shift the SoC temperature, which is the
regressor in the self-heating compensation, so a weak power supply presents as
an unexplained temperature bias rather than as anything resembling a power
problem. get_throttled is now sampled hourly and logged when set.

Measured and deliberately not done: colour features. r, g and b are logged and
74% of rows carry usable colour, but adding blue/red, green/red and saturation
made MAE 1.50% worse and helped in only 13 of 72 cases. Three more regressors
on a 33-feature model whose longest horizon trains on 13 independent pairs is
straightforwardly overfitting. That also prompted a sweep of the RLS prior and
forgetting factor in both directions; delta = 100 with lambda = 0.9985 is a
local optimum on both axes, so neither moved.

ashvale/api.py

@@ -61,6 +61,7 @@ async def lifespan(app: FastAPI):
     station.start()
     if CONFIG.server.led_enabled:
         display = LedDisplay(station, CONFIG.server.led_cycle_s, CONFIG.server.led_fps)
+        station.display = display          # lets the joystick drive the panel
         display.start()
     try:
         yield

ashvale/led.py

@@ -1025,6 +1025,28 @@ class LedDisplay:
         except Exception:
             pass
 
+    def next_scene(self) -> str:
+        """Advance to the next ambient scene, skipping the glyph track."""
+        self._prev = self._current()
+        self._show_glyph = False
+        self._idx = (self._idx + 1) % len(self.scenes)
+        now = time.monotonic()
+        self._fade_started = now
+        self._scene_started = now
+        return type(self.scenes[self._idx]).__name__
+
+    def flash(self, colour, ms: int = 450) -> None:
+        """Blocking confirmation flash. Used to acknowledge a joystick label.
+
+        A headless station gives no other feedback that a press registered, and
+        a button you cannot tell worked gets pressed twice.
+        """
+        try:
+            self.station.board.set_pixels([tuple(colour)] * 64)
+            time.sleep(ms / 1000.0)
+        except Exception:
+            pass
+
     def start(self) -> None:
         self._stop.clear()
         self._task = asyncio.create_task(self._run())

ashvale/sensors.py

@@ -26,9 +26,10 @@ from __future__ import annotations
 
 import logging
 import math
+import subprocess
 import time
 from pathlib import Path
-from typing import Any, Dict, Optional
+from typing import Any, Dict, List, Optional, Tuple
 
 import numpy as np
 
@@ -42,6 +43,37 @@ TCS3400_CONTROL = 0x8F
 TCS3400_CDATA = 0x94
 
 
+def read_throttled() -> Optional[Dict[str, Any]]:
+    """Raspberry Pi undervoltage and throttling flags, or None if all clear.
+
+    Bit 0 is undervoltage now, 16 is undervoltage since boot, 2 is arm
+    frequency capped, 3 is thermal throttling. A capped or browning-out board
+    runs its SoC at a different temperature, and the SoC temperature is the
+    regressor in the self-heating compensation, so the visible symptom is a
+    temperature bias with no apparent cause.
+    """
+    try:
+        out = subprocess.run(["vcgencmd", "get_throttled"], capture_output=True,
+                             text=True, timeout=5).stdout.strip()
+    except (OSError, subprocess.SubprocessError):
+        return None
+    if "=" not in out:
+        return None
+    try:
+        bits = int(out.split("=", 1)[1], 0)
+    except ValueError:
+        return None
+    if bits == 0:
+        return None
+    now = {0: "undervoltage", 1: "arm_capped", 2: "throttled", 3: "soft_temp_limit"}
+    ever = {16: "undervoltage_since_boot", 17: "arm_capped_since_boot",
+            18: "throttled_since_boot", 19: "soft_temp_limit_since_boot"}
+    active = [name for bit, name in now.items() if bits & (1 << bit)]
+    historic = [name for bit, name in ever.items() if bits & (1 << bit)]
+    return {"raw": hex(bits), "active": active, "since_boot": historic,
+            "severity": "warn" if active else "info"}
+
+
 def read_cpu_temperature() -> float:
     """Core temperature in C. This is the single most important nuisance
     variable on a Sense HAT: the HTS221 and LPS25HB sit millimetres above a
@@ -387,6 +419,19 @@ class SenseBoard:
             "simulated": False,
         }
 
+    def stick_events(self) -> List[Tuple[str, str]]:
+        """Pending joystick events as (direction, action), oldest first.
+
+        Non-blocking, and returns [] when nothing has happened. The library
+        buffers events, so polling slowly loses none of them.
+        """
+        if self.sense is None:
+            return []
+        try:
+            return [(e.direction, e.action) for e in self.sense.stick.get_events()]
+        except Exception:
+            return []
+
     # --------------------------------------------------------------- LED
 
     def clear(self, *args):

ashvale/station.py

@@ -49,7 +49,7 @@ from .models.anomaly import AnomalyMonitor
 from .models.climatology import HarmonicClimatology
 from .models.nowcast import NowcastEnsemble
 from .models.precip import PrecipitationModel, proxy_wet_label, zambretti
-from .sensors import OutdoorProbe, SenseBoard, enrich
+from .sensors import OutdoorProbe, SenseBoard, enrich, read_throttled
 from .storage import Store, resample
 
 STATE_VERSION = 1
@@ -67,6 +67,9 @@ class Station:
             longitude=cfg.site.longitude,
         )
         # Optional and entirely absent on a board without one wired up.
+        # Set by the API layer once the LED display exists, so the joystick
+        # can acknowledge a press and cycle scenes. None when there is no HAT.
+        self.display = None
         self.probe = (OutdoorProbe(cfg.sensor.outdoor_probe_period_s)
                       if cfg.sensor.outdoor_probe else None)
         self.tracker = SignalTracker(cfg)
@@ -490,6 +493,22 @@ class Station:
 
     # ------------------------------------------------------------ train
 
+    # 2025-01-01. Any clock below this has not been set since boot, because
+    # this project did not exist before it.
+    CLOCK_FLOOR = 1735689600.0
+
+    def clock_sanity(self) -> Dict[str, Any]:
+        """Is the wall clock usable for anything time-dependent?"""
+        now = time.time()
+        if now < self.CLOCK_FLOOR:
+            return {"ok": False, "now": now,
+                    "reason": "clock is before 2025, so it has not been set since boot"}
+        newest = self.store.newest_ts()
+        if newest is not None and now < newest - 60.0:
+            return {"ok": False, "now": now, "newest": newest,
+                    "reason": f"clock is {newest - now:.0f}s behind the newest stored row"}
+        return {"ok": True, "now": now}
+
     def build_training_grid(self, hours: float = 24 * 30):
         raw = self.store.window(hours, ["ts", "temp_smooth", "hum_smooth",
                                         "press_slp", "lux"])
@@ -513,6 +532,16 @@ class Station:
 
     def train(self, hours: float = 24 * 30) -> Dict:
         t_start = time.time()
+        clock = self.clock_sanity()
+        if not clock["ok"]:
+            # The board has no RTC. A power cut without a network gives a clock
+            # somewhere in 1970 on the next boot, and every feature that depends
+            # on absolute time then lies with total confidence: solar elevation,
+            # the diurnal harmonics, the position of a sample on the 5-minute
+            # grid. Training on that poisons the weights, and unlike a gap in
+            # the record it cannot be spotted afterwards.
+            self.store.log_event("clock", "error", json.dumps(clock))
+            return {"trained": False, "reason": clock["reason"]}
         built = self.build_training_grid(hours)
         if built is None:
             return {"trained": False,
@@ -699,10 +728,61 @@ class Station:
                 self.store.log_event("verify", "error", repr(exc))
             await asyncio.sleep(300)
 
+    # Joystick bindings. Left and right are the two answers to the only
+    # question the precipitation model cannot answer for itself.
+    STICK_LABELS = {"left": 0.0, "right": 1.0}
+    STICK_COLOURS = {0.0: (90, 90, 110), 1.0: (40, 110, 220)}
+
+    async def _loop_joystick(self):
+        """Rain labels without a browser.
+
+        Precipitation is the weakest model in the bank and it is starved of the
+        only thing that would fix it. This station has 80 strong labels against
+        thousands of proxy ones, because the label button lives in a web page
+        and a web page is not where anyone is standing when it starts raining.
+        A button on the device is the whole difference between labelling and
+        intending to label.
+
+        Middle cycles the LED scene, which is the other thing you want from a
+        headless box and otherwise requires a laptop.
+        """
+        while not self._stop.is_set():
+            try:
+                for direction, action in self.board.stick_events():
+                    if action != "pressed":
+                        continue
+                    if direction in self.STICK_LABELS:
+                        value = self.STICK_LABELS[direction]
+                        result = await asyncio.to_thread(
+                            self.add_label, "rain", value, None, "joystick")
+                        self.store.log_event(
+                            "joystick", "info",
+                            json.dumps({"direction": direction, "rain": value,
+                                        "strong_labels": result.get("strong_labels")}))
+                        if self.display is not None:
+                            await asyncio.to_thread(
+                                self.display.flash, self.STICK_COLOURS[value])
+                    elif direction == "middle" and self.display is not None:
+                        name = self.display.next_scene()
+                        self.store.log_event("joystick", "info",
+                                             json.dumps({"scene": name}))
+            except Exception as exc:
+                self.store.log_event("joystick", "error", repr(exc))
+            await asyncio.sleep(0.25)
+
     async def _loop_maintenance(self):
         while not self._stop.is_set():
             await asyncio.sleep(3600)
             now = time.time()
+            # Undervoltage and thermal capping both move the SoC temperature,
+            # which is the input to the self-heating compensation, so a weak
+            # supply shows up as a temperature bias rather than as anything
+            # that looks like a power problem. Recorded so the anomaly is
+            # labelled rather than mysterious.
+            flags = read_throttled()
+            if flags:
+                self.store.log_event("throttled", flags["severity"],
+                                     json.dumps(flags))
             if now - self.last_compact >= self.cfg.storage.vacuum_period_s:
                 try:
                     removed = await asyncio.to_thread(
@@ -723,6 +803,7 @@ class Station:
             asyncio.create_task(self._loop_train()),
             asyncio.create_task(self._loop_verify()),
             asyncio.create_task(self._loop_maintenance()),
+            asyncio.create_task(self._loop_joystick()),
         ]
 
     async def stop(self) -> None:

ashvale/storage.py

@@ -417,6 +417,11 @@ class Store:
         with self._conn() as conn:
             return int(conn.execute("SELECT COUNT(*) FROM telemetry").fetchone()[0])
 
+    def newest_ts(self) -> Optional[float]:
+        with self._conn() as conn:
+            row = conn.execute("SELECT MAX(ts) FROM telemetry").fetchone()
+        return float(row[0]) if row and row[0] is not None else None
+
     def span_days(self) -> float:
         with self._conn() as conn:
             row = conn.execute("SELECT MIN(ts), MAX(ts) FROM telemetry").fetchone()

tests/test_recompute.py

@@ -122,3 +122,111 @@ def test_calibration_logs_a_discontinuity_marker(client):
     kinds = [e["kind"] for e in client.get("/api/status").json()["events"]]
     assert "discontinuity" in kinds
     client.post("/api/calibrate/humidity", json={"reset": True})
+
+
+# ------------------------------------------------------------ clock guard
+
+def test_training_refuses_a_clock_that_has_not_been_set(tmp_path, monkeypatch):
+    """The board has no RTC.
+
+    A power cut without a network gives a clock somewhere in 1970 on the next
+    boot. Solar elevation, the diurnal harmonics and a sample's position on the
+    5-minute grid all then lie with total confidence, and unlike a gap in the
+    record the damage cannot be spotted afterwards.
+    """
+    import time as _time
+
+    from ashvale.config import load_config
+    from ashvale.station import Station
+
+    cfg = load_config()
+    cfg.storage.db_path = str(tmp_path / "clock.db")
+    st = Station(cfg)
+
+    assert st.clock_sanity()["ok"], "a correct clock must pass"
+
+    monkeypatch.setattr(_time, "time", lambda: 1000.0)      # 1970
+    verdict = st.clock_sanity()
+    assert not verdict["ok"]
+    assert "2025" in verdict["reason"]
+    result = st.train()
+    assert result["trained"] is False
+    # and specifically for the clock, not because the database is empty
+    assert "2025" in result["reason"], result["reason"]
+
+
+def test_training_refuses_a_clock_that_went_backwards(tmp_path, monkeypatch):
+    """NTP stepping backwards past stored data is equally unusable."""
+    import time as _time
+
+    from ashvale.config import load_config
+    from ashvale.station import Station
+
+    cfg = load_config()
+    cfg.storage.db_path = str(tmp_path / "back.db")
+    st = Station(cfg)
+    future = _time.time() + 7200.0
+    st.store.insert_telemetry({"ts": future, "temp_raw": 20.0})
+
+    verdict = st.clock_sanity()
+    assert not verdict["ok"]
+    assert "behind" in verdict["reason"]
+
+
+# ------------------------------------------------------------ joystick
+
+def test_joystick_left_and_right_record_rain_labels(tmp_path):
+    """The button that fixes the precipitation model.
+
+    Strong labels are the binding constraint on that head: 80 against thousands
+    of proxy ones on a real station, because the only label control lives in a
+    web page. Left is dry, right is wet.
+    """
+    import asyncio
+    import sqlite3
+
+    from ashvale.config import load_config
+    from ashvale.station import Station
+
+    cfg = load_config()
+    cfg.storage.db_path = str(tmp_path / "stick.db")
+    st = Station(cfg)
+    st.sample_once()
+
+    pending = [("left", "pressed"), ("right", "pressed"),
+               ("up", "pressed"), ("right", "released")]
+
+    def fake_events():
+        out, pending[:] = list(pending), []
+        return out
+
+    st.board.stick_events = fake_events
+
+    async def one_pass():
+        task = asyncio.create_task(st._loop_joystick())
+        await asyncio.sleep(0.6)
+        st._stop.set()
+        task.cancel()
+        try:
+            await task
+        except (asyncio.CancelledError, Exception):
+            pass
+
+    asyncio.run(one_pass())
+
+    with sqlite3.connect(cfg.storage.db_path) as c:
+        rows = sorted(r[0] for r in c.execute("SELECT value FROM labels WHERE kind='rain'"))
+    assert rows == [0.0, 1.0], f"expected one dry and one wet label, got {rows}"
+    # 'up' is unbound and 'released' is not a press: neither may label anything.
+
+
+def test_joystick_survives_a_board_with_no_hat(tmp_path):
+    """The simulator path has no stick. The loop must not spin on exceptions."""
+    from ashvale.config import load_config
+    from ashvale.station import Station
+
+    cfg = load_config()
+    cfg.storage.db_path = str(tmp_path / "nohat.db")
+    st = Station(cfg)
+    assert st.board.stick_events() == []
+    assert st.display is None