kemal/ashvale-station
1
0
Fork 0
mirror of https://github.com/lynchaos/ashvale-station.git synced 2026-09-12 20:52:23 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
8515f4fb030df609fb4853b550fb6d4f1be7b97d
ashvale-station/ashvale/storage.py
T
kemalandClaude Opus 5 d5b3cdad7c Joystick labelling, a clock guard, and throttle logging 
Three things the hardware offers that the code ignored.

The joystick has never had a line of code. Left records a dry label, right a
wet one, middle cycles the LED scene, and a full-panel flash acknowledges the
press because a headless box gives no other sign and a button you cannot tell
worked gets pressed twice. Precipitation is the weakest head in the bank and
strong labels are its binding constraint: this station has 80 of them against
thousands of proxy ones, entirely because the only label control lives in a web
page, and a web page is not where anyone is standing when it starts raining.

The board has no RTC, so a power cut without a network gives a clock somewhere
in 1970 on the next boot. Solar elevation, the diurnal harmonics and a sample's
position on the 5-minute grid then all lie with complete confidence, and unlike
a gap in the record the damage cannot be identified afterwards. train() now
refuses a clock below 2025 or one that has stepped behind the newest stored
row, and logs the refusal rather than training on fiction.

Undervoltage and thermal capping both shift the SoC temperature, which is the
regressor in the self-heating compensation, so a weak power supply presents as
an unexplained temperature bias rather than as anything resembling a power
problem. get_throttled is now sampled hourly and logged when set.

Measured and deliberately not done: colour features. r, g and b are logged and
74% of rows carry usable colour, but adding blue/red, green/red and saturation
made MAE 1.50% worse and helped in only 13 of 72 cases. Three more regressors
on a 33-feature model whose longest horizon trains on 13 independent pairs is
straightforwardly overfitting. That also prompted a sweep of the RLS prior and
forgetting factor in both directions; delta = 100 with lambda = 0.9985 is a
local optimum on both axes, so neither moved.

Co-Authored-By: Claude Opus 5 <[email protected]>
2026-08-19 19:45:47 +01:00

562 lines
24 KiB
Python
Raw Blame History

# Copyright 2026 Kemal Yaylali
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Durable storage: SQLite in WAL mode with tiered downsampling.
An SD card is a consumable. The write pattern here is deliberately
gentle: one row every `persist_period_s`, WAL journalling, a compaction
pass that folds week-old raw rows into 5-minute means and quarter-old
5-minute rows into hourly means. A year of station history lands around
30 MB, which the Pi will not notice.
"""
from __future__ import annotations
import sqlite3
import threading
import time
from typing import Any, Dict, Iterable, List, Optional
import numpy as np
TIER_RAW = 0
TIER_5MIN = 1
TIER_HOUR = 2
COLUMNS = [
"ts", "temp_raw", "temp_h", "temp_p", "temp_c", "temp_smooth", "temp_rate",
"hum", "hum_smooth",
"press", "press_slp", "press_smooth", "press_rate", "cpu_temp", "dew_c",
"lux", "r", "g", "b", "pitch", "roll", "yaw", "compass",
"ax", "ay", "az", "gx", "gy", "gz",
]
SCHEMA = f"""
PRAGMA journal_mode=WAL;
PRAGMA synchronous=NORMAL;
PRAGMA temp_store=MEMORY;
CREATE TABLE IF NOT EXISTS telemetry (
ts REAL PRIMARY KEY,
{", ".join(f"{c} REAL" for c in COLUMNS if c != "ts")},
tier INTEGER NOT NULL DEFAULT 0
);
CREATE INDEX IF NOT EXISTS idx_telemetry_tier_ts ON telemetry(tier, ts);
CREATE TABLE IF NOT EXISTS forecasts (
issued_ts REAL NOT NULL,
valid_ts REAL NOT NULL,
horizon_s INTEGER NOT NULL,
target TEXT NOT NULL,
mu REAL, lo REAL, hi REAL,
model TEXT,
PRIMARY KEY (issued_ts, horizon_s, target)
);
CREATE INDEX IF NOT EXISTS idx_forecast_valid ON forecasts(valid_ts);
CREATE TABLE IF NOT EXISTS labels (
ts REAL NOT NULL,
kind TEXT NOT NULL,
value REAL NOT NULL,
note TEXT,
PRIMARY KEY (ts, kind)
);
CREATE TABLE IF NOT EXISTS scores (
ts REAL NOT NULL,
target TEXT NOT NULL,
horizon_s INTEGER NOT NULL,
mae REAL, rmse REAL, bias REAL,
mae_persistence REAL, skill REAL, coverage REAL, n INTEGER,
PRIMARY KEY (ts, target, horizon_s)
);
CREATE TABLE IF NOT EXISTS events (
ts REAL NOT NULL,
kind TEXT NOT NULL,
severity TEXT,
detail TEXT
);
CREATE INDEX IF NOT EXISTS idx_events_ts ON events(ts);
"""
class Store:
def __init__(self, path: str):
self.path = path
self._local = threading.local()
with self._conn() as conn:
conn.executescript(SCHEMA)
self._migrate(conn)
@staticmethod
def _migrate(conn: sqlite3.Connection) -> None:
"""Add any column COLUMNS has gained since this database was created.
CREATE TABLE IF NOT EXISTS is a no-op against a table that already
exists, so a new entry in COLUMNS reaches a fresh install and silently
misses every station that has been running. The failure then surfaces
as an OperationalError inside insert_telemetry, which is on the sample
loop, so a column addition would take a live station down rather than
merely leaving a gap in its record.
"""
have = {row[1] for row in conn.execute("PRAGMA table_info(telemetry)")}
if not have:
return
for col in COLUMNS:
if col in have:
continue
if not col.isidentifier():
raise ValueError(f"refusing to splice a non-identifier column: {col!r}")
conn.execute(f"ALTER TABLE telemetry ADD COLUMN {col} REAL")
def _conn(self) -> sqlite3.Connection:
conn = getattr(self._local, "conn", None)
if conn is None:
conn = sqlite3.connect(self.path, timeout=20.0, check_same_thread=False)
conn.row_factory = sqlite3.Row
self._local.conn = conn
return conn
# ------------------------------------------------------------ writes
def insert_telemetry(self, row: Dict[str, Any], tier: int = TIER_RAW) -> None:
payload = {c: float(row.get(c)) if row.get(c) is not None else None for c in COLUMNS}
payload["tier"] = tier
cols = ", ".join(payload.keys())
marks = ", ".join("?" for _ in payload)
with self._conn() as conn:
conn.execute(
f"INSERT OR REPLACE INTO telemetry ({cols}) VALUES ({marks})",
list(payload.values()),
)
def insert_forecast(self, issued_ts: float, horizon_s: int, target: str,
mu: float, lo: float, hi: float, model: str) -> None:
with self._conn() as conn:
conn.execute(
"INSERT OR REPLACE INTO forecasts "
"(issued_ts, valid_ts, horizon_s, target, mu, lo, hi, model) "
"VALUES (?,?,?,?,?,?,?,?)",
(issued_ts, issued_ts + horizon_s, horizon_s, target,
float(mu), float(lo), float(hi), model),
)
def insert_label(self, ts: float, kind: str, value: float, note: str = "") -> None:
with self._conn() as conn:
conn.execute(
"INSERT OR REPLACE INTO labels (ts, kind, value, note) VALUES (?,?,?,?)",
(ts, kind, float(value), note),
)
def insert_score(self, ts: float, target: str, horizon_s: int, **kw) -> None:
with self._conn() as conn:
conn.execute(
"INSERT OR REPLACE INTO scores "
"(ts, target, horizon_s, mae, rmse, bias, mae_persistence, skill, coverage, n) "
"VALUES (?,?,?,?,?,?,?,?,?,?)",
(ts, target, horizon_s, kw.get("mae"), kw.get("rmse"), kw.get("bias"),
kw.get("mae_persistence"), kw.get("skill"), kw.get("coverage"), kw.get("n")),
)
def log_event(self, kind: str, severity: str, detail: str, ts: Optional[float] = None) -> None:
with self._conn() as conn:
conn.execute("INSERT INTO events (ts, kind, severity, detail) VALUES (?,?,?,?)",
(ts or time.time(), kind, severity, detail))
# ------------------------------------------------------------- reads
def all_for_recompute(self) -> Dict[str, np.ndarray]:
"""Every stored row's *raw* inputs, oldest first.
Only the columns a re-derivation actually needs. The raw sensor values
are never overwritten, which is precisely what makes recomputation
possible after a calibration changes k or the humidity offset.
"""
cols = ["ts", "temp_raw", "cpu_temp", "hum", "press"]
with self._conn() as conn:
rows = conn.execute(
f"SELECT {', '.join(cols)} FROM telemetry ORDER BY ts ASC").fetchall()
if not rows:
return {c: np.empty(0) for c in cols}
arr = np.array(rows, dtype=object)
out = {}
for i, c in enumerate(cols):
out[c] = np.array([np.nan if v is None else float(v) for v in arr[:, i]],
dtype=float)
return out
def apply_recompute(self, ts: np.ndarray, updates: Dict[str, np.ndarray],
chunk: int = 2000) -> int:
"""Write recomputed derived columns back, in chunks.
Chunked because a year of tiered history is a six-figure row count and a
single statement would hold the whole parameter list in memory on a
512 MB board.
"""
names = list(updates.keys())
sql = (f"UPDATE telemetry SET {', '.join(n + ' = ?' for n in names)} "
f"WHERE ts = ?")
n_written = 0
with self._conn() as conn:
for start in range(0, ts.size, chunk):
stop = min(start + chunk, ts.size)
batch = [
tuple(
[None if not np.isfinite(updates[n][i]) else float(updates[n][i])
for n in names] + [float(ts[i])]
)
for i in range(start, stop)
]
conn.executemany(sql, batch)
n_written += len(batch)
return n_written
def window(self, hours: float, columns: Optional[Iterable[str]] = None) -> Dict[str, np.ndarray]:
"""Return the last `hours` of telemetry as column arrays, oldest first."""
cols = list(columns) if columns else COLUMNS
since = time.time() - hours * 3600.0
with self._conn() as conn:
cur = conn.execute(
f"SELECT {', '.join(cols)} FROM telemetry WHERE ts >= ? ORDER BY ts ASC",
(since,),
)
rows = cur.fetchall()
if not rows:
return {c: np.empty(0, dtype=float) for c in cols}
arr = np.array([[r[c] if r[c] is not None else np.nan for c in cols] for r in rows],
dtype=float)
return {c: arr[:, i] for i, c in enumerate(cols)}
# ------------------------------------------------- historical access
@staticmethod
def auto_bucket(start: float, end: float, target_points: int = 700) -> int:
"""Pick a sensible aggregation bucket for a requested span.
The browser cannot draw more than about a thousand points usefully
and the Pi should not serialise more than it must, so the bucket
grows with the span. Snapped to familiar durations so the x-axis
reads in round numbers rather than 437-second increments.
"""
span = max(float(end) - float(start), 1.0)
raw = span / max(int(target_points), 1)
ladder = [30, 60, 120, 300, 600, 900, 1800, 3600, 7200,
10800, 21600, 43200, 86400, 604800]
for step in ladder:
if raw <= step:
return step
return ladder[-1]
def range_series(self, start: float, end: float,
bucket_s: Optional[int] = None) -> Dict[str, Any]:
"""Bucket-aggregated telemetry between two epoch timestamps.
Aggregation happens in SQLite rather than numpy: pulling 90 days of
rows into Python to average them would cost more memory than the
Zero 2 W has to spare. Min and max travel alongside the mean so the
UI can shade a true range band instead of implying the mean was the
whole story.
"""
start, end = float(start), float(end)
if end <= start:
return {"n": 0, "bucket_s": 0, "series": {}}
bucket = int(bucket_s or self.auto_bucket(start, end))
# The alias must not be a bare single letter: the telemetry table has
# r, g and b colour columns, and SQLite resolves an unqualified name in
# GROUP BY to a real column before a result alias. `GROUP BY b` silently
# grouped by the blue channel and returned one row per sample while
# cheerfully reporting the requested bucket size.
sql = f"""
SELECT CAST(ts / {bucket} AS INTEGER) * {bucket} AS bucket_ts,
AVG(temp_smooth) AS temp, MIN(temp_smooth) AS temp_lo,
MAX(temp_smooth) AS temp_hi,
AVG(hum_smooth) AS hum, MIN(hum_smooth) AS hum_lo,
MAX(hum_smooth) AS hum_hi,
AVG(press_slp) AS press, MIN(press_slp) AS press_lo,
MAX(press_slp) AS press_hi,
AVG(dew_c) AS dew, AVG(lux) AS lux,
AVG(temp_rate) AS temp_rate,
AVG(press_rate) AS press_rate,
AVG(cpu_temp) AS cpu, COUNT(*) AS n
FROM telemetry
WHERE ts >= ? AND ts <= ?
GROUP BY bucket_ts ORDER BY bucket_ts ASC
"""
with self._conn() as conn:
rows = conn.execute(sql, (start, end)).fetchall()
if not rows:
return {"n": 0, "bucket_s": bucket, "series": {}}
keys = ["temp", "temp_lo", "temp_hi", "hum", "hum_lo", "hum_hi",
"press", "press_lo", "press_hi", "dew", "lux",
"temp_rate", "press_rate", "cpu", "n"]
out: Dict[str, list] = {"ts": [float(r["bucket_ts"]) for r in rows]}
for k in keys:
out[k] = [r[k] for r in rows]
return {"n": len(rows), "bucket_s": bucket,
"start": start, "end": end, "series": out}
def daily_summary(self, start: float, end: float) -> List[Dict[str, Any]]:
"""Per-calendar-day extremes and means, in the station's local time.
Local time, not UTC: a `daily minimum` that straddles midnight in
the wrong timezone is the kind of quiet wrongness nobody notices
until they compare against the Met Office and lose an afternoon.
"""
sql = """
SELECT date(ts, 'unixepoch', 'localtime') AS day,
MIN(ts) AS first_ts, MAX(ts) AS last_ts, COUNT(*) AS n,
MIN(temp_smooth) AS temp_min, MAX(temp_smooth) AS temp_max,
AVG(temp_smooth) AS temp_mean,
MIN(hum_smooth) AS hum_min, MAX(hum_smooth) AS hum_max,
AVG(hum_smooth) AS hum_mean,
MIN(press_slp) AS press_min, MAX(press_slp) AS press_max,
AVG(press_slp) AS press_mean,
AVG(dew_c) AS dew_mean, MAX(lux) AS lux_max
FROM telemetry
WHERE ts >= ? AND ts <= ?
GROUP BY day ORDER BY day DESC
"""
with self._conn() as conn:
return [dict(r) for r in conn.execute(sql, (float(start), float(end))).fetchall()]
def extremes(self) -> Dict[str, Any]:
"""All-time records held by the station, each with when it happened."""
pairs = [
("temp_max", "temp_smooth", "DESC"), ("temp_min", "temp_smooth", "ASC"),
("hum_max", "hum_smooth", "DESC"), ("hum_min", "hum_smooth", "ASC"),
("press_max", "press_slp", "DESC"), ("press_min", "press_slp", "ASC"),
("dew_max", "dew_c", "DESC"), ("dew_min", "dew_c", "ASC"),
("rate_rise", "press_rate", "DESC"), ("rate_fall", "press_rate", "ASC"),
]
# Physical sanity bounds. A Kalman filter's rate estimate is garbage
# for the first few samples after it initialises, which happens on
# every restart, and an unfiltered MAX() will faithfully enshrine that
# transient as an all-time record of -37 hPa/h forever. The most
# extreme real sea-level pressure changes on Earth are around
# 10 hPa/h in an explosively deepening cyclone.
bounds = {"press_rate": 10.0, "temp_rate": 25.0}
out: Dict[str, Any] = {}
with self._conn() as conn:
for name, col, order in pairs:
guard = ""
if col in bounds:
guard = f" AND ABS({col}) <= {bounds[col]}"
row = conn.execute(
f"SELECT ts, {col} AS v FROM telemetry "
f"WHERE {col} IS NOT NULL{guard} ORDER BY {col} {order} LIMIT 1"
).fetchone()
out[name] = {"ts": row["ts"], "value": row["v"]} if row else None
span = conn.execute("SELECT MIN(ts) AS a, MAX(ts) AS b, COUNT(*) AS n "
"FROM telemetry").fetchone()
out["coverage"] = {"first_ts": span["a"], "last_ts": span["b"],
"rows": span["n"]}
return out
def iter_csv(self, start: float, end: float):
"""Yield CSV lines for export. Generator, so a year of history does
not have to exist in memory at once on a 512 MB board."""
cols = ["ts", "temp_smooth", "hum_smooth", "press_slp", "dew_c",
"temp_rate", "press_rate", "cpu_temp", "lux", "tier"]
yield "iso_time," + ",".join(cols) + "\n"
with self._conn() as conn:
cur = conn.execute(
f"SELECT {', '.join(cols)} FROM telemetry "
f"WHERE ts >= ? AND ts <= ? ORDER BY ts ASC",
(float(start), float(end)),
)
while True:
chunk = cur.fetchmany(500)
if not chunk:
break
for r in chunk:
iso = time.strftime("%Y-%m-%dT%H:%M:%S", time.localtime(r["ts"]))
vals = ["" if r[c] is None else
(f"{r[c]:.4f}" if isinstance(r[c], float) else str(r[c]))
for c in cols]
yield iso + "," + ",".join(vals) + "\n"
def storage_stats(self) -> Dict[str, Any]:
"""Rows per resolution tier, so the retention policy is visible."""
with self._conn() as conn:
rows = conn.execute(
"SELECT tier, COUNT(*) AS n, MIN(ts) AS a, MAX(ts) AS b "
"FROM telemetry GROUP BY tier ORDER BY tier"
).fetchall()
page = conn.execute("PRAGMA page_count").fetchone()[0]
size = conn.execute("PRAGMA page_size").fetchone()[0]
names = {TIER_RAW: "raw", TIER_5MIN: "5 minute", TIER_HOUR: "hourly"}
return {
"tiers": [{"tier": r["tier"], "label": names.get(r["tier"], "?"),
"rows": r["n"], "first_ts": r["a"], "last_ts": r["b"]}
for r in rows],
"bytes": int(page) * int(size),
}
def latest(self) -> Optional[Dict[str, Any]]:
with self._conn() as conn:
cur = conn.execute("SELECT * FROM telemetry ORDER BY ts DESC LIMIT 1")
row = cur.fetchone()
return dict(row) if row else None
def row_count(self) -> int:
with self._conn() as conn:
return int(conn.execute("SELECT COUNT(*) FROM telemetry").fetchone()[0])
def newest_ts(self) -> Optional[float]:
with self._conn() as conn:
row = conn.execute("SELECT MAX(ts) FROM telemetry").fetchone()
return float(row[0]) if row and row[0] is not None else None
def span_days(self) -> float:
with self._conn() as conn:
row = conn.execute("SELECT MIN(ts), MAX(ts) FROM telemetry").fetchone()
if not row or row[0] is None:
return 0.0
return (row[1] - row[0]) / 86400.0
def due_forecasts(self, now: Optional[float] = None) -> List[sqlite3.Row]:
"""Forecasts whose validity time has passed and can now be scored."""
now = now or time.time()
with self._conn() as conn:
return conn.execute(
"SELECT * FROM forecasts WHERE valid_ts <= ? AND valid_ts >= ? ORDER BY valid_ts",
(now, now - 7 * 86400),
).fetchall()
def scorecard(self) -> List[Dict[str, Any]]:
with self._conn() as conn:
rows = conn.execute(
"SELECT s.* FROM scores s JOIN ("
" SELECT target, horizon_s, MAX(ts) AS mts FROM scores GROUP BY target, horizon_s"
") m ON s.target = m.target AND s.horizon_s = m.horizon_s AND s.ts = m.mts "
"ORDER BY s.target, s.horizon_s"
).fetchall()
return [dict(r) for r in rows]
def recent_events(self, limit: int = 25) -> List[Dict[str, Any]]:
with self._conn() as conn:
rows = conn.execute(
"SELECT * FROM events ORDER BY ts DESC LIMIT ?", (limit,)
).fetchall()
return [dict(r) for r in rows]
def labels(self, kind: str, hours: float = 24 * 30) -> Dict[str, np.ndarray]:
since = time.time() - hours * 3600.0
with self._conn() as conn:
rows = conn.execute(
"SELECT ts, value FROM labels WHERE kind = ? AND ts >= ? ORDER BY ts",
(kind, since),
).fetchall()
if not rows:
return {"ts": np.empty(0), "value": np.empty(0)}
return {
"ts": np.array([r["ts"] for r in rows], dtype=float),
"value": np.array([r["value"] for r in rows], dtype=float),
}
# -------------------------------------------------------- compaction
def compact(self, raw_retention_days: float, five_min_retention_days: float) -> Dict[str, int]:
"""Fold old high-resolution rows into means. Returns rows removed per tier."""
now = time.time()
removed = {"raw": 0, "5min": 0}
removed["raw"] = self._fold(TIER_RAW, TIER_5MIN, 300,
now - raw_retention_days * 86400)
removed["5min"] = self._fold(TIER_5MIN, TIER_HOUR, 3600,
now - five_min_retention_days * 86400)
with self._conn() as conn:
conn.execute("PRAGMA incremental_vacuum")
return removed
def _fold(self, from_tier: int, to_tier: int, bucket_s: int, older_than: float) -> int:
agg_cols = [c for c in COLUMNS if c != "ts"]
select = ", ".join(f"AVG({c}) AS {c}" for c in agg_cols)
with self._conn() as conn:
rows = conn.execute(
f"SELECT CAST(ts / {bucket_s} AS INTEGER) * {bucket_s} AS bucket, {select} "
f"FROM telemetry WHERE tier = ? AND ts < ? GROUP BY bucket",
(from_tier, older_than),
).fetchall()
if not rows:
return 0
cur = conn.execute("SELECT COUNT(*) FROM telemetry WHERE tier = ? AND ts < ?",
(from_tier, older_than))
n_before = int(cur.fetchone()[0])
conn.execute("DELETE FROM telemetry WHERE tier = ? AND ts < ?",
(from_tier, older_than))
payload = [
tuple([float(r["bucket"])] + [r[c] for c in agg_cols] + [to_tier])
for r in rows
]
marks = ", ".join("?" for _ in range(len(agg_cols) + 2))
conn.executemany(
f"INSERT OR REPLACE INTO telemetry (ts, {', '.join(agg_cols)}, tier) "
f"VALUES ({marks})",
payload,
)
return n_before - len(rows)
def resample(ts: np.ndarray, values: Dict[str, np.ndarray], grid_s: int,
max_gap_grid: int = 3):
"""Bin irregular samples onto a regular grid, mean-aggregating each bin.
Returns (grid_ts, {name: array}) with NaN in bins that had no data and
linear interpolation across gaps no longer than `max_gap_grid` bins.
Anything longer stays NaN so the learner never trains on invention.
"""
if ts.size == 0:
return np.empty(0), {k: np.empty(0) for k in values}
start = np.floor(ts[0] / grid_s) * grid_s
stop = np.floor(ts[-1] / grid_s) * grid_s
grid = np.arange(start, stop + grid_s, grid_s, dtype=float)
if grid.size == 0:
return np.empty(0), {k: np.empty(0) for k in values}
idx = np.clip(((ts - start) / grid_s).astype(int), 0, grid.size - 1)
out = {}
counts = np.bincount(idx, minlength=grid.size).astype(float)
for name, arr in values.items():
clean = np.nan_to_num(arr, nan=0.0)
mask = (~np.isnan(arr)).astype(float)
total = np.bincount(idx, weights=clean, minlength=grid.size)
n = np.bincount(idx, weights=mask, minlength=grid.size)
with np.errstate(invalid="ignore", divide="ignore"):
binned = np.where(n > 0, total / np.maximum(n, 1e-9), np.nan)
out[name] = _interp_short_gaps(binned, max_gap_grid)
out["_count"] = counts
return grid, out
def _interp_short_gaps(arr: np.ndarray, max_gap: int) -> np.ndarray:
"""Linear fill for runs of NaN up to `max_gap` long; leave longer runs alone."""
a = arr.copy()
isnan = np.isnan(a)
if not isnan.any() or isnan.all():
return a
valid = np.flatnonzero(~isnan)
filled = np.interp(np.arange(a.size), valid, a[valid])
# find NaN runs and only accept the short ones
edges = np.flatnonzero(np.diff(np.concatenate(([0], isnan.view(np.int8), [0]))))
for start, stop in zip(edges[::2], edges[1::2]):
if (stop - start) <= max_gap and start > 0 and stop < a.size:
a[start:stop] = filled[start:stop]
return a
Reference in New Issue View Git Blame Copy Permalink