diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml new file mode 100644 index 0000000..66fe850 --- /dev/null +++ b/.github/workflows/image.yml @@ -0,0 +1,110 @@ +name: Pi image + +# Builds a ready-to-flash Raspberry Pi OS Lite image with Ashvale Station +# preinstalled, and attaches it to a GitHub Release. +# +# Built in CI rather than on a laptop on purpose. The artifact is something +# other people flash onto their own hardware, so it should be reproducible from +# a public log by anyone who wants to check what went into it, rather than +# appearing from a machine only I can see. + +on: + workflow_dispatch: + inputs: + publish: + description: "Attach the image to a release" + type: boolean + default: false + push: + tags: + - "v*" + +permissions: + contents: write + +jobs: + build: + runs-on: ubuntu-latest + timeout-minutes: 180 + + steps: + - name: Check out Ashvale + uses: actions/checkout@v4 + with: + path: ashvale + + # pi-gen needs about 10 GB and a stock runner does not have it spare. + - name: Reclaim disk space + run: | + sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc \ + /usr/local/share/boost "$AGENT_TOOLSDIRECTORY" + df -h / | tail -1 + + - name: Install build dependencies + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + coreutils quilt parted qemu-user-static debootstrap zerofree zip \ + dosfstools libarchive-tools libcap2-bin grep rsync xz-utils file \ + git curl bc gpg pigz kpartx arch-test + + # Pinned to a commit, not a branch. An image other people flash should not + # change because an upstream branch moved between builds. + - name: Check out pi-gen + uses: actions/checkout@v4 + with: + repository: RPi-Distro/pi-gen + ref: ca8aeed0ae300c2a89f55ce9617d5f96a27e99e5 # arm64 branch, pinned + path: pi-gen + fetch-depth: 1 + + - name: Assemble the custom stage + run: | + cp ashvale/deploy/pi-image/config pi-gen/config + cp -r ashvale/deploy/pi-image/stage-ashvale pi-gen/stage-ashvale + # Lite only: everything from stage3 up is the desktop. + touch pi-gen/stage3/SKIP pi-gen/stage4/SKIP pi-gen/stage5/SKIP + touch pi-gen/stage4/SKIP_IMAGES pi-gen/stage5/SKIP_IMAGES + # stage2 stops exporting so ours is the only image produced. + rm -f pi-gen/stage2/EXPORT_IMAGE + echo "ASHVALE_SRC=${GITHUB_WORKSPACE}/ashvale" >> pi-gen/config + echo "--- config ---" && cat pi-gen/config + + - name: Build + working-directory: pi-gen + run: sudo -E ./build.sh + + - name: Collect artifact + id: artifact + run: | + IMG=$(find pi-gen/deploy -name '*.img.xz' | head -1) + test -n "$IMG" || { echo "no image produced"; ls -R pi-gen/deploy; exit 1; } + mkdir -p out && mv "$IMG" out/ + cd out + NAME=$(basename *.img.xz) + sha256sum "$NAME" > "$NAME.sha256" + echo "name=$NAME" >> "$GITHUB_OUTPUT" + ls -lh + # A release asset is capped at 2 GB; Lite compresses to well under that, + # but fail loudly here rather than at upload time. + SIZE=$(stat -c%s "$NAME") + echo "compressed size: $((SIZE/1024/1024)) MiB" + test "$SIZE" -lt 2000000000 || { echo "image exceeds the 2 GB release limit"; exit 1; } + + - name: Upload as a workflow artifact + uses: actions/upload-artifact@v4 + with: + name: ashvale-pi-image + path: out/* + retention-days: 14 + + - name: Attach to release + if: startsWith(github.ref, 'refs/tags/') || inputs.publish + env: + GH_TOKEN: ${{ github.token }} + run: | + TAG="${GITHUB_REF_NAME}" + gh release view "$TAG" --repo "$GITHUB_REPOSITORY" >/dev/null 2>&1 \ + || gh release create "$TAG" --repo "$GITHUB_REPOSITORY" \ + --title "$TAG" --notes "Ashvale Station image for Raspberry Pi." + gh release upload "$TAG" out/* --repo "$GITHUB_REPOSITORY" --clobber diff --git a/README.md b/README.md index aa2e342..3871e51 100644 --- a/README.md +++ b/README.md @@ -131,6 +131,22 @@ after a fortnight of real data and believe those instead. --- +## Getting it onto a Pi + +Already have a working Raspberry Pi: + +```bash +curl -fsSL https://raw.githubusercontent.com/lynchaos/ashvale-station/main/deploy/install.sh | sudo bash +``` + +Blank SD card: download the prebuilt Raspberry Pi OS Lite (Trixie, arm64) image +from [Releases](https://github.com/lynchaos/ashvale-station/releases), verify the +checksum, and flash it with Raspberry Pi Imager, setting your own username, +password and WiFi in the customisation dialog. The image carries no credentials +and no SSH host keys: those are generated on first boot. + +Either way, see [deploy/README.md](deploy/README.md). + ## Install on the Pi ```bash diff --git a/deploy/README.md b/deploy/README.md new file mode 100644 index 0000000..6f33d87 --- /dev/null +++ b/deploy/README.md @@ -0,0 +1,116 @@ +# Getting Ashvale Station onto a Pi + +Two routes. Pick the second one unless the card is empty. + +| | Use when | Cost | +|---|---|---| +| **Prebuilt image** | A blank SD card, or you want a known-good starting point | ~500 MB download, reflashes the card | +| **Install script** | You already have a working Pi OS | ~2 minutes, keeps everything else | + +--- + +## Install script (recommended) + +Works on any Raspberry Pi already running Raspberry Pi OS (Bookworm or Trixie). + +```bash +curl -fsSL https://raw.githubusercontent.com/lynchaos/ashvale-station/main/deploy/install.sh | sudo bash +``` + +Installs the apt dependencies, clones into `/opt/ashvale`, builds a venv with +`--system-site-packages`, enables I2C, and installs and starts a systemd unit. +Then it **polls the HTTP endpoint** rather than trusting systemd, because a unit +that is crash-looping on a port clash reports `active` for the instant between +exec and its first failed bind. + +Re-running it upgrades in place. It never touches `data/` or `config.yaml`, +because those are your history and your coordinates and neither is recoverable. + +Environment overrides, mostly useful for running a second instance beside a +live one: + +```bash +ASHVALE_DEST=/opt/ashvale-2 ASHVALE_SERVICE=ashvale-2 ASHVALE_PORT=8099 \ + sudo -E bash deploy/install.sh +``` + +Reboot afterwards if it told you it enabled I2C: the Sense HAT is not detected +until you do, and until then the station silently runs its simulator, which +looks like it is working and is measuring nothing. + +--- + +## Prebuilt image + +Download the `.img.xz` and its `.sha256` from +[Releases](https://github.com/lynchaos/ashvale-station/releases), verify, and +flash with Raspberry Pi Imager. + +```bash +sha256sum -c ashvale-station-*.img.xz.sha256 +``` + +**Set your username, password and WiFi in Imager's customisation dialog.** The +image deliberately contains none of them. It also contains no SSH host keys: +those are generated on first boot, because an image shipping real host keys +would give every person who flashed it the same identity and make them trivially +impersonable on their own network. + +Boot it, wait a minute or two for the first-boot expansion, then open +`http://:8000`. + +### What is in it + +Raspberry Pi OS Lite, Trixie, arm64, plus: + +- the application in `/opt/ashvale` with its venv already built +- `ashvale.service`, enabled +- I2C enabled in `config.txt`, which the Sense HAT needs +- a login banner with the address and the security caveat +- `/opt/ashvale/README.first-boot` + +Lite, not Desktop: the station is headless and a desktop would eat the 512 MB +budget the whole project is designed around. + +### What is deliberately not in it + +No password, no WiFi credentials, no SSH host keys, no database, no trained +model state, and coordinates set to Greenwich at 0 m. That last one is wrong for +everybody on purpose: altitude feeds the sea-level pressure reduction on every +stored row, and pressure tendency is what drives the precipitation forecast, so +a plausible-looking wrong altitude is worse than an obviously wrong one. + +The build copies the working tree through a `.gitignore` filter rather than a +hand-written exclude list. That is a security property: a hand-written list +missed `HANDOVER.md`, which is gitignored precisely because it holds LAN +addresses and SSH details, and a local build would have baked one person's +network into an image other people flash. + +--- + +## Building the image yourself + +CI does it on every tag via `.github/workflows/image.yml`, on a pinned pi-gen +commit so the output does not move when an upstream branch does. To build +locally you need a Linux host (or Docker) with `qemu-user-static`: + +```bash +git clone --branch arm64 https://github.com/RPi-Distro/pi-gen +cp deploy/pi-image/config pi-gen/config +cp -r deploy/pi-image/stage-ashvale pi-gen/ +touch pi-gen/stage3/SKIP pi-gen/stage4/SKIP pi-gen/stage5/SKIP +rm -f pi-gen/stage2/EXPORT_IMAGE +echo "ASHVALE_SRC=$PWD" >> pi-gen/config +cd pi-gen && sudo -E ./build.sh +``` + +Expect roughly an hour and about 10 GB of scratch space. + +--- + +## Licensing + +Ashvale Station is Apache 2.0. The image also contains Raspberry Pi OS and +Debian, which carry their own licences including some non-free firmware. It is +an unofficial image and is not endorsed by or affiliated with Raspberry Pi Ltd +or the Debian project. diff --git a/deploy/install.sh b/deploy/install.sh new file mode 100755 index 0000000..5a6fea4 --- /dev/null +++ b/deploy/install.sh @@ -0,0 +1,147 @@ +#!/usr/bin/env bash +# Install Ashvale Station onto a Raspberry Pi that is already running. +# +# Most people already have a working Pi and should not have to reflash a card to +# try this. The prebuilt image exists for a fresh board; this exists for +# everything else, and it is the same install the image performs. +# +# curl -fsSL https://raw.githubusercontent.com/lynchaos/ashvale-station/main/deploy/install.sh | bash +# +# Idempotent: safe to re-run to upgrade. It never touches data/ or config.yaml +# on a machine that already has them, because those are your history and your +# coordinates and neither can be regenerated. +set -euo pipefail + +REPO="${ASHVALE_REPO:-https://github.com/lynchaos/ashvale-station.git}" +DEST="${ASHVALE_DEST:-/opt/ashvale}" +BRANCH="${ASHVALE_BRANCH:-main}" +SERVICE="${ASHVALE_SERVICE:-ashvale}" +# Overridable so a second instance can be installed alongside a live one, +# which is also the only way to test this script without stopping the real +# station. Empty means "whatever config.yaml says". +PORT_OVERRIDE="${ASHVALE_PORT:-}" # overridable so the installer can be tested without clobbering a live unit + +say() { printf '\n\033[1;36m==>\033[0m %s\n' "$*"; } +warn() { printf '\033[1;33m !\033[0m %s\n' "$*"; } +die() { printf '\033[1;31m x\033[0m %s\n' "$*" >&2; exit 1; } + +[ "$(id -u)" -eq 0 ] || die "run with sudo: curl ... | sudo bash" + +if ! grep -qi raspberry /proc/device-tree/model 2>/dev/null; then + warn "this does not look like a Raspberry Pi; continuing anyway" +fi + +RUN_USER="${SUDO_USER:-$(getent passwd 1000 | cut -d: -f1)}" +[ -n "$RUN_USER" ] || die "could not determine a non-root user to run the service as" + +say "Installing system packages" +# Hardware libraries come from apt, never pip. sense-hat pulls in RTIMULib, and +# building that inside a clean venv on ARM is a genuine ordeal. +apt-get update -qq +apt-get install -y --no-install-recommends \ + git python3-venv python3-numpy python3-smbus2 sense-hat sqlite3 + +say "Fetching the application into $DEST" +if [ -d "$DEST/.git" ]; then + git -C "$DEST" fetch --depth 1 origin "$BRANCH" + git -C "$DEST" reset --hard "origin/$BRANCH" +else + install -d "$DEST" + git clone --depth 1 --branch "$BRANCH" "$REPO" "$DEST" +fi + +say "Creating the virtual environment" +# --system-site-packages so numpy and the Sense HAT stack come from apt rather +# than being compiled here. Plain uvicorn, never uvicorn[standard]: that extra +# drags in watchfiles and uvloop, which compile Rust and C from source on ARM +# for features this does not use. +[ -d "$DEST/.venv" ] || python3 -m venv --system-site-packages "$DEST/.venv" +"$DEST/.venv/bin/pip" install -q --upgrade pip +"$DEST/.venv/bin/pip" install -q --no-cache-dir -r "$DEST/requirements.txt" + +install -d -o "$RUN_USER" -g "$RUN_USER" "$DEST/data" "$DEST/data/state" +chown -R "$RUN_USER":"$RUN_USER" "$DEST" + +say "Enabling I2C for the Sense HAT" +CFG=/boot/firmware/config.txt +[ -f "$CFG" ] || CFG=/boot/config.txt +if [ -f "$CFG" ] && ! grep -q '^dtparam=i2c_arm=on' "$CFG"; then + printf '\n# --- Ashvale Station ---\ndtparam=i2c_arm=on\n' >> "$CFG" + warn "I2C enabled: reboot before the Sense HAT is detected" +fi + +say "Installing the service" +cat > "/etc/systemd/system/$SERVICE.service" </dev/null +systemctl restart "$SERVICE" + +# Poll the endpoint, not systemd. "active" is true for the instant between +# exec and the first failed bind, so a unit that is crash-looping on a port +# clash reports healthy and the installer congratulates you on a broken install. +# Asking the thing whether it answers is the only check that means anything. +PORT="$PORT_OVERRIDE" +[ -n "$PORT" ] || PORT="$(sed -n 's/^ *port: *\([0-9]\+\).*/\1/p' "$DEST/config.yaml" | head -1)" +PORT="${PORT:-8000}" +IP=$(hostname -I 2>/dev/null | awk '{print $1}') +for _ in $(seq 1 30); do + if curl -fsS -o /dev/null --max-time 2 "http://127.0.0.1:${PORT}/api/status"; then + say "Running: http://${IP:-}:${PORT}" + HEALTHY=1; break + fi + sleep 2 +done +if [ "${HEALTHY:-0}" != 1 ]; then + systemctl is-active --quiet "$SERVICE" \ + && warn "unit is up but nothing is answering on port ${PORT}; is it already in use?" \ + || warn "unit is not running" + die "install finished but the station is not serving: journalctl -u $SERVICE -n 40" +fi + +cat <> "$CONFIG_TXT" <<'CFG' + +# --- Ashvale Station --- +# Sense HAT sits on I2C. Without this the board is not detected at all. +dtparam=i2c_arm=on +# Uncomment for a DS18B20 outdoor probe on GPIO4. Left off by default because +# it claims that pin whether or not a sensor is attached. +#dtoverlay=w1-gpio +CFG +fi diff --git a/deploy/pi-image/stage-ashvale/00-install/files/README.first-boot b/deploy/pi-image/stage-ashvale/00-install/files/README.first-boot new file mode 100644 index 0000000..6462dce --- /dev/null +++ b/deploy/pi-image/stage-ashvale/00-install/files/README.first-boot @@ -0,0 +1,41 @@ +Ashvale Station +=============== + +The service is already running. Open http://:8000 from the same network. + +Three things to do, in order of how much they matter. + +1. Set your location. Settings tab -> Site and model. Altitude feeds the + sea-level pressure reduction on every stored row, and pressure tendency is + what drives the precipitation forecast, so a wrong altitude quietly biases + the thing this station is best at. The image ships Greenwich at 0 m, which + is wrong for you on purpose. + +2. Calibrate the temperature. Models and Calibration tab. Put any thermometer + next to the board, wait ten minutes, type the number in. The Sense HAT sits + millimetres above a SoC running 20 C hotter than the room, and one reading + fixes the bias on every forecast that follows. This is the highest value per + minute of anything you can do. + +3. Tell it where it lives. Settings tab -> Surroundings. Indoors the building + governs temperature and humidity rather than the sky. Say so, and say when + you open a window or turn the heating on, because those are changes of + regime and the models carry about 55 hours of memory. + +Forecasts appear after about 10 hours of history and the scorecard fills in over +the following day as each horizon matures. Until a head has been scored, its +forecast is an opinion. + +Security +-------- +No authentication, no TLS, binds 0.0.0.0. It is built for a trusted home +network. Do not port-forward it: /api/train, /api/calibrate, /api/label and +/api/settings all mutate model state. + +Where things are +---------------- + /opt/ashvale application + /opt/ashvale/config.yaml site configuration + /opt/ashvale/data database and trained state (never in the image) + systemctl status ashvale service + journalctl -u ashvale -f logs diff --git a/deploy/pi-image/stage-ashvale/00-install/files/ashvale-firstboot b/deploy/pi-image/stage-ashvale/00-install/files/ashvale-firstboot new file mode 100755 index 0000000..6232327 --- /dev/null +++ b/deploy/pi-image/stage-ashvale/00-install/files/ashvale-firstboot @@ -0,0 +1,24 @@ +#!/bin/bash +# Runs once, before the station starts. Everything here is deliberately absent +# from the image itself, because baking it in would mean every person who +# flashed this card shared the same secrets or the same location. +set -e +STATE=/opt/ashvale/data +CONF=/opt/ashvale/config.yaml +UID_MAIN=$(getent passwd 1000 | cut -d: -f1) + +install -d -o 1000 -g 1000 "$STATE" "$STATE/state" + +# Site coordinates default to Greenwich, not to the author's house. They are +# wrong for everyone, which is the point: the Methods tab and the sea-level +# reduction both depend on them, so they should be conspicuously wrong until set. +if [ -f "$CONF" ] && ! grep -q 'ASHVALE_FIRSTBOOT_DONE' "$CONF"; then + sed -i 's/^\( *latitude:\).*/\1 51.4779 # CHANGE ME: Settings tab or this file/' "$CONF" + sed -i 's/^\( *longitude:\).*/\1 0.0015 # CHANGE ME/' "$CONF" + sed -i 's/^\( *altitude_m:\).*/\1 0.0 # CHANGE ME: wrong altitude skews sea-level pressure/' "$CONF" + echo "# ASHVALE_FIRSTBOOT_DONE" >> "$CONF" + chown 1000:1000 "$CONF" +fi + +logger -t ashvale-firstboot "prepared state for user ${UID_MAIN:-uid1000}" +systemctl disable ashvale-firstboot.service || true diff --git a/deploy/pi-image/stage-ashvale/00-install/files/ashvale-firstboot.service b/deploy/pi-image/stage-ashvale/00-install/files/ashvale-firstboot.service new file mode 100644 index 0000000..78d6aad --- /dev/null +++ b/deploy/pi-image/stage-ashvale/00-install/files/ashvale-firstboot.service @@ -0,0 +1,13 @@ +[Unit] +Description=Ashvale Station first-boot preparation +After=local-fs.target +Before=ashvale.service +ConditionPathExists=!/opt/ashvale/data/state + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/ashvale-firstboot +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target diff --git a/deploy/pi-image/stage-ashvale/00-install/files/ashvale.service b/deploy/pi-image/stage-ashvale/00-install/files/ashvale.service new file mode 100644 index 0000000..dcf51e0 --- /dev/null +++ b/deploy/pi-image/stage-ashvale/00-install/files/ashvale.service @@ -0,0 +1,27 @@ +[Unit] +Description=Ashvale Station forecast service +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=1000 +Group=1000 +WorkingDirectory=/opt/ashvale +ExecStart=/opt/ashvale/.venv/bin/python run.py +Restart=always +RestartSec=10 + +# A Zero 2 W has 512 MB. Cap the service so a runaway allocation takes the +# service down instead of the whole board. +MemoryMax=280M +CPUWeight=70 +Nice=5 + +# The SD card is a consumable: keep journald from writing every heartbeat. +StandardOutput=journal +StandardError=journal +SyslogIdentifier=ashvale + +[Install] +WantedBy=multi-user.target diff --git a/deploy/pi-image/stage-ashvale/00-install/files/motd.sh b/deploy/pi-image/stage-ashvale/00-install/files/motd.sh new file mode 100755 index 0000000..635387c --- /dev/null +++ b/deploy/pi-image/stage-ashvale/00-install/files/motd.sh @@ -0,0 +1,7 @@ +#!/bin/sh +IP=$(hostname -I 2>/dev/null | awk '{print $1}') +printf '\n Ashvale Station -> http://%s:8000\n' "${IP:-}" +printf ' status: %s\n' "$(systemctl is-active ashvale 2>/dev/null || echo unknown)" +printf '\n No authentication and no TLS. Trusted LAN only: do not port-forward it.\n' +printf ' Set your coordinates and altitude on the Settings tab before trusting\n' +printf ' the pressure readings. See /opt/ashvale/README.first-boot\n\n' diff --git a/deploy/pi-image/stage-ashvale/00-packages b/deploy/pi-image/stage-ashvale/00-packages new file mode 100644 index 0000000..0e359bf --- /dev/null +++ b/deploy/pi-image/stage-ashvale/00-packages @@ -0,0 +1,5 @@ +python3-venv +python3-numpy +python3-smbus2 +sense-hat +sqlite3 diff --git a/deploy/pi-image/stage-ashvale/EXPORT_IMAGE b/deploy/pi-image/stage-ashvale/EXPORT_IMAGE new file mode 100644 index 0000000..2b0f09d --- /dev/null +++ b/deploy/pi-image/stage-ashvale/EXPORT_IMAGE @@ -0,0 +1 @@ +IMG_SUFFIX="" diff --git a/deploy/pi-image/stage-ashvale/prerun.sh b/deploy/pi-image/stage-ashvale/prerun.sh new file mode 100755 index 0000000..2bd18db --- /dev/null +++ b/deploy/pi-image/stage-ashvale/prerun.sh @@ -0,0 +1,4 @@ +#!/bin/bash -e +if [ ! -d "${ROOTFS_DIR}" ]; then + copy_previous +fi