kemal
98f981b59e
scripts/pi_run.sh runs heavy jobs in their own process group and stops them when MemAvailable drops below a floor (Raspberry Pi OS boots with cgroup_disable=memory, so a systemd MemoryMax cap is not available); a 360-sample genotype run with cohort QC had exhausted memory and the OOM killer took down the desktop session.
31 lines
1.3 KiB
Bash
Executable File
31 lines
1.3 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Run a heavy command on the 8 GB Pi with a memory guard. Raspberry Pi OS boots with cgroup_disable=memory, so
|
|
# systemd MemoryMax cannot be used; instead the command runs in its own process group and is stopped (TERM, then
|
|
# KILL) as soon as the system's MemAvailable drops below MIN_FREE_MB -- before the kernel OOM killer would pick
|
|
# victims such as the desktop session. Prints the peak resident memory of the command's process tree.
|
|
# MIN_FREE_MB=1200 bash scripts/pi_run.sh <command...>
|
|
set -uo pipefail
|
|
MIN_FREE_MB=${MIN_FREE_MB:-1200}
|
|
setsid "$@" &
|
|
pid=$!
|
|
peak=0
|
|
tripped=0
|
|
tree_rss() { # kB, summed over the process group
|
|
ps -o rss= -g "$pid" 2>/dev/null | awk '{s+=$1} END {print s+0}'
|
|
}
|
|
while kill -0 "$pid" 2>/dev/null; do
|
|
rss=$(tree_rss); (( rss > peak )) && peak=$rss
|
|
avail=$(awk '/^MemAvailable:/ {print int($2/1024)}' /proc/meminfo)
|
|
if (( avail < MIN_FREE_MB )); then
|
|
echo "pi_run: MemAvailable ${avail} MiB < ${MIN_FREE_MB} MiB -- stopping the job (process group $pid)" >&2
|
|
kill -TERM -- "-$pid" 2>/dev/null; sleep 5; kill -KILL -- "-$pid" 2>/dev/null
|
|
tripped=1
|
|
break
|
|
fi
|
|
sleep 1
|
|
done
|
|
wait "$pid"; rc=$?
|
|
(( tripped )) && rc=137
|
|
echo "pi_run: exit $rc, peak RSS $((peak / 1024)) MiB, guard ${MIN_FREE_MB} MiB free" >&2
|
|
exit $rc
|