P9: scaling on a RAM disk (bench --work/--clean, cloud task p9b)

bench/scaling.py

@@ -15,6 +15,7 @@ import argparse
 import datetime as dt
 import json
 import os
+import shutil
 import subprocess
 import sys
 import time
@@ -61,6 +62,8 @@ def main() -> int:
     ap.add_argument("--samples", type=int, default=0, help="use the first N samples (0 = all)")
     ap.add_argument("--simurg", default=str(Path(sys.executable).with_name("simurg")))
     ap.add_argument("--pure-python", action="store_true", help="run with SIMURG_PURE_PYTHON=1")
+    ap.add_argument("--work", type=Path, help="folder for the GTC outputs (e.g. a tmpfs); default: --out")
+    ap.add_argument("--clean", action="store_true", help="delete each run's GTCs after timing it")
     a = ap.parse_args()
     a.out.mkdir(parents=True, exist_ok=True)
     idats = subset(a.idats, a.samples, a.out / "idats")
@@ -68,7 +71,7 @@ def main() -> int:
     env = {**os.environ, "SIMURG_PURE_PYTHON": "1" if a.pure_python else "0"}
     runs = []
     for t in (int(x) for x in a.threads.split(",")):
-        out = a.out / f"t{t}"
+        out = (a.work or a.out) / f"t{t}"
         t0 = time.time()
         cmd = [a.simurg, "genotype", "--bpm", a.bpm, "--egt", a.egt, "--idats", str(idats), "--out", str(out),
                "--threads", str(t), "--no-cohort-qc", "--autocall-date", "1/1/2026 12:00:00 AM",
@@ -76,6 +79,8 @@ def main() -> int:
         rc = subprocess.run(cmd, env=env).returncode
         wall = time.time() - t0
         setup, phase = phase_seconds(out / "logs" / "genotype.log")
+        if a.clean:
+            shutil.rmtree(out / "samples", ignore_errors=True)
         runs.append(
             {
                 "threads": t,

scripts/cloud/tasks/p9b.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+# Phase P9 follow-up: thread scaling with the GTC outputs on a RAM disk (/dev/shm), to separate compute scaling
+# from the gp3 disk's write throughput (125 MB/s baseline; one GSA GTC is ~13 MB). Run with OMNI=0.
+set -uo pipefail
+OUT=$2
+W=/work/p9; R=$OUT/p9b; mkdir -p $W $R
+N=$(nproc)
+DEBIAN_FRONTEND=noninteractive apt-get install -y -q gcc > /dev/null
+curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal > /dev/null 2>&1
+export PATH="$HOME/.cargo/bin:$PATH"
+VIRTUAL_ENV=$PWD/.venv uv pip install -q ./rust/simurg-core || { echo "!! simurg-core build failed"; exit 1; }
+.venv/bin/python -c "from simurg._core import backend; print('backend', backend())"
+GSA=https://webdata.illumina.com/downloads/productfiles/global-screening-array/v1-0
+mkdir -p $W/gsa && cd $W/gsa
+for z in infinium-global-screening-array-24-v1-0-c2-manifest-file-bpm-build38.zip \
+         infinium-global-screening-array-v1-0-c1-cluster-file.zip; do
+  curl -sSfL --retry 5 -O "$GSA/$z" && unzip -oq $z
+done
+cd - > /dev/null
+BPM=$W/gsa/GSA-24v1-0_C2.bpm; EGT=$W/gsa/GSA-24v1-0_C1_ClusterFile.egt
+.venv/bin/python scripts/p8/prepare.py --tsv scripts/p8/gse165845_samples.tsv --dl /dev/shm/geo --out /dev/shm/idats \
+  --workers 24 2>&1 | tail -2 || exit 1
+df -h /dev/shm | tail -1
+echo "=== scaling, outputs on /dev/shm $(date -u +%T) ==="
+.venv/bin/python bench/scaling.py --bpm $BPM --egt $EGT --idats /dev/shm/idats --out $W/shm --work /dev/shm/out \
+  --clean --threads 1,4,8,16,24,$N 2>&1 | grep -v WARNING
+cp $W/shm/scaling.json $R/scaling_rust_shm.json
+lscpu | grep -i "model name\|^CPU(s)" > $R/cpu.txt
+echo "=== task done $(date -u +%T) ==="