Tidy the repository: remove docs/research and bench, fix references
This commit is contained in:
kemal committed
1 parent
8ee0c46bba
commit
0562a31562
20 files changed
+7
-8032
No files matched your search
@@ -1,54 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Phase P9 performance gate on a 32-vCPU Graviton instance, GSE165845 GSA arrays (360 samples):
|
||||
# 1. build the compiled kernels (rust/simurg-core) into the venv;
|
||||
# 2. thread scaling 1 → 32 with the kernels (bench/scaling.py, per-sample phase from the run log);
|
||||
# 3. pure-Python reference at 1 and 32 threads on a subset, and byte-identity of all 360 GTCs (kernels vs pure).
|
||||
# Args: <unused data dir> <results dir>. Run with OMNI=0.
|
||||
set -uo pipefail
|
||||
OUT=$2
|
||||
W=/work/p9; R=$OUT/p9; mkdir -p $W $R
|
||||
N=$(nproc)
|
||||
|
||||
echo "=== build simurg-core $(date -u +%T) ==="
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y -q gcc > /dev/null # Rust links through the C toolchain
|
||||
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal > /dev/null 2>&1
|
||||
export PATH="$HOME/.cargo/bin:$PATH"
|
||||
rustc --version
|
||||
VIRTUAL_ENV=$PWD/.venv uv pip install -q ./rust/simurg-core || { echo "!! simurg-core build failed"; exit 1; }
|
||||
.venv/bin/python -c "from simurg._core import backend; print('backend', backend())"
|
||||
.venv/bin/python -m pytest tests/unit/test_core_kernels.py -q -p no:cacheprovider 2>&1 | tail -2
|
||||
|
||||
echo "=== data $(date -u +%T) ==="
|
||||
GSA=https://webdata.illumina.com/downloads/productfiles/global-screening-array/v1-0
|
||||
mkdir -p $W/gsa && cd $W/gsa
|
||||
for z in infinium-global-screening-array-24-v1-0-c2-manifest-file-bpm-build38.zip \
|
||||
infinium-global-screening-array-v1-0-c1-cluster-file.zip; do
|
||||
curl -sSfL --retry 5 -O "$GSA/$z" && unzip -oq $z
|
||||
done
|
||||
cd - > /dev/null
|
||||
BPM=$W/gsa/GSA-24v1-0_C2.bpm; EGT=$W/gsa/GSA-24v1-0_C1_ClusterFile.egt
|
||||
.venv/bin/python scripts/p8/prepare.py --tsv scripts/p8/gse165845_samples.tsv --dl $W/geo --out $W/idats --workers 24 \
|
||||
|| exit 1
|
||||
|
||||
echo "=== scaling with kernels $(date -u +%T) ==="
|
||||
.venv/bin/python bench/scaling.py --bpm $BPM --egt $EGT --idats $W/idats --out $W/rust \
|
||||
--threads 1,2,4,8,16,$N 2>&1 | grep -v WARNING
|
||||
cp $W/rust/scaling.json $R/scaling_rust.json
|
||||
|
||||
echo "=== pure Python reference $(date -u +%T) ==="
|
||||
.venv/bin/python bench/scaling.py --bpm $BPM --egt $EGT --idats $W/idats --out $W/python --threads 1 --samples 64 \
|
||||
--pure-python 2>&1 | grep -v WARNING
|
||||
cp $W/python/scaling.json $R/scaling_python_t1.json
|
||||
.venv/bin/python bench/scaling.py --bpm $BPM --egt $EGT --idats $W/idats --out $W/python_all --threads $N \
|
||||
--pure-python 2>&1 | grep -v WARNING
|
||||
cp $W/python_all/scaling.json $R/scaling_python_t$N.json
|
||||
|
||||
echo "=== identity: 360 GTCs, kernels vs pure Python $(date -u +%T) ==="
|
||||
n=0; d=0
|
||||
for f in $(cd $W/rust/t$N && find samples -name '*.gtc'); do
|
||||
n=$((n+1)); cmp -s $W/rust/t$N/$f $W/python_all/t$N/$f || { d=$((d+1)); echo "differs: $f"; }
|
||||
done
|
||||
s=$(cmp -s $W/rust/t$N/sample_summary.tsv $W/python_all/t$N/sample_summary.tsv && echo identical || echo DIFFERS)
|
||||
echo "{\"gtcs\": $n, \"differ\": $d, \"sample_summary\": \"$s\"}" | tee $R/identity.json
|
||||
lscpu | grep -i "model name\|^CPU(s)" > $R/cpu.txt; free -g >> $R/cpu.txt
|
||||
echo "=== task done $(date -u +%T) ==="
|
||||
@@ -1,29 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Phase P9 follow-up: thread scaling with the GTC outputs on a RAM disk (/dev/shm), to separate compute scaling
|
||||
# from the gp3 disk's write throughput (125 MB/s baseline; one GSA GTC is ~13 MB). Run with OMNI=0.
|
||||
set -uo pipefail
|
||||
OUT=$2
|
||||
W=/work/p9; R=$OUT/p9b; mkdir -p $W $R
|
||||
N=$(nproc)
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y -q gcc > /dev/null
|
||||
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal > /dev/null 2>&1
|
||||
export PATH="$HOME/.cargo/bin:$PATH"
|
||||
VIRTUAL_ENV=$PWD/.venv uv pip install -q ./rust/simurg-core || { echo "!! simurg-core build failed"; exit 1; }
|
||||
.venv/bin/python -c "from simurg._core import backend; print('backend', backend())"
|
||||
GSA=https://webdata.illumina.com/downloads/productfiles/global-screening-array/v1-0
|
||||
mkdir -p $W/gsa && cd $W/gsa
|
||||
for z in infinium-global-screening-array-24-v1-0-c2-manifest-file-bpm-build38.zip \
|
||||
infinium-global-screening-array-v1-0-c1-cluster-file.zip; do
|
||||
curl -sSfL --retry 5 -O "$GSA/$z" && unzip -oq $z
|
||||
done
|
||||
cd - > /dev/null
|
||||
BPM=$W/gsa/GSA-24v1-0_C2.bpm; EGT=$W/gsa/GSA-24v1-0_C1_ClusterFile.egt
|
||||
.venv/bin/python scripts/p8/prepare.py --tsv scripts/p8/gse165845_samples.tsv --dl /dev/shm/geo --out /dev/shm/idats \
|
||||
--workers 24 2>&1 | tail -2 || exit 1
|
||||
df -h /dev/shm | tail -1
|
||||
echo "=== scaling, outputs on /dev/shm $(date -u +%T) ==="
|
||||
.venv/bin/python bench/scaling.py --bpm $BPM --egt $EGT --idats /dev/shm/idats --out $W/shm --work /dev/shm/out \
|
||||
--clean --threads 1,4,8,16,24,$N 2>&1 | grep -v WARNING
|
||||
cp $W/shm/scaling.json $R/scaling_rust_shm.json
|
||||
lscpu | grep -i "model name\|^CPU(s)" > $R/cpu.txt
|
||||
echo "=== task done $(date -u +%T) ==="
|
||||
Reference in new issue
Block a user