kemal
2622dc5b88
- tests/oracle/test_p2_raw_xy.py: IDAT-assembled raw X/Y and controls equal the arrays in oracle GTCs - validate_p1_large.py: phase-P2 comparison against vendor GTCs with diagnostics (channel swap, bead-count disagreements) - cloud trial: also runs `simurg ingest` on all samples (time/memory), disk 250 GB
74 lines
3.5 KiB
Bash
Executable File
74 lines
3.5 KiB
Bash
Executable File
#!/bin/bash
|
|
# EC2 user-data for a self-destructing SimurgArray validation run (Ubuntu 24.04, arm64).
|
|
# Placeholders (__X__) are filled in by scripts/cloud/trial.sh.
|
|
#
|
|
# Safety layers inside the instance:
|
|
# 1. `shutdown -h +MAX_MINUTES` is scheduled first -> hard time cap
|
|
# 2. an EXIT trap uploads the log and shuts down -> ends as soon as the job ends, success or failure
|
|
# Shutdown = TERMINATE (instance-initiated-shutdown-behavior) and the only disk has DeleteOnTermination=true,
|
|
# so nothing keeps costing money afterwards. A CloudWatch alarm (layer 3) terminates an idle instance from
|
|
# outside, in case the OS hangs.
|
|
set -uo pipefail
|
|
export HOME=/root # cloud-init runs user-data without HOME
|
|
export PATH="/root/.local/bin:/snap/bin:$PATH"
|
|
BUCKET="__BUCKET__"
|
|
RUN="__RUN__"
|
|
MAX_MINUTES="__MAX_MINUTES__"
|
|
shutdown -h +"$MAX_MINUTES" "SimurgArray trial hard time limit"
|
|
|
|
LOG=/var/log/simurg-trial.log
|
|
exec > >(tee -a "$LOG") 2>&1
|
|
S3=""
|
|
upload_log() { [[ -n "$S3" ]] && $S3 cp "$LOG" "s3://$BUCKET/$RUN/log.txt" --only-show-errors || true; }
|
|
finish() {
|
|
rc=$?
|
|
echo "=== finished rc=$rc at $(date -u +%FT%TZ) ==="
|
|
echo "$rc" > /tmp/rc && [[ -n "$S3" ]] && $S3 cp /tmp/rc "s3://$BUCKET/$RUN/exit_code.txt" --only-show-errors
|
|
upload_log
|
|
shutdown -h now
|
|
}
|
|
trap finish EXIT
|
|
set -e
|
|
echo "=== start $(date -u +%FT%TZ) on $(uname -m), $(nproc) cpus ==="
|
|
|
|
export DEBIAN_FRONTEND=noninteractive
|
|
apt-get update -q
|
|
apt-get install -y -q pigz git python3-venv curl unzip time
|
|
snap install aws-cli --classic
|
|
S3="/snap/bin/aws s3"
|
|
( while sleep 300; do upload_log; done ) &
|
|
|
|
mkdir -p /work && cd /work
|
|
$S3 cp "s3://$BUCKET/$RUN/repo.tar.gz" . --only-show-errors
|
|
mkdir repo && tar -xzf repo.tar.gz -C repo && cd repo
|
|
curl -LsSf https://astral.sh/uv/install.sh | sh
|
|
uv venv -q --python 3.12 .venv && uv pip install -q -e ".[dev]"
|
|
mkdir -p refs && git clone -q --depth 1 https://github.com/Illumina/BeadArrayFiles.git refs/BeadArrayFiles
|
|
|
|
SRC=s3://1000genomes/release/20130502/supporting/hd_genotype_chip/broad_intensities
|
|
D=/work/omni && mkdir -p $D/data
|
|
for f in HumanOmni2.5-4v1-Multi_B.bpm HumanOmni2.5-4v1-Multi_B.egt HumanOmni2.5-4v1_B.csv; do
|
|
$S3 cp --no-sign-request "$SRC/$f" $D/ --only-show-errors
|
|
done
|
|
echo "=== streaming 107 GB IDAT+GTC archive (no copy of the .tgz is kept) $(date -u +%T) ==="
|
|
$S3 cp --no-sign-request "$SRC/Omni25_idats_gtcs_2141_samples.tgz" - | pigz -dc | tar -x -C $D/data
|
|
echo "=== extracted $(du -sh $D/data | cut -f1) at $(date -u +%T) ==="
|
|
|
|
echo "=== validation $(date -u +%T) ==="
|
|
.venv/bin/python -m pytest tests/unit -q -p no:cacheprovider || echo "unit tests reported failures (continuing)"
|
|
.venv/bin/python scripts/validate_p1_large.py --root $D/data --bpm $D/HumanOmni2.5-4v1-Multi_B.bpm \
|
|
--egt $D/HumanOmni2.5-4v1-Multi_B.egt --csv $D/HumanOmni2.5-4v1_B.csv --out /work/summary.json \
|
|
--workers "$(nproc)" || echo "validation reported differences (see summary.json)"
|
|
$S3 cp /work/summary.json "s3://$BUCKET/$RUN/summary.json" --only-show-errors
|
|
|
|
echo "=== simurg ingest at scale $(date -u +%T) ==="
|
|
/usr/bin/time -v .venv/bin/simurg ingest --bpm $D/HumanOmni2.5-4v1-Multi_B.bpm --idats $D/data \
|
|
--out /work/ingest --threads "$(nproc)" --quiet 2> /work/ingest_time.txt || echo "ingest exit code $?"
|
|
tail -25 /work/ingest_time.txt
|
|
for f in ingest_summary.tsv run_manifest.json; do
|
|
$S3 cp "/work/ingest/$f" "s3://$BUCKET/$RUN/ingest_$f" --only-show-errors || true
|
|
done
|
|
$S3 cp /work/ingest_time.txt "s3://$BUCKET/$RUN/ingest_time.txt" --only-show-errors || true
|
|
echo "ingest output size: $(du -sh /work/ingest | cut -f1)"
|
|
echo "=== done $(date -u +%T) ==="
|