Add self-destructing AWS trial and large-scale P1 validation script
scripts/cloud/trial.sh launches one spot instance that validates the parsers on the 1000 Genomes Omni2.5 set (2141 samples, vendor GTCs) streamed from the public S3 bucket, uploads the summary and terminates itself. Cost guards: spot price cap, hard time limit (shutdown = terminate), EXIT-trap shutdown, CloudWatch idle alarm that terminates from outside, disk deleted with the instance, monthly budget alert, and an `audit` command listing anything still billable. scripts/validate_p1_large.py compares every GTC with BeadArrayFiles, checks byte-identical round-trips, IDAT pairs and BPM/EGT/CSV.
This commit is contained in:
1 parent
5a20e7131c
commit
3bfb7b5757
4 files changed
+412
No files matched your search
Executable
+62
@@ -0,0 +1,62 @@
|
||||
#!/bin/bash
|
||||
# EC2 user-data for a self-destructing SimurgArray validation run (Ubuntu 24.04, arm64).
|
||||
# Placeholders (__X__) are filled in by scripts/cloud/trial.sh.
|
||||
#
|
||||
# Safety layers inside the instance:
|
||||
# 1. `shutdown -h +MAX_MINUTES` is scheduled first -> hard time cap
|
||||
# 2. an EXIT trap uploads the log and shuts down -> ends as soon as the job ends, success or failure
|
||||
# Shutdown = TERMINATE (instance-initiated-shutdown-behavior) and the only disk has DeleteOnTermination=true,
|
||||
# so nothing keeps costing money afterwards. A CloudWatch alarm (layer 3) terminates an idle instance from
|
||||
# outside, in case the OS hangs.
|
||||
set -uo pipefail
|
||||
BUCKET="__BUCKET__"
|
||||
RUN="__RUN__"
|
||||
MAX_MINUTES="__MAX_MINUTES__"
|
||||
shutdown -h +"$MAX_MINUTES" "SimurgArray trial hard time limit"
|
||||
|
||||
LOG=/var/log/simurg-trial.log
|
||||
exec > >(tee -a "$LOG") 2>&1
|
||||
S3=""
|
||||
upload_log() { [[ -n "$S3" ]] && $S3 cp "$LOG" "s3://$BUCKET/$RUN/log.txt" --only-show-errors || true; }
|
||||
finish() {
|
||||
rc=$?
|
||||
echo "=== finished rc=$rc at $(date -u +%FT%TZ) ==="
|
||||
echo "$rc" > /tmp/rc && [[ -n "$S3" ]] && $S3 cp /tmp/rc "s3://$BUCKET/$RUN/exit_code.txt" --only-show-errors
|
||||
upload_log
|
||||
shutdown -h now
|
||||
}
|
||||
trap finish EXIT
|
||||
set -e
|
||||
echo "=== start $(date -u +%FT%TZ) on $(uname -m), $(nproc) cpus ==="
|
||||
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get update -q
|
||||
apt-get install -y -q pigz git python3-venv curl unzip
|
||||
snap install aws-cli --classic
|
||||
S3="/snap/bin/aws s3"
|
||||
( while sleep 300; do upload_log; done ) &
|
||||
|
||||
mkdir -p /work && cd /work
|
||||
$S3 cp "s3://$BUCKET/$RUN/repo.tar.gz" . --only-show-errors
|
||||
mkdir repo && tar -xzf repo.tar.gz -C repo && cd repo
|
||||
curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
export PATH="$HOME/.local/bin:$PATH"
|
||||
uv venv -q --python 3.12 .venv && uv pip install -q -e ".[dev]"
|
||||
mkdir -p refs && git clone -q --depth 1 https://github.com/Illumina/BeadArrayFiles.git refs/BeadArrayFiles
|
||||
|
||||
SRC=s3://1000genomes/release/20130502/supporting/hd_genotype_chip/broad_intensities
|
||||
D=/work/omni && mkdir -p $D/data
|
||||
for f in HumanOmni2.5-4v1-Multi_B.bpm HumanOmni2.5-4v1-Multi_B.egt HumanOmni2.5-4v1_B.csv; do
|
||||
$S3 cp --no-sign-request "$SRC/$f" $D/ --only-show-errors
|
||||
done
|
||||
echo "=== streaming 107 GB IDAT+GTC archive (no copy of the .tgz is kept) $(date -u +%T) ==="
|
||||
$S3 cp --no-sign-request "$SRC/Omni25_idats_gtcs_2141_samples.tgz" - | pigz -dc | tar -x -C $D/data
|
||||
echo "=== extracted $(du -sh $D/data | cut -f1) at $(date -u +%T) ==="
|
||||
|
||||
echo "=== validation $(date -u +%T) ==="
|
||||
.venv/bin/python -m pytest tests/unit -q -p no:cacheprovider
|
||||
.venv/bin/python scripts/validate_p1_large.py --root $D/data --bpm $D/HumanOmni2.5-4v1-Multi_B.bpm \
|
||||
--egt $D/HumanOmni2.5-4v1-Multi_B.egt --csv $D/HumanOmni2.5-4v1_B.csv --out /work/summary.json \
|
||||
--workers "$(nproc)" || echo "validation reported differences (see summary.json)"
|
||||
$S3 cp /work/summary.json "s3://$BUCKET/$RUN/summary.json" --only-show-errors
|
||||
echo "=== done $(date -u +%T) ==="
|
||||
Reference in new issue
Block a user