kemalandClaude Opus 5.5
a9aa6dc080
scripts/cloud/trial.sh launches one spot instance that validates the parsers on the 1000 Genomes Omni2.5 set (2141 samples, vendor GTCs) streamed from the public S3 bucket, uploads the summary and terminates itself. Cost guards: spot price cap, hard time limit (shutdown = terminate), EXIT-trap shutdown, CloudWatch idle alarm that terminates from outside, disk deleted with the instance, monthly budget alert, and an `audit` command listing anything still billable. scripts/validate_p1_large.py compares every GTC with BeadArrayFiles, checks byte-identical round-trips, IDAT pairs and BPM/EGT/CSV. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01814GKVzZVL3FuF3DeiZXhv
63 lines
2.8 KiB
Bash
Executable File
63 lines
2.8 KiB
Bash
Executable File
#!/bin/bash
|
|
# EC2 user-data for a self-destructing SimurgArray validation run (Ubuntu 24.04, arm64).
|
|
# Placeholders (__X__) are filled in by scripts/cloud/trial.sh.
|
|
#
|
|
# Safety layers inside the instance:
|
|
# 1. `shutdown -h +MAX_MINUTES` is scheduled first -> hard time cap
|
|
# 2. an EXIT trap uploads the log and shuts down -> ends as soon as the job ends, success or failure
|
|
# Shutdown = TERMINATE (instance-initiated-shutdown-behavior) and the only disk has DeleteOnTermination=true,
|
|
# so nothing keeps costing money afterwards. A CloudWatch alarm (layer 3) terminates an idle instance from
|
|
# outside, in case the OS hangs.
|
|
set -uo pipefail
|
|
BUCKET="__BUCKET__"
|
|
RUN="__RUN__"
|
|
MAX_MINUTES="__MAX_MINUTES__"
|
|
shutdown -h +"$MAX_MINUTES" "SimurgArray trial hard time limit"
|
|
|
|
LOG=/var/log/simurg-trial.log
|
|
exec > >(tee -a "$LOG") 2>&1
|
|
S3=""
|
|
upload_log() { [[ -n "$S3" ]] && $S3 cp "$LOG" "s3://$BUCKET/$RUN/log.txt" --only-show-errors || true; }
|
|
finish() {
|
|
rc=$?
|
|
echo "=== finished rc=$rc at $(date -u +%FT%TZ) ==="
|
|
echo "$rc" > /tmp/rc && [[ -n "$S3" ]] && $S3 cp /tmp/rc "s3://$BUCKET/$RUN/exit_code.txt" --only-show-errors
|
|
upload_log
|
|
shutdown -h now
|
|
}
|
|
trap finish EXIT
|
|
set -e
|
|
echo "=== start $(date -u +%FT%TZ) on $(uname -m), $(nproc) cpus ==="
|
|
|
|
export DEBIAN_FRONTEND=noninteractive
|
|
apt-get update -q
|
|
apt-get install -y -q pigz git python3-venv curl unzip
|
|
snap install aws-cli --classic
|
|
S3="/snap/bin/aws s3"
|
|
( while sleep 300; do upload_log; done ) &
|
|
|
|
mkdir -p /work && cd /work
|
|
$S3 cp "s3://$BUCKET/$RUN/repo.tar.gz" . --only-show-errors
|
|
mkdir repo && tar -xzf repo.tar.gz -C repo && cd repo
|
|
curl -LsSf https://astral.sh/uv/install.sh | sh
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
uv venv -q --python 3.12 .venv && uv pip install -q -e ".[dev]"
|
|
mkdir -p refs && git clone -q --depth 1 https://github.com/Illumina/BeadArrayFiles.git refs/BeadArrayFiles
|
|
|
|
SRC=s3://1000genomes/release/20130502/supporting/hd_genotype_chip/broad_intensities
|
|
D=/work/omni && mkdir -p $D/data
|
|
for f in HumanOmni2.5-4v1-Multi_B.bpm HumanOmni2.5-4v1-Multi_B.egt HumanOmni2.5-4v1_B.csv; do
|
|
$S3 cp --no-sign-request "$SRC/$f" $D/ --only-show-errors
|
|
done
|
|
echo "=== streaming 107 GB IDAT+GTC archive (no copy of the .tgz is kept) $(date -u +%T) ==="
|
|
$S3 cp --no-sign-request "$SRC/Omni25_idats_gtcs_2141_samples.tgz" - | pigz -dc | tar -x -C $D/data
|
|
echo "=== extracted $(du -sh $D/data | cut -f1) at $(date -u +%T) ==="
|
|
|
|
echo "=== validation $(date -u +%T) ==="
|
|
.venv/bin/python -m pytest tests/unit -q -p no:cacheprovider
|
|
.venv/bin/python scripts/validate_p1_large.py --root $D/data --bpm $D/HumanOmni2.5-4v1-Multi_B.bpm \
|
|
--egt $D/HumanOmni2.5-4v1-Multi_B.egt --csv $D/HumanOmni2.5-4v1_B.csv --out /work/summary.json \
|
|
--workers "$(nproc)" || echo "validation reported differences (see summary.json)"
|
|
$S3 cp /work/summary.json "s3://$BUCKET/$RUN/summary.json" --only-show-errors
|
|
echo "=== done $(date -u +%T) ==="
|