#!/bin/bash
# EC2 user-data for a self-destructing SimurgArray validation run (Ubuntu 24.04, arm64).
# Placeholders (__X__) are filled in by scripts/cloud/trial.sh.
#
# Safety layers inside the instance:
#   1. `shutdown -h +MAX_MINUTES` is scheduled first  -> hard time cap
#   2. an EXIT trap uploads the log and shuts down      -> ends as soon as the job ends, success or failure
#   Shutdown = TERMINATE (instance-initiated-shutdown-behavior) and the only disk has DeleteOnTermination=true,
#   so nothing keeps costing money afterwards. A CloudWatch alarm (layer 3) terminates an idle instance from
#   outside, in case the OS hangs.
set -uo pipefail
export HOME=/root  # cloud-init runs user-data without HOME
export PATH="/root/.local/bin:/snap/bin:$PATH"
BUCKET="__BUCKET__"
RUN="__RUN__"
MAX_MINUTES="__MAX_MINUTES__"
shutdown -h +"$MAX_MINUTES" "SimurgArray trial hard time limit"

LOG=/var/log/simurg-trial.log
exec > >(tee -a "$LOG") 2>&1
S3=""
upload_log() { [[ -n "$S3" ]] && $S3 cp "$LOG" "s3://$BUCKET/$RUN/log.txt" --only-show-errors || true; }
finish() {
  rc=$?
  echo "=== finished rc=$rc at $(date -u +%FT%TZ) ==="
  echo "$rc" > /tmp/rc && [[ -n "$S3" ]] && $S3 cp /tmp/rc "s3://$BUCKET/$RUN/exit_code.txt" --only-show-errors
  upload_log
  shutdown -h now
}
trap finish EXIT
set -e
echo "=== start $(date -u +%FT%TZ) on $(uname -m), $(nproc) cpus ==="

export DEBIAN_FRONTEND=noninteractive
apt-get update -q
apt-get install -y -q pigz git python3-venv curl unzip time
snap install aws-cli --classic
S3="/snap/bin/aws s3"
( while sleep 300; do upload_log; done ) &

mkdir -p /work && cd /work
$S3 cp "s3://$BUCKET/$RUN/repo.tar.gz" . --only-show-errors
mkdir repo && tar -xzf repo.tar.gz -C repo && cd repo
curl -LsSf https://astral.sh/uv/install.sh | sh
uv venv -q --python 3.12 .venv && uv pip install -q -e ".[dev]"
mkdir -p refs && git clone -q --depth 1 https://github.com/Illumina/BeadArrayFiles.git refs/BeadArrayFiles

SRC=s3://1000genomes/release/20130502/supporting/hd_genotype_chip/broad_intensities
D=/work/omni && mkdir -p $D/data
for f in HumanOmni2.5-4v1-Multi_B.bpm HumanOmni2.5-4v1-Multi_B.egt HumanOmni2.5-4v1_B.csv; do
  $S3 cp --no-sign-request "$SRC/$f" $D/ --only-show-errors
done
echo "=== streaming 107 GB IDAT+GTC archive (no copy of the .tgz is kept) $(date -u +%T) ==="
$S3 cp --no-sign-request "$SRC/Omni25_idats_gtcs_2141_samples.tgz" - | pigz -dc | tar -x -C $D/data
echo "=== extracted $(du -sh $D/data | cut -f1) at $(date -u +%T) ==="

echo "=== validation $(date -u +%T) ==="
.venv/bin/python -m pytest tests/unit -q -p no:cacheprovider || echo "unit tests reported failures (continuing)"
.venv/bin/python scripts/validate_p1_large.py --root $D/data --bpm $D/HumanOmni2.5-4v1-Multi_B.bpm \
  --egt $D/HumanOmni2.5-4v1-Multi_B.egt --csv $D/HumanOmni2.5-4v1_B.csv --out /work/summary.json \
  --workers "$(nproc)" || echo "validation reported differences (see summary.json)"
$S3 cp /work/summary.json "s3://$BUCKET/$RUN/summary.json" --only-show-errors

echo "=== simurg ingest at scale $(date -u +%T) ==="
/usr/bin/time -v .venv/bin/simurg ingest --bpm $D/HumanOmni2.5-4v1-Multi_B.bpm --idats $D/data \
  --out /work/ingest --threads "$(nproc)" --quiet 2> /work/ingest_time.txt || echo "ingest exit code $?"
tail -25 /work/ingest_time.txt
for f in ingest_summary.tsv run_manifest.json; do
  $S3 cp "/work/ingest/$f" "s3://$BUCKET/$RUN/ingest_$f" --only-show-errors || true
done
$S3 cp /work/ingest_time.txt "s3://$BUCKET/$RUN/ingest_time.txt" --only-show-errors || true
echo "ingest output size: $(du -sh /work/ingest | cut -f1)"
echo "=== done $(date -u +%T) ==="