Add self-destructing AWS trial and large-scale P1 validation script

scripts/cloud/trial.sh launches one spot instance that validates the parsers on the 1000 Genomes Omni2.5 set (2141 samples, vendor GTCs) streamed from the public S3 bucket, uploads the summary and terminates itself. Cost guards: spot price cap, hard time limit (shutdown = terminate), EXIT-trap shutdown, CloudWatch idle alarm that terminates from outside, disk deleted with the instance, monthly budget alert, and an `audit` command listing anything still billable. scripts/validate_p1_large.py compares every GTC with BeadArrayFiles, checks byte-identical round-trips, IDAT pairs and BPM/EGT/CSV. Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01814GKVzZVL3FuF3DeiZXhv
2026-09-27 19:30:27 +01:00
1 parent 4357d3deef
commit a9aa6dc080
@@ -0,0 +1,123 @@
+#!/usr/bin/env bash
+# Self-destructing, budget-guarded AWS validation run for SimurgArray.
+#
+#   bash scripts/cloud/trial.sh setup  <alert-email>   # one-time: budget alert, results bucket, instance role
+#   bash scripts/cloud/trial.sh launch                  # start one spot run (terminates itself when done)
+#   bash scripts/cloud/trial.sh status                  # instance state + last lines of the remote log
+#   bash scripts/cloud/trial.sh results                 # download log/summary of the latest run
+#   bash scripts/cloud/trial.sh kill                    # emergency: terminate every trial instance now
+#   bash scripts/cloud/trial.sh audit                   # list anything still costing money
+#
+# Cost guards: spot price cap, hard time cap (shutdown = terminate), EXIT trap shutdown, CloudWatch idle alarm
+# that terminates from outside, disk deleted with the instance, monthly budget e-mail alert.
+set -euo pipefail
+cd "$(dirname "$0")/../.."
+REGION=${REGION:-us-east-1}
+TYPE=${TYPE:-m7g.4xlarge}
+MAX_PRICE=${MAX_PRICE:-0.30}        # USD/hour spot cap (on-demand is 0.6528)
+MAX_MINUTES=${MAX_MINUTES:-360}     # hard cap: 6 hours
+DISK_GB=${DISK_GB:-200}
+BUDGET_USD=${BUDGET_USD:-10}
+TAG=simurg-trial
+ACCOUNT=$(aws sts get-caller-identity --query Account --output text)
+BUCKET=simurg-trial-$ACCOUNT
+ROLE=simurg-trial-role
+AWS="aws --region $REGION"
+
+cmd=${1:-help}
+case "$cmd" in
+setup)
+  EMAIL=${2:?usage: trial.sh setup <alert-email>}
+  echo "-> budget alert: \$$BUDGET_USD/month, e-mail $EMAIL at 50%, 100% actual and 100% forecast"
+  cat > /tmp/simurg-budget.json <<EOF
+{"BudgetName":"$TAG-monthly","BudgetLimit":{"Amount":"$BUDGET_USD","Unit":"USD"},"TimeUnit":"MONTHLY","BudgetType":"COST"}
+EOF
+  notif() { echo "{\"Notification\":{\"NotificationType\":\"$1\",\"ComparisonOperator\":\"GREATER_THAN\",\"Threshold\":$2,\"ThresholdType\":\"PERCENTAGE\"},\"Subscribers\":[{\"SubscriptionType\":\"EMAIL\",\"Address\":\"$EMAIL\"}]}"; }
+  echo "[$(notif ACTUAL 50),$(notif ACTUAL 100),$(notif FORECASTED 100)]" > /tmp/simurg-notif.json
+  aws budgets create-budget --account-id "$ACCOUNT" --budget file:///tmp/simurg-budget.json \
+    --notifications-with-subscribers file:///tmp/simurg-notif.json 2>/dev/null || echo "   (budget already exists)"
+  echo "-> results bucket s3://$BUCKET (objects expire after 30 days)"
+  $AWS s3api create-bucket --bucket "$BUCKET" >/dev/null 2>&1 || true
+  $AWS s3api put-public-access-block --bucket "$BUCKET" --public-access-block-configuration \
+    BlockPublicAcls=true,IgnorePublicAcls=true,BlockPublicPolicy=true,RestrictPublicBuckets=true
+  $AWS s3api put-bucket-lifecycle-configuration --bucket "$BUCKET" --lifecycle-configuration \
+    '{"Rules":[{"ID":"expire","Status":"Enabled","Filter":{},"Expiration":{"Days":30}}]}'
+  echo "-> instance role $ROLE (read/write only this bucket)"
+  aws iam create-role --role-name $ROLE --assume-role-policy-document \
+    '{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"Service":"ec2.amazonaws.com"},"Action":"sts:AssumeRole"}]}' \
+    >/dev/null 2>&1 || true
+  aws iam put-role-policy --role-name $ROLE --policy-name bucket-only --policy-document \
+    "{\"Version\":\"2012-10-17\",\"Statement\":[{\"Effect\":\"Allow\",\"Action\":[\"s3:GetObject\",\"s3:PutObject\"],\"Resource\":\"arn:aws:s3:::$BUCKET/*\"}]}"
+  aws iam create-instance-profile --instance-profile-name $ROLE >/dev/null 2>&1 || true
+  aws iam add-role-to-instance-profile --instance-profile-name $ROLE --role-name $ROLE 2>/dev/null || true
+  echo "setup done (budget alerts need you to confirm nothing; AWS e-mails $EMAIL when thresholds are crossed)"
+  ;;
+launch)
+  running=$($AWS ec2 describe-instances --filters Name=tag:Project,Values=$TAG \
+    Name=instance-state-name,Values=pending,running --query 'length(Reservations[].Instances[])')
+  [[ "$running" == "0" ]] || { echo "a trial instance is already running; refusing to start another"; exit 1; }
+  RUN=run-$(date -u +%Y%m%dT%H%M%SZ)
+  echo "-> packing repository at $(git rev-parse --short HEAD) -> s3://$BUCKET/$RUN/"
+  git archive --format=tar.gz -o /tmp/simurg-repo.tar.gz HEAD
+  $AWS s3 cp /tmp/simurg-repo.tar.gz "s3://$BUCKET/$RUN/repo.tar.gz" --only-show-errors
+  sed -e "s/__BUCKET__/$BUCKET/" -e "s/__RUN__/$RUN/" -e "s/__MAX_MINUTES__/$MAX_MINUTES/" \
+    scripts/cloud/userdata.sh > /tmp/simurg-userdata.sh
+  AMI=$($AWS ssm get-parameter --name /aws/service/canonical/ubuntu/server/24.04/stable/current/arm64/hvm/ebs-gp3/ami-id \
+    --query Parameter.Value --output text)
+  echo "-> launching $TYPE spot (cap \$$MAX_PRICE/h, hard limit ${MAX_MINUTES} min, ${DISK_GB} GB disk deleted on exit)"
+  ID=$($AWS ec2 run-instances --image-id "$AMI" --instance-type "$TYPE" --count 1 \
+    --iam-instance-profile Name=$ROLE \
+    --instance-initiated-shutdown-behavior terminate \
+    --instance-market-options "MarketType=spot,SpotOptions={MaxPrice=$MAX_PRICE,SpotInstanceType=one-time,InstanceInterruptionBehavior=terminate}" \
+    --block-device-mappings "DeviceName=/dev/sda1,Ebs={VolumeSize=$DISK_GB,VolumeType=gp3,DeleteOnTermination=true}" \
+    --metadata-options HttpTokens=required \
+    --user-data file:///tmp/simurg-userdata.sh \
+    --tag-specifications "ResourceType=instance,Tags=[{Key=Project,Value=$TAG},{Key=Run,Value=$RUN}]" \
+      "ResourceType=volume,Tags=[{Key=Project,Value=$TAG},{Key=Run,Value=$RUN}]" \
+    --query 'Instances[0].InstanceId' --output text)
+  echo "$ID $RUN" > .trial-last
+  echo "-> idle alarm: terminate if average CPU < 2% for 30 minutes"
+  $AWS cloudwatch put-metric-alarm --alarm-name "$TAG-idle-$ID" --namespace AWS/EC2 --metric-name CPUUtilization \
+    --dimensions Name=InstanceId,Value="$ID" --statistic Average --period 300 --evaluation-periods 6 \
+    --threshold 2 --comparison-operator LessThanThreshold --treat-missing-data notBreaching \
+    --alarm-actions "arn:aws:automate:$REGION:ec2:terminate"
+  echo "launched $ID ($RUN). Check with: bash scripts/cloud/trial.sh status"
+  ;;
+status)
+  read -r ID RUN < .trial-last
+  $AWS ec2 describe-instances --instance-ids "$ID" \
+    --query 'Reservations[0].Instances[0].[InstanceId,State.Name,InstanceType,LaunchTime]' --output text
+  $AWS s3 cp "s3://$BUCKET/$RUN/log.txt" - 2>/dev/null | tail -n "${LINES_:-15}" || echo "(no log uploaded yet)"
+  ;;
+results)
+  read -r ID RUN < .trial-last
+  mkdir -p "results/$RUN"
+  $AWS s3 cp "s3://$BUCKET/$RUN/" "results/$RUN/" --recursive --exclude repo.tar.gz --only-show-errors
+  ls -la "results/$RUN"
+  ;;
+kill)
+  ids=$($AWS ec2 describe-instances --filters Name=tag:Project,Values=$TAG \
+    Name=instance-state-name,Values=pending,running,stopping,stopped --query 'Reservations[].Instances[].InstanceId' --output text)
+  [[ -n "$ids" ]] && $AWS ec2 terminate-instances --instance-ids $ids --output text || echo "no trial instances"
+  ;&
+audit)
+  echo "-- instances (any project) --"
+  $AWS ec2 describe-instances --filters Name=instance-state-name,Values=pending,running,stopping,stopped \
+    --query 'Reservations[].Instances[].[InstanceId,State.Name,InstanceType]' --output text
+  echo "-- volumes --"
+  $AWS ec2 describe-volumes --query 'Volumes[].[VolumeId,State,Size]' --output text
+  echo "-- snapshots owned --"
+  $AWS ec2 describe-snapshots --owner-ids self --query 'Snapshots[].[SnapshotId,VolumeSize]' --output text
+  echo "-- elastic IPs --"
+  $AWS ec2 describe-addresses --query 'Addresses[].PublicIp' --output text
+  echo "-- trial alarms (deleting alarms of terminated instances) --"
+  for a in $($AWS cloudwatch describe-alarms --alarm-name-prefix "$TAG-idle-" --query 'MetricAlarms[].AlarmName' --output text); do
+    iid=${a#"$TAG-idle-"}
+    st=$($AWS ec2 describe-instances --instance-ids "$iid" --query 'Reservations[0].Instances[0].State.Name' --output text 2>/dev/null || echo gone)
+    if [[ "$st" == "terminated" || "$st" == "gone" ]]; then $AWS cloudwatch delete-alarms --alarm-names "$a"; echo "deleted $a"; else echo "$a ($st)"; fi
+  done
+  ;;
+*)
+  sed -n '2,15p' "$0"
+  ;;
+esac
@@ -0,0 +1,62 @@
+#!/bin/bash
+# EC2 user-data for a self-destructing SimurgArray validation run (Ubuntu 24.04, arm64).
+# Placeholders (__X__) are filled in by scripts/cloud/trial.sh.
+#
+# Safety layers inside the instance:
+#   1. `shutdown -h +MAX_MINUTES` is scheduled first  -> hard time cap
+#   2. an EXIT trap uploads the log and shuts down      -> ends as soon as the job ends, success or failure
+#   Shutdown = TERMINATE (instance-initiated-shutdown-behavior) and the only disk has DeleteOnTermination=true,
+#   so nothing keeps costing money afterwards. A CloudWatch alarm (layer 3) terminates an idle instance from
+#   outside, in case the OS hangs.
+set -uo pipefail
+BUCKET="__BUCKET__"
+RUN="__RUN__"
+MAX_MINUTES="__MAX_MINUTES__"
+shutdown -h +"$MAX_MINUTES" "SimurgArray trial hard time limit"
+
+LOG=/var/log/simurg-trial.log
+exec > >(tee -a "$LOG") 2>&1
+S3=""
+upload_log() { [[ -n "$S3" ]] && $S3 cp "$LOG" "s3://$BUCKET/$RUN/log.txt" --only-show-errors || true; }
+finish() {
+  rc=$?
+  echo "=== finished rc=$rc at $(date -u +%FT%TZ) ==="
+  echo "$rc" > /tmp/rc && [[ -n "$S3" ]] && $S3 cp /tmp/rc "s3://$BUCKET/$RUN/exit_code.txt" --only-show-errors
+  upload_log
+  shutdown -h now
+}
+trap finish EXIT
+set -e
+echo "=== start $(date -u +%FT%TZ) on $(uname -m), $(nproc) cpus ==="
+
+export DEBIAN_FRONTEND=noninteractive
+apt-get update -q
+apt-get install -y -q pigz git python3-venv curl unzip
+snap install aws-cli --classic
+S3="/snap/bin/aws s3"
+( while sleep 300; do upload_log; done ) &
+
+mkdir -p /work && cd /work
+$S3 cp "s3://$BUCKET/$RUN/repo.tar.gz" . --only-show-errors
+mkdir repo && tar -xzf repo.tar.gz -C repo && cd repo
+curl -LsSf https://astral.sh/uv/install.sh | sh
+export PATH="$HOME/.local/bin:$PATH"
+uv venv -q --python 3.12 .venv && uv pip install -q -e ".[dev]"
+mkdir -p refs && git clone -q --depth 1 https://github.com/Illumina/BeadArrayFiles.git refs/BeadArrayFiles
+
+SRC=s3://1000genomes/release/20130502/supporting/hd_genotype_chip/broad_intensities
+D=/work/omni && mkdir -p $D/data
+for f in HumanOmni2.5-4v1-Multi_B.bpm HumanOmni2.5-4v1-Multi_B.egt HumanOmni2.5-4v1_B.csv; do
+  $S3 cp --no-sign-request "$SRC/$f" $D/ --only-show-errors
+done
+echo "=== streaming 107 GB IDAT+GTC archive (no copy of the .tgz is kept) $(date -u +%T) ==="
+$S3 cp --no-sign-request "$SRC/Omni25_idats_gtcs_2141_samples.tgz" - | pigz -dc | tar -x -C $D/data
+echo "=== extracted $(du -sh $D/data | cut -f1) at $(date -u +%T) ==="
+
+echo "=== validation $(date -u +%T) ==="
+.venv/bin/python -m pytest tests/unit -q -p no:cacheprovider
+.venv/bin/python scripts/validate_p1_large.py --root $D/data --bpm $D/HumanOmni2.5-4v1-Multi_B.bpm \
+  --egt $D/HumanOmni2.5-4v1-Multi_B.egt --csv $D/HumanOmni2.5-4v1_B.csv --out /work/summary.json \
+  --workers "$(nproc)" || echo "validation reported differences (see summary.json)"
+$S3 cp /work/summary.json "s3://$BUCKET/$RUN/summary.json" --only-show-errors
+echo "=== done $(date -u +%T) ==="