SimurgArray: master plan, repo scaffolding and data/ref fetch scripts

Phase 0 deliverables: full technical specification and roadmap
(SIMURGARRAY_MASTER_PLAN.md), CLAUDE.md, bootstrap script for reference
open-source repos and tiered dataset fetch script with checksums.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01814GKVzZVL3FuF3DeiZXhv

scripts/fetch_data.sh

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+# Download SimurgArray datasets into data/<tier>/ and record sha256 checksums.
+# Vendor files are for personal research use only; never commit them.
+# Usage: bash scripts/fetch_data.sh p0|p1|p2
+set -euo pipefail
+cd "$(dirname "$0")/.."
+tier="${1:?usage: fetch_data.sh p0|p1|p2}"
+mkdir -p "data/$tier"
+sums="data/CHECKSUMS.sha256"; touch "$sums"
+
+get() {  # get <url> <dest-dir>
+  local url="$1" dir="$2" f
+  f="$dir/$(basename "${url%%\?*}")"
+  if [[ -s "$f" ]]; then echo "have $f"; else
+    if command -v aria2c >/dev/null; then aria2c -x8 -s8 -c -d "$dir" "$url"
+    else wget -c -P "$dir" "$url"; fi
+  fi
+  grep -q " $f\$" "$sums" || sha256sum "$f" >> "$sums"
+}
+
+GDA=https://webdata.illumina.com/downloads/productfiles/global-diversity-array
+G1K=https://ftp.1000genomes.ebi.ac.uk/vol1/ftp/release/20130502/supporting/hd_genotype_chip
+
+case "$tier" in
+p0)
+  d=data/p0/gda_demo36; mkdir -p "$d"
+  get "$GDA/infinium-global-diversity-array-8-v1-0-D1-demo-data-36.zip" "$d"
+  get "https://support.illumina.com/content/dam/illumina-support/documents/downloads/productfiles/global-diversity-array/infinium-global-diversity-array-8-v1-0_D1_SampleSheet_Demo_36.csv" "$d"
+  get "$GDA/infinium-global-diversity-array-8-v1-0-D1-manifest-file-bpm.zip" "$d"
+  # CSV manifest and EGT: file names can change; if these fail, download manually from
+  # https://support.illumina.com/array/array_kits/infinium-global-diversity-array/product-files.html
+  get "$GDA/infinium-global-diversity-array-8-v1-0-D1-manifest-file-csv.zip" "$d" || echo "!! fetch CSV manifest manually"
+  get "$GDA/infinium-global-diversity-array-8-v1-0-D1-cluster-file.zip" "$d" || echo "!! fetch EGT manually"
+  if command -v Rscript >/dev/null; then
+    Rscript -e 'if(!requireNamespace("BiocManager",quietly=TRUE)) install.packages("BiocManager",repos="https://cloud.r-project.org"); BiocManager::install(c("IlluminaDataTestFiles","hapmap370k"),ask=FALSE,update=FALSE)'
+    Rscript -e 'cat(system.file("extdata",package="IlluminaDataTestFiles"),"\n")' > data/p0/bioc_fixture_path.txt
+  else
+    echo "!! R not found: install IlluminaDataTestFiles + hapmap370k from Bioconductor for parser fixtures"
+  fi
+  echo "NOTE: GSA v4 PGx demo (36 samples) + product files: download manually from"
+  echo "  https://support.illumina.com/downloads/infinium-gsa-v4-pgx-demo-data-set-iscan.html"
+  echo "  https://support.illumina.com/array/array_kits/infinium-global-screening-array-v4-pgx/product-files.html"
+  ;;
+p1)
+  d=data/p1/1kg_omni_broad; mkdir -p "$d"
+  for f in HumanOmni2.5-4v1-Multi_B.bpm HumanOmni2.5-4v1_B.csv; do get "$G1K/broad_intensities/$f" "$d" || true; done
+  echo "Listing broad_intensities/ (pick EGT + the 107 GB Omni25_idats_gtcs_2141_samples.tgz):"
+  curl -s "$G1K/broad_intensities/" | grep -o 'href="[^"]*"' | cut -d'"' -f2 | grep -v '^?' || true
+  echo "Then: bash scripts/fetch_data.sh p1-big   (downloads the 107 GB archive)"
+  d=data/p1/1kg_omni_sanger; mkdir -p "$d"
+  echo "Sanger set index:"; curl -s "$G1K/sanger_intensities/" | grep -o 'href="[^"]*"' | cut -d'"' -f2 | grep -v '^?' || true
+  d=data/p1/gse165845; mkdir -p "$d"
+  get "https://ftp.ncbi.nlm.nih.gov/geo/series/GSE165nnn/GSE165845/suppl/GSE165845_RAW.tar" "$d"
+  ;;
+p1-big)
+  get "$G1K/broad_intensities/Omni25_idats_gtcs_2141_samples.tgz" data/p1/1kg_omni_broad
+  ;;
+p2)
+  d=data/p2; mkdir -p "$d/gse96909" "$d/gda_pgx480"
+  get "https://ftp.ncbi.nlm.nih.gov/geo/series/GSE96nnn/GSE96909/suppl/GSE96909_RAW.tar" "$d/gse96909"
+  get "https://s3.amazonaws.com/webdata.illumina.com/downloads/productfiles/global-diversity-array-pgx/GDAPGx_Demo_idats.zip" "$d/gda_pgx480"
+  ;;
+*) echo "unknown tier $tier"; exit 2;;
+esac
+echo "checksums in $sums"