SimurgArray: master plan, repo scaffolding and data/ref fetch scripts
Phase 0 deliverables: full technical specification and roadmap (SIMURGARRAY_MASTER_PLAN.md), CLAUDE.md, bootstrap script for reference open-source repos and tiered dataset fetch script with checksums.
This commit is contained in:
kemal committed
commit
631c1d030d
6 files changed
+1139
No files matched your search
+18
@@ -0,0 +1,18 @@
|
||||
# vendor / large data — never commit
|
||||
data/
|
||||
refs/
|
||||
_phase0_archive/
|
||||
*.idat
|
||||
*.idat.gz
|
||||
*.bpm
|
||||
*.egt
|
||||
*.gtc
|
||||
# python
|
||||
__pycache__/
|
||||
.venv/
|
||||
*.egg-info/
|
||||
.pytest_cache/
|
||||
.mypy_cache/
|
||||
.ruff_cache/
|
||||
# rust
|
||||
target/
|
||||
@@ -0,0 +1,17 @@
|
||||
# SimurgArray
|
||||
Personal research toolkit for Illumina Infinium genotyping arrays (IDAT→GTC→VCF, QC, CN, PGx).
|
||||
The full spec and roadmap is SIMURGARRAY_MASTER_PLAN.md — read the relevant section before changing code.
|
||||
|
||||
## Rules
|
||||
- Work phase by phase (§18). Don't start a phase until the previous gate passes.
|
||||
- Science modules (signal/, genotype/, cn/, qc/, pgx/) are pure functions; no I/O inside them.
|
||||
- Every algorithm cites its source (§7 tags) in its docstring.
|
||||
- MIT/BSD/Apache/Unlicense code may be ported with attribution in THIRD_PARTY_NOTICES.md.
|
||||
GPL tools (plink2, illuminaio, verifyIDintensity) = subprocess oracles only; never copy their code.
|
||||
- Never commit vendor data (BPM/CSV/EGT/IDAT) — only tests/fixtures/ (Artistic-2.0 Bioconductor files, synthetic data).
|
||||
- Never hide oracle discrepancies; record them in docs/known_unknowns.md.
|
||||
- Default mode is bit-exact vs Illumina-produced GTCs; keep a clean-math mode too.
|
||||
|
||||
## Commands
|
||||
- uv sync ; uv run pytest -q ; uv run ruff check . ; uv run mypy src
|
||||
- bash scripts/fetch_data.sh p0|p1|p2 ; bash scripts/bootstrap_refs.sh
|
||||
@@ -0,0 +1,11 @@
|
||||
# SimurgArray
|
||||
|
||||
Personal research toolkit for Illumina Infinium genotyping BeadChip data
|
||||
(IDAT → normalization → genotype calling → BAF/LRR → QC → VCF/PLINK; later CN/LOH, PGx, cluster training).
|
||||
|
||||
Start with **SIMURGARRAY_MASTER_PLAN.md** (full specification and roadmap).
|
||||
|
||||
```bash
|
||||
bash scripts/bootstrap_refs.sh # clone reference open-source repos into refs/
|
||||
bash scripts/fetch_data.sh p0 # download small test datasets into data/
|
||||
```
|
||||
File diff suppressed because it is too large.
Load diff
Executable
+24
@@ -0,0 +1,24 @@
|
||||
#!/usr/bin/env bash
|
||||
# Shallow-clone reference open-source repositories into refs/ (git-ignored).
|
||||
# GPL repos are for use as external oracles only — do not copy their code into SimurgArray.
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
mkdir -p refs
|
||||
repos=(
|
||||
"Illumina/BeadArrayFiles" # BSD-2
|
||||
"Illumina/GTCtoVCF" # Apache-2.0
|
||||
"freeseek/gtc2vcf" # MIT
|
||||
"freeseek/mocha" # MIT
|
||||
"broadinstitute/picard" # MIT (see src/main/java/picard/arrays)
|
||||
"WGLab/PennCNV" # Unlicense
|
||||
"sbslee/pypgx" # MIT
|
||||
"PharmGKB/PharmCAT" # MPL-2.0 (oracle)
|
||||
"HenrikBengtsson/illuminaio" # GPL-2 (oracle only)
|
||||
"gjun/verifyIDintensity" # GPL-3 (oracle only)
|
||||
)
|
||||
for r in "${repos[@]}"; do
|
||||
d="refs/${r#*/}"
|
||||
if [[ -d "$d/.git" ]]; then echo "skip $d"; continue; fi
|
||||
git clone --depth 1 "https://github.com/$r.git" "$d"
|
||||
done
|
||||
echo "done: $(ls refs | wc -l) repos in refs/"
|
||||
Executable
+65
@@ -0,0 +1,65 @@
|
||||
#!/usr/bin/env bash
|
||||
# Download SimurgArray datasets into data/<tier>/ and record sha256 checksums.
|
||||
# Vendor files are for personal research use only; never commit them.
|
||||
# Usage: bash scripts/fetch_data.sh p0|p1|p2
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
tier="${1:?usage: fetch_data.sh p0|p1|p2}"
|
||||
mkdir -p "data/$tier"
|
||||
sums="data/CHECKSUMS.sha256"; touch "$sums"
|
||||
|
||||
get() { # get <url> <dest-dir>
|
||||
local url="$1" dir="$2" f
|
||||
f="$dir/$(basename "${url%%\?*}")"
|
||||
if [[ -s "$f" ]]; then echo "have $f"; else
|
||||
if command -v aria2c >/dev/null; then aria2c -x8 -s8 -c -d "$dir" "$url"
|
||||
else wget -c -P "$dir" "$url"; fi
|
||||
fi
|
||||
grep -q " $f\$" "$sums" || sha256sum "$f" >> "$sums"
|
||||
}
|
||||
|
||||
GDA=https://webdata.illumina.com/downloads/productfiles/global-diversity-array
|
||||
G1K=https://ftp.1000genomes.ebi.ac.uk/vol1/ftp/release/20130502/supporting/hd_genotype_chip
|
||||
|
||||
case "$tier" in
|
||||
p0)
|
||||
d=data/p0/gda_demo36; mkdir -p "$d"
|
||||
get "$GDA/infinium-global-diversity-array-8-v1-0-D1-demo-data-36.zip" "$d"
|
||||
get "https://support.illumina.com/content/dam/illumina-support/documents/downloads/productfiles/global-diversity-array/infinium-global-diversity-array-8-v1-0_D1_SampleSheet_Demo_36.csv" "$d"
|
||||
get "$GDA/infinium-global-diversity-array-8-v1-0-D1-manifest-file-bpm.zip" "$d"
|
||||
# CSV manifest and EGT: file names can change; if these fail, download manually from
|
||||
# https://support.illumina.com/array/array_kits/infinium-global-diversity-array/product-files.html
|
||||
get "$GDA/infinium-global-diversity-array-8-v1-0-D1-manifest-file-csv.zip" "$d" || echo "!! fetch CSV manifest manually"
|
||||
get "$GDA/infinium-global-diversity-array-8-v1-0-D1-cluster-file.zip" "$d" || echo "!! fetch EGT manually"
|
||||
if command -v Rscript >/dev/null; then
|
||||
Rscript -e 'if(!requireNamespace("BiocManager",quietly=TRUE)) install.packages("BiocManager",repos="https://cloud.r-project.org"); BiocManager::install(c("IlluminaDataTestFiles","hapmap370k"),ask=FALSE,update=FALSE)'
|
||||
Rscript -e 'cat(system.file("extdata",package="IlluminaDataTestFiles"),"\n")' > data/p0/bioc_fixture_path.txt
|
||||
else
|
||||
echo "!! R not found: install IlluminaDataTestFiles + hapmap370k from Bioconductor for parser fixtures"
|
||||
fi
|
||||
echo "NOTE: GSA v4 PGx demo (36 samples) + product files: download manually from"
|
||||
echo " https://support.illumina.com/downloads/infinium-gsa-v4-pgx-demo-data-set-iscan.html"
|
||||
echo " https://support.illumina.com/array/array_kits/infinium-global-screening-array-v4-pgx/product-files.html"
|
||||
;;
|
||||
p1)
|
||||
d=data/p1/1kg_omni_broad; mkdir -p "$d"
|
||||
for f in HumanOmni2.5-4v1-Multi_B.bpm HumanOmni2.5-4v1_B.csv; do get "$G1K/broad_intensities/$f" "$d" || true; done
|
||||
echo "Listing broad_intensities/ (pick EGT + the 107 GB Omni25_idats_gtcs_2141_samples.tgz):"
|
||||
curl -s "$G1K/broad_intensities/" | grep -o 'href="[^"]*"' | cut -d'"' -f2 | grep -v '^?' || true
|
||||
echo "Then: bash scripts/fetch_data.sh p1-big (downloads the 107 GB archive)"
|
||||
d=data/p1/1kg_omni_sanger; mkdir -p "$d"
|
||||
echo "Sanger set index:"; curl -s "$G1K/sanger_intensities/" | grep -o 'href="[^"]*"' | cut -d'"' -f2 | grep -v '^?' || true
|
||||
d=data/p1/gse165845; mkdir -p "$d"
|
||||
get "https://ftp.ncbi.nlm.nih.gov/geo/series/GSE165nnn/GSE165845/suppl/GSE165845_RAW.tar" "$d"
|
||||
;;
|
||||
p1-big)
|
||||
get "$G1K/broad_intensities/Omni25_idats_gtcs_2141_samples.tgz" data/p1/1kg_omni_broad
|
||||
;;
|
||||
p2)
|
||||
d=data/p2; mkdir -p "$d/gse96909" "$d/gda_pgx480"
|
||||
get "https://ftp.ncbi.nlm.nih.gov/geo/series/GSE96nnn/GSE96909/suppl/GSE96909_RAW.tar" "$d/gse96909"
|
||||
get "https://s3.amazonaws.com/webdata.illumina.com/downloads/productfiles/global-diversity-array-pgx/GDAPGx_Demo_idats.zip" "$d/gda_pgx480"
|
||||
;;
|
||||
*) echo "unknown tier $tier"; exit 2;;
|
||||
esac
|
||||
echo "checksums in $sums"
|
||||
Reference in new issue
Block a user