kemal
e65d2426bb
- formats/fasta: plain/BGZF FASTA, .fai/.gzi built identical to htslib, index check, build detection, contig aliases, mmap genome cache (IN-060..064) - formats/bgzf, formats/bcf: pure-Python BGZF, tabix .tbi, BCF 2.2 and CSI (OUT-010, OUT-038, OUT-040) - harmonize/sites: plus-strand mapping with audit rows, indel resolution from SourceSeq (port of GTCtoVCF, Apache-2.0) and left normalisation (OUT-014..021, OUT-030, OUT-042) - harmonize/combine: OUT-031 genotype combination (ploidy-aware, Infinium I/II signal classes, OUT-037) - pipeline/vcf: `simurg gtc-to-vcf` (per-sample VCF.gz+tbi, --merge cohort BCF+csi, audit parquet) - pipeline/export: `simurg export bedgraph|finalreport|plink|locus-summary` (OUT-043, OUT-050..052, OUT-071) - validation: 36 GSA samples vs gtc2vcf: 23.66 M identical calls, 0 unexplained differences; PLINK identical to plink2 on 637,397 variants; bcftools validates VCF/BCF and our indexes answer region queries correctly - fetch_data.sh p6 (Ensembl GRCh38 with checksum), make_oracle_vcf.sh, validate_p6_vcf.py; K-022/K-023
92 lines
4.7 KiB
Bash
Executable File
92 lines
4.7 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Download SimurgArray datasets into data/<tier>/ and record sha256 checksums.
|
|
# Vendor files are for personal research use only; never commit them.
|
|
# Usage: bash scripts/fetch_data.sh p0|p1|p2
|
|
set -euo pipefail
|
|
cd "$(dirname "$0")/.."
|
|
tier="${1:?usage: fetch_data.sh p0|p1|p2}"
|
|
mkdir -p "data/$tier"
|
|
sums="data/CHECKSUMS.sha256"; touch "$sums"
|
|
|
|
get() { # get <url> <dest-dir>
|
|
local url="$1" dir="$2" f
|
|
f="$dir/$(basename "${url%%\?*}")"
|
|
if [[ -s "$f" ]]; then echo "have $f"; else
|
|
if command -v aria2c >/dev/null; then aria2c -x8 -s8 -c -d "$dir" "$url"
|
|
else wget -c -P "$dir" "$url"; fi
|
|
fi
|
|
grep -q " $f\$" "$sums" || sha256sum "$f" >> "$sums"
|
|
}
|
|
|
|
unpack() { # unzip/untar every archive under <dir> next to itself (idempotent)
|
|
find "$1" -name '*.zip' -o -name '*.tar.gz' | sort | while read -r a; do
|
|
out="${a%.zip}"; out="${out%.tar.gz}"
|
|
[[ -d "$out" ]] && continue
|
|
mkdir -p "$out"
|
|
case "$a" in *.zip) unzip -q -o "$a" -d "$out";; *.tar.gz) tar -xzf "$a" -C "$out";; esac
|
|
done
|
|
}
|
|
|
|
GDA=https://webdata.illumina.com/downloads/productfiles/global-diversity-array
|
|
G1K=https://ftp.1000genomes.ebi.ac.uk/vol1/ftp/release/20130502/supporting/hd_genotype_chip
|
|
|
|
case "$tier" in
|
|
p0)
|
|
# GSA v4 PGx demo: 36 samples, real IDATs + BPM/CSV/EGT (primary small fixture)
|
|
SUP=https://support.illumina.com/content/dam/illumina-support/documents/documentation/chemistry_documentation/infinium_assays/infinium-gsa-pgx
|
|
d=data/p0/gsa_v4_pgx; mkdir -p "$d"
|
|
for f in idats_Demo_36.zip GSA-PGx-48v4-0_20079540_E2_bpm.zip GSA-PGx-48v4-0_20079540_E2_csv.zip \
|
|
GSA-PGx-48v4-0_20079540_E2_ClusterFile.zip SampleSheetTemplate_GSAv4PGx_2beadchips.csv; do
|
|
get "$SUP/$f" "$d"
|
|
done
|
|
# GDA v1 manifest/cluster files (note: the "Demo 36" bundle is a GenomeStudio project, not IDATs)
|
|
d=data/p0/gda_v1; mkdir -p "$d"
|
|
get "$GDA/infinium-global-diversity-array-8-v1-0-D1-manifest-file-bpm.zip" "$d"
|
|
get "$GDA/infinium-global-diversity-array-8-v1-0-D1-manifest-file-csv.zip" "$d"
|
|
get "$GDA/infinium-global-diversity-array-8-v1-0-D1-cluster-file.zip" "$d"
|
|
get "https://support.illumina.com/content/dam/illumina-support/documents/downloads/productfiles/global-diversity-array/infinium-global-diversity-array-8-v1-0_D1_SampleSheet_Demo_36.csv" "$d"
|
|
# Bioconductor IDAT fixtures (Artistic-2.0, redistributable) — fetched as source tarballs, no R needed
|
|
d=data/p0/bioc; mkdir -p "$d"
|
|
for pkg in IlluminaDataTestFiles hapmap370k; do
|
|
url=$(curl -sL "https://bioconductor.org/packages/release/data/experiment/html/$pkg.html" \
|
|
| grep -oE "\.\./src/contrib/${pkg}_[0-9.]+\.tar\.gz" | head -1 || true)
|
|
[[ -n "$url" ]] && get "https://bioconductor.org/packages/release/data/experiment/${url#../}" "$d" \
|
|
|| echo "!! could not resolve $pkg tarball"
|
|
done
|
|
unpack data/p0
|
|
;;
|
|
p1)
|
|
d=data/p1/1kg_omni_broad; mkdir -p "$d"
|
|
for f in HumanOmni2.5-4v1-Multi_B.bpm HumanOmni2.5-4v1_B.csv; do get "$G1K/broad_intensities/$f" "$d" || true; done
|
|
echo "Listing broad_intensities/ (pick EGT + the 107 GB Omni25_idats_gtcs_2141_samples.tgz):"
|
|
curl -s "$G1K/broad_intensities/" | grep -o 'href="[^"]*"' | cut -d'"' -f2 | grep -v '^?' || true
|
|
echo "Then: bash scripts/fetch_data.sh p1-big (downloads the 107 GB archive)"
|
|
d=data/p1/1kg_omni_sanger; mkdir -p "$d"
|
|
echo "Sanger set index:"; curl -s "$G1K/sanger_intensities/" | grep -o 'href="[^"]*"' | cut -d'"' -f2 | grep -v '^?' || true
|
|
d=data/p1/gse165845; mkdir -p "$d"
|
|
get "https://ftp.ncbi.nlm.nih.gov/geo/series/GSE165nnn/GSE165845/suppl/GSE165845_RAW.tar" "$d"
|
|
;;
|
|
p1-big)
|
|
get "$G1K/broad_intensities/Omni25_idats_gtcs_2141_samples.tgz" data/p1/1kg_omni_broad
|
|
;;
|
|
p2)
|
|
d=data/p2; mkdir -p "$d/gse96909" "$d/gda_pgx480"
|
|
get "https://ftp.ncbi.nlm.nih.gov/geo/series/GSE96nnn/GSE96909/suppl/GSE96909_RAW.tar" "$d/gse96909"
|
|
get "https://s3.amazonaws.com/webdata.illumina.com/downloads/productfiles/global-diversity-array-pgx/GDAPGx_Demo_idats.zip" "$d/gda_pgx480"
|
|
;;
|
|
p6)
|
|
# GRCh38 primary assembly (Ensembl release 110; contigs named 1..22, X, Y, MT like the GSA/GDA manifests)
|
|
d=data/p6; mkdir -p "$d"
|
|
ENS=https://ftp.ensembl.org/pub/release-110/fasta/homo_sapiens/dna
|
|
f=Homo_sapiens.GRCh38.dna.primary_assembly.fa.gz
|
|
[[ -f "$d/$f" ]] || curl -sSfL -o "$d/$f" "$ENS/$f"
|
|
want=$(curl -sSfL "$ENS/CHECKSUMS" | awk -v f="$f" '$3==f {print $1" "$2}')
|
|
got=$(sum "$d/$f" | awk '{print $1" "$2}')
|
|
[[ "$want" == "$got" ]] || { echo "!! checksum mismatch for $f ($got vs $want)"; exit 1; }
|
|
[[ -f "$d/GRCh38.primary.fa" ]] || gzip -dc "$d/$f" > "$d/GRCh38.primary.fa"
|
|
echo "FASTA ready: $d/GRCh38.primary.fa (the .fai is built on first use)"
|
|
;;
|
|
*) echo "unknown tier $tier"; exit 2;;
|
|
esac
|
|
echo "checksums in $sums"
|