From f10c651f0ff105153ce200caa432ed5b2c5c9af9 Mon Sep 17 00:00:00 2001 From: Kemal Yaylali Date: Sun, 27 Sep 2026 16:56:39 +0100 Subject: [PATCH] P0: project setup (packaging, CI, licences, data/oracle scripts) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - pyproject (uv/hatch, ruff, mypy strict, pytest markers), MIT LICENSE, THIRD_PARTY_NOTICES with verbatim BSD-2 (BeadArrayFiles) and MIT (gtc2vcf) - CI workflow for Gitea/GitHub Actions: lint, type check, unit tests - fetch_data.sh p0 now fetches the GSA v4 PGx demo IDATs (the GDA "Demo 36" bundle is a GenomeStudio project), unpacks archives, fixes pipefail bug - make_oracle_gtcs.sh builds bcftools + gtc2vcf and writes oracle GTCs - plan §17: corrected P0 dataset table Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01814GKVzZVL3FuF3DeiZXhv --- .github/workflows/ci.yml | 25 ++++++++++++++ LICENSE | 21 +++++++++++ README.md | 23 ++++++++++--- SIMURGARRAY_MASTER_PLAN.md | 14 ++++---- THIRD_PARTY_NOTICES.md | 69 +++++++++++++++++++++++++++++++++++++ pyproject.toml | 64 ++++++++++++++++++++++++++++++++++ scripts/fetch_data.sh | 46 ++++++++++++++++--------- scripts/make_oracle_gtcs.sh | 34 ++++++++++++++++++ 8 files changed, 269 insertions(+), 27 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 LICENSE create mode 100644 THIRD_PARTY_NOTICES.md create mode 100644 pyproject.toml create mode 100755 scripts/make_oracle_gtcs.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..125afeb --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,25 @@ +# Runs on Gitea Actions (reads .github/workflows) and GitHub Actions. +name: ci +on: + push: + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + python: ["3.12", "3.13"] + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v5 + with: + python-version: ${{ matrix.python }} + - name: Install + run: uv venv && uv pip install -e ".[dev]" + - name: Lint + run: uv run ruff check src tests && uv run ruff format --check src tests + - name: Type check + run: uv run mypy + - name: Unit tests (no downloaded data needed) + run: uv run pytest tests/unit -q diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..33a742d --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Kemal Yaylali + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index b1f9b29..c1b7008 100644 --- a/README.md +++ b/README.md @@ -3,10 +3,25 @@ Personal research toolkit for Illumina Infinium genotyping BeadChip data (IDAT → normalization → genotype calling → BAF/LRR → QC → VCF/PLINK; later CN/LOH, PGx, cluster training). -Start with **SIMURGARRAY_MASTER_PLAN.md** (architecture, algorithms, roadmap) and -**SIMURGARRAY_FUNCTIONAL_SPEC.md** (416 itemised requirements: CLI, inputs, outputs, QC, CN, PGx, methylation, edge cases). +- **SIMURGARRAY_MASTER_PLAN.md** — architecture, algorithms, roadmap +- **SIMURGARRAY_FUNCTIONAL_SPEC.md** — 416 itemised requirements (CLI, inputs, outputs, QC, CN, PGx, methylation, edge cases) +- `docs/validation/` — per-phase validation reports · `docs/known_unknowns.md` — findings log · `docs/research/` — background + +## Status + +Phases **P0 (setup)** and **P1 (file parsers)** are complete — see `docs/validation/P1.md`. +Available commands: `simurg version`, `simurg inspect [--json]`. + +## Quick start ```bash -bash scripts/bootstrap_refs.sh # clone reference open-source repos into refs/ -bash scripts/fetch_data.sh p0 # download small test datasets into data/ +uv venv && uv pip install -e ".[dev]" +uv run pytest tests/unit # no downloads needed +bash scripts/bootstrap_refs.sh # reference open-source repos → refs/ (oracles) +bash scripts/fetch_data.sh p0 # ~1.4 GB of public test data → data/ +bash scripts/make_oracle_gtcs.sh # builds gtc2vcf, writes oracle GTCs +uv run pytest tests/oracle # real-data acceptance tests (add -m "not slow" to skip the long ones) +uv run simurg inspect data/p0/gsa_v4_pgx/idats_Demo_36/idats_Demo_36/207895980019/207895980019_R03C01_Grn.idat ``` + +Vendor data files are downloaded for personal research use only and are never committed. See `THIRD_PARTY_NOTICES.md`. diff --git a/SIMURGARRAY_MASTER_PLAN.md b/SIMURGARRAY_MASTER_PLAN.md index bd49271..f5fad40 100644 --- a/SIMURGARRAY_MASTER_PLAN.md +++ b/SIMURGARRAY_MASTER_PLAN.md @@ -817,17 +817,17 @@ precision/recall vs 1000G WGS CNV/SV callsets; PGx diplotype concordance vs WGS- Vendor files are for personal research use only and are **never committed**. `scripts/fetch_data.sh ` downloads, records SHA-256 in `data/CHECKSUMS.sha256`, and unpacks. -### Tier P0 — small, start here (~3 GB) +### Tier P0 — small, start here (~1.4 GB; verified 2026-09-27) | Dataset | URL | Notes | |---|---|---| -| Bioconductor IlluminaDataTestFiles | https://bioconductor.org/packages/IlluminaDataTestFiles | Artistic-2.0 → **may be committed** as IDAT parser fixtures | -| Bioconductor hapmap370k | https://bioconductor.org/packages/hapmap370k | Artistic-2.0; 40 HapMap IDATs | -| GDA v1 Demo 36 (IDATs) | https://webdata.illumina.com/downloads/productfiles/global-diversity-array/infinium-global-diversity-array-8-v1-0-D1-demo-data-36.zip | 465 MB; 3 CEU trios incl. NA12878 + replicates | -| GDA v1 sample sheet | https://support.illumina.com/content/dam/illumina-support/documents/downloads/productfiles/global-diversity-array/infinium-global-diversity-array-8-v1-0_D1_SampleSheet_Demo_36.csv | has Parent1/Parent2/Replicate | -| GDA v1 BPM / CSV / EGT | https://webdata.illumina.com/downloads/productfiles/global-diversity-array/infinium-global-diversity-array-8-v1-0-D1-manifest-file-bpm.zip (and `…-csv.zip`, `…-D1-cluster-file.zip`; page: https://support.illumina.com/array/array_kits/infinium-global-diversity-array/product-files.html) | | -| GSA v4 PGx demo (36 samples) | https://support.illumina.com/downloads/infinium-gsa-v4-pgx-demo-data-set-iscan.html | 872 MB; product files at https://support.illumina.com/array/array_kits/infinium-global-screening-array-v4-pgx/product-files.html | +| **GSA v4 PGx demo IDATs (36 samples, 72 IDATs)** | https://support.illumina.com/content/dam/illumina-support/documents/documentation/chemistry_documentation/infinium_assays/infinium-gsa-pgx/idats_Demo_36.zip | 872 MB; primary small fixture. BPM/CSV/EGT from the same folder (`GSA-PGx-48v4-0_20079540_E2_{bpm,csv,ClusterFile}.zip`) | +| Bioconductor IlluminaDataTestFiles | https://bioconductor.org/packages/IlluminaDataTestFiles | Artistic-2.0 → **may be committed** as IDAT parser fixtures; includes 2 encrypted IDATs (negative tests) | +| GDA v1 BPM / CSV / EGT + demo sample sheet | https://support.illumina.com/array/array_kits/infinium-global-diversity-array/product-files.html | second design (1.9 M loci). Note: the "GDA Demo 36" bundle is a **GenomeStudio project, not IDATs** | | GRCh37 & GRCh38 FASTA | e.g. GRCh38 no-alt analysis set (NCBI) | contigs named per manifest | +(Bioconductor `hapmap370k` is no longer in the current release.) Oracle GTCs for P1–P4 are generated locally with +gtc2vcf `idat2gtc` (MIT) from the GSA demo IDATs — see `docs/validation/P1.md`. + ### Tier P1 — validation core (~170 GB) | Dataset | URL | Notes | |---|---|---| diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md new file mode 100644 index 0000000..3146235 --- /dev/null +++ b/THIRD_PARTY_NOTICES.md @@ -0,0 +1,69 @@ +# Third-party notices + +SimurgArray is MIT-licensed. It uses the following sources under their licences (plan §2). GPL-licensed tools are +used only as external subprocess oracles and are not copied or linked. + +| Source | Licence | How it is used | +|---|---|---| +| [Illumina BeadArrayFiles](https://github.com/Illumina/BeadArrayFiles) — Copyright (c) 2020, Illumina | BSD-2-Clause | File-format layouts (GTC, BPM, EGT), normalization-bin rule, `GTC_File_Format_v5.pdf`; test oracle | +| [gtc2vcf / idat2gtc](https://github.com/freeseek/gtc2vcf) — Copyright (C) 2018-2025 Giulio Genovese | MIT | IDAT field codes, BPM locus-entry order, EGT data-block variants, GTC write order, assay-type inference (`infer_assay_type`); test oracle | +| [Illumina GTCtoVCF](https://github.com/Illumina/GTCtoVCF) | Apache-2.0 | VCF conversion rules (phase P6) | +| [Picard `picard.arrays`](https://github.com/broadinstitute/picard) | MIT | Second format reference | +| [illuminaio](https://github.com/HenrikBengtsson/illuminaio) | GPL-2 | Documentation of IDAT field codes only; oracle via subprocess | + +## BSD-2-Clause (BeadArrayFiles, verbatim from its README) + +``` +Copyright (c) 2020, Illumina +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The views and conclusions contained in the software and documentation are those +of the authors and should not be interpreted as representing official policies, +either expressed or implied, of the FreeBSD Project. +``` + +## MIT (gtc2vcf, verbatim LICENSE) + +``` +The MIT License + +Copyright (C) 2018-2025 Giulio Genovese + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +``` diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..5574e89 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,64 @@ +[build-system] +requires = ["hatchling>=1.24"] +build-backend = "hatchling.build" + +[project] +name = "simurg" +description = "SimurgArray: open research toolkit for Illumina Infinium genotyping BeadChip data" +readme = "README.md" +license = { text = "MIT" } +requires-python = ">=3.12" +authors = [{ name = "Kemal Yaylali" }] +dynamic = ["version"] +dependencies = [ + "numpy>=1.26", # all numerics + "typer>=0.12", # CLI + "rich>=13", # progress / human-readable output + "pyyaml>=6", # --config files +] + +[project.optional-dependencies] +dev = [ + "pytest>=8", + "pytest-benchmark>=4", + "hypothesis>=6.100", + "ruff>=0.6", + "mypy>=1.11", + "types-PyYAML", +] + +[project.scripts] +simurg = "simurg.cli.main:app" + +[tool.hatch.version] +path = "src/simurg/__init__.py" + +[tool.hatch.build.targets.wheel] +packages = ["src/simurg"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "-q --strict-markers" +markers = [ + "data: needs downloaded datasets under data/ (skipped when absent)", + "oracle: compares against a reference implementation under refs/ (skipped when absent)", + "slow: long-running test", +] + +[tool.ruff] +line-length = 120 +target-version = "py312" +src = ["src", "tests"] + +[tool.ruff.lint] +select = ["E", "F", "W", "I", "B", "UP", "N", "SIM", "RUF"] +ignore = ["N802", "N806", "RUF001", "RUF002", "RUF003"] + +[tool.ruff.lint.per-file-ignores] +"tests/**" = ["E501"] + +[tool.mypy] +python_version = "3.12" +strict = true +files = ["src/simurg"] +ignore_missing_imports = true diff --git a/scripts/fetch_data.sh b/scripts/fetch_data.sh index c94b31e..614a792 100755 --- a/scripts/fetch_data.sh +++ b/scripts/fetch_data.sh @@ -18,28 +18,42 @@ get() { # get grep -q " $f\$" "$sums" || sha256sum "$f" >> "$sums" } +unpack() { # unzip/untar every archive under next to itself (idempotent) + find "$1" -name '*.zip' -o -name '*.tar.gz' | sort | while read -r a; do + out="${a%.zip}"; out="${out%.tar.gz}" + [[ -d "$out" ]] && continue + mkdir -p "$out" + case "$a" in *.zip) unzip -q -o "$a" -d "$out";; *.tar.gz) tar -xzf "$a" -C "$out";; esac + done +} + GDA=https://webdata.illumina.com/downloads/productfiles/global-diversity-array G1K=https://ftp.1000genomes.ebi.ac.uk/vol1/ftp/release/20130502/supporting/hd_genotype_chip case "$tier" in p0) - d=data/p0/gda_demo36; mkdir -p "$d" - get "$GDA/infinium-global-diversity-array-8-v1-0-D1-demo-data-36.zip" "$d" - get "https://support.illumina.com/content/dam/illumina-support/documents/downloads/productfiles/global-diversity-array/infinium-global-diversity-array-8-v1-0_D1_SampleSheet_Demo_36.csv" "$d" + # GSA v4 PGx demo: 36 samples, real IDATs + BPM/CSV/EGT (primary small fixture) + SUP=https://support.illumina.com/content/dam/illumina-support/documents/documentation/chemistry_documentation/infinium_assays/infinium-gsa-pgx + d=data/p0/gsa_v4_pgx; mkdir -p "$d" + for f in idats_Demo_36.zip GSA-PGx-48v4-0_20079540_E2_bpm.zip GSA-PGx-48v4-0_20079540_E2_csv.zip \ + GSA-PGx-48v4-0_20079540_E2_ClusterFile.zip SampleSheetTemplate_GSAv4PGx_2beadchips.csv; do + get "$SUP/$f" "$d" + done + # GDA v1 manifest/cluster files (note: the "Demo 36" bundle is a GenomeStudio project, not IDATs) + d=data/p0/gda_v1; mkdir -p "$d" get "$GDA/infinium-global-diversity-array-8-v1-0-D1-manifest-file-bpm.zip" "$d" - # CSV manifest and EGT: file names can change; if these fail, download manually from - # https://support.illumina.com/array/array_kits/infinium-global-diversity-array/product-files.html - get "$GDA/infinium-global-diversity-array-8-v1-0-D1-manifest-file-csv.zip" "$d" || echo "!! fetch CSV manifest manually" - get "$GDA/infinium-global-diversity-array-8-v1-0-D1-cluster-file.zip" "$d" || echo "!! fetch EGT manually" - if command -v Rscript >/dev/null; then - Rscript -e 'if(!requireNamespace("BiocManager",quietly=TRUE)) install.packages("BiocManager",repos="https://cloud.r-project.org"); BiocManager::install(c("IlluminaDataTestFiles","hapmap370k"),ask=FALSE,update=FALSE)' - Rscript -e 'cat(system.file("extdata",package="IlluminaDataTestFiles"),"\n")' > data/p0/bioc_fixture_path.txt - else - echo "!! R not found: install IlluminaDataTestFiles + hapmap370k from Bioconductor for parser fixtures" - fi - echo "NOTE: GSA v4 PGx demo (36 samples) + product files: download manually from" - echo " https://support.illumina.com/downloads/infinium-gsa-v4-pgx-demo-data-set-iscan.html" - echo " https://support.illumina.com/array/array_kits/infinium-global-screening-array-v4-pgx/product-files.html" + get "$GDA/infinium-global-diversity-array-8-v1-0-D1-manifest-file-csv.zip" "$d" + get "$GDA/infinium-global-diversity-array-8-v1-0-D1-cluster-file.zip" "$d" + get "https://support.illumina.com/content/dam/illumina-support/documents/downloads/productfiles/global-diversity-array/infinium-global-diversity-array-8-v1-0_D1_SampleSheet_Demo_36.csv" "$d" + # Bioconductor IDAT fixtures (Artistic-2.0, redistributable) — fetched as source tarballs, no R needed + d=data/p0/bioc; mkdir -p "$d" + for pkg in IlluminaDataTestFiles hapmap370k; do + url=$(curl -sL "https://bioconductor.org/packages/release/data/experiment/html/$pkg.html" \ + | grep -oE "\.\./src/contrib/${pkg}_[0-9.]+\.tar\.gz" | head -1 || true) + [[ -n "$url" ]] && get "https://bioconductor.org/packages/release/data/experiment/${url#../}" "$d" \ + || echo "!! could not resolve $pkg tarball" + done + unpack data/p0 ;; p1) d=data/p1/1kg_omni_broad; mkdir -p "$d" diff --git a/scripts/make_oracle_gtcs.sh b/scripts/make_oracle_gtcs.sh new file mode 100755 index 0000000..5417165 --- /dev/null +++ b/scripts/make_oracle_gtcs.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Build bcftools + gtc2vcf plugins (MIT) under refs/tools and generate oracle GTC files from the P0 GSA demo IDATs. +# Usage: bash scripts/make_oracle_gtcs.sh [chip-folder ...] (default: every chip in the demo set) +set -euo pipefail +cd "$(dirname "$0")/.." +V=1.22 +T=refs/tools +B=$T/bcftools-$V +if [[ ! -x $B/bcftools || ! -f $B/plugins/idat2gtc.so ]]; then + [[ -d refs/gtc2vcf ]] || bash scripts/bootstrap_refs.sh + mkdir -p $T + [[ -f $T/bcftools.tar.bz2 ]] || curl -sL -o $T/bcftools.tar.bz2 \ + "https://github.com/samtools/bcftools/releases/download/$V/bcftools-$V.tar.bz2" + tar -xjf $T/bcftools.tar.bz2 -C $T + rm -f $B/plugins/{idat2gtc,gtc2vcf,affy2vcf}.{c,h} + cp refs/gtc2vcf/{idat2gtc,gtc2vcf,affy2vcf}.c refs/gtc2vcf/*.h $B/plugins/ + (cd $B/htslib-$V && ./configure --disable-libcurl --disable-bz2 --disable-lzma --disable-ref-cache >/dev/null) + (cd $B && ./configure --without-gsl >/dev/null && make -j"$(nproc)" >/dev/null) +fi +export BCFTOOLS_PLUGINS=$B/plugins +D=data/p0/gsa_v4_pgx +IDATS=$D/idats_Demo_36/idats_Demo_36 +OUT=data/p0/oracle/gsa_gtc_gtc2vcf +mkdir -p "$OUT" +chips=("$@") +[[ ${#chips[@]} -eq 0 ]] && mapfile -t chips < <(find "$IDATS" -mindepth 1 -maxdepth 1 -type d | sort) +for chip in "${chips[@]}"; do + $B/bcftools +idat2gtc \ + --bpm $D/GSA-PGx-48v4-0_20079540_E2_bpm/GSA-PGx-48v4-0_20079540_E2.bpm \ + --egt $D/GSA-PGx-48v4-0_20079540_E2_ClusterFile/GSA-PGx-48v4-0_20079540_E2_ClusterFile.egt \ + --idats "$chip" --output "$OUT" 2>&1 | grep -E "^(Call rate|Gender|Writing)" || true +done +$B/bcftools +idat2gtc --version 2>&1 | head -2 || true +echo "oracle GTCs in $OUT: $(ls "$OUT"/*.gtc | wc -l)"