From ab92c6ec2037ad26097c40af0c2b11c0cf33735f Mon Sep 17 00:00:00 2001 From: Kemal Yaylali Date: Wed, 30 Sep 2026 08:13:07 +0100 Subject: [PATCH] Plan: drop the assistant-specific appendices; numeric helpers become Appendix A --- .gitignore | 2 +- SIMURGARRAY_MASTER_PLAN.md | 53 ++++----------------- src/simurg/signal/normalization/numerics.py | 2 +- tests/unit/test_normalization.py | 2 +- 4 files changed, 11 insertions(+), 48 deletions(-) diff --git a/.gitignore b/.gitignore index 9943a88..ca6a902 100644 --- a/.gitignore +++ b/.gitignore @@ -20,7 +20,7 @@ target/ results/ # editor lock files .~lock.*# -# Claude Code files (kept local only) +# local assistant files (kept out of the repository) CLAUDE.md CLAUDE.local.md .claude/ diff --git a/SIMURGARRAY_MASTER_PLAN.md b/SIMURGARRAY_MASTER_PLAN.md index 94e80fa..57893f4 100644 --- a/SIMURGARRAY_MASTER_PLAN.md +++ b/SIMURGARRAY_MASTER_PLAN.md @@ -4,7 +4,7 @@ > genotyping BeadChip data: IDAT → normalization → genotype calling → BAF/LRR → QC → VCF/PLINK, with later > modules for copy number, LOH/mosaicism, pharmacogenomics and cluster-file training. > -> This document is **self-contained**: a developer (human or Claude Code) on a fresh machine should be able to +> This document is **self-contained**: a developer on a fresh machine should be able to > build the whole project from it without any other notes. Version 1.0 · 2026-09-27. > > *Name:* the Simurg (Anka / Zümrüdüanka) is the mythical bird of Turkish–Persian legend. CLI and Python package: `simurg`. @@ -36,9 +36,7 @@ 20. Known unknowns & planned experiments 21. Risk register 22. References -- Appendix A — `CLAUDE.md` for the new repository -- Appendix B — first-session prompt for Claude Code -- Appendix C — numeric helper definitions (percentile, trimmean, robustfit, fuzzy memberships) +- Appendix A — numeric helper definitions (percentile, trimmean, robustfit, fuzzy memberships) --- @@ -189,7 +187,6 @@ bash scripts/bootstrap_refs.sh && bash scripts/fetch_data.sh p0 ``` simurg/ ├── pyproject.toml # uv/hatch; package "simurg"; entry point `simurg` -├── CLAUDE.md # Appendix A (local only, git-ignored) ├── SIMURGARRAY_MASTER_PLAN.md # this file ├── THIRD_PARTY_NOTICES.md ├── scripts/ @@ -203,7 +200,7 @@ simurg/ │ │ ├── vcf.py plink.py bedgraph.py finalreport.py │ ├── signal/ │ │ ├── raw_xy.py # IDAT→locus X/Y assembly (§7.1) -│ │ ├── normalization/ # fit.py (§7.2), apply.py (§7.3), numerics.py (Appendix C) +│ │ ├── normalization/ # fit.py (§7.2), apply.py (§7.3), numerics.py (Appendix A) │ │ └── polar.py # θ, R (§7.4) │ ├── genotype/ │ │ ├── gencall.py # EGT-based caller + score (§7.5) @@ -411,7 +408,7 @@ GC Score, Theta, R, X, Y, X Raw, Y Raw, B Allele Freq, Log R Ratio. Used only as ## 7. Core algorithms (exact math) -Notation: per locus i raw intensities (x, y) as float; P_p(v) = Illumina percentile (Appendix C); +Notation: per locus i raw intensities (x, y) as float; P_p(v) = Illumina percentile (Appendix A); trimmeanK(v) = mean of v within [P_{K/2}, P_{100−K/2}]. ### 7.1 Raw X/Y assembly [OSS: gtc2vcf `fill_array` idat2gtc.c:3711-3745; LIT: Steemers 2006] @@ -445,13 +442,13 @@ In bit-exact mode, missing loci (0,0) **stay in** the bin as zeros (vendor quirk candidates; y-sweep analogously → BB candidates. (For ≥2000 points the vendor uses a binned approximate nearest-neighbour search `findClosestSitesToPointsAlongAxis` — reproduce it in bit-exact mode; see gtc2vcf `nearest_neighbor.c`.) - - robust fits (bisquare, Appendix C): y = b_a + m_a·x on AA; x = b_b + m_b·y on BB; + - robust fits (bisquare, Appendix A): y = b_a + m_a·x on AA; x = b_b + m_b·y on BB; - intersection o_x = (b_b + m_b·b_a)/(1 − m_b·m_a), o_y = b_a + m_a·o_x; - offset = (o_x + mx, o_y + my); subtract. 3. **nn12 / rr12** diagnostics: nn12 = fraction with x < 0.2·P_95(x) and y < 0.2·P_95(y); rr12 = mean(x+y of those) / mean(x+y of all). 4. **Rotation** (if n < 192: θ = 0): new x-sweep over [min x, max x], union with AA; robust fit through origin - y = m·x (`robustfit0`, see quirk in Appendix C); θ = atan(m); rotate x′ = cosθ·x + sinθ·y, y′ = −sinθ·x + cosθ·y. + y = m·x (`robustfit0`, see quirk in Appendix A); θ = atan(m); rotate x′ = cosθ·x + sinθ·y, y′ = −sinθ·x + cosθ·y. 5. **Shear** (if n < 192: shear = 0): y-sweep over [0, max y], union with BB; fit x = m·y through origin; shear = m; x ← x − shear·y. 6. **Scale**: @@ -870,7 +867,7 @@ https://github.com/gjun/verifyIDintensity # GPL-3 — oracle only, do not | **P0 Setup** (≈1 wk) | repo skeleton, uv project, CI (ruff, mypy, pytest), fetch scripts, refs bootstrap, `THIRD_PARTY_NOTICES.md` | `pytest` green on empty suite; P0 data fetched with checksums | | **P1 Parsers** (≈2–3 wk) | `binutil`, IDAT, BPM, CSV, EGT, GTC read + GTC write, sample sheet, `simurg inspect` | all fields equal to BeadArrayFiles/illuminaio on P0 + Omni data; GTC read→write round-trip byte-identical; fuzz tests never crash (clean errors) | | **P2 Data model** (≈1 wk) | Arrow schemas, raw X/Y assembly, norm-bin rule | raw X/Y **exactly** equal to GTC TOC 1000/1001 of 1000G Illumina GTCs for 100% loci × ≥50 samples | -| **P3 Normalization** (≈3–4 wk) | numerics (Appendix C), fit v1.2.0 (+1.1.2), apply, polar | transforms vs Illumina GTC TOC 400 within 2 ULP for ≥99.9% bins; normalized X/Y within 1e-5 | +| **P3 Normalization** (≈3–4 wk) | numerics (Appendix A), fit v1.2.0 (+1.1.2), apply, polar | transforms vs Illumina GTC TOC 400 within 2 ULP for ≥99.9% bins; normalized X/Y within 1e-5 | | **P4 Genotyping** (≈3 wk) | GenCall-equivalent caller, score, BAF/LRR, LogRDev, GC10/50, sex, percentiles, GTC v5 writer | vs Illumina GTCs: genotypes ≥99.999% identical, scores ≤1e-5, sample stats ≤1e-6, sex 100%; md5-identical GTC vs gtc2vcf output (timestamps masked) | | **P5 QC** (≈2 wk) | control metrics, sample/locus summaries, contamination, relatedness, HTML report | control metrics equal reference formulas on synthetic controls; 1000G sex 100% vs pedigree; replicate/duplicate detection 100% on demo | | **P6 Outputs** (≈2 wk) | harmonization + audit, VCF/BCF, PLINK, bedGraph, final-report export | `bcftools` validates; vs gtc2vcf/GTCtoVCF VCF: 0 unexplained genotype differences; strand audit complete | @@ -956,41 +953,7 @@ Haraksingh 2017 BMC Genomics 18:321 (GSE96909). --- -## Appendix A — `CLAUDE.md` for the new repository - -```markdown -# SimurgArray -Personal research toolkit for Illumina Infinium genotyping arrays (IDAT→GTC→VCF, QC, CN, PGx). -The full spec and roadmap is SIMURGARRAY_MASTER_PLAN.md — read the relevant section before changing code. - -## Rules -- Work phase by phase (§18). Don't start a phase until the previous gate passes. -- Science modules (signal/, genotype/, cn/, qc/, pgx/) are pure functions; no I/O inside them. -- Every algorithm cites its source (§7 tags) in its docstring. -- MIT/BSD/Apache/Unlicense code may be ported with attribution in THIRD_PARTY_NOTICES.md. - GPL tools (plink2, illuminaio, verifyIDintensity) = subprocess oracles only; never copy their code. -- Never commit vendor data (BPM/CSV/EGT/IDAT) — only tests/fixtures/ (Artistic-2.0 Bioconductor files, synthetic data). -- Never hide oracle discrepancies; record them in docs/known_unknowns.md. -- Default mode is bit-exact vs Illumina-produced GTCs; keep a clean-math mode too. - -## Commands -- uv sync ; uv run pytest -q ; uv run ruff check . ; uv run mypy src -- bash scripts/fetch_data.sh p0|p1|p2 ; bash scripts/bootstrap_refs.sh -``` - -## Appendix B — first-session prompt for Claude Code (on the new machine) - -> Read `SIMURGARRAY_MASTER_PLAN.md` fully. We are starting **Phase P0 (Setup)** then **P1 (Parsers)**. -> 1. Create the repository skeleton of §4.2 with a uv-managed `pyproject.toml` (Python 3.12), ruff, mypy, pytest, -> and a GitHub Actions CI workflow. -> 2. Write `scripts/bootstrap_refs.sh` (shallow clones of §17 repos into `refs/`) and `scripts/fetch_data.sh` -> (tiers p0/p1/p2, sha256 recording, resumable downloads with aria2c/wget). -> 3. Run bootstrap + fetch p0. -> 4. Implement `formats/binutil.py` and `formats/idat.py` per §6.1 with tests (Bioconductor fixtures, malformed-file -> tests, property tests), then `simurg inspect`. -> Stop after the IDAT reader passes the P1 criteria for IDAT and report results. - -## Appendix C — numeric helper definitions (bit-exact mode) +## Appendix A — numeric helper definitions (bit-exact mode) **Illumina percentile** (used for normalization, GC10/GC50): sorted v of length n, target p ∈ [0,100]: ``` diff --git a/src/simurg/signal/normalization/numerics.py b/src/simurg/signal/normalization/numerics.py index 12afe93..64e005b 100644 --- a/src/simurg/signal/normalization/numerics.py +++ b/src/simurg/signal/normalization/numerics.py @@ -1,4 +1,4 @@ -"""Numerics with the exact rounding behaviour of the reference normalization (plan Appendix C). +"""Numerics with the exact rounding behaviour of the reference normalization (plan Appendix A). Ported from gtc2vcf ``idat2gtc.c`` (MIT, © Giulio Genovese), which reproduces the vendor's MATLAB-derived routines. Rules that matter for bit-exactness: diff --git a/tests/unit/test_normalization.py b/tests/unit/test_normalization.py index 9dcc4ce..8286ada 100644 --- a/tests/unit/test_normalization.py +++ b/tests/unit/test_normalization.py @@ -1,4 +1,4 @@ -"""Phase P3: normalization numerics, fit and apply (plan §7.2–§7.4, Appendix C).""" +"""Phase P3: normalization numerics, fit and apply (plan §7.2–§7.4, Appendix A).""" from __future__ import annotations