diff --git a/CHANGELOG.md b/CHANGELOG.md
index 182ec3e..38bafa8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,10 @@
## Unreleased
+**Repository tidy-up.** Removed the Phase-0 research notes (`docs/research/`), the P9 benchmark scripts (`bench/`,
+`scripts/cloud/tasks/p9*.sh`) and a vendor probe-name list that should never have been committed (also purged from
+the git history).
+
**Branding.** A visual identity (`branding/`, guide in `branding/README.md`): a dotted simurg mark in detailed, simple
and micro drawings, wordmark and lockups, icons, README banner, social preview and featured-image templates, with
contrast and colour-vision checks. HTML reports (QC report, compare dashboard, methylation QC) get a favicon and a
diff --git a/README.md b/README.md
index 179ceaf..64d1a4b 100644
--- a/README.md
+++ b/README.md
@@ -8,7 +8,7 @@ methylation control QC).
- **SIMURGARRAY_MASTER_PLAN.md** — architecture, algorithms, roadmap
- **SIMURGARRAY_FUNCTIONAL_SPEC.md** — 416 itemised requirements (CLI, inputs, outputs, QC, CN, PGx, methylation, edge cases)
-- `docs/validation/` — per-phase validation reports · `docs/known_unknowns.md` — findings log · `docs/research/` — background
+- `docs/validation/` — per-phase validation reports · `docs/known_unknowns.md` — findings log
- `branding/` — logo, icons, templates and the brand guide · `blog/` — five posts about the project
## Status
diff --git a/SIMURGARRAY_MASTER_PLAN.md b/SIMURGARRAY_MASTER_PLAN.md
index 2e0c255..57893f4 100644
--- a/SIMURGARRAY_MASTER_PLAN.md
+++ b/SIMURGARRAY_MASTER_PLAN.md
@@ -218,7 +218,6 @@ simurg/
├── tests/
│ ├── unit/ property/ integration/ regression/ oracle/ perf/
│ └── fixtures/ # ONLY redistributable data (Bioconductor Artistic-2.0 files, synthetic)
-├── bench/
├── docs/ # user guide, algorithm notes, validation reports
├── refs/ # (git-ignored) cloned third-party repos
└── data/ # (git-ignored) downloaded datasets
diff --git a/bench/profile_genotype.py b/bench/profile_genotype.py
deleted file mode 100644
index 0c522e7..0000000
--- a/bench/profile_genotype.py
+++ /dev/null
@@ -1,83 +0,0 @@
-#!/usr/bin/env python3
-"""Per-stage timing of `simurg genotype` for one sample (plan §19 measurements).
-
-python bench/profile_genotype.py BPM EGT GRN_IDAT [--repeat N]
-"""
-
-from __future__ import annotations
-
-import argparse
-import datetime
-import time
-from dataclasses import replace
-from pathlib import Path
-
-import numpy as np
-
-from simurg.formats.bpm import read_bpm
-from simurg.formats.egt import read_egt
-from simurg.formats.gtc import write_gtc
-from simurg.formats.idat import read_idat
-from simurg.genotype.baf_lrr import baf_lrr, logr_dev
-from simurg.genotype.clusters import align_clusters
-from simurg.genotype.gencall import call_genotypes
-from simurg.genotype.sample import genotype_sample, manifest_context, params_for_preset
-from simurg.genotype.sex import estimate_sex
-from simurg.signal.normalization.apply import normalize, polar
-from simurg.signal.normalization.fit import fit_transforms
-from simurg.signal.raw_xy import LocusMapCache, assemble_raw_xy
-
-
-def main() -> None:
- ap = argparse.ArgumentParser()
- ap.add_argument("bpm")
- ap.add_argument("egt")
- ap.add_argument("grn")
- ap.add_argument("--repeat", type=int, default=3)
- ap.add_argument("--out", default="/tmp/profile.gtc")
- a = ap.parse_args()
- m = read_bpm(a.bpm)
- ctx = manifest_context(m, align_clusters(read_egt(a.egt), m))
- lite = m.lite()
- ctx = replace(ctx, manifest=lite, cache=LocusMapCache(lite))
- params = params_for_preset("current")
- red_path = a.grn.replace("_Grn", "_Red")
- t: dict[str, list[float]] = {}
-
- def lap(name: str, t0: float) -> float:
- now = time.perf_counter()
- t.setdefault(name, []).append(now - t0)
- return now
-
- for _ in range(a.repeat):
- t0 = time.perf_counter()
- grn, red = read_idat(a.grn), read_idat(red_path)
- t0 = lap("read_idat x2", t0)
- raw = assemble_raw_xy(grn, red, lite, cache=ctx.cache)
- t0 = lap("assemble_raw_xy", t0)
- xf = fit_transforms(raw.raw_x, raw.raw_y, ctx.norm_bins, params.norm_version)
- t0 = lap("fit_transforms", t0)
- nx, ny = normalize(raw.raw_x, raw.raw_y, ctx.lookup, xf)
- th, r = polar(nx, ny)
- t0 = lap("normalize+polar", t0)
- z = (raw.raw_x == 0) & (raw.raw_y == 0)
- thc = np.where(z, np.float32(np.nan), th).astype(np.float32)
- rc = np.where(z, np.float32(np.nan), r).astype(np.float32)
- calls = call_genotypes(thc, rc, z, ctx.clusters, lite.intensity_only, params.cutoff)
- t0 = lap("call_genotypes", t0)
- estimate_sex(calls.genotypes, r, ctx.clusters.gentrain, ctx.clusters.present, ctx.sex_classes, params.sex)
- t0 = lap("estimate_sex", t0)
- _, lrr = baf_lrr(thc, rc, z, ctx.clusters, lite.intensity_only)
- logr_dev(lrr, ctx.autosomal, lite.intensity_only, ctx.clusters.present)
- t0 = lap("baf_lrr+logr_dev", t0)
- g = genotype_sample(grn, red, ctx, params, datetime.datetime(2026, 1, 1))
- t0 = lap("genotype_sample (whole)", t0)
- write_gtc(g, Path(a.out))
- t0 = lap("write_gtc", t0)
- print(f"{'stage':28s} {'median s':>9s}")
- for k, v in t.items():
- print(f"{k:28s} {sorted(v)[len(v) // 2]:9.3f}")
-
-
-if __name__ == "__main__":
- main()
diff --git a/bench/scaling.py b/bench/scaling.py
deleted file mode 100644
index 9f97948..0000000
--- a/bench/scaling.py
+++ /dev/null
@@ -1,114 +0,0 @@
-#!/usr/bin/env python3
-"""Thread scaling of ``simurg genotype`` (plan §18 P9 gate: ≥ 20 samples/min/core on GSA-size arrays, linear
-scaling to 32 threads).
-
-For every worker count the same N samples are genotyped (``--no-cohort-qc``); the per-sample phase is timed from
-the run log ("genotyping … sample(s)" → "done"), so manifest loading is reported separately and does not blur
-the scaling. Writes a JSON summary.
-
- python bench/scaling.py --bpm M.bpm --egt C.egt --idats DIR --out OUT --threads 1,2,4,8 [--samples N]
-"""
-
-from __future__ import annotations
-
-import argparse
-import datetime as dt
-import json
-import os
-import shutil
-import subprocess
-import sys
-import time
-from pathlib import Path
-
-
-def phase_seconds(log: Path) -> tuple[float, float]:
- """(manifest+setup seconds, per-sample phase seconds) from a genotype run log."""
- ts: dict[str, dt.datetime] = {}
- for line in log.read_text().splitlines():
- rec = json.loads(line)
- msg = rec.get("msg") or ""
- t = dt.datetime.fromisoformat(rec["ts"].replace("Z", "+00:00"))
- ts.setdefault("first", t)
- if msg.startswith("genotyping "):
- ts["start"] = t
- elif msg.startswith("done:"):
- ts["done"] = t
- return (ts["start"] - ts["first"]).total_seconds(), (ts["done"] - ts["start"]).total_seconds()
-
-
-def subset(idats: Path, n: int, dest: Path) -> Path:
- grn = sorted(p for p in idats.rglob("*_Grn.idat*"))
- if n <= 0 or n >= len(grn):
- return idats
- dest.mkdir(parents=True, exist_ok=True)
- for g in grn[:n]:
- d = dest / g.parent.name
- d.mkdir(exist_ok=True)
- for p in (g, g.with_name(g.name.replace("_Grn.", "_Red."))):
- link = d / p.name
- if not link.exists():
- link.symlink_to(p.resolve())
- return dest
-
-
-def main() -> int:
- ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
- ap.add_argument("--bpm", required=True)
- ap.add_argument("--egt", required=True)
- ap.add_argument("--idats", required=True, type=Path)
- ap.add_argument("--out", required=True, type=Path)
- ap.add_argument("--threads", default="1,2,4")
- ap.add_argument("--samples", type=int, default=0, help="use the first N samples (0 = all)")
- ap.add_argument("--simurg", default=str(Path(sys.executable).with_name("simurg")))
- ap.add_argument("--pure-python", action="store_true", help="run with SIMURG_PURE_PYTHON=1")
- ap.add_argument("--work", type=Path, help="folder for the GTC outputs (e.g. a tmpfs); default: --out")
- ap.add_argument("--clean", action="store_true", help="delete each run's GTCs after timing it")
- a = ap.parse_args()
- a.out.mkdir(parents=True, exist_ok=True)
- idats = subset(a.idats, a.samples, a.out / "idats")
- n = len(list(idats.rglob("*_Grn.idat*")))
- env = {**os.environ, "SIMURG_PURE_PYTHON": "1" if a.pure_python else "0"}
- runs = []
- for t in (int(x) for x in a.threads.split(",")):
- out = (a.work or a.out) / f"t{t}"
- t0 = time.time()
- cmd = [a.simurg, "genotype", "--bpm", a.bpm, "--egt", a.egt, "--idats", str(idats), "--out", str(out),
- "--threads", str(t), "--no-cohort-qc", "--autocall-date", "1/1/2026 12:00:00 AM",
- "--quiet", "--force"] # fmt: skip
- rc = subprocess.run(cmd, env=env).returncode
- wall = time.time() - t0
- setup, phase = phase_seconds(out / "logs" / "genotype.log")
- if a.clean:
- shutil.rmtree(out / "samples", ignore_errors=True)
- runs.append(
- {
- "threads": t,
- "rc": rc,
- "wall_s": round(wall, 1),
- "setup_s": round(setup, 1),
- "per_sample_phase_s": round(phase, 2),
- "samples_per_min": round(60 * n / phase, 1) if phase else None,
- "samples_per_min_per_core": round(60 * n / phase / t, 1) if phase else None,
- }
- )
- print(json.dumps(runs[-1]), flush=True)
- base = runs[0]["per_sample_phase_s"] * runs[0]["threads"]
- for r in runs:
- r["speedup_vs_first"] = round(base / r["per_sample_phase_s"], 2) if r["per_sample_phase_s"] else None
- r["efficiency"] = round(r["speedup_vs_first"] / r["threads"], 3) if r["speedup_vs_first"] else None
- from simurg._core import backend
-
- summary = {
- "samples": n,
- "backend": "python" if a.pure_python else backend(),
- "cpu_count": os.cpu_count(),
- "machine": os.uname().machine,
- "runs": runs,
- }
- (a.out / "scaling.json").write_text(json.dumps(summary, indent=1))
- return 0
-
-
-if __name__ == "__main__":
- raise SystemExit(main())
diff --git a/docs/research/00_phase0_report.md b/docs/research/00_phase0_report.md
deleted file mode 100644
index e318a4c..0000000
--- a/docs/research/00_phase0_report.md
+++ /dev/null
@@ -1,282 +0,0 @@
-# SimurgArray — Phase 0 Research Report: can the reference array-analysis product be independently reimplemented?
-
-Status: PHASE 0 (research only, no implementation code) · Research date: 2026-09-27 · Sanitized and brought into the
-repository afterwards (see `README.md` in this folder).
-Project: **SimurgArray**, a personal, non-commercial research toolkit. Canonical repository:
-https://gitea.yaylali.uk/kemal/SimurgArray. The roadmap and ground rules that came out of this study are in
-`SIMURGARRAY_MASTER_PLAN.md` (plan) and `SIMURGARRAY_FUNCTIONAL_SPEC.md` (spec).
-
-Evidence base: `R1_reference_product.md` (product dossier), `R2_open_source_ecosystem.md` (GitHub ecosystem,
-source-level math with file:line refs), `R3_science_formats_algorithms.md` (formats + statistics), `R4_datasets.md`
-(public data), and the documentation walk-through files `A_cli_inputs.md` … `E_release_history.md`. Every claim below
-carries a tag; the URL / file:line sits in the referenced file section.
-
-Tags: **[DOC]** public vendor documentation · **[LIT]** scientific literature/patents · **[OSS]** open-source
-implementation · **[OBS]** experimentally observed (none yet) · **[INF]** inferred · **[HYP]** hypothesis.
-
-Naming: "the reference product" = the vendor's current Infinium array secondary-analysis software; `refcli` = its CLI
-binary; "ACLI" = the vendor's earlier array CLI (its predecessor); "the vendor cloud platform" = the vendor's hosted
-analysis and data platform.
-
----
-
-## Executive verdict
-
-| Question | Answer |
-|---|---|
-| Is a reference-compatible *genotyping* pipeline (IDAT → GTC → VCF + QC) feasible? | **Yes, with high confidence.** Formats are publicly specified [DOC/OSS]; the math of normalization, GenCall, BAF/LRR is reproduced in MIT-licensed open source that claims md5-identical GTCs vs ACLI [OSS: gtc2vcf]. The reference product itself does **not** cluster — it consumes Illumina EGT files [DOC]. |
-| Is numerical (bit-level) GTC compatibility achievable? | **Probably**, pending experiment. Whether the reference product's `genotype call` equals ACLI 2.1 output is **unverified [HYP]**. The practical target is the vendor-produced GTCs already published with 1000 Genomes (plan §16.2). |
-| Can PGx CNV / star alleles / cyto CNV-LOH be cloned? | **Not cloned — re-engineered as compatible alternatives.** Algorithms and `.dat` model formats are undocumented [DOC gap]. Output *schemas* are documented and can be matched; results can only be matched *semantically* (same calls), not numerically. |
-| Hardware acceleration? | The reference product is **CPU-only**, explicitly "no FPGA" [DOC]. Workload is I/O- and memory-bound linear algebra per sample; FPGA is **not justified** [INF]. GPU is optional only for large cohort re-clustering. |
-| Main risk | At study time: **legal/provenance**, not science — (1) the vendor EULA family forbids deriving "underlying ideas, algorithms" from the Software [DOC]; (2) the best open reference (gtc2vcf) states it ports Illumina's GenCall C# code [OSS]. **Resolved by the decisions below (plan §2):** no vendor binary is run or inspected, and MIT code is reused openly with attribution. |
-
----
-
-## 1. What the reference product actually is
-
-- The vendor's analysis suite for **Infinium BeadChip** data; CLI binary **`refcli`**, a self-contained **.NET 9**
- app for linux-x64 / win10-x64; no ARM build [DOC; INF from RIDs]. Recommended 8 cores / 32 GB RAM [DOC].
-- Lineage: AutoConvert → IAAP-CLI → ACLI (the vendor's earlier array CLI, retired 2024-12) → the reference product
- (v1.0 2023-12 … v1.4 2026-05) [DOC]. Stack traces in release notes show `ArrayAnalysis.Core.*` namespaces
- [DOC] ⇒ genotyping core is very likely inherited from ACLI [INF].
-- Deployed locally (free, offline except star-allele) and on the vendor cloud platform [DOC].
-- **Not** a clustering tool: custom EGT generation is still done in GenomeStudio [DOC].
-
-## 2. Workflows
-
-| Workflow | Command | Local? | Algorithm disclosed? |
-|---|---|---|---|
-| Control-probe QC | `qc call --array-type genotyping\|methylation` | yes | **Formulas fully tabulated** [DOC] |
-| QC report | `qc report` (HTML + xlsx/csv, thresholds callRate 0.98, logRDev 0.20) | yes | yes (thresholding) [DOC] |
-| Genotyping | `genotype call` (IDAT+BPM+EGT → GTC v5, cutoff 0.15) | yes | qualitatively [DOC]; quantitatively only via patents + OSS |
-| VCF export | `genotype gtc-to-vcf` (→ `*.snv.vcf.gz`, `GT:GS:BAF:LRR`) | yes | yes — mirrors Illumina/GTCtoVCF (Apache-2.0) [DOC/OSS] |
-| BedGraph | `genotype gtc-to-bedgraph` | yes | trivial |
-| PGx CN | `pgx copy-number train/call` (≥24 samples batch, `.dat` model, 7 genes/10 regions) | yes | **no** [DOC gap] |
-| PGx star alleles | `pgx star-allele call/annotate` (PharmVar/CPIC/DPWG) | yes, **paid, license-server** | **no** |
-| Cytogenetics CNV/LOH/mosaic | `cyto call/annotate` (`cyto_model.dat`, ISCN output) | yes | **no** (outputs suggest HMM/segmentation) [HYP] |
-| Methylation full QC (beta, M, detP) | cloud only | no | method named: Noob + nonlinear dye bias + negative-control detP [DOC] |
-| Final Report / Locus Summary | cloud only | no | columns documented [DOC] |
-
-## 3. Inputs consumed [DOC]
-
-IDAT pairs (`Barcode_RxxCxx_Grn/Red.idat`), BPM manifest, CSV manifest (SourceSeq, RefStrand), EGT cluster file,
-genome FASTA (+fai, contigs `1` not `chr1`), sample sheet (GenomeStudio-compatible), PGx CN model `.dat` + `.msk`,
-cyto model `.dat`, PGx database `.zip`, cyto annotation `.zip`, QC YAML.
-
-## 4. Outputs produced [DOC]
-
-Per sample: GTC v5; `*.snv.vcf.gz`+`.tbi` (VCFv4.1, FILTER always PASS, ploidy-adjusted GT, **BAF/LRR oriented to
-REF=A/ALT=B → may be 1−GenomeStudio BAF**); LRR/BAF bedgraphs; PGx `*.cnv.vcf.gz` (`CN:NR`, CN≤5, QUAL≤60);
-cyto `*.cnv.vcf.gz` (+ JSON ISCN annotation); star-allele JSON.
-Per batch: `gt_sample_summary.csv/json` (Autosomal Call Rate, Call Rate, LogRDev, Sex Estimate, TGA control),
-`controls.raw_metrics.csv`, `controls.qc_metrics.csv`, QC HTML/xlsx, `cn_batch_summary.csv`, five PGx CSVs.
-
-## 5. What is publicly documented (sufficient to implement)
-
-| Area | Source |
-|---|---|
-| IDAT v3 binary layout (field codes, TOC) | illuminaio/BeadArrayFiles/gtc2vcf docs & code [OSS] |
-| BPM, EGT, GTC v3–v5 layouts, incl. 52-byte normalization transform record | BeadArrayFiles + `GTC_File_Format_v5.pdf` [DOC/OSS] |
-| Application of normalization: offset→rotate→shear→scale→clamp; θ=(2/π)atan2(y,x), R=x+y | BeadArrayFiles `GenotypeCalls.py` [DOC/OSS] |
-| Normalization bins `(norm_id + 100·assay_type) mod 256` | BeadArrayFiles `BeadPoolManifest.py:118-130` [OSS] |
-| BAF / LRR from cluster centres (piecewise-linear), LRR=log2(R/R_exp) | Peiffer 2006 Genome Res [LIT] |
-| TOP/BOT, PLUS/MINUS, indel `[I/D]` strand rules | Illumina strand tech note [DOC] |
-| GenCall cutoff 0.15, GC10/GC50 definitions | tech notes [DOC] |
-| Control-probe QC formulas (all ratios, offset 3000) | reference-product output docs `[ref-docs:output-files]` [DOC] |
-| VCF conversion incl. multi-allelic merging, indel resolution | Illumina/GTCtoVCF + reference-product docs [DOC/OSS] |
-| Normalization fit & GenTrain concepts | expired patents US7035740 (2024-05), US7467117 (2024-03); GenTrain 2/3 tech notes [LIT/DOC] |
-| cnvPartition model | Illumina CNV tech note [DOC] |
-
-## 6. Implementable from published science (independent of vendor code)
-
-Normalization fitting (patent + Teo 2007 description of the 400-homozygote approach) · Bayesian genotype
-calling in θ/R given EGT · BAF/LRR/LogRDev · sex inference (X het + Y intensity) · CNV/LOH/mosaic
-(PennCNV HMM [public domain], MoChA [MIT], cnvPartition tech note) · contamination (BAFregress, VerifyIDintensity
-model) · relatedness (KING-robust math) · methylation Noob/dye-bias/detP (Triche 2013, sesame) · star-allele
-matching (PharmCAT/PyPGx logic) · **own** cluster training (GMM / t-mixture à la Illuminus/CRLMM).
-
-## 7. Behaviours that cannot be read from documentation
-
-At study time these were listed as "requires black-box characterization of the vendor binary". Per decision D2
-below, the vendor binary is **not** run; each item is instead resolved from published vendor-produced GTCs, open
-oracles and synthetic experiments, or recorded as a known unknown in `docs/known_unknowns.md`.
-
-1. Is the reference product's `genotype call` GTC ≡ ACLI GTC (md5 with timestamps masked)? — *not testable without
- the binary; bit-exact target is the published vendor-produced GTCs instead.*
-2. Exact GenCall score mapping & edge cases (low bead counts, zero intensities, Omni5 normID overflow).
-3. **Sex Estimate**, **Autosomal Call Rate** locus set, **LogRDev** locus set/trimming.
-4. VCF details: REF/ALT choice for indels, ploidy rules on X/Y/MT, `GS` Phred scaling, BAF orientation flip.
-5. PGx CN: CN calls vs regions, NR distribution, QUAL calibration (targets for our *own* model).
-6. Cyto CNV/LOH: event boundaries, MF estimates, filters (targets for our own caller).
-7. Star alleles: diplotype/score behaviour on known samples.
-
-## 8. Hardware-specific parts
-
-None. CPU-only by vendor statement [DOC]. The product *brand* is FPGA-associated for NGS, but the array product is not
-[DOC]. ⇒ No FPGA work in scope; revisit only if profiling shows a hotspot (unlikely: per-sample work is
-~O(#probes) ≈ 1–5 M floats).
-
-## 9. Open-source projects (see R2 table for ~45 entries)
-
-| Project | License | Role in SimurgArray |
-|---|---|---|
-| Illumina/BeadArrayFiles | BSD-2 | format spec + oracle (GTC/BPM/EGT readers); may be ported with attribution |
-| Illumina/GTCtoVCF | Apache-2.0 (archived) | VCF conversion reference + oracle; may be ported with attribution |
-| freeseek/gtc2vcf (idat2gtc, gtc2vcf, BAFregress) | MIT | **test oracle** and algorithm reference; may be ported with attribution |
-| broadinstitute/picard `picard.arrays` | MIT | second independent format reader; QC metric definitions (WARP) |
-| freeseek/mocha | MIT | mosaic CNV reference |
-| WGLab/PennCNV | Unlicense | CNV HMM; PFB/GC model |
-| PharmGKB/PharmCAT | MPL-2.0 | star-allele oracle (subprocess) |
-| sbslee/pypgx | MIT | array-VCF star allele reference |
-| bcftools | MIT | VCF I/O oracle, `+fixref` |
-| plink2, verifyIDintensity, GenotypeHarmonizer, illuminaio | GPL | **subprocess oracles only, never linked or copied** |
-| zCall, argyle, Aldy, Cyrius | none / NC / PolyForm | **excluded** (ideas from their papers only) |
-
-Gap: **no open-source GenTrain / EGT writer exists** [OSS survey].
-
-## 10. Realistically cloneable (target: numerical compatibility)
-
-IDAT/BPM/EGT/CSV readers · GTC v5 writer · normalization · GenCall calling+score · BAF/LRR/LogRDev · sex estimate ·
-call rates · control-probe QC · GTC→VCF · bedgraph · `gt_sample_summary` · QC report table.
-
-## 11. Implement as compatible alternatives (target: semantic compatibility, same output schema)
-
-Cluster-file generation (own "ArrayTrain", explicitly *not* GenTrain) · PGx CNV (own GC-corrected batch model) ·
-star alleles (PharmVar-based solver, PharmCAT/PyPGx as oracles) · cyto CNV/LOH/mosaic (HMM on LRR+BAF) ·
-methylation QC (Noob etc.) · contamination/relatedness (not in the reference product; value-add).
-
-## 12. Proposed architecture (as sketched in Phase 0; the binding layout is plan §4)
-
-```
-simurg/
- core/ canonical data model (Arrow schemas: Probe, Locus, SampleIntensity, Call, Provenance)
- formats/ idat/ bpm/ csv_manifest/ egt/ gtc/ samplesheet/ vcf/ plink/ bedgraph/
- signal/ raw_xy (probe→locus assembly, bead-count rules), normalization/ (fit, apply), polar
- genotype/ gencall/ (EGT-based caller), train/ (ArrayTrain: own clustering), rescue/ (optional)
- cn/ baf_lrr/, gc_wave/, cyto_hmm/, pgx_cn/
- pgx/ star_allele/ (PharmVar db builder, solver), annotate/
- qc/ controls/, sample/ (call rate, LogRDev, sex), contamination/, relatedness/, report/
- harmonize/ strand (TOP/BOT/PLUS), REF/ALT, build liftover hooks, audit log of every flip
- pipeline/ DAG runner, resumable, per-step provenance (hashes, versions, params, seed)
- cli/ `simurg qc|genotype|gtc-to-vcf|cn|cyto|pgx|pipeline` (reference-like verbs)
- compat/ reference-CLI option aliases + output-name compatibility layer
- difftest/ oracle adapters (published vendor GTCs, gtc2vcf, BeadArrayFiles, GTCtoVCF, PennCNV, PharmCAT)
- synth/ synthetic IDAT/intensity generator (ground-truth genotypes, CN, contamination, dye bias)
- bench/ tests/ docs/
-```
-Principles: science modules are pure functions over Arrow/NumPy arrays; no I/O inside algorithms; every
-transform returns data **plus** an audit record. Reference implementation in Python (NumPy/SciPy, pyarrow,
-pysam/cyvcf2); performance tier later in **Rust** (PyO3) for IDAT parsing + normalization + calling — the
-hot, embarrassingly-parallel per-sample path.
-
-## 13. Phased plan (as proposed in Phase 0; the binding roadmap and gates are plan §18)
-
-| Phase | Deliverable | Exit gate (measurable) |
-|---|---|---|
-| **0 Research** (this) | research docs, decisions D1–D3, dataset fetch scripts | decisions taken (below); datasets checksummed |
-| **1 Parsers** | IDAT, BPM, EGT, CSV, GTC read/write, sample sheet | 100% field equality vs BeadArrayFiles + illuminaio on ≥254-series GEO sweep + Omni2141; GTC round-trip byte-identical |
-| **2 Canonical model** | Arrow schemas, locus assembly (Inf I/II, bead count ≥2 rule) | raw X/Y identical to GTC RAW_X/RAW_Y (TOC 1000/1001) for 1000G Broad GTCs |
-| **3 Normalization** | fit + apply per bin | 13-float transforms vs Illumina GTC TOC 400: |Δ| ≤1e-5 rel on ≥99.9% bins; normalized X/Y match |
-| **4 Genotyping (EGT-based)** | GenCall-compatible caller + score, BAF/LRR, sex | vs Illumina GTCs: genotype concordance ≥99.99%, GC score |Δ|≤1e-4 |
-| **4b ArrayTrain** | own clustering → EGT | vs Illumina EGT calls on Omni2141: ≥99.5% concordance; vs 30x WGS non-ref discordance ≤ GenCall's |
-| **5 QC** | controls, sample summary, QC report | control metrics equal the documented formulas; Sex Estimate 100% on 1000G known sex |
-| **6 Outputs** | VCF (reference-compatible layout), PLINK, bedgraph, JSON/CSV summaries | `bcftools` field-level diff vs gtc2vcf/GTCtoVCF: 0 unexplained discrepancies |
-| **7 Differential testing** | automated EXACT/NEAR/DISCREPANCY/UNKNOWN dashboard | nightly on demo + Omni subsets |
-| **8 Validation** | science validation vs WGS truth, trios, replicates | concordance, Mendelian error, reproducibility reported |
-| **8b CN/Cyto/PGx** | own CN HMM, PGx CN, star alleles | CNV vs the vendor's public 1000 Genomes WGS reanalysis (AWS Open Data) & PennCNV; CYP2D6 CN vs WGS truth; star alleles vs PharmCAT-on-WGS |
-| **9 Optimization** | Rust core, mmap, multithreading | throughput at least on par with the vendor's documented figure (35 s/sample VCF claim) on a comparable x86 box |
-| **10 Packaging** | conda/pip/Docker, docs, CI | reproducible builds, provenance manifest per run |
-
-## 14. Validation strategy
-
-Three independent axes, never conflated:
-1. **Format compatibility** — parses/writes files other tools accept (BeadArrayFiles, bcftools, plink2).
-2. **Numerical compatibility** — field-level diff vs oracles with explicit tolerances (float32 ULP-aware).
- Oracles: (a) Illumina GTCs **already published** in the 1000G Broad set (the vendor ground truth, no licensed
- binary needed), (b) GenomeStudio FinalReport in the 1000G Sanger set, (c) gtc2vcf (MIT), (d) BeadArrayFiles /
- GTCtoVCF / Picard. Outputs of the reference product's binary are **not** an oracle (decision D2).
-3. **Scientific validity** — vs truth independent of the vendor: NYGC 30x / GIAB v4.2.1 (concordance, NRD,
- sensitivity/precision per genotype class & MAF bin), 337 Omni trios + GDA-demo trios (Mendelian error),
- GSE165845 triplicates (reproducibility), the vendor's public 1000 Genomes WGS reanalysis (AWS Open Data) + GIAB SV
- (CNV), synthetic data (exact truth).
-Reporting: EXACT / NEAR (within tolerance) / DISCREPANCY / UNKNOWN per field, per locus class (Inf I/II,
-indel, X/Y/MT, PAR, low-intensity).
-
-## 15. Datasets required (all verified public, see R4)
-
-| Priority | Dataset | Use |
-|---|---|---|
-| P0 | GDA v1 Demo 36 (465 MB; 3 CEU trios + replicates, incl. NA12878) | smallest end-to-end fixture |
-| P0 | GSA v4 ePGx Demo 36 (872 MB) + BPM/CSV/EGT + CN/cyto models | primary reference-compatibility target (inputs and documented schemas) |
-| P0 | Bioconductor IlluminaDataTestFiles, hapmap370k (Artistic-2.0) | only fixtures we may vendor in CI |
-| P1 | 1000G Broad Omni2.5 IDAT+**GTC** 2,141 samples (107 GB) + BPM/EGT/CSV | GenCall oracle w/o licensed software; trios; clustering training |
-| P1 | 1000G Sanger Omni2.5-8 (6.2 GB) + GenomeStudio FinalReport | GenomeStudio-output oracle |
-| P1 | GEO GSE165845 GSA 120×3 (45 GB) | reproducibility, GSA-class concordance |
-| P1 | NYGC 30x 1kGP VCFs, GIAB v4.2.1, the vendor's public 1000 Genomes WGS reanalysis (AWS Open Data) (CNV/PGx) | truth |
-| P2 | GSE96909 (NA12878 on 9 designs), GDA ePGx 480, OneK1K GSA 1,104, 254 GEO series | cross-design, scale, parser fuzzing |
-
-Vendor product files/demo data: **fetch-with-checksum script only, never committed or redistributed** (site terms;
-plan §2 rule 4).
-
-## 16. Experiments (each: input / variable / observable / confidence)
-
-Experiments that needed the vendor binary (original E1 "GTC md5 of `refcli` vs ACLI vs gtc2vcf", and the `refcli`
-side of E4–E8, E11) are **dropped** under decision D2. The rest run against published vendor-produced GTCs, open
-oracles and synthetic data:
-
-E1 *(dropped: required the vendor binary)* — replaced by E2 against published GTCs.
-E2 Normalization transform per bin: SimurgArray vs Illumina GTC (1000G Broad) — tolerance study.
-E3 Synthetic single-SNP perturbation in IDAT (shift X/Y, bead count 0/1/2, NaN, extreme) → call/score response of
- SimurgArray vs gtc2vcf.
-E4 Subset-of-samples invariance: is genotype calling strictly per-sample? (expect yes [INF]; check on published GTCs.)
-E5 Sex Estimate decision boundary: sweep X-het and Y-R in synthetic IDATs (our definition; documented thresholds).
-E6 LogRDev / Autosomal call-rate locus sets: remove chromosomes/loci classes, compare with published GTC metrics.
-E7 VCF: indel REF/ALT, duplicates, MAV merging, `--use-infI-nc-info`, ploidy on X/Y/MT/PAR — vs GTCtoVCF/gtc2vcf and
- the documented reference behaviour.
-E8 PGx CN: batch-size and batch-composition sensitivity (24→500), GC-correction on/off — our own model.
-E9 Cyto: spiked-in synthetic LRR/BAF events (size, mosaic fraction 5–100%, LOH) → detection curves.
-E10 Star alleles: known 1000G samples with WGS-derived truth (PharmCAT-on-WGS as reference).
-E11 Determinism: repeat runs, thread counts, OS/architecture (x86-64 vs aarch64) for SimurgArray itself (the vendor
- reports tiny cross-platform PGx differences [DOC]).
-
-## 17. Known unknowns (tracked in `docs/known_unknowns.md`)
-
-U1 reference-product GTC ≡ ACLI GTC? (not testable under D2) · U2 GenCall score mapping details beyond OSS ·
-U3 Sex Estimate algorithm · U4 LogRDev/Autosomal CR locus set · U5 PGx CN algorithm & `.dat` format · U6 cyto
-caller & `.dat` format · U7 star-allele scoring/solver · U8 GenTrain internals (NN cluster imputation, scoring
-weights, 14–15 unused EGT floats) · U9 GTC PDF says Raw X = green vs OSS red — test · U10 IDAT MidBlock int32 vs
-uint8 · U11 EGT intensity-threshold version gate (≥7 vs 9) · U12 reference-product EULA text (not public; moot under
-D2).
-
-## 18. Risk register (updated to the decisions taken)
-
-| # | Risk | L | I | Mitigation / status |
-|---|---|---|---|---|
-| R1 | EULA prohibits deriving algorithms from the vendor binary | — | — | **Closed by D2**: the vendor binary is never run, benchmarked, inspected or decompiled; oracles are published vendor-produced GTCs |
-| R2 | Provenance: gtc2vcf ports Illumina GenCall C# code | L | M | **D1**: MIT code may be ported with attribution in `THIRD_PARTY_NOTICES.md`; no strict clean room is required for a personal research project |
-| R3 | Research notes (R2) contain gtc2vcf-derived details | L | L | **Closed by D1**: allowed; cite sources (plan §7 tags) |
-| R4 | GenTrain can't be matched ⇒ dependence on Illumina EGT files (non-redistributable) | H | M | ArrayTrain own clusters; user supplies EGT; fetch script |
-| R5 | Illumina manifest/EGT/demo licensing (non-commercial, no redistribution) | H | M | never commit; checksum fetcher; CI on Artistic-2.0 fixtures + synthetic data |
-| R6 | PGx/cyto calls diverge from the reference product | H | M | "compatible schema, independently validated"; publish concordance vs truth, never claim equivalence |
-| R7 | Encrypted v1 IDATs (DES, only GPL illuminaio reads them) | L | L | out of scope; detect and error clearly |
-| R8 | Floating-point non-determinism across platforms/threads | M | M | deterministic reductions, fixed ordering, float32/64 policy per field, ULP tolerances |
-| R9 | Trademark misuse | M | M | project named SimurgArray; docs refer to Illumina *file formats* only and use no vendor product brand (plan §2 rule 5) |
-| R10 | Scale (100k samples) memory | M | M | per-sample streaming; only clustering needs cohort-wide data (column-chunked Arrow) |
-| R11 | Vendor binary needs x86-64; dev box is aarch64 | — | — | **Moot under D2** (the binary is not used) |
-
-## Decisions taken (see plan §2)
-
-- **D1 — Provenance policy: taken.** SimurgArray is a personal, non-commercial research project. MIT / BSD /
- Apache-2.0 / Unlicense code (gtc2vcf/idat2gtc/BAFregress, mocha, BeadArrayFiles, GTCtoVCF, Picard `picard.arrays`,
- PennCNV, PyPGx, bcftools) **may be ported or vendored with attribution** in `THIRD_PARTY_NOTICES.md`. GPL tools
- (plink2, illuminaio, verifyIDintensity, GenotypeHarmonizer, akt) are **external subprocess oracles only**, never
- copied or linked. Code with no licence or non-commercial/PolyForm terms (zCall, argyle, Aldy, Cyrius, Stargazer,
- QuantiSNP binaries) is not used; ideas from their papers are fine. The Phase-0 "Path B strict clean room" option
- is not needed.
-- **D2 — Black-box access: taken.** Black-box runs of the vendor binary are **not part of the plan**, and vendor
- software is never decompiled, disassembled or inspected, nor licence checks bypassed. Oracles are the **published
- vendor-produced GTCs** (1000 Genomes Omni2.5 Broad set; GenomeStudio FinalReport in the Sanger set),
- **gtc2vcf** and **BeadArrayFiles** (plus GTCtoVCF/Picard for formats). Reference-product behaviour is taken only
- from its public documentation (files A–E); what cannot be settled that way is logged as a known unknown.
-- **D3 — Scope of v1.0: taken.** Genotyping + QC + VCF/PLINK first (plan phases P0–P8, then v1.0); ArrayTrain,
- CN/LOH/mosaic, PGx and methylation control QC are v2 modules (plan §18).
diff --git a/docs/research/A_cli_inputs.md b/docs/research/A_cli_inputs.md
deleted file mode 100644
index de92fb2..0000000
--- a/docs/research/A_cli_inputs.md
+++ /dev/null
@@ -1,814 +0,0 @@
-# A — Local CLI & Input Files: exhaustive analysis
-
-> **Archived Phase-0 study (sanitized copy, see `README.md`).** Written 2026-09-27, before the decisions in
-> `SIMURGARRAY_MASTER_PLAN.md` §2. The analysed vendor product is called "the reference product" and its CLI
-> `refcli`; its documentation URLs are replaced by `[ref-docs:…]` tokens. Where this file speaks of a clean room,
-> legal clearance, or running/benchmarking the vendor binary, plan §2 supersedes it: personal research; MIT/BSD/Apache
-> code may be reused with attribution; GPL tools are subprocess oracles only; vendor binaries are never run for
-> comparison, inspected or decompiled; oracles are published vendor-produced GTCs, gtc2vcf and BeadArrayFiles.
-
-Sources (both read in full, line by line):
-- `site/pages/product-guides/local-analysis.md` (647 lines, "[product] Local Analysis")
-- `site/pages/product-guides/input-files.md` (205 lines, "Input Files")
-
-Conventions in this document: "the vendor tool" = the analysed product; executable name is `refcli`.
-Line references `L:n` = local-analysis page, `I:n` = input-files page. Verbatim quotes are in `code` or "quotes".
-Items marked **[DOC-INCONSISTENCY]** are contradictions/omissions in the source docs themselves.
-
-**Images:** neither page contains any image reference (no `![]`, `
`, `.png/.jpg/.svg`, or `gitbook/assets` URLs).
-Only external hyperlinks exist (listed in §8).
-
----
-
-## 1. Complete command reference
-
-### 1.1 General syntax (L:143-147)
-```
-refcli [module] [sub-module (not needed for cyto)] [command] [required parameters] [optional parameters]
-```
-- Command names are lowercase; **CLI options are case-sensitive** and must be typed exactly, notably mixed-case
- `--threshold-callRate` and `--threshold-logRdev` (L:147, repeated L:564).
-- Top-level modules (L:149-156):
-
-| Module | Description (verbatim) |
-|---|---|
-| qc | Extract control probe intensities, compute QC metrics and generate QC reports. |
-| genotype | Call genotypes, single nucleotide variants, and various related file conversions. |
-| pgx | Pharmacogenomics CNV, star allele calling and metabolizer status annotation. |
-| cyto | Cytogenetics CNV/LOH/mosaic calling and annotation. |
-| help | Display more information on a specific command. |
-| version | Display version information. |
-
-- `refcli help` — "Displays the first-layer help information." (L:160)
-- `refcli version` — "Displays current [product] Local version." (L:164). Used as installation check (L:55).
-- Every module and sub-module also has its own `help` and `version` commands; every leaf command has `--help` and
- (almost all) `--version` flags.
-
-Full command tree:
-```
-refcli
-├── help
-├── version
-├── qc
-│ ├── call
-│ ├── report
-│ ├── help
-│ └── version
-├── genotype
-│ ├── call
-│ ├── gtc-to-bedgraph
-│ ├── gtc-to-vcf
-│ ├── help
-│ └── version
-├── pgx
-│ ├── copy-number
-│ │ ├── call
-│ │ ├── train
-│ │ ├── help
-│ │ └── version
-│ ├── star-allele
-│ │ ├── call
-│ │ ├── annotate
-│ │ ├── help
-│ │ └── version
-│ ├── help
-│ └── version
-└── cyto (no sub-module)
- ├── call
- ├── annotate
- ├── help
- └── version
-```
-
-### 1.2 Common option semantics (apply to all leaf commands where present)
-| Flag | Type | Default | Meaning |
-|---|---|---|---|
-| `--debug` | boolean switch | false | "Includes stack traces in logs." / "Logs will include stack traces." |
-| `--json-log` | boolean switch | false | "Outputs logs in JSON format." (JSON schema not documented on these pages) |
-| `--help` | switch | — | Command help screen ("Display this help screen.") |
-| `--version` | switch | — | "Displays version information." (absent from `qc call` table) |
-| `--output-folder` | path | CWD for qc call, qc report, pgx copy-number train, star-allele call/annotate, cyto call/annotate; **not stated** for genotype call, gtc-to-bedgraph, gtc-to-vcf, pgx copy-number call | Output directory. Re-using an output folder overwrites older files (L:87). |
-| `--sample-sheet` | path (CSV) | — | Filter samples and supply metadata; see §3.9 |
-| `--idat-folder` / `--gtc-folder` | directory | — | Root folder; **recursive** (includes subdirectories, stated for idat-folder). Overrides sample-sheet `RootFolder`. |
-
-Rule repeated for every sample-set command: **"Either --idat-folder, --sample-sheet, or both are required inputs."**
-(qc call L:192, genotype call L:256) / **"Either --gtc-folder, --sample-sheet, or both are required inputs."**
-(gtc-to-bedgraph L:273, gtc-to-vcf L:324). For pgx copy-number call/train and cyto call the same pattern is implied
-(`--gtc-folder` not marked required) but not stated explicitly.
-
-### 1.3 `qc call` (L:177-192)
-"Performs QC sample analysis given raw iScan data (IDAT)." / "Extracts raw control probe intensities and computes QC
-metrics from IDAT files."
-
-| Option | Req | Type | Default | Allowed | Description |
-|---|---|---|---|---|---|
-| `--array-type` | **Required** | enum | — | `genotyping`, `methylation` | Array type for QC analysis |
-| `--csv-manifest` | **Required** | path | — | — | CSV manifest |
-| `--idat-folder` | "[Required]" in table, but see note | dir | — | — | IDAT dir incl. all subdirectories; overrides sheet RootFolder |
-| `--sample-sheet` | optional | path | — | — | Sample sheet for filtering |
-| `--output-folder` | optional | dir | CWD | — | Output dir |
-| `--debug` | optional | bool | false | — | stack traces |
-| `--help` | — | — | — | — | help |
-| `--json-log` | optional | bool | false | — | JSON logs |
-
-Note: "Either --idat-folder, --sample-sheet, or both are required inputs." **[DOC-INCONSISTENCY]** table says
-`--idat-folder` "[Required]" while the note says sheet alone suffices. No `--version`, no `--num-threads`, no BPM
-(QC uses the CSV manifest's `[Controls]` section, not BPM). Outputs (from L:106-107,196): `controls.raw_metrics.csv`,
-`controls.qc_metrics.csv`.
-
-### 1.4 `qc report` (L:194-216)
-Generates QC Report (interactive HTML) + QC Table (spreadsheet) from one or more dataset folders.
-Inputs per dataset folder: `controls.raw_metrics.csv` (required), `controls.qc_metrics.csv` (required), from
-`qc call` **or** `genotype call`; `gt_sample_summary.csv` ("highly recommended", optional) from `genotype call` for
-"functional QC". Supports "parent-folder discovery, dataset labeling, and interactive dashboards".
-
-| Option | Req | Type | Default | Description |
-|---|---|---|---|---|
-| `--data` | **Required** | `` | — | One or more dataset folders, comma-separated; **or one parent folder** → datasets auto-discovered beneath it |
-| `--config` | optional | `` YAML | built-ins | Default QC thresholds. CLI thresholds override config. Built-in defaults `callRate = 0.98`, `logRDev = 0.20`; "All other thresholds are unset by default." |
-| `--label` | optional | `