From 08e67cff292a7d7ad7c48f3a7317f9c378031de5 Mon Sep 17 00:00:00 2001 From: Kemal Yaylali Date: Tue, 29 Sep 2026 23:04:16 +0100 Subject: [PATCH] CN: final full-cohort cloud validation (GSE165845, CYTO-041/042 defaults) 360 arrays vs NYGC 30x SV: all samples OK (K-027 gate fix), PASS precision DEL 44 % / DUP 57 % (first run 39 % / 34 % with filters ignored), 61 % without the one excess_events array; rare DEL sensitivity 30 %; LOH replicated 85 %. m7g.4xlarge spot, 17 min. docs/validation/V2_cn.md, K-027. --- docs/known_unknowns.md | 2 +- docs/validation/V2_cn.md | 26 ++++- docs/validation/V2_cn_gse165845_final.json | 123 +++++++++++++++++++++ 3 files changed, 149 insertions(+), 2 deletions(-) create mode 100644 docs/validation/V2_cn_gse165845_final.json diff --git a/docs/known_unknowns.md b/docs/known_unknowns.md index 5aa59e9..9affca4 100644 --- a/docs/known_unknowns.md +++ b/docs/known_unknowns.md @@ -7,7 +7,7 @@ Plan §20 lists the open questions; this file records what the implementation ha | K-030 | 2026-09-29 | ArrayTrain (v2) | The clustering is ours, not the vendor's (plan §12): unbounded 1/2/3-component θ mixtures chosen by BIC, with components labelled by HWE goodness of fit plus weak position and gap priors. Fixed θ windows (v1) mislabelled compressed and asymmetric loci. On GSE165845 (train on replicate 3, evaluate on replicates 1–2, chr22 vs 1000G 30x) the result is 99.44 % concordance, 1.30 % NRD and 99.61 % call rate, against 99.46 % / 1.23 % / 99.15 % with the vendor EGT. The parameters were tuned on this evaluation (a flat optimum). GenTrain and cluster-separation scores are ours, so GenCall scores are not comparable with vendor-file scores. | Open: second cohort/chip; score calibration (`docs/validation/V2_arraytrain.md`). | | K-029 | 2026-09-29 | PGx star alleles (v2) | (1) PGX-012 error model: ε(GS) = 0.5·(1 − GS)² was replaced by ε = 1e-3 (GS ≥ 0.15) / 0.05 — after probe masking 0 of 9 740 GSA v4 PGx calls disagreed with 1000G 30x at any GS, and the old model made correct low-GS calls lose to the prior (spec amended). (2) PGX-010 masks derived from 14 individuals (spec: ≥ 100); a variant absent from the filtered 1000G panel counts as hom-ref only if rare (CYP2B6 rs2279343 is filtered out, not absent). (3) CPIC core-allele definitions use IUPAC wildcards; alleles without definitions (NUDT15 *2, structural CYP2D6) are not callable from SNVs. (4) GeT-RM uses legacy UGT1A1/SLCO1B1 names. Result vs GeT-RM (16 CEPH): 100/109 gene calls, CYP2D6/2C19/2C9 41/45. (5) DPWG: no public machine-readable genotype→phenotype mapping; CPIC mappings are used for genes CPIC covers (as ClinPGx does), DPWG-only genes (CYP1A2, CYP3A4, F5) get no phenotype. (6) GeT-RM definitions are older than CPIC/PharmVar for UGT1A1, SLCO1B1, CYP1A2 and CYP2C8 (e.g. PharmVar CYP2C8 *19 = rs2071426 alone). Result vs GeT-RM (16 CEPH, 21 genes): 154/168 comparable gene calls. Independent check on GSE165845 (GSA v1 C2, 120 individuals × 3, WGS-derived truth, 2-fold masks): 95.7 % array-resolvable concordance, 67.9 % top-solution (the chip lacks many defining variants), replicate concordance 99.2 % without the failed array. | Open: KNMP DPWG tables if they become available (`docs/validation/V2_pgx_star.md`). | | K-028 | 2026-09-29 | PGx CN (v2) | Deviations from PCN-009: the per-probe reference is the training cohort's median stored in the model (a calling-batch median is not CN 2 where deletions are common, e.g. GSTM1 null ≈ 50 %); no GC-correction step; the CN mixture is ours: µ_c = δ + γ·log2((c+1)/3) per region with a free lower CN-0 component and Hardy–Weinberg-constrained weights (probe compression differs by region: γ ≈ 0.19 CYP2D6.p5 … 0.56 GSTT1). Interim validation on the GSA v4 PGx demo — whose 36 arrays turned out to be 30 CEPH/1000G individuals — against NYGC 30x SV RD_CN: 224/240 region calls (GSTM1, UGT2B17 30/30), but in-sample (trained on the same arrays, < 96 samples, priors chosen with this truth in view). | Open: validate on an independent GSA-PGx cohort; CYP2D6 via GeT-RM star alleles (`docs/validation/V2_pgx_cn.md`). | -| K-027 | 2026-09-29 | CN/LOH (v2) | CYTO-014 noise gate: the summary `logr_dev` also measures cluster-file mismatch (GSE165845: 0.246, all samples `SampleQuality`) while the LRR used after the batch reference has SD ≈ 0.1; the gate now uses the SD of the normalised LRR (the JSON still reports the summary value). Spec: built-in mask/cytobands from UCSC; the annotation bundle uses UCSC ncbiRefSeq instead of RefSeq GFF + Ensembl GTF + HGNC. Real-data result: constitutional DEL/DUP calls 74–77 % in NYGC truth, mosaic calls mostly false positives. Pi tuning: mosaic prior / switch distance barely matter; CYTO-041 (noise-normalised BAF split, complete-loss promotion) cuts PASS mosaic calls by ~30 % without losing implanted ≥ 15 % mosaics. Two limits remain: (1) without phasing, < 20 % mosaic losses are not separable from regional BAF noise; (2) common CNVs (carrier frequency > 20 %) are almost never detected (1/236 DEL) because the median-LRR batch reference absorbs them; rare DEL (< 5 %) 38 %. | Open: rare-event sensitivity as primary metric; phasing-based mosaic BAF (`docs/validation/V2_cn.md`). | +| K-027 | 2026-09-29 | CN/LOH (v2) | CYTO-014 noise gate: the summary `logr_dev` also measures cluster-file mismatch (GSE165845: 0.246, all samples `SampleQuality`) while the LRR used after the batch reference has SD ≈ 0.1; the gate now uses the SD of the normalised LRR (the JSON still reports the summary value). Spec: built-in mask/cytobands from UCSC; the annotation bundle uses UCSC ncbiRefSeq instead of RefSeq GFF + Ensembl GTF + HGNC. Real-data result: constitutional DEL/DUP calls 74–77 % in NYGC truth, mosaic calls mostly false positives. Pi tuning: mosaic prior / switch distance barely matter; CYTO-041 (noise-normalised BAF split, complete-loss promotion) cuts PASS mosaic calls by ~30 % without losing implanted ≥ 15 % mosaics. Two limits remain: (1) without phasing, < 20 % mosaic losses are not separable from regional BAF noise; (2) common CNVs (carrier frequency > 20 %) are almost never detected (1/236 DEL) because the median-LRR batch reference absorbs them; rare DEL (< 5 %) 38 %. Final cloud run (360 arrays, CYTO-041/042 defaults): all samples `OK`; PASS precision DEL 44 %, DUP 57 % (61 % for CNV calls without one bad array); rare DEL sensitivity 30 %. | Open: rare-event sensitivity as primary metric; phasing-based mosaic BAF (`docs/validation/V2_cn.md`). | | K-026 | 2026-09-29 | Methylation QC (v2) | Spec deviations: vendor EPIC manifests put a 5th field on some control lines (≥ 4 fields accepted); detection p follows sesame's pOOBAH (1 − max(F(M), F(U))) rather than the ambiguous "1 − ECDF(M + U)"; minfi-style `Sentrix_ID`/`Sentrix_Position` sheet keys accepted. Against sesame 1.28.1 on minfiDataEPIC: β r = 0.99998 (max \|Δβ\| 0.011), detection calls 99.1 % identical, noob signals within 0.5 % (median) / 5 % (max). The systematic remainder is unexplained; it is not the negative controls (sesame did not attach the supplied control table). | Open; `docs/validation/V2_methylation.md`. Next step: dump sesame's background vectors and noob parameters to find the difference. | | K-025 | 2026-09-28 | Scaling limits (P9) | With the Rust kernels a Graviton3 core genotypes 78–80 GSA samples/min. At > 8 workers a 125 MB/s gp3 volume caps the run at ~620 samples/min (GTC ≈ 13 MB each); on a RAM disk efficiency is 0.77 at 16 and 0.48 at 32 workers for 360 samples, explained by a fixed start-up cost (≈ 2.5 s + 0.2 s per worker). Linear scaling for large batches is inferred from that model, not measured. | `docs/validation/P9.md`; measure a ≥ 5000-sample batch when one is at hand. | | K-024 | 2026-09-28 | GSE165845 (P8) | GSA-MD v1 chips have no public cluster file; the GSA v1 C1 EGT with the C2 build-38 BPM covers 99.996 % of bead addresses and gives median LogR deviation 0.246 (cluster mismatch, not noise). GSM5051309 (NA20362 rep 2) is a failed array (call rate 0.762, concordance 0.336 with its own replicates). 27 510 array records have no NYGC 30x record and are scored as hom-ref; 1444 of them are systematically called alt. | Recorded in `docs/validation/P8.md`; results reported with and without these. | diff --git a/docs/validation/V2_cn.md b/docs/validation/V2_cn.md index 8dde62a..38e0afb 100644 --- a/docs/validation/V2_cn.md +++ b/docs/validation/V2_cn.md @@ -83,9 +83,33 @@ duplications labelled mosaic (43 % in truth). cluster file, trained on the population, already absorbs part of it). Array CN calling targets rare events; the headline sensitivity figures above are dominated by common polymorphic deletions. +## Final full-cohort run (cloud, 2026-09-29, commit ccf7f4f, CYTO-041/042 defaults) + +The run used m7g.4xlarge spot: 17 min in total (`cn build-model` 1 min 17 s, 6.0 GB; `cn call` of 360 samples 7 min 44 s, 1.7 GB). +Truth and matching are the same as above. PASS calls only, QUAL ≥ 20, no filters ignored +(`V2_cn_gse165845_final.json`). + +* **Sample QC.** All 360 samples are `OK`; the first cloud run flagged every sample `SampleQuality` (the K-027 gate + fix). 14 samples carry `high_noise` and one carries `excess_events`. +* **The `excess_events` array.** 203610890020_R06C01 (NA19923) has 238 DEL and 688 mosaic calls, and none of its + 147 CNV calls is in truth. It is a bad array, and it is the only one flagged. +* **Per-sample call counts (median).** 1 DEL, 0 DUP, 4 mosaic. + +| | DEL | DUP | +|---|---|---| +| precision, PASS calls | 213/480 = **44 %** (first run, filters ignored: 39 %) | 142/249 = **57 %** (first run, filters ignored: 34 %) | +| precision, CNV calls without R06C01 | 355/582 = **61 %** (DEL + DUP) | | +| sensitivity 25–50 / 50–100 / 100–500 / 500–1000 kb | 5 / 7 / 8 / 9 % | 0 / 3 / 10 / 14 % | +| sensitivity by carrier frequency < 5 % / 5–20 % / ≥ 20 % | **30 %** / 5 % / 0.05 % | 13 % / 21 % / 0.6 % | + +* **Replicate reproducibility.** CNV 35 %, LOH 85 %. +* **What CYTO-041/042 changed.** Precision is clearly higher and sensitivity is unchanged. The low overall + sensitivity is the common-CNV effect described above, not missed rare events. +* **Rare deletions.** 30 % of rare (< 5 %) truth deletions are found, the same as in the Pi tuning. + ## Next steps 1. Mosaic losses below ~20 %: phasing-based BAF (MoChA-style, MIT) is the principled fix. 2. Rare-event sensitivity (currently DEL 30 %, DUP 18 % for events ≥ 25 kb with ≥ 10 probes): exclude reference samples carrying the event when computing the per-probe median (iterative reference), size-aware transitions. -3. A final full-cohort run (cloud, ~$0.10, needs approval). +3. ~~A final full-cohort run~~ — done (section above). diff --git a/docs/validation/V2_cn_gse165845_final.json b/docs/validation/V2_cn_gse165845_final.json new file mode 100644 index 0000000..384a317 --- /dev/null +++ b/docs/validation/V2_cn_gse165845_final.json @@ -0,0 +1,123 @@ +{ + "samples": 360, + "min_qual": 20, + "ignored_filters": [], + "sensitivity": { + "DEL": { + "100-500kb": { + "found": 128, + "truth": 1554, + "rate": 0.0824 + }, + "1000-infkb": { + "found": 0, + "truth": 15, + "rate": 0.0 + }, + "25-50kb": { + "found": 12, + "truth": 225, + "rate": 0.0533 + }, + "50-100kb": { + "found": 59, + "truth": 894, + "rate": 0.066 + }, + "500-1000kb": { + "found": 16, + "truth": 180, + "rate": 0.0889 + } + }, + "DUP": { + "100-500kb": { + "found": 109, + "truth": 1059, + "rate": 0.1029 + }, + "1000-infkb": { + "found": 9, + "truth": 21, + "rate": 0.4286 + }, + "25-50kb": { + "found": 0, + "truth": 198, + "rate": 0.0 + }, + "50-100kb": { + "found": 13, + "truth": 462, + "rate": 0.0281 + }, + "500-1000kb": { + "found": 11, + "truth": 78, + "rate": 0.141 + } + } + }, + "sensitivity_by_carrier_frequency": { + "DEL": { + "5-20%": { + "found": 15, + "truth": 285, + "rate": 0.0526 + }, + "common>=20%": { + "found": 1, + "truth": 1923, + "rate": 0.0005 + }, + "rare<5%": { + "found": 199, + "truth": 660, + "rate": 0.3015 + } + }, + "DUP": { + "5-20%": { + "found": 21, + "truth": 102, + "rate": 0.2059 + }, + "common>=20%": { + "found": 5, + "truth": 855, + "rate": 0.0058 + }, + "rare<5%": { + "found": 116, + "truth": 861, + "rate": 0.1347 + } + } + }, + "precision": { + "DEL": { + "in_truth": 213, + "calls": 480, + "rate": 0.4437 + }, + "DUP": { + "in_truth": 142, + "calls": 249, + "rate": 0.5703 + } + }, + "replicate_reproducibility": { + "CNV": { + "matched": 512, + "comparisons": 1458, + "rate": 0.3512 + }, + "LOH": { + "matched": 356, + "comparisons": 420, + "rate": 0.8476 + } + }, + "calls_per_sample_median": 1.0, + "loh_calls_per_sample_median": 0.0 +} \ No newline at end of file