P5a: QC — control metrics, thresholds, status, contamination, relatedness, locus summary
- qc/controls: canonical control names (QC-002), bkg/offset (QC-003), 13 genotyping metrics (QC-004/005), missing/zero-divisor handling (QC-007/008); controls_raw.tsv / controls_qc.tsv (QC-009/011) incl. compat CSVs - qc/thresholds + profiles: YAML schema, validation codes, precedence, shipped profiles, auto (QC-020..025) - qc/status: functional PASS/FAIL/NA, control CLEAR/FLAGGED, flags (QC-030..033) - qc/contamination: BAF regression; bit-compatible with gtc2vcf BAFregress (36/36, K-017), exact mode too - qc/relatedness: KING-robust, duplicates, Mendelian errors, pedigree from sheet (IN-018) - qc/locus_summary: OUT-072, bit-identical to BeadArrayFiles except ChiTest (K-018) - qc/suggest: suggest-thresholds (QC-026) - pipeline/qc: disk-backed cohort store; CLI qc controls|samples|contamination|relatedness|loci|suggest-thresholds - genotype: control tables (OUT-070), het_rate, QC columns, cohort QC - tests: unit + oracle (BAFregress plugin, BeadArrayFiles locus summary)
This commit is contained in:
kemal committed
1 parent
d11226ca31
commit
a8caa26aa4
28 files changed
+3355
-22
No files matched your search
@@ -0,0 +1,179 @@
|
||||
"""Contamination estimate by B-allele-frequency regression (spec QC-064, plan §9.3).
|
||||
|
||||
[LIT: Jun G. et al. 2012, AJHG 91:839 — BAF regression on homozygous calls]
|
||||
[OSS: gtc2vcf ``BAFregress.c`` (MIT) — reference numerics: per-site median adjustment of AA/BB BAF when a cluster
|
||||
has >= 5 calls, float32 accumulators updated with double-precision addends, OLS slope in double.]
|
||||
|
||||
For every site with cohort B-allele frequency p in [t, 1 - t] (t = 0.1f):
|
||||
AA call: x = p, y = BAF - median_AA(BAF)
|
||||
BB call: x = 1 - p, y = 1 - (BAF - (median_BB(BAF) - 1))
|
||||
The contamination fraction is the slope m = (n·Σxy − Σx·Σy) / (n·Σx² − (Σx)²).
|
||||
|
||||
QUIRK (K-017): for an even number of calls the reference averages the quickselect result with whatever element sits
|
||||
just below it after the partial partition (klib ``ks_ksmall``), which is not always the lower middle value.
|
||||
``median="reference"`` (default) reproduces it exactly; ``median="exact"`` uses the true median.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from collections.abc import Callable, Iterable
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import numpy.typing as npt
|
||||
|
||||
F32 = np.float32
|
||||
CHUNK = 4096
|
||||
|
||||
|
||||
@dataclass
|
||||
class ContaminationResult:
|
||||
estimate: npt.NDArray[np.float64] # per sample; NaN when undefined
|
||||
n_hom: npt.NDArray[np.int64]
|
||||
n_sites: int
|
||||
|
||||
|
||||
def b_allele_frequency(genotypes: npt.NDArray[np.uint8]) -> npt.NDArray[np.float64]:
|
||||
"""Per-site B-allele frequency over called samples (rows = sites); NaN where nobody is called."""
|
||||
n_ab = (genotypes == 2).sum(axis=1)
|
||||
n_bb = (genotypes == 3).sum(axis=1)
|
||||
an = 2 * ((genotypes == 1).sum(axis=1) + n_ab + n_bb)
|
||||
with np.errstate(invalid="ignore", divide="ignore"):
|
||||
return np.where(an > 0, (n_ab + 2 * n_bb) / an.astype(np.float64), np.nan)
|
||||
|
||||
|
||||
def ks_ksmall(arr: list[float], kk: int) -> float:
|
||||
"""k-th smallest by in-place quickselect, partitioning exactly as klib ``ksort.h`` (MIT; htslib copy)
|
||||
so that the element left at ``arr[kk-1]`` matches the reference."""
|
||||
low, high, k = 0, len(arr) - 1, kk
|
||||
while True:
|
||||
if high <= low:
|
||||
return arr[k]
|
||||
if high == low + 1:
|
||||
if arr[high] < arr[low]:
|
||||
arr[low], arr[high] = arr[high], arr[low]
|
||||
return arr[k]
|
||||
mid = low + (high - low) // 2
|
||||
if arr[high] < arr[mid]:
|
||||
arr[mid], arr[high] = arr[high], arr[mid]
|
||||
if arr[high] < arr[low]:
|
||||
arr[low], arr[high] = arr[high], arr[low]
|
||||
if arr[low] < arr[mid]:
|
||||
arr[mid], arr[low] = arr[low], arr[mid]
|
||||
arr[mid], arr[low + 1] = arr[low + 1], arr[mid]
|
||||
ll, hh = low + 1, high
|
||||
pivot = arr[low]
|
||||
while True:
|
||||
ll += 1
|
||||
while arr[ll] < pivot:
|
||||
ll += 1
|
||||
hh -= 1
|
||||
while pivot < arr[hh]:
|
||||
hh -= 1
|
||||
if hh < ll:
|
||||
break
|
||||
arr[ll], arr[hh] = arr[hh], arr[ll]
|
||||
arr[low], arr[hh] = arr[hh], arr[low]
|
||||
if hh <= k:
|
||||
low = ll
|
||||
if hh >= k:
|
||||
high = hh - 1
|
||||
|
||||
|
||||
def _median_reference(values: npt.NDArray[np.float32]) -> np.float32:
|
||||
t = values.tolist() # float32 → exact Python floats; comparisons are unchanged
|
||||
j = len(t)
|
||||
m = F32(ks_ksmall(t, j // 2))
|
||||
if j % 2 == 0:
|
||||
m = F32((m + F32(t[j // 2 - 1])) * F32(0.5))
|
||||
return m
|
||||
|
||||
|
||||
def _median_exact(baf: npt.NDArray[np.float32], mask: npt.NDArray[np.bool_]) -> npt.NDArray[np.float32]:
|
||||
"""Row-wise true median of ``baf`` where ``mask``; float32, even counts average the middle pair as (a+b)*0.5f."""
|
||||
v = np.where(mask, baf, F32(np.inf)).astype(F32)
|
||||
v.sort(axis=1)
|
||||
k = mask.sum(axis=1)
|
||||
rows = np.arange(len(v))
|
||||
hi = v[rows, k // 2]
|
||||
lo = v[rows, np.maximum(k // 2 - 1, 0)]
|
||||
return np.where(k % 2 == 0, ((lo + hi) * F32(0.5)).astype(F32), hi).astype(F32)
|
||||
|
||||
|
||||
def cluster_adjustments(
|
||||
task: tuple[npt.NDArray[np.uint8], npt.NDArray[np.float32], int, str],
|
||||
) -> tuple[npt.NDArray[np.float32], npt.NDArray[np.float32]]:
|
||||
"""Per-site AA and BB BAF adjustments for one chunk of sites (picklable for process pools)."""
|
||||
g, b, adjust_min, median = task
|
||||
adj_aa = np.zeros(len(g), F32)
|
||||
adj_bb = np.zeros(len(g), F32)
|
||||
if adjust_min == -1:
|
||||
return adj_aa, adj_bb
|
||||
for code, out, shift in ((1, adj_aa, F32(0.0)), (3, adj_bb, F32(1.0))):
|
||||
mask = g == code
|
||||
use = np.flatnonzero(mask.sum(axis=1) >= adjust_min)
|
||||
if not len(use):
|
||||
continue
|
||||
if median == "exact":
|
||||
out[use] = _median_exact(b[use], mask[use]) - shift
|
||||
else:
|
||||
for i in use:
|
||||
out[i] = _median_reference(b[i][mask[i]]) - shift
|
||||
return adj_aa, adj_bb
|
||||
|
||||
|
||||
MapFn = Callable[[Callable[[Any], Any], Iterable[Any]], Iterable[Any]]
|
||||
|
||||
|
||||
def baf_regress(
|
||||
genotypes: npt.NDArray[np.uint8],
|
||||
baf: npt.NDArray[np.float32],
|
||||
af: npt.NDArray[np.float64] | None = None,
|
||||
*,
|
||||
threshold: float = 0.1,
|
||||
adjust_min: int = 5,
|
||||
median: str = "reference",
|
||||
map_fn: MapFn = map,
|
||||
) -> ContaminationResult:
|
||||
"""Contamination per sample. ``genotypes`` (0 no call, 1 AA, 2 AB, 3 BB) and ``baf`` are (sites × samples);
|
||||
``af`` defaults to the cohort B-allele frequency. Sites are used in row order (the reference's float32
|
||||
accumulation is order-dependent). ``map_fn`` (e.g. an executor's ``map``) parallelises the per-site medians."""
|
||||
if median not in ("reference", "exact"):
|
||||
raise ValueError(f"median must be 'reference' or 'exact', got {median!r}")
|
||||
n = genotypes.shape[1]
|
||||
if af is None:
|
||||
af = b_allele_frequency(genotypes)
|
||||
t = float(F32(threshold)) # the reference stores the threshold as float: 0.1f = 0.10000000149…
|
||||
keep = np.flatnonzero(np.isfinite(af) & (af >= t) & (af <= 1.0 - t))
|
||||
chunks = [keep[s : s + CHUNK] for s in range(0, len(keep), CHUNK)]
|
||||
tasks = ((np.asarray(genotypes[r]), np.asarray(baf[r], dtype=F32), adjust_min, median) for r in chunks)
|
||||
sx2, sxy, sx, sy = (np.zeros(n, F32) for _ in range(4))
|
||||
cnt = np.zeros(n, np.int64)
|
||||
for rows, (adj_aa, adj_bb) in zip(chunks, map_fn(cluster_adjustments, tasks), strict=True):
|
||||
g = np.asarray(genotypes[rows])
|
||||
b = np.asarray(baf[rows], dtype=F32)
|
||||
p = af[rows]
|
||||
is_aa, is_bb = g == 1, g == 3
|
||||
y_aa = (b - adj_aa[:, None]).astype(F32).astype(np.float64)
|
||||
y_bb = (b - adj_bb[:, None]).astype(F32).astype(np.float64)
|
||||
for i in range(len(rows)):
|
||||
a, m = is_aa[i], is_aa[i] | is_bb[i]
|
||||
if not m.any():
|
||||
continue
|
||||
pa, pb = p[i], 1.0 - p[i]
|
||||
x = np.where(a, pa, pb)[m]
|
||||
xy = np.where(a, pa * y_aa[i], pb * (1.0 - y_bb[i]))[m]
|
||||
y = np.where(a, y_aa[i], 1.0 - y_bb[i])[m]
|
||||
sx2[m] = (sx2[m].astype(np.float64) + x * x).astype(F32)
|
||||
sxy[m] = (sxy[m].astype(np.float64) + xy).astype(F32)
|
||||
sx[m] = (sx[m].astype(np.float64) + x).astype(F32)
|
||||
sy[m] = (sy[m].astype(np.float64) + y).astype(F32)
|
||||
cnt[m] += 1
|
||||
nf = cnt.astype(np.float64)
|
||||
fx2, fxy, fx, fy = (s.astype(np.float64) for s in (sx2, sxy, sx, sy))
|
||||
denom = nf * fx2 - fx * fx
|
||||
with np.errstate(invalid="ignore", divide="ignore"):
|
||||
est = np.where(denom != 0, (nf * fxy - fx * fy) / denom, math.nan)
|
||||
return ContaminationResult(estimate=est, n_hom=cnt, n_sites=len(keep))
|
||||
@@ -0,0 +1,84 @@
|
||||
# Canonical control-probe IDs (spec QC-002). Keys are *normalised* names: case-folded, whitespace collapsed,
|
||||
# no spaces next to or inside parentheses, "bgnd" written as "bkg". Several manifest spellings map to one ID.
|
||||
canonical:
|
||||
# staining
|
||||
biotin(high): biotin_high
|
||||
biotin(bkg): biotin_bkg
|
||||
biotin(5k): biotin_5k
|
||||
dnp(high): dnp_high
|
||||
dnp(bkg): dnp_bkg
|
||||
dnp(20k): dnp_20k
|
||||
# extension
|
||||
extension(a): ext_A
|
||||
extension(t): ext_T
|
||||
extension(c): ext_C
|
||||
extension(g): ext_G
|
||||
# hybridization
|
||||
hyb(high): hyb_high
|
||||
hyb(medium): hyb_medium
|
||||
hyb(low): hyb_low
|
||||
# target removal
|
||||
target removal: target_removal_1
|
||||
target removal(1): target_removal_1
|
||||
target removal 1: target_removal_1
|
||||
target removal(2): target_removal_2
|
||||
target removal 2: target_removal_2
|
||||
# stringency
|
||||
string(pm): string_pm
|
||||
string(mm): string_mm
|
||||
# non-specific binding (one bead type per colour label on genotyping chips)
|
||||
nsb(bkg): nsb_bkg
|
||||
nsb(bkg) red: nsb_bkg
|
||||
nsb(bkg) green: nsb_bkg
|
||||
nsb(bkg) blue: nsb_bkg
|
||||
nsb(bkg) purple: nsb_bkg
|
||||
# non-polymorphic
|
||||
np(a): np_A
|
||||
np(t): np_T
|
||||
np(c): np_C
|
||||
np(g): np_G
|
||||
# restoration (FFPE)
|
||||
restoration: restoration
|
||||
restore: restoration
|
||||
# methylation (metrics in v2; names mapped already so controls_raw.tsv is labelled consistently)
|
||||
bs conversion i c1: bs1_C1
|
||||
bs conversion i c2: bs1_C2
|
||||
bs conversion i c3: bs1_C3
|
||||
bs conversion i c4: bs1_C4
|
||||
bs conversion i c5: bs1_C5
|
||||
bs conversion i u1: bs1_U1
|
||||
bs conversion i u2: bs1_U2
|
||||
bs conversion i u3: bs1_U3
|
||||
bs conversion i u4: bs1_U4
|
||||
bs conversion i u5: bs1_U5
|
||||
bs conversion ii-1: bs2_1
|
||||
bs conversion ii-2: bs2_2
|
||||
bs conversion ii-3: bs2_3
|
||||
bs conversion ii-4: bs2_4
|
||||
gt mismatch 1(pm): gtmm_1_pm
|
||||
gt mismatch 2(pm): gtmm_2_pm
|
||||
gt mismatch 3(pm): gtmm_3_pm
|
||||
gt mismatch 4(pm): gtmm_4_pm
|
||||
gt mismatch 5(pm): gtmm_5_pm
|
||||
gt mismatch 6(pm): gtmm_6_pm
|
||||
gt mismatch 1(mm): gtmm_1_mm
|
||||
gt mismatch 2(mm): gtmm_2_mm
|
||||
gt mismatch 3(mm): gtmm_3_mm
|
||||
gt mismatch 4(mm): gtmm_4_mm
|
||||
gt mismatch 5(mm): gtmm_5_mm
|
||||
gt mismatch 6(mm): gtmm_6_mm
|
||||
specificity 1: spec2_1
|
||||
specificity 2: spec2_2
|
||||
specificity 3: spec2_3
|
||||
|
||||
# Categories treated as methylation-only (dataset type detection, QC-043).
|
||||
methylation_categories:
|
||||
- bisulfite conversion i
|
||||
- bisulfite conversion ii
|
||||
- specificity i
|
||||
- specificity ii
|
||||
|
||||
# Report exclusion list (QC-010): raw-probe views leave these out; derived metrics still use them.
|
||||
report_exclude:
|
||||
categories: [negative, norm]
|
||||
names: ["np(g) 1", "np(g) 2", "np(g) 3", "np(g) 4", "np(g) 5", "biotin(5k)", "dnp(20k)"]
|
||||
@@ -0,0 +1,257 @@
|
||||
"""Control-probe QC metrics (spec QC-001 … QC-011; plan §9.1).
|
||||
|
||||
Controls are the non-polymorphic bead types listed in the manifest's control section. Each gives a raw green and
|
||||
red intensity (IDAT means, no normalization). Derived metrics are signal/background ratios per assay step.
|
||||
[DOC: Illumina BeadArray Controls Reporter software guide and Infinium controls training material — formulas only]
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import re
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from functools import cache
|
||||
from importlib import resources
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import numpy.typing as npt
|
||||
import yaml
|
||||
|
||||
from simurg.formats.idat import Idat
|
||||
from simurg.signal.raw_xy import AddressIndex
|
||||
|
||||
DEFAULT_BG_OFFSET = 3000.0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ControlBead:
|
||||
"""One control bead type: a (manifest control line, distinct address) pair, in manifest order."""
|
||||
|
||||
category: str
|
||||
color: str
|
||||
name: str
|
||||
address: int
|
||||
canonical: str # canonical ID, "" when the name is not in control_names.yaml
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Metric:
|
||||
column: str # controls_qc.tsv column
|
||||
key: str # threshold YAML key (QC-020)
|
||||
genotyping: bool
|
||||
methylation: bool
|
||||
|
||||
|
||||
# QC-004, in table order.
|
||||
METRICS: tuple[Metric, ...] = (
|
||||
Metric("restoration", "restoration", True, True),
|
||||
Metric("staining_green", "stainingGreen", True, True),
|
||||
Metric("staining_red", "stainingRed", True, True),
|
||||
Metric("extension_green", "extensionGreen", True, True),
|
||||
Metric("extension_red", "extensionRed", True, True),
|
||||
Metric("hyb_high_med", "hybGreenHighMed", True, True),
|
||||
Metric("hyb_med_low", "hybGreenMediumLow", True, True),
|
||||
Metric("target_removal_1", "targetRemovalI", True, True),
|
||||
Metric("target_removal_2", "targetRemovalII", False, True),
|
||||
Metric("stringency", "stringency", True, False),
|
||||
Metric("nsb_green", "nonSpecificBindingGreen", True, False),
|
||||
Metric("nsb_red", "nonSpecificBindingRed", True, False),
|
||||
Metric("np_green", "npGreen", True, True),
|
||||
Metric("np_red", "npRed", True, True),
|
||||
Metric("specificity_1_green", "specificityIGreen", False, True),
|
||||
Metric("specificity_1_red", "specificityIRed", False, True),
|
||||
Metric("specificity_2", "specificityII", False, True),
|
||||
Metric("specificity_2_bg", "specificityIIBg", False, True),
|
||||
Metric("bs_conv_1_green", "bisulfiteConversionIGreen", False, True),
|
||||
Metric("bs_conv_1_green_bg", "bisulfiteConversionIGreenBg", False, True),
|
||||
Metric("bs_conv_1_red", "bisulfiteConversionIRed", False, True),
|
||||
Metric("bs_conv_1_red_bg", "bisulfiteConversionIRedBg", False, True),
|
||||
Metric("bs_conv_2", "bisulfiteConversionII", False, True),
|
||||
Metric("bs_conv_2_bg", "bisulfiteConversionIIBg", False, True),
|
||||
)
|
||||
ASSAY_CONTROL_COLUMN = "assay_control_norm_r" # QC-005: copied from the sample summary
|
||||
COMPAT_METRIC_NAMES = {
|
||||
"restoration": "RestorationQC",
|
||||
"staining_green": "StainingGreenQC",
|
||||
"staining_red": "StainingRedQC",
|
||||
"extension_green": "ExtensionGreenQC",
|
||||
"extension_red": "ExtensionRedQC",
|
||||
"hyb_high_med": "HybridizationHighMediumQC",
|
||||
"hyb_med_low": "HybridizationMediumLowQC",
|
||||
"target_removal_1": "TargetRemovalQC",
|
||||
"target_removal_2": "TargetRemoval2QC",
|
||||
"stringency": "StringencyQC",
|
||||
"nsb_green": "NonSpecificBindingGreenQC",
|
||||
"nsb_red": "NonSpecificBindingRedQC",
|
||||
"np_green": "NonPolymorphicGreenQC",
|
||||
"np_red": "NonPolymorphicRedQC",
|
||||
ASSAY_CONTROL_COLUMN: "AssayControlQC",
|
||||
}
|
||||
|
||||
|
||||
def metrics_for(array_type: str) -> list[Metric]:
|
||||
"""QC-005 / QC-006 metric sets."""
|
||||
if array_type == "genotyping":
|
||||
return [m for m in METRICS if m.genotyping]
|
||||
return [m for m in METRICS if m.methylation]
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------ names
|
||||
@cache
|
||||
def name_config() -> dict[str, Any]:
|
||||
text = resources.files("simurg.qc").joinpath("control_names.yaml").read_text(encoding="utf-8")
|
||||
cfg: dict[str, Any] = yaml.safe_load(text)
|
||||
return cfg
|
||||
|
||||
|
||||
def normalise_name(name: str) -> str:
|
||||
"""QC-002 normalisation: case-fold, collapse whitespace, no spaces before or inside parentheses, bgnd = bkg."""
|
||||
s = " ".join(name.casefold().split())
|
||||
s = re.sub(r"\s*\(\s*", "(", s)
|
||||
s = re.sub(r"\s*\)", ")", s)
|
||||
return re.sub(r"\bbgnd\b|\bbkgd\b", "bkg", s)
|
||||
|
||||
|
||||
def canonical_id(name: str) -> str:
|
||||
return str(name_config()["canonical"].get(normalise_name(name), ""))
|
||||
|
||||
|
||||
def excluded_from_report(bead: ControlBead) -> bool:
|
||||
"""QC-010."""
|
||||
ex = name_config()["report_exclude"]
|
||||
return bead.category.casefold() in ex["categories"] or normalise_name(bead.name) in ex["names"]
|
||||
|
||||
|
||||
def is_methylation_controls(beads: Sequence[ControlBead]) -> bool:
|
||||
cats = set(name_config()["methylation_categories"])
|
||||
return any(b.category.casefold() in cats for b in beads)
|
||||
|
||||
|
||||
def beads_from_definitions(defs: Sequence[dict[str, str]]) -> list[ControlBead]:
|
||||
"""Manifest control lines (``address`` may be ``a:b:…``) → bead types in manifest order (QC-001)."""
|
||||
out: list[ControlBead] = []
|
||||
for d in defs:
|
||||
seen: set[int] = set()
|
||||
for part in str(d["address"]).split(":"):
|
||||
part = part.strip()
|
||||
if not part:
|
||||
continue
|
||||
addr = int(part)
|
||||
if addr in seen:
|
||||
continue
|
||||
seen.add(addr)
|
||||
out.append(ControlBead(d["category"], d["color"], d["name"], addr, canonical_id(d["name"])))
|
||||
return out
|
||||
|
||||
|
||||
def compare_definitions(a: Sequence[ControlBead], b: Sequence[ControlBead]) -> list[str]:
|
||||
"""Differences between two control lists (BPM vs CSV, QC-001); empty when they agree."""
|
||||
ka = {(x.address, normalise_name(x.name)) for x in a}
|
||||
kb = {(x.address, normalise_name(x.name)) for x in b}
|
||||
diffs = [f"only in BPM: {addr} {nm}" for addr, nm in sorted(ka - kb)[:5]]
|
||||
diffs += [f"only in CSV: {addr} {nm}" for addr, nm in sorted(kb - ka)[:5]]
|
||||
return diffs
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------ intensities
|
||||
def control_intensities(
|
||||
grn: Idat, red: Idat, beads: Sequence[ControlBead]
|
||||
) -> tuple[npt.NDArray[np.float64], npt.NDArray[np.float64]]:
|
||||
"""Raw green/red means per bead type; NaN where the IDAT lacks the address."""
|
||||
addrs = np.array([b.address for b in beads], dtype=np.int64)
|
||||
out = []
|
||||
for idat in (grn, red):
|
||||
rows, found = AddressIndex(idat).lookup(addrs)
|
||||
v = idat.mean[rows].astype(np.float64)
|
||||
v[~found] = math.nan
|
||||
out.append(v)
|
||||
return out[0], out[1]
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------ metrics
|
||||
@dataclass
|
||||
class ControlQC:
|
||||
values: dict[str, float] # column -> value (NaN when not evaluable or degenerate)
|
||||
missing: dict[str, list[str]] = field(default_factory=dict) # column -> canonical IDs absent from the manifest
|
||||
zero_divisor: list[str] = field(default_factory=list) # columns with a zero denominator (QC-008)
|
||||
|
||||
@property
|
||||
def na_metrics(self) -> list[str]:
|
||||
return list(self.missing)
|
||||
|
||||
|
||||
class _Probes:
|
||||
def __init__(self, beads: Sequence[ControlBead], green: npt.NDArray[np.float64], red: npt.NDArray[np.float64]):
|
||||
self.idx: dict[str, list[int]] = {}
|
||||
for i, b in enumerate(beads):
|
||||
if b.canonical:
|
||||
self.idx.setdefault(b.canonical, []).append(i)
|
||||
self.ch = {"G": green, "R": red}
|
||||
self.absent: set[str] = set()
|
||||
|
||||
def vals(self, ids: Sequence[str], ch: str) -> npt.NDArray[np.float64]:
|
||||
rows: list[int] = []
|
||||
for cid in ids:
|
||||
if cid not in self.idx:
|
||||
self.absent.add(cid)
|
||||
rows.extend(self.idx.get(cid, []))
|
||||
return self.ch[ch][rows]
|
||||
|
||||
def mean(self, cid: str, ch: str) -> float:
|
||||
v = self.vals([cid], ch)
|
||||
return float(v.mean()) if len(v) else math.nan
|
||||
|
||||
def min(self, ids: Sequence[str], ch: str) -> float:
|
||||
v = self.vals(ids, ch)
|
||||
return float(v.min()) if len(v) else math.nan
|
||||
|
||||
def max(self, ids: Sequence[str], ch: str) -> float:
|
||||
v = self.vals(ids, ch)
|
||||
return float(v.max()) if len(v) else math.nan
|
||||
|
||||
|
||||
def compute_control_metrics(
|
||||
beads: Sequence[ControlBead],
|
||||
green: npt.NDArray[np.float64],
|
||||
red: npt.NDArray[np.float64],
|
||||
array_type: str = "genotyping",
|
||||
bg_offset: float = DEFAULT_BG_OFFSET,
|
||||
) -> ControlQC:
|
||||
"""QC-003/QC-004 metrics of one sample. Single-probe values are bead means; min/max run over all beads (QC-002)."""
|
||||
if array_type != "genotyping":
|
||||
from simurg.core.errors import SimurgError
|
||||
|
||||
raise SimurgError("E_QC_METHYLATION", "methylation control metrics are planned for v2 (QC-006)")
|
||||
p = _Probes(beads, green, red)
|
||||
x = bg_offset
|
||||
formulas: dict[str, tuple[Any, Any]] = {
|
||||
# column: (numerator thunk, denominator thunk); each thunk records the probes it needs
|
||||
"restoration": (lambda: p.mean("restoration", "G"), lambda: p.max(["ext_A", "ext_T"], "G") + x),
|
||||
"staining_green": (lambda: p.mean("biotin_high", "G"), lambda: p.mean("biotin_bkg", "G")),
|
||||
"staining_red": (lambda: p.mean("dnp_high", "R"), lambda: p.mean("dnp_bkg", "R")),
|
||||
"extension_green": (lambda: p.min(["ext_C", "ext_G"], "G"), lambda: p.max(["ext_A", "ext_T"], "G")),
|
||||
"extension_red": (lambda: p.min(["ext_A", "ext_T"], "R"), lambda: p.max(["ext_C", "ext_G"], "R")),
|
||||
"hyb_high_med": (lambda: p.mean("hyb_high", "G"), lambda: p.mean("hyb_medium", "G")),
|
||||
"hyb_med_low": (lambda: p.mean("hyb_medium", "G"), lambda: p.mean("hyb_low", "G")),
|
||||
"target_removal_1": (lambda: p.max(["ext_A", "ext_T"], "G") + x, lambda: p.mean("target_removal_1", "G")),
|
||||
"stringency": (lambda: p.mean("string_pm", "R"), lambda: p.mean("string_mm", "R")),
|
||||
"nsb_green": (lambda: p.max(["ext_A", "ext_T"], "G") + x, lambda: p.max(["nsb_bkg"], "G")),
|
||||
"nsb_red": (lambda: p.max(["ext_C", "ext_G"], "R") + x, lambda: p.max(["nsb_bkg"], "R")),
|
||||
"np_green": (lambda: p.min(["np_C", "np_G"], "G"), lambda: p.max(["np_A", "np_T"], "G")),
|
||||
"np_red": (lambda: p.min(["np_A", "np_T"], "R"), lambda: p.max(["np_C", "np_G"], "R")),
|
||||
}
|
||||
res = ControlQC(values={})
|
||||
for m in metrics_for(array_type):
|
||||
p.absent = set()
|
||||
num, den = formulas[m.column][0](), formulas[m.column][1]()
|
||||
if p.absent:
|
||||
res.missing[m.column] = sorted(p.absent)
|
||||
res.values[m.column] = math.nan
|
||||
elif den == 0:
|
||||
res.zero_divisor.append(m.column)
|
||||
res.values[m.column] = math.nan
|
||||
else:
|
||||
res.values[m.column] = num / den # NaN propagates when a bead's address is absent from the IDAT
|
||||
return res
|
||||
@@ -0,0 +1,135 @@
|
||||
"""Per-locus summary statistics across samples (spec OUT-072, CLI-025; plan §7.9).
|
||||
|
||||
[OSS: BeadArrayFiles ``examples/locus_summary.py`` (BSD-2) — formulas: call/genotype frequencies, minor frequency,
|
||||
k-regularised heterozygote excess with k = (0.013/q)^4, ChiTest_P100 = 2·Φ(±dh; 0, 0.1), GenomeStudio percentile of
|
||||
the called GenCall scores.] Vectorised over a block of loci; the caller streams blocks for large cohorts.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
|
||||
import numpy as np
|
||||
import numpy.typing as npt
|
||||
|
||||
COLUMNS = [
|
||||
"locus_name",
|
||||
"locus_id",
|
||||
"n_nocalls",
|
||||
"n_calls",
|
||||
"call_freq",
|
||||
"aa_freq",
|
||||
"ab_freq",
|
||||
"bb_freq",
|
||||
"minor_freq",
|
||||
"gentrain_score",
|
||||
"gc50",
|
||||
"gc10",
|
||||
"het_excess",
|
||||
"chi_test_p100",
|
||||
"cluster_sep",
|
||||
"aa_t_mean",
|
||||
"aa_t_sd",
|
||||
"ab_t_mean",
|
||||
"ab_t_sd",
|
||||
"bb_t_mean",
|
||||
"bb_t_sd",
|
||||
"aa_r_mean",
|
||||
"aa_r_sd",
|
||||
"ab_r_mean",
|
||||
"ab_r_sd",
|
||||
"bb_r_mean",
|
||||
"bb_r_sd",
|
||||
"plus_minus_strand",
|
||||
"het_excess_simple",
|
||||
]
|
||||
HET_EXCESS_EPS = 1e-9 # OQ-030: denominator guard of the simple form
|
||||
M_SQRT1_2 = 0.70710678118654752440 # C literal (1/sqrt(2) in Python is 1 ULP lower)
|
||||
|
||||
|
||||
def _norm_cdf_sd01(x: npt.NDArray[np.float64]) -> npt.NDArray[np.float64]:
|
||||
"""Φ(x; 0, 0.1) with the branch structure of the cephes ``ndtr`` used by the reference."""
|
||||
z = (x / 0.1) * M_SQRT1_2
|
||||
out = np.empty_like(z)
|
||||
for i, v in enumerate(z.tolist()):
|
||||
a = abs(v)
|
||||
if a < M_SQRT1_2:
|
||||
y = 0.5 + 0.5 * math.erf(v)
|
||||
else:
|
||||
y = 0.5 * math.erfc(a)
|
||||
if v > 0:
|
||||
y = 1.0 - y
|
||||
out[i] = y
|
||||
return out
|
||||
|
||||
|
||||
def gs_percentile_columns(
|
||||
sorted_scores: npt.NDArray[np.float32], counts: npt.NDArray[np.int64], p: int
|
||||
) -> npt.NDArray[np.float64]:
|
||||
"""GenomeStudio percentile of the first ``counts[j]`` values of each column of ``sorted_scores`` (double math)."""
|
||||
n = counts.astype(np.int64)
|
||||
pos = n * p / 100.0
|
||||
idx = pos.astype(np.int64) # int(num*p/100) truncates like the reference
|
||||
idx = np.where(pos - idx < 0.5, idx - 1, idx)
|
||||
cols = np.arange(sorted_scores.shape[1])
|
||||
last = np.maximum(n - 1, 0)
|
||||
lo = np.clip(idx, 0, np.maximum(last - 1, 0))
|
||||
y1 = sorted_scores[lo, cols].astype(np.float64)
|
||||
y2 = sorted_scores[np.minimum(lo + 1, last), cols].astype(np.float64)
|
||||
with np.errstate(invalid="ignore", divide="ignore"):
|
||||
x1 = 100 * (lo + 0.5) / n
|
||||
x2 = 100 * (lo + 1 + 0.5) / n
|
||||
mid = y1 + (y2 - y1) / (x2 - x1) * (p - x1)
|
||||
first = sorted_scores[0, cols].astype(np.float64)
|
||||
lastv = sorted_scores[last, cols].astype(np.float64)
|
||||
out = np.where(idx < 0, first, np.where(idx >= n - 1, lastv, mid))
|
||||
return np.where(n == 0, np.nan, out)
|
||||
|
||||
|
||||
def summarize_block(genotypes: npt.NDArray[np.uint8], scores: npt.NDArray[np.float32]) -> dict[str, npt.NDArray]: # type: ignore[type-arg]
|
||||
"""Statistics for a block of loci; inputs are (samples × loci)."""
|
||||
g = np.asarray(genotypes)
|
||||
nc = (g == 0).sum(axis=0).astype(np.int64)
|
||||
aa = (g == 1).sum(axis=0).astype(np.int64)
|
||||
ab = (g == 2).sum(axis=0).astype(np.int64)
|
||||
bb = (g == 3).sum(axis=0).astype(np.int64)
|
||||
calls = aa + ab + bb
|
||||
with np.errstate(invalid="ignore", divide="ignore"):
|
||||
call_freq = np.where(calls + nc > 0, calls / (calls + nc), np.nan)
|
||||
cf = calls.astype(np.float64)
|
||||
aa_f = np.where(calls > 0, aa / cf, np.nan)
|
||||
ab_f = np.where(calls > 0, ab / cf, np.nan)
|
||||
bb_f = np.where(calls > 0, bb / cf, np.nan)
|
||||
a_freq = np.where(calls > 0, (aa * 2 + ab) / (2 * cf), np.nan)
|
||||
q = np.minimum(a_freq, 1.0 - a_freq)
|
||||
p = 1 - q
|
||||
temp = 0.013 / q
|
||||
k = temp * temp * temp * temp
|
||||
dh = ((ab / cf + k) / (2 * p * q + k)) - 1
|
||||
simple = (ab_f - 2 * p * q) / (2 * p * q + HET_EXCESS_EPS)
|
||||
mono = (aa + ab == 0) | (ab + bb == 0)
|
||||
regular = (calls > 0) & ~mono
|
||||
hw = np.zeros(len(calls))
|
||||
het = np.zeros(len(calls))
|
||||
if regular.any():
|
||||
d = dh[regular]
|
||||
cdf = _norm_cdf_sd01(d)
|
||||
hw[regular] = np.where(d < 0, 2 * cdf, 2 * (1 - cdf))
|
||||
het[regular] = d
|
||||
hw[(calls > 0) & mono] = 1.0
|
||||
called_scores = np.where(g != 0, scores.astype(np.float32), np.float32(np.inf))
|
||||
called_scores.sort(axis=0)
|
||||
return {
|
||||
"n_nocalls": nc,
|
||||
"n_calls": calls,
|
||||
"call_freq": call_freq,
|
||||
"aa_freq": aa_f,
|
||||
"ab_freq": ab_f,
|
||||
"bb_freq": bb_f,
|
||||
"minor_freq": q,
|
||||
"gc50": gs_percentile_columns(called_scores, calls, 50),
|
||||
"gc10": gs_percentile_columns(called_scores, calls, 10),
|
||||
"het_excess": het,
|
||||
"chi_test_p100": hw,
|
||||
"het_excess_simple": np.where(calls > 0, simple, np.nan),
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
# Starting point for review, not a truth (spec QC-024): recalibrate on your own data with
|
||||
# `simurg qc suggest-thresholds`. For LCG/HTS genotyping chips.
|
||||
qc:
|
||||
report:
|
||||
callRate: 0.98
|
||||
logRDev: 0.20
|
||||
stainingGreen: 5
|
||||
stainingRed: 5
|
||||
extensionGreen: 5
|
||||
extensionRed: 5
|
||||
hybGreenHighMed: 1
|
||||
hybGreenMediumLow: 1
|
||||
targetRemovalI: 1
|
||||
restoration: null
|
||||
stringency: 2
|
||||
nonSpecificBindingGreen: 1
|
||||
nonSpecificBindingRed: 1
|
||||
npGreen: 5
|
||||
npRed: 5
|
||||
assayControl: 1
|
||||
bgOffset: 3000
|
||||
@@ -0,0 +1,21 @@
|
||||
# Starting point for review, not a truth (spec QC-024): recalibrate on your own data with
|
||||
# `simurg qc suggest-thresholds`. For EX-format genotyping chips (provisional, OQ-046).
|
||||
qc:
|
||||
report:
|
||||
callRate: 0.98
|
||||
logRDev: 0.20
|
||||
stainingGreen: 5
|
||||
stainingRed: 5
|
||||
extensionGreen: 5
|
||||
extensionRed: 5
|
||||
hybGreenHighMed: 1
|
||||
hybGreenMediumLow: 1
|
||||
targetRemovalI: 1
|
||||
restoration: null
|
||||
stringency: 2
|
||||
nonSpecificBindingGreen: 1
|
||||
nonSpecificBindingRed: 1
|
||||
npGreen: 5
|
||||
npRed: 5
|
||||
assayControl: 1
|
||||
bgOffset: 3000
|
||||
@@ -0,0 +1,20 @@
|
||||
# Alternative example values from the reference documentation (spec QC-024). Starting point for review only.
|
||||
qc:
|
||||
report:
|
||||
callRate: 0.98
|
||||
logRDev: 0.15
|
||||
stainingGreen: 5
|
||||
stainingRed: 5
|
||||
extensionGreen: 5
|
||||
extensionRed: 5
|
||||
hybGreenHighMed: 1.25
|
||||
hybGreenMediumLow: 1.25
|
||||
targetRemovalI: 5
|
||||
restoration: null
|
||||
stringency: 1.5
|
||||
nonSpecificBindingGreen: 5
|
||||
nonSpecificBindingRed: 5
|
||||
npGreen: 5
|
||||
npRed: 5
|
||||
assayControl: 1
|
||||
bgOffset: 3000
|
||||
@@ -0,0 +1,25 @@
|
||||
# Methylation EPIC profile (spec QC-024). Methylation metrics are computed from v2 on.
|
||||
qc:
|
||||
report:
|
||||
stainingGreen: 5
|
||||
stainingRed: 5
|
||||
extensionGreen: 5
|
||||
extensionRed: 5
|
||||
hybGreenHighMed: 1
|
||||
hybGreenMediumLow: 1
|
||||
targetRemovalI: 1
|
||||
targetRemovalII: 1
|
||||
restoration: 0
|
||||
npGreen: 5
|
||||
npRed: 5
|
||||
bisulfiteConversionIGreen: 1
|
||||
bisulfiteConversionIRed: 1
|
||||
bisulfiteConversionIGreenBg: 1
|
||||
bisulfiteConversionIRedBg: 1
|
||||
bisulfiteConversionII: 1
|
||||
bisulfiteConversionIIBg: 1
|
||||
specificityIGreen: 1
|
||||
specificityIRed: 1
|
||||
specificityII: 1
|
||||
specificityIIBg: 1
|
||||
bgOffset: 3000
|
||||
@@ -0,0 +1,25 @@
|
||||
# Methylation MSA profile (spec QC-024). Methylation metrics are computed from v2 on.
|
||||
qc:
|
||||
report:
|
||||
stainingGreen: 5
|
||||
stainingRed: 5
|
||||
extensionGreen: 5
|
||||
extensionRed: 5
|
||||
hybGreenHighMed: 1
|
||||
hybGreenMediumLow: 1
|
||||
targetRemovalI: 1
|
||||
targetRemovalII: 1
|
||||
restoration: 0
|
||||
npGreen: 2.5
|
||||
npRed: 3
|
||||
bisulfiteConversionIGreen: 1
|
||||
bisulfiteConversionIRed: 1
|
||||
bisulfiteConversionIGreenBg: 0.5
|
||||
bisulfiteConversionIRedBg: 0.5
|
||||
bisulfiteConversionII: 0.5
|
||||
bisulfiteConversionIIBg: 0.5
|
||||
specificityIGreen: 1
|
||||
specificityIRed: 1
|
||||
specificityII: 1
|
||||
specificityIIBg: 1
|
||||
bgOffset: 3000
|
||||
@@ -0,0 +1,113 @@
|
||||
"""Identity and relatedness (spec QC-033, QC-064; plan §9.4).
|
||||
|
||||
* KING-robust kinship between every pair of samples
|
||||
[LIT: Manichaikul A. et al. 2010, Bioinformatics 26:2867, eq. for the within-family (robust) estimator]:
|
||||
φ_ij = [N(Aa,Aa) − 2·N(AA,aa)] / [2·min(N_Aa(i), N_Aa(j))] + 1/2 − [N_Aa(i) + N_Aa(j)] / [4·min(N_Aa(i), N_Aa(j))]
|
||||
with every count taken over the sites called in both samples.
|
||||
* Duplicates: genotype concordance over jointly called sites >= 0.99 (plan §9.4) [OURS threshold].
|
||||
* Mendelian errors of child/parent trios from the sample sheet (Parent1/Parent2).
|
||||
|
||||
Counts are formed with matrix products of per-genotype indicator matrices, so a cohort is O(n²·m) arithmetic in a
|
||||
few BLAS calls. Relationship degrees use KING's published cut-offs (> 0.354 duplicate/MZ twin, > 0.177 first
|
||||
degree, > 0.0884 second, > 0.0442 third).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
import numpy as np
|
||||
import numpy.typing as npt
|
||||
|
||||
DEGREE_CUTS = ((0.354, "duplicate"), (0.177, "first"), (0.0884, "second"), (0.0442, "third"))
|
||||
DUPLICATE_CONCORDANCE = 0.99
|
||||
|
||||
|
||||
@dataclass
|
||||
class PairStats:
|
||||
kinship: npt.NDArray[np.float64] # (n, n); NaN on the diagonal or where undefined
|
||||
concordance: npt.NDArray[np.float64] # (n, n)
|
||||
n_sites: npt.NDArray[np.int64] # jointly called sites
|
||||
|
||||
|
||||
def degree(phi: float) -> str:
|
||||
for cut, name in DEGREE_CUTS:
|
||||
if phi > cut:
|
||||
return name
|
||||
return "unrelated"
|
||||
|
||||
|
||||
def select_sites(
|
||||
counts: npt.NDArray[np.int64],
|
||||
usable: npt.NDArray[np.bool_],
|
||||
*,
|
||||
min_call_rate: float = 0.95,
|
||||
min_maf: float = 0.01,
|
||||
max_sites: int = 100_000,
|
||||
) -> npt.NDArray[np.intp]:
|
||||
"""High-quality sites from per-locus (no-call, AA, AB, BB) ``counts``: ``usable`` (autosomal, not
|
||||
intensity-only), cohort call rate and MAF filters, thinned evenly to at most ``max_sites``."""
|
||||
called = counts[:, 1:].sum(axis=1)
|
||||
total = counts.sum(axis=1)
|
||||
with np.errstate(invalid="ignore", divide="ignore"):
|
||||
cr = np.where(total > 0, called / total, 0.0)
|
||||
af = np.where(called > 0, (counts[:, 2] + 2 * counts[:, 3]) / (2.0 * called), np.nan)
|
||||
maf = np.minimum(af, 1 - af)
|
||||
idx = np.flatnonzero(usable & (cr >= min_call_rate) & (maf >= min_maf))
|
||||
if len(idx) > max_sites:
|
||||
idx = idx[np.linspace(0, len(idx) - 1, max_sites).round().astype(np.intp)]
|
||||
return idx
|
||||
|
||||
|
||||
def pair_stats(genotypes: npt.NDArray[np.uint8]) -> PairStats:
|
||||
"""KING-robust kinship and concordance for every pair; ``genotypes`` is (sites × samples), 0 = no call."""
|
||||
g = np.asarray(genotypes)
|
||||
aa = (g == 1).astype(np.float32)
|
||||
ab = (g == 2).astype(np.float32)
|
||||
bb = (g == 3).astype(np.float32)
|
||||
c = (g > 0).astype(np.float32)
|
||||
# float32 products are exact while counts stay below 2**24 sites
|
||||
t = np.float64
|
||||
n_het_het = (ab.T @ ab).astype(t)
|
||||
n_opp = (aa.T @ bb).astype(t)
|
||||
n_opp = n_opp + n_opp.T
|
||||
het_i = (ab.T @ c).astype(t) # [i, j] = hets of i over sites called in j
|
||||
het_j = het_i.T
|
||||
joint = (c.T @ c).astype(t)
|
||||
same = (aa.T @ aa + ab.T @ ab + bb.T @ bb).astype(t)
|
||||
mn = np.minimum(het_i, het_j)
|
||||
with np.errstate(invalid="ignore", divide="ignore"):
|
||||
phi = (n_het_het - 2 * n_opp) / (2 * mn) + 0.5 - (het_i + het_j) / (4 * mn)
|
||||
conc = same / joint
|
||||
phi[mn == 0] = np.nan
|
||||
np.fill_diagonal(phi, np.nan)
|
||||
np.fill_diagonal(conc, np.nan)
|
||||
return PairStats(kinship=phi, concordance=conc, n_sites=joint.astype(np.int64))
|
||||
|
||||
|
||||
def mendel_errors(
|
||||
child: npt.NDArray[np.uint8], father: npt.NDArray[np.uint8] | None, mother: npt.NDArray[np.uint8] | None
|
||||
) -> tuple[int, int]:
|
||||
"""(errors, informative sites) for one child with one or two genotyped parents; autosomal sites only.
|
||||
|
||||
A site is informative when the child and every given parent are called. Errors: the child carries an allele
|
||||
that no parent can transmit (duo: opposite homozygotes; trio: child genotype outside the parental combinations).
|
||||
"""
|
||||
parents = [p for p in (father, mother) if p is not None]
|
||||
if not parents:
|
||||
return 0, 0
|
||||
ok = child > 0
|
||||
for p in parents:
|
||||
ok &= p > 0
|
||||
ch = child[ok].astype(np.int8) - 1 # B-allele count 0/1/2
|
||||
ps = [p[ok].astype(np.int8) - 1 for p in parents]
|
||||
if len(ps) == 1:
|
||||
par = ps[0]
|
||||
err = ((ch == 0) & (par == 2)) | ((ch == 2) & (par == 0))
|
||||
else:
|
||||
f, m = ps
|
||||
# B alleles a parent can transmit: hom-A → {0}, het → {0, 1}, hom-B → {1}
|
||||
lo = f // 2 + m // 2
|
||||
hi = (f + 1) // 2 + (m + 1) // 2
|
||||
err = (ch < lo) | (ch > hi)
|
||||
return int(err.sum()), int(ok.sum())
|
||||
@@ -0,0 +1,117 @@
|
||||
"""Two-axis sample QC status and informational flags (spec QC-030, QC-031, QC-033, QC-034).
|
||||
|
||||
* functional status (PASS/FAIL/NA) — call rate and LogR deviation only;
|
||||
* control status (CLEAR/FLAGGED) — control-probe metrics including the assay control;
|
||||
* flags — informational, never change either status.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
from simurg.qc.thresholds import CONTROL_KEYS, KEY_TO_COLUMN, Thresholds
|
||||
|
||||
CONTAMINATION_FLAG = 0.02 # plan §9.3 [OURS]
|
||||
HET_Z = 3.0
|
||||
ASSAY_CONTROL_MIN = 1.0 # OUT-062
|
||||
|
||||
|
||||
@dataclass
|
||||
class SampleStatus:
|
||||
functional_status: str = "NA"
|
||||
control_status: str = ""
|
||||
control_flag_codes: list[str] = field(default_factory=list)
|
||||
na_metrics: list[str] = field(default_factory=list)
|
||||
flags: list[str] = field(default_factory=list)
|
||||
|
||||
def columns(self) -> dict[str, str]:
|
||||
return {
|
||||
"functional_status": self.functional_status,
|
||||
"control_status": self.control_status,
|
||||
"control_flag_codes": ";".join(self.control_flag_codes),
|
||||
"flags": ";".join(self.flags),
|
||||
}
|
||||
|
||||
|
||||
def _num(v: Any) -> float:
|
||||
try:
|
||||
return float(v)
|
||||
except (TypeError, ValueError):
|
||||
return math.nan
|
||||
|
||||
|
||||
def functional_status(row: Mapping[str, Any] | None, th: Thresholds) -> str:
|
||||
"""QC-030. ``row`` is a sample-summary row; None means no genotyping summary (NA)."""
|
||||
if row is None:
|
||||
return "NA"
|
||||
if row.get("status", "OK") != "OK":
|
||||
return "FAIL"
|
||||
cr, ld = th.get("callRate"), th.get("logRDev")
|
||||
if cr.fails(_num(row.get("autosomal_call_rate"))) or ld.fails(_num(row.get("logr_dev"))):
|
||||
return "FAIL"
|
||||
return "PASS"
|
||||
|
||||
|
||||
def control_status(
|
||||
values: Mapping[str, float] | None, na_metrics: Sequence[str], th: Thresholds
|
||||
) -> tuple[str, list[str]]:
|
||||
"""QC-031: FLAGGED iff an enabled, evaluable control metric is out of range. Codes in QC-004 order."""
|
||||
if values is None:
|
||||
return "", []
|
||||
codes = []
|
||||
for key in CONTROL_KEYS:
|
||||
col = KEY_TO_COLUMN[key]
|
||||
t = th.get(key)
|
||||
if not t.enabled or col not in values or col in na_metrics:
|
||||
continue
|
||||
if t.fails(_num(values[col])):
|
||||
codes.append(key)
|
||||
return ("FLAGGED" if codes else "CLEAR"), codes
|
||||
|
||||
|
||||
def het_zscores(het: Sequence[float]) -> list[float]:
|
||||
"""z-scores of heterozygosity within the batch (NaN where undefined or with < 3 finite values)."""
|
||||
a = np.array([_num(h) for h in het], dtype=np.float64)
|
||||
ok = np.isfinite(a)
|
||||
if ok.sum() < 3:
|
||||
return [math.nan] * len(a)
|
||||
mu, sd = float(a[ok].mean()), float(a[ok].std(ddof=1))
|
||||
if sd == 0:
|
||||
return [0.0 if o else math.nan for o in ok]
|
||||
return [float((x - mu) / sd) if o else math.nan for x, o in zip(a, ok, strict=True)]
|
||||
|
||||
|
||||
def sample_flags(
|
||||
row: Mapping[str, Any],
|
||||
th: Thresholds,
|
||||
*,
|
||||
het_z: float = math.nan,
|
||||
probe_columns: Sequence[str] = (),
|
||||
duplicates: Sequence[str] = (),
|
||||
unexpected_relatedness: bool = False,
|
||||
) -> list[str]:
|
||||
"""QC-033 flags in a fixed order."""
|
||||
flags: list[str] = []
|
||||
ok = row.get("status", "OK") == "OK"
|
||||
if ok and th.get("callRate").fails(_num(row.get("autosomal_call_rate"))):
|
||||
flags.append("low_call_rate")
|
||||
if ok and th.get("logRDev").fails(_num(row.get("logr_dev"))):
|
||||
flags.append("high_lrr_sd")
|
||||
if row.get("sex_check") == "mismatch":
|
||||
flags.append("sex_mismatch")
|
||||
if math.isfinite(het_z) and abs(het_z) > HET_Z:
|
||||
flags.append("het_outlier")
|
||||
c = _num(row.get("contamination"))
|
||||
if math.isfinite(c) and c > CONTAMINATION_FLAG:
|
||||
flags.append("contaminated")
|
||||
if ok and any(_num(row.get(p)) < ASSAY_CONTROL_MIN for p in probe_columns):
|
||||
flags.append("assay_control_fail")
|
||||
flags.extend(f"duplicate_of:{d}" for d in duplicates)
|
||||
if unexpected_relatedness:
|
||||
flags.append("unexpected_relatedness")
|
||||
return flags
|
||||
@@ -0,0 +1,98 @@
|
||||
"""Data-driven threshold suggestion (spec QC-026, CLI-027) [OURS].
|
||||
|
||||
For each metric: robust centre m = median and spread s = 1.4826·MAD over samples, on the log scale for ratio
|
||||
metrics (every control metric and the assay-control norm R); suggestion exp(m − k·s) for ``below`` keys and
|
||||
exp(m + k·s) for ``above`` keys (linear scale for call rate and LogR deviation). At least ``min_samples`` finite
|
||||
values are required; otherwise the key is left unset with a note.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
|
||||
import numpy as np
|
||||
import numpy.typing as npt
|
||||
|
||||
from simurg.qc.thresholds import ABOVE_KEYS, Thresholds
|
||||
|
||||
LINEAR_KEYS = {"callRate", "logRDev"}
|
||||
MAD_SCALE = 1.4826
|
||||
|
||||
|
||||
@dataclass
|
||||
class Suggestion:
|
||||
key: str
|
||||
n: int
|
||||
centre: float # on the analysis scale (log for ratio metrics)
|
||||
spread: float
|
||||
value: float | None
|
||||
pct_flagged_current: float
|
||||
pct_flagged_suggested: float
|
||||
note: str = ""
|
||||
|
||||
|
||||
def _flagged(vals: npt.NDArray[np.float64], thr: float | None, above: bool) -> float:
|
||||
if thr is None or len(vals) == 0:
|
||||
return math.nan
|
||||
bad = vals > thr if above else vals < thr
|
||||
return float(100.0 * bad.mean())
|
||||
|
||||
|
||||
def suggest(
|
||||
values: Mapping[str, Sequence[float]],
|
||||
current: Thresholds,
|
||||
*,
|
||||
k: float = 4.0,
|
||||
min_samples: int = 30,
|
||||
) -> list[Suggestion]:
|
||||
out = []
|
||||
for key, raw in values.items():
|
||||
v = np.array([float(x) for x in raw], dtype=np.float64)
|
||||
v = v[np.isfinite(v)]
|
||||
above = key in ABOVE_KEYS
|
||||
linear = key in LINEAR_KEYS
|
||||
cur = current.get(key).value
|
||||
if len(v) < min_samples:
|
||||
out.append(
|
||||
Suggestion(
|
||||
key,
|
||||
len(v),
|
||||
math.nan,
|
||||
math.nan,
|
||||
None,
|
||||
_flagged(v, cur, above),
|
||||
math.nan,
|
||||
f"fewer than {min_samples} samples",
|
||||
)
|
||||
)
|
||||
continue
|
||||
x = v if linear else np.log(np.maximum(v, 1e-12))
|
||||
m = float(np.median(x))
|
||||
s = MAD_SCALE * float(np.median(np.abs(x - m)))
|
||||
t = m + k * s if above else m - k * s
|
||||
val = t if linear else math.exp(t)
|
||||
if key == "callRate":
|
||||
val = min(max(val, 0.0), 1.0)
|
||||
val = max(val, 0.0)
|
||||
out.append(Suggestion(key, len(v), m, s, val, _flagged(v, cur, above), _flagged(v, val, above)))
|
||||
return out
|
||||
|
||||
|
||||
def render_yaml(sugs: Sequence[Suggestion], k: float) -> str:
|
||||
lines = [
|
||||
"# Suggested QC thresholds (simurg qc suggest-thresholds, spec QC-026).",
|
||||
f"# centre = median, spread = {MAD_SCALE}*MAD (log scale for ratio metrics), k = {k}.",
|
||||
"# Review before use: these are starting points computed from your own data.",
|
||||
"qc:",
|
||||
" report:",
|
||||
]
|
||||
for s in sugs:
|
||||
info = (
|
||||
f"n={s.n} m={s.centre:.4g} s={s.spread:.4g} flagged: current={s.pct_flagged_current:.2f}% "
|
||||
f"suggested={s.pct_flagged_suggested:.2f}%"
|
||||
)
|
||||
val = "null" if s.value is None else f"{s.value:.6g}"
|
||||
lines.append(f" {s.key}: {val} # {s.note or info}")
|
||||
return "\n".join(lines) + "\n"
|
||||
@@ -0,0 +1,190 @@
|
||||
"""QC thresholds: YAML schema, validation, profiles and precedence (spec QC-020 … QC-025).
|
||||
|
||||
Precedence, highest first: CLI > ``--config`` YAML > profile > built-in. Every effective value keeps its source.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from importlib import resources
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
|
||||
from simurg.core.errors import SimurgError
|
||||
from simurg.qc.controls import DEFAULT_BG_OFFSET, METRICS
|
||||
|
||||
FUNCTIONAL_KEYS = ("callRate", "logRDev")
|
||||
CONTROL_KEYS = (*(m.key for m in METRICS), "assayControl")
|
||||
KNOWN_KEYS = (*FUNCTIONAL_KEYS, *CONTROL_KEYS, "bgOffset")
|
||||
ALIASES = {"tgaControl": "assayControl", "call_rate_min": "callRate", "logr_dev_max": "logRDev"}
|
||||
ABOVE_KEYS = {"logRDev"} # flagged when value > threshold; every other key is flagged when value < threshold
|
||||
BUILTIN: dict[str, float | None] = {"callRate": 0.98, "logRDev": 0.20} # QC-023 [EMU]; every other key unset
|
||||
PROFILES = ("genotyping", "genotyping_ex", "genotyping_example", "methylation_epic", "methylation_msa")
|
||||
KEY_TO_COLUMN = {m.key: m.column for m in METRICS} | {"assayControl": "assay_control_norm_r"}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Threshold:
|
||||
value: float | None # None: check disabled (QC-022)
|
||||
direction: str # "below" | "above"
|
||||
source: str # builtin | profile:<name> | config | cli
|
||||
|
||||
@property
|
||||
def enabled(self) -> bool:
|
||||
return self.value is not None
|
||||
|
||||
def fails(self, v: float) -> bool:
|
||||
"""True when ``v`` is out of range. Equality passes (QC-020); NaN signal fails (QC-008)."""
|
||||
if self.value is None:
|
||||
return False
|
||||
if v is None or math.isnan(v):
|
||||
return True
|
||||
return v < self.value if self.direction == "below" else v > self.value
|
||||
|
||||
|
||||
@dataclass
|
||||
class Thresholds:
|
||||
items: dict[str, Threshold] = field(default_factory=dict)
|
||||
profile: str = ""
|
||||
|
||||
def get(self, key: str) -> Threshold:
|
||||
return self.items.get(key, Threshold(None, "above" if key in ABOVE_KEYS else "below", "unset"))
|
||||
|
||||
@property
|
||||
def bg_offset(self) -> float:
|
||||
t = self.items.get("bgOffset")
|
||||
return float(t.value) if t is not None and t.value is not None else DEFAULT_BG_OFFSET
|
||||
|
||||
def as_records(self) -> list[dict[str, Any]]:
|
||||
return [
|
||||
{"key": k, "value": t.value, "direction": t.direction, "source": t.source}
|
||||
for k, t in sorted(self.items.items(), key=lambda kv: KNOWN_KEYS.index(kv[0]))
|
||||
]
|
||||
|
||||
|
||||
def _canon_key(key: str) -> str:
|
||||
key = ALIASES.get(key, key)
|
||||
if key not in KNOWN_KEYS:
|
||||
raise SimurgError(
|
||||
"E_QC_THRESHOLD_KEY", f"unknown QC threshold key {key!r}; valid keys: {', '.join(KNOWN_KEYS)}"
|
||||
)
|
||||
return key
|
||||
|
||||
|
||||
def _parse_value(key: str, raw: Any, source: str) -> Threshold:
|
||||
direction = "above" if key in ABOVE_KEYS else "below"
|
||||
if isinstance(raw, Mapping):
|
||||
direction = str(raw.get("direction", direction))
|
||||
if direction not in ("below", "above"):
|
||||
raise SimurgError("E_QC_THRESHOLD_VALUE", f"{key}: direction must be 'below' or 'above', got {direction!r}")
|
||||
raw = raw.get("threshold")
|
||||
if raw is None or (isinstance(raw, str) and raw.strip().lower() in ("", "null", "none", "~")):
|
||||
return Threshold(None, direction, source)
|
||||
if isinstance(raw, bool):
|
||||
raise SimurgError("E_QC_THRESHOLD_VALUE", f"{key}: value must be numeric or null, got {raw!r}")
|
||||
try:
|
||||
v = float(raw)
|
||||
except (TypeError, ValueError):
|
||||
raise SimurgError("E_QC_THRESHOLD_VALUE", f"{key}: value must be numeric or null, got {raw!r}") from None
|
||||
if not math.isfinite(v):
|
||||
raise SimurgError("E_QC_THRESHOLD_VALUE", f"{key}: value must be finite, got {raw!r}")
|
||||
if key == "callRate" and not 0.0 <= v <= 1.0:
|
||||
raise SimurgError("E_QC_THRESHOLD_RANGE", f"callRate must be within [0, 1], got {v}")
|
||||
if v < 0:
|
||||
raise SimurgError("E_QC_THRESHOLD_RANGE", f"{key} must be >= 0, got {v}")
|
||||
return Threshold(v, direction, source)
|
||||
|
||||
|
||||
def parse_report_mapping(doc: Any, where: str) -> tuple[dict[str, Any], str | None]:
|
||||
"""``qc: {profile, report: {...}}`` (or ``QC:``/``Report:``) → (report mapping, profile)."""
|
||||
if not isinstance(doc, Mapping):
|
||||
raise SimurgError("E_QC_CONFIG_STRUCTURE", f"{where}: expected a 'qc:' mapping with a 'report:' section")
|
||||
top = doc.get("qc", doc.get("QC"))
|
||||
if not isinstance(top, Mapping):
|
||||
raise SimurgError("E_QC_CONFIG_STRUCTURE", f"{where}: missing top-level 'qc:' (or 'QC:') mapping")
|
||||
report = top.get("report", top.get("Report"))
|
||||
if report is None:
|
||||
report = {}
|
||||
if not isinstance(report, Mapping):
|
||||
raise SimurgError("E_QC_CONFIG_STRUCTURE", f"{where}: 'report:' (or 'Report:') must be a mapping")
|
||||
profile = top.get("profile")
|
||||
return dict(report), (str(profile) if profile else None)
|
||||
|
||||
|
||||
def load_config(path: Path) -> tuple[dict[str, Any], str | None]:
|
||||
"""Read and structurally validate a threshold YAML (QC-021)."""
|
||||
p = Path(path)
|
||||
if not p.exists():
|
||||
raise SimurgError("E_QC_CONFIG_MISSING", f"QC config file was provided but does not exist: {p}", path=str(p))
|
||||
if p.suffix.lower() not in (".yaml", ".yml"):
|
||||
raise SimurgError("E_QC_CONFIG_EXT", f"QC config file must have a .yaml or .yml extension: {p}", path=str(p))
|
||||
try:
|
||||
doc = yaml.safe_load(p.read_text(encoding="utf-8-sig"))
|
||||
except yaml.YAMLError as e:
|
||||
raise SimurgError("E_QC_CONFIG_PARSE", f"QC config file is not valid YAML: {p}: {e}", path=str(p)) from None
|
||||
return parse_report_mapping(doc, str(p))
|
||||
|
||||
|
||||
def profile_values(name: str) -> dict[str, Any]:
|
||||
"""Shipped profile (QC-024); ``+ffpe`` suffix enables restoration: 1."""
|
||||
base, _, suffix = name.partition("+")
|
||||
if base not in PROFILES or suffix not in ("", "ffpe"):
|
||||
raise SimurgError("E_QC_PROFILE", f"unknown QC profile {name!r}; choose from {', '.join(PROFILES)} or auto")
|
||||
text = resources.files("simurg.qc").joinpath("profiles", f"{base}.yaml").read_text(encoding="utf-8")
|
||||
report, _ = parse_report_mapping(yaml.safe_load(text), f"profile {base}")
|
||||
if suffix == "ffpe":
|
||||
report["restoration"] = 1
|
||||
return report
|
||||
|
||||
|
||||
def auto_profile(array_type: str, assay_format: str = "", manifest_name: str = "") -> str:
|
||||
"""QC-025."""
|
||||
if array_type == "methylation":
|
||||
return "methylation_msa" if "msa" in manifest_name.lower() else "methylation_epic"
|
||||
return "genotyping_ex" if assay_format.strip().upper() == "EX" else "genotyping"
|
||||
|
||||
|
||||
def parse_cli_pairs(pairs: list[str]) -> dict[str, Any]:
|
||||
out: dict[str, Any] = {}
|
||||
for item in pairs:
|
||||
key, sep, val = item.partition("=")
|
||||
if not sep:
|
||||
raise SimurgError("E_QC_THRESHOLD_VALUE", f"--threshold expects KEY=VALUE, got {item!r}")
|
||||
out[key.strip()] = val.strip()
|
||||
return out
|
||||
|
||||
|
||||
def resolve_thresholds(
|
||||
*,
|
||||
config: Path | None = None,
|
||||
profile: str | None = None,
|
||||
cli: Mapping[str, Any] | None = None,
|
||||
array_type: str = "genotyping",
|
||||
assay_format: str = "",
|
||||
manifest_name: str = "",
|
||||
) -> Thresholds:
|
||||
"""Merge built-in < profile < config < CLI (QC-023). A ``profile:`` in the YAML is used when no CLI profile."""
|
||||
layers: list[tuple[str, Mapping[str, Any]]] = [("builtin", BUILTIN)]
|
||||
cfg_report: dict[str, Any] = {}
|
||||
cfg_profile = None
|
||||
if config is not None:
|
||||
cfg_report, cfg_profile = load_config(config)
|
||||
chosen = profile or cfg_profile
|
||||
if chosen == "auto":
|
||||
chosen = auto_profile(array_type, assay_format, manifest_name)
|
||||
if chosen:
|
||||
layers.append((f"profile:{chosen}", profile_values(chosen)))
|
||||
if cfg_report:
|
||||
layers.append(("config", cfg_report))
|
||||
if cli:
|
||||
layers.append(("cli", cli))
|
||||
th = Thresholds(profile=chosen or "")
|
||||
for source, mapping in layers:
|
||||
for raw_key, raw in mapping.items():
|
||||
key = _canon_key(str(raw_key))
|
||||
th.items[key] = _parse_value(key, raw, source)
|
||||
return th
|
||||
Reference in new issue
Block a user