PGx CN: pgx copy-number train|call (CLI-031/032, PCN-001..017, IN-050), region BED v1, demo identities (K-028)
- simurg/pgx/cn.py: per-region CN mixture, mu_c = delta + gamma*log2((c+1)/3), free lower CN-0 component, Hardy-Weinberg-constrained weights over haplotype copy numbers, EM from a grid of label starts. - pipeline/pgx_cn.py: model file (.simurg-pgxcn), batch rules (PCN-004), per-sample CN VCF + tbi, batch and sample summaries. - The GSA v4 PGx demo arrays are 30 CEPH/1000G individuals (scripts/pgx/demo36_identity.tsv); against NYGC 30x SV RD_CN 224/240 region calls agree (GSTM1, UGT2B17 30/30), in-sample (docs/validation/V2_pgx_cn.md).
This commit is contained in:
kemal committed
1 parent
518bfbe059
commit
c3e0681b6e
14 files changed
+1306
-1
No files matched your search
@@ -0,0 +1,510 @@
|
||||
"""``simurg pgx copy-number train|call`` — targeted PGx copy number (spec PCN-001 … PCN-017, CLI-031/032, IN-050).
|
||||
|
||||
I/O and orchestration only; the numerics live in ``simurg.pgx.cn``.
|
||||
|
||||
Deviations (K-028): the per-probe reference is the training cohort's median, stored in the model (not the calling
|
||||
batch's median, which is not CN 2 in regions with common deletions such as GSTM1); no GC correction step (the region
|
||||
median over many probes plus the sample scale already removes most of it); GSTT1 needs alt-contig probe coordinates,
|
||||
which the GSA v4 PGx GRCh38 manifest carries (160 probes on `chr22_KI270879v1_alt`); manifests without them give
|
||||
`NoCall` for GSTT1.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import math
|
||||
import zipfile
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime
|
||||
from importlib import resources
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import numpy.typing as npt
|
||||
import pyarrow as pa
|
||||
import pyarrow.parquet as pq
|
||||
|
||||
from simurg import __version__
|
||||
from simurg.core.errors import ExitCode, SimurgError
|
||||
from simurg.core.log import Reporter
|
||||
from simurg.formats.bpm import read_bpm
|
||||
from simurg.pgx.cn import CN_MAX, Region, RegionFit, call_cn, fit_mixture, parse_regions
|
||||
|
||||
MODEL_SCHEMA = 1
|
||||
REGIONS_FILE = "pgx_cn_regions.GRCh38.v1.bed"
|
||||
N_SCALE_PROBES = 20_000
|
||||
MIN_REGION_PROBES = 10 # PCN-008
|
||||
MIN_FINITE_FRAC = 0.5 # PCN-002: fewer finite region values → NoCall [OURS]
|
||||
F32 = npt.NDArray[np.float32]
|
||||
|
||||
|
||||
def default_regions() -> tuple[str, str]:
|
||||
"""(file name, text) of the packaged region BED (PCN-001)."""
|
||||
return REGIONS_FILE, resources.files("simurg.pgx.data").joinpath(REGIONS_FILE).read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def _core(chrom: str) -> str:
|
||||
c = chrom[3:] if chrom.lower().startswith("chr") else chrom
|
||||
return "X" if c == "XY" else c
|
||||
|
||||
|
||||
@dataclass
|
||||
class PgxCnModel:
|
||||
header: dict[str, Any]
|
||||
regions: list[Region]
|
||||
regions_text: str
|
||||
probe_idx: npt.NDArray[np.int64] # manifest indices of all probes used (region probes, then scale probes)
|
||||
probe_region: list[str] # region name, "" for scale probes
|
||||
probe_ref: F32 # training reference (median sample-centred LRR)
|
||||
fits: dict[str, RegionFit] = field(default_factory=dict)
|
||||
|
||||
def region_sel(self, name: str) -> npt.NDArray[np.int64]:
|
||||
return np.flatnonzero(np.asarray(self.probe_region) == name)
|
||||
|
||||
|
||||
def region_probes(
|
||||
regions: list[Region], chroms: list[str], pos: npt.NDArray[np.int64], masked: set[int]
|
||||
) -> dict[str, npt.NDArray[np.int64]]:
|
||||
"""Manifest indices inside each region (1-based position in (start0, end]), mask removed (PCN-008)."""
|
||||
core = np.array([_core(c) for c in chroms])
|
||||
out = {}
|
||||
for r in regions:
|
||||
sel = np.flatnonzero((core == _core(r.chrom)) & (pos > r.start0) & (pos <= r.end))
|
||||
out[r.name] = np.array([i for i in sel.tolist() if i not in masked], dtype=np.int64)
|
||||
return out
|
||||
|
||||
|
||||
def scale_probes(
|
||||
chroms: list[str], pos: npt.NDArray[np.int64], exclude: npt.NDArray[np.int64], n: int = N_SCALE_PROBES
|
||||
) -> npt.NDArray[np.int64]:
|
||||
"""Seeded random autosomal non-target probes for the per-sample scale (PCN-009 step 2)."""
|
||||
core = np.array([_core(c) for c in chroms])
|
||||
ok = np.isin(core, [str(i) for i in range(1, 23)]) & (pos > 0)
|
||||
ok[exclude] = False
|
||||
cand = np.flatnonzero(ok)
|
||||
rng = np.random.default_rng(20260929)
|
||||
return np.sort(rng.choice(cand, min(n, len(cand)), replace=False)).astype(np.int64)
|
||||
|
||||
|
||||
def _centred(lrr: npt.NDArray[np.float32], auto_idx: npt.NDArray[np.int64]) -> npt.NDArray[np.float64]:
|
||||
x = lrr.astype(np.float64)
|
||||
x[~np.isfinite(x)] = np.nan
|
||||
med = float(np.nanmedian(x[auto_idx])) if np.isfinite(x[auto_idx]).any() else 0.0
|
||||
return x - med
|
||||
|
||||
|
||||
def region_values(d: npt.NDArray[np.float64], model: PgxCnModel) -> dict[str, tuple[float, int]]:
|
||||
"""Per region: (x, n finite probes). ``d`` = centred LRR at ``model.probe_idx`` minus the reference."""
|
||||
scale_sel = model.region_sel("")
|
||||
sc = float(np.nanmedian(d[scale_sel])) if np.isfinite(d[scale_sel]).any() else 0.0
|
||||
out = {}
|
||||
for r in model.regions:
|
||||
sel = model.region_sel(r.name)
|
||||
v = d[sel]
|
||||
nf = int(np.isfinite(v).sum())
|
||||
out[r.name] = (float(np.nanmedian(v)) - sc if nf else math.nan, nf)
|
||||
return out
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------ model file
|
||||
def write_model(path: Path, m: PgxCnModel) -> None:
|
||||
table = pa.table({"index": m.probe_idx, "region": m.probe_region, "ref": m.probe_ref,
|
||||
"name": m.header.pop("_names")}) # fmt: skip
|
||||
buf = io.BytesIO()
|
||||
pq.write_table(table, buf, compression="zstd")
|
||||
mixture = {name: {"delta": f.delta, "gamma": f.gamma, "sigma": f.sigma, "weights": f.weights, "n": f.n,
|
||||
"cn0_shift": f.cn0_shift, "cn0_sigma": f.cn0_sigma} for name, f in m.fits.items()} # fmt: skip
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.with_suffix(path.suffix + ".part")
|
||||
with zipfile.ZipFile(tmp, "w", compression=zipfile.ZIP_STORED) as z:
|
||||
z.writestr("header.json", json.dumps(m.header, indent=1))
|
||||
z.writestr("probes.parquet", buf.getvalue())
|
||||
z.writestr("regions.bed", m.regions_text)
|
||||
z.writestr("mixture.json", json.dumps(mixture, indent=1))
|
||||
tmp.replace(path)
|
||||
|
||||
|
||||
def read_model(path: Path) -> PgxCnModel:
|
||||
"""IN-050."""
|
||||
with zipfile.ZipFile(path) as z:
|
||||
header = json.loads(z.read("header.json"))
|
||||
table = pq.read_table(io.BytesIO(z.read("probes.parquet")))
|
||||
regions_text = z.read("regions.bed").decode()
|
||||
mixture = json.loads(z.read("mixture.json"))
|
||||
if int(header.get("schema_version", 0)) < MODEL_SCHEMA:
|
||||
raise SimurgError("E_MODEL_VERSION", f"{path}: PGx CN model schema {header.get('schema_version')} is too old",
|
||||
hint="retrain it with simurg pgx copy-number train") # fmt: skip
|
||||
fits = {
|
||||
k: RegionFit(
|
||||
v["delta"],
|
||||
v["gamma"],
|
||||
v["sigma"],
|
||||
list(v["weights"]),
|
||||
int(v["n"]),
|
||||
v.get("cn0_shift", 0.0),
|
||||
v.get("cn0_sigma", 0.0),
|
||||
)
|
||||
for k, v in mixture.items()
|
||||
}
|
||||
return PgxCnModel(header, parse_regions(regions_text), regions_text,
|
||||
table.column("index").to_numpy().astype(np.int64), table.column("region").to_pylist(),
|
||||
table.column("ref").to_numpy().astype(np.float32), fits) # fmt: skip
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------ train
|
||||
@dataclass
|
||||
class PgxTrainOptions:
|
||||
gtcs: Path
|
||||
bpm: Path
|
||||
fasta: Path
|
||||
out: Path
|
||||
mask: Path | None = None
|
||||
regions: Path | None = None
|
||||
summary: Path | None = None
|
||||
platform: str = "auto"
|
||||
min_samples: int = 96
|
||||
min_passing: int = 90
|
||||
max_samples: int = 3000
|
||||
lrr_sd_max: float = 0.20
|
||||
call_rate_min: float = 0.99
|
||||
assay_control_min: float = 1.0
|
||||
force: bool = False
|
||||
argv: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
def read_mask_tsv(path: Path | None) -> tuple[set[str], str]:
|
||||
"""IN-052: ``probe_name<TAB>reason``; returns (names, sha256)."""
|
||||
if path is None:
|
||||
return set(), ""
|
||||
data = path.read_bytes()
|
||||
names = set()
|
||||
for ln in data.decode("utf-8").splitlines():
|
||||
if ln.strip() and not ln.startswith("#"):
|
||||
names.add(ln.split("\t")[0].strip())
|
||||
return names, hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def _summary(path: Path | None) -> dict[str, dict[str, str]]:
|
||||
if path is None or not path.exists():
|
||||
return {}
|
||||
lines = path.read_text(encoding="utf-8").splitlines()
|
||||
head = lines[0].split("\t")
|
||||
return {r[0]: dict(zip(head, r, strict=False)) for r in (ln.split("\t") for ln in lines[1:] if ln)}
|
||||
|
||||
|
||||
def _assay_control(row: dict[str, str]) -> float:
|
||||
for k, v in row.items():
|
||||
if k.endswith("_norm_r") and k.startswith("TGA_Ctrl"):
|
||||
try:
|
||||
return float(v)
|
||||
except ValueError:
|
||||
return math.nan
|
||||
return math.nan
|
||||
|
||||
|
||||
def _probe_set_sha(names: list[str], chroms: list[str], pos: npt.NDArray[np.int64]) -> str:
|
||||
text = "\n".join(f"{n}\t{c}\t{p}" for n, c, p in zip(names, chroms, pos.tolist(), strict=True))
|
||||
return hashlib.sha256(text.encode()).hexdigest()
|
||||
|
||||
|
||||
def run_train(o: PgxTrainOptions, rep: Reporter) -> int:
|
||||
"""``simurg pgx copy-number train`` (CLI-032, PCN-005)."""
|
||||
from simurg.core.atomic import check_outputs_absent
|
||||
from simurg.formats.discovery import discover_gtcs
|
||||
from simurg.formats.fasta import Genome
|
||||
from simurg.formats.gtc import read_gtc
|
||||
from simurg.pipeline.qc import write_tsv
|
||||
|
||||
man = read_bpm(o.bpm)
|
||||
out_model = o.out / f"{o.bpm.stem}.simurg-pgxcn"
|
||||
check_outputs_absent([out_model, o.out / "train_exclusions.tsv"], force=o.force)
|
||||
genome = Genome(o.fasta)
|
||||
if genome.build not in ("GRCh38", "unknown", ""):
|
||||
raise SimurgError("E_BUILD_MISMATCH", f"PGx CN supports GRCh38 only (FASTA is {genome.build})") # PCN-015
|
||||
reg_name, reg_text = (o.regions.name, o.regions.read_text()) if o.regions else default_regions()
|
||||
regions = parse_regions(reg_text)
|
||||
mask_path = o.mask or (
|
||||
o.bpm.with_name(o.bpm.stem + ".mask.tsv") if (o.bpm.with_name(o.bpm.stem + ".mask.tsv")).exists() else None
|
||||
)
|
||||
mask_names, mask_sha = read_mask_tsv(mask_path)
|
||||
masked = {i for i, n in enumerate(man.names) if n in mask_names}
|
||||
pos = man.map_infos.astype(np.int64)
|
||||
rp = region_probes(regions, list(man.chroms), pos, masked)
|
||||
for r in regions:
|
||||
if len(rp[r.name]) < MIN_REGION_PROBES:
|
||||
rep.warn("W_PGX_REGION_FEW_PROBES", f"region {r.name}: {len(rp[r.name])} unmasked probe(s) on "
|
||||
f"{man.manifest_name}; it will be NoCall") # fmt: skip
|
||||
reg_idx = np.concatenate([rp[r.name] for r in regions]) if regions else np.zeros(0, np.int64)
|
||||
sc_idx = scale_probes(list(man.chroms), pos, reg_idx)
|
||||
probe_idx = np.concatenate([reg_idx, sc_idx])
|
||||
probe_region = [r.name for r in regions for _ in range(len(rp[r.name]))] + [""] * len(sc_idx)
|
||||
gtcs = discover_gtcs(o.gtcs)
|
||||
keys = sorted(gtcs)
|
||||
if len(keys) > o.max_samples:
|
||||
rng = np.random.default_rng(42)
|
||||
keys = sorted(rng.choice(keys, o.max_samples, replace=False).tolist())
|
||||
rep.info(f"training on a seeded random subset of {o.max_samples} of {len(gtcs)} samples (PCN-005)")
|
||||
summ = _summary(o.summary or (o.gtcs / "sample_summary.tsv"))
|
||||
rows = np.full((len(keys), len(probe_idx)), np.nan, dtype=np.float32)
|
||||
exclusions: list[dict[str, str]] = []
|
||||
passing = np.zeros(len(keys), dtype=np.bool_)
|
||||
cluster_files = set()
|
||||
for i, k in enumerate(keys):
|
||||
g = read_gtc(gtcs[k])
|
||||
if len(g.logr_ratios) != man.num_loci:
|
||||
raise SimurgError("E_MODEL_MANIFEST_MISMATCH", f"{gtcs[k]} has {len(g.logr_ratios)} loci, the manifest "
|
||||
f"{man.num_loci}") # fmt: skip
|
||||
cluster_files.add(g.cluster_file)
|
||||
rows[i] = _centred(g.logr_ratios, sc_idx)[probe_idx]
|
||||
reasons = []
|
||||
if not g.logr_dev <= o.lrr_sd_max:
|
||||
reasons.append(f"logr_dev {g.logr_dev:.3f} > {o.lrr_sd_max}")
|
||||
if not g.call_rate >= o.call_rate_min:
|
||||
reasons.append(f"call_rate {g.call_rate:.4f} < {o.call_rate_min}")
|
||||
ac = _assay_control(summ.get(k, {}))
|
||||
if math.isfinite(ac) and ac < o.assay_control_min:
|
||||
reasons.append(f"assay_control_norm_r {ac:.3f} < {o.assay_control_min}")
|
||||
if reasons:
|
||||
exclusions.append({"sample_id": k, "reason": "; ".join(reasons)})
|
||||
else:
|
||||
passing[i] = True
|
||||
n_pass = int(passing.sum())
|
||||
if len(keys) < o.min_samples or n_pass < o.min_passing:
|
||||
raise SimurgError("E_PGX_TRAIN_TOO_SMALL", f"{n_pass} of {len(keys)} sample(s) passed QC; training requires "
|
||||
f"at least {o.min_samples} samples of which {o.min_passing} pass",
|
||||
hint="lower --min-samples/--min-passing only for research use") # fmt: skip
|
||||
ref = np.nanmedian(rows[passing], axis=0)
|
||||
d = rows - ref
|
||||
sc_sel = np.flatnonzero(np.asarray(probe_region) == "")
|
||||
sc = np.nanmedian(d[:, sc_sel], axis=1)
|
||||
mad = float(np.nanmedian(np.abs(sc[passing] - np.nanmedian(sc[passing])))) * 1.4826 or 1e-9
|
||||
z = np.abs(sc - np.nanmedian(sc[passing])) / mad
|
||||
use = passing & (z <= 5) # PCN-009: outlier samples out of the reference
|
||||
for i in np.flatnonzero(passing & ~use).tolist():
|
||||
exclusions.append({"sample_id": keys[i], "reason": f"genome-wide NR outlier (|z| = {z[i]:.1f} > 5)"})
|
||||
if (~use & passing).any():
|
||||
ref = np.nanmedian(rows[use], axis=0)
|
||||
d = rows - ref
|
||||
model = PgxCnModel({}, regions, reg_text, probe_idx, probe_region, ref.astype(np.float32))
|
||||
values = {}
|
||||
for r in regions:
|
||||
if len(rp[r.name]) < MIN_REGION_PROBES:
|
||||
continue
|
||||
values[r.name] = np.array([region_values(d[i], model)[r.name][0] for i in np.flatnonzero(use)])
|
||||
model.fits = fit_mixture(values)
|
||||
csv_fmt = "unknown" if o.platform == "auto" else o.platform
|
||||
model.header = {
|
||||
"schema": "simurg.pgx_cn_model", "schema_version": MODEL_SCHEMA, "software_version": __version__,
|
||||
"manifest_name": man.manifest_name, "manifest_revision": getattr(man, "version", None),
|
||||
"probe_set_sha256": _probe_set_sha(man.names, list(man.chroms), pos),
|
||||
"n_probes": man.num_loci,
|
||||
"cluster_file_name": sorted(cluster_files)[0] if len(cluster_files) == 1 else sorted(cluster_files),
|
||||
"genome_build": "GRCh38", "fasta_contigs_sha256": genome.sha256, "platform": csv_fmt,
|
||||
"mask_sha256": mask_sha, "regions_bed_version": reg_name, "n_training_samples": int(use.sum()),
|
||||
"training_exclusions": len(exclusions), "created": datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ"),
|
||||
"regions_few_probes": [r.name for r in regions if len(rp[r.name]) < MIN_REGION_PROBES],
|
||||
"model": "mu_c = delta_r + gamma_r * log2((c + 1) / 3), c = 0..5 (simurg.pgx.cn)",
|
||||
"_names": [man.names[i] for i in probe_idx.tolist()],
|
||||
} # fmt: skip
|
||||
o.out.mkdir(parents=True, exist_ok=True)
|
||||
write_model(out_model, model)
|
||||
write_tsv(o.out / "train_exclusions.tsv", ["sample_id", "reason"], exclusions)
|
||||
for name, f in model.fits.items():
|
||||
rep.info(f"{name}: CN2 at {f.delta:+.3f}, gamma {f.gamma:.3f}, sd {f.sigma:.3f}, "
|
||||
f"weights {' '.join(f'{w:.2f}' for w in f.weights)}") # fmt: skip
|
||||
rep.info(f"PGx CN model from {int(use.sum())} sample(s) written to {out_model}")
|
||||
return ExitCode.OK
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------ call
|
||||
@dataclass
|
||||
class PgxCallOptions:
|
||||
gtcs: Path
|
||||
model: Path
|
||||
fasta: Path
|
||||
out: Path
|
||||
min_batch: int = 24
|
||||
min_passing: int = 22
|
||||
recommended_passing: int = 96
|
||||
lrr_sd_max: float = 0.20
|
||||
cluster_mismatch: str = "error"
|
||||
bgzip: bool = True
|
||||
force: bool = False
|
||||
argv: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
def _vcf_header(sample: str, contigs: list[tuple[str, int]], meta: dict[str, str]) -> str:
|
||||
h = ["##fileformat=VCFv4.3", f"##source=SimurgArray {__version__} pgx-cn", f"##reference=file://{meta['fasta']}",
|
||||
"##simurg_genome_build=GRCh38", f"##simurg_manifest={meta['manifest']}",
|
||||
f"##simurg_pgx_cn_model={meta['model']}", f"##simurg_batch={meta['batch']}",
|
||||
f"##OverallPloidy={meta['ploidy']}", "##GCCorrect=false"] # fmt: skip
|
||||
h += [f"##contig=<ID={c},length={n}>" for c, n in contigs]
|
||||
h += ['##ALT=<ID=CNV,Description="Copy number variable region (targeted genotype)">',
|
||||
'##FILTER=<ID=PASS,Description="All filters passed">',
|
||||
'##FILTER=<ID=Q7,Description="Quality below 7">',
|
||||
'##FILTER=<ID=SampleQuality,Description="Sample LRR SD above 0.2">',
|
||||
'##FILTER=<ID=NoCall,Description="Region could not be assessed">',
|
||||
'##INFO=<ID=END,Number=1,Type=Integer,Description="Last base of the region (1-based, inclusive)">',
|
||||
'##INFO=<ID=SVTYPE,Number=1,Type=String,Description="CNV">',
|
||||
'##INFO=<ID=CNVLEN,Number=1,Type=Integer,Description="Region length in bp">',
|
||||
'##INFO=<ID=PROBE,Number=1,Type=Integer,Description="Number of probes used for the region">',
|
||||
'##INFO=<ID=REGION,Number=1,Type=String,Description="Target region name">',
|
||||
'##INFO=<ID=GENE,Number=1,Type=String,Description="Gene symbol">',
|
||||
'##FORMAT=<ID=CN,Number=1,Type=Integer,Description="Copy number; 5 means 5 or more">',
|
||||
'##FORMAT=<ID=NR,Number=1,Type=Float,Description="Aggregated normalized intensity (about CN/2)">',
|
||||
"\t".join(["#CHROM", "POS", "ID", "REF", "ALT", "QUAL", "FILTER", "INFO", "FORMAT", sample])] # fmt: skip
|
||||
return "\n".join(h) + "\n"
|
||||
|
||||
|
||||
def call_sample_regions(d: npt.NDArray[np.float64], model: PgxCnModel, passing: bool) -> list[dict[str, Any]]:
|
||||
"""PCN-002/009: one result per region."""
|
||||
vals = region_values(d, model)
|
||||
out = []
|
||||
for r in model.regions:
|
||||
x, nf = vals[r.name]
|
||||
n_all = len(model.region_sel(r.name))
|
||||
fit = model.fits.get(r.name)
|
||||
filters: list[str] = []
|
||||
if fit is None or n_all < MIN_REGION_PROBES or nf < MIN_FINITE_FRAC * n_all or not math.isfinite(x):
|
||||
cn, qual, nr = None, 0, math.nan
|
||||
filters.append("NoCall")
|
||||
else:
|
||||
cn, qual, _ = call_cn(x, fit)
|
||||
# NR ≈ CN / 2: the continuous CN that the region's mixture means imply for x [OURS]
|
||||
nr = max(0.0, 3.0 * 2.0 ** ((x - fit.delta) / fit.gamma) - 1.0) / 2.0 if cn else 0.0
|
||||
if qual < 7:
|
||||
filters.append("Q7")
|
||||
if not passing:
|
||||
filters.append("SampleQuality")
|
||||
out.append(
|
||||
{"region": r, "cn": cn, "qual": qual, "nr": nr, "x": x, "probes": nf, "filters": filters or ["PASS"]}
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def run_call(o: PgxCallOptions, rep: Reporter) -> int:
|
||||
"""``simurg pgx copy-number call`` (CLI-031)."""
|
||||
from simurg.core.atomic import atomic_write_text, check_outputs_absent
|
||||
from simurg.formats.discovery import discover_gtcs
|
||||
from simurg.formats.fasta import ContigMap, Genome
|
||||
from simurg.formats.gtc import read_gtc
|
||||
from simurg.pipeline.cn import write_vcf
|
||||
from simurg.pipeline.qc import write_tsv
|
||||
|
||||
model = read_model(o.model)
|
||||
genome = Genome(o.fasta)
|
||||
if genome.build not in ("GRCh38", "unknown", ""):
|
||||
raise SimurgError("E_BUILD_MISMATCH", f"PGx CN supports GRCh38 only (FASTA is {genome.build})")
|
||||
cmap = ContigMap([e.name for e in genome.entries], "auto")
|
||||
gtcs = discover_gtcs(o.gtcs)
|
||||
ext = ".pgx_cn.vcf.gz" if o.bgzip else ".pgx_cn.vcf"
|
||||
check_outputs_absent([o.out / f"{k}{ext}" for k in gtcs] + [o.out / "pgx_cn_batch_summary.tsv"], force=o.force)
|
||||
n_probes = int(model.header["n_probes"])
|
||||
mcl = model.header.get("cluster_file_name")
|
||||
# pass 1: batch QC from GTC headers (PCN-004)
|
||||
heads = {}
|
||||
for k, p in gtcs.items():
|
||||
g = read_gtc(p)
|
||||
heads[k] = (g.logr_dev, g.sample_name or k, g.cluster_file, len(g.logr_ratios))
|
||||
n_pass = sum(1 for v in heads.values() if v[0] < o.lrr_sd_max)
|
||||
if len(heads) < o.min_batch or n_pass < o.min_passing:
|
||||
raise SimurgError("E_PGX_BATCH_TOO_SMALL", f"{n_pass} sample(s) passed QC. Requires at least "
|
||||
f"{max(o.min_passing, 1)} samples to proceed.") # fmt: skip
|
||||
if n_pass < o.recommended_passing:
|
||||
rep.warn("W_PGX_BATCH_BELOW_RECOMMENDED", f"{n_pass} passing sample(s); {o.recommended_passing} recommended")
|
||||
for k, (_, _, cf, nl) in heads.items():
|
||||
if nl != n_probes:
|
||||
raise SimurgError("E_MODEL_MANIFEST_MISMATCH", f"{k} has {nl} loci, the model's manifest {n_probes}")
|
||||
if isinstance(mcl, str) and cf and cf != mcl:
|
||||
msg = f"Cluster file ({cf}) differs from the CN model's cluster file ({mcl})"
|
||||
if o.cluster_mismatch == "error":
|
||||
raise SimurgError("E_MODEL_CLUSTER_MISMATCH", msg, hint="retrain the model (PCN-005)")
|
||||
rep.warn("W_MODEL_CLUSTER_MISMATCH", msg)
|
||||
break
|
||||
contigs: list[tuple[str, int]] = []
|
||||
contig_name: dict[str, str | None] = {}
|
||||
for r in model.regions:
|
||||
if r.chrom in contig_name:
|
||||
continue
|
||||
c = cmap.get(r.chrom)
|
||||
contig_name[r.chrom] = c
|
||||
if c is None:
|
||||
rep.warn("W_PGX_NO_ALT_CONTIG", f"{r.chrom} is not in {o.fasta.name}; REF written as N (IN-062)")
|
||||
contigs.append((r.chrom, genome.contig_length(c) if c else r.end))
|
||||
order = {c: i for i, (c, _) in enumerate(contigs)}
|
||||
model_sha = hashlib.sha256(o.model.read_bytes()).hexdigest()
|
||||
mh = model.header
|
||||
meta = {
|
||||
"fasta": str(o.fasta.resolve()),
|
||||
"manifest": f"{mh['manifest_name']};sha256={mh['probe_set_sha256']}",
|
||||
"model": f"{o.model.name};sha256={model_sha};cluster_file={mcl};regions={mh['regions_bed_version']}",
|
||||
"batch": f"n_samples={len(heads)};n_passing={n_pass};lrr_sd_max={o.lrr_sd_max}",
|
||||
}
|
||||
centre_idx = model.probe_idx[model.region_sel("")] # the scale probes (autosomal), as in training
|
||||
o.out.mkdir(parents=True, exist_ok=True)
|
||||
srows, per_region = [], {r.name: {"n_gain": 0, "n_loss": 0, "n_neutral": 0, "n_nocall": 0, "n_filtered": 0}
|
||||
for r in model.regions} # fmt: skip
|
||||
failed = 0
|
||||
for k, p in gtcs.items():
|
||||
try:
|
||||
g = read_gtc(p)
|
||||
d = _centred(g.logr_ratios, centre_idx)[model.probe_idx] - model.probe_ref
|
||||
passing = heads[k][0] < o.lrr_sd_max
|
||||
res = call_sample_regions(d, model, passing)
|
||||
except SimurgError:
|
||||
raise
|
||||
except Exception as e: # PCN: one sample never stops the batch
|
||||
failed += 1
|
||||
rep.error("E_PGX_CN_FAILED", f"{k}: {e}", sample_id=k)
|
||||
continue
|
||||
called = [(x["region"], x["cn"]) for x in res if x["cn"] is not None]
|
||||
tot = sum(r.end - r.start0 for r, _ in called)
|
||||
ploidy = sum((r.end - r.start0) * min(cn, CN_MAX) for r, cn in called) / tot if tot else math.nan
|
||||
meta_s = {**meta, "ploidy": f"{ploidy:.3f}" if math.isfinite(ploidy) else "."}
|
||||
recs = []
|
||||
for x in sorted(res, key=lambda y: (order[y["region"].chrom], y["region"].start0)):
|
||||
r = x["region"]
|
||||
c = contig_name.get(r.chrom)
|
||||
ref = (genome.fetch(c, r.start0 - 1, r.start0) or "N").upper() if c and r.start0 >= 1 else "N"
|
||||
s, e_ = r.start0 + 1, r.end
|
||||
cn = "." if x["cn"] is None else str(x["cn"])
|
||||
nr = "." if not math.isfinite(x["nr"]) else f"{x['nr']:.6f}"
|
||||
line = "\t".join([r.chrom, str(s - 1), f"CNV:{r.name}:{r.chrom}:{s}-{e_}", ref, "<CNV>", str(x["qual"]),
|
||||
";".join(x["filters"]), f"END={e_};SVTYPE=CNV;CNVLEN={e_ - s + 1};PROBE={x['probes']};"
|
||||
f"REGION={r.name};GENE={r.gene}", "CN:NR", f"{cn}:{nr}"]) # fmt: skip
|
||||
recs.append((r.chrom, s - 2, e_, line))
|
||||
pr = per_region[r.name]
|
||||
if x["cn"] is None:
|
||||
pr["n_nocall"] += 1
|
||||
elif x["cn"] > 2:
|
||||
pr["n_gain"] += 1
|
||||
elif x["cn"] < 2:
|
||||
pr["n_loss"] += 1
|
||||
else:
|
||||
pr["n_neutral"] += 1
|
||||
pr["n_filtered"] += x["filters"] != ["PASS"]
|
||||
write_vcf(o.out / f"{k}{ext}", _vcf_header(heads[k][1], contigs, meta_s), recs, o.bgzip)
|
||||
srows.append({"sample_id": k, "sample_name": heads[k][1], "sample_folder": str(Path(p).parent),
|
||||
"logr_dev": f"{heads[k][0]:.6f}", "passed_batch_qc": str(passing).lower(),
|
||||
"used_in_reference": "false",
|
||||
**{f"cn_{x['region'].name}": ("." if x["cn"] is None else x["cn"]) for x in res},
|
||||
"flags": ""}) # fmt: skip
|
||||
cols = ["region", "gene", "n_gain", "n_loss", "n_neutral", "n_nocall", "n_filtered"]
|
||||
rows: list[dict[str, Any]] = [
|
||||
{"region": r.name, "gene": r.gene, **per_region[r.name]} for r in sorted(model.regions, key=lambda r: r.name)
|
||||
]
|
||||
totals = {c: sum(per_region[r.name][c] for r in model.regions) for c in cols[2:]}
|
||||
rows.append({"region": "ALL", "gene": "", **totals})
|
||||
write_tsv(o.out / "pgx_cn_batch_summary.tsv", cols, rows)
|
||||
scols = ["sample_id", "sample_name", "sample_folder", "logr_dev", "passed_batch_qc", "used_in_reference"]
|
||||
scols += [f"cn_{r.name}" for r in model.regions] + ["flags"]
|
||||
write_tsv(o.out / "pgx_cn_sample_summary.tsv", scols, srows)
|
||||
run = {"software": f"SimurgArray {__version__}", **meta, "samples": len(gtcs), "failed": failed}
|
||||
atomic_write_text(o.out / "pgx_cn_run.json", json.dumps(run, indent=1))
|
||||
rep.info(f"PGx CN: {len(srows)}/{len(gtcs)} sample(s) called")
|
||||
if failed == 0:
|
||||
return ExitCode.OK
|
||||
return ExitCode.PARTIAL if srows else ExitCode.ALL_FAILED
|
||||
Reference in new issue
Block a user