P6: gtc-to-vcf (VCF/BCF + indexes), harmonisation, exports (plink, bedgraph, finalreport)
- formats/fasta: plain/BGZF FASTA, .fai/.gzi built identical to htslib, index check, build detection, contig aliases, mmap genome cache (IN-060..064) - formats/bgzf, formats/bcf: pure-Python BGZF, tabix .tbi, BCF 2.2 and CSI (OUT-010, OUT-038, OUT-040) - harmonize/sites: plus-strand mapping with audit rows, indel resolution from SourceSeq (port of GTCtoVCF, Apache-2.0) and left normalisation (OUT-014..021, OUT-030, OUT-042) - harmonize/combine: OUT-031 genotype combination (ploidy-aware, Infinium I/II signal classes, OUT-037) - pipeline/vcf: `simurg gtc-to-vcf` (per-sample VCF.gz+tbi, --merge cohort BCF+csi, audit parquet) - pipeline/export: `simurg export bedgraph|finalreport|plink|locus-summary` (OUT-043, OUT-050..052, OUT-071) - validation: 36 GSA samples vs gtc2vcf: 23.66 M identical calls, 0 unexplained differences; PLINK identical to plink2 on 637,397 variants; bcftools validates VCF/BCF and our indexes answer region queries correctly - fetch_data.sh p6 (Ensembl GRCh38 with checksum), make_oracle_vcf.sh, validate_p6_vcf.py; K-022/K-023
This commit is contained in:
kemal committed
1 parent
c1fcb088c5
commit
c1699b4465
19 files changed
+3181
-9
No files matched your search
@@ -0,0 +1,422 @@
|
||||
"""``simurg export bedgraph | finalreport | plink | locus-summary`` (spec CLI-023, OUT-043, OUT-050 … OUT-052,
|
||||
OUT-071, OUT-072). None of them re-genotypes: they read GTCs (or SimurgArray VCFs) plus the manifest.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import gzip
|
||||
import io
|
||||
import math
|
||||
import re
|
||||
from collections.abc import Iterator, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import IO, Any, cast
|
||||
|
||||
import numpy as np
|
||||
import numpy.typing as npt
|
||||
|
||||
from simurg import PRODUCT_NAME, __version__
|
||||
from simurg.core.atomic import atomic_open, check_outputs_absent
|
||||
from simurg.core.errors import ExitCode, SimurgError
|
||||
from simurg.core.log import Reporter
|
||||
from simurg.core.provenance import RunManifest
|
||||
from simurg.formats.bpm import Manifest
|
||||
from simurg.formats.samplesheet import output_stem
|
||||
|
||||
_COMP = bytes.maketrans(b"ACGTDI", b"TGCADI")
|
||||
LARGE_OUTPUT = 100_000_000
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------ inputs
|
||||
@dataclass
|
||||
class GtcSample:
|
||||
sample_id: str
|
||||
sample_name: str
|
||||
path: str
|
||||
reported_sex: str = "U"
|
||||
|
||||
|
||||
def gtc_samples(gtcs: Path | None, sheet_path: Path | None, rep: Reporter) -> list[GtcSample]:
|
||||
"""Sheet order when a sheet is given, else sorted sample IDs (OUT-076)."""
|
||||
from simurg.formats.discovery import discover_gtcs
|
||||
from simurg.formats.samplesheet import parse_samplesheet
|
||||
|
||||
sheet = parse_samplesheet(sheet_path) if sheet_path else None
|
||||
root = gtcs or (sheet.root_folder if sheet else None)
|
||||
if root is None:
|
||||
raise SimurgError("E_NO_INPUT", "give --gtcs or a sample sheet with a RootFolder")
|
||||
found = discover_gtcs(root)
|
||||
if sheet is None:
|
||||
return [GtcSample(k, k, str(v)) for k, v in sorted(found.items())]
|
||||
out = []
|
||||
for r in sheet.rows:
|
||||
if r.sample_id in found:
|
||||
out.append(GtcSample(r.sample_id, r.sample_name or r.sample_id, str(found[r.sample_id]), r.reported_sex))
|
||||
else:
|
||||
rep.warn("W_QC_UNMATCHED_SAMPLES", f"{r.sample_id}: no GTC found", sample_id=r.sample_id)
|
||||
if not out:
|
||||
raise SimurgError("E_NO_INPUT", "none of the sheet's samples has a GTC")
|
||||
return out
|
||||
|
||||
|
||||
def chrom_rank(chrom: str) -> tuple[int, str]:
|
||||
c = chrom[3:] if chrom.lower().startswith("chr") else chrom
|
||||
if c.isdigit():
|
||||
return (int(c) if c != "0" else 1000, "")
|
||||
return ({"X": 23, "XY": 24, "Y": 25, "MT": 26, "M": 26}.get(c, 100), c)
|
||||
|
||||
|
||||
def genome_order(m: Manifest) -> npt.NDArray[np.int64]:
|
||||
"""Loci by (chromosome, position, name); unmapped (Chr 0) last."""
|
||||
keys = [
|
||||
((1000, "") if c in ("0", "") or p <= 0 else chrom_rank(c), int(p), n)
|
||||
for c, p, n in zip(m.chroms, m.map_infos.tolist(), m.names, strict=True)
|
||||
]
|
||||
return np.array(sorted(range(len(keys)), key=keys.__getitem__), dtype=np.int64)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------ bedGraph
|
||||
def _chrom_label(chrom: str, style: str) -> str:
|
||||
c = "X" if chrom == "XY" else chrom
|
||||
if style == "ucsc":
|
||||
c = "M" if c == "MT" else c
|
||||
return c if c.startswith("chr") else f"chr{c}"
|
||||
return c
|
||||
|
||||
|
||||
def moving_median(v: npt.NDArray[np.float64], k: int) -> npt.NDArray[np.float64]:
|
||||
if k <= 0 or len(v) == 0:
|
||||
return v.copy()
|
||||
pad = np.pad(v, k, mode="edge")
|
||||
win = np.lib.stride_tricks.sliding_window_view(pad, 2 * k + 1)
|
||||
return np.nanmedian(win, axis=1)
|
||||
|
||||
|
||||
@dataclass
|
||||
class BedgraphLayout:
|
||||
"""Sample-independent part of OUT-050: loci in output order with their labels (computed once per manifest)."""
|
||||
|
||||
order: npt.NDArray[np.int64] # mapped loci, genome order
|
||||
chrom: list[str] # output chromosome label per ordered locus
|
||||
pos: npt.NDArray[np.int64]
|
||||
|
||||
|
||||
def bedgraph_layout(m: Manifest, style: str) -> BedgraphLayout:
|
||||
order = genome_order(m)
|
||||
mapped = np.array([c not in ("0", "") for c in m.chroms]) & (m.map_infos > 0)
|
||||
order = order[mapped[order]]
|
||||
return BedgraphLayout(order, [_chrom_label(m.chroms[i], style) for i in order.tolist()], m.map_infos[order])
|
||||
|
||||
|
||||
def bedgraph_rows(m: Manifest, g: Any, layout: BedgraphLayout, which: str) -> list[tuple[str, int, float]]:
|
||||
"""OUT-050 rows (chrom, pos, value) for one track, in output order, same-position probes averaged."""
|
||||
vals = g.b_allele_freqs if which == "baf" else g.logr_ratios
|
||||
if len(vals) != m.num_loci:
|
||||
raise SimurgError("E_GTC_UNSUITABLE", "GTC has no BAF/LRR (GTC v3?)")
|
||||
o = layout.order
|
||||
v = vals[o].astype(np.float64)
|
||||
keep = ~((g.raw_x[o] == 0) & (g.raw_y[o] == 0)) & np.isfinite(v)
|
||||
if which == "baf":
|
||||
keep &= ~m.intensity_only[o]
|
||||
idx = np.flatnonzero(keep)
|
||||
if not len(idx):
|
||||
return []
|
||||
chrom = [layout.chrom[i] for i in idx.tolist()]
|
||||
pos = layout.pos[idx]
|
||||
new = np.ones(len(idx), np.bool_)
|
||||
new[1:] = (pos[1:] != pos[:-1]) | np.array([chrom[k] != chrom[k - 1] for k in range(1, len(chrom))])
|
||||
starts = np.flatnonzero(new)
|
||||
sums = np.add.reduceat(v[idx], starts)
|
||||
counts = np.diff(np.append(starts, len(idx)))
|
||||
means = sums / counts
|
||||
return [(chrom[k], int(pos[k]), float(x)) for k, x in zip(starts.tolist(), means.tolist(), strict=True)]
|
||||
|
||||
|
||||
def write_bedgraph(path: Path, name: str, which: str, rows: Sequence[tuple[str, int, float]], gz: bool) -> None:
|
||||
limits = "0:1" if which == "baf" else "-2:2"
|
||||
head = (
|
||||
f'track type=bedGraph name="{name} {which.upper()}" description="{PRODUCT_NAME} {__version__}" '
|
||||
f"visibility=full autoScale=off viewLimits={limits}\n"
|
||||
)
|
||||
body = "".join(f"{c}\t{p - 1}\t{p}\t{v:.6g}\n" for c, p, v in rows)
|
||||
data = (head + body).encode()
|
||||
with atomic_open(path, "wb") as fh:
|
||||
cast(IO[bytes], fh).write(gzip.compress(data, mtime=0) if gz else data)
|
||||
|
||||
|
||||
@dataclass
|
||||
class BedgraphOptions:
|
||||
out: Path
|
||||
bpm: Path
|
||||
gtcs: Path | None = None
|
||||
sheet: Path | None = None
|
||||
gzip: bool = True
|
||||
chrom_style: str = "manifest"
|
||||
smoothed_lrr: int = 0
|
||||
force: bool = False
|
||||
argv: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
def run_export_bedgraph(opts: BedgraphOptions, rep: Reporter) -> int:
|
||||
from simurg.formats.bpm import read_bpm
|
||||
from simurg.formats.gtc import read_gtc
|
||||
|
||||
if opts.chrom_style not in ("manifest", "ucsc"):
|
||||
raise SimurgError("E_CONFIG_KEY", "--chrom-style must be manifest or ucsc")
|
||||
rm = RunManifest(command="export bedgraph", argv=opts.argv)
|
||||
samples = gtc_samples(opts.gtcs, opts.sheet, rep)
|
||||
ext = ".bedgraph.gz" if opts.gzip else ".bedgraph"
|
||||
outs = []
|
||||
for s in samples:
|
||||
stem = output_stem(s.sample_name)
|
||||
outs += [opts.out / f"{stem}.baf{ext}", opts.out / f"{stem}.lrr{ext}"]
|
||||
if opts.smoothed_lrr:
|
||||
outs.append(opts.out / f"{stem}.lrr_smoothed{ext}")
|
||||
check_outputs_absent(outs, force=opts.force)
|
||||
m = read_bpm(opts.bpm)
|
||||
layout = bedgraph_layout(m, opts.chrom_style)
|
||||
opts.out.mkdir(parents=True, exist_ok=True)
|
||||
failed = 0
|
||||
for s in samples:
|
||||
try:
|
||||
g = read_gtc(s.path)
|
||||
stem = output_stem(s.sample_name)
|
||||
for which in ("baf", "lrr"):
|
||||
rows = bedgraph_rows(m, g, layout, which)
|
||||
write_bedgraph(opts.out / f"{stem}.{which}{ext}", s.sample_name, which, rows, opts.gzip)
|
||||
if which == "lrr" and opts.smoothed_lrr:
|
||||
sm: list[tuple[str, int, float]] = []
|
||||
for chrom in dict.fromkeys(c for c, _, _ in rows):
|
||||
sub = [r for r in rows if r[0] == chrom]
|
||||
med = moving_median(np.array([r[2] for r in sub]), opts.smoothed_lrr)
|
||||
sm += [(c, p, float(v)) for (c, p, _), v in zip(sub, med, strict=True)]
|
||||
write_bedgraph(opts.out / f"{stem}.lrr_smoothed{ext}", s.sample_name, "lrr", sm, opts.gzip)
|
||||
except SimurgError as e:
|
||||
failed += 1
|
||||
rep.error(e.code, f"{s.sample_id}: {e.message}", sample_id=s.sample_id)
|
||||
rm.status = "completed"
|
||||
rm.summary = {"samples": len(samples), "failed": failed}
|
||||
rm.write(opts.out / "run_manifest.export_bedgraph.json")
|
||||
return ExitCode.OK if not failed else (ExitCode.PARTIAL if failed < len(samples) else ExitCode.ALL_FAILED)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------ final report
|
||||
FR_COLUMNS = ["SNP Name", "SNP", "Sample ID", "Allele1 - Top", "Allele2 - Top", "Allele1 - Forward",
|
||||
"Allele2 - Forward", "Allele1 - Plus", "Allele2 - Plus", "GC Score", "GT Score", "Log R Ratio",
|
||||
"B Allele Freq", "Chr", "Position"] # fmt: skip
|
||||
|
||||
|
||||
def allele_letters(m: Manifest) -> dict[str, list[tuple[str, str]]]:
|
||||
"""Per locus (A, B) letters on the TOP, Forward (source) and Plus (reference) strands."""
|
||||
top, fwd, plus = [], [], []
|
||||
for snp, ilmn, src, ref in zip(m.snps, m.ilmn_strands, m.source_strands, m.ref_strands, strict=True):
|
||||
mm = re.match(r"^\[([^/\]]+)/([^\]]+)\]$", snp.strip())
|
||||
a, b = (mm.group(1), mm.group(2)) if mm else ("-", "-")
|
||||
|
||||
def comp(x: str) -> str:
|
||||
return x.encode().translate(_COMP).decode() if len(x) == 1 else x
|
||||
|
||||
d = (ilmn or "")[:1].upper()
|
||||
top.append((a, b) if d in ("T", "P", "") or len(a) > 1 else (comp(a), comp(b)))
|
||||
same_src = (src or "")[:1].upper() == d or not src
|
||||
fwd.append((a, b) if same_src else (comp(a), comp(b)))
|
||||
plus.append((a, b) if (ref or "+").strip() == "+" else (comp(a), comp(b)))
|
||||
return {"top": top, "fwd": fwd, "plus": plus}
|
||||
|
||||
|
||||
def _pair(call: int, ab: tuple[str, str]) -> tuple[str, str]:
|
||||
a, b = ab
|
||||
return {1: (a, a), 2: (a, b), 3: (b, b)}.get(call, ("-", "-"))
|
||||
|
||||
|
||||
@dataclass
|
||||
class FinalReportOptions:
|
||||
out: Path
|
||||
bpm: Path
|
||||
gtcs: Path | None = None
|
||||
sheet: Path | None = None
|
||||
csv: Path | None = None
|
||||
egt: Path | None = None
|
||||
matrix: bool = False
|
||||
force: bool = False
|
||||
argv: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
def run_export_finalreport(opts: FinalReportOptions, rep: Reporter) -> int:
|
||||
from simurg.formats.bpm import read_bpm
|
||||
from simurg.formats.gtc import read_gtc
|
||||
|
||||
rm = RunManifest(command="export finalreport", argv=opts.argv)
|
||||
samples = gtc_samples(opts.gtcs, opts.sheet, rep)
|
||||
target = opts.out / ("final_report_matrix.tsv.gz" if opts.matrix else "final_report.tsv.gz")
|
||||
check_outputs_absent([target], force=opts.force)
|
||||
m = read_bpm(opts.bpm)
|
||||
if opts.csv is not None:
|
||||
from simurg.formats.csv_manifest import read_csv_manifest
|
||||
|
||||
cm = read_csv_manifest(opts.csv, load_sequences=False)
|
||||
m.source_strands, m.ref_strands = cm.source_strands, cm.ref_strands
|
||||
cells = m.num_loci * len(samples)
|
||||
if cells > LARGE_OUTPUT:
|
||||
rep.warn("W_LARGE_OUTPUT", f"{cells:,} rows (~{cells * 90 / 1e9:.1f} GB uncompressed); consider --matrix")
|
||||
gt_score = np.full(m.num_loci, np.nan)
|
||||
if opts.egt is not None:
|
||||
from simurg.formats.egt import read_egt
|
||||
|
||||
egt = read_egt(opts.egt)
|
||||
idx = egt.name_index()
|
||||
for i, nm in enumerate(m.names):
|
||||
j = idx.get(nm)
|
||||
if j is not None:
|
||||
gt_score[i] = float(egt.gentrain_score[j])
|
||||
letters = allele_letters(m)
|
||||
order = genome_order(m).tolist()
|
||||
opts.out.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def f4(v: float) -> str:
|
||||
return "NaN" if math.isnan(v) else f"{v:.4f}"
|
||||
|
||||
with atomic_open(target, "wb") as raw:
|
||||
gz = gzip.GzipFile(fileobj=cast(IO[bytes], raw), mode="wb", mtime=0)
|
||||
fh = io.TextIOWrapper(gz, encoding="utf-8", newline="\n")
|
||||
fh.write("#simurg_schema=simurg.finalreport/1\n")
|
||||
if opts.matrix:
|
||||
fh.write("\t".join(["SNP Name", "Chr", "Position", *[s.sample_name for s in samples]]) + "\n")
|
||||
calls = []
|
||||
for s in samples:
|
||||
calls.append(read_gtc(s.path).genotypes)
|
||||
for i in order:
|
||||
row = [m.names[i], m.chroms[i], str(int(m.map_infos[i]))]
|
||||
row += ["".join(_pair(int(c[i]), letters["top"][i])) for c in calls]
|
||||
fh.write("\t".join(row) + "\n")
|
||||
else:
|
||||
fh.write("\t".join(FR_COLUMNS) + "\n")
|
||||
for s in samples:
|
||||
g = read_gtc(s.path)
|
||||
has = len(g.b_allele_freqs) == m.num_loci
|
||||
bc = g.base_calls
|
||||
for i in order:
|
||||
c = int(g.genotypes[i])
|
||||
f, p = (_pair(c, letters[k][i]) for k in ("fwd", "plus"))
|
||||
t = (chr(bc[i, 0]), chr(bc[i, 1])) if c else ("-", "-") # TOP-strand base calls (OUT-071)
|
||||
baf = float(g.b_allele_freqs[i]) if has and not m.intensity_only[i] else math.nan
|
||||
lrr = float(g.logr_ratios[i]) if has else math.nan
|
||||
fh.write(
|
||||
f"{m.names[i]}\t{m.snps[i]}\t{s.sample_id}\t{t[0]}\t{t[1]}\t{f[0]}\t{f[1]}\t{p[0]}\t{p[1]}\t"
|
||||
f"{f4(float(g.genotype_scores[i]))}\t{f4(gt_score[i])}\t{f4(lrr)}\t{f4(baf)}\t"
|
||||
f"{m.chroms[i]}\t{int(m.map_infos[i])}\n"
|
||||
)
|
||||
fh.flush()
|
||||
fh.detach()
|
||||
gz.close()
|
||||
rm.status = "completed"
|
||||
rm.summary = {"samples": len(samples), "loci": m.num_loci}
|
||||
rm.write(opts.out / "run_manifest.export_finalreport.json")
|
||||
return ExitCode.OK
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------ PLINK
|
||||
PLINK_MAGIC = bytes([0x6C, 0x1B, 0x01])
|
||||
|
||||
|
||||
def parse_vcf_records(path: Path) -> Iterator[list[str]]:
|
||||
opener = gzip.open if str(path).endswith(".gz") else open
|
||||
with opener(path, "rt") as fh:
|
||||
for line in fh:
|
||||
if not line.startswith("#"):
|
||||
yield line.rstrip("\n").split("\t")
|
||||
|
||||
|
||||
def bed_code(gt: str, alt_index: int) -> int:
|
||||
"""PLINK 1 two-bit code with A1 = the split ALT, A2 = REF: 0 hom A1, 1 missing, 2 het, 3 hom A2."""
|
||||
if gt in (".", "./."):
|
||||
return 1
|
||||
al = gt.replace("|", "/").split("/")
|
||||
if "." in al:
|
||||
return 1
|
||||
n_alt = sum(int(x) == alt_index for x in al)
|
||||
if len(al) == 1:
|
||||
n_alt *= 2
|
||||
return {0: 3, 1: 2, 2: 0}[n_alt]
|
||||
|
||||
|
||||
@dataclass
|
||||
class PlinkOptions:
|
||||
out: Path
|
||||
vcfs: Path
|
||||
sheet: Path | None = None
|
||||
gtcs: Path | None = None
|
||||
multiallelic: str = "split"
|
||||
prefix: str = "cohort"
|
||||
force: bool = False
|
||||
argv: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
def run_export_plink(opts: PlinkOptions, rep: Reporter) -> int:
|
||||
"""OUT-043: SimurgArray per-sample VCFs (identical record order) → PLINK 1 .bed/.bim/.fam."""
|
||||
if opts.multiallelic not in ("split", "drop"):
|
||||
raise SimurgError("E_CONFIG_KEY", "--multiallelic must be split or drop")
|
||||
rm = RunManifest(command="export plink", argv=opts.argv)
|
||||
vcfs = sorted(p for p in opts.vcfs.glob("*.snv.vcf*") if not p.name.endswith((".tbi", ".csi")))
|
||||
if not vcfs:
|
||||
raise SimurgError("E_NO_INPUT", f"no *.snv.vcf(.gz) files in {opts.vcfs}")
|
||||
base = opts.out / opts.prefix
|
||||
check_outputs_absent([base.with_suffix(x) for x in (".bed", ".bim", ".fam")], force=opts.force)
|
||||
names, sex = [], {}
|
||||
for v in vcfs:
|
||||
opener = gzip.open if str(v).endswith(".gz") else open
|
||||
with opener(v, "rt") as fh:
|
||||
for line in fh:
|
||||
if line.startswith("#CHROM"):
|
||||
names.append(line.rstrip("\n").split("\t")[9])
|
||||
break
|
||||
if opts.gtcs is not None:
|
||||
from simurg.formats.discovery import discover_gtcs
|
||||
from simurg.formats.gtc import read_gtc
|
||||
|
||||
for key, gpath in discover_gtcs(opts.gtcs).items():
|
||||
sex[key] = read_gtc(gpath, check_write_complete=False).gender_char
|
||||
if opts.sheet is not None:
|
||||
from simurg.formats.samplesheet import parse_samplesheet
|
||||
|
||||
for r in parse_samplesheet(opts.sheet).rows:
|
||||
if r.reported_sex in ("M", "F"):
|
||||
sex[r.sample_name or r.sample_id] = sex[r.sample_id] = r.reported_sex
|
||||
streams = [parse_vcf_records(v) for v in vcfs]
|
||||
n = len(vcfs)
|
||||
nbytes = (n + 3) // 4
|
||||
opts.out.mkdir(parents=True, exist_ok=True)
|
||||
n_var = dropped = 0
|
||||
with (
|
||||
atomic_open(base.with_suffix(".bed"), "wb") as bed_raw,
|
||||
atomic_open(base.with_suffix(".bim"), "w", "utf-8") as bim_raw,
|
||||
):
|
||||
bed, bim = cast(IO[bytes], bed_raw), cast(IO[str], bim_raw)
|
||||
bed.write(PLINK_MAGIC)
|
||||
for recs in zip(*streams, strict=True):
|
||||
r0 = recs[0]
|
||||
if any(r[:5] != r0[:5] for r in recs[1:]):
|
||||
raise SimurgError("E_GTC_UNSUITABLE", f"VCF record order differs between samples at {r0[0]}:{r0[1]}")
|
||||
alts = [] if r0[4] == "." else r0[4].split(",")
|
||||
if len(alts) > 1 and opts.multiallelic == "drop":
|
||||
dropped += 1
|
||||
continue
|
||||
gts = [r[9].split(":", 1)[0] for r in recs]
|
||||
for k, alt in enumerate(alts, start=1):
|
||||
codes = [bed_code(g, k) for g in gts]
|
||||
buf = bytearray(nbytes)
|
||||
for j, c in enumerate(codes):
|
||||
buf[j >> 2] |= c << (2 * (j & 3))
|
||||
bed.write(bytes(buf))
|
||||
vid = r0[2] if len(alts) == 1 else f"{r0[2]}:{alt}"
|
||||
bim.write(f"{r0[0]}\t{vid}\t0\t{r0[1]}\t{alt}\t{r0[3]}\n")
|
||||
n_var += 1
|
||||
fam = "".join(f"{s}\t{s}\t0\t0\t{ {'M': 1, 'F': 2}.get(sex.get(s, 'U'), 0) }\t-9\n" for s in names)
|
||||
with atomic_open(base.with_suffix(".fam"), "w", "utf-8") as fam_raw:
|
||||
cast(IO[str], fam_raw).write(fam)
|
||||
rm.status = "completed"
|
||||
rm.summary = {"samples": n, "variants": n_var, "multiallelic_dropped": dropped}
|
||||
rm.write(opts.out / "run_manifest.export_plink.json")
|
||||
rep.info(f"PLINK fileset {base}.bed/.bim/.fam: {n} samples, {n_var} variants")
|
||||
return ExitCode.OK
|
||||
@@ -308,26 +308,27 @@ class SampleInfo:
|
||||
class CohortStore:
|
||||
"""Disk-backed (samples × loci) matrices: genotypes (uint8), BAF and GenCall scores (float32, optional)."""
|
||||
|
||||
def __init__(self, root: Path, n_samples: int, n_loci: int, *, baf: bool, scores: bool) -> None:
|
||||
def __init__(self, root: Path, n_samples: int, n_loci: int, *, baf: bool, scores: bool, lrr: bool = False) -> None:
|
||||
self.root, self.shape = root, (n_samples, n_loci)
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
self.gt = np.lib.format.open_memmap(root / "gt.npy", "w+", np.uint8, self.shape)
|
||||
self.baf = np.lib.format.open_memmap(root / "baf.npy", "w+", np.float32, self.shape) if baf else None
|
||||
self.score = np.lib.format.open_memmap(root / "score.npy", "w+", np.float32, self.shape) if scores else None
|
||||
self.lrr = np.lib.format.open_memmap(root / "lrr.npy", "w+", np.float32, self.shape) if lrr else None
|
||||
|
||||
def close(self) -> None:
|
||||
for name in ("gt", "baf", "score"):
|
||||
for name in ("gt", "baf", "score", "lrr"):
|
||||
a = getattr(self, name)
|
||||
if a is not None:
|
||||
a.flush()
|
||||
self.gt = self.baf = self.score = None # type: ignore[assignment]
|
||||
self.gt = self.baf = self.score = self.lrr = None # type: ignore[assignment]
|
||||
shutil.rmtree(self.root, ignore_errors=True)
|
||||
|
||||
|
||||
def _load_one(task: tuple[int, str, str, str, bool, bool]) -> SampleInfo:
|
||||
def _load_one(task: tuple[int, str, str, str, bool, bool, bool]) -> SampleInfo:
|
||||
from simurg.formats.gtc import read_gtc
|
||||
|
||||
i, sid, path, root, want_baf, want_scores = task
|
||||
i, sid, path, root, want_baf, want_scores, want_lrr = task
|
||||
info = SampleInfo(sid, path)
|
||||
try:
|
||||
g = read_gtc(path)
|
||||
@@ -347,6 +348,10 @@ def _load_one(task: tuple[int, str, str, str, bool, bool]) -> SampleInfo:
|
||||
info.has_baf = len(g.b_allele_freqs) == gt.shape[1]
|
||||
baf[i] = g.b_allele_freqs if info.has_baf else np.nan
|
||||
baf.flush()
|
||||
if want_lrr:
|
||||
lr = np.load(Path(root) / "lrr.npy", mmap_mode="r+")
|
||||
lr[i] = g.logr_ratios if len(g.logr_ratios) == gt.shape[1] else np.nan
|
||||
lr.flush()
|
||||
if want_scores:
|
||||
sc = np.load(Path(root) / "score.npy", mmap_mode="r+")
|
||||
sc[i] = g.genotype_scores
|
||||
@@ -359,12 +364,19 @@ def _load_one(task: tuple[int, str, str, str, bool, bool]) -> SampleInfo:
|
||||
|
||||
|
||||
def load_cohort(
|
||||
samples: Sequence[tuple[str, str]], n_loci: int, tmp: Path, *, baf: bool, scores: bool, workers: int
|
||||
samples: Sequence[tuple[str, str]],
|
||||
n_loci: int,
|
||||
tmp: Path,
|
||||
*,
|
||||
baf: bool,
|
||||
scores: bool,
|
||||
workers: int,
|
||||
lrr: bool = False,
|
||||
) -> tuple[CohortStore, list[SampleInfo]]:
|
||||
"""Read GTCs (sample_id, path) into a :class:`CohortStore` with a process pool."""
|
||||
store = CohortStore(tmp, len(samples), n_loci, baf=baf, scores=scores)
|
||||
store = CohortStore(tmp, len(samples), n_loci, baf=baf, scores=scores, lrr=lrr)
|
||||
store.gt.flush()
|
||||
tasks = [(i, sid, p, str(tmp), baf, scores) for i, (sid, p) in enumerate(samples)]
|
||||
tasks = [(i, sid, p, str(tmp), baf, scores, lrr) for i, (sid, p) in enumerate(samples)]
|
||||
if workers <= 1 or len(tasks) <= 1:
|
||||
infos = [_load_one(t) for t in tasks]
|
||||
else:
|
||||
|
||||
@@ -0,0 +1,656 @@
|
||||
"""``simurg gtc-to-vcf``: GTC + manifest + FASTA → one bgzipped, indexed SNV VCF per sample (spec CLI-022,
|
||||
OUT-010 … OUT-042, IN-060 … IN-064; plan §8).
|
||||
|
||||
The manifest is mapped to plus-strand sites once (``simurg.harmonize``); each sample is then written by a worker
|
||||
process with vectorised genotype lookup for single-probe sites and the OUT-031 algorithm for merged sites.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import math
|
||||
import multiprocessing as mp
|
||||
import os
|
||||
from collections import Counter
|
||||
from concurrent.futures import ProcessPoolExecutor
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import numpy.typing as npt
|
||||
|
||||
from simurg import PRODUCT_NAME, __version__
|
||||
from simurg.core.atomic import atomic_open, check_outputs_absent
|
||||
from simurg.core.errors import ExitCode, SimurgError
|
||||
from simurg.core.log import Reporter
|
||||
from simurg.core.provenance import RunManifest, sha256_file
|
||||
from simurg.formats.bgzf import BgzfWriter, TabixBuilder, write_tbi
|
||||
from simurg.harmonize.combine import genotype as combine_genotype
|
||||
from simurg.harmonize.combine import gt_text
|
||||
from simurg.harmonize.sites import AuditRow, ManifestView, Site, group_sites, map_locus
|
||||
|
||||
PAR = { # 1-based inclusive pseudo-autosomal intervals on X (IN-065, human)
|
||||
"GRCh38": ((10_001, 2_781_479), (155_701_383, 156_030_895)),
|
||||
"GRCh37": ((60_001, 2_699_520), (154_931_044, 155_260_560)),
|
||||
}
|
||||
CLASS_AUTO, CLASS_X, CLASS_Y, CLASS_MT, CLASS_PAR = 0, 1, 2, 3, 4
|
||||
|
||||
|
||||
@dataclass
|
||||
class VcfOptions:
|
||||
out: Path
|
||||
bpm: Path
|
||||
fasta: Path
|
||||
gtcs: Path | None = None
|
||||
sheet: Path | None = None
|
||||
csv: Path | None = None
|
||||
bgzip: bool = True
|
||||
genome_cache: bool = True
|
||||
exclude_loci: Path | None = None
|
||||
include_loci: Path | None = None
|
||||
unsquash_duplicates: bool = False
|
||||
inf1_nocall_informative: bool = False
|
||||
drop_ambiguous: bool = False
|
||||
filter_gs: float | None = None
|
||||
contig_alias: str = "strict"
|
||||
ploidy_policy: str = "sex-aware"
|
||||
threads: int = 0
|
||||
merge: bool = False
|
||||
force: bool = False
|
||||
argv: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------ site table
|
||||
@dataclass
|
||||
class SiteTable:
|
||||
"""Everything a worker needs, precomputed once (picklable)."""
|
||||
|
||||
prefix: list[bytes] # "CHROM\tPOS\tID\tREF\tALT\t.\t" per site
|
||||
single_rows: npt.NDArray[np.int64] # site indices of single-probe sites
|
||||
single_locus: npt.NDArray[np.int64] # their manifest locus index
|
||||
single_gt: npt.NDArray[np.object_] # (n_single, 2 ploidy, 4 calls) GT text
|
||||
single_flip: npt.NDArray[np.bool_] # BAF must be 1 − BAF
|
||||
merged_sites: list[Site]
|
||||
merged_rows: npt.NDArray[np.int64]
|
||||
site_class: npt.NDArray[np.int8]
|
||||
contigs: list[str]
|
||||
header: str
|
||||
site_beg: npt.NDArray[np.int64]
|
||||
site_end: npt.NDArray[np.int64]
|
||||
site_contig: list[str]
|
||||
site_id: list[str] = field(default_factory=list)
|
||||
site_alleles: list[list[str]] = field(default_factory=list)
|
||||
single_ab: npt.NDArray[np.int16] | None = None # (n_single, 2) allele indices of the probe's A and B alleles
|
||||
inf1_nc_informative: bool = False
|
||||
ploidy_policy: str = "sex-aware"
|
||||
filter_gs: float | None = None
|
||||
|
||||
|
||||
def _site_class(contig: str, pos: int, par_flag: bool, build: str) -> int:
|
||||
c = contig[3:] if contig.lower().startswith("chr") else contig
|
||||
if c in ("MT", "M"):
|
||||
return CLASS_MT
|
||||
if c == "Y":
|
||||
return CLASS_Y
|
||||
if c == "X":
|
||||
if par_flag or any(a <= pos <= b for a, b in PAR.get(build or "GRCh38", ())):
|
||||
return CLASS_PAR
|
||||
return CLASS_X
|
||||
return CLASS_AUTO
|
||||
|
||||
|
||||
def ploidy_for(cls: npt.NDArray[np.int8], sex: str, policy: str) -> npt.NDArray[np.int8]:
|
||||
"""OUT-018 per site; 0 marks 'haploid and always missing' (Y of female samples)."""
|
||||
p = np.full(len(cls), 2, dtype=np.int8)
|
||||
p[cls == CLASS_MT] = 1
|
||||
if policy == "diploid":
|
||||
return p
|
||||
p[cls == CLASS_Y] = 0 if sex == "F" else 1
|
||||
if sex == "M":
|
||||
p[cls == CLASS_X] = 1
|
||||
return p
|
||||
|
||||
|
||||
def read_locus_list(path: Path) -> list[str]:
|
||||
out = []
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
line = line.split("#", 1)[0].strip()
|
||||
if line:
|
||||
out.append(line)
|
||||
return out
|
||||
|
||||
|
||||
def build_sites(
|
||||
opts: VcfOptions, rep: Reporter, rm: RunManifest
|
||||
) -> tuple[SiteTable, list[AuditRow], Counter[str], Any]:
|
||||
from simurg.formats.bpm import read_bpm
|
||||
from simurg.formats.fasta import ContigMap, Genome
|
||||
|
||||
bpm = read_bpm(opts.bpm)
|
||||
csvm = None
|
||||
if opts.csv is not None:
|
||||
from simurg.formats.csv_manifest import compare_manifests, read_csv_manifest
|
||||
|
||||
csvm = read_csv_manifest(opts.csv)
|
||||
problems = compare_manifests(bpm, csvm)
|
||||
if problems:
|
||||
raise SimurgError("E_BPM_CSV_MISMATCH", "BPM and CSV manifests differ:\n " + "\n ".join(problems))
|
||||
genome = Genome(opts.fasta, use_cache=opts.genome_cache)
|
||||
for n in genome.notes:
|
||||
rep.info(n)
|
||||
builds = Counter(bpm.genome_builds)
|
||||
mbuild = builds.most_common(1)[0][0] if builds else ""
|
||||
want = {"37": "GRCh37", "38": "GRCh38"}.get(mbuild.strip(), mbuild.strip())
|
||||
if genome.build and want and want != genome.build:
|
||||
raise SimurgError(
|
||||
"E_BUILD_MISMATCH", f"manifest genome build {want} but FASTA looks like {genome.build} ({opts.fasta})"
|
||||
)
|
||||
build = genome.build or want
|
||||
src = csvm or bpm
|
||||
mv = ManifestView(
|
||||
bpm.names,
|
||||
bpm.snps,
|
||||
bpm.chroms,
|
||||
bpm.map_infos.tolist(),
|
||||
src.ilmn_strands,
|
||||
src.source_strands,
|
||||
src.ref_strands if any(src.ref_strands) else bpm.ref_strands,
|
||||
csvm.source_seqs if csvm else [""] * bpm.num_loci,
|
||||
bpm.assay_types.tolist(),
|
||||
bpm.intensity_only.tolist(),
|
||||
)
|
||||
if csvm is None and any(("D" in s) for s in bpm.snps):
|
||||
rep.error("W_SKIP_INDEL_CONTEXT", "no CSV manifest: indels are skipped (give --csv with SourceSeq)")
|
||||
names = set(bpm.names)
|
||||
excl: set[str] = set()
|
||||
incl: set[str] | None = None
|
||||
for path, kind in ((opts.exclude_loci, "exclude"), (opts.include_loci, "include")):
|
||||
if path is None:
|
||||
continue
|
||||
lst = read_locus_list(path)
|
||||
unknown = [x for x in lst if x not in names]
|
||||
if unknown:
|
||||
rep.warn("W_UNKNOWN_LOCI", f"{len(unknown)} name(s) in --{kind}-loci are not in the manifest")
|
||||
if kind == "exclude":
|
||||
excl = set(lst)
|
||||
else:
|
||||
incl = set(lst)
|
||||
cmap = ContigMap([e.name for e in genome.entries], opts.contig_alias)
|
||||
probes = []
|
||||
audit: list[AuditRow] = []
|
||||
skips: Counter[str] = Counter()
|
||||
for i in range(bpm.num_loci):
|
||||
nm = bpm.names[i]
|
||||
if nm in excl or (incl is not None and nm not in incl):
|
||||
a = AuditRow(nm, bpm.snps[i], "", "", "", "", "", False, False, "W_SKIP_EXCLUDED", build, "")
|
||||
p = None
|
||||
else:
|
||||
p, a = map_locus(i, mv, genome.fetch, cmap.get, build)
|
||||
if p is not None and opts.drop_ambiguous and p.confidence == "low":
|
||||
a.reason, p = "W_SKIP_AMBIGUOUS", None
|
||||
audit.append(a)
|
||||
if p is None or a.reason.startswith("W_"):
|
||||
skips[a.reason] += 1
|
||||
if p is not None:
|
||||
probes.append(p)
|
||||
for code, count in sorted(skips.items()):
|
||||
rep.warn(code, f"{count} locus/loci: {code}")
|
||||
order = {e.name: k for k, e in enumerate(genome.entries)}
|
||||
sites = group_sites(probes, order, unsquash=opts.unsquash_duplicates)
|
||||
used = {s.contig for s in sites}
|
||||
contigs = [e.name for e in genome.entries if e.name in used]
|
||||
# static per-site text and single-probe genotype tables
|
||||
prefix, cls = [], np.zeros(len(sites), np.int8)
|
||||
single_rows, single_locus, single_flip, gts, single_ab = [], [], [], [], []
|
||||
merged_sites, merged_rows = [], []
|
||||
beg = np.zeros(len(sites), np.int64)
|
||||
end = np.zeros(len(sites), np.int64)
|
||||
for k, s in enumerate(sites):
|
||||
prefix.append(f"{s.contig}\t{s.pos}\t{s.id}\t{s.ref}\t{','.join(s.alts) if s.alts else '.'}\t.\t".encode())
|
||||
cls[k] = _site_class(s.contig, s.pos, any(p.par for p in s.probes), build)
|
||||
beg[k], end[k] = s.pos - 1, s.pos - 1 + len(s.ref)
|
||||
if len(s.probes) == 1:
|
||||
from simurg.harmonize.combine import single
|
||||
|
||||
single_rows.append(k)
|
||||
single_locus.append(s.probes[0].index)
|
||||
al_a, al_b = s.probes[0].alleles
|
||||
single_flip.append(al_a != s.ref and al_b == s.ref)
|
||||
single_ab.append((s.allele_index(al_a), s.allele_index(al_b)))
|
||||
gts.append([[gt_text(single(s, c, pl), pl) for c in range(4)] for pl in (1, 2)])
|
||||
else:
|
||||
merged_sites.append(s)
|
||||
merged_rows.append(k)
|
||||
header = vcf_header(opts, genome, build, contigs, rm)
|
||||
table = SiteTable(
|
||||
prefix=prefix,
|
||||
single_rows=np.array(single_rows, np.int64),
|
||||
single_locus=np.array(single_locus, np.int64),
|
||||
single_gt=np.array(gts, dtype=object).reshape(len(single_rows), 2, 4),
|
||||
single_flip=np.array(single_flip, np.bool_),
|
||||
merged_sites=merged_sites,
|
||||
merged_rows=np.array(merged_rows, np.int64),
|
||||
site_class=cls,
|
||||
contigs=contigs,
|
||||
header=header,
|
||||
site_beg=beg,
|
||||
site_end=end,
|
||||
site_contig=[s.contig for s in sites],
|
||||
site_id=[s.id for s in sites],
|
||||
site_alleles=[[s.ref, *s.alts] for s in sites],
|
||||
single_ab=np.array(single_ab, np.int16).reshape(-1, 2),
|
||||
inf1_nc_informative=opts.inf1_nocall_informative,
|
||||
ploidy_policy=opts.ploidy_policy,
|
||||
filter_gs=opts.filter_gs,
|
||||
)
|
||||
return table, audit, skips, genome
|
||||
|
||||
|
||||
def _sha(path: Path | None) -> str:
|
||||
return sha256_file(path) if path else ""
|
||||
|
||||
|
||||
def vcf_header(opts: VcfOptions, genome: Any, build: str, contigs: list[str], rm: RunManifest) -> str:
|
||||
"""OUT-011 (without the ##SAMPLE and #CHROM lines, which are per sample)."""
|
||||
|
||||
def lst(p: Path | None) -> str:
|
||||
return "none" if p is None else f"{p.name}:{hashlib.sha256(p.read_bytes()).hexdigest()[:16]}"
|
||||
|
||||
lines = [
|
||||
"##fileformat=VCFv4.3",
|
||||
f"##source={PRODUCT_NAME} {__version__}",
|
||||
f"##reference=file://{Path(opts.fasta).resolve()}",
|
||||
f"##simurg_genome_build={build}",
|
||||
f"##simurg_manifest={opts.bpm.name};sha256={_sha(opts.bpm)}",
|
||||
]
|
||||
if opts.csv:
|
||||
lines.append(f"##simurg_csv_manifest={opts.csv.name};sha256={_sha(opts.csv)}")
|
||||
lines.append("##simurg_cluster_file=__CLUSTER__")
|
||||
lines.append(
|
||||
"##simurg_options="
|
||||
f"unsquash_duplicates={str(opts.unsquash_duplicates).lower()};exclude_loci={lst(opts.exclude_loci)};"
|
||||
f"include_loci={lst(opts.include_loci)};inf1_nocall_informative={str(opts.inf1_nocall_informative).lower()};"
|
||||
f"auxiliary_loci=none;ploidy_policy={opts.ploidy_policy};contig_alias={opts.contig_alias}"
|
||||
+ (";drop_ambiguous=true" if opts.drop_ambiguous else "")
|
||||
+ (";unknown_sex_policy=diploid_x" if opts.ploidy_policy == "sex-aware" else "")
|
||||
)
|
||||
lines.append(f"##simurg_run={rm.run_id}")
|
||||
for c in contigs:
|
||||
lines.append(f"##contig=<ID={c},length={genome.contig_length(c)}>")
|
||||
lines.append('##FILTER=<ID=PASS,Description="All filters passed">')
|
||||
if opts.filter_gs is not None:
|
||||
lines.append(f'##FILTER=<ID=LowGS,Description="GenCall score below {opts.filter_gs}">')
|
||||
lines += [
|
||||
'##FORMAT=<ID=GT,Number=1,Type=String,Description="Genotype">',
|
||||
'##FORMAT=<ID=GS,Number=1,Type=Float,Description="GenCall score. For records merged from several assays, '
|
||||
'the minimum GenCall score over the contributing assays">',
|
||||
'##FORMAT=<ID=BAF,Number=1,Type=Float,Description="B allele frequency, oriented so that 1 means homozygous '
|
||||
'ALT (REF=A, ALT=B relative to the reference genome)">',
|
||||
'##FORMAT=<ID=LRR,Number=1,Type=Float,Description="Log R ratio">',
|
||||
]
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------ per sample
|
||||
_W: dict[str, Any] = {}
|
||||
|
||||
|
||||
def _init(table: SiteTable) -> None:
|
||||
_W["t"] = table
|
||||
|
||||
|
||||
def _f4(v: float) -> str:
|
||||
return "." if math.isnan(v) else f"{v:.4f}"
|
||||
|
||||
|
||||
def _g8(v: float) -> str:
|
||||
return "." if math.isnan(v) else f"{v:.8g}"
|
||||
|
||||
|
||||
def write_sample(task: tuple[str, str, str, str, bool]) -> dict[str, Any]:
|
||||
"""Worker: one GTC → one VCF (+ index). Returns counts for the run summary."""
|
||||
from simurg.formats.gtc import read_gtc
|
||||
|
||||
sample_id, sample_name, gtc_path, out_path, bgzip = task
|
||||
t: SiteTable = _W["t"]
|
||||
res: dict[str, Any] = {"sample_id": sample_id, "records": 0, "reasons": Counter(), "merge_failures": []}
|
||||
try:
|
||||
g = read_gtc(gtc_path)
|
||||
n = len(t.prefix)
|
||||
sex = g.gender_char
|
||||
ploidy = ploidy_for(t.site_class, sex, t.ploidy_policy)
|
||||
gt = np.empty(n, dtype=object)
|
||||
gs = np.full(n, np.nan, np.float64)
|
||||
baf = np.full(n, np.nan, np.float64)
|
||||
lrr = np.full(n, np.nan, np.float64)
|
||||
has_baf = len(g.b_allele_freqs) == g.num_snps
|
||||
# single-probe sites (vectorised)
|
||||
rows, loc = t.single_rows, t.single_locus
|
||||
calls = g.genotypes[loc].astype(np.int64)
|
||||
pl = ploidy[rows]
|
||||
pidx = np.where(pl == 2, 1, 0)
|
||||
gt[rows] = t.single_gt[np.arange(len(rows)), pidx, calls]
|
||||
gt[rows[pl == 0]] = "."
|
||||
gs[rows] = g.genotype_scores[loc].astype(np.float64)
|
||||
if has_baf:
|
||||
b = g.b_allele_freqs[loc].astype(np.float32)
|
||||
b = np.where(t.single_flip, (np.float32(1.0) - b).astype(np.float32), b)
|
||||
baf[rows] = b.astype(np.float64)
|
||||
lrr[rows] = g.logr_ratios[loc].astype(np.float64)
|
||||
het_haploid = int(((calls == 2) & (pl == 1)).sum())
|
||||
if het_haploid:
|
||||
res["reasons"]["het_in_haploid"] += het_haploid
|
||||
# merged sites
|
||||
for s, k in zip(t.merged_sites, t.merged_rows.tolist(), strict=True):
|
||||
c = tuple(int(g.genotypes[p.index]) for p in s.probes)
|
||||
p_k = int(ploidy[k])
|
||||
gs[k] = float(min(float(g.genotype_scores[p.index]) for p in s.probes))
|
||||
if p_k == 0:
|
||||
gt[k] = "."
|
||||
continue
|
||||
o = combine_genotype(s, c, p_k, t.inf1_nc_informative)
|
||||
gt[k] = gt_text(o, p_k)
|
||||
if o.reason:
|
||||
res["reasons"][o.reason] += 1
|
||||
if o.reason.startswith("W_MERGE") and o.reason != "W_MERGE_ALL_NC":
|
||||
res["merge_failures"].append((s.id, c, o.reason))
|
||||
merged_mask = np.zeros(n, np.bool_)
|
||||
merged_mask[t.merged_rows] = True
|
||||
header = t.header.replace("__CLUSTER__", g.cluster_file or "") + (
|
||||
f"##SAMPLE=<ID={sample_name},SentrixID={sample_id}>\n"
|
||||
f"#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t{sample_name}\n"
|
||||
)
|
||||
low = t.filter_gs
|
||||
gsl, bafl, lrrl = gs.tolist(), baf.tolist(), lrr.tolist()
|
||||
tbx = TabixBuilder()
|
||||
out = Path(out_path)
|
||||
with atomic_open(out, "wb") as raw:
|
||||
if bgzip:
|
||||
w = BgzfWriter(raw) # type: ignore[arg-type]
|
||||
w.write_record(header.encode())
|
||||
else:
|
||||
raw.write(header.encode()) # type: ignore[arg-type]
|
||||
for k in range(n):
|
||||
filt = "LowGS" if low is not None and not math.isnan(gsl[k]) and gsl[k] < low else "PASS"
|
||||
if merged_mask[k]:
|
||||
body = f"{filt}\t.\tGT:GS\t{gt[k]}:{_f4(gsl[k])}\n"
|
||||
else:
|
||||
body = f"{filt}\t.\tGT:GS:BAF:LRR\t{gt[k]}:{_f4(gsl[k])}:{_g8(bafl[k])}:{_g8(lrrl[k])}\n"
|
||||
line = t.prefix[k] + body.encode()
|
||||
if bgzip:
|
||||
vs, ve = w.write_record(line)
|
||||
tbx.add(t.site_contig[k], int(t.site_beg[k]), int(t.site_end[k]), vs, ve)
|
||||
else:
|
||||
raw.write(line) # type: ignore[arg-type]
|
||||
if bgzip:
|
||||
w.close()
|
||||
if bgzip:
|
||||
write_tbi(Path(str(out) + ".tbi"), tbx)
|
||||
res["records"] = n
|
||||
res["sex"] = sex
|
||||
except SimurgError as e:
|
||||
res["error"] = f"{e.code}: {e.message}"
|
||||
except Exception as e:
|
||||
res["error"] = f"E_INTERNAL: {type(e).__name__}: {e}"
|
||||
return res
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------ command
|
||||
def run_gtc_to_vcf(opts: VcfOptions, rep: Reporter) -> int:
|
||||
from simurg.formats.discovery import discover_gtcs
|
||||
from simurg.formats.samplesheet import output_stem, parse_samplesheet
|
||||
from simurg.pipeline.inputs import check_paths
|
||||
|
||||
rm = RunManifest(command="gtc-to-vcf", argv=opts.argv)
|
||||
rep.run_id = rm.run_id
|
||||
check_paths(
|
||||
[
|
||||
(opts.bpm, "BPM"),
|
||||
(opts.fasta, "FASTA"),
|
||||
(opts.csv, "CSV manifest"),
|
||||
(opts.gtcs, "GTC folder"),
|
||||
(opts.sheet, "sample sheet"),
|
||||
(opts.exclude_loci, "locus list"),
|
||||
(opts.include_loci, "locus list"),
|
||||
]
|
||||
)
|
||||
if opts.ploidy_policy not in ("sex-aware", "diploid"):
|
||||
raise SimurgError("E_CONFIG_KEY", "--ploidy-policy must be sex-aware or diploid")
|
||||
sheet = parse_samplesheet(opts.sheet) if opts.sheet else None
|
||||
root = opts.gtcs or (sheet.root_folder if sheet else None)
|
||||
if root is None:
|
||||
raise SimurgError("E_NO_INPUT", "give --gtcs or a sample sheet with a RootFolder")
|
||||
found = discover_gtcs(root)
|
||||
samples: list[tuple[str, str, str]] = [] # (sample_id, sample_name, path)
|
||||
if sheet is not None:
|
||||
for r in sheet.rows:
|
||||
if r.sample_id in found:
|
||||
samples.append((r.sample_id, r.sample_name or r.sample_id, str(found[r.sample_id])))
|
||||
else:
|
||||
rep.warn("W_QC_UNMATCHED_SAMPLES", f"{r.sample_id}: no GTC under {root}", sample_id=r.sample_id)
|
||||
else:
|
||||
samples = [(k, k, str(v)) for k, v in sorted(found.items())]
|
||||
suffix = ".snv.vcf.gz" if opts.bgzip else ".snv.vcf"
|
||||
outs = (
|
||||
[opts.out / "cohort.snv.bcf"]
|
||||
if opts.merge
|
||||
else [opts.out / f"{output_stem(name)}{suffix}" for _, name, _ in samples]
|
||||
)
|
||||
check_outputs_absent([*outs, opts.out / "audit" / "loci.parquet"], force=opts.force)
|
||||
if not opts.bgzip:
|
||||
rep.warn("W_NO_BGZIP", "writing plain VCF without index; pgx and cn commands need bgzipped, indexed VCFs")
|
||||
rep.info(f"mapping manifest {opts.bpm.name} to {opts.fasta.name}")
|
||||
table, audit, skips, _genome = build_sites(opts, rep, rm)
|
||||
for role, p in (("bpm", opts.bpm), ("csv_manifest", opts.csv), ("fasta", opts.fasta), ("sample_sheet", opts.sheet)):
|
||||
if p is not None:
|
||||
rm.add_input(role, p)
|
||||
opts.out.mkdir(parents=True, exist_ok=True)
|
||||
write_audit(opts.out / "audit" / "loci.parquet", audit)
|
||||
if opts.merge:
|
||||
n = write_cohort_bcf(table, samples, opts.out / "cohort.snv.bcf", opts, rep)
|
||||
rm.summary = {"samples": len(samples), "records": n, "skipped_loci_by_code": dict(skips)}
|
||||
rm.add_output(opts.out / "cohort.snv.bcf", "bcf/2.2")
|
||||
rm.status = "completed"
|
||||
rm.write(opts.out / "run_manifest.json")
|
||||
rep.info(f"done: cohort BCF with {len(samples)} sample(s), {n} records")
|
||||
return ExitCode.OK
|
||||
workers = max(1, min(opts.threads or (os.cpu_count() or 1), len(samples)))
|
||||
tasks = [(sid, name, path, str(o), opts.bgzip) for (sid, name, path), o in zip(samples, outs, strict=True)]
|
||||
rep.info(f"{len(table.prefix)} VCF records; writing {len(tasks)} sample(s) with {workers} worker(s)")
|
||||
if workers == 1:
|
||||
_init(table)
|
||||
results = [write_sample(t) for t in tasks]
|
||||
else:
|
||||
with ProcessPoolExecutor(
|
||||
workers, mp_context=mp.get_context("spawn"), initializer=_init, initargs=(table,)
|
||||
) as ex:
|
||||
results = list(ex.map(write_sample, tasks))
|
||||
reasons: Counter[str] = Counter()
|
||||
failures = []
|
||||
for res in results:
|
||||
if "error" in res:
|
||||
rep.error(res["error"].split(":", 1)[0], f"{res['sample_id']}: {res['error']}", sample_id=res["sample_id"])
|
||||
continue
|
||||
reasons.update(res["reasons"])
|
||||
failures += [(res["sample_id"], *f) for f in res["merge_failures"]]
|
||||
for code, n in sorted(reasons.items()):
|
||||
if code.startswith("W_"):
|
||||
rep.warn(code, f"{n} genotype(s): {code}")
|
||||
write_merge_audit(opts.out / "audit" / "merges.parquet", failures)
|
||||
ok = sum("error" not in r for r in results)
|
||||
rm.summary = {"samples": len(results), "succeeded": ok, "records_per_sample": len(table.prefix),
|
||||
"skipped_loci_by_code": dict(skips), "genotype_notes": dict(reasons)} # fmt: skip
|
||||
for o in outs:
|
||||
if o.exists():
|
||||
rm.add_output(o, "vcf/4.3")
|
||||
rm.status = "completed"
|
||||
rm.write(opts.out / "run_manifest.json")
|
||||
rep.info(f"done: {ok}/{len(results)} VCF(s) in {opts.out}")
|
||||
if ok == len(results):
|
||||
return ExitCode.OK
|
||||
return ExitCode.PARTIAL if ok else ExitCode.ALL_FAILED
|
||||
|
||||
|
||||
def write_audit(path: Path, audit: list[AuditRow]) -> None:
|
||||
import pyarrow as pa
|
||||
|
||||
from simurg.core.schemas import write_parquet
|
||||
|
||||
cols = list(AuditRow.__dataclass_fields__)
|
||||
table = pa.table({c: [getattr(a, c) for a in audit] for c in cols})
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
write_parquet(table, path)
|
||||
|
||||
|
||||
def write_merge_audit(path: Path, rows: list[tuple[str, str, tuple[int, ...], str]]) -> None:
|
||||
import pyarrow as pa
|
||||
|
||||
from simurg.core.schemas import write_parquet
|
||||
|
||||
table = pa.table(
|
||||
{
|
||||
"sample_id": [r[0] for r in rows],
|
||||
"site_id": [r[1] for r in rows],
|
||||
"calls": ["/".join(str(c) for c in r[2]) for r in rows],
|
||||
"reason": [r[3] for r in rows],
|
||||
}
|
||||
)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
write_parquet(table, path)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------ cohort BCF
|
||||
def _f32_payload(v: npt.NDArray[np.float32]) -> bytes:
|
||||
u = np.asarray(v, dtype="<f4").view("<u4").copy()
|
||||
u[np.isnan(np.asarray(v, dtype=np.float32))] = 0x7F800001
|
||||
return u.tobytes()
|
||||
|
||||
|
||||
def write_cohort_bcf(
|
||||
t: SiteTable, samples: list[tuple[str, str, str]], out: Path, opts: VcfOptions, rep: Reporter
|
||||
) -> int:
|
||||
"""OUT-040: one multi-sample BCF (+ .csi), same records and FORMAT rules as the per-sample VCFs."""
|
||||
from simurg.formats.bcf import (
|
||||
FLOAT,
|
||||
INT8,
|
||||
BcfWriter,
|
||||
CsiBuilder,
|
||||
_type_byte,
|
||||
encode_record,
|
||||
header_dictionaries,
|
||||
)
|
||||
from simurg.formats.bpm import read_bpm
|
||||
from simurg.pipeline.qc import default_tmp, load_cohort
|
||||
|
||||
n_loci = read_bpm(opts.bpm).num_loci
|
||||
workers = max(1, opts.threads or (os.cpu_count() or 1))
|
||||
store, infos = load_cohort(
|
||||
[(sid, path) for sid, _, path in samples],
|
||||
n_loci,
|
||||
default_tmp(out.parent),
|
||||
baf=True,
|
||||
scores=True,
|
||||
lrr=True,
|
||||
workers=workers,
|
||||
)
|
||||
try:
|
||||
for s in infos:
|
||||
if s.error:
|
||||
raise SimurgError("E_GTC_UNSUITABLE", f"{s.sample_id}: {s.error}")
|
||||
names = [name for _, name, _ in samples]
|
||||
cluster = read_cluster_name(samples[0][2])
|
||||
header = t.header.replace("__CLUSTER__", cluster)
|
||||
header += "".join(f"##SAMPLE=<ID={name},SentrixID={sid}>\n" for sid, name, _ in samples)
|
||||
header += "#CHROM\tPOS\tID\tREF\tALT\tQUAL\tFILTER\tINFO\tFORMAT\t" + "\t".join(names) + "\n"
|
||||
strings, contigs = header_dictionaries(header)
|
||||
k_gt, k_gs, k_baf, k_lrr = strings["GT"], strings["GS"], strings["BAF"], strings["LRR"]
|
||||
f_pass, f_low = strings["PASS"], strings.get("LowGS", -1)
|
||||
n = len(names)
|
||||
sexes = [s.sex_est for s in infos]
|
||||
ploidy = np.stack([ploidy_for(t.site_class, sx, t.ploidy_policy) for sx in sexes], axis=1) # sites × samples
|
||||
single_pos = np.full(len(t.prefix), -1, np.int64)
|
||||
single_pos[t.single_rows] = np.arange(len(t.single_rows))
|
||||
merged_pos = {int(k): j for j, k in enumerate(t.merged_rows.tolist())}
|
||||
csi = CsiBuilder(len(contigs))
|
||||
fdesc = _type_byte(1, FLOAT)
|
||||
written = 0
|
||||
with atomic_open(out, "wb") as raw:
|
||||
w = BcfWriter(raw, header)
|
||||
chunk = 20000
|
||||
for start in range(0, len(t.prefix), chunk):
|
||||
rows = range(start, min(len(t.prefix), start + chunk))
|
||||
need = {int(t.single_locus[single_pos[k]]) for k in rows if single_pos[k] >= 0}
|
||||
for k in rows:
|
||||
if k in merged_pos:
|
||||
need.update(p.index for p in t.merged_sites[merged_pos[k]].probes)
|
||||
loci = sorted(need)
|
||||
col = {loc: j for j, loc in enumerate(loci)}
|
||||
sel = np.array(loci, np.int64)
|
||||
gt = np.asarray(store.gt[:, sel]) if len(sel) else np.zeros((n, 0), np.uint8)
|
||||
sc = np.asarray(store.score[:, sel]) if len(sel) else np.zeros((n, 0), np.float32) # type: ignore[index]
|
||||
bf = np.asarray(store.baf[:, sel]) if len(sel) else np.zeros((n, 0), np.float32) # type: ignore[index]
|
||||
lr = np.asarray(store.lrr[:, sel]) if len(sel) else np.zeros((n, 0), np.float32) # type: ignore[index]
|
||||
for k in rows:
|
||||
pl = ploidy[k]
|
||||
if single_pos[k] >= 0:
|
||||
j = int(single_pos[k])
|
||||
c = col[int(t.single_locus[j])]
|
||||
calls = gt[:, c].astype(np.int64)
|
||||
ia, ib = (int(x) for x in t.single_ab[j]) # type: ignore[index]
|
||||
a1 = np.where(calls == 3, ib, ia)
|
||||
a2 = np.where(calls == 1, ia, ib)
|
||||
lo, hi = np.minimum(a1, a2), np.maximum(a1, a2)
|
||||
width = max(1, int(pl.max()))
|
||||
cells = np.full((n, width), -127, np.int8)
|
||||
miss = (calls == 0) | (pl == 0) | ((pl == 1) & (calls == 2))
|
||||
hap = pl <= 1
|
||||
cells[:, 0] = np.where(miss, 0, ((np.where(hap & (calls == 3), ib, lo) + 1) << 1)).astype(
|
||||
np.int8
|
||||
)
|
||||
if width == 2:
|
||||
second = np.where(miss, 0, (hi + 1) << 1).astype(np.int8)
|
||||
cells[:, 1] = np.where(hap, np.int8(-127), second)
|
||||
gs = sc[:, c]
|
||||
b = bf[:, c]
|
||||
b = np.where(t.single_flip[j], (np.float32(1.0) - b).astype(np.float32), b)
|
||||
fmt = [(k_gt, _type_byte(width, INT8), cells.tobytes()), (k_gs, fdesc, _f32_payload(gs)),
|
||||
(k_baf, fdesc, _f32_payload(b)), (k_lrr, fdesc, _f32_payload(lr[:, c]))] # fmt: skip
|
||||
gs_min = gs
|
||||
else:
|
||||
site = t.merged_sites[merged_pos[k]]
|
||||
cs = [col[p.index] for p in site.probes]
|
||||
g_all = gt[:, cs].astype(np.int64)
|
||||
gs_min = sc[:, cs].min(axis=1)
|
||||
width = max(1, int(pl.max()))
|
||||
cells = np.full((n, width), -127, np.int8)
|
||||
for si in range(n):
|
||||
p_k = int(pl[si])
|
||||
if p_k == 0:
|
||||
cells[si, 0] = 0
|
||||
continue
|
||||
o = combine_genotype(site, tuple(int(x) for x in g_all[si]), p_k, t.inf1_nc_informative)
|
||||
if o.alleles is None:
|
||||
cells[si, :p_k] = 0
|
||||
else:
|
||||
for q, a in enumerate(o.alleles):
|
||||
cells[si, q] = (a + 1) << 1
|
||||
fmt = [(k_gt, _type_byte(width, INT8), cells.tobytes()), (k_gs, fdesc, _f32_payload(gs_min))]
|
||||
low = opts.filter_gs is not None and bool(np.nanmin(gs_min) < opts.filter_gs) if n else False
|
||||
rec = encode_record(contigs[t.site_contig[k]], int(t.site_beg[k]),
|
||||
int(t.site_end[k] - t.site_beg[k]), t.site_id[k], t.site_alleles[k],
|
||||
f_low if low and f_low >= 0 else f_pass, n, fmt) # fmt: skip
|
||||
vs, ve = w.write(rec)
|
||||
csi.add(contigs[t.site_contig[k]], int(t.site_beg[k]), int(t.site_end[k]), vs, ve)
|
||||
written += 1
|
||||
w.close()
|
||||
Path(str(out) + ".csi").write_bytes(csi.encode())
|
||||
finally:
|
||||
store.close()
|
||||
return written
|
||||
|
||||
|
||||
def read_cluster_name(gtc_path: str) -> str:
|
||||
from simurg.formats.gtc import read_gtc
|
||||
|
||||
return read_gtc(gtc_path, check_write_complete=False).cluster_file or ""
|
||||
Reference in new issue
Block a user