P6 complete: optional VCF fields, SNV+indel merging, auxiliary MNV loci, PLINK 2 .pgen
- gtc-to-vcf --info-fields ASSAY_TYPE,NORM_BIN,GENTRAIN,CLUSTER_SEP,ILMN_STRAND,PROBE_IDS (--egt) and --extra-format X,Y,NORMX,NORMY,THETA,R, in per-sample VCFs and the --merge BCF (OUT-012, plan §8.3); equal to gtc2vcf's values (THETA/R missing where both normalized intensities are 0, K-023) - --merge-snv-indel (OUT-021, EDGE-013): alleles extended to the longest REF; SNV probes read the base at POS - --auxiliary-loci (OUT-035, EDGE-015): rules (a)-(e), E_AUX_INVALID; rule (d) uses the last plus-strand base for minus-strand designs (spec amended, as GTCtoVCF) - export plink --format bed|pgen|both: PLINK 2 fixed-width .pgen/.pvar/.psam; plink2 reads it back to a .bed byte-identical to ours - cohort store keeps raw/normalized intensities for the BCF path; tests (unit + gtc2vcf oracle); docs
This commit is contained in:
kemal committed
1 parent
d3add008f7
commit
1001bf951e
13 files changed
+611
-50
No files matched your search
@@ -148,3 +148,74 @@ def test_bedgraph_equals_gtc(tmp_path: Path) -> None: # OUT-050, OUT-052, EDGE-
|
||||
assert v == f"{float(g.logr_ratios[single[key]]):.6g}"
|
||||
checked += 1
|
||||
assert checked > 500_000
|
||||
|
||||
|
||||
def test_info_and_extra_format_match_gtc2vcf(tmp_path: Path) -> None: # OUT-012, plan §8.3
|
||||
"""INFO ASSAY_TYPE/GENTRAIN/CLUSTER_SEP and FORMAT X/Y/NORMX/NORMY equal gtc2vcf's; THETA/R too, except where the
|
||||
normalized intensities are both 0 (SimurgArray: missing, plan §7.4; gtc2vcf: 0)."""
|
||||
from tests.conftest import GSA_EGT
|
||||
|
||||
plugin = BCFTOOLS.parent / "plugins" / "gtc2vcf.so"
|
||||
need(GSA_BPM, GSA_CSV, GSA_EGT, FASTA, ORACLE_GTCS)
|
||||
if not plugin.exists():
|
||||
pytest.skip("gtc2vcf plugin not built")
|
||||
import os
|
||||
|
||||
from typer.testing import CliRunner
|
||||
|
||||
from simurg.cli.main import app
|
||||
|
||||
gtcs = tmp_path / "g"
|
||||
gtcs.mkdir()
|
||||
src = sorted(ORACLE_GTCS.glob("*.gtc"))[0]
|
||||
(gtcs / src.name).symlink_to(src)
|
||||
out = tmp_path / "v"
|
||||
res = CliRunner().invoke(
|
||||
app,
|
||||
["gtc-to-vcf", "--bpm", str(GSA_BPM), "--csv", str(GSA_CSV), "--fasta", str(FASTA), "--gtcs", str(gtcs),
|
||||
"--out", str(out), "--threads", "1", "--unsquash-duplicates", "--quiet", "--egt", str(GSA_EGT),
|
||||
"--info-fields", "ASSAY_TYPE,GENTRAIN,CLUSTER_SEP", "--extra-format", "X,Y,NORMX,NORMY,THETA,R"],
|
||||
) # fmt: skip
|
||||
assert res.exit_code == 0, res.output
|
||||
env = dict(os.environ, BCFTOOLS_PLUGINS=str(plugin.parent))
|
||||
g2v = subprocess.run(
|
||||
[str(BCFTOOLS), "+gtc2vcf", "--no-version", "-t", "GT,X,Y,NORMX,NORMY,THETA,R", "-b", str(GSA_BPM), "-c",
|
||||
str(GSA_CSV), "-e", str(GSA_EGT), "-f", str(FASTA), "-g", str(gtcs), "-O", "v"],
|
||||
capture_output=True, text=True, env=env, check=True,
|
||||
).stdout # fmt: skip
|
||||
ref = {}
|
||||
for line in g2v.splitlines():
|
||||
if line.startswith("#"):
|
||||
continue
|
||||
c = line.split("\t")
|
||||
info = dict(kv.split("=", 1) for kv in c[7].split(";") if "=" in kv)
|
||||
ref[c[2]] = (info, dict(zip(c[8].split(":"), c[9].split(":"), strict=True)))
|
||||
|
||||
def close(a: str, b: str) -> bool:
|
||||
na, nb = a in (".", "") or "nan" in a.lower(), b in (".", "") or "nan" in b.lower()
|
||||
if na or nb:
|
||||
return na and nb
|
||||
return abs(float(a) - float(b)) <= 1e-5 * max(1.0, abs(float(b)))
|
||||
|
||||
checked = zero_norm = 0
|
||||
with gzip.open(out / f"{src.stem}.snv.vcf.gz", "rt") as fh:
|
||||
for line in fh:
|
||||
if line.startswith("#"):
|
||||
continue
|
||||
c = line.rstrip("\n").split("\t")
|
||||
if "," in c[2] or ";" in c[2] or c[2] not in ref:
|
||||
continue
|
||||
info = dict(kv.split("=", 1) for kv in c[7].split(";"))
|
||||
fmt = dict(zip(c[8].split(":"), c[9].split(":"), strict=True))
|
||||
ri, rf = ref[c[2]]
|
||||
assert info["ASSAY_TYPE"] == ri["ASSAY_TYPE"]
|
||||
assert close(info["GENTRAIN"], ri["GenTrain_Score"]) and close(info["CLUSTER_SEP"], ri["Cluster_Sep"])
|
||||
for k in ("X", "Y", "NORMX", "NORMY"):
|
||||
assert close(fmt[k], rf[k]), (c[2], k, fmt[k], rf[k])
|
||||
if fmt["NORMX"] == "0" and fmt["NORMY"] == "0":
|
||||
zero_norm += 1
|
||||
assert fmt["THETA"] == "." and float(rf["THETA"]) == 0.0
|
||||
else:
|
||||
assert close(fmt["THETA"], rf["THETA"]) and close(fmt["R"], rf["R"]), (c[2], fmt, rf)
|
||||
checked += 1
|
||||
assert checked > 600_000 and zero_norm < 1000
|
||||
@@ -232,3 +232,64 @@ def test_ploidy_rules() -> None: # OUT-018, EDGE-038
|
||||
def test_plink_codes() -> None: # OUT-043
|
||||
assert [bed_code(g, 1) for g in ("0/0", "0/1", "1/1", "./.", "1", "0")] == [3, 2, 0, 1, 0, 3]
|
||||
assert bed_code("0/2", 1) == 3 and bed_code("0/2", 2) == 2 # split multi-allelic record
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------------ P6 options
|
||||
def test_merge_snv_indel_edge_013() -> None: # OUT-021
|
||||
snv = ProbeSite(0, "snv", "1", 5, "A", ("A", "G"), "snv", 2, "1")
|
||||
dele = ProbeSite(1, "del", "1", 5, "ACT", ("ACT", "A"), "indel", 1, "1")
|
||||
sep = group_sites([snv, dele], {"1": 0})
|
||||
assert len(sep) == 2 # default: separate records
|
||||
(s,) = group_sites([snv, dele], {"1": 0}, merge_snv_indel=True)
|
||||
assert s.ref == "ACT" and s.alts == ["GCT", "A"] and s.id == "snv;del"
|
||||
ext_snv, ext_del = s.probes
|
||||
assert ext_del.alleles == ("ACT", "A")
|
||||
assert ext_snv.alleles == ("ACT", "GCT")
|
||||
# SNV probe homozygous A (sees the anchor base of the deletion too), deletion probe het → REF/DEL
|
||||
assert gt_text(genotype(s, (AA, AB), 2), 2) == "0/2"
|
||||
# SNV het A/G, deletion hom REF → REF/SNV
|
||||
assert gt_text(genotype(s, (AB, AA), 2), 2) == "0/1"
|
||||
# incompatible REFs (not prefixes of each other) stay separate
|
||||
other = ProbeSite(2, "x", "1", 5, "GT", ("GT", "G"), "indel", 1, "1")
|
||||
assert len(group_sites([snv, other], {"1": 0}, merge_snv_indel=True)) == 2
|
||||
|
||||
|
||||
def _aux_probe(alleles: tuple[str, str] = ("A", "C"), idx: int = 0, name: str = "mnv") -> ProbeSite:
|
||||
return ProbeSite(idx, name, "1", 5, "A", alleles, "snv", 2, "1")
|
||||
|
||||
|
||||
def test_auxiliary_loci_rules() -> None: # OUT-035, EDGE-015
|
||||
from simurg.harmonize.sites import AuxRecord, apply_auxiliary
|
||||
|
||||
genome = "TTTTATGCC" # positions 5-6 = AT
|
||||
f = fetcher(genome)
|
||||
plus = lambda i: "+" # noqa: E731
|
||||
ok = AuxRecord("mnv", "1", 5, "AT", ["CG"])
|
||||
probes, errs = apply_auxiliary([_aux_probe()], [ok], f, lambda c: c, plus)
|
||||
assert not errs and probes[0].ref == "AT" and probes[0].alleles == ("AT", "CG") and probes[0].reason == "aux_locus"
|
||||
cases = {
|
||||
"(a)": AuxRecord("mnv", "1", 5, "AT", ["CG", "GG"]),
|
||||
"(b)": AuxRecord("mnv", "1", 5, "AT", ["C"]),
|
||||
"(d)": AuxRecord("mnv", "1", 5, "AT", ["TG"]),
|
||||
"(e)": AuxRecord("mnv", "1", 5, "AG", ["CG"]),
|
||||
}
|
||||
for rule, rec in cases.items():
|
||||
_, errs = apply_auxiliary([_aux_probe()], [rec], f, lambda c: c, plus)
|
||||
assert len(errs) == 1 and rule in errs[0], (rule, errs)
|
||||
dup = [_aux_probe(), _aux_probe(idx=1, name="twin")]
|
||||
_, errs = apply_auxiliary(dup, [ok], f, lambda c: c, plus)
|
||||
assert "(c)" in errs[0]
|
||||
_, errs = apply_auxiliary([_aux_probe()], [AuxRecord("nope", "1", 5, "AT", ["CG"])], f, lambda c: c, plus)
|
||||
assert "not a probe" in errs[0]
|
||||
# minus-strand design: the interrogated base is the MNV's last base on the plus strand
|
||||
probes, errs = apply_auxiliary([_aux_probe(("T", "G"))], [AuxRecord("mnv", "1", 5, "AT", ["CG"])], f,
|
||||
lambda c: c, lambda i: "-") # fmt: skip
|
||||
assert not errs and probes[0].alleles == ("AT", "CG")
|
||||
|
||||
|
||||
def test_pgen_codes() -> None: # plan §8.4
|
||||
from simurg.pipeline.export import PGEN_FROM_BED, _pack
|
||||
|
||||
# PLINK 1 (A1 = ALT): 0 hom ALT, 1 missing, 2 het, 3 hom REF → PLINK 2: 2, 3, 1, 0
|
||||
assert list(PGEN_FROM_BED) == [2, 3, 1, 0]
|
||||
assert _pack([0, 1, 2, 3, 0], 2) == bytes([0b11100100, 0b00000000])
|
||||
Reference in new issue
Block a user