PGx star alleles: pgx star-allele call (PGX-011..021), probe masks vs 1000G, GeT-RM validation (K-029)

- simurg/pgx/star.py: solver-free diplotype enumeration incl. CN forms (deletion, xN), wildcards, array-equivalence
  classes, raw_score/score with CPIC frequency priors; calibrated error model (spec PGX-012 amended).
- pipeline/pgx_star.py: DB gene models, SNV + PGx CN VCF evidence, masks, diplotype_summary.tsv + pgx_calls.json.
- scripts/pgx: derive_masks.py (array vs 1000G 30x over HTTPS, GS calibration), getrm_truth.py,
  validate_star_getrm.py.
- GSA v4 PGx demo vs GeT-RM (16 CEPH): 100/109 gene calls; CYP2D6/2C19/2C9 41/45 (docs/validation/V2_pgx_star.md).

scripts/pgx/build_cn_regions.py

@@ -49,12 +49,12 @@ def main() -> int:
     rows: list[tuple[str, int, int, str, str]] = []
 
     def exon(gene: str, k: int) -> tuple[int, int]:
-        chrom, strand, s, e = tx[gene]
+        _, strand, s, e = tx[gene]
         i = k - 1 if strand == "+" else len(s) - k
         return s[i], e[i]
 
     def intron(gene: str, k: int) -> tuple[int, int]:
-        chrom, strand, s, e = tx[gene]
+        strand = tx[gene][1]
         a_, b_ = exon(gene, k), exon(gene, k + 1)
         return (a_[1], b_[0]) if strand == "+" else (b_[1], a_[0])
 

scripts/pgx/derive_masks.py

@@ -0,0 +1,157 @@
+#!/usr/bin/env python3
+"""Derive a PGx probe mask from array-vs-WGS concordance (spec PGX-010) for the individuals of a GSA PGx batch.
+
+    python scripts/pgx/derive_masks.py --db DB.sqlite --vcfs VCF_DIR --identity demo36_identity.tsv \
+        --exclude-individuals getrm.txt --manifest NAME --out MASK.tsv [--min-concordance 0.95]
+
+Truth: the 1000 Genomes 30x phased panel (NYGC, 3202 samples), fetched per gene window over HTTPS with pysam (MIT;
+a script-only dependency: ``uv pip install pysam``). For every PGx-DB defining variant the array record(s) at that
+site are compared with the truth alt dosage; records whose concordance is below the threshold (≥ 5 comparisons)
+are masked by their probe names. Individuals listed in ``--exclude-individuals`` (the validation set) are not used,
+so the mask stays independent of the star-allele validation. The spec asks for ≥ 100 samples; this batch has fewer
+(recorded as K-029).
+"""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import gzip
+import sqlite3
+from collections import defaultdict
+from pathlib import Path
+
+PANEL = (
+    "https://ftp.1000genomes.ebi.ac.uk/vol1/ftp/data_collections/1000G_2504_high_coverage/working/"
+    "20220422_3202_phased_SNV_INDEL_SV/1kGP_high_coverage_Illumina.{c}.filtered.SNV_INDEL_SV_phased_panel.vcf.gz"
+)
+
+
+def main() -> int:
+    import pysam
+
+    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--db", required=True, type=Path)
+    ap.add_argument("--vcfs", required=True, type=Path)
+    ap.add_argument("--identity", required=True, type=Path)
+    ap.add_argument("--exclude-individuals", type=Path)
+    ap.add_argument("--manifest", required=True)
+    ap.add_argument("--min-concordance", type=float, default=0.95)
+    ap.add_argument("--out", required=True, type=Path)
+    a = ap.parse_args()
+    ident = {}
+    with a.identity.open() as fh:
+        for r in csv.DictReader((ln for ln in fh if not ln.startswith("#")), delimiter="\t"):
+            ident[r["sample_id"]] = r["individual"]
+    excl = set(a.exclude_individuals.read_text().split()) if a.exclude_individuals else set()
+    use = {s: i for s, i in ident.items() if i not in excl}
+    con = sqlite3.connect(a.db)
+    # the highest population frequency among the alleles that carry each variant (CPIC frequencies)
+    vfreq: dict[tuple[str, int, str, str], float] = defaultdict(float)
+    for c, p, r, al, f in con.execute(
+        "SELECT v.chrom, v.pos, v.ref, v.alt, MAX(f.freq) FROM allele_variants v JOIN frequencies f "
+        "ON f.gene = v.gene AND f.allele = v.allele GROUP BY v.chrom, v.pos, v.ref, v.alt"
+    ):
+        vfreq[(c, p, r, al)] = float(f or 0.0)
+    keys = sorted({(c, p, r, al, g) for g, c, p, r, al in con.execute(
+        "SELECT gene, chrom, pos, ref, alt FROM allele_variants")})  # fmt: skip
+    # array dosages per (chrom, pos, ref, alt) and record ID
+    wanted = defaultdict(set)
+    for c, p, *_ in keys:
+        wanted[c].add(p)
+    arr: dict[tuple[str, int, str, str], dict[str, dict[str, int]]] = defaultdict(lambda: defaultdict(dict))
+    gsd: dict[tuple[str, int, str, str, str, str], float] = {}
+    for sid in sorted(use):
+        path = a.vcfs / f"{sid}.snv.vcf.gz"
+        if not path.exists():
+            continue
+        with gzip.open(path, "rt") as fh:
+            for ln in fh:
+                if ln[0] == "#":
+                    continue
+                c, p, rid, ref, alts = ln.split("\t", 5)[:5]
+                if int(p) not in wanted.get(c, ()):
+                    continue
+                sample = ln.rstrip("\n").split("\t")[9].split(":")
+                gt = sample[0].replace("|", "/").split("/")
+                if "." in gt:
+                    continue
+                gs_v = float(sample[1]) if len(sample) > 1 and sample[1] not in (".", "") else float("nan")
+                for ai, alt in enumerate(alts.split(","), start=1):
+                    arr[(c, int(p), ref, alt)][rid][sid] = sum(int(x) == ai for x in gt)
+                    gsd[(c, int(p), ref, alt, rid, sid)] = gs_v
+    # truth per gene window
+    truth: dict[tuple[str, int, str, str], dict[str, int]] = {}
+    by_gene = defaultdict(list)
+    for c, p, r, al, g in keys:
+        by_gene[(g, c)].append((p, r, al))
+    inds = sorted(set(use.values()))
+    for (g, c), vs in sorted(by_gene.items()):
+        core = c.removeprefix("chr")
+        chrom = "chrM" if core in ("MT", "M") else f"chr{core}"
+        if chrom == "chrM":
+            continue
+        try:
+            vf = pysam.VariantFile(PANEL.format(c=chrom))
+        except (OSError, ValueError):
+            print(f"{g} {chrom}: panel file not available, skipped")
+            continue
+        samples = [s for s in inds if s in vf.header.samples]
+        vf.subset_samples(samples)
+        lo, hi = min(p for p, _, _ in vs) - 10, max(p + len(r) for p, r, _ in vs) + 10
+        want = {(p, r, al) for p, r, al in vs}
+        seen = set()
+        for rec in vf.fetch(chrom, lo, hi):
+            for ai, alt in enumerate(rec.alts or (), start=1):
+                k = (rec.pos, rec.ref, alt)
+                if k in want:
+                    truth[(c, *k)] = {s: sum(x == ai for x in rec.samples[s]["GT"] if x is not None) for s in samples}
+                    seen.add(k)
+        for p, r, al in want - seen:
+            # absent from the (filtered) panel: homozygous reference only for a rare variant; a common one was
+            # most likely filtered out (e.g. CYP2B6 rs2279343 next to CYP2B7) and is not assessed
+            if vfreq.get((c, p, r, al), 0.0) < 0.05:
+                truth[(c, p, r, al)] = dict.fromkeys(samples, 0)
+        print(f"{g} {chrom}: {len(want)} variant(s), {len(seen)} in the panel")
+    rows = []
+    stats = [0, 0]
+    bins = [0.0, 0.15, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 1.01]
+    calib = [[0, 0] for _ in bins[:-1]]  # [discordant, n] per GS bin over unmasked records (error-model calibration)
+    for k, recs in sorted(arr.items()):
+        t = truth.get(k)
+        if t is None:
+            continue
+        for rid, dos in sorted(recs.items()):
+            pairs = [(d, t[use[s]]) for s, d in dos.items() if use[s] in t]
+            if len(pairs) < 5:
+                continue
+            conc = sum(x == y for x, y in pairs) / len(pairs)
+            stats[1] += 1
+            if conc >= a.min_concordance:
+                for s_, d in dos.items():
+                    if use[s_] not in t:
+                        continue
+                    gv = gsd.get((*k, rid, s_), float("nan"))
+                    for bi in range(len(bins) - 1):
+                        if bins[bi] <= gv < bins[bi + 1]:
+                            calib[bi][0] += d != t[use[s_]]
+                            calib[bi][1] += 1
+            if conc < a.min_concordance:
+                stats[0] += 1
+                for probe in rid.split(","):
+                    rows.append((probe, f"concordance {conc:.2f} vs 1000G 30x at {k[0]}:{k[1]} {k[2]}>{k[3]} "
+                                        f"(n={len(pairs)})", a.manifest))  # fmt: skip
+    with a.out.open("w") as fh:
+        fh.write(f"# PGx probe mask for {a.manifest} (PGX-010): array vs 1000G 30x, {len(inds)} individuals, "
+                 f"threshold {a.min_concordance}\n")  # fmt: skip
+        for r in sorted(set(rows)):
+            fh.write("\t".join(r) + "\n")
+    print(f"masked {stats[0]} of {stats[1]} assessed record(s); {len(set(rows))} probe name(s) -> {a.out}")
+    for bi, (dsc, n) in enumerate(calib):
+        if n:
+            print(f"GS [{bins[bi]:.2f}, {bins[bi + 1]:.2f}): discordance {dsc}/{n} = {dsc / n:.4f} (unmasked records)")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/pgx/getrm_truth.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+"""Extract GeT-RM consensus diplotypes (CDC Genetic Testing Reference Material program, consolidated PGx/HLA table,
+public) for the individuals of an identity table.
+
+    python scripts/pgx/getrm_truth.py --xlsx Consolidated_PGx-HLA_table_1-22-25-V4.xlsx \
+        --identity scripts/pgx/demo36_identity.tsv --out getrm_truth.tsv
+Writes ``individual, gene, diplotype`` (the table's text, trimmed) for the star-allele genes.
+"""
+
+from __future__ import annotations
+
+import argparse
+import csv
+from pathlib import Path
+
+GENES = ("CYP2B6", "CYP2C9", "CYP2C19", "CYP2D6", "CYP3A4", "CYP3A5", "CYP4F2", "NAT2", "NUDT15", "SLCO1B1", "TPMT",
+         "UGT1A1", "DPYD", "CYP2A6", "CYP1A2", "CYP2C8", "GSTM1", "GSTT1", "UGT2B17")  # fmt: skip
+
+
+def main() -> int:
+    import openpyxl
+
+    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--xlsx", required=True, type=Path)
+    ap.add_argument("--identity", required=True, type=Path)
+    ap.add_argument("--out", required=True, type=Path)
+    a = ap.parse_args()
+    with a.identity.open() as fh:
+        inds = {r["individual"] for r in csv.DictReader((ln for ln in fh if not ln.startswith("#")), delimiter="\t")}
+    wb = openpyxl.load_workbook(a.xlsx, read_only=True)
+    rows = list(wb[wb.sheetnames[0]].iter_rows(values_only=True))
+    head = [str(h or "").split("\n")[0].strip() for h in rows[1]]
+    col = {g: head.index(g) for g in GENES if g in head}
+    out = []
+    for r in rows[2:]:
+        if not r or not r[1] or str(r[1]).strip() not in inds:
+            continue
+        for g, i in col.items():
+            v = r[i]
+            if v is not None and str(v).strip():
+                out.append((str(r[1]).strip(), g, " ".join(str(v).split())))
+    with a.out.open("w") as fh:
+        fh.write("individual\tgene\tdiplotype\n")
+        for t in sorted(out):
+            fh.write("\t".join(t) + "\n")
+    print(f"{len({t[0] for t in out})} individual(s), {len(out)} gene call(s) -> {a.out}")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/pgx/validate_cn_truth.py

@@ -59,10 +59,12 @@ def main() -> int:
         tot[1] += len(pairs)
         res[region] = {"record": rid, "concordant": ok, "n": len(pairs),
                        "rate": round(ok / len(pairs), 4) if pairs else None,
-                       "truth_to_call_mismatches": dict(collections.Counter(f"{t}->{c}" for t, c in pairs if t != c))}  # fmt: skip
+                       "truth_to_call_mismatches": dict(collections.Counter(
+                           f"{t}->{c}" for t, c in pairs if t != c))}  # fmt: skip
     summary = {"individuals": len(calls), "regions": res, "overall": {"concordant": tot[0], "n": tot[1],
                "rate": round(tot[0] / tot[1], 4) if tot[1] else None},
-               "gstt1_cn_distribution": dict(sorted(collections.Counter(c.get("GSTT1", ".") for c in calls.values()).items()))}  # fmt: skip
+               "gstt1_cn_distribution": dict(sorted(collections.Counter(
+                   c.get("GSTT1", ".") for c in calls.values()).items()))}  # fmt: skip
     a.out.write_text(json.dumps(summary, indent=1))
     print(json.dumps(summary, indent=1))
     return 0

scripts/pgx/validate_star_getrm.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+"""Compare `simurg pgx star-allele call` with GeT-RM consensus diplotypes (PGX-032).
+
+    python scripts/pgx/validate_star_getrm.py --calls STAR_DIR/pgx_calls.json --truth getrm_truth.tsv \
+        --identity scripts/pgx/demo36_identity.tsv --out summary.json
+
+GeT-RM notation is normalised per allele: "*2(*35)" / "*2 (*35)" accept *2 or *35; a trailing sub-allele letter
+("*1F") also matches the core allele ("*1") when the call has no letter; "(*68)+*4" tandems and "xN" duplications
+reduce to the core allele (structural forms are reported separately); "(*28 + *60)" keeps the star alleles the DB
+defines. "A/B or C/D" accepts either. UGT1A1 and SLCO1B1 are reported but left out of the total: GeT-RM uses pre-2023
+names (UGT1A1 *60, SLCO1B1 *1B/*17/*21) that the CPIC definitions no longer carry. A sample's gene counts as **top**
+concordant when the top solution matches, and as **equivalent**
+concordant when the truth is among the array-equivalent members of any feasible solution. Replicate arrays of one
+individual are counted once (the first).
+"""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+import re
+from collections import defaultdict
+from pathlib import Path
+
+
+def truth_options(text: str) -> list[set[str]] | None:
+    """Per haplotype, the set of acceptable core alleles; None when the text is not a diplotype."""
+    t = text.replace(" ", "")
+    if "/" not in t:
+        return None
+    out = []
+    for side in t.split("/", 1):
+        opts = set(re.findall(r"\*\d+[A-Z]?", side))
+        if not opts:
+            return None
+        cores = {re.sub(r"x\d+|xN", "", o) for o in opts}
+        cores |= {re.sub(r"[A-Z]$", "", o) for o in cores}
+        if "+" in side:  # tandem / hybrid: the non-parenthesised last unit is the core allele
+            last = re.findall(r"\*\d+[A-Z]?", side.split("+")[-1])
+            cores |= set(last)
+        out.append(cores)
+    return out
+
+
+LEGACY_NOMENCLATURE = {"UGT1A1", "SLCO1B1"}  # GeT-RM uses pre-2023 names (*60, SLCO1B1 *1B/*17/*21) not in CPIC
+
+
+def truth_alternatives(text: str) -> list[list[set[str]]]:
+    """GeT-RM "A/B or C/D" → one option list per alternative diplotype."""
+    alts = [truth_options(x) for x in re.split(r"\bor\b", text)]
+    return [x for x in alts if x is not None]
+
+
+def matches(call: tuple[str, str], truth: list[list[set[str]]]) -> bool:
+    a, b = (re.sub(r"x\d+$", "", x) for x in call)
+    return any((a in t[0] and b in t[1]) or (a in t[1] and b in t[0]) for t in truth)
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("--calls", required=True, type=Path)
+    ap.add_argument("--truth", required=True, type=Path)
+    ap.add_argument("--identity", required=True, type=Path)
+    ap.add_argument("--out", required=True, type=Path)
+    a = ap.parse_args()
+    with a.identity.open() as fh:
+        ident = {r["sample_id"]: r["individual"] for r in csv.DictReader((ln for ln in fh if not ln.startswith("#")),
+                                                                           delimiter="\t")}  # fmt: skip
+    truth: dict[tuple[str, str], str] = {}
+    with a.truth.open() as fh:
+        for r in csv.DictReader(fh, delimiter="\t"):
+            truth[(r["individual"], r["gene"])] = r["diplotype"]
+    calls = json.loads(a.calls.read_text())
+    per_gene: dict[str, dict[str, int]] = defaultdict(lambda: defaultdict(int))
+    detail = []
+    seen = set()
+    for s in calls["samples"]:
+        ind = ident.get(s["sample_id"])
+        if ind is None or ind in seen:
+            continue
+        seen.add(ind)
+        for g in s["genes"]:
+            t = truth.get((ind, g["gene"]))
+            if t is None:
+                continue
+            opts = truth_alternatives(t)
+            st = per_gene[g["gene"]]
+            if not opts:
+                st["not_comparable"] += 1
+                continue
+            st["n"] += 1
+            top = g["candidates"][0]["genotype"].split("/") if g["candidates"] else ["?", "?"]
+            ok_top = len(top) == 2 and matches((top[0], top[1]), opts)
+            feas = [c for c in g["candidates"] if c["genotype"] in g["feasible_solutions"]]
+            ok_eq = ok_top or any(matches(tuple(e.split("/")[:2]), opts) for c in feas for e in c["equivalents"]
+                                  if len(e.split("/")) >= 2)  # fmt: skip
+            st["top"] += ok_top
+            st["equivalent"] += ok_eq
+            if not ok_top:
+                detail.append({"individual": ind, "gene": g["gene"], "truth": t, "call": g["genotype"],
+                               "within_equivalents": ok_eq})  # fmt: skip
+    comparable = {k: v for k, v in per_gene.items() if k not in LEGACY_NOMENCLATURE}
+    summary = {"individuals": len(seen), "genes": {k: dict(v) for k, v in sorted(per_gene.items())},
+               "legacy_nomenclature_genes": sorted(LEGACY_NOMENCLATURE),
+               "total": {k: sum(v.get(k, 0) for v in comparable.values()) for k in ("n", "top", "equivalent")},
+               "discordant": detail}  # fmt: skip
+    a.out.write_text(json.dumps(summary, indent=1))
+    for g, v in sorted(per_gene.items()):
+        print(f"{g:8s} top {v['top']}/{v['n']}  within equivalents {v['equivalent']}/{v['n']}"
+              + (f"  (not comparable {v['not_comparable']})" if v.get("not_comparable") else "")
+              + ("  [legacy nomenclature]" if g in LEGACY_NOMENCLATURE else ""))  # fmt: skip
+    print("total (excluding legacy-nomenclature genes)", summary["total"])
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())