P6: gtc-to-vcf (VCF/BCF + indexes), harmonisation, exports (plink, bedgraph, finalreport)

- formats/fasta: plain/BGZF FASTA, .fai/.gzi built identical to htslib, index check, build detection,
  contig aliases, mmap genome cache (IN-060..064)
- formats/bgzf, formats/bcf: pure-Python BGZF, tabix .tbi, BCF 2.2 and CSI (OUT-010, OUT-038, OUT-040)
- harmonize/sites: plus-strand mapping with audit rows, indel resolution from SourceSeq (port of GTCtoVCF,
  Apache-2.0) and left normalisation (OUT-014..021, OUT-030, OUT-042)
- harmonize/combine: OUT-031 genotype combination (ploidy-aware, Infinium I/II signal classes, OUT-037)
- pipeline/vcf: `simurg gtc-to-vcf` (per-sample VCF.gz+tbi, --merge cohort BCF+csi, audit parquet)
- pipeline/export: `simurg export bedgraph|finalreport|plink|locus-summary` (OUT-043, OUT-050..052, OUT-071)
- validation: 36 GSA samples vs gtc2vcf: 23.66 M identical calls, 0 unexplained differences; PLINK identical
  to plink2 on 637,397 variants; bcftools validates VCF/BCF and our indexes answer region queries correctly
- fetch_data.sh p6 (Ensembl GRCh38 with checksum), make_oracle_vcf.sh, validate_p6_vcf.py; K-022/K-023
This commit is contained in:
kemal committed 2026-09-28 12:17:53 +01:00
1 parent 1b6f834583
commit e65d2426bb
19 files changed
+3184 -9

No files matched your search

+20 -8
View File
@@ -308,26 +308,27 @@ class SampleInfo:
class CohortStore:
"""Disk-backed (samples × loci) matrices: genotypes (uint8), BAF and GenCall scores (float32, optional)."""
def __init__(self, root: Path, n_samples: int, n_loci: int, *, baf: bool, scores: bool) -> None:
def __init__(self, root: Path, n_samples: int, n_loci: int, *, baf: bool, scores: bool, lrr: bool = False) -> None:
self.root, self.shape = root, (n_samples, n_loci)
root.mkdir(parents=True, exist_ok=True)
self.gt = np.lib.format.open_memmap(root / "gt.npy", "w+", np.uint8, self.shape)
self.baf = np.lib.format.open_memmap(root / "baf.npy", "w+", np.float32, self.shape) if baf else None
self.score = np.lib.format.open_memmap(root / "score.npy", "w+", np.float32, self.shape) if scores else None
self.lrr = np.lib.format.open_memmap(root / "lrr.npy", "w+", np.float32, self.shape) if lrr else None
def close(self) -> None:
for name in ("gt", "baf", "score"):
for name in ("gt", "baf", "score", "lrr"):
a = getattr(self, name)
if a is not None:
a.flush()
self.gt = self.baf = self.score = None # type: ignore[assignment]
self.gt = self.baf = self.score = self.lrr = None # type: ignore[assignment]
shutil.rmtree(self.root, ignore_errors=True)
def _load_one(task: tuple[int, str, str, str, bool, bool]) -> SampleInfo:
def _load_one(task: tuple[int, str, str, str, bool, bool, bool]) -> SampleInfo:
from simurg.formats.gtc import read_gtc
i, sid, path, root, want_baf, want_scores = task
i, sid, path, root, want_baf, want_scores, want_lrr = task
info = SampleInfo(sid, path)
try:
g = read_gtc(path)
@@ -347,6 +348,10 @@ def _load_one(task: tuple[int, str, str, str, bool, bool]) -> SampleInfo:
info.has_baf = len(g.b_allele_freqs) == gt.shape[1]
baf[i] = g.b_allele_freqs if info.has_baf else np.nan
baf.flush()
if want_lrr:
lr = np.load(Path(root) / "lrr.npy", mmap_mode="r+")
lr[i] = g.logr_ratios if len(g.logr_ratios) == gt.shape[1] else np.nan
lr.flush()
if want_scores:
sc = np.load(Path(root) / "score.npy", mmap_mode="r+")
sc[i] = g.genotype_scores
@@ -359,12 +364,19 @@ def _load_one(task: tuple[int, str, str, str, bool, bool]) -> SampleInfo:
def load_cohort(
samples: Sequence[tuple[str, str]], n_loci: int, tmp: Path, *, baf: bool, scores: bool, workers: int
samples: Sequence[tuple[str, str]],
n_loci: int,
tmp: Path,
*,
baf: bool,
scores: bool,
workers: int,
lrr: bool = False,
) -> tuple[CohortStore, list[SampleInfo]]:
"""Read GTCs (sample_id, path) into a :class:`CohortStore` with a process pool."""
store = CohortStore(tmp, len(samples), n_loci, baf=baf, scores=scores)
store = CohortStore(tmp, len(samples), n_loci, baf=baf, scores=scores, lrr=lrr)
store.gt.flush()
tasks = [(i, sid, p, str(tmp), baf, scores) for i, (sid, p) in enumerate(samples)]
tasks = [(i, sid, p, str(tmp), baf, scores, lrr) for i, (sid, p) in enumerate(samples)]
if workers <= 1 or len(tasks) <= 1:
infos = [_load_one(t) for t in tasks]
else: