P7: differential-testing harness (simurg compare) and nightly run on the Pi

- compare/core: EXACT/NEAR/DISCREPANCY/UNKNOWN with the plan §16.3 tolerances (abs, float32 ULP), NaN = NaN,
  locus classes (assay, variant type, chromosome group, intensity-only); fields absent on both sides are
  not applicable
- compare/records: GTC vs GTC (per-locus arrays, transforms per bin, controls, sample statistics; metadata
  listed, not counted) and VCF vs VCF (records keyed incl. ID; SNV/MNV/indel, merged, multi-allelic classes)
- simurg compare A B (files or folders paired by name): summary.json, discrepancies.tsv.gz (never truncated),
  self-contained dashboard.html; --tolerance, --fail-on-discrepancy (exit 3)
- scripts/nightly.py + install_nightly.sh (crontab 02:30): unit tests, GSA vs gtc2vcf (GTC + VCF), VCF
  regression vs golden, Omni2.5 subset vs vendor GTCs; first run PASS on every step
- tests/unit/test_compare.py; docs/validation/P7.md, README
This commit is contained in:
kemal committed 2026-09-28 17:00:37 +01:00
1 parent 1001bf951e
commit 01129ded08
10 files changed
+1268 -1

No files matched your search

+55
View File
@@ -444,6 +444,61 @@ def gtc_to_vcf_cmd(
run_guarded(rep, lambda: run_gtc_to_vcf(opts, rep))
# --------------------------------------------------------------------------------------------------------
@app.command("compare")
def compare_cmd(
a: Annotated[Path, typer.Argument(help="GTC/VCF file or folder (A).")],
b: Annotated[Path, typer.Argument(help="GTC/VCF file or folder (B), paired with A by relative path.")],
out: Annotated[Path, typer.Option("--out", help="Output folder (summary.json, discrepancies, dashboard).")],
bpm: Annotated[Path | None, typer.Option("--bpm", help="Manifest, for per-locus-class counts of GTCs.")] = None,
tolerance: Annotated[
list[str] | None,
typer.Option("--tolerance", help="FIELD=exact|abs=X|ulp=N (repeatable); VCF floats: float=abs=X"),
] = None,
sample: Annotated[
list[str] | None, typer.Option("--sample", help="VCF sample(s) to compare (default: all shared).")
] = None,
title: Annotated[str, typer.Option("--title", help="Dashboard title.")] = "",
threads: Annotated[int, typer.Option("--threads", min=0, max=1024, help="Worker processes (0 = auto).")] = 0,
fail_on_discrepancy: Annotated[
bool, typer.Option("--fail-on-discrepancy", help="Exit 3 when any DISCREPANCY is found.")
] = False,
force: Annotated[bool, typer.Option("--force", help="Replace existing outputs.")] = False,
log_json: Annotated[bool, typer.Option("--log-json", "--json-log")] = False,
debug: Annotated[bool, typer.Option("--debug")] = False,
quiet: Annotated[bool, typer.Option("--quiet", "--no-progress")] = False,
) -> None:
"""Field-level differential test of two GTCs/VCFs (or folders): EXACT/NEAR/DISCREPANCY/UNKNOWN (plan §16.3)."""
from simurg.pipeline.compare import CompareOptions, run_compare
rep = Reporter(
command="compare", level="debug" if debug else "info", json_stderr=log_json, debug=debug, quiet=quiet
)
def body() -> int:
tol: dict[str, str] = {}
for item in tolerance or []:
key, sep, val = item.partition("=")
if not sep:
raise SimurgError("E_CONFIG_KEY", f"--tolerance expects FIELD=VALUE, got {item!r}")
tol[key.strip()] = val.strip()
opts = CompareOptions(
a=a,
b=b,
out=out,
bpm=bpm,
tolerances=tol,
samples=list(sample) if sample else None,
threads=threads,
fail_on_discrepancy=fail_on_discrepancy,
title=title,
force=force,
)
return run_compare(opts, rep)
run_guarded(rep, body)
# --------------------------------------------------------------------------------------------------------
export_app = typer.Typer(help="Exports from GTCs or VCFs: bedgraph, finalreport, plink, locus-summary.")
app.add_typer(export_app, name="export")