Jev maintenance tools: shared client, spec coverage audit, voice check, nightly triage

scripts/jev/nightly_triage.py

@@ -0,0 +1,132 @@
+"""Triage of a failed nightly run: which known finding explains each failing step, if any?
+
+Code collects the evidence of every step that did not pass (its verdict and counts, the fields of the compare
+summary that differ with their first example rows, or the failing lines of the run log) and the 30 or so entries of
+docs/known_unknowns.md. Jev answers three questions per step in one request:
+
+* which known finding explains the failure, an environment problem (memory, disk, network, missing data), or a new
+  issue (Choice);
+* does it look like a behaviour change introduced by recent code (Noul);
+* how severe is it, from metadata only to a crash (Score).
+
+Only new issues and likely regressions need a person. The evidence comes from public test data (the GSA demo and the
+1000 Genomes Omni2.5 subset).
+
+    python scripts/jev/nightly_triage.py data/nightly/2026-10-01     # writes triage.md in that folder
+"""
+
+from __future__ import annotations
+
+import argparse
+import gzip
+import json
+import re
+import sys
+from pathlib import Path
+from typing import Any
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from client import ROOT, choice, noul, score, system_one
+
+KNOWN = ROOT / "docs" / "known_unknowns.md"
+SEVERITY = [
+    "Only metadata, timestamps or report formatting differ; every computed value is the same",
+    "Small numeric differences in some values, genotype calls unchanged",
+    "Genotype calls, file contents or counts differ for some samples or loci",
+    "A step crashed or produced no output",
+]
+
+
+def known_findings(text: str) -> dict[str, str]:
+    """``K-NNN -> 'topic: finding'`` from the table of docs/known_unknowns.md."""
+    out: dict[str, str] = {}
+    for ln in text.splitlines():
+        m = re.match(r"\|\s*(K-\d+)\s*\|[^|]*\|\s*([^|]*)\|\s*([^|]*)\|", ln)
+        if m:
+            out[m.group(1)] = f"{m.group(2).strip()}: {m.group(3).strip()}"[:420]
+    return out
+
+
+def _log_lines(log: Path, n: int = 30) -> list[str]:
+    if not log.exists():
+        return []
+    keep = [ln for ln in log.read_text(errors="replace").splitlines()
+            if re.search(r"FAILED|ERROR|Error|Traceback|Killed|MemAvailable|No space|exit [1-9]", ln)]  # fmt: skip
+    return keep[-n:]
+
+
+def evidence(base: Path, name: str, step: dict[str, Any]) -> dict[str, Any]:
+    ev: dict[str, Any] = {"step": name, "result": {k: v for k, v in step.items() if k not in ("report", "seconds")}}
+    rep = step.get("report") or (step.get("regression") or {}).get("report")
+    if rep:
+        folder = (base / rep).parent
+        s = folder / "summary.json"
+        if s.exists():
+            summ = json.loads(s.read_text())
+            fields = {f: {"tolerance": d.get("tolerance"), "discrepancies": d.get("discrepancies")}
+                      for f, d in summ.get("fields", {}).items() if d.get("discrepancies")}  # fmt: skip
+            ev["differing_fields"] = fields
+            ev["notes"] = summ.get("notes", [])[:10]
+            tsv = folder / "discrepancies.tsv.gz"
+            if tsv.exists():
+                with gzip.open(tsv, "rt") as fh:
+                    ev["first_rows"] = [ln.rstrip("\n")[:300] for _, ln in zip(range(16), fh, strict=False)]
+    if "differing_fields" not in ev and (step.get("verdict") in ("FAIL", "ERROR") or "error" in step):
+        ev["log"] = _log_lines(base / "run.log")  # the shared log only when the step left no compare evidence
+    return ev
+
+
+def triage_step(ev: dict[str, Any], findings: dict[str, str]) -> dict[str, Any]:
+    criteria: dict[str, str | None] = dict(findings)
+    criteria["environment"] = ("The run itself failed: out of memory, disk full, network, missing data or tools, a "
+                               "killed or timed out process")  # fmt: skip
+    criteria["new_issue"] = "None of the known findings or environment problems explains it: a new difference"
+    qs = {
+        "match": choice("Which known finding explains `failure`, if any?", criteria),
+        "regression": noul(
+            "Does `failure` look like a behaviour change introduced by recent code (outputs changed), rather than a "
+            "known quirk or an environment problem?"
+        ),
+        "severity": score("How severe is `failure`?", SEVERITY),
+    }
+    ans = system_one({"failure": ev}, qs)
+    m = ans["match"]
+    return {
+        "match": m["choice"],
+        "match_confidence": round(float(m["confidence"]), 2),
+        "regression": round(float(ans["regression"]["noul"]), 2),
+        "severity": round(float(ans["severity"]["score"]), 2),
+        "needs_person": m["choice"] == "new_issue"
+        or float(ans["regression"]["noul"]) >= 0.5
+        or float(m["confidence"]) < 0.6,
+    }
+
+
+def triage(base: Path) -> dict[str, Any]:
+    night = json.loads((base / "summary.json").read_text())
+    findings = known_findings(KNOWN.read_text(encoding="utf-8"))
+    out: dict[str, Any] = {}
+    for name, step in night["steps"].items():
+        if step.get("verdict") in ("PASS", "SKIPPED"):
+            continue
+        out[name] = triage_step(evidence(base, name, step), findings)
+    lines = [f"# Nightly triage {night['date']}", ""]
+    for name, t in out.items():
+        who = "a person should look" if t["needs_person"] else "explained"
+        lines.append(f"* **{name}**: {t['match']} (confidence {t['match_confidence']}), regression p "
+                     f"{t['regression']}, severity {t['severity']} of 3: {who}")  # fmt: skip
+    (base / "triage.md").write_text("\n".join(lines) + "\n")
+    return out
+
+
+def main(argv: list[str] | None = None) -> int:
+    ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
+    ap.add_argument("night", type=Path, help="folder of one night, for example data/nightly/2026-10-01")
+    a = ap.parse_args(argv)
+    res = triage(a.night)
+    print(json.dumps(res, indent=1) if res else "every step passed; nothing to triage")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())