133 lines
5.8 KiB
Python
133 lines
5.8 KiB
Python
"""Triage of a failed nightly run: which known finding explains each failing step, if any?
|
|
|
|
Code collects the evidence of every step that did not pass (its verdict and counts, the fields of the compare
|
|
summary that differ with their first example rows, or the failing lines of the run log) and the 30 or so entries of
|
|
docs/known_unknowns.md. Jev answers three questions per step in one request:
|
|
|
|
* which known finding explains the failure, an environment problem (memory, disk, network, missing data), or a new
|
|
issue (Choice);
|
|
* does it look like a behaviour change introduced by recent code (Noul);
|
|
* how severe is it, from metadata only to a crash (Score).
|
|
|
|
Only new issues and likely regressions need a person. The evidence comes from public test data (the GSA demo and the
|
|
1000 Genomes Omni2.5 subset).
|
|
|
|
python scripts/jev/nightly_triage.py data/nightly/2026-10-01 # writes triage.md in that folder
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import gzip
|
|
import json
|
|
import re
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
from client import ROOT, choice, noul, score, system_one
|
|
|
|
KNOWN = ROOT / "docs" / "known_unknowns.md"
|
|
SEVERITY = [
|
|
"Only metadata, timestamps or report formatting differ; every computed value is the same",
|
|
"Small numeric differences in some values, genotype calls unchanged",
|
|
"Genotype calls, file contents or counts differ for some samples or loci",
|
|
"A step crashed or produced no output",
|
|
]
|
|
|
|
|
|
def known_findings(text: str) -> dict[str, str]:
|
|
"""``K-NNN -> 'topic: finding'`` from the table of docs/known_unknowns.md."""
|
|
out: dict[str, str] = {}
|
|
for ln in text.splitlines():
|
|
m = re.match(r"\|\s*(K-\d+)\s*\|[^|]*\|\s*([^|]*)\|\s*([^|]*)\|", ln)
|
|
if m:
|
|
out[m.group(1)] = f"{m.group(2).strip()}: {m.group(3).strip()}"[:420]
|
|
return out
|
|
|
|
|
|
def _log_lines(log: Path, n: int = 30) -> list[str]:
|
|
if not log.exists():
|
|
return []
|
|
keep = [ln for ln in log.read_text(errors="replace").splitlines()
|
|
if re.search(r"FAILED|ERROR|Error|Traceback|Killed|MemAvailable|No space|exit [1-9]", ln)] # fmt: skip
|
|
return keep[-n:]
|
|
|
|
|
|
def evidence(base: Path, name: str, step: dict[str, Any]) -> dict[str, Any]:
|
|
ev: dict[str, Any] = {"step": name, "result": {k: v for k, v in step.items() if k not in ("report", "seconds")}}
|
|
rep = step.get("report") or (step.get("regression") or {}).get("report")
|
|
if rep:
|
|
folder = (base / rep).parent
|
|
s = folder / "summary.json"
|
|
if s.exists():
|
|
summ = json.loads(s.read_text())
|
|
fields = {f: {"tolerance": d.get("tolerance"), "discrepancies": d.get("discrepancies")}
|
|
for f, d in summ.get("fields", {}).items() if d.get("discrepancies")} # fmt: skip
|
|
ev["differing_fields"] = fields
|
|
ev["notes"] = summ.get("notes", [])[:10]
|
|
tsv = folder / "discrepancies.tsv.gz"
|
|
if tsv.exists():
|
|
with gzip.open(tsv, "rt") as fh:
|
|
ev["first_rows"] = [ln.rstrip("\n")[:300] for _, ln in zip(range(16), fh, strict=False)]
|
|
if "differing_fields" not in ev and (step.get("verdict") in ("FAIL", "ERROR") or "error" in step):
|
|
ev["log"] = _log_lines(base / "run.log") # the shared log only when the step left no compare evidence
|
|
return ev
|
|
|
|
|
|
def triage_step(ev: dict[str, Any], findings: dict[str, str]) -> dict[str, Any]:
|
|
criteria: dict[str, str | None] = dict(findings)
|
|
criteria["environment"] = ("The run itself failed: out of memory, disk full, network, missing data or tools, a "
|
|
"killed or timed out process") # fmt: skip
|
|
criteria["new_issue"] = "None of the known findings or environment problems explains it: a new difference"
|
|
qs = {
|
|
"match": choice("Which known finding explains `failure`, if any?", criteria),
|
|
"regression": noul(
|
|
"Does `failure` look like a behaviour change introduced by recent code (outputs changed), rather than a "
|
|
"known quirk or an environment problem?"
|
|
),
|
|
"severity": score("How severe is `failure`?", SEVERITY),
|
|
}
|
|
ans = system_one({"failure": ev}, qs)
|
|
m = ans["match"]
|
|
return {
|
|
"match": m["choice"],
|
|
"match_confidence": round(float(m["confidence"]), 2),
|
|
"regression": round(float(ans["regression"]["noul"]), 2),
|
|
"severity": round(float(ans["severity"]["score"]), 2),
|
|
"needs_person": m["choice"] == "new_issue"
|
|
or float(ans["regression"]["noul"]) >= 0.5
|
|
or float(m["confidence"]) < 0.6,
|
|
}
|
|
|
|
|
|
def triage(base: Path) -> dict[str, Any]:
|
|
night = json.loads((base / "summary.json").read_text())
|
|
findings = known_findings(KNOWN.read_text(encoding="utf-8"))
|
|
out: dict[str, Any] = {}
|
|
for name, step in night["steps"].items():
|
|
if step.get("verdict") in ("PASS", "SKIPPED"):
|
|
continue
|
|
out[name] = triage_step(evidence(base, name, step), findings)
|
|
lines = [f"# Nightly triage {night['date']}", ""]
|
|
for name, t in out.items():
|
|
who = "a person should look" if t["needs_person"] else "explained"
|
|
lines.append(f"* **{name}**: {t['match']} (confidence {t['match_confidence']}), regression p "
|
|
f"{t['regression']}, severity {t['severity']} of 3: {who}") # fmt: skip
|
|
(base / "triage.md").write_text("\n".join(lines) + "\n")
|
|
return out
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
|
ap.add_argument("night", type=Path, help="folder of one night, for example data/nightly/2026-10-01")
|
|
a = ap.parse_args(argv)
|
|
res = triage(a.night)
|
|
print(json.dumps(res, indent=1) if res else "every step passed; nothing to triage")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|