From 783e2a00398c26a2eea8967c87874bfd55687509 Mon Sep 17 00:00:00 2001 From: Kemal Yaylali Date: Thu, 1 Oct 2026 13:05:37 +0100 Subject: [PATCH] Claim checker for public texts (TypeSafe), measured manifest load times, corrected README timings --- .gitea/workflows/ci.yml | 2 +- .gitignore | 2 + CHANGELOG.md | 5 + CONTRIBUTING.md | 17 ++ README.md | 7 +- docs/user_guide.md | 2 +- docs/validation/P9.md | 3 + scripts/docs/check_claims.py | 368 ++++++++++++++++++++++++++++++++ tests/unit/test_check_claims.py | 75 +++++++ 9 files changed, 477 insertions(+), 4 deletions(-) create mode 100644 scripts/docs/check_claims.py create mode 100644 tests/unit/test_check_claims.py diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 8e57e5c..4ed4dc1 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: python-version: ${{ matrix.python }} - name: Secret scan (no cloud keys or private keys in the repository) run: | - if git grep -nIE '(AKIA|ASIA)[A-Z0-9]{16}|aws_secret_access_key *=|BEGIN [A-Z ]*PRIVATE KEY'; then + if git grep -nIE '(AKIA|ASIA)[A-Z0-9]{16}|aws_secret_access_key *=|BEGIN [A-Z ]*PRIVATE KEY|apikey_[0-9a-f]{20,}'; then echo "::error::credential-like content found"; exit 1 fi - name: Install diff --git a/.gitignore b/.gitignore index 7bb3b6a..9ccc9f5 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,5 @@ results/ credentials .git-credentials .netrc +# local caches (claim checker and others) +.cache/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 8101b60..97ae72a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## Unreleased +**Documentation checks.** `scripts/docs/check_claims.py` checks the numeric claims of public texts against the +validation reports (code retrieves evidence, a TypeSafe System One model judges support). Its first run corrected the +manifest loading times (Omni2.5 about 18 s on a Pi, now measured in `docs/validation/P9.md`) and a demo timing +sentence in the README. + **Packaging.** The distributions are now `simurgarray` and `simurgarray-core` (the name `simurg` is taken on PyPI); the `simurg` command and the `simurg` / `simurg_core` import names are unchanged. The conda recipe no longer pins the kernel package to the Python package's version (it required `simurg-core == 2.0.0` while the kernels are 1.1.0). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e658519..8d579ff 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -50,6 +50,23 @@ and 3.13. If your change touches file formats, calling, QC or outputs, also run 8. **Plain writing.** Documentation follows the voice in `branding/README.md`: short sentences, numbers instead of adjectives, and every claim next to its limitation. +## Checking numbers before publishing + +Text that quotes results (README, user guide, blog posts, announcements) is checked against the validation reports +before it goes out: + +```bash +python scripts/docs/check_claims.py README.md blog/*.md docs/user_guide.md my_draft.txt +``` + +Code finds every sentence or table row with a number and the passages in `docs/validation/`, +`docs/known_unknowns.md` and the spec that share it; a TypeSafe System One model only judges whether each passage +supports, contradicts or says nothing about the claim. `claims_report.md` lists contradicted claims, claims without +a source, low confidence cases for a person to read, and verified claims. Only this repository's documentation text +is sent to the API, never data. The key comes from `TYPESAFE_API_KEY` or `~/.config/typesafe/credentials` and never +enters the repository; a full check costs well under one cent. `--dry-run` shows the claims and evidence without +calling the API. + ## Pull requests * One topic per pull request, with tests. diff --git a/README.md b/README.md index 175b7bb..8c2f494 100644 --- a/README.md +++ b/README.md @@ -169,7 +169,7 @@ bash examples/gsa_demo.sh results/gsa_demo 6 # end to end on 6 samples; drop th ``` The demo writes GTCs, a sample summary, an HTML QC report, per sample VCFs and a cohort BCF, PLINK files, bedGraph -tracks and a final report to `results/gsa_demo/`. On a Raspberry Pi 5 all 36 samples take about ten minutes. +tracks and a final report to `results/gsa_demo/`. On a Raspberry Pi 5 the whole demo for all 36 samples (genotyping, QC report, VCFs and exports) takes about ten minutes with two workers. ## Usage @@ -303,7 +303,7 @@ Every HTML report carries the software version and a checksum of the data it sho * With the kernels a GSA sample takes about 1.1 s on one Raspberry Pi 5 core (3.4 s pure Python) and a server core handles about 80 samples a minute (`docs/validation/P9.md`). -* Loading a manifest is a fixed cost per command: about 5 s for GSA and 15 s for Omni2.5 on a Pi. +* Loading a manifest is a fixed cost per command: about 4 s for GSA and 18 s for Omni2.5 on a Pi with the kernels. * Cohort QC steps grow with samples × loci. On small machines use `--threads 2`, and `--no-cohort-qc` on very large `genotype` runs, then run the `qc` commands separately. * `simurg train` needs about 2 GB for 120 GSA samples and takes about 70 minutes on a Pi. @@ -328,6 +328,9 @@ bash scripts/fetch_data.sh p6 && bash scripts/make_oracle_vcf.sh uv run pytest tests/oracle # add -m "not slow" to skip the long ones ``` +Before publishing text with numbers, `python scripts/docs/check_claims.py FILES` checks every claim against the +validation reports (see `CONTRIBUTING.md`). + CI runs lint, type checks, a secret scan and the unit tests with and without the Rust kernels on Python 3.12 and 3.13 for every push. `scripts/nightly.py` reruns the differential tests against the oracles. diff --git a/docs/user_guide.md b/docs/user_guide.md index e2476c3..b22a7ae 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -304,7 +304,7 @@ A nightly differential run (`scripts/nightly.py`) re-checks these against the or QC steps; on an 8 GB machine keep `--threads` low (≈ 2–3 GB per worker for GSA/Omni-size arrays). * Speed: with the compiled kernels a GSA sample takes ~1.1 s on a Raspberry Pi 5 core (pure Python ~3.4 s); see `docs/validation/P9.md` for server numbers and thread scaling. Loading a manifest/cluster file is a fixed cost - per run (GSA ~5 s, Omni2.5 ~15 s on the Pi). + per run (GSA ~4 s, Omni2.5 ~18 s on the Pi with the kernels). * Methylation arrays: control-probe QC, detection and β values only (no differential analysis). ## Troubleshooting diff --git a/docs/validation/P9.md b/docs/validation/P9.md index 2ed032b..b416bd5 100644 --- a/docs/validation/P9.md +++ b/docs/validation/P9.md @@ -80,3 +80,6 @@ Manifest loading: GSA 13 s → 3 s, Omni2.5 53 s → 13 s, GDA 40 s → 10 s (Pi calling 0.24 s, normalize/polar 0.08 s, raw X/Y 0.06 s. A full Rust normalization fit is the next step if needed. * Pi thread scaling is poor (36 samples, 3 workers ≈ 1.7× one core): the Pi 5's memory bandwidth, not the code. * CI (`.gitea/workflows/ci.yml`) builds the extension and runs the unit tests with and without it. +* Manifest loading on the Raspberry Pi 5 with the kernels (2026-10-01, two runs each, Python imports included): + GSA v1 C2 BPM (618 540 loci) 4.1 to 4.4 s; Omni2.5 BPM (2 450 000 loci) 17.4 to 19.4 s, so about 4 s and 18 s. + Before the kernels the same step took 13 s and 53 s. diff --git a/scripts/docs/check_claims.py b/scripts/docs/check_claims.py new file mode 100644 index 0000000..04635ed --- /dev/null +++ b/scripts/docs/check_claims.py @@ -0,0 +1,368 @@ +"""Check the numeric claims in public texts (README, blog posts, guide, drafts) against the validation reports. + +Code finds the claims and the candidate evidence; a TypeSafe System One model (Jev) only judges whether a passage +supports, contradicts or says nothing about a claim (the "double checking citations" pattern, +https://docs.typesafe.ai/cookbooks/citation_check). Only this repository's own documentation text is sent; no data, +no sample sheets, no vendor files. Needs ``TYPESAFE_API_KEY`` in the environment, or in +``~/.config/typesafe/credentials`` (mode 600), for the judgment step. + + python scripts/docs/check_claims.py README.md blog/*.md # check, write claims_report.md + python scripts/docs/check_claims.py --dry-run linkedin_draft.txt # extraction and retrieval only, no API calls + +Verdicts: verified (a passage supports the claim, confidence >= threshold), contradicted, unsupported (passages found +but none addresses the claim), no_source (no passage contains any of the claim's numbers), review (low confidence). +Exit code 1 when any claim is contradicted or has no source, so the script can gate a publishing step. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import sys +import time +import urllib.error +import urllib.request +from dataclasses import dataclass, field +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +API_URL = "https://api.typesafe.ai/v1/systemone" +MODEL = "jev-latest" +AUTO_ACCEPT = 0.8 # cookbook default; evaluate on our own claims before trusting it +TOP_K = 3 +DEFAULT_SOURCES = ["docs/validation", "docs/known_unknowns.md", "SIMURGARRAY_FUNCTIONAL_SPEC.md"] +CACHE = ROOT / ".cache" / "claims" + +RELATION = { + "type": "choice", + "criteria": { + "supports": "The section states the claim's numbers and meaning, or directly implies that the claim is true", + "contradicts": "The section gives a different value for the same quantity, or implies the claim is false", + "says_nothing": "The section does not address what the claim asserts, either way", + }, +} + +# ------------------------------------------------------------------ numbers + +_NUM = re.compile(r"(? set[str]: + """Normalised number tokens of a text, with percent/fraction twins: '99.34%' -> {'99.34', '0.9934'}.""" + out: set[str] = set() + for m in _NUM.finditer(text): + raw, pct = m.group(1), m.group(2) + v = raw.replace(",", "").replace(" ", "") + try: + f = float(v) + except ValueError: + continue + out.add(_fmt(f)) + if pct: + out.add(_fmt(f / 100)) + elif 0 < f < 1: + out.add(_fmt(f * 100)) + return out + + +def _fmt(f: float) -> str: + s = f"{f:.6f}".rstrip("0").rstrip(".") + return s or "0" + + +_MONTHS = "January|February|March|April|May|June|July|August|September|October|November|December" +_DATE = re.compile( + rf"\b({_MONTHS})\s+\d{{4}}\b|\b({_MONTHS})\s+\d{{1,2}}(,\s*\d{{4}})?|\b(19|20)\d{{2}}\b|\d{{4}}-\d{{2}}-\d{{2}}" + r"|\b(version|v)\s?\d+(\.\d+)+\b" +) + + +def significant(nums: set[str]) -> set[str]: + """Numbers worth matching: drop tiny integers that appear everywhere (0 to 12).""" + return {n for n in nums if not (n.isdigit() and int(n) <= 12)} + + +def claim_numbers(text: str) -> set[str]: + """Significant numbers of a claim; dates and years are context, not claims.""" + return significant(numbers(_DATE.sub(" ", text))) + + +# ------------------------------------------------------------------ claims + + +@dataclass +class Claim: + path: str + line: int + text: str + context: str + nums: set[str] + + +def _strip_md(s: str) -> str: + s = re.sub(r"<[^>]+>", " ", s) + s = re.sub(r"!\[[^\]]*\]\([^)]*\)", " ", s) + s = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", s) + s = re.sub(r"https?://\S+", " ", s) + return re.sub(r"\s+", " ", s.replace("**", "").replace("`", "")).strip() + + +def extract_claims(path: Path, text: str) -> list[Claim]: + """Sentences of paragraphs and bullets, and table rows, that carry significant numbers. Lines of one paragraph + or bullet are joined first, so a sentence wrapped over several lines stays whole. Code blocks and commands are + skipped.""" + claims: list[Claim] = [] + heading, table_head, in_code = "", "", False + buf: list[str] = [] + start = 0 + + def add(unit: str, line: int) -> None: + nums = claim_numbers(unit) + if nums and not re.search(r"\b(simurg|uv|pip|bash|docker|conda) ", unit): + ctx = " / ".join(x for x in (heading, table_head) if x) + claims.append(Claim(str(path), line, unit, ctx, nums)) + + def flush() -> None: + if buf: + para = _strip_md(" ".join(buf)) + for u in re.split(r"(?<=[.!?])\s+(?=[A-Z])", para): + if u: + add(u, start) + buf.clear() + + for i, raw in enumerate([*text.splitlines(), ""], 1): + ln = raw.rstrip() + if ln.lstrip().startswith("```"): + flush() + in_code = not in_code + continue + if in_code: + continue + if not ln.strip(): + flush() + table_head = "" + continue + if ln.startswith("#"): + flush() + heading = _strip_md(ln.lstrip("#")) + continue + if ln.lstrip().startswith("|"): + flush() + if re.fullmatch(r"\s*\|[\s:|-]+\|?\s*", ln): + continue + if not table_head: + table_head = _strip_md(ln) + continue + add(_strip_md(ln), i) + continue + bullet = re.match(r"\s*([*>-]|\d+\.)\s+", ln) + if bullet: + flush() + ln = ln[bullet.end() :] + if not buf: + start = i + buf.append(ln.strip()) + return claims + + +# ------------------------------------------------------------------ evidence + + +@dataclass +class Passage: + path: str + line: int + text: str + nums: set[str] = field(default_factory=set) + + +def load_passages(sources: list[Path]) -> list[Passage]: + """Paragraphs and table rows (with their header) of the source documents.""" + out: list[Passage] = [] + files = [p for s in sources for p in (sorted(s.rglob("*.md")) if s.is_dir() else [s])] + for f in files: + rel = str(f.relative_to(ROOT)) if f.is_relative_to(ROOT) else str(f) + heading, table_head = "", "" + buf: list[str] = [] + start = 0 + for i, ln in enumerate([*f.read_text(encoding="utf-8").splitlines(), ""], 1): + is_row = ln.lstrip().startswith("|") + if (ln.startswith("#") or is_row or not ln.strip()) and buf: + t = (heading + ": " if heading else "") + " ".join(buf) + out.append(Passage(rel, start, t, numbers(t))) + buf = [] + if ln.startswith("#"): + heading, table_head = _strip_md(ln.lstrip("#")), "" + elif is_row: + if re.fullmatch(r"\s*\|[\s:|-]+\|?\s*", ln): + continue + if not table_head: + table_head = _strip_md(ln) + continue + t = f"{heading}: {table_head} || {_strip_md(ln)}" + out.append(Passage(rel, i, t, numbers(t))) + elif not ln.strip(): + table_head = "" + else: + if not buf: + start = i + buf.append(_strip_md(ln)) + return out + + +def _shared(a: set[str], b: set[str], tol: float = 0.03) -> int: + """Numbers of ``a`` found in ``b`` exactly, or within 3 % for values of 10 and more (rounded figures such as + 'about 550 MB' for 545 and 551 MB).""" + fb = [float(x) for x in b] + n = 0 + for x in a: + if x in b: + n += 1 + continue + v = float(x) + if v >= 10 and any(w >= 10 and abs(v - w) <= tol * max(v, w) for w in fb): + n += 1 + return n + + +def candidates(claim: Claim, passages: list[Passage], k: int = TOP_K) -> list[Passage]: + """Passages sharing the most of the claim's numbers (exactly or approximately), then the most words.""" + words = set(re.findall(r"[a-z]{4,}", claim.text.lower())) + scored = [] + for p in passages: + shared = _shared(claim.nums, p.nums) + if shared: + overlap = len(words & set(re.findall(r"[a-z]{4,}", p.text.lower()))) + scored.append((shared, overlap, p)) + scored.sort(key=lambda t: (-t[0], -t[1], t[2].path, t[2].line)) + return [p for _, _, p in scored[:k]] + + +# ------------------------------------------------------------------ judgment + + +def build_request(claim: Claim, cands: list[Passage]) -> dict[str, object]: + state = { + "claim": claim.text, + "claim_context": claim.context, + "sections": [{"source": f"{p.path}:{p.line}", "text": p.text[:1500]} for p in cands], + } + ask = "How does `sections[{i}].text` relate to `claim` (read `claim` together with `claim_context`)?" + questions = {f"s{i}": {**RELATION, "instructions": ask.format(i=i)} for i in range(len(cands))} + return {"model": MODEL, "state": state, "questions": questions} + + +def call_api(payload: dict[str, object], key: str, retries: int = 5) -> dict[str, object]: + body = json.dumps(payload).encode() + digest = hashlib.sha256(body).hexdigest() + cached = CACHE / f"{digest}.json" + if cached.exists(): + return json.loads(cached.read_text()) # type: ignore[no-any-return] + headers = { + "Authorization": f"Bearer {key}", + "Content-Type": "application/json", + "User-Agent": "simurgarray-claims/1", + } + req = urllib.request.Request(API_URL, data=body, method="POST", headers=headers) + for attempt in range(retries): + try: + with urllib.request.urlopen(req, timeout=60) as r: + res = json.loads(r.read()) + CACHE.mkdir(parents=True, exist_ok=True) + cached.write_text(json.dumps(res)) + return res # type: ignore[no-any-return] + except urllib.error.HTTPError as e: + if e.code in (429, 529) and attempt < retries - 1: + time.sleep(2**attempt) + continue + raise RuntimeError(f"TypeSafe API error {e.code}: {e.read()[:200]!r}") from e + raise RuntimeError("TypeSafe API: retries exhausted") + + +def verdict(answers: dict[str, dict[str, object]], threshold: float = AUTO_ACCEPT) -> tuple[str, str | None, float]: + """Combine per-section answers: any confident 'supports' wins; else a confident 'contradicts'; else a confident + 'says_nothing' from every section; anything else goes to a person.""" + best: dict[str, tuple[float, str]] = {} + for qid, a in answers.items(): + ch, conf = str(a["choice"]), float(a["confidence"]) # type: ignore[arg-type] + if conf >= threshold and conf > best.get(ch, (-1.0, ""))[0]: + best[ch] = (conf, qid) + if "supports" in best: + return "verified", best["supports"][1], best["supports"][0] + support = max((float(a["probabilities"].get("supports", 0.0)) for a in answers.values()), default=0.0) # type: ignore[attr-defined] + if "contradicts" in best: + if support >= 0.3: # another section leans towards support: a person decides + return "review", best["contradicts"][1], best["contradicts"][0] + return "contradicted", best["contradicts"][1], best["contradicts"][0] + confident = [float(a["confidence"]) >= threshold and a["choice"] == "says_nothing" for a in answers.values()] # type: ignore[arg-type] + if confident and all(confident): + return "unsupported", None, min(float(a["confidence"]) for a in answers.values()) # type: ignore[arg-type] + return "review", None, max((float(a["confidence"]) for a in answers.values()), default=0.0) # type: ignore[arg-type] + + +# ------------------------------------------------------------------ report + + +def _key_from_file(path: Path) -> str: + """``TYPESAFE_API_KEY=...`` from a private local file (never from the repository).""" + if path.exists(): + for ln in path.read_text().splitlines(): + if ln.startswith("TYPESAFE_API_KEY="): + return ln.split("=", 1)[1].strip() + return "" + + +def main(argv: list[str] | None = None) -> int: + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument("files", nargs="+", type=Path, help="texts to check (markdown or plain text)") + ap.add_argument("--sources", nargs="+", type=Path, default=[ROOT / s for s in DEFAULT_SOURCES]) + ap.add_argument("--out", type=Path, default=Path("claims_report.md")) + ap.add_argument("--threshold", type=float, default=AUTO_ACCEPT) + ap.add_argument("--dry-run", action="store_true", help="extract claims and evidence, no API calls") + a = ap.parse_args(argv) + key = os.environ.get("TYPESAFE_API_KEY", "") or _key_from_file(Path.home() / ".config" / "typesafe" / "credentials") + if not a.dry_run and not key: + print("TYPESAFE_API_KEY is not set (use --dry-run to check extraction without the API).", file=sys.stderr) + return 2 + passages = load_passages(a.sources) + rows: list[tuple[str, Claim, Passage | None, float, str]] = [] + for f in a.files: + for c in extract_claims(f, f.read_text(encoding="utf-8")): + cands = candidates(c, passages) + if not cands: + rows.append(("no_source", c, None, 1.0, "")) + continue + if a.dry_run: + rows.append(("planned", c, cands[0], 0.0, f"{len(cands)} candidate(s)")) + continue + res = call_api(build_request(c, cands), key) + answers = res["answers"] # type: ignore[index] + v, qid, conf = verdict(answers, a.threshold) # type: ignore[arg-type] + src = cands[int(qid[1:])] if qid else cands[0] + probs = ", ".join(f"{q}: {answers[q]['choice']} {answers[q]['confidence']:.2f}" for q in answers) # type: ignore[index] + rows.append((v, c, src, conf, probs)) + order = ["contradicted", "no_source", "review", "unsupported", "verified", "planned"] + counts = {v: sum(r[0] == v for r in rows) for v in order} + lines = ["# Claim check", "", " · ".join(f"{v} {n}" for v, n in counts.items() if n), ""] + for v in order: + sel = [r for r in rows if r[0] == v] + if not sel: + continue + lines += [f"## {v} ({len(sel)})", ""] + for _, c, src, conf, extra in sel: + lines.append(f"* `{c.path}:{c.line}` {c.text}") + if src: + lines.append(f" * evidence `{src.path}:{src.line}` (confidence {conf:.2f}) {src.text[:220]}") + if extra: + lines.append(f" * {extra}") + lines.append("") + a.out.write_text("\n".join(lines)) + print(f"{len(rows)} claims: " + ", ".join(f"{v} {n}" for v, n in counts.items() if n) + f" → {a.out}") + return 1 if counts["contradicted"] or counts["no_source"] else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/unit/test_check_claims.py b/tests/unit/test_check_claims.py new file mode 100644 index 0000000..3fecad8 --- /dev/null +++ b/tests/unit/test_check_claims.py @@ -0,0 +1,75 @@ +"""The documentation claim checker (scripts/docs/check_claims.py): everything except the API call.""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] +_spec = importlib.util.spec_from_file_location("check_claims", ROOT / "scripts" / "docs" / "check_claims.py") +assert _spec is not None and _spec.loader is not None +cc = importlib.util.module_from_spec(_spec) +sys.modules["check_claims"] = cc # dataclasses look their module up +_spec.loader.exec_module(cc) + + +def test_numbers_have_percent_and_fraction_twins() -> None: + assert {"99.34", "0.9934"} <= cc.numbers("concordance 99.34 %") + assert {"0.9934", "99.34"} <= cc.numbers("concordance 0.9934") + assert "2141" in cc.numbers("2,141 arrays") and "2917" in cc.numbers("2 917 sites") + + +def test_dates_versions_and_small_integers_are_not_claims() -> None: + assert cc.claim_numbers("Kemal Yaylali, September 2026") == set() + assert cc.claim_numbers("Week of January 11, 2027: version 2.1") == set() + assert cc.claim_numbers("3 of 5 posts") == set() + assert cc.claim_numbers("about 4 s and 18 s") == {"18"} + + +def test_wrapped_sentences_stay_whole_and_code_is_skipped() -> None: + text = ( + "# Accuracy\n\nScores from a trained file are not\ncomparable with 99.46 % from vendor files.\n\n" + "```bash\nsimurg genotype --threads 4096\n```\n\n| Check | Result |\n|---|---|\n| GIAB | 0.9985 |\n" + ) + claims = cc.extract_claims(Path("x.md"), text) + assert [c.line for c in claims] == [3, 12] + assert "not comparable with 99.46 %" in claims[0].text and claims[0].context == "Accuracy" + assert claims[1].context == "Accuracy / | Check | Result |" and "0.9985" in claims[1].nums + + +def test_candidates_match_rounded_figures(tmp_path: Path) -> None: + src = tmp_path / "r.md" + src.write_text( + "# Docker\n\n| Artifact | Size |\n|---|---|\n| image | 545 MB (x86_64), 551 MB (aarch64) |\n\nUnrelated 12345.\n" + ) + passages = cc.load_passages([src]) + claim = cc.Claim("README.md", 1, "The image is about 550 MB", "", cc.claim_numbers("The image is about 550 MB")) + found = cc.candidates(claim, passages) + assert len(found) == 1 and "545 MB" in found[0].text + + +def _ans(choice: str, conf: float, supports: float = 0.0) -> dict[str, object]: + probs = {"supports": supports, "contradicts": 0.0, "says_nothing": 0.0} + probs[choice] = max(probs[choice], conf) + return {"choice": choice, "confidence": conf, "probabilities": probs} + + +def test_verdicts() -> None: + assert cc.verdict({"s0": _ans("says_nothing", 0.9), "s1": _ans("supports", 0.95)})[0] == "verified" + assert cc.verdict({"s0": _ans("contradicts", 0.95)})[0] == "contradicted" + # a confident contradiction while another section leans towards support goes to a person + assert cc.verdict({"s0": _ans("contradicts", 0.95), "s1": _ans("says_nothing", 0.5, supports=0.4)})[0] == "review" + assert cc.verdict({"s0": _ans("says_nothing", 0.9), "s1": _ans("says_nothing", 0.85)})[0] == "unsupported" + assert cc.verdict({"s0": _ans("supports", 0.6)})[0] == "review" + + +def test_request_shape() -> None: + claim = cc.Claim("README.md", 1, "99.44 % concordance", "Validation", {"99.44"}) + p = cc.Passage("docs/validation/V2_arraytrain.md", 7, "v2 gives 99.44 %", {"99.44"}) + req = cc.build_request(claim, [p, p]) + assert req["model"] == "jev-latest" and set(req["questions"]) == {"s0", "s1"} # type: ignore[arg-type] + q = req["questions"]["s1"] # type: ignore[index] + assert q["type"] == "choice" and "`sections[1].text`" in q["instructions"] + assert set(q["criteria"]) == {"supports", "contradicts", "says_nothing"} + assert req["state"]["sections"][0]["source"] == "docs/validation/V2_arraytrain.md:7" # type: ignore[index]