76 lines
3.7 KiB
Python
76 lines
3.7 KiB
Python
"""The documentation claim checker (scripts/docs/check_claims.py): everything except the API call."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
_spec = importlib.util.spec_from_file_location("check_claims", ROOT / "scripts" / "docs" / "check_claims.py")
|
|
assert _spec is not None and _spec.loader is not None
|
|
cc = importlib.util.module_from_spec(_spec)
|
|
sys.modules["check_claims"] = cc # dataclasses look their module up
|
|
_spec.loader.exec_module(cc)
|
|
|
|
|
|
def test_numbers_have_percent_and_fraction_twins() -> None:
|
|
assert {"99.34", "0.9934"} <= cc.numbers("concordance 99.34 %")
|
|
assert {"0.9934", "99.34"} <= cc.numbers("concordance 0.9934")
|
|
assert "2141" in cc.numbers("2,141 arrays") and "2917" in cc.numbers("2 917 sites")
|
|
|
|
|
|
def test_dates_versions_and_small_integers_are_not_claims() -> None:
|
|
assert cc.claim_numbers("Kemal Yaylali, September 2026") == set()
|
|
assert cc.claim_numbers("Week of January 11, 2027: version 2.1") == set()
|
|
assert cc.claim_numbers("3 of 5 posts") == set()
|
|
assert cc.claim_numbers("about 4 s and 18 s") == {"18"}
|
|
|
|
|
|
def test_wrapped_sentences_stay_whole_and_code_is_skipped() -> None:
|
|
text = (
|
|
"# Accuracy\n\nScores from a trained file are not\ncomparable with 99.46 % from vendor files.\n\n"
|
|
"```bash\nsimurg genotype --threads 4096\n```\n\n| Check | Result |\n|---|---|\n| GIAB | 0.9985 |\n"
|
|
)
|
|
claims = cc.extract_claims(Path("x.md"), text)
|
|
assert [c.line for c in claims] == [3, 12]
|
|
assert "not comparable with 99.46 %" in claims[0].text and claims[0].context == "Accuracy"
|
|
assert claims[1].context == "Accuracy / | Check | Result |" and "0.9985" in claims[1].nums
|
|
|
|
|
|
def test_candidates_match_rounded_figures(tmp_path: Path) -> None:
|
|
src = tmp_path / "r.md"
|
|
src.write_text(
|
|
"# Docker\n\n| Artifact | Size |\n|---|---|\n| image | 545 MB (x86_64), 551 MB (aarch64) |\n\nUnrelated 12345.\n"
|
|
)
|
|
passages = cc.load_passages([src])
|
|
claim = cc.Claim("README.md", 1, "The image is about 550 MB", "", cc.claim_numbers("The image is about 550 MB"))
|
|
found = cc.candidates(claim, passages)
|
|
assert len(found) == 1 and "545 MB" in found[0].text
|
|
|
|
|
|
def _ans(choice: str, conf: float, supports: float = 0.0) -> dict[str, object]:
|
|
probs = {"supports": supports, "contradicts": 0.0, "says_nothing": 0.0}
|
|
probs[choice] = max(probs[choice], conf)
|
|
return {"choice": choice, "confidence": conf, "probabilities": probs}
|
|
|
|
|
|
def test_verdicts() -> None:
|
|
assert cc.verdict({"s0": _ans("says_nothing", 0.9), "s1": _ans("supports", 0.95)})[0] == "verified"
|
|
assert cc.verdict({"s0": _ans("contradicts", 0.95)})[0] == "contradicted"
|
|
# a confident contradiction while another section leans towards support goes to a person
|
|
assert cc.verdict({"s0": _ans("contradicts", 0.95), "s1": _ans("says_nothing", 0.5, supports=0.4)})[0] == "review"
|
|
assert cc.verdict({"s0": _ans("says_nothing", 0.9), "s1": _ans("says_nothing", 0.85)})[0] == "unsupported"
|
|
assert cc.verdict({"s0": _ans("supports", 0.6)})[0] == "review"
|
|
|
|
|
|
def test_request_shape() -> None:
|
|
claim = cc.Claim("README.md", 1, "99.44 % concordance", "Validation", {"99.44"})
|
|
p = cc.Passage("docs/validation/V2_arraytrain.md", 7, "v2 gives 99.44 %", {"99.44"})
|
|
req = cc.build_request(claim, [p, p])
|
|
assert req["model"] == "jev-latest" and set(req["questions"]) == {"s0", "s1"} # type: ignore[arg-type]
|
|
q = req["questions"]["s1"] # type: ignore[index]
|
|
assert q["type"] == "choice" and "`sections[1].text`" in q["instructions"]
|
|
assert set(q["criteria"]) == {"supports", "contradicts", "says_nothing"}
|
|
assert req["state"]["sections"][0]["source"] == "docs/validation/V2_arraytrain.md:7" # type: ignore[index]
|