Kemal Yaylali
e76ae847a1
A review of the ranking's arithmetic found four things wrong, all of which made the score look better informed than it was. Measurements below are from this repo, not estimates. **Components now abstain instead of inventing a number.** A run without a VEP cache returns no allele frequencies, and rarity_score(None) read that as "absent from gnomAD, therefore maximally rare" and awarded every variant a free 0.25. jobs.has_frequencies / has_effect_scores record what the run actually produced, absent components are dropped from the weighted mean, and the remaining weights are renormalised so the score keeps its meaning. The UI shows "not looked up" rather than a bar, and the funnel stops calling a step "rare" when nothing was filtered. **Allele frequency is no longer a model feature.** It dominated: the same missense variant scored 0.887 at AF 0 and 0.0003 at AF 0.01. That double- counted, because the ranking already scores frequency explicitly, putting ~45% of every rank on one measurement; and it was circular, because ACMG assigns ClinVar's benign labels using frequency (BA1/BS1). Retraining without it moves missense AUROC from 0.872 to 0.500 — exactly random. The old figure was allele frequency, not variant-effect knowledge. The model therefore abstains unless CADD or AlphaMissense is present, since otherwise it only restates the consequence class. **Phenotype matching is weighted by information content** and HPO annotations are propagated up the ontology. Counting terms alike let "global developmental delay" (IC 0.93) count as much as "dilated left subclavian artery" (IC 7.88). **A real bug in the propagation, found by checking it.** The ancestor walk read a pre-order DFS backwards, which on a DAG lets a term resolve before one of its parents and inherit that parent alone instead of its lineage. It dropped 399 terms out of the phenotype branch, Camptodactyly and Chiari malformation among them. Now a true post-order, tested against a reference transitive closure. The ontology arithmetic moved to rarelens_ml.hpo so it is covered by tests, and rarelens_ml.benchmark measures the whole thing: across 10,178 published cases the causal gene ranks first 45.9-81.0% of the time against 5,269 genes, versus 0.02% for chance. docs/data.md reports that with its contamination (HPO's annotations come from these same case reports), and includes the measurement showing information-content weighting earns its place while propagation does not - kept anyway, for a reason the docs argue rather than assume.
164 lines
7.3 KiB
Python
164 lines
7.3 KiB
Python
"""The triage workflow: a case with a phenotype, ranked candidates, decisions, a report."""
|
|
import uuid
|
|
|
|
import pytest
|
|
from factories import seed_case
|
|
from httpx import AsyncClient
|
|
|
|
NF2_TERMS = [("HP:0000365", "Hearing impairment"), ("HP:0009592", "Vestibular schwannoma")]
|
|
CASE_TERMS = [*NF2_TERMS, ("HP:0002321", "Vertigo")]
|
|
|
|
|
|
async def a_case_with_candidates() -> tuple[uuid.UUID, uuid.UUID]:
|
|
return await seed_case(
|
|
phenotypes=CASE_TERMS,
|
|
gene_terms={"NF2": NF2_TERMS},
|
|
variants=[
|
|
{"id": 1, "gene": "NF2", "pos": 1000, "impact": "HIGH", "score": 0.94,
|
|
"clinvar_sig": "pathogenic"},
|
|
{"id": 2, "gene": "CHEK2", "pos": 2000, "impact": "MODERATE", "gnomad_af": 0.0004,
|
|
"score": 0.55},
|
|
{"id": 3, "gene": "TTN", "pos": 3000, "impact": "MODIFIER", "gnomad_af": None},
|
|
{"id": 4, "gene": "APOE", "pos": 4000, "impact": "HIGH", "gnomad_af": 0.3},
|
|
],
|
|
)
|
|
|
|
|
|
@pytest.mark.usefixtures("db")
|
|
async def test_a_case_keeps_its_phenotype(client: AsyncClient) -> None:
|
|
body = {
|
|
"name": "PROBAND-01",
|
|
"vcf_uri": "gs://bucket/proband.vcf.gz",
|
|
"phenotypes": [{"hpo_id": h, "label": lab} for h, lab in NF2_TERMS],
|
|
}
|
|
r = await client.post("/api/cases", json=body)
|
|
assert r.status_code == 201, r.text
|
|
assert [p["label"] for p in r.json()["phenotypes"]] == [lab for _, lab in NF2_TERMS]
|
|
|
|
listed = (await client.get("/api/cases")).json()
|
|
assert listed[0]["name"] == "PROBAND-01"
|
|
assert listed[0]["shortlisted"] == 0
|
|
|
|
|
|
@pytest.mark.usefixtures("db")
|
|
async def test_the_funnel_shows_the_narrowing(client: AsyncClient) -> None:
|
|
case_id, _ = await a_case_with_candidates()
|
|
page = (await client.get(f"/api/cases/{case_id}/candidates")).json()
|
|
# 4 variants -> 3 rare -> 2 rare and coding -> 1 of those in a phenotype-matched gene
|
|
assert page["funnel"] == {
|
|
"total": 4, "rare": 3, "candidates": 2, "phenotype_matched": 1, "frequencies": True,
|
|
}
|
|
|
|
|
|
@pytest.mark.usefixtures("db")
|
|
async def test_the_phenotype_matched_variant_ranks_first_with_its_reasons(
|
|
client: AsyncClient,
|
|
) -> None:
|
|
case_id, _ = await a_case_with_candidates()
|
|
page = (await client.get(f"/api/cases/{case_id}/candidates")).json()
|
|
|
|
assert [c["variant"]["gene"] for c in page["items"]] == ["NF2", "CHEK2"]
|
|
top = page["items"][0]
|
|
assert [t["label"] for t in top["matched_terms"]] == ["Hearing impairment", "Vestibular schwannoma"]
|
|
assert top["components"]["phenotype"] == pytest.approx(2 / 3, abs=1e-4)
|
|
assert top["components"]["rarity"] == 1.0
|
|
assert page["weights"]["phenotype"] == 0.35
|
|
# ClinVar is evidence, not an input to the rank.
|
|
assert top["variant"]["clinvar_sig"] == "pathogenic"
|
|
|
|
|
|
@pytest.mark.usefixtures("db")
|
|
async def test_a_reviewer_decides_and_the_decision_sticks(client: AsyncClient) -> None:
|
|
case_id, _ = await a_case_with_candidates()
|
|
variant_id = (await client.get(f"/api/cases/{case_id}/candidates")).json()["items"][0]["variant"]["id"]
|
|
|
|
r = await client.post(
|
|
f"/api/variants/{variant_id}/decision",
|
|
json={"state": "shortlisted", "reason": "fits the phenotype", "note": "confirm by Sanger"},
|
|
)
|
|
assert r.status_code == 200, r.text
|
|
|
|
# Changing your mind replaces the decision rather than failing.
|
|
r = await client.post(f"/api/variants/{variant_id}/decision", json={"state": "dismissed"})
|
|
assert r.status_code == 200
|
|
detail = (await client.get(f"/api/variants/{variant_id}")).json()
|
|
assert detail["decision"]["state"] == "dismissed"
|
|
assert (await client.get("/api/cases")).json()[0]["shortlisted"] == 0
|
|
|
|
|
|
@pytest.mark.usefixtures("db")
|
|
async def test_the_variant_panel_carries_the_evidence(client: AsyncClient) -> None:
|
|
case_id, _ = await a_case_with_candidates()
|
|
variant_id = (await client.get(f"/api/cases/{case_id}/candidates")).json()["items"][0]["variant"]["id"]
|
|
|
|
detail = (await client.get(f"/api/variants/{variant_id}")).json()
|
|
assert detail["variant"]["hgvsp"] is None or isinstance(detail["variant"]["hgvsp"], str)
|
|
assert detail["score"] > 0
|
|
assert [t["hpo_id"] for t in detail["matched_terms"]] == [h for h, _ in NF2_TERMS]
|
|
assert "annotations" in detail
|
|
|
|
|
|
@pytest.mark.usefixtures("db")
|
|
async def test_the_report_is_the_decision_trail(client: AsyncClient) -> None:
|
|
case_id, _ = await a_case_with_candidates()
|
|
items = (await client.get(f"/api/cases/{case_id}/candidates")).json()["items"]
|
|
await client.post(f"/api/variants/{items[0]['variant']['id']}/decision",
|
|
json={"state": "shortlisted", "reason": "fits the phenotype"})
|
|
await client.post(f"/api/variants/{items[1]['variant']['id']}/decision",
|
|
json={"state": "dismissed", "reason": "gene unrelated to phenotype"})
|
|
|
|
report = (await client.get(f"/api/cases/{case_id}/report")).json()
|
|
assert report["funnel"]["candidates"] == 2
|
|
assert [v["variant"]["gene"] for v in report["shortlisted"]] == ["NF2"]
|
|
assert report["shortlisted"][0]["decision"]["reason"] == "fits the phenotype"
|
|
assert [v["variant"]["gene"] for v in report["dismissed"]] == ["CHEK2"]
|
|
assert report["provenance"]["vep_version"] == "113.0"
|
|
|
|
|
|
@pytest.mark.usefixtures("db")
|
|
async def test_the_report_shows_the_top_candidates_before_anyone_decides(
|
|
client: AsyncClient,
|
|
) -> None:
|
|
"""The first thing anyone opens is a report with no decisions in it; it must still say something."""
|
|
case_id, _ = await a_case_with_candidates()
|
|
|
|
report = (await client.get(f"/api/cases/{case_id}/report")).json()
|
|
assert report["shortlisted"] == []
|
|
assert [c["variant"]["gene"] for c in report["top"]] == ["NF2", "CHEK2"]
|
|
|
|
|
|
@pytest.mark.usefixtures("db")
|
|
async def test_the_report_drops_decided_variants_from_the_top_list(client: AsyncClient) -> None:
|
|
case_id, _ = await a_case_with_candidates()
|
|
items = (await client.get(f"/api/cases/{case_id}/candidates")).json()["items"]
|
|
await client.post(f"/api/variants/{items[0]['variant']['id']}/decision",
|
|
json={"state": "shortlisted", "reason": "fits"})
|
|
|
|
report = (await client.get(f"/api/cases/{case_id}/report")).json()
|
|
assert [c["variant"]["gene"] for c in report["shortlisted"]] == ["NF2"]
|
|
assert [c["variant"]["gene"] for c in report["top"]] == ["CHEK2"]
|
|
|
|
|
|
@pytest.mark.usefixtures("db")
|
|
async def test_candidates_can_still_be_filtered(client: AsyncClient) -> None:
|
|
case_id, _ = await a_case_with_candidates()
|
|
page = (await client.get(f"/api/cases/{case_id}/candidates", params={"gene": "chek2"})).json()
|
|
assert [c["variant"]["gene"] for c in page["items"]] == ["CHEK2"]
|
|
# The funnel still describes the whole case, not the filtered view.
|
|
assert page["funnel"]["total"] == 4
|
|
|
|
|
|
@pytest.mark.usefixtures("db")
|
|
async def test_phenotype_search_backs_the_picker(client: AsyncClient) -> None:
|
|
await seed_case(gene_terms={"NF2": NF2_TERMS}, variants=[])
|
|
found = (await client.get("/api/phenotypes", params={"q": "vestibular"})).json()
|
|
assert found == [{"hpo_id": "HP:0009592", "label": "Vestibular schwannoma"}]
|
|
|
|
|
|
@pytest.mark.usefixtures("db")
|
|
async def test_candidates_before_the_pipeline_has_run(client: AsyncClient) -> None:
|
|
r = await client.post("/api/cases", json={"name": "empty", "vcf_uri": "gs://bucket/x.vcf.gz"})
|
|
page = (await client.get(f"/api/cases/{r.json()['id']}/candidates")).json()
|
|
assert page["items"] == []
|
|
assert page["funnel"]["total"] == 0
|