fix(science): stop scoring evidence that was never looked up
A review of the ranking's arithmetic found four things wrong, all of which made the score look better informed than it was. Measurements below are from this repo, not estimates. **Components now abstain instead of inventing a number.** A run without a VEP cache returns no allele frequencies, and rarity_score(None) read that as "absent from gnomAD, therefore maximally rare" and awarded every variant a free 0.25. jobs.has_frequencies / has_effect_scores record what the run actually produced, absent components are dropped from the weighted mean, and the remaining weights are renormalised so the score keeps its meaning. The UI shows "not looked up" rather than a bar, and the funnel stops calling a step "rare" when nothing was filtered. **Allele frequency is no longer a model feature.** It dominated: the same missense variant scored 0.887 at AF 0 and 0.0003 at AF 0.01. That double- counted, because the ranking already scores frequency explicitly, putting ~45% of every rank on one measurement; and it was circular, because ACMG assigns ClinVar's benign labels using frequency (BA1/BS1). Retraining without it moves missense AUROC from 0.872 to 0.500 — exactly random. The old figure was allele frequency, not variant-effect knowledge. The model therefore abstains unless CADD or AlphaMissense is present, since otherwise it only restates the consequence class. **Phenotype matching is weighted by information content** and HPO annotations are propagated up the ontology. Counting terms alike let "global developmental delay" (IC 0.93) count as much as "dilated left subclavian artery" (IC 7.88). **A real bug in the propagation, found by checking it.** The ancestor walk read a pre-order DFS backwards, which on a DAG lets a term resolve before one of its parents and inherit that parent alone instead of its lineage. It dropped 399 terms out of the phenotype branch, Camptodactyly and Chiari malformation among them. Now a true post-order, tested against a reference transitive closure. The ontology arithmetic moved to rarelens_ml.hpo so it is covered by tests, and rarelens_ml.benchmark measures the whole thing: across 10,178 published cases the causal gene ranks first 45.9-81.0% of the time against 5,269 genes, versus 0.02% for chance. docs/data.md reports that with its contamination (HPO's annotations come from these same case reports), and includes the measurement showing information-content weighting earns its place while propagation does not - kept anyway, for a reason the docs argue rather than assume.
This commit is contained in:
Kemal Yaylali
@@ -0,0 +1,99 @@
|
||||
"""The benchmark is the project's strongest scientific claim, so its arithmetic is tested."""
|
||||
import json
|
||||
import math
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from rarelens_ml.benchmark import cases, gene_annotations, rank_of, report
|
||||
from rarelens_ml.hpo import information_content
|
||||
from rarelens_ml.hpo import phenotype_score as score
|
||||
|
||||
|
||||
def annotations_file(tmp_path: Path, genes: dict[str, list[str]]) -> str:
|
||||
path = tmp_path / "gp.tsv"
|
||||
path.write_text("".join(f"{g}\t{t}\n" for g, terms in genes.items() for t in terms))
|
||||
return str(path)
|
||||
|
||||
|
||||
def phenopacket(gene: str, terms: list[str], excluded: list[str] | None = None) -> dict:
|
||||
return {
|
||||
"phenotypicFeatures": [{"type": {"id": t}} for t in terms]
|
||||
+ [{"type": {"id": t}, "excluded": True} for t in (excluded or [])],
|
||||
"interpretations": [
|
||||
{
|
||||
"diagnosis": {
|
||||
"genomicInterpretations": [
|
||||
{"variantInterpretation": {"variationDescriptor": {
|
||||
"geneContext": {"symbol": gene}}}}
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def store(tmp_path: Path, packets: dict[str, dict]) -> str:
|
||||
path = tmp_path / "pps.zip"
|
||||
with zipfile.ZipFile(path, "w") as z:
|
||||
for name, packet in packets.items():
|
||||
z.writestr(name, json.dumps(packet))
|
||||
return str(path)
|
||||
|
||||
|
||||
def test_information_content_makes_a_universal_term_worthless() -> None:
|
||||
genes = {"A": {"HP:1", "HP:2"}, "B": {"HP:1"}, "C": {"HP:1"}}
|
||||
ic = information_content(genes)
|
||||
assert ic["HP:1"] == pytest.approx(0.0) # every gene has it
|
||||
assert ic["HP:2"] == pytest.approx(math.log(3)) # one gene in three
|
||||
|
||||
|
||||
def test_score_is_recall_weighted_by_specificity() -> None:
|
||||
ic = {"HP:1": 0.0, "HP:2": 4.0}
|
||||
assert score(["HP:1", "HP:2"], {"HP:2"}, ic, 1.0) == pytest.approx(1.0)
|
||||
assert score(["HP:1", "HP:2"], {"HP:1"}, ic, 1.0) == pytest.approx(0.0)
|
||||
|
||||
|
||||
def test_rank_separates_optimistic_from_pessimistic_on_ties() -> None:
|
||||
"""Every gene carrying the same term ties; the report must not hide that."""
|
||||
genes = {"RIGHT": {"HP:1"}, "TIED": {"HP:1"}, "WRONG": {"HP:9"}}
|
||||
ic = {"HP:1": 1.0, "HP:9": 1.0}
|
||||
optimistic, pessimistic, target = rank_of("RIGHT", ["HP:1"], genes, ic, 1.0)
|
||||
assert (optimistic, pessimistic) == (1, 2)
|
||||
assert target == pytest.approx(1.0)
|
||||
|
||||
|
||||
def test_a_uniquely_matching_gene_ranks_first_either_way() -> None:
|
||||
genes = {"RIGHT": {"HP:1", "HP:2"}, "PARTIAL": {"HP:1"}, "WRONG": set()}
|
||||
ic = {"HP:1": 1.0, "HP:2": 1.0}
|
||||
assert rank_of("RIGHT", ["HP:1", "HP:2"], genes, ic, 1.0)[:2] == (1, 1)
|
||||
|
||||
|
||||
def test_cases_reads_the_causal_gene_and_drops_excluded_terms(tmp_path: Path) -> None:
|
||||
"""An excluded feature means the authors looked and did not find it."""
|
||||
path = store(tmp_path, {
|
||||
"a/one.json": phenopacket("TGFBR2", ["HP:1", "HP:2"], excluded=["HP:3"]),
|
||||
"a/notes.txt": {},
|
||||
})
|
||||
assert list(cases(path)) == [("TGFBR2", ["HP:1", "HP:2"])]
|
||||
|
||||
|
||||
def test_cases_skips_packets_without_exactly_one_causal_gene(tmp_path: Path) -> None:
|
||||
two = phenopacket("A", ["HP:1"])
|
||||
two["interpretations"][0]["diagnosis"]["genomicInterpretations"].append(
|
||||
{"variantInterpretation": {"variationDescriptor": {"geneContext": {"symbol": "B"}}}}
|
||||
)
|
||||
path = store(tmp_path, {"two.json": two, "none.json": phenopacket("C", [])})
|
||||
assert list(cases(path)) == []
|
||||
|
||||
|
||||
def test_gene_annotations_reads_the_export(tmp_path: Path) -> None:
|
||||
path = annotations_file(tmp_path, {"A": ["HP:1", "HP:2"], "B": ["HP:1"]})
|
||||
assert gene_annotations(path) == {"A": {"HP:1", "HP:2"}, "B": {"HP:1"}}
|
||||
|
||||
|
||||
def test_report_states_the_contamination_and_both_bounds() -> None:
|
||||
text = report([(1, 2, 1.0), (1, 1, 1.0), (3, 5, 0.5)], n_genes=100)
|
||||
assert "optimistic" in text and "pessimistic" in text
|
||||
assert "HPO already carries" in text # the caveat travels with the number
|
||||
@@ -9,7 +9,6 @@ def raw(**overrides: list) -> pd.DataFrame:
|
||||
base = {
|
||||
"impact": ["HIGH", "LOW", None],
|
||||
"consequence": ["stop_gained", "synonymous_variant", None],
|
||||
"gnomad_af": [None, "0.12", 0.001],
|
||||
"cadd_phred": ["35", "2.1", "-"],
|
||||
"am_pathogenicity": ["0.98", None, "-"],
|
||||
}
|
||||
@@ -18,13 +17,22 @@ def raw(**overrides: list) -> pd.DataFrame:
|
||||
|
||||
|
||||
def test_raw_columns_are_the_serving_contract() -> None:
|
||||
assert RAW_COLUMNS == ["impact", "consequence", "gnomad_af", "cadd_phred", "am_pathogenicity"]
|
||||
assert RAW_COLUMNS == ["impact", "consequence", "cadd_phred", "am_pathogenicity"]
|
||||
|
||||
|
||||
def test_allele_frequency_is_not_a_feature() -> None:
|
||||
"""It dominated the model and the ranking already scores it, auditably and only once.
|
||||
|
||||
Keeping it here also meant learning ACMG's own frequency-based benign rule from labels that
|
||||
rule produced, which is most of why the headline AUROC looked so good.
|
||||
"""
|
||||
assert "gnomad_af" not in RAW_COLUMNS
|
||||
assert "gnomad_af" not in build(raw(gnomad_af=[0.0, 0.5, None])).columns
|
||||
|
||||
|
||||
def test_build_ranks_impact_and_coerces_numbers() -> None:
|
||||
out = build(raw())
|
||||
assert out["impact_rank"].tolist() == [3, 1, 0]
|
||||
assert out["gnomad_af"].tolist() == [0.0, 0.12, 0.001] # missing AF means absent from gnomAD
|
||||
assert out["cadd_phred"].iloc[0] == 35.0
|
||||
assert math.isnan(out["cadd_phred"].iloc[2]) # VEP writes "-" for missing
|
||||
assert math.isnan(out["am_pathogenicity"].iloc[1])
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
"""The ontology arithmetic sits underneath the phenotype half of the ranking, so it is tested."""
|
||||
import io
|
||||
import math
|
||||
import random
|
||||
|
||||
import pytest
|
||||
|
||||
from rarelens_ml.hpo import (
|
||||
PHENOTYPIC_ABNORMALITY,
|
||||
ancestors_of,
|
||||
information_content,
|
||||
parse_obo,
|
||||
phenotype_score,
|
||||
propagate,
|
||||
)
|
||||
|
||||
OBO = f"""format-version: 1.2
|
||||
|
||||
[Term]
|
||||
id: {PHENOTYPIC_ABNORMALITY}
|
||||
name: Phenotypic abnormality
|
||||
|
||||
[Term]
|
||||
id: HP:0001
|
||||
name: Abnormality of the vasculature
|
||||
is_a: {PHENOTYPIC_ABNORMALITY} ! Phenotypic abnormality
|
||||
|
||||
[Term]
|
||||
id: HP:0002
|
||||
name: Aortic aneurysm
|
||||
is_a: HP:0001 ! Abnormality of the vasculature
|
||||
|
||||
[Term]
|
||||
id: HP:0003
|
||||
name: Aortic root aneurysm
|
||||
is_a: HP:0002 ! Aortic aneurysm
|
||||
|
||||
[Term]
|
||||
id: HP:0004
|
||||
name: Autosomal dominant inheritance
|
||||
|
||||
[Term]
|
||||
id: HP:0005
|
||||
name: Obsolete thing
|
||||
is_a: HP:0001 ! Abnormality of the vasculature
|
||||
is_obsolete: true
|
||||
"""
|
||||
|
||||
|
||||
def ontology() -> tuple[dict[str, set[str]], dict[str, str]]:
|
||||
return parse_obo(io.StringIO(OBO))
|
||||
|
||||
|
||||
def test_parse_obo_reads_parents_and_drops_obsolete_terms() -> None:
|
||||
parents, names = ontology()
|
||||
assert parents["HP:0003"] == {"HP:0002"}
|
||||
assert names["HP:0002"] == "Aortic aneurysm"
|
||||
assert "HP:0005" not in parents
|
||||
|
||||
|
||||
def test_ancestors_include_the_term_itself_and_the_whole_lineage() -> None:
|
||||
ancestors = ancestors_of(ontology()[0])
|
||||
assert ancestors["HP:0003"] == {"HP:0003", "HP:0002", "HP:0001", PHENOTYPIC_ABNORMALITY}
|
||||
assert ancestors["HP:0004"] == {"HP:0004"} # its own branch, not under phenotypic abnormality
|
||||
|
||||
|
||||
def closure(parents: dict[str, set[str]]) -> dict[str, set[str]]:
|
||||
"""Reference transitive closure by relaxation: obviously correct, too slow for 20k terms."""
|
||||
result = {node: {node} | set(ps) for node, ps in parents.items()}
|
||||
changed = True
|
||||
while changed:
|
||||
changed = False
|
||||
for node, found in result.items():
|
||||
grown = set(found)
|
||||
for parent in found - {node}:
|
||||
grown |= result.get(parent, {parent})
|
||||
if grown != found:
|
||||
result[node] = grown
|
||||
changed = True
|
||||
return result
|
||||
|
||||
|
||||
def test_ancestors_match_a_reference_closure_on_a_tangled_dag() -> None:
|
||||
"""The regression this guards cost 399 HPO terms, Camptodactyly and Chiari malformation among
|
||||
them: on a DAG a term can be reached before one of its parents, and the old walk then gave it
|
||||
that parent alone instead of the parent's whole lineage. It only shows up when a node shares
|
||||
ancestors by several routes, so the test needs a genuinely tangled graph rather than a
|
||||
hand-drawn diamond.
|
||||
"""
|
||||
rng = random.Random(0)
|
||||
nodes = [PHENOTYPIC_ABNORMALITY] + [f"HP:{i:04d}" for i in range(1, 80)]
|
||||
parents = {PHENOTYPIC_ABNORMALITY: set()}
|
||||
for i, node in enumerate(nodes[1:], start=1):
|
||||
# only earlier nodes may be parents, which keeps it acyclic
|
||||
parents[node] = set(rng.sample(nodes[:i], k=min(i, rng.randint(1, 3))))
|
||||
|
||||
for _ in range(5): # dict order decides the traversal, so try several
|
||||
shuffled = list(parents.items())
|
||||
rng.shuffle(shuffled)
|
||||
assert ancestors_of(dict(shuffled)) == closure(dict(shuffled))
|
||||
|
||||
|
||||
def test_every_descendant_of_the_root_keeps_the_root() -> None:
|
||||
"""The property that actually matters: losing it drops the term out of the phenotype branch."""
|
||||
parents = {
|
||||
PHENOTYPIC_ABNORMALITY: set(),
|
||||
"HP:P": {PHENOTYPIC_ABNORMALITY},
|
||||
"HP:X": {"HP:P"},
|
||||
"HP:N": {"HP:P"},
|
||||
"HP:A": {"HP:X", "HP:N"},
|
||||
}
|
||||
ancestors = ancestors_of(parents)
|
||||
for term in ("HP:P", "HP:X", "HP:N", "HP:A"):
|
||||
assert PHENOTYPIC_ABNORMALITY in ancestors[term], term
|
||||
|
||||
|
||||
def test_propagation_lets_a_parent_term_match_a_gene_annotated_with_a_child() -> None:
|
||||
ancestors = ancestors_of(ontology()[0])
|
||||
genes = propagate([("TGFBR2", "HP:0003")], ancestors)
|
||||
assert genes["TGFBR2"] == {"HP:0003", "HP:0002", "HP:0001"}
|
||||
|
||||
|
||||
def test_propagation_drops_the_root_and_anything_outside_the_phenotype_branch() -> None:
|
||||
ancestors = ancestors_of(ontology()[0])
|
||||
genes = propagate([("A", "HP:0003"), ("A", "HP:0004")], ancestors)
|
||||
assert PHENOTYPIC_ABNORMALITY not in genes["A"] # every gene has it; it carries no information
|
||||
assert "HP:0004" not in genes["A"] # inheritance is not a patient finding
|
||||
|
||||
|
||||
def test_information_content_is_zero_for_a_term_every_gene_carries() -> None:
|
||||
ic = information_content({"A": {"HP:1", "HP:2"}, "B": {"HP:1"}, "C": {"HP:1"}})
|
||||
assert ic["HP:1"] == pytest.approx(0.0)
|
||||
assert ic["HP:2"] == pytest.approx(math.log(3))
|
||||
|
||||
|
||||
def test_phenotype_score_weights_by_specificity() -> None:
|
||||
ic = {"HP:common": 0.1, "HP:rare": 6.0}
|
||||
terms = ["HP:common", "HP:rare"]
|
||||
assert phenotype_score(terms, {"HP:rare"}, ic, 1.0) == pytest.approx(6.0 / 6.1)
|
||||
assert phenotype_score(terms, {"HP:common"}, ic, 1.0) == pytest.approx(0.1 / 6.1)
|
||||
|
||||
|
||||
def test_phenotype_score_treats_an_unscored_term_as_maximally_specific() -> None:
|
||||
"""It can never match, so it must depress every gene equally rather than vanish."""
|
||||
assert phenotype_score(["HP:1", "HP:unknown"], {"HP:1"}, {"HP:1": 5.0}, 5.0) == pytest.approx(0.5)
|
||||
Reference in New Issue
Block a user