Kemal Yaylali
197975cc42
"Variants are unscored" was accurate: nothing was ever trained, so a quarter of every rank was dead weight and the UI leaked a connection error at the reader. - scripts/make-training-set.sh derives a training table from ClinVar directly. ClinVar already carries the molecular consequence, the gene and an allele frequency, which is the feature set serving sends, so this avoids running VEP over hundreds of thousands of variants. 2-star records only. - train.py now holds out whole genes (GroupShuffleSplit). docs/data.md had said to do this since the data pass; the code was still doing a random split, which is the leak Grimm 2015 describes. - evaluate() reports missense on its own. On the last run: AUROC 0.986 over 74,239 held-out variants, but 0.872 over the 13,553 missense ones, and the docs say plainly why even that is flattered — within missense the only live feature is allele frequency, and ClinVar's benign calls often use allele frequency as evidence (ACMG BA1/BS1), so the feature partly caused the label. - the 503 now names what is missing (model@alias via tracking URI) and leaves the exception in the server log instead of the UI. - make training-set / make train; the 58 MB table is gitignored. Verified end to end: model registered as v2, the simulated NF2 case scores 0.999 on the planted variant, and it now ranks 1.00 with all four components live. Tests: api 77, ml 22, loader 16, web 32; ruff, mypy, svelte-check clean.
106 lines
3.7 KiB
Python
106 lines
3.7 KiB
Python
import math
|
|
import uuid
|
|
|
|
import numpy as np
|
|
import pandas as pd
|
|
import pytest
|
|
from factories import seed_case
|
|
from httpx import AsyncClient
|
|
from sqlalchemy import select
|
|
|
|
from app.db import SessionLocal
|
|
from app.models import JobStatus, Prediction, Variant
|
|
from app.services import scoring
|
|
|
|
|
|
def variant(**kw: object) -> Variant:
|
|
fields: dict = {"chrom": "22", "pos": 1, "ref": "A", "alt": "G", "annotations": {}}
|
|
fields.update(kw)
|
|
return Variant(**fields)
|
|
|
|
|
|
def test_raw_frame_sends_the_model_contract_columns() -> None:
|
|
frame = scoring.raw_frame([
|
|
variant(impact="HIGH", consequence="stop_gained", gnomad_af=None,
|
|
annotations={"CADD_PHRED": "35", "am_pathogenicity": "0.98"}),
|
|
variant(impact="LOW", consequence="synonymous_variant", gnomad_af=0.2, annotations={}),
|
|
])
|
|
assert list(frame.columns) == scoring.RAW_COLUMNS
|
|
assert frame["impact"].tolist() == ["HIGH", "LOW"]
|
|
assert frame["cadd_phred"].iloc[0] == "35"
|
|
assert pd.isna(frame["cadd_phred"].iloc[1])
|
|
assert math.isnan(frame["gnomad_af"].iloc[0])
|
|
|
|
|
|
class FakeModel:
|
|
def __init__(self, score: float) -> None:
|
|
self.score = score
|
|
|
|
def predict(self, frame: pd.DataFrame) -> np.ndarray:
|
|
assert list(frame.columns) == scoring.RAW_COLUMNS
|
|
return np.full(len(frame), self.score)
|
|
|
|
|
|
async def make_case(status: JobStatus, n_variants: int) -> tuple[uuid.UUID, uuid.UUID]:
|
|
return await seed_case(
|
|
status=status,
|
|
variants=[{"pos": i + 1, "annotations": {}} for i in range(n_variants)],
|
|
)
|
|
|
|
|
|
async def predictions(job_id: uuid.UUID) -> list[Prediction]:
|
|
async with SessionLocal() as s:
|
|
rows = await s.scalars(select(Prediction).join(Variant).where(Variant.job_id == job_id))
|
|
return list(rows)
|
|
|
|
|
|
@pytest.mark.usefixtures("db")
|
|
async def test_scoring_twice_updates_instead_of_failing(
|
|
client: AsyncClient, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
case_id, job_id = await make_case(JobStatus.succeeded, n_variants=3)
|
|
|
|
monkeypatch.setattr(scoring, "load_model", lambda: (FakeModel(0.9), "7"))
|
|
r = await client.post(f"/api/cases/{case_id}/score")
|
|
assert r.status_code == 200, r.text
|
|
assert r.json() == {"case_id": str(case_id), "scored": 3, "model_version": "7"}
|
|
|
|
monkeypatch.setattr(scoring, "load_model", lambda: (FakeModel(0.2), "8"))
|
|
r = await client.post(f"/api/cases/{case_id}/score")
|
|
assert r.status_code == 200, r.text
|
|
|
|
preds = await predictions(job_id)
|
|
assert len(preds) == 3
|
|
assert {(p.score, p.model_version) for p in preds} == {(0.2, "8")}
|
|
|
|
|
|
@pytest.mark.usefixtures("db")
|
|
async def test_scoring_an_unknown_case_is_404(client: AsyncClient) -> None:
|
|
r = await client.post(f"/api/cases/{uuid.uuid4()}/score")
|
|
assert r.status_code == 404
|
|
|
|
|
|
@pytest.mark.usefixtures("db")
|
|
async def test_scoring_before_the_annotation_finishes_is_409(client: AsyncClient) -> None:
|
|
case_id, _ = await make_case(JobStatus.running, n_variants=1)
|
|
r = await client.post(f"/api/cases/{case_id}/score")
|
|
assert r.status_code == 409
|
|
|
|
|
|
@pytest.mark.usefixtures("db")
|
|
async def test_an_unreachable_model_registry_is_explained_not_a_500(
|
|
client: AsyncClient, monkeypatch: pytest.MonkeyPatch
|
|
) -> None:
|
|
case_id, _ = await make_case(JobStatus.succeeded, n_variants=1)
|
|
|
|
def unreachable() -> tuple:
|
|
raise ConnectionError("connection refused to http://localhost:5000")
|
|
|
|
monkeypatch.setattr(scoring, "load_model", unreachable)
|
|
r = await client.post(f"/api/cases/{case_id}/score")
|
|
assert r.status_code == 503
|
|
detail = r.json()["detail"]
|
|
# Names what is missing; the exception itself belongs in the server log, not the UI.
|
|
assert "rarelens-pathogenicity@production" in detail
|
|
assert "Max retries" not in detail and "Traceback" not in detail
|