fix: overhaul the platform skeleton, add a serverless deployment track

An end-to-end audit found the repo could not build, test or run as shipped. This fixes every finding, then adds a Cloud Run track so the demo costs about £1/month idle instead of ~£150. CI (red on its first run) - api: setuptools could not build the package (flat layout with app/ and alembic/) - web: missing @types/node; `vitest run` exited 1 with no test files - pipeline: the stub run needed a gitignored VCF, and no process had a stub block - ruff pinned, mypy configured, DB tests on real Postgres (pgserver locally, service in CI) ML serving (scores were meaningless) - the registered model now carries its own feature engineering and returns predict_proba, so serving sends raw columns and cannot drift from training - resolve by registry alias (stages are deprecated in MLflow 3) and record the real version; re-scoring upserts instead of failing on the unique constraint - ClinVar labels parsed from VEP's lowercase terms Pipeline - exact ref/alt recovered from a CHROM_POS_REF_ALT VCF ID; loading is idempotent - job status reaches running/failed/succeeded, so the UI stops polling dead jobs - DATABASE_URL travels in the environment or a Nextflow secret, never on a command line - VEP cache and plugins staged as inputs; the gcp profile runs tasks on Google Batch Deployment - the API serves /api (matching the ingress); the web app reads its API URL at runtime - migrations run in an init container under a Postgres advisory lock - terraform: custom VPC shared with Batch, private Cloud SQL, API enablement, Workload Identity bindings, Secret Manager, deletion protection - serverless track, now the default: Cloud Run services scaling to zero, a Cloud Run job for the Nextflow driver, and Neon or Cloud SQL behind one DATABASE_URL secret. GKE and Argo remain, behind -var deploy_kubernetes=true. See docs/cloud.md. Correctness and security - 409 on duplicate sample names, 422 on bad paging, natural chromosome ordering, wider VEP text columns, enum dropped on downgrade, the sample's assembly actually used - vcf_uri restricted to gs:// objects or files under the data root, blocking option injection - CORS restricted to configured origins; `make down` no longer deletes volumes Data - docs/data.md records the peer-reviewed, openly licensed sources (GIAB HG002, ClinVar, gnomAD) with citations and an honest evaluation plan; `make data` fetches a chr22 slice Verified: api 50 tests, ml 18, loader 16, web 12; ruff, mypy, svelte-check, terraform validate and both kustomize overlays clean.
2026-09-12 07:21:11 +01:00
parent 5463f489a3
commit 11fb6b3d73
100 changed files with 3431 additions and 340 deletions
@@ -0,0 +1,38 @@
+import math
+
+import pandas as pd
+
+from rarelens_ml.features import RAW_COLUMNS, build
+
+
+def raw(**overrides: list) -> pd.DataFrame:
+    base = {
+        "impact": ["HIGH", "LOW", None],
+        "consequence": ["stop_gained", "synonymous_variant", None],
+        "gnomad_af": [None, "0.12", 0.001],
+        "cadd_phred": ["35", "2.1", "-"],
+        "am_pathogenicity": ["0.98", None, "-"],
+    }
+    base.update(overrides)
+    return pd.DataFrame(base, index=[10, 11, 12])
+
+
+def test_raw_columns_are_the_serving_contract() -> None:
+    assert RAW_COLUMNS == ["impact", "consequence", "gnomad_af", "cadd_phred", "am_pathogenicity"]
+
+
+def test_build_ranks_impact_and_coerces_numbers() -> None:
+    out = build(raw())
+    assert out["impact_rank"].tolist() == [3, 1, 0]
+    assert out["gnomad_af"].tolist() == [0.0, 0.12, 0.001]  # missing AF means absent from gnomAD
+    assert out["cadd_phred"].iloc[0] == 35.0
+    assert math.isnan(out["cadd_phred"].iloc[2])  # VEP writes "-" for missing
+    assert math.isnan(out["am_pathogenicity"].iloc[1])
+
+
+def test_build_keeps_the_input_index() -> None:
+    assert build(raw()).index.tolist() == [10, 11, 12]
+
+
+def test_build_makes_consequence_categorical() -> None:
+    assert isinstance(build(raw())["consequence"].dtype, pd.CategoricalDtype)
@@ -0,0 +1,104 @@
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import pytest
+
+from rarelens_ml.train import label, read_vep_tab
+
+HEADER = [
+    "Uploaded_variation", "Location", "Allele", "Consequence", "IMPACT", "SYMBOL",
+    "gnomADe_AF", "CLIN_SIG", "CADD_PHRED", "am_pathogenicity",
+]
+
+
+def write_vep_tab(path: Path, rows: list[list[str]]) -> Path:
+    lines = [
+        "## ENSEMBL VARIANT EFFECT PREDICTOR v113.0",
+        "## Column descriptions:",
+        "#" + "\t".join(HEADER),
+        *("\t".join(r) for r in rows),
+    ]
+    path.write_text("\n".join(lines) + "\n")
+    return path
+
+
+@pytest.mark.parametrize(
+    ("clin_sig", "expected"),
+    [
+        # VEP writes lowercase, comma-separated terms from co-located ClinVar records.
+        ("pathogenic", 1),
+        ("pathogenic,likely_pathogenic", 1),
+        ("likely_benign", 0),
+        ("benign,likely_benign", 0),
+        # ClinVar VCF CLNSIG spelling must keep working too.
+        ("Pathogenic/Likely_pathogenic", 1),
+        ("Benign", 0),
+        ("uncertain_significance", None),
+        ("pathogenic,benign", None),  # conflicting evidence is not a label
+        ("-", None),
+        ("", None),
+        (np.nan, None),
+    ],
+)
+def test_label(clin_sig: object, expected: int | None) -> None:
+    assert label(clin_sig) == expected
+
+
+def test_read_vep_tab_uses_the_hash_header_and_keeps_dashes(tmp_path: Path) -> None:
+    tsv = write_vep_tab(
+        tmp_path / "x.vep.tsv",
+        [["22_1_A_G", "22:1", "G", "missense_variant", "MODERATE", "TBX1", "-", "pathogenic", "28", "0.9"]],
+    )
+    df = read_vep_tab(tsv)
+    assert list(df.columns) == HEADER
+    assert df.loc[0, "gnomADe_AF"] == "-"
+    assert df.loc[0, "CLIN_SIG"] == "pathogenic"
+
+
+def test_load_returns_raw_serving_columns_and_labels(tmp_path: Path) -> None:
+    from rarelens_ml.features import RAW_COLUMNS
+    from rarelens_ml.train import load
+
+    tsv = write_vep_tab(
+        tmp_path / "x.vep.tsv",
+        [
+            ["a", "22:1", "G", "missense_variant", "MODERATE", "TBX1", "0.0001", "pathogenic", "28", "0.9"],
+            ["b", "22:2", "A", "synonymous_variant", "LOW", "CHEK2", "0.12", "benign", "3", "-"],
+            ["c", "22:3", "T", "intron_variant", "MODIFIER", "CHEK2", "0.3", "uncertain_significance", "1", "-"],
+        ],
+    )
+    X, y = load(str(tsv))
+    assert list(X.columns) == RAW_COLUMNS
+    assert y.tolist() == [1, 0]  # the VUS row is dropped
+
+
+def test_logged_model_returns_probabilities_from_raw_columns(tmp_path: Path) -> None:
+    """The registered model must take the raw columns serving sends and return P(pathogenic)."""
+    import mlflow
+
+    from rarelens_ml.train import fit, log_and_register
+
+    rng = np.random.default_rng(0)
+    n = 400
+    impact = rng.choice(["HIGH", "MODERATE", "LOW", "MODIFIER"], n)
+    y = pd.Series(((impact == "HIGH") | (rng.random(n) < 0.1)).astype(int))
+    X = pd.DataFrame({
+        "impact": impact,
+        "consequence": rng.choice(["stop_gained", "missense_variant", "intron_variant"], n),
+        "gnomad_af": rng.random(n).round(4).astype(str),   # strings, as read from the DB
+        "cadd_phred": (rng.random(n) * 40).round(1).astype(str),
+        "am_pathogenicity": "-",
+    })
+
+    mlflow.set_tracking_uri(f"sqlite:///{tmp_path}/mlflow.db")
+    mlflow.set_experiment("test")
+    clf = fit(X, y)
+    version = log_and_register(clf, model_name="rarelens-test", alias="production")
+
+    model = mlflow.pyfunc.load_model("models:/rarelens-test@production")
+    scores = np.asarray(model.predict(X.head(50)))
+    assert version == "1"
+    assert scores.shape == (50,)
+    assert ((scores >= 0) & (scores <= 1)).all()
+    assert not set(np.unique(scores)) <= {0.0, 1.0}, "got class labels, expected probabilities"