fix: overhaul the platform skeleton, add a serverless deployment track

An end-to-end audit found the repo could not build, test or run as shipped. This fixes every finding, then adds a Cloud Run track so the demo costs about £1/month idle instead of ~£150. CI (red on its first run) - api: setuptools could not build the package (flat layout with app/ and alembic/) - web: missing @types/node; `vitest run` exited 1 with no test files - pipeline: the stub run needed a gitignored VCF, and no process had a stub block - ruff pinned, mypy configured, DB tests on real Postgres (pgserver locally, service in CI) ML serving (scores were meaningless) - the registered model now carries its own feature engineering and returns predict_proba, so serving sends raw columns and cannot drift from training - resolve by registry alias (stages are deprecated in MLflow 3) and record the real version; re-scoring upserts instead of failing on the unique constraint - ClinVar labels parsed from VEP's lowercase terms Pipeline - exact ref/alt recovered from a CHROM_POS_REF_ALT VCF ID; loading is idempotent - job status reaches running/failed/succeeded, so the UI stops polling dead jobs - DATABASE_URL travels in the environment or a Nextflow secret, never on a command line - VEP cache and plugins staged as inputs; the gcp profile runs tasks on Google Batch Deployment - the API serves /api (matching the ingress); the web app reads its API URL at runtime - migrations run in an init container under a Postgres advisory lock - terraform: custom VPC shared with Batch, private Cloud SQL, API enablement, Workload Identity bindings, Secret Manager, deletion protection - serverless track, now the default: Cloud Run services scaling to zero, a Cloud Run job for the Nextflow driver, and Neon or Cloud SQL behind one DATABASE_URL secret. GKE and Argo remain, behind -var deploy_kubernetes=true. See docs/cloud.md. Correctness and security - 409 on duplicate sample names, 422 on bad paging, natural chromosome ordering, wider VEP text columns, enum dropped on downgrade, the sample's assembly actually used - vcf_uri restricted to gs:// objects or files under the data root, blocking option injection - CORS restricted to configured origins; `make down` no longer deletes volumes Data - docs/data.md records the peer-reviewed, openly licensed sources (GIAB HG002, ClinVar, gnomAD) with citations and an honest evaluation plan; `make data` fetches a chr22 slice Verified: api 50 tests, ml 18, loader 16, web 12; ruff, mypy, svelte-check, terraform validate and both kustomize overlays clean.
2026-09-12 07:21:11 +01:00
parent 5463f489a3
commit 11fb6b3d73
100 changed files with 3431 additions and 340 deletions
@@ -1,54 +1,130 @@
-"""Train a pathogenicity classifier on ClinVar labels (Pathogenic/Likely pathogenic vs Benign/Likely benign).
+"""Train a pathogenicity classifier on ClinVar labels ((likely) pathogenic vs (likely) benign).

 Label leakage warning: CLIN_SIG must never be a feature. This is a learning exercise, not a clinical model.
-Usage: python -m rarelens_ml.train --tsv results/clinvar.vep.tsv
+Usage: python -m rarelens_ml.train --tsv results/clinvar.vep.tsv --register
 """
 import argparse
+import re
+from pathlib import Path

 import lightgbm as lgb
 import mlflow
-import mlflow.lightgbm
 import pandas as pd
+import sklearn
+from mlflow import MlflowClient
 from sklearn.metrics import average_precision_score, roc_auc_score
 from sklearn.model_selection import train_test_split

-from rarelens_ml.features import build
+from rarelens_ml.features import RAW_COLUMNS, build
+from rarelens_ml.model import PathogenicityModel

-POS = {"Pathogenic", "Likely_pathogenic", "Pathogenic/Likely_pathogenic"}
-NEG = {"Benign", "Likely_benign", "Benign/Likely_benign"}
+PACKAGE_DIR = Path(__file__).resolve().parent
+MODEL_NAME = "rarelens-pathogenicity"
+PARAMS = {
+    "n_estimators": 400, "learning_rate": 0.05, "num_leaves": 31, "class_weight": "balanced",
+    "verbose": -1,
+}
+
+POS = {"pathogenic", "likely_pathogenic"}
+NEG = {"benign", "likely_benign"}
+# VEP --tab column -> raw feature column (am_pathogenicity already matches).
+VEP_TO_RAW = {
+    "IMPACT": "impact", "Consequence": "consequence", "gnomADe_AF": "gnomad_af",
+    "CADD_PHRED": "cadd_phred",
+}
+
+
+def label(clin_sig: object) -> int | None:
+    """1 / 0 when every ClinVar term agrees, None for VUS, conflicts and missing values.
+
+    Accepts VEP's lowercase comma-separated form ("pathogenic,likely_pathogenic") and ClinVar's
+    CLNSIG form ("Pathogenic/Likely_pathogenic").
+    """
+    if not isinstance(clin_sig, str):
+        return None
+    terms = {t for t in re.split(r"[,&/|]", clin_sig.strip().lower()) if t and t != "-"}
+    if terms and terms <= POS:
+        return 1
+    if terms and terms <= NEG:
+        return 0
+    return None
+
+
+def read_vep_tab(path: str | Path) -> pd.DataFrame:
+    """Read VEP --tab output as strings, keeping "-" (VEP's missing marker) verbatim.
+
+    Skips the "##" preamble by position instead of comment="#", which would also cut any value
+    containing "#".
+    """
+    with open(path) as fh:
+        for n, line in enumerate(fh):
+            if line.startswith("#Uploaded_variation"):
+                break
+        else:
+            raise ValueError(f"{path}: no #Uploaded_variation header; is this VEP --tab output?")
+    df = pd.read_csv(path, sep="\t", skiprows=n, dtype=str, keep_default_na=False)
+    return df.rename(columns={"#Uploaded_variation": "Uploaded_variation"})


 def load(tsv: str) -> tuple[pd.DataFrame, pd.Series]:
-    df = pd.read_csv(tsv, sep="\t", comment="#", header=None, dtype=str)
-    with open(tsv) as fh:
-        df.columns = next(l for l in fh if l.startswith("#Uploaded")).lstrip("#").rstrip().split("\t")
-    df = df.rename(columns={"IMPACT": "impact", "Consequence": "consequence", "gnomADe_AF": "gnomad_af",
-                            "CADD_PHRED": "cadd_phred"})
-    y = df["CLIN_SIG"].map(lambda s: 1 if s in POS else 0 if s in NEG else None)
+    df = read_vep_tab(tsv).rename(columns=VEP_TO_RAW)
+    for col in RAW_COLUMNS:  # plugin columns are absent when VEP ran without CADD/AlphaMissense
+        if col not in df:
+            df[col] = pd.NA
+    y = df["CLIN_SIG"].map(label)
    keep = y.notna()
-    return build(df[keep]), y[keep].astype(int)
+    return (
+        df.loc[keep, RAW_COLUMNS].reset_index(drop=True),
+        y[keep].astype(int).reset_index(drop=True),
+    )
+
+
+def fit(X: pd.DataFrame, y: pd.Series) -> lgb.LGBMClassifier:
+    return lgb.LGBMClassifier(**PARAMS).fit(build(X), y)
+
+
+def log_and_register(clf: lgb.LGBMClassifier, model_name: str, alias: str) -> str:
+    """Log the pyfunc, register it and point `alias` at the new version. Returns the version."""
+    info = mlflow.pyfunc.log_model(
+        name="model",
+        python_model=PathogenicityModel(clf),
+        code_paths=[str(PACKAGE_DIR)],
+        registered_model_name=model_name,
+        pip_requirements=[
+            f"lightgbm=={lgb.__version__}",
+            f"pandas=={pd.__version__}",
+            f"scikit-learn=={sklearn.__version__}",
+        ],
+    )
+    version = str(info.registered_model_version)
+    MlflowClient().set_registered_model_alias(model_name, alias, version)
+    return version


 def main() -> None:
    p = argparse.ArgumentParser()
    p.add_argument("--tsv", required=True)
-    p.add_argument("--register", action="store_true")
+    p.add_argument("--register", action="store_true",
+                   help="register the model and move the alias to the new version")
+    p.add_argument("--alias", default="production")
    a = p.parse_args()

    X, y = load(a.tsv)
    Xtr, Xte, ytr, yte = train_test_split(X, y, test_size=0.2, stratify=y, random_state=42)

-    mlflow.set_experiment("rarelens-pathogenicity")
+    mlflow.set_experiment(MODEL_NAME)
    with mlflow.start_run():
-        params = {"n_estimators": 400, "learning_rate": 0.05, "num_leaves": 31, "class_weight": "balanced"}
-        mlflow.log_params(params)
-        model = lgb.LGBMClassifier(**params).fit(Xtr, ytr)
-        proba = model.predict_proba(Xte)[:, 1]
-        mlflow.log_metrics({"auroc": roc_auc_score(yte, proba), "auprc": average_precision_score(yte, proba)})
-        mlflow.lightgbm.log_model(
-            model, "model",
-            registered_model_name="rarelens-pathogenicity" if a.register else None,
-        )
+        mlflow.log_params(PARAMS)
+        clf = fit(Xtr, ytr)
+        proba = clf.predict_proba(build(Xte))[:, 1]
+        mlflow.log_metrics({"auroc": roc_auc_score(yte, proba),
+                            "auprc": average_precision_score(yte, proba)})
+        if a.register:
+            version = log_and_register(clf, MODEL_NAME, a.alias)
+            print(f"registered {MODEL_NAME} v{version} as @{a.alias}")
+        else:
+            mlflow.pyfunc.log_model(name="model", python_model=PathogenicityModel(clf),
+                                    code_paths=[str(PACKAGE_DIR)])


 if __name__ == "__main__":