fix: overhaul the platform skeleton, add a serverless deployment track

An end-to-end audit found the repo could not build, test or run as shipped. This fixes every finding, then adds a Cloud Run track so the demo costs about £1/month idle instead of ~£150. CI (red on its first run) - api: setuptools could not build the package (flat layout with app/ and alembic/) - web: missing @types/node; `vitest run` exited 1 with no test files - pipeline: the stub run needed a gitignored VCF, and no process had a stub block - ruff pinned, mypy configured, DB tests on real Postgres (pgserver locally, service in CI) ML serving (scores were meaningless) - the registered model now carries its own feature engineering and returns predict_proba, so serving sends raw columns and cannot drift from training - resolve by registry alias (stages are deprecated in MLflow 3) and record the real version; re-scoring upserts instead of failing on the unique constraint - ClinVar labels parsed from VEP's lowercase terms Pipeline - exact ref/alt recovered from a CHROM_POS_REF_ALT VCF ID; loading is idempotent - job status reaches running/failed/succeeded, so the UI stops polling dead jobs - DATABASE_URL travels in the environment or a Nextflow secret, never on a command line - VEP cache and plugins staged as inputs; the gcp profile runs tasks on Google Batch Deployment - the API serves /api (matching the ingress); the web app reads its API URL at runtime - migrations run in an init container under a Postgres advisory lock - terraform: custom VPC shared with Batch, private Cloud SQL, API enablement, Workload Identity bindings, Secret Manager, deletion protection - serverless track, now the default: Cloud Run services scaling to zero, a Cloud Run job for the Nextflow driver, and Neon or Cloud SQL behind one DATABASE_URL secret. GKE and Argo remain, behind -var deploy_kubernetes=true. See docs/cloud.md. Correctness and security - 409 on duplicate sample names, 422 on bad paging, natural chromosome ordering, wider VEP text columns, enum dropped on downgrade, the sample's assembly actually used - vcf_uri restricted to gs:// objects or files under the data root, blocking option injection - CORS restricted to configured origins; `make down` no longer deletes volumes Data - docs/data.md records the peer-reviewed, openly licensed sources (GIAB HG002, ClinVar, gnomAD) with citations and an honest evaluation plan; `make data` fetches a chr22 slice Verified: api 50 tests, ml 18, loader 16, web 12; ruff, mypy, svelte-check, terraform validate and both kustomize overlays clean.
2026-09-12 07:21:11 +01:00
parent 5463f489a3
commit 11fb6b3d73
100 changed files with 3431 additions and 340 deletions
@@ -1,59 +1,133 @@
-#!/usr/bin/env python
-"""Load a VEP --tab output into the rarelens Postgres schema and mark the job succeeded."""
+#!/usr/bin/env python3
+"""Load VEP --tab output into the rarelens Postgres schema and mark the job succeeded.
+
+Variant identity (chrom/pos/ref/alt) comes from the VCF ID, which NORMALISE sets to
+CHROM_POS_REF_ALT: VEP's own Location/Allele columns trim indel alleles and shift positions.
+The database URL is read from $DATABASE_URL so it never appears on a command line or in .command.sh.
+Loading replaces any rows already stored for the job, so a retried task cannot duplicate variants.
+"""
 import argparse
 import json
+import os
+import re
 import sys
+from pathlib import Path

 import pandas as pd
-from sqlalchemy import create_engine, text
+from sqlalchemy import Engine, create_engine, text
+from sqlalchemy.engine import make_url
+
+INSERT_CHUNK = 5000
+VEP_VERSION = re.compile(r"^## ENSEMBL VARIANT EFFECT PREDICTOR v(\S+)")
+
+
+def engine_for(url: str) -> Engine:
+    """Accept the API's asyncpg URL (or a plain postgresql:// one) and use psycopg."""
+    parsed = make_url(url).set(drivername="postgresql+psycopg")
+    query = dict(parsed.query)
+    if "ssl" in query:  # asyncpg's spelling; libpq (psycopg) wants sslmode
+        query["sslmode"] = query.pop("ssl")
+    return create_engine(parsed.set(query=query))
+
+
+def read_vep_tab(path: str | Path) -> pd.DataFrame:
+    """Read VEP --tab output as strings, keeping "-" (VEP's missing marker) verbatim."""
+    with open(path) as fh:
+        for n, line in enumerate(fh):
+            if line.startswith("#Uploaded_variation"):
+                break
+        else:
+            raise ValueError(f"{path}: no #Uploaded_variation header; is this VEP --tab output?")
+    df = pd.read_csv(path, sep="\t", skiprows=n, dtype=str, keep_default_na=False)
+    return df.rename(columns={"#Uploaded_variation": "Uploaded_variation"})
+
+
+def vep_version(path: str | Path) -> str | None:
+    with open(path) as fh:
+        for line in fh:
+            if not line.startswith("##"):
+                return None
+            if m := VEP_VERSION.match(line):
+                return m.group(1)
+    return None
+
+
+def parse_variant_id(uid: str) -> tuple[str, int, str, str]:
+    # rsplit: contig names may contain "_" (chrUn_KI270742v1); positions and alleles never do.
+    parts = uid.rsplit("_", 3)
+    if len(parts) != 4 or not parts[1].isdigit():
+        raise ValueError(
+            f"unexpected Uploaded_variation {uid!r}; NORMALISE must set VCF IDs to CHROM_POS_REF_ALT"
+        )
+    chrom, pos, ref, alt = parts
+    return chrom, int(pos), ref, alt
+
+
+def _value(v: str | None) -> str | None:
+    return None if v in (None, "", "-") else v
+
+
+def to_rows(df: pd.DataFrame, job_id: str) -> list[dict]:
+    rows = []
+    for rec in df.to_dict("records"):
+        chrom, pos, ref, alt = parse_variant_id(rec["Uploaded_variation"])
+        af = _value(rec.get("gnomADe_AF"))
+        rows.append({
+            "job_id": job_id,
+            "chrom": chrom,
+            "pos": pos,
+            "ref": ref,
+            "alt": alt,
+            "gene": _value(rec.get("SYMBOL")),
+            "consequence": _value(rec.get("Consequence")),
+            "impact": _value(rec.get("IMPACT")),
+            "hgvsc": _value(rec.get("HGVSc")),
+            "hgvsp": _value(rec.get("HGVSp")),
+            "gnomad_af": float(af) if af is not None else None,
+            "clinvar_sig": _value(rec.get("CLIN_SIG")),
+            "annotations": json.dumps({k: v for k, v in rec.items() if _value(v) is not None}),
+        })
+    return rows
+
+
+def load(engine: Engine, job_id: str, rows: list[dict], vep: str | None) -> None:
+    with engine.begin() as conn:
+        conn.execute(text("DELETE FROM variants WHERE job_id = :id"), {"id": job_id})
+        for start in range(0, len(rows), INSERT_CHUNK):
+            conn.execute(
+                text("""
+                    INSERT INTO variants (job_id, chrom, pos, ref, alt, gene, consequence, impact,
+                                          hgvsc, hgvsp, gnomad_af, clinvar_sig, annotations)
+                    VALUES (:job_id, :chrom, :pos, :ref, :alt, :gene, :consequence, :impact,
+                            :hgvsc, :hgvsp, :gnomad_af, :clinvar_sig, CAST(:annotations AS jsonb))
+                """),
+                rows[start:start + INSERT_CHUNK],
+            )
+        conn.execute(
+            text("""
+                UPDATE jobs SET status = 'succeeded', vep_version = :vep, log = NULL,
+                                finished_at = now()
+                WHERE id = :id
+            """),
+            {"id": job_id, "vep": vep},
+        )


 def main() -> None:
    p = argparse.ArgumentParser()
    p.add_argument("--tsv", required=True)
    p.add_argument("--job-id", required=True)
-    p.add_argument("--db-url", required=True)
+    p.add_argument("--dry-run", action="store_true", help="parse only; do not touch the database")
    a = p.parse_args()

-    df = pd.read_csv(a.tsv, sep="\t", comment="#", header=None, dtype=str)
-    with open(a.tsv) as fh:
-        header = next(l for l in fh if l.startswith("#Uploaded_variation")).lstrip("#").rstrip().split("\t")
-    df.columns = header
-
-    chrom_pos = df["Location"].str.split(":", expand=True)
-    rows = []
-    for i, r in df.iterrows():
-        ref, _, alt = r["Uploaded_variation"].partition("/") if "/" in r["Uploaded_variation"] else ("", "", r["Allele"])
-        rows.append({
-            "job_id": a.job_id,
-            "chrom": chrom_pos.iloc[i, 0],
-            "pos": int(chrom_pos.iloc[i, 1].split("-")[0]),
-            "ref": ref or "-",
-            "alt": r["Allele"],
-            "gene": r.get("SYMBOL") if r.get("SYMBOL") != "-" else None,
-            "consequence": r["Consequence"],
-            "impact": r["IMPACT"],
-            "hgvsc": None if r.get("HGVSc") == "-" else r.get("HGVSc"),
-            "hgvsp": None if r.get("HGVSp") == "-" else r.get("HGVSp"),
-            "gnomad_af": None if r.get("gnomADe_AF", "-") == "-" else float(r["gnomADe_AF"]),
-            "clinvar_sig": None if r.get("CLIN_SIG", "-") == "-" else r["CLIN_SIG"],
-            "annotations": json.dumps({k: v for k, v in r.items() if v != "-"}),
-        })
-
-    if a.db_url == "none":
+    rows = to_rows(read_vep_tab(a.tsv), job_id=a.job_id)
+    if a.dry_run:
        print(f"{len(rows)} variants parsed (dry run, no DB)")
        return
-
-    engine = create_engine(a.db_url.replace("+asyncpg", "+psycopg"))
-    with engine.begin() as conn:
-        conn.execute(text("""
-            INSERT INTO variants (job_id, chrom, pos, ref, alt, gene, consequence, impact,
-                                  hgvsc, hgvsp, gnomad_af, clinvar_sig, annotations)
-            VALUES (:job_id, :chrom, :pos, :ref, :alt, :gene, :consequence, :impact,
-                    :hgvsc, :hgvsp, :gnomad_af, :clinvar_sig, CAST(:annotations AS jsonb))
-        """), rows)
-        conn.execute(text("UPDATE jobs SET status='succeeded', finished_at=now() WHERE id=:id"),
-                     {"id": a.job_id})
+    url = os.environ.get("DATABASE_URL")
+    if not url:
+        sys.exit("DATABASE_URL is not set")
+    load(engine_for(url), a.job_id, rows, vep=vep_version(a.tsv))
    print(f"loaded {len(rows)} variants for job {a.job_id}", file=sys.stderr)


@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+"""Set a job's status, e.g. from the Argo exit handler when a workflow fails.
+
+A succeeded job is never overwritten: the loader's success is the source of truth.
+"""
+import argparse
+import os
+import sys
+
+from load_db import engine_for
+from sqlalchemy import Engine, text
+
+
+def set_status(engine: Engine, job_id: str, status: str, log: str | None) -> None:
+    with engine.begin() as conn:
+        conn.execute(
+            text("""
+                UPDATE jobs
+                SET status = CAST(:status AS jobstatus), log = :log,
+                    finished_at = CASE WHEN :status IN ('succeeded', 'failed') THEN now() END
+                WHERE id = :id AND status <> 'succeeded'
+            """),
+            {"id": job_id, "status": status, "log": log},
+        )
+
+
+def main() -> None:
+    p = argparse.ArgumentParser()
+    p.add_argument("--job-id", required=True)
+    p.add_argument("--status", required=True, choices=["running", "failed"])
+    p.add_argument("--log")
+    a = p.parse_args()
+    url = os.environ.get("DATABASE_URL")
+    if not url:
+        sys.exit("DATABASE_URL is not set")
+    set_status(engine_for(url), a.job_id, a.status, a.log)
+
+
+if __name__ == "__main__":
+    main()