fix: overhaul the platform skeleton, add a serverless deployment track
An end-to-end audit found the repo could not build, test or run as shipped. This fixes every finding, then adds a Cloud Run track so the demo costs about £1/month idle instead of ~£150. CI (red on its first run) - api: setuptools could not build the package (flat layout with app/ and alembic/) - web: missing @types/node; `vitest run` exited 1 with no test files - pipeline: the stub run needed a gitignored VCF, and no process had a stub block - ruff pinned, mypy configured, DB tests on real Postgres (pgserver locally, service in CI) ML serving (scores were meaningless) - the registered model now carries its own feature engineering and returns predict_proba, so serving sends raw columns and cannot drift from training - resolve by registry alias (stages are deprecated in MLflow 3) and record the real version; re-scoring upserts instead of failing on the unique constraint - ClinVar labels parsed from VEP's lowercase terms Pipeline - exact ref/alt recovered from a CHROM_POS_REF_ALT VCF ID; loading is idempotent - job status reaches running/failed/succeeded, so the UI stops polling dead jobs - DATABASE_URL travels in the environment or a Nextflow secret, never on a command line - VEP cache and plugins staged as inputs; the gcp profile runs tasks on Google Batch Deployment - the API serves /api (matching the ingress); the web app reads its API URL at runtime - migrations run in an init container under a Postgres advisory lock - terraform: custom VPC shared with Batch, private Cloud SQL, API enablement, Workload Identity bindings, Secret Manager, deletion protection - serverless track, now the default: Cloud Run services scaling to zero, a Cloud Run job for the Nextflow driver, and Neon or Cloud SQL behind one DATABASE_URL secret. GKE and Argo remain, behind -var deploy_kubernetes=true. See docs/cloud.md. Correctness and security - 409 on duplicate sample names, 422 on bad paging, natural chromosome ordering, wider VEP text columns, enum dropped on downgrade, the sample's assembly actually used - vcf_uri restricted to gs:// objects or files under the data root, blocking option injection - CORS restricted to configured origins; `make down` no longer deletes volumes Data - docs/data.md records the peer-reviewed, openly licensed sources (GIAB HG002, ClinVar, gnomAD) with citations and an honest evaluation plan; `make data` fetches a chr22 slice Verified: api 50 tests, ml 18, loader 16, web 12; ruff, mypy, svelte-check, terraform validate and both kustomize overlays clean.
This commit is contained in:
Kemal Yaylali
+115
-41
@@ -1,59 +1,133 @@
|
||||
#!/usr/bin/env python
|
||||
"""Load a VEP --tab output into the rarelens Postgres schema and mark the job succeeded."""
|
||||
#!/usr/bin/env python3
|
||||
"""Load VEP --tab output into the rarelens Postgres schema and mark the job succeeded.
|
||||
|
||||
Variant identity (chrom/pos/ref/alt) comes from the VCF ID, which NORMALISE sets to
|
||||
CHROM_POS_REF_ALT: VEP's own Location/Allele columns trim indel alleles and shift positions.
|
||||
The database URL is read from $DATABASE_URL so it never appears on a command line or in .command.sh.
|
||||
Loading replaces any rows already stored for the job, so a retried task cannot duplicate variants.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
from sqlalchemy import create_engine, text
|
||||
from sqlalchemy import Engine, create_engine, text
|
||||
from sqlalchemy.engine import make_url
|
||||
|
||||
INSERT_CHUNK = 5000
|
||||
VEP_VERSION = re.compile(r"^## ENSEMBL VARIANT EFFECT PREDICTOR v(\S+)")
|
||||
|
||||
|
||||
def engine_for(url: str) -> Engine:
|
||||
"""Accept the API's asyncpg URL (or a plain postgresql:// one) and use psycopg."""
|
||||
parsed = make_url(url).set(drivername="postgresql+psycopg")
|
||||
query = dict(parsed.query)
|
||||
if "ssl" in query: # asyncpg's spelling; libpq (psycopg) wants sslmode
|
||||
query["sslmode"] = query.pop("ssl")
|
||||
return create_engine(parsed.set(query=query))
|
||||
|
||||
|
||||
def read_vep_tab(path: str | Path) -> pd.DataFrame:
|
||||
"""Read VEP --tab output as strings, keeping "-" (VEP's missing marker) verbatim."""
|
||||
with open(path) as fh:
|
||||
for n, line in enumerate(fh):
|
||||
if line.startswith("#Uploaded_variation"):
|
||||
break
|
||||
else:
|
||||
raise ValueError(f"{path}: no #Uploaded_variation header; is this VEP --tab output?")
|
||||
df = pd.read_csv(path, sep="\t", skiprows=n, dtype=str, keep_default_na=False)
|
||||
return df.rename(columns={"#Uploaded_variation": "Uploaded_variation"})
|
||||
|
||||
|
||||
def vep_version(path: str | Path) -> str | None:
|
||||
with open(path) as fh:
|
||||
for line in fh:
|
||||
if not line.startswith("##"):
|
||||
return None
|
||||
if m := VEP_VERSION.match(line):
|
||||
return m.group(1)
|
||||
return None
|
||||
|
||||
|
||||
def parse_variant_id(uid: str) -> tuple[str, int, str, str]:
|
||||
# rsplit: contig names may contain "_" (chrUn_KI270742v1); positions and alleles never do.
|
||||
parts = uid.rsplit("_", 3)
|
||||
if len(parts) != 4 or not parts[1].isdigit():
|
||||
raise ValueError(
|
||||
f"unexpected Uploaded_variation {uid!r}; NORMALISE must set VCF IDs to CHROM_POS_REF_ALT"
|
||||
)
|
||||
chrom, pos, ref, alt = parts
|
||||
return chrom, int(pos), ref, alt
|
||||
|
||||
|
||||
def _value(v: str | None) -> str | None:
|
||||
return None if v in (None, "", "-") else v
|
||||
|
||||
|
||||
def to_rows(df: pd.DataFrame, job_id: str) -> list[dict]:
|
||||
rows = []
|
||||
for rec in df.to_dict("records"):
|
||||
chrom, pos, ref, alt = parse_variant_id(rec["Uploaded_variation"])
|
||||
af = _value(rec.get("gnomADe_AF"))
|
||||
rows.append({
|
||||
"job_id": job_id,
|
||||
"chrom": chrom,
|
||||
"pos": pos,
|
||||
"ref": ref,
|
||||
"alt": alt,
|
||||
"gene": _value(rec.get("SYMBOL")),
|
||||
"consequence": _value(rec.get("Consequence")),
|
||||
"impact": _value(rec.get("IMPACT")),
|
||||
"hgvsc": _value(rec.get("HGVSc")),
|
||||
"hgvsp": _value(rec.get("HGVSp")),
|
||||
"gnomad_af": float(af) if af is not None else None,
|
||||
"clinvar_sig": _value(rec.get("CLIN_SIG")),
|
||||
"annotations": json.dumps({k: v for k, v in rec.items() if _value(v) is not None}),
|
||||
})
|
||||
return rows
|
||||
|
||||
|
||||
def load(engine: Engine, job_id: str, rows: list[dict], vep: str | None) -> None:
|
||||
with engine.begin() as conn:
|
||||
conn.execute(text("DELETE FROM variants WHERE job_id = :id"), {"id": job_id})
|
||||
for start in range(0, len(rows), INSERT_CHUNK):
|
||||
conn.execute(
|
||||
text("""
|
||||
INSERT INTO variants (job_id, chrom, pos, ref, alt, gene, consequence, impact,
|
||||
hgvsc, hgvsp, gnomad_af, clinvar_sig, annotations)
|
||||
VALUES (:job_id, :chrom, :pos, :ref, :alt, :gene, :consequence, :impact,
|
||||
:hgvsc, :hgvsp, :gnomad_af, :clinvar_sig, CAST(:annotations AS jsonb))
|
||||
"""),
|
||||
rows[start:start + INSERT_CHUNK],
|
||||
)
|
||||
conn.execute(
|
||||
text("""
|
||||
UPDATE jobs SET status = 'succeeded', vep_version = :vep, log = NULL,
|
||||
finished_at = now()
|
||||
WHERE id = :id
|
||||
"""),
|
||||
{"id": job_id, "vep": vep},
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--tsv", required=True)
|
||||
p.add_argument("--job-id", required=True)
|
||||
p.add_argument("--db-url", required=True)
|
||||
p.add_argument("--dry-run", action="store_true", help="parse only; do not touch the database")
|
||||
a = p.parse_args()
|
||||
|
||||
df = pd.read_csv(a.tsv, sep="\t", comment="#", header=None, dtype=str)
|
||||
with open(a.tsv) as fh:
|
||||
header = next(l for l in fh if l.startswith("#Uploaded_variation")).lstrip("#").rstrip().split("\t")
|
||||
df.columns = header
|
||||
|
||||
chrom_pos = df["Location"].str.split(":", expand=True)
|
||||
rows = []
|
||||
for i, r in df.iterrows():
|
||||
ref, _, alt = r["Uploaded_variation"].partition("/") if "/" in r["Uploaded_variation"] else ("", "", r["Allele"])
|
||||
rows.append({
|
||||
"job_id": a.job_id,
|
||||
"chrom": chrom_pos.iloc[i, 0],
|
||||
"pos": int(chrom_pos.iloc[i, 1].split("-")[0]),
|
||||
"ref": ref or "-",
|
||||
"alt": r["Allele"],
|
||||
"gene": r.get("SYMBOL") if r.get("SYMBOL") != "-" else None,
|
||||
"consequence": r["Consequence"],
|
||||
"impact": r["IMPACT"],
|
||||
"hgvsc": None if r.get("HGVSc") == "-" else r.get("HGVSc"),
|
||||
"hgvsp": None if r.get("HGVSp") == "-" else r.get("HGVSp"),
|
||||
"gnomad_af": None if r.get("gnomADe_AF", "-") == "-" else float(r["gnomADe_AF"]),
|
||||
"clinvar_sig": None if r.get("CLIN_SIG", "-") == "-" else r["CLIN_SIG"],
|
||||
"annotations": json.dumps({k: v for k, v in r.items() if v != "-"}),
|
||||
})
|
||||
|
||||
if a.db_url == "none":
|
||||
rows = to_rows(read_vep_tab(a.tsv), job_id=a.job_id)
|
||||
if a.dry_run:
|
||||
print(f"{len(rows)} variants parsed (dry run, no DB)")
|
||||
return
|
||||
|
||||
engine = create_engine(a.db_url.replace("+asyncpg", "+psycopg"))
|
||||
with engine.begin() as conn:
|
||||
conn.execute(text("""
|
||||
INSERT INTO variants (job_id, chrom, pos, ref, alt, gene, consequence, impact,
|
||||
hgvsc, hgvsp, gnomad_af, clinvar_sig, annotations)
|
||||
VALUES (:job_id, :chrom, :pos, :ref, :alt, :gene, :consequence, :impact,
|
||||
:hgvsc, :hgvsp, :gnomad_af, :clinvar_sig, CAST(:annotations AS jsonb))
|
||||
"""), rows)
|
||||
conn.execute(text("UPDATE jobs SET status='succeeded', finished_at=now() WHERE id=:id"),
|
||||
{"id": a.job_id})
|
||||
url = os.environ.get("DATABASE_URL")
|
||||
if not url:
|
||||
sys.exit("DATABASE_URL is not set")
|
||||
load(engine_for(url), a.job_id, rows, vep=vep_version(a.tsv))
|
||||
print(f"loaded {len(rows)} variants for job {a.job_id}", file=sys.stderr)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user