fix: overhaul the platform skeleton, add a serverless deployment track

An end-to-end audit found the repo could not build, test or run as shipped. This
fixes every finding, then adds a Cloud Run track so the demo costs about £1/month
idle instead of ~£150.

CI (red on its first run)
- api: setuptools could not build the package (flat layout with app/ and alembic/)
- web: missing @types/node; `vitest run` exited 1 with no test files
- pipeline: the stub run needed a gitignored VCF, and no process had a stub block
- ruff pinned, mypy configured, DB tests on real Postgres (pgserver locally, service in CI)

ML serving (scores were meaningless)
- the registered model now carries its own feature engineering and returns predict_proba,
  so serving sends raw columns and cannot drift from training
- resolve by registry alias (stages are deprecated in MLflow 3) and record the real
  version; re-scoring upserts instead of failing on the unique constraint
- ClinVar labels parsed from VEP's lowercase terms

Pipeline
- exact ref/alt recovered from a CHROM_POS_REF_ALT VCF ID; loading is idempotent
- job status reaches running/failed/succeeded, so the UI stops polling dead jobs
- DATABASE_URL travels in the environment or a Nextflow secret, never on a command line
- VEP cache and plugins staged as inputs; the gcp profile runs tasks on Google Batch

Deployment
- the API serves /api (matching the ingress); the web app reads its API URL at runtime
- migrations run in an init container under a Postgres advisory lock
- terraform: custom VPC shared with Batch, private Cloud SQL, API enablement, Workload
  Identity bindings, Secret Manager, deletion protection
- serverless track, now the default: Cloud Run services scaling to zero, a Cloud Run job
  for the Nextflow driver, and Neon or Cloud SQL behind one DATABASE_URL secret. GKE and
  Argo remain, behind -var deploy_kubernetes=true. See docs/cloud.md.

Correctness and security
- 409 on duplicate sample names, 422 on bad paging, natural chromosome ordering, wider
  VEP text columns, enum dropped on downgrade, the sample's assembly actually used
- vcf_uri restricted to gs:// objects or files under the data root, blocking option injection
- CORS restricted to configured origins; `make down` no longer deletes volumes

Data
- docs/data.md records the peer-reviewed, openly licensed sources (GIAB HG002, ClinVar,
  gnomAD) with citations and an honest evaluation plan; `make data` fetches a chr22 slice

Verified: api 50 tests, ml 18, loader 16, web 12; ruff, mypy, svelte-check, terraform
validate and both kustomize overlays clean.

api/app/config.py

@@ -1,5 +1,10 @@
+from pathlib import Path
+
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
+# api/app/config.py -> repo root locally; "/" in the API image, where compose mounts /pipeline.
+REPO_ROOT = Path(__file__).resolve().parents[2]
+
 
 class Settings(BaseSettings):
     model_config = SettingsConfigDict(env_file=".env", extra="ignore")
@@ -7,9 +12,22 @@ class Settings(BaseSettings):
     database_url: str = "postgresql+asyncpg://rarelens:rarelens@localhost:5432/rarelens"
     mlflow_tracking_uri: str = "http://localhost:5000"
     model_name: str = "rarelens-pathogenicity"
-    model_stage: str = "Production"
+    # Registry alias set by `rarelens_ml.train --register` (stages are deprecated in MLflow 3).
+    model_alias: str = "production"
+    # A model artifact URI (gs://...) scores without an MLflow server running; wins over the registry.
+    model_uri: str | None = None
     gcs_bucket: str | None = None          # set in GCP; local uses ./data
     pubsub_topic: str | None = None        # "vcf-uploaded" in GCP; local runs pipeline inline
+    # Serverless track: run the Nextflow driver as a Cloud Run job instead of Argo + Pub/Sub.
+    cloudrun_job: str | None = None
+    gcp_project: str | None = None         # required with pubsub_topic or cloudrun_job
+    gcp_region: str = "europe-west2"
+    pipeline_dir: Path = REPO_ROOT / "pipeline"
+    nextflow_profile: str = "docker"
+    # Local (non-gs://) VCFs must live under this directory.
+    local_data_root: Path = Path("/data")
+    # Browsers calling the API cross-origin; behind the ingress the UI is same-origin.
+    cors_origins: list[str] = ["http://localhost:5173"]
 
 
 settings = Settings()

api/app/db.py

@@ -1,13 +1,33 @@
 from collections.abc import AsyncIterator
+from typing import Annotated
 
+from fastapi import Depends
+from sqlalchemy.engine import make_url
 from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
 
 from app.config import settings
 
-engine = create_async_engine(settings.database_url, pool_pre_ping=True)
+
+def normalize_async_url(url: str) -> str:
+    """Translate libpq's `sslmode=` into the `ssl=` asyncpg understands.
+
+    Hosted Postgres (Neon, Supabase) hands out sslmode= URLs, which asyncpg rejects outright.
+    """
+    parsed = make_url(url)
+    if "asyncpg" not in parsed.drivername or "sslmode" not in parsed.query:
+        return url
+    query = dict(parsed.query)
+    query["ssl"] = query.pop("sslmode")
+    return parsed.set(query=query).render_as_string(hide_password=False)
+
+
+engine = create_async_engine(normalize_async_url(settings.database_url), pool_pre_ping=True)
 SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
 
 
 async def get_session() -> AsyncIterator[AsyncSession]:
     async with SessionLocal() as session:
         yield session
+
+
+SessionDep = Annotated[AsyncSession, Depends(get_session)]

api/app/main.py

@@ -1,8 +1,9 @@
 from contextlib import asynccontextmanager
 
-from fastapi import FastAPI
+from fastapi import APIRouter, FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 
+from app.config import settings
 from app.routers import jobs, predictions, samples, variants
 
 
@@ -14,13 +15,19 @@ async def lifespan(app: FastAPI):
 
 app = FastAPI(title="rarelens API", version="0.1.0", lifespan=lifespan)
 app.add_middleware(
-    CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"]
+    CORSMiddleware,
+    allow_origins=settings.cors_origins,
+    allow_methods=["GET", "POST"],
+    allow_headers=["content-type"],
 )
 
-app.include_router(samples.router, prefix="/samples", tags=["samples"])
-app.include_router(jobs.router, prefix="/jobs", tags=["jobs"])
-app.include_router(variants.router, prefix="/variants", tags=["variants"])
-app.include_router(predictions.router, prefix="/predictions", tags=["predictions"])
+# The ingress forwards /api/* to this service unchanged, and local dev uses the same prefix.
+api = APIRouter(prefix="/api")
+api.include_router(samples.router, prefix="/samples", tags=["samples"])
+api.include_router(jobs.router, prefix="/jobs", tags=["jobs"])
+api.include_router(variants.router, prefix="/variants", tags=["variants"])
+api.include_router(predictions.router, prefix="/predictions", tags=["predictions"])
+app.include_router(api)
 
 
 @app.get("/health", tags=["ops"])

api/app/models.py

@@ -2,9 +2,9 @@
 
 One sample -> many jobs; one job -> many variants; one variant -> one prediction (latest).
 """
-from datetime import datetime
 import enum
 import uuid
+from datetime import datetime
 
 from sqlalchemy import DateTime, Enum, Float, ForeignKey, Integer, String, Text, func
 from sqlalchemy.dialects.postgresql import JSONB, UUID
@@ -55,12 +55,12 @@ class Variant(Base):
     ref: Mapped[str] = mapped_column(Text)
     alt: Mapped[str] = mapped_column(Text)
     gene: Mapped[str | None] = mapped_column(String(60), index=True)
-    consequence: Mapped[str | None] = mapped_column(String(120))
+    consequence: Mapped[str | None] = mapped_column(Text)   # "&"-joined VEP terms
     impact: Mapped[str | None] = mapped_column(String(20))
     hgvsc: Mapped[str | None] = mapped_column(Text)
     hgvsp: Mapped[str | None] = mapped_column(Text)
     gnomad_af: Mapped[float | None] = mapped_column(Float)
-    clinvar_sig: Mapped[str | None] = mapped_column(String(120))
+    clinvar_sig: Mapped[str | None] = mapped_column(Text)   # ","-joined co-located ClinVar terms
     annotations: Mapped[dict] = mapped_column(JSONB, default=dict)   # full VEP CSQ record
     job: Mapped[Job] = relationship(back_populates="variants")
     prediction: Mapped["Prediction | None"] = relationship(back_populates="variant", uselist=False)

api/app/routers/jobs.py

@@ -1,9 +1,8 @@
 import uuid
 
-from fastapi import APIRouter, Depends, HTTPException
-from sqlalchemy.ext.asyncio import AsyncSession
+from fastapi import APIRouter, HTTPException
 
-from app.db import get_session
+from app.db import SessionDep
 from app.models import Job
 from app.schemas import JobOut
 
@@ -11,7 +10,7 @@ router = APIRouter()
 
 
 @router.get("/{job_id}", response_model=JobOut)
-async def get_job(job_id: uuid.UUID, session: AsyncSession = Depends(get_session)):
+async def get_job(job_id: uuid.UUID, session: SessionDep):
     job = await session.get(Job, job_id)
     if job is None:
         raise HTTPException(404, "job not found")

api/app/routers/predictions.py

@@ -1,16 +1,22 @@
 import uuid
 
-from fastapi import APIRouter, Depends
-from sqlalchemy.ext.asyncio import AsyncSession
+from fastapi import APIRouter, HTTPException
 
-from app.db import get_session
+from app.db import SessionDep
+from app.models import Job, JobStatus
+from app.schemas import ScoreOut
 from app.services.scoring import score_job
 
 router = APIRouter()
 
 
-@router.post("/score/{job_id}")
-async def score(job_id: uuid.UUID, session: AsyncSession = Depends(get_session)) -> dict:
-    """Load the registered MLflow model and score every variant of a job."""
-    n = await score_job(job_id, session)
-    return {"job_id": str(job_id), "scored": n}
+@router.post("/score/{job_id}", response_model=ScoreOut)
+async def score(job_id: uuid.UUID, session: SessionDep) -> ScoreOut:
+    """Score every variant of a finished job with the model behind the registry alias."""
+    job = await session.get(Job, job_id)
+    if job is None:
+        raise HTTPException(404, "job not found")
+    if job.status != JobStatus.succeeded:
+        raise HTTPException(409, f"job is {job.status.value}; only succeeded jobs can be scored")
+    n, version = await score_job(job_id, session)
+    return ScoreOut(job_id=job_id, scored=n, model_version=version)

api/app/routers/samples.py

@@ -1,34 +1,39 @@
 import uuid
+from datetime import UTC, datetime
 
-from fastapi import APIRouter, Depends, HTTPException, status
+from fastapi import APIRouter, HTTPException, status
 from sqlalchemy import select
-from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.exc import IntegrityError
 
-from app.db import get_session
-from app.models import Job, Sample
+from app.db import SessionDep
+from app.models import Job, JobStatus, Sample
 from app.schemas import JobOut, SampleCreate, SampleOut
-from app.services.events import publish_vcf_uploaded
+from app.services import events
 
 router = APIRouter()
 
 
 @router.get("", response_model=list[SampleOut])
-async def list_samples(session: AsyncSession = Depends(get_session)):
+async def list_samples(session: SessionDep):
     result = await session.scalars(select(Sample).order_by(Sample.created_at.desc()))
     return result.all()
 
 
 @router.post("", response_model=SampleOut, status_code=status.HTTP_201_CREATED)
-async def create_sample(payload: SampleCreate, session: AsyncSession = Depends(get_session)):
+async def create_sample(payload: SampleCreate, session: SessionDep):
     sample = Sample(**payload.model_dump())
     session.add(sample)
-    await session.commit()
+    try:
+        await session.commit()
+    except IntegrityError:  # samples.name is unique
+        await session.rollback()
+        raise HTTPException(409, f"a sample named {payload.name!r} already exists") from None
     await session.refresh(sample)
     return sample
 
 
 @router.get("/{sample_id}/jobs", response_model=list[JobOut])
-async def list_jobs(sample_id: uuid.UUID, session: AsyncSession = Depends(get_session)):
+async def list_jobs(sample_id: uuid.UUID, session: SessionDep):
     result = await session.scalars(
         select(Job).where(Job.sample_id == sample_id).order_by(Job.created_at.desc())
     )
@@ -36,14 +41,21 @@ async def list_jobs(sample_id: uuid.UUID, session: AsyncSession = Depends(get_se
 
 
 @router.post("/{sample_id}/annotate", response_model=JobOut, status_code=status.HTTP_202_ACCEPTED)
-async def annotate(sample_id: uuid.UUID, session: AsyncSession = Depends(get_session)):
+async def annotate(sample_id: uuid.UUID, session: SessionDep):
     sample = await session.get(Sample, sample_id)
     if sample is None:
         raise HTTPException(404, "sample not found")
-    job = Job(sample_id=sample.id)
+    # Commit `running` before launching: a local run that dies instantly is marked failed by its
+    # watcher, and a later status write here would overwrite that.
+    job = Job(sample_id=sample.id, status=JobStatus.running)
     session.add(job)
     await session.commit()
+    try:
+        job.workflow_ref = await events.launch(job.id, sample.vcf_uri, sample.assembly)
+    except events.LaunchError as e:
+        job.status = JobStatus.failed
+        job.log = str(e)
+        job.finished_at = datetime.now(UTC)
+    await session.commit()
     await session.refresh(job)
-    # Emits to Pub/Sub in GCP; runs the Nextflow pipeline inline for local dev.
-    await publish_vcf_uploaded(job_id=job.id, vcf_uri=sample.vcf_uri)
     return job

api/app/routers/variants.py

@@ -1,28 +1,37 @@
 import uuid
 
-from fastapi import APIRouter, Depends, Query
-from sqlalchemy import func, select
-from sqlalchemy.ext.asyncio import AsyncSession
+from fastapi import APIRouter, Query
+from sqlalchemy import Integer, case, cast, func, select
 from sqlalchemy.orm import selectinload
 
-from app.db import get_session
+from app.db import SessionDep
 from app.models import Prediction, Variant
-from app.schemas import VariantPage
+from app.schemas import VariantOut, VariantPage
 
 router = APIRouter()
 
+# Karyotype order (1..22, X, Y, MT) instead of text order, where "10" sorts before "2".
+_chrom = func.regexp_replace(Variant.chrom, "^chr", "", "i")
+CHROM_ORDER = case(
+    (_chrom.regexp_match("^[0-9]+$"), cast(_chrom, Integer)),
+    (_chrom == "X", 23),
+    (_chrom == "Y", 24),
+    (_chrom.in_(["M", "MT"]), 25),
+    else_=26,
+)
+
 
 @router.get("", response_model=VariantPage)
 async def list_variants(
     job_id: uuid.UUID,
+    session: SessionDep,
     gene: str | None = None,
     impact: str | None = Query(None, pattern="^(HIGH|MODERATE|LOW|MODIFIER)$"),
     max_af: float | None = Query(None, ge=0, le=1),
     min_score: float | None = Query(None, ge=0, le=1),
-    limit: int = Query(50, le=500),
-    offset: int = 0,
-    session: AsyncSession = Depends(get_session),
-):
+    limit: int = Query(50, ge=1, le=500),
+    offset: int = Query(0, ge=0),
+) -> VariantPage:
     stmt = select(Variant).where(Variant.job_id == job_id)
     if gene:
         stmt = stmt.where(Variant.gene == gene.upper())
@@ -31,13 +40,21 @@ async def list_variants(
     if max_af is not None:
         stmt = stmt.where((Variant.gnomad_af.is_(None)) | (Variant.gnomad_af <= max_af))
     if min_score is not None:
-        stmt = stmt.join(Prediction, Prediction.variant_id == Variant.id).where(Prediction.score >= min_score)
+        stmt = stmt.join(Prediction, Prediction.variant_id == Variant.id).where(
+            Prediction.score >= min_score
+        )
 
     total = await session.scalar(select(func.count()).select_from(stmt.subquery()))
     rows = await session.scalars(
         stmt.options(selectinload(Variant.prediction))
-        .order_by(Variant.chrom, Variant.pos)
+        # id breaks ties between split multiallelics at one position, keeping pages stable.
+        .order_by(CHROM_ORDER, Variant.chrom, Variant.pos, Variant.id)
         .limit(limit)
         .offset(offset)
     )
-    return VariantPage(items=rows.all(), total=total or 0, limit=limit, offset=offset)
+    return VariantPage(
+        items=[VariantOut.model_validate(v) for v in rows],
+        total=total or 0,
+        limit=limit,
+        offset=offset,
+    )

api/app/schemas.py

@@ -1,8 +1,12 @@
-from datetime import datetime
+import re
 import uuid
+from datetime import datetime
+from pathlib import PurePosixPath
+from typing import Literal
 
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, ConfigDict, Field, field_validator
 
+from app.config import settings
 from app.models import JobStatus
 
 
@@ -10,10 +14,34 @@ class ORMModel(BaseModel):
     model_config = ConfigDict(from_attributes=True)
 
 
+Assembly = Literal["GRCh38", "GRCh37"]
+VCF_SUFFIXES = (".vcf", ".vcf.gz", ".vcf.bgz", ".bcf")
+GCS_URI = re.compile(r"gs://[a-z0-9][a-z0-9._-]{1,220}[a-z0-9]/\S+")
+
+
 class SampleCreate(BaseModel):
     name: str = Field(min_length=1, max_length=120)
     vcf_uri: str
-    assembly: str = "GRCh38"
+    # Passed to VEP --assembly; the VEP cache must contain it.
+    assembly: Assembly = "GRCh38"
+
+    @field_validator("vcf_uri")
+    @classmethod
+    def _gcs_object_or_file_under_data_root(cls, v: str) -> str:
+        # The URI becomes a Nextflow argument and a path the pipeline reads: accept a GCS object or
+        # a file under the local data root, never something that parses as an option.
+        if not v or any(ord(c) < 32 for c in v):
+            raise ValueError("vcf_uri must be a non-empty single line")
+        if not v.lower().endswith(VCF_SUFFIXES):
+            raise ValueError(f"vcf_uri must end in one of {', '.join(VCF_SUFFIXES)}")
+        if v.startswith("gs://"):
+            if not GCS_URI.fullmatch(v):
+                raise ValueError("vcf_uri is not a valid gs://bucket/object URI")
+            return v
+        path, root = PurePosixPath(v), PurePosixPath(settings.local_data_root)
+        if not path.is_absolute() or ".." in path.parts or not path.is_relative_to(root):
+            raise ValueError(f"local VCFs must be absolute paths under {root}")
+        return v
 
 
 class SampleOut(ORMModel):
@@ -30,6 +58,7 @@ class JobOut(ORMModel):
     status: JobStatus
     workflow_ref: str | None
     vep_version: str | None
+    log: str | None
     created_at: datetime
     finished_at: datetime | None
 
@@ -56,6 +85,12 @@ class VariantOut(ORMModel):
     prediction: PredictionOut | None = None
 
 
+class ScoreOut(BaseModel):
+    job_id: uuid.UUID
+    scored: int
+    model_version: str
+
+
 class VariantPage(BaseModel):
     items: list[VariantOut]
     total: int

api/app/services/events.py

@@ -1,31 +1,142 @@
-"""Event publishing.
+"""Start the annotation pipeline for a job.
 
-GCP: publish a JSON message to Pub/Sub; an Argo Events sensor turns it into an Argo Workflow.
-Local: run Nextflow directly in a background task so `docker compose up` gives a working demo.
+Serverless (CLOUDRUN_JOB set): execute a Cloud Run job that runs the Nextflow driver, which
+submits the pipeline tasks to Google Batch. Nothing runs, or is billed, between annotations.
+Kubernetes (PUBSUB_TOPIC set): publish to Pub/Sub; an Argo Events sensor submits the annotate-vcf
+WorkflowTemplate, whose exit handler marks the job failed if the workflow does not succeed.
+Local: run Nextflow as a child process and watch it. The docker-compose API image has neither
+Nextflow nor Docker, so there the job fails at once with instructions to run it from the host.
 """
 import asyncio
 import json
+import logging
+import os
+import shutil
 import uuid
+from datetime import UTC, datetime
+from functools import cache
+from typing import Any
 
 from app.config import settings
+from app.db import SessionLocal
+from app.models import Job, JobStatus
+
+logger = logging.getLogger(__name__)
+
+PIPELINE_ENTRYPOINT = "/pipeline/main.nf"
+LOG_TAIL_CHARS = 4000
+PUBLISH_TIMEOUT_S = 30
+_watchers: set[asyncio.Task[None]] = set()
 
 
-async def publish_vcf_uploaded(job_id: uuid.UUID, vcf_uri: str) -> None:
-    message = {"job_id": str(job_id), "vcf_uri": vcf_uri}
+class LaunchError(Exception):
+    """The pipeline could not be started; the message is stored on the job for the user."""
+
+
+async def launch(job_id: uuid.UUID, vcf_uri: str, assembly: str) -> str:
+    """Start the pipeline and return a reference to the run (stored as jobs.workflow_ref)."""
+    if settings.cloudrun_job:
+        return await _run_cloud_run_job(job_id, vcf_uri, assembly)
     if settings.pubsub_topic:
-        from google.cloud import pubsub_v1  # optional dependency, installed in the GCP image
+        message = {"job_id": str(job_id), "vcf_uri": vcf_uri, "assembly": assembly}
+        return await _publish(settings.pubsub_topic, message)
+    return await _run_local(job_id, vcf_uri, assembly)
 
-        publisher = pubsub_v1.PublisherClient()
-        publisher.publish(settings.pubsub_topic, json.dumps(message).encode())
-        return
 
-    cmd = [
-        "nextflow", "run", "/pipeline/main.nf", "-profile", "docker",
-        "--vcf", vcf_uri, "--job_id", str(job_id), "--db_url", settings.database_url,
+async def drain() -> None:
+    """Wait for local pipeline watchers (tests, graceful shutdown)."""
+    await asyncio.gather(*_watchers)
+
+
+@cache
+def _jobs_client() -> Any:
+    from google.cloud import run_v2
+
+    return run_v2.JobsClient()
+
+
+async def _run_cloud_run_job(job_id: uuid.UUID, vcf_uri: str, assembly: str) -> str:
+    if not settings.gcp_project:
+        raise LaunchError("CLOUDRUN_JOB is set but GCP_PROJECT is not")
+    name = (
+        f"projects/{settings.gcp_project}/locations/{settings.gcp_region}"
+        f"/jobs/{settings.cloudrun_job}"
+    )
+    args = [
+        "run", PIPELINE_ENTRYPOINT, "-profile", "gcp",
+        "--vcf", vcf_uri, "--job_id", str(job_id), "--assembly", assembly,
     ]
+    request = {"name": name, "overrides": {"container_overrides": [{"args": args}]}}
     try:
-        await asyncio.create_subprocess_exec(*cmd)
-    except FileNotFoundError:
-        # Nextflow is not installed in this environment (no Java toolchain yet);
-        # leave the job queued instead of crashing the request.
-        print(f"[rarelens] nextflow not found; cannot start pipeline for job {job_id}", flush=True)
+        operation = await asyncio.to_thread(_jobs_client().run_job, request=request)
+    except Exception as e:
+        logger.exception("starting Cloud Run job %s failed", name)
+        raise LaunchError(f"could not start {settings.cloudrun_job}: {e}") from e
+    name_or_job = getattr(getattr(operation, "metadata", None), "name", "")
+    execution = name_or_job or settings.cloudrun_job or "started"
+    return f"cloudrun:{execution.rsplit('/', 1)[-1]}"
+
+
+@cache
+def _publisher() -> Any:
+    from google.cloud import pubsub_v1
+
+    return pubsub_v1.PublisherClient()
+
+
+async def _publish(topic_name: str, message: dict[str, str]) -> str:
+    if not settings.gcp_project:
+        raise LaunchError("PUBSUB_TOPIC is set but GCP_PROJECT is not")
+    publisher = _publisher()
+    topic = publisher.topic_path(settings.gcp_project, topic_name)
+    try:
+        future = publisher.publish(topic, json.dumps(message).encode())
+        message_id = await asyncio.to_thread(future.result, timeout=PUBLISH_TIMEOUT_S)
+    except Exception as e:
+        logger.exception("publishing to %s failed", topic)
+        raise LaunchError(f"could not publish to {topic}: {e}") from e
+    return f"pubsub:{message_id}"
+
+
+async def _run_local(job_id: uuid.UUID, vcf_uri: str, assembly: str) -> str:
+    nextflow = shutil.which("nextflow")
+    if nextflow is None:
+        raise LaunchError(
+            "Nextflow is not installed where the API runs. Start the pipeline from the host with "
+            f"`make annotate JOB={job_id} VCF=<path to the VCF>`."
+        )
+    cmd = [
+        nextflow, "run", str(settings.pipeline_dir / "main.nf"),
+        "-profile", settings.nextflow_profile,
+        "--vcf", vcf_uri, "--job_id", str(job_id), "--assembly", assembly,
+    ]
+    # The loader reads DATABASE_URL from its environment; keep it off the command line.
+    env = {**os.environ, "DATABASE_URL": settings.database_url}
+    try:
+        proc = await asyncio.create_subprocess_exec(
+            *cmd,
+            cwd=settings.pipeline_dir,
+            env=env,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.STDOUT,
+        )
+    except OSError as e:
+        raise LaunchError(f"could not start Nextflow: {e}") from e
+    task = asyncio.create_task(_watch(job_id, proc))
+    _watchers.add(task)
+    task.add_done_callback(_watchers.discard)
+    return f"nextflow:pid-{proc.pid}"
+
+
+async def _watch(job_id: uuid.UUID, proc: Any) -> None:
+    out, err = await proc.communicate()
+    if proc.returncode == 0:
+        return  # the loader marks the job succeeded
+    tail = ((out or b"") + (err or b"")).decode(errors="replace")[-LOG_TAIL_CHARS:]
+    async with SessionLocal() as session:
+        job = await session.get(Job, job_id)
+        if job is not None and job.status != JobStatus.succeeded:
+            job.status = JobStatus.failed
+            job.log = f"nextflow exited with code {proc.returncode}\n{tail}"
+            job.finished_at = datetime.now(UTC)
+            await session.commit()

api/app/services/scoring.py

@@ -1,45 +1,107 @@
+"""Score a job's variants with the registered MLflow model.
+
+The registered model is a pyfunc that owns its feature engineering (rarelens_ml.features travels
+with it as model code) and returns P(pathogenic). Serving therefore only sends the raw columns
+below and cannot drift from training.
+"""
+import asyncio
 import uuid
+from collections.abc import Sequence
+from typing import Any
 
 import mlflow
 import pandas as pd
-from sqlalchemy import select
+from mlflow import MlflowClient
+from sqlalchemy import func, select
+from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.config import settings
 from app.models import Prediction, Variant
 
-_model = None
+# Must match rarelens_ml.features.RAW_COLUMNS.
+RAW_COLUMNS = ["impact", "consequence", "gnomad_af", "cadd_phred", "am_pathogenicity"]
+CHUNK_SIZE = 5000
+
+_models: dict[str, Any] = {}  # model version -> loaded pyfunc
 
 
-def load_model():
-    global _model
-    if _model is None:
+def load_model() -> tuple[Any, str]:
+    """Return (model, version), from MODEL_URI if set, else from the registry alias."""
+    if settings.model_uri:
+        return _load_uri(settings.model_uri)
+    client = MlflowClient(tracking_uri=settings.mlflow_tracking_uri)
+    version = client.get_model_version_by_alias(settings.model_name, settings.model_alias).version
+    if version not in _models:
         mlflow.set_tracking_uri(settings.mlflow_tracking_uri)
-        _model = mlflow.pyfunc.load_model(f"models:/{settings.model_name}/{settings.model_stage}")
-    return _model
+        _models.clear()
+        _models[version] = mlflow.pyfunc.load_model(f"models:/{settings.model_name}/{version}")
+    return _models[version], version
 
 
-def featurise(variants: list[Variant]) -> pd.DataFrame:
-    # Mirror ml/rarelens_ml/features.py exactly; shared package later.
+def _load_uri(uri: str) -> tuple[Any, str]:
+    """Load a model artifact directly (gs://...): no tracking server, nothing running when idle."""
+    version = uri.rstrip("/").rsplit("/", 1)[-1][:40] or "uri"
+    if version not in _models:
+        _models.clear()
+        _models[version] = mlflow.pyfunc.load_model(uri)
+    return _models[version], version
+
+
+def raw_frame(variants: Sequence[Variant]) -> pd.DataFrame:
     return pd.DataFrame(
         {
             "impact": [v.impact for v in variants],
             "consequence": [v.consequence for v in variants],
-            "gnomad_af": [v.gnomad_af if v.gnomad_af is not None else 0.0 for v in variants],
+            "gnomad_af": [v.gnomad_af if v.gnomad_af is not None else float("nan") for v in variants],
             "cadd_phred": [v.annotations.get("CADD_PHRED") for v in variants],
             "am_pathogenicity": [v.annotations.get("am_pathogenicity") for v in variants],
-        }
+        },
+        columns=RAW_COLUMNS,
     )
 
 
-async def score_job(job_id: uuid.UUID, session: AsyncSession) -> int:
-    variants = (await session.scalars(select(Variant).where(Variant.job_id == job_id))).all()
-    if not variants:
-        return 0
-    model = load_model()
-    scores = model.predict(featurise(variants))
-    for v, s in zip(variants, scores):
-        session.add(Prediction(variant_id=v.id, model_name=settings.model_name,
-                               model_version=settings.model_stage, score=float(s)))
+async def score_job(job_id: uuid.UUID, session: AsyncSession) -> tuple[int, str]:
+    model, version = await asyncio.to_thread(load_model)
+    scored = 0
+    last_id = 0
+    # Keyset pagination keeps memory flat for whole-genome jobs.
+    while True:
+        variants = (
+            await session.scalars(
+                select(Variant)
+                .where(Variant.job_id == job_id, Variant.id > last_id)
+                .order_by(Variant.id)
+                .limit(CHUNK_SIZE)
+            )
+        ).all()
+        if not variants:
+            break
+        scores = await asyncio.to_thread(model.predict, raw_frame(variants))
+        stmt = insert(Prediction).values(
+            [
+                {
+                    "variant_id": v.id,
+                    "model_name": settings.model_name,
+                    "model_version": version,
+                    "score": float(s),
+                }
+                for v, s in zip(variants, scores, strict=True)
+            ]
+        )
+        # Re-scoring (e.g. after a new model version) replaces the previous prediction.
+        await session.execute(
+            stmt.on_conflict_do_update(
+                index_elements=[Prediction.variant_id],
+                set_={
+                    "model_name": stmt.excluded.model_name,
+                    "model_version": stmt.excluded.model_version,
+                    "score": stmt.excluded.score,
+                    "created_at": func.now(),
+                },
+            )
+        )
+        scored += len(variants)
+        last_id = variants[-1].id
     await session.commit()
-    return len(variants)
+    return scored, version