fix: overhaul the platform skeleton, add a serverless deployment track

An end-to-end audit found the repo could not build, test or run as shipped. This
fixes every finding, then adds a Cloud Run track so the demo costs about £1/month
idle instead of ~£150.

CI (red on its first run)
- api: setuptools could not build the package (flat layout with app/ and alembic/)
- web: missing @types/node; `vitest run` exited 1 with no test files
- pipeline: the stub run needed a gitignored VCF, and no process had a stub block
- ruff pinned, mypy configured, DB tests on real Postgres (pgserver locally, service in CI)

ML serving (scores were meaningless)
- the registered model now carries its own feature engineering and returns predict_proba,
  so serving sends raw columns and cannot drift from training
- resolve by registry alias (stages are deprecated in MLflow 3) and record the real
  version; re-scoring upserts instead of failing on the unique constraint
- ClinVar labels parsed from VEP's lowercase terms

Pipeline
- exact ref/alt recovered from a CHROM_POS_REF_ALT VCF ID; loading is idempotent
- job status reaches running/failed/succeeded, so the UI stops polling dead jobs
- DATABASE_URL travels in the environment or a Nextflow secret, never on a command line
- VEP cache and plugins staged as inputs; the gcp profile runs tasks on Google Batch

Deployment
- the API serves /api (matching the ingress); the web app reads its API URL at runtime
- migrations run in an init container under a Postgres advisory lock
- terraform: custom VPC shared with Batch, private Cloud SQL, API enablement, Workload
  Identity bindings, Secret Manager, deletion protection
- serverless track, now the default: Cloud Run services scaling to zero, a Cloud Run job
  for the Nextflow driver, and Neon or Cloud SQL behind one DATABASE_URL secret. GKE and
  Argo remain, behind -var deploy_kubernetes=true. See docs/cloud.md.

Correctness and security
- 409 on duplicate sample names, 422 on bad paging, natural chromosome ordering, wider
  VEP text columns, enum dropped on downgrade, the sample's assembly actually used
- vcf_uri restricted to gs:// objects or files under the data root, blocking option injection
- CORS restricted to configured origins; `make down` no longer deletes volumes

Data
- docs/data.md records the peer-reviewed, openly licensed sources (GIAB HG002, ClinVar,
  gnomAD) with citations and an honest evaluation plan; `make data` fetches a chr22 slice

Verified: api 50 tests, ml 18, loader 16, web 12; ruff, mypy, svelte-check, terraform
validate and both kustomize overlays clean.

api/.dockerignore

@@ -0,0 +1,5 @@
+.venv/
+.env
+tests/
+**/__pycache__/
+*.egg-info/

api/Dockerfile

@@ -2,7 +2,8 @@ FROM python:3.12-slim
 WORKDIR /app
 RUN pip install --no-cache-dir uv
 COPY pyproject.toml .
-RUN uv pip install --system -e .
+# Dependencies only, so this layer is cached until pyproject.toml changes; `app` runs from WORKDIR.
+RUN uv pip install --system -r pyproject.toml --extra gcp
 COPY . .
 EXPOSE 8000
 CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]

api/alembic/env.py

@@ -2,8 +2,8 @@ import asyncio
 from logging.config import fileConfig
 
 from alembic import context
-from sqlalchemy.ext.asyncio import async_engine_from_config
 from sqlalchemy import pool
+from sqlalchemy.ext.asyncio import async_engine_from_config
 
 from app.config import settings
 from app.models import Base
@@ -14,10 +14,15 @@ if config.config_file_name:
     fileConfig(config.config_file_name)
 target_metadata = Base.metadata
 
+# Arbitrary constant shared by every `alembic upgrade` (one per API replica's init container).
+MIGRATION_LOCK_ID = 72150001
+
 
 def run_migrations(connection):
     context.configure(connection=connection, target_metadata=target_metadata)
     with context.begin_transaction():
+        # Held until this transaction commits; a concurrent run waits, then finds nothing to do.
+        connection.exec_driver_sql(f"SELECT pg_advisory_xact_lock({MIGRATION_LOCK_ID})")
         context.run_migrations()
 
 
@@ -27,6 +32,22 @@ async def run_async():
     )
     async with engine.connect() as conn:
         await conn.run_sync(run_migrations)
+    await engine.dispose()
 
 
-asyncio.run(run_async())
+def run_offline():
+    """`alembic upgrade head --sql`: emit the SQL instead of running it."""
+    context.configure(
+        url=settings.database_url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_offline()
+else:
+    asyncio.run(run_async())

api/alembic/versions/5f2c8e1b9d04_widen_vep_text_columns.py

@@ -0,0 +1,34 @@
+"""widen VEP text columns
+
+VEP joins multiple consequences with "&" and co-located ClinVar significances with ",", which
+overflowed VARCHAR(120) and aborted the whole load transaction.
+
+Revision ID: 5f2c8e1b9d04
+Revises: a3e9ead256a5
+Create Date: 2026-09-11 18:00:00.000000
+
+"""
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = '5f2c8e1b9d04'
+down_revision: str | None = 'a3e9ead256a5'
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+COLUMNS = ('consequence', 'clinvar_sig')
+
+
+def upgrade() -> None:
+    for column in COLUMNS:
+        op.alter_column('variants', column, type_=sa.Text(), existing_type=sa.String(length=120))
+
+
+def downgrade() -> None:
+    for column in COLUMNS:
+        op.alter_column(
+            'variants', column, type_=sa.String(length=120), existing_type=sa.Text(),
+            postgresql_using=f'left({column}, 120)',
+        )

api/alembic/versions/a3e9ead256a5_initial_schema.py

@@ -5,17 +5,17 @@ Revises:
 Create Date: 2026-09-11 15:36:04.335440
 
 """
-from typing import Sequence, Union
+from collections.abc import Sequence
 
-from alembic import op
 import sqlalchemy as sa
+from alembic import op
 from sqlalchemy.dialects import postgresql
 
 # revision identifiers, used by Alembic.
 revision: str = 'a3e9ead256a5'
-down_revision: Union[str, None] = None
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
+down_revision: str | None = None
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
 
 
 def upgrade() -> None:
@@ -87,4 +87,7 @@ def downgrade() -> None:
     op.drop_table('variants')
     op.drop_table('jobs')
     op.drop_table('samples')
-    # ### end Alembic commands ###
+    # ### end Alembic commands ###
+    # Autogenerate does not drop the type created by sa.Enum; without this, upgrading again
+    # fails with "type jobstatus already exists".
+    sa.Enum(name='jobstatus').drop(op.get_bind(), checkfirst=True)

api/app/config.py

@@ -1,5 +1,10 @@
+from pathlib import Path
+
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
+# api/app/config.py -> repo root locally; "/" in the API image, where compose mounts /pipeline.
+REPO_ROOT = Path(__file__).resolve().parents[2]
+
 
 class Settings(BaseSettings):
     model_config = SettingsConfigDict(env_file=".env", extra="ignore")
@@ -7,9 +12,22 @@ class Settings(BaseSettings):
     database_url: str = "postgresql+asyncpg://rarelens:rarelens@localhost:5432/rarelens"
     mlflow_tracking_uri: str = "http://localhost:5000"
     model_name: str = "rarelens-pathogenicity"
-    model_stage: str = "Production"
+    # Registry alias set by `rarelens_ml.train --register` (stages are deprecated in MLflow 3).
+    model_alias: str = "production"
+    # A model artifact URI (gs://...) scores without an MLflow server running; wins over the registry.
+    model_uri: str | None = None
     gcs_bucket: str | None = None          # set in GCP; local uses ./data
     pubsub_topic: str | None = None        # "vcf-uploaded" in GCP; local runs pipeline inline
+    # Serverless track: run the Nextflow driver as a Cloud Run job instead of Argo + Pub/Sub.
+    cloudrun_job: str | None = None
+    gcp_project: str | None = None         # required with pubsub_topic or cloudrun_job
+    gcp_region: str = "europe-west2"
+    pipeline_dir: Path = REPO_ROOT / "pipeline"
+    nextflow_profile: str = "docker"
+    # Local (non-gs://) VCFs must live under this directory.
+    local_data_root: Path = Path("/data")
+    # Browsers calling the API cross-origin; behind the ingress the UI is same-origin.
+    cors_origins: list[str] = ["http://localhost:5173"]
 
 
 settings = Settings()

api/app/db.py

@@ -1,13 +1,33 @@
 from collections.abc import AsyncIterator
+from typing import Annotated
 
+from fastapi import Depends
+from sqlalchemy.engine import make_url
 from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
 
 from app.config import settings
 
-engine = create_async_engine(settings.database_url, pool_pre_ping=True)
+
+def normalize_async_url(url: str) -> str:
+    """Translate libpq's `sslmode=` into the `ssl=` asyncpg understands.
+
+    Hosted Postgres (Neon, Supabase) hands out sslmode= URLs, which asyncpg rejects outright.
+    """
+    parsed = make_url(url)
+    if "asyncpg" not in parsed.drivername or "sslmode" not in parsed.query:
+        return url
+    query = dict(parsed.query)
+    query["ssl"] = query.pop("sslmode")
+    return parsed.set(query=query).render_as_string(hide_password=False)
+
+
+engine = create_async_engine(normalize_async_url(settings.database_url), pool_pre_ping=True)
 SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
 
 
 async def get_session() -> AsyncIterator[AsyncSession]:
     async with SessionLocal() as session:
         yield session
+
+
+SessionDep = Annotated[AsyncSession, Depends(get_session)]

api/app/main.py

@@ -1,8 +1,9 @@
 from contextlib import asynccontextmanager
 
-from fastapi import FastAPI
+from fastapi import APIRouter, FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 
+from app.config import settings
 from app.routers import jobs, predictions, samples, variants
 
 
@@ -14,13 +15,19 @@ async def lifespan(app: FastAPI):
 
 app = FastAPI(title="rarelens API", version="0.1.0", lifespan=lifespan)
 app.add_middleware(
-    CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"]
+    CORSMiddleware,
+    allow_origins=settings.cors_origins,
+    allow_methods=["GET", "POST"],
+    allow_headers=["content-type"],
 )
 
-app.include_router(samples.router, prefix="/samples", tags=["samples"])
-app.include_router(jobs.router, prefix="/jobs", tags=["jobs"])
-app.include_router(variants.router, prefix="/variants", tags=["variants"])
-app.include_router(predictions.router, prefix="/predictions", tags=["predictions"])
+# The ingress forwards /api/* to this service unchanged, and local dev uses the same prefix.
+api = APIRouter(prefix="/api")
+api.include_router(samples.router, prefix="/samples", tags=["samples"])
+api.include_router(jobs.router, prefix="/jobs", tags=["jobs"])
+api.include_router(variants.router, prefix="/variants", tags=["variants"])
+api.include_router(predictions.router, prefix="/predictions", tags=["predictions"])
+app.include_router(api)
 
 
 @app.get("/health", tags=["ops"])

api/app/models.py

@@ -2,9 +2,9 @@
 
 One sample -> many jobs; one job -> many variants; one variant -> one prediction (latest).
 """
-from datetime import datetime
 import enum
 import uuid
+from datetime import datetime
 
 from sqlalchemy import DateTime, Enum, Float, ForeignKey, Integer, String, Text, func
 from sqlalchemy.dialects.postgresql import JSONB, UUID
@@ -55,12 +55,12 @@ class Variant(Base):
     ref: Mapped[str] = mapped_column(Text)
     alt: Mapped[str] = mapped_column(Text)
     gene: Mapped[str | None] = mapped_column(String(60), index=True)
-    consequence: Mapped[str | None] = mapped_column(String(120))
+    consequence: Mapped[str | None] = mapped_column(Text)   # "&"-joined VEP terms
     impact: Mapped[str | None] = mapped_column(String(20))
     hgvsc: Mapped[str | None] = mapped_column(Text)
     hgvsp: Mapped[str | None] = mapped_column(Text)
     gnomad_af: Mapped[float | None] = mapped_column(Float)
-    clinvar_sig: Mapped[str | None] = mapped_column(String(120))
+    clinvar_sig: Mapped[str | None] = mapped_column(Text)   # ","-joined co-located ClinVar terms
     annotations: Mapped[dict] = mapped_column(JSONB, default=dict)   # full VEP CSQ record
     job: Mapped[Job] = relationship(back_populates="variants")
     prediction: Mapped["Prediction | None"] = relationship(back_populates="variant", uselist=False)

api/app/routers/jobs.py

@@ -1,9 +1,8 @@
 import uuid
 
-from fastapi import APIRouter, Depends, HTTPException
-from sqlalchemy.ext.asyncio import AsyncSession
+from fastapi import APIRouter, HTTPException
 
-from app.db import get_session
+from app.db import SessionDep
 from app.models import Job
 from app.schemas import JobOut
 
@@ -11,7 +10,7 @@ router = APIRouter()
 
 
 @router.get("/{job_id}", response_model=JobOut)
-async def get_job(job_id: uuid.UUID, session: AsyncSession = Depends(get_session)):
+async def get_job(job_id: uuid.UUID, session: SessionDep):
     job = await session.get(Job, job_id)
     if job is None:
         raise HTTPException(404, "job not found")

api/app/routers/predictions.py

@@ -1,16 +1,22 @@
 import uuid
 
-from fastapi import APIRouter, Depends
-from sqlalchemy.ext.asyncio import AsyncSession
+from fastapi import APIRouter, HTTPException
 
-from app.db import get_session
+from app.db import SessionDep
+from app.models import Job, JobStatus
+from app.schemas import ScoreOut
 from app.services.scoring import score_job
 
 router = APIRouter()
 
 
-@router.post("/score/{job_id}")
-async def score(job_id: uuid.UUID, session: AsyncSession = Depends(get_session)) -> dict:
-    """Load the registered MLflow model and score every variant of a job."""
-    n = await score_job(job_id, session)
-    return {"job_id": str(job_id), "scored": n}
+@router.post("/score/{job_id}", response_model=ScoreOut)
+async def score(job_id: uuid.UUID, session: SessionDep) -> ScoreOut:
+    """Score every variant of a finished job with the model behind the registry alias."""
+    job = await session.get(Job, job_id)
+    if job is None:
+        raise HTTPException(404, "job not found")
+    if job.status != JobStatus.succeeded:
+        raise HTTPException(409, f"job is {job.status.value}; only succeeded jobs can be scored")
+    n, version = await score_job(job_id, session)
+    return ScoreOut(job_id=job_id, scored=n, model_version=version)

api/app/routers/samples.py

@@ -1,34 +1,39 @@
 import uuid
+from datetime import UTC, datetime
 
-from fastapi import APIRouter, Depends, HTTPException, status
+from fastapi import APIRouter, HTTPException, status
 from sqlalchemy import select
-from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.exc import IntegrityError
 
-from app.db import get_session
-from app.models import Job, Sample
+from app.db import SessionDep
+from app.models import Job, JobStatus, Sample
 from app.schemas import JobOut, SampleCreate, SampleOut
-from app.services.events import publish_vcf_uploaded
+from app.services import events
 
 router = APIRouter()
 
 
 @router.get("", response_model=list[SampleOut])
-async def list_samples(session: AsyncSession = Depends(get_session)):
+async def list_samples(session: SessionDep):
     result = await session.scalars(select(Sample).order_by(Sample.created_at.desc()))
     return result.all()
 
 
 @router.post("", response_model=SampleOut, status_code=status.HTTP_201_CREATED)
-async def create_sample(payload: SampleCreate, session: AsyncSession = Depends(get_session)):
+async def create_sample(payload: SampleCreate, session: SessionDep):
     sample = Sample(**payload.model_dump())
     session.add(sample)
-    await session.commit()
+    try:
+        await session.commit()
+    except IntegrityError:  # samples.name is unique
+        await session.rollback()
+        raise HTTPException(409, f"a sample named {payload.name!r} already exists") from None
     await session.refresh(sample)
     return sample
 
 
 @router.get("/{sample_id}/jobs", response_model=list[JobOut])
-async def list_jobs(sample_id: uuid.UUID, session: AsyncSession = Depends(get_session)):
+async def list_jobs(sample_id: uuid.UUID, session: SessionDep):
     result = await session.scalars(
         select(Job).where(Job.sample_id == sample_id).order_by(Job.created_at.desc())
     )
@@ -36,14 +41,21 @@ async def list_jobs(sample_id: uuid.UUID, session: AsyncSession = Depends(get_se
 
 
 @router.post("/{sample_id}/annotate", response_model=JobOut, status_code=status.HTTP_202_ACCEPTED)
-async def annotate(sample_id: uuid.UUID, session: AsyncSession = Depends(get_session)):
+async def annotate(sample_id: uuid.UUID, session: SessionDep):
     sample = await session.get(Sample, sample_id)
     if sample is None:
         raise HTTPException(404, "sample not found")
-    job = Job(sample_id=sample.id)
+    # Commit `running` before launching: a local run that dies instantly is marked failed by its
+    # watcher, and a later status write here would overwrite that.
+    job = Job(sample_id=sample.id, status=JobStatus.running)
     session.add(job)
     await session.commit()
+    try:
+        job.workflow_ref = await events.launch(job.id, sample.vcf_uri, sample.assembly)
+    except events.LaunchError as e:
+        job.status = JobStatus.failed
+        job.log = str(e)
+        job.finished_at = datetime.now(UTC)
+    await session.commit()
     await session.refresh(job)
-    # Emits to Pub/Sub in GCP; runs the Nextflow pipeline inline for local dev.
-    await publish_vcf_uploaded(job_id=job.id, vcf_uri=sample.vcf_uri)
     return job

api/app/routers/variants.py

@@ -1,28 +1,37 @@
 import uuid
 
-from fastapi import APIRouter, Depends, Query
-from sqlalchemy import func, select
-from sqlalchemy.ext.asyncio import AsyncSession
+from fastapi import APIRouter, Query
+from sqlalchemy import Integer, case, cast, func, select
 from sqlalchemy.orm import selectinload
 
-from app.db import get_session
+from app.db import SessionDep
 from app.models import Prediction, Variant
-from app.schemas import VariantPage
+from app.schemas import VariantOut, VariantPage
 
 router = APIRouter()
 
+# Karyotype order (1..22, X, Y, MT) instead of text order, where "10" sorts before "2".
+_chrom = func.regexp_replace(Variant.chrom, "^chr", "", "i")
+CHROM_ORDER = case(
+    (_chrom.regexp_match("^[0-9]+$"), cast(_chrom, Integer)),
+    (_chrom == "X", 23),
+    (_chrom == "Y", 24),
+    (_chrom.in_(["M", "MT"]), 25),
+    else_=26,
+)
+
 
 @router.get("", response_model=VariantPage)
 async def list_variants(
     job_id: uuid.UUID,
+    session: SessionDep,
     gene: str | None = None,
     impact: str | None = Query(None, pattern="^(HIGH|MODERATE|LOW|MODIFIER)$"),
     max_af: float | None = Query(None, ge=0, le=1),
     min_score: float | None = Query(None, ge=0, le=1),
-    limit: int = Query(50, le=500),
-    offset: int = 0,
-    session: AsyncSession = Depends(get_session),
-):
+    limit: int = Query(50, ge=1, le=500),
+    offset: int = Query(0, ge=0),
+) -> VariantPage:
     stmt = select(Variant).where(Variant.job_id == job_id)
     if gene:
         stmt = stmt.where(Variant.gene == gene.upper())
@@ -31,13 +40,21 @@ async def list_variants(
     if max_af is not None:
         stmt = stmt.where((Variant.gnomad_af.is_(None)) | (Variant.gnomad_af <= max_af))
     if min_score is not None:
-        stmt = stmt.join(Prediction, Prediction.variant_id == Variant.id).where(Prediction.score >= min_score)
+        stmt = stmt.join(Prediction, Prediction.variant_id == Variant.id).where(
+            Prediction.score >= min_score
+        )
 
     total = await session.scalar(select(func.count()).select_from(stmt.subquery()))
     rows = await session.scalars(
         stmt.options(selectinload(Variant.prediction))
-        .order_by(Variant.chrom, Variant.pos)
+        # id breaks ties between split multiallelics at one position, keeping pages stable.
+        .order_by(CHROM_ORDER, Variant.chrom, Variant.pos, Variant.id)
         .limit(limit)
         .offset(offset)
     )
-    return VariantPage(items=rows.all(), total=total or 0, limit=limit, offset=offset)
+    return VariantPage(
+        items=[VariantOut.model_validate(v) for v in rows],
+        total=total or 0,
+        limit=limit,
+        offset=offset,
+    )

api/app/schemas.py

@@ -1,8 +1,12 @@
-from datetime import datetime
+import re
 import uuid
+from datetime import datetime
+from pathlib import PurePosixPath
+from typing import Literal
 
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, ConfigDict, Field, field_validator
 
+from app.config import settings
 from app.models import JobStatus
 
 
@@ -10,10 +14,34 @@ class ORMModel(BaseModel):
     model_config = ConfigDict(from_attributes=True)
 
 
+Assembly = Literal["GRCh38", "GRCh37"]
+VCF_SUFFIXES = (".vcf", ".vcf.gz", ".vcf.bgz", ".bcf")
+GCS_URI = re.compile(r"gs://[a-z0-9][a-z0-9._-]{1,220}[a-z0-9]/\S+")
+
+
 class SampleCreate(BaseModel):
     name: str = Field(min_length=1, max_length=120)
     vcf_uri: str
-    assembly: str = "GRCh38"
+    # Passed to VEP --assembly; the VEP cache must contain it.
+    assembly: Assembly = "GRCh38"
+
+    @field_validator("vcf_uri")
+    @classmethod
+    def _gcs_object_or_file_under_data_root(cls, v: str) -> str:
+        # The URI becomes a Nextflow argument and a path the pipeline reads: accept a GCS object or
+        # a file under the local data root, never something that parses as an option.
+        if not v or any(ord(c) < 32 for c in v):
+            raise ValueError("vcf_uri must be a non-empty single line")
+        if not v.lower().endswith(VCF_SUFFIXES):
+            raise ValueError(f"vcf_uri must end in one of {', '.join(VCF_SUFFIXES)}")
+        if v.startswith("gs://"):
+            if not GCS_URI.fullmatch(v):
+                raise ValueError("vcf_uri is not a valid gs://bucket/object URI")
+            return v
+        path, root = PurePosixPath(v), PurePosixPath(settings.local_data_root)
+        if not path.is_absolute() or ".." in path.parts or not path.is_relative_to(root):
+            raise ValueError(f"local VCFs must be absolute paths under {root}")
+        return v
 
 
 class SampleOut(ORMModel):
@@ -30,6 +58,7 @@ class JobOut(ORMModel):
     status: JobStatus
     workflow_ref: str | None
     vep_version: str | None
+    log: str | None
     created_at: datetime
     finished_at: datetime | None
 
@@ -56,6 +85,12 @@ class VariantOut(ORMModel):
     prediction: PredictionOut | None = None
 
 
+class ScoreOut(BaseModel):
+    job_id: uuid.UUID
+    scored: int
+    model_version: str
+
+
 class VariantPage(BaseModel):
     items: list[VariantOut]
     total: int

api/app/services/events.py

@@ -1,31 +1,142 @@
-"""Event publishing.
+"""Start the annotation pipeline for a job.
 
-GCP: publish a JSON message to Pub/Sub; an Argo Events sensor turns it into an Argo Workflow.
-Local: run Nextflow directly in a background task so `docker compose up` gives a working demo.
+Serverless (CLOUDRUN_JOB set): execute a Cloud Run job that runs the Nextflow driver, which
+submits the pipeline tasks to Google Batch. Nothing runs, or is billed, between annotations.
+Kubernetes (PUBSUB_TOPIC set): publish to Pub/Sub; an Argo Events sensor submits the annotate-vcf
+WorkflowTemplate, whose exit handler marks the job failed if the workflow does not succeed.
+Local: run Nextflow as a child process and watch it. The docker-compose API image has neither
+Nextflow nor Docker, so there the job fails at once with instructions to run it from the host.
 """
 import asyncio
 import json
+import logging
+import os
+import shutil
 import uuid
+from datetime import UTC, datetime
+from functools import cache
+from typing import Any
 
 from app.config import settings
+from app.db import SessionLocal
+from app.models import Job, JobStatus
+
+logger = logging.getLogger(__name__)
+
+PIPELINE_ENTRYPOINT = "/pipeline/main.nf"
+LOG_TAIL_CHARS = 4000
+PUBLISH_TIMEOUT_S = 30
+_watchers: set[asyncio.Task[None]] = set()
 
 
-async def publish_vcf_uploaded(job_id: uuid.UUID, vcf_uri: str) -> None:
-    message = {"job_id": str(job_id), "vcf_uri": vcf_uri}
+class LaunchError(Exception):
+    """The pipeline could not be started; the message is stored on the job for the user."""
+
+
+async def launch(job_id: uuid.UUID, vcf_uri: str, assembly: str) -> str:
+    """Start the pipeline and return a reference to the run (stored as jobs.workflow_ref)."""
+    if settings.cloudrun_job:
+        return await _run_cloud_run_job(job_id, vcf_uri, assembly)
     if settings.pubsub_topic:
-        from google.cloud import pubsub_v1  # optional dependency, installed in the GCP image
+        message = {"job_id": str(job_id), "vcf_uri": vcf_uri, "assembly": assembly}
+        return await _publish(settings.pubsub_topic, message)
+    return await _run_local(job_id, vcf_uri, assembly)
 
-        publisher = pubsub_v1.PublisherClient()
-        publisher.publish(settings.pubsub_topic, json.dumps(message).encode())
-        return
 
-    cmd = [
-        "nextflow", "run", "/pipeline/main.nf", "-profile", "docker",
-        "--vcf", vcf_uri, "--job_id", str(job_id), "--db_url", settings.database_url,
+async def drain() -> None:
+    """Wait for local pipeline watchers (tests, graceful shutdown)."""
+    await asyncio.gather(*_watchers)
+
+
+@cache
+def _jobs_client() -> Any:
+    from google.cloud import run_v2
+
+    return run_v2.JobsClient()
+
+
+async def _run_cloud_run_job(job_id: uuid.UUID, vcf_uri: str, assembly: str) -> str:
+    if not settings.gcp_project:
+        raise LaunchError("CLOUDRUN_JOB is set but GCP_PROJECT is not")
+    name = (
+        f"projects/{settings.gcp_project}/locations/{settings.gcp_region}"
+        f"/jobs/{settings.cloudrun_job}"
+    )
+    args = [
+        "run", PIPELINE_ENTRYPOINT, "-profile", "gcp",
+        "--vcf", vcf_uri, "--job_id", str(job_id), "--assembly", assembly,
     ]
+    request = {"name": name, "overrides": {"container_overrides": [{"args": args}]}}
     try:
-        await asyncio.create_subprocess_exec(*cmd)
-    except FileNotFoundError:
-        # Nextflow is not installed in this environment (no Java toolchain yet);
-        # leave the job queued instead of crashing the request.
-        print(f"[rarelens] nextflow not found; cannot start pipeline for job {job_id}", flush=True)
+        operation = await asyncio.to_thread(_jobs_client().run_job, request=request)
+    except Exception as e:
+        logger.exception("starting Cloud Run job %s failed", name)
+        raise LaunchError(f"could not start {settings.cloudrun_job}: {e}") from e
+    name_or_job = getattr(getattr(operation, "metadata", None), "name", "")
+    execution = name_or_job or settings.cloudrun_job or "started"
+    return f"cloudrun:{execution.rsplit('/', 1)[-1]}"
+
+
+@cache
+def _publisher() -> Any:
+    from google.cloud import pubsub_v1
+
+    return pubsub_v1.PublisherClient()
+
+
+async def _publish(topic_name: str, message: dict[str, str]) -> str:
+    if not settings.gcp_project:
+        raise LaunchError("PUBSUB_TOPIC is set but GCP_PROJECT is not")
+    publisher = _publisher()
+    topic = publisher.topic_path(settings.gcp_project, topic_name)
+    try:
+        future = publisher.publish(topic, json.dumps(message).encode())
+        message_id = await asyncio.to_thread(future.result, timeout=PUBLISH_TIMEOUT_S)
+    except Exception as e:
+        logger.exception("publishing to %s failed", topic)
+        raise LaunchError(f"could not publish to {topic}: {e}") from e
+    return f"pubsub:{message_id}"
+
+
+async def _run_local(job_id: uuid.UUID, vcf_uri: str, assembly: str) -> str:
+    nextflow = shutil.which("nextflow")
+    if nextflow is None:
+        raise LaunchError(
+            "Nextflow is not installed where the API runs. Start the pipeline from the host with "
+            f"`make annotate JOB={job_id} VCF=<path to the VCF>`."
+        )
+    cmd = [
+        nextflow, "run", str(settings.pipeline_dir / "main.nf"),
+        "-profile", settings.nextflow_profile,
+        "--vcf", vcf_uri, "--job_id", str(job_id), "--assembly", assembly,
+    ]
+    # The loader reads DATABASE_URL from its environment; keep it off the command line.
+    env = {**os.environ, "DATABASE_URL": settings.database_url}
+    try:
+        proc = await asyncio.create_subprocess_exec(
+            *cmd,
+            cwd=settings.pipeline_dir,
+            env=env,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.STDOUT,
+        )
+    except OSError as e:
+        raise LaunchError(f"could not start Nextflow: {e}") from e
+    task = asyncio.create_task(_watch(job_id, proc))
+    _watchers.add(task)
+    task.add_done_callback(_watchers.discard)
+    return f"nextflow:pid-{proc.pid}"
+
+
+async def _watch(job_id: uuid.UUID, proc: Any) -> None:
+    out, err = await proc.communicate()
+    if proc.returncode == 0:
+        return  # the loader marks the job succeeded
+    tail = ((out or b"") + (err or b"")).decode(errors="replace")[-LOG_TAIL_CHARS:]
+    async with SessionLocal() as session:
+        job = await session.get(Job, job_id)
+        if job is not None and job.status != JobStatus.succeeded:
+            job.status = JobStatus.failed
+            job.log = f"nextflow exited with code {proc.returncode}\n{tail}"
+            job.finished_at = datetime.now(UTC)
+            await session.commit()

api/app/services/scoring.py

@@ -1,45 +1,107 @@
+"""Score a job's variants with the registered MLflow model.
+
+The registered model is a pyfunc that owns its feature engineering (rarelens_ml.features travels
+with it as model code) and returns P(pathogenic). Serving therefore only sends the raw columns
+below and cannot drift from training.
+"""
+import asyncio
 import uuid
+from collections.abc import Sequence
+from typing import Any
 
 import mlflow
 import pandas as pd
-from sqlalchemy import select
+from mlflow import MlflowClient
+from sqlalchemy import func, select
+from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.config import settings
 from app.models import Prediction, Variant
 
-_model = None
+# Must match rarelens_ml.features.RAW_COLUMNS.
+RAW_COLUMNS = ["impact", "consequence", "gnomad_af", "cadd_phred", "am_pathogenicity"]
+CHUNK_SIZE = 5000
+
+_models: dict[str, Any] = {}  # model version -> loaded pyfunc
 
 
-def load_model():
-    global _model
-    if _model is None:
+def load_model() -> tuple[Any, str]:
+    """Return (model, version), from MODEL_URI if set, else from the registry alias."""
+    if settings.model_uri:
+        return _load_uri(settings.model_uri)
+    client = MlflowClient(tracking_uri=settings.mlflow_tracking_uri)
+    version = client.get_model_version_by_alias(settings.model_name, settings.model_alias).version
+    if version not in _models:
         mlflow.set_tracking_uri(settings.mlflow_tracking_uri)
-        _model = mlflow.pyfunc.load_model(f"models:/{settings.model_name}/{settings.model_stage}")
-    return _model
+        _models.clear()
+        _models[version] = mlflow.pyfunc.load_model(f"models:/{settings.model_name}/{version}")
+    return _models[version], version
 
 
-def featurise(variants: list[Variant]) -> pd.DataFrame:
-    # Mirror ml/rarelens_ml/features.py exactly; shared package later.
+def _load_uri(uri: str) -> tuple[Any, str]:
+    """Load a model artifact directly (gs://...): no tracking server, nothing running when idle."""
+    version = uri.rstrip("/").rsplit("/", 1)[-1][:40] or "uri"
+    if version not in _models:
+        _models.clear()
+        _models[version] = mlflow.pyfunc.load_model(uri)
+    return _models[version], version
+
+
+def raw_frame(variants: Sequence[Variant]) -> pd.DataFrame:
     return pd.DataFrame(
         {
             "impact": [v.impact for v in variants],
             "consequence": [v.consequence for v in variants],
-            "gnomad_af": [v.gnomad_af if v.gnomad_af is not None else 0.0 for v in variants],
+            "gnomad_af": [v.gnomad_af if v.gnomad_af is not None else float("nan") for v in variants],
             "cadd_phred": [v.annotations.get("CADD_PHRED") for v in variants],
             "am_pathogenicity": [v.annotations.get("am_pathogenicity") for v in variants],
-        }
+        },
+        columns=RAW_COLUMNS,
     )
 
 
-async def score_job(job_id: uuid.UUID, session: AsyncSession) -> int:
-    variants = (await session.scalars(select(Variant).where(Variant.job_id == job_id))).all()
-    if not variants:
-        return 0
-    model = load_model()
-    scores = model.predict(featurise(variants))
-    for v, s in zip(variants, scores):
-        session.add(Prediction(variant_id=v.id, model_name=settings.model_name,
-                               model_version=settings.model_stage, score=float(s)))
+async def score_job(job_id: uuid.UUID, session: AsyncSession) -> tuple[int, str]:
+    model, version = await asyncio.to_thread(load_model)
+    scored = 0
+    last_id = 0
+    # Keyset pagination keeps memory flat for whole-genome jobs.
+    while True:
+        variants = (
+            await session.scalars(
+                select(Variant)
+                .where(Variant.job_id == job_id, Variant.id > last_id)
+                .order_by(Variant.id)
+                .limit(CHUNK_SIZE)
+            )
+        ).all()
+        if not variants:
+            break
+        scores = await asyncio.to_thread(model.predict, raw_frame(variants))
+        stmt = insert(Prediction).values(
+            [
+                {
+                    "variant_id": v.id,
+                    "model_name": settings.model_name,
+                    "model_version": version,
+                    "score": float(s),
+                }
+                for v, s in zip(variants, scores, strict=True)
+            ]
+        )
+        # Re-scoring (e.g. after a new model version) replaces the previous prediction.
+        await session.execute(
+            stmt.on_conflict_do_update(
+                index_elements=[Prediction.variant_id],
+                set_={
+                    "model_name": stmt.excluded.model_name,
+                    "model_version": stmt.excluded.model_version,
+                    "score": stmt.excluded.score,
+                    "created_at": func.now(),
+                },
+            )
+        )
+        scored += len(variants)
+        last_id = variants[-1].id
     await session.commit()
-    return len(variants)
+    return scored, version

api/pyproject.toml

@@ -1,3 +1,7 @@
+[build-system]
+requires = ["setuptools>=69"]
+build-backend = "setuptools.build_meta"
+
 [project]
 name = "rarelens-api"
 version = "0.1.0"
@@ -12,17 +16,50 @@ dependencies = [
   "pydantic>=2.8",
   "pydantic-settings>=2.4",
   "httpx>=0.27",
-  "mlflow-skinny>=2.16",
+  # mlflow major must match rarelens-ml and the tracking server image.
+  "mlflow-skinny>=3,<4",
   "lightgbm>=4.5",
+  "scikit-learn>=1.5",  # the pickled LGBMClassifier inside the pyfunc needs it to load
   "pandas>=2.2",
 ]
 
 [project.optional-dependencies]
-dev = ["pytest", "pytest-asyncio", "ruff", "mypy", "aiosqlite"]
+# pubsub: Kubernetes track; run: serverless track; storage: load a model from gs://
+gcp = ["google-cloud-pubsub>=2.23", "google-cloud-run>=0.10", "google-cloud-storage>=2.18"]
+dev = [
+  "pytest>=8",
+  "pytest-asyncio>=1.0",
+  "ruff==0.16.2",
+  "mypy>=1.11",
+  "pandas-stubs",
+  # Throwaway Postgres for `pytest` when DATABASE_URL is unset (no Docker needed).
+  "pgserver; sys_platform != 'win32'",
+]
+
+# The repo also has top-level `alembic/` and `tests/` dirs; only `app` is the package.
+[tool.setuptools.packages.find]
+include = ["app*"]
 
 [tool.ruff]
 line-length = 100
 target-version = "py312"
 
+[tool.ruff.lint.isort]
+# Without this the local `alembic/` migrations dir makes ruff treat the alembic library as first-party.
+known-third-party = ["alembic"]
+
+[tool.mypy]
+python_version = "3.12"
+
+[[tool.mypy.overrides]]
+module = ["google.cloud.*", "mlflow.*"]
+ignore_missing_imports = true
+
 [tool.pytest.ini_options]
 asyncio_mode = "auto"
+# asyncpg connections are bound to the loop that opened them; one loop for the whole run
+# lets the app's pooled engine be shared across tests.
+asyncio_default_fixture_loop_scope = "session"
+asyncio_default_test_loop_scope = "session"
+pythonpath = ["."]
+testpaths = ["tests"]

api/tests/conftest.py

@@ -0,0 +1,83 @@
+"""Test bootstrap.
+
+CI provides DATABASE_URL (a Postgres service container). Locally, when it is unset, a throwaway
+Postgres is started with `pgserver` so `pytest` needs no Docker. Set REQUIRE_DB=1 to turn a missing
+database into a failure instead of a skip (CI does this).
+
+DATABASE_URL must be settled before anything imports `app`, because `app.config.settings` and the
+engine in `app.db` are created at import time.
+"""
+import os
+import tempfile
+from collections.abc import AsyncIterator, Iterator
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import pytest
+from httpx import ASGITransport, AsyncClient
+
+if TYPE_CHECKING:
+    from alembic.config import Config
+
+API_DIR = Path(__file__).resolve().parents[1]
+_pg_server = None  # keeps the pgserver handle alive for the whole session
+
+
+def _resolve_database_url() -> str | None:
+    global _pg_server
+    if url := os.environ.get("DATABASE_URL"):
+        return url
+    try:
+        import pgserver
+    except ImportError:
+        return None
+    _pg_server = pgserver.get_server(tempfile.mkdtemp(prefix="rarelens-pg-"), cleanup_mode="delete")
+    socket_dir = _pg_server.get_uri().split("host=", 1)[1]
+    return f"postgresql+asyncpg://postgres@/postgres?host={socket_dir}"
+
+
+DATABASE_URL = _resolve_database_url()
+if DATABASE_URL:
+    os.environ["DATABASE_URL"] = DATABASE_URL
+
+
+@pytest.fixture(scope="session")
+def alembic_cfg() -> "Config":
+    if not DATABASE_URL:
+        if os.environ.get("REQUIRE_DB"):
+            pytest.fail("REQUIRE_DB is set but no database is available")
+        pytest.skip("no DATABASE_URL and pgserver is not installed")
+    from alembic.config import Config
+
+    cfg = Config(str(API_DIR / "alembic.ini"))
+    cfg.set_main_option("script_location", str(API_DIR / "alembic"))
+    return cfg
+
+
+@pytest.fixture(scope="session")
+def migrated_db(alembic_cfg: "Config") -> Iterator[None]:
+    from alembic import command
+
+    command.upgrade(alembic_cfg, "head")
+    yield
+
+
+@pytest.fixture
+async def db(migrated_db: None) -> AsyncIterator[None]:
+    from sqlalchemy import text
+
+    from app.db import engine
+
+    async with engine.begin() as conn:
+        await conn.execute(
+            text("TRUNCATE samples, jobs, variants, predictions RESTART IDENTITY CASCADE")
+        )
+    yield
+
+
+@pytest.fixture
+async def client() -> AsyncIterator[AsyncClient]:
+    from app.main import app
+
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
+        yield c

api/tests/factories.py

@@ -0,0 +1,21 @@
+import uuid
+from typing import Any
+
+from app.db import SessionLocal
+from app.models import Job, JobStatus, Sample, Variant
+
+
+async def seed_job(
+    variants: list[dict[str, Any]], status: JobStatus = JobStatus.succeeded
+) -> uuid.UUID:
+    """Insert a sample, a job and its variants; each variant dict overrides the defaults."""
+    async with SessionLocal() as s:
+        sample = Sample(name=f"s-{uuid.uuid4()}", vcf_uri="gs://b/x.vcf.gz", assembly="GRCh38")
+        job = Job(sample=sample, status=status)
+        rows = [
+            Variant(job=job, **{"chrom": "22", "pos": 1, "ref": "A", "alt": "G", "annotations": {}} | v)
+            for v in variants
+        ]
+        s.add_all([sample, job, *rows])
+        await s.commit()
+        return job.id

api/tests/test_annotate.py

@@ -0,0 +1,126 @@
+import uuid
+from typing import Any
+
+import pytest
+from httpx import AsyncClient
+
+from app.config import settings
+from app.services import events
+
+
+async def new_sample(client: AsyncClient) -> str:
+    r = await client.post(
+        "/api/samples",
+        json={"name": f"s-{uuid.uuid4()}", "vcf_uri": "gs://bucket/x.vcf.gz", "assembly": "GRCh37"},
+    )
+    assert r.status_code == 201, r.text
+    return r.json()["id"]
+
+
+@pytest.mark.usefixtures("db")
+async def test_local_without_nextflow_fails_fast_with_instructions(
+    client: AsyncClient, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    monkeypatch.setattr(settings, "pubsub_topic", None)
+    monkeypatch.setattr(events.shutil, "which", lambda _: None)
+    sample_id = await new_sample(client)
+
+    r = await client.post(f"/api/samples/{sample_id}/annotate")
+    assert r.status_code == 202
+    job = r.json()
+    assert job["status"] == "failed"
+    assert "make annotate" in job["log"]
+    assert job["finished_at"] is not None
+
+
+class FakeProcess:
+    def __init__(self, returncode: int, stderr: bytes) -> None:
+        self.returncode = returncode
+        self._stderr = stderr
+        self.pid = 4242
+
+    async def communicate(self) -> tuple[bytes, bytes]:
+        return b"", self._stderr
+
+
+@pytest.mark.usefixtures("db")
+async def test_local_run_is_watched_and_a_crash_marks_the_job_failed(
+    client: AsyncClient, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    monkeypatch.setattr(settings, "pubsub_topic", None)
+    monkeypatch.setattr(events.shutil, "which", lambda _: "/usr/bin/nextflow")
+    launched: dict[str, Any] = {}
+
+    async def fake_exec(*cmd: str, **kw: Any) -> FakeProcess:
+        launched["cmd"], launched["env"] = cmd, kw["env"]
+        return FakeProcess(1, b"ERROR ~ VEP cache not found")
+
+    monkeypatch.setattr(events.asyncio, "create_subprocess_exec", fake_exec)
+    sample_id = await new_sample(client)
+
+    r = await client.post(f"/api/samples/{sample_id}/annotate")
+    assert r.status_code == 202
+    await events.drain()
+
+    job = (await client.get(f"/api/jobs/{r.json()['id']}")).json()
+    assert job["status"] == "failed"
+    assert "VEP cache not found" in job["log"]
+    # The DB password travels in the environment, never on the command line.
+    assert settings.database_url not in " ".join(launched["cmd"])
+    assert launched["env"]["DATABASE_URL"] == settings.database_url
+    assert launched["cmd"][launched["cmd"].index("--assembly") + 1] == "GRCh37"
+
+
+class FakeFuture:
+    def __init__(self, result: str | Exception) -> None:
+        self._result = result
+
+    def result(self, timeout: float | None = None) -> str:
+        if isinstance(self._result, Exception):
+            raise self._result
+        return self._result
+
+
+class FakePublisher:
+    def __init__(self, result: str | Exception) -> None:
+        self.result = result
+        self.published: list[tuple[str, bytes]] = []
+
+    def topic_path(self, project: str, topic: str) -> str:
+        return f"projects/{project}/topics/{topic}"
+
+    def publish(self, topic: str, data: bytes) -> FakeFuture:
+        self.published.append((topic, data))
+        return FakeFuture(self.result)
+
+
+@pytest.mark.usefixtures("db")
+async def test_pubsub_publishes_to_the_full_topic_path(
+    client: AsyncClient, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    publisher = FakePublisher("msg-123")
+    monkeypatch.setattr(settings, "pubsub_topic", "vcf-uploaded")
+    monkeypatch.setattr(settings, "gcp_project", "my-proj")
+    monkeypatch.setattr(events, "_publisher", lambda: publisher)
+    sample_id = await new_sample(client)
+
+    r = await client.post(f"/api/samples/{sample_id}/annotate")
+    job = r.json()
+    assert (job["status"], job["workflow_ref"]) == ("running", "pubsub:msg-123")
+    [(topic, data)] = publisher.published
+    assert topic == "projects/my-proj/topics/vcf-uploaded"
+    assert b'"assembly": "GRCh37"' in data and job["id"].encode() in data
+
+
+@pytest.mark.usefixtures("db")
+async def test_pubsub_failure_marks_the_job_failed(
+    client: AsyncClient, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    monkeypatch.setattr(settings, "pubsub_topic", "vcf-uploaded")
+    monkeypatch.setattr(settings, "gcp_project", "my-proj")
+    monkeypatch.setattr(events, "_publisher", lambda: FakePublisher(RuntimeError("403 denied")))
+    sample_id = await new_sample(client)
+
+    job = (await client.post(f"/api/samples/{sample_id}/annotate")).json()
+    assert job["status"] == "failed"
+    assert "403 denied" in job["log"]

api/tests/test_cloudrun.py

@@ -0,0 +1,97 @@
+"""The serverless track: the API starts a Cloud Run job instead of publishing to Pub/Sub."""
+import uuid
+from typing import Any
+
+import pytest
+from httpx import AsyncClient
+
+from app.config import settings
+from app.services import events
+
+
+class FakeOperation:
+    def __init__(self, execution: str) -> None:
+        self.metadata = type("Meta", (), {"name": execution})()
+
+
+class FakeJobsClient:
+    def __init__(self, result: Any = None) -> None:
+        self.result = result
+        self.requests: list[dict] = []
+
+    def run_job(self, request: dict) -> FakeOperation:
+        self.requests.append(request)
+        if isinstance(self.result, Exception):
+            raise self.result
+        return self.result
+
+
+async def new_sample(client: AsyncClient) -> str:
+    r = await client.post(
+        "/api/samples",
+        json={"name": f"s-{uuid.uuid4()}", "vcf_uri": "gs://bucket/x.vcf.gz", "assembly": "GRCh37"},
+    )
+    assert r.status_code == 201, r.text
+    return r.json()["id"]
+
+
+@pytest.fixture
+def cloudrun(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(settings, "pubsub_topic", None)
+    monkeypatch.setattr(settings, "cloudrun_job", "rarelens-nextflow")
+    monkeypatch.setattr(settings, "gcp_project", "my-proj")
+    monkeypatch.setattr(settings, "gcp_region", "europe-west2")
+
+
+@pytest.mark.usefixtures("db", "cloudrun")
+async def test_annotate_executes_the_job_with_pipeline_arguments(
+    client: AsyncClient, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    jobs = FakeJobsClient(FakeOperation("projects/p/locations/l/executions/rarelens-nextflow-abc12"))
+    monkeypatch.setattr(events, "_jobs_client", lambda: jobs)
+    sample_id = await new_sample(client)
+
+    r = await client.post(f"/api/samples/{sample_id}/annotate")
+    assert r.status_code == 202
+    job = r.json()
+    assert job["status"] == "running"
+    assert job["workflow_ref"] == "cloudrun:rarelens-nextflow-abc12"
+
+    [request] = jobs.requests
+    assert request["name"] == "projects/my-proj/locations/europe-west2/jobs/rarelens-nextflow"
+    [override] = request["overrides"]["container_overrides"]
+    args = override["args"]
+    assert args[:4] == ["run", "/pipeline/main.nf", "-profile", "gcp"]
+    assert args[args.index("--job_id") + 1] == job["id"]
+    assert args[args.index("--assembly") + 1] == "GRCh37"
+    assert args[args.index("--vcf") + 1] == "gs://bucket/x.vcf.gz"
+
+
+@pytest.mark.usefixtures("db", "cloudrun")
+async def test_a_failed_execution_call_marks_the_job_failed(
+    client: AsyncClient, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    monkeypatch.setattr(
+        events, "_jobs_client", lambda: FakeJobsClient(RuntimeError("403 permission denied"))
+    )
+    sample_id = await new_sample(client)
+
+    job = (await client.post(f"/api/samples/{sample_id}/annotate")).json()
+    assert job["status"] == "failed"
+    assert "403 permission denied" in job["log"]
+
+
+@pytest.mark.usefixtures("db", "cloudrun")
+async def test_cloud_run_job_takes_precedence_over_pubsub(
+    client: AsyncClient, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    monkeypatch.setattr(settings, "pubsub_topic", "vcf-uploaded")
+    jobs = FakeJobsClient(FakeOperation("projects/p/locations/l/executions/x-1"))
+    monkeypatch.setattr(events, "_jobs_client", lambda: jobs)
+    monkeypatch.setattr(
+        events, "_publisher", lambda: pytest.fail("Pub/Sub must not be used in the serverless track")
+    )
+    sample_id = await new_sample(client)
+
+    assert (await client.post(f"/api/samples/{sample_id}/annotate")).json()["status"] == "running"
+    assert len(jobs.requests) == 1

api/tests/test_db_url.py

@@ -0,0 +1,34 @@
+"""Neon-style URLs use libpq's sslmode, which asyncpg does not understand."""
+import pytest
+
+from app.db import normalize_async_url
+
+
+@pytest.mark.parametrize(
+    ("raw", "expected"),
+    [
+        (
+            "postgresql+asyncpg://u:[email protected]/neondb?sslmode=require",
+            "postgresql+asyncpg://u:[email protected]/neondb?ssl=require",
+        ),
+        # Already asyncpg-shaped, or nothing to do.
+        (
+            "postgresql+asyncpg://u:p@h/db?ssl=require",
+            "postgresql+asyncpg://u:p@h/db?ssl=require",
+        ),
+        ("postgresql+asyncpg://u:p@h:5432/db", "postgresql+asyncpg://u:p@h:5432/db"),
+        # Only the asyncpg driver needs the rewrite.
+        ("postgresql+psycopg://u:p@h/db?sslmode=require", "postgresql+psycopg://u:p@h/db?sslmode=require"),
+    ],
+)
+def test_normalize_async_url(raw: str, expected: str) -> None:
+    assert normalize_async_url(raw) == expected
+
+
+def test_disable_is_preserved() -> None:
+    assert normalize_async_url("postgresql+asyncpg://u:p@h/db?sslmode=disable").endswith("ssl=disable")
+
+
+def test_unix_socket_url_is_untouched() -> None:
+    raw = "postgresql+asyncpg://postgres@/postgres?host=/tmp/pg"
+    assert normalize_async_url(raw) == raw

api/tests/test_health.py

@@ -1,12 +1,7 @@
-import pytest
-from httpx import ASGITransport, AsyncClient
-
-from app.main import app
+from httpx import AsyncClient
 
 
-@pytest.mark.asyncio
-async def test_health():
-    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
-        r = await c.get("/health")
+async def test_health(client: AsyncClient) -> None:
+    r = await client.get("/health")
     assert r.status_code == 200
     assert r.json() == {"status": "ok"}

api/tests/test_migrations.py

@@ -0,0 +1,15 @@
+import asyncio
+
+from alembic import command
+from alembic.config import Config
+
+from app.db import engine
+
+
+async def test_downgrade_to_base_then_upgrade_again(alembic_cfg: Config) -> None:
+    # alembic's env.py runs its own event loop, so keep it off this one.
+    await asyncio.to_thread(command.downgrade, alembic_cfg, "base")
+    await asyncio.to_thread(command.upgrade, alembic_cfg, "head")
+    # The recreated enum type has a new OID that pooled asyncpg connections have not seen;
+    # reconnect so later tests (and the database left at head) are unaffected.
+    await engine.dispose()

api/tests/test_model_loading.py

@@ -0,0 +1,47 @@
+"""A model artifact URI (gs://...) lets the API score without an MLflow server running."""
+# Imported eagerly: mlflow loads .pyfunc lazily, so patching it by name can hit the proxy.
+import mlflow.pyfunc
+import pytest
+
+from app.config import settings
+from app.services import scoring
+
+
+@pytest.fixture(autouse=True)
+def clear_cache() -> None:
+    scoring._models.clear()
+
+
+def no_registry(**kwargs: object) -> None:
+    pytest.fail("the registry must not be contacted when MODEL_URI is set")
+
+
+def test_model_uri_skips_the_registry(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(settings, "model_uri", "gs://bucket/models/pathogenicity/3")
+    monkeypatch.setattr(scoring, "MlflowClient", no_registry)
+    monkeypatch.setattr(mlflow.pyfunc, "load_model", lambda uri: f"model@{uri}")
+
+    model, version = scoring.load_model()
+    assert model == "model@gs://bucket/models/pathogenicity/3"
+    assert version == "3"
+
+
+def test_model_uri_without_a_version_segment_still_labels_the_prediction(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(settings, "model_uri", "gs://bucket/models/pathogenicity/")
+    monkeypatch.setattr(scoring, "MlflowClient", no_registry)
+    monkeypatch.setattr(mlflow.pyfunc, "load_model", lambda uri: "model")
+
+    assert scoring.load_model()[1] == "pathogenicity"
+
+
+def test_model_uri_is_loaded_once(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(settings, "model_uri", "gs://bucket/models/pathogenicity/3")
+    monkeypatch.setattr(scoring, "MlflowClient", no_registry)
+    calls: list[str] = []
+    monkeypatch.setattr(mlflow.pyfunc, "load_model", lambda uri: calls.append(uri) or "model")
+
+    scoring.load_model()
+    scoring.load_model()
+    assert len(calls) == 1

api/tests/test_routes.py

@@ -0,0 +1,13 @@
+import pytest
+from httpx import AsyncClient
+
+
+@pytest.mark.usefixtures("db")
+async def test_resources_live_under_api_prefix(client: AsyncClient) -> None:
+    # The ingress forwards /api/* unchanged, so the app itself must serve that prefix.
+    assert (await client.get("/api/samples")).status_code == 200
+    assert (await client.get("/samples")).status_code == 404
+
+
+async def test_health_stays_at_root_for_probes(client: AsyncClient) -> None:
+    assert (await client.get("/health")).status_code == 200

api/tests/test_samples.py

@@ -0,0 +1,18 @@
+import pytest
+from httpx import AsyncClient
+
+
+@pytest.mark.usefixtures("db")
+async def test_duplicate_sample_name_is_409_not_500(client: AsyncClient) -> None:
+    body = {"name": "HG002", "vcf_uri": "gs://bucket/hg002.vcf.gz"}
+    assert (await client.post("/api/samples", json=body)).status_code == 201
+    r = await client.post("/api/samples", json=body)
+    assert r.status_code == 409
+    assert "HG002" in r.json()["detail"]
+
+
+async def test_unknown_assembly_is_rejected(client: AsyncClient) -> None:
+    r = await client.post(
+        "/api/samples", json={"name": "a", "vcf_uri": "gs://bucket/a.vcf.gz", "assembly": "hg19"}
+    )
+    assert r.status_code == 422

api/tests/test_scoring.py

@@ -0,0 +1,90 @@
+import math
+import uuid
+
+import numpy as np
+import pandas as pd
+import pytest
+from httpx import AsyncClient
+from sqlalchemy import select
+
+from app.db import SessionLocal
+from app.models import Job, JobStatus, Prediction, Sample, Variant
+from app.services import scoring
+
+
+def variant(**kw: object) -> Variant:
+    fields: dict = {"chrom": "22", "pos": 1, "ref": "A", "alt": "G", "annotations": {}}
+    fields.update(kw)
+    return Variant(**fields)
+
+
+def test_raw_frame_sends_the_model_contract_columns() -> None:
+    frame = scoring.raw_frame([
+        variant(impact="HIGH", consequence="stop_gained", gnomad_af=None,
+                annotations={"CADD_PHRED": "35", "am_pathogenicity": "0.98"}),
+        variant(impact="LOW", consequence="synonymous_variant", gnomad_af=0.2, annotations={}),
+    ])
+    assert list(frame.columns) == scoring.RAW_COLUMNS
+    assert frame["impact"].tolist() == ["HIGH", "LOW"]
+    assert frame["cadd_phred"].iloc[0] == "35"
+    assert pd.isna(frame["cadd_phred"].iloc[1])
+    assert math.isnan(frame["gnomad_af"].iloc[0])
+
+
+class FakeModel:
+    def __init__(self, score: float) -> None:
+        self.score = score
+
+    def predict(self, frame: pd.DataFrame) -> np.ndarray:
+        assert list(frame.columns) == scoring.RAW_COLUMNS
+        return np.full(len(frame), self.score)
+
+
+async def make_job(status: JobStatus, n_variants: int) -> uuid.UUID:
+    async with SessionLocal() as s:
+        sample = Sample(name=f"s-{uuid.uuid4()}", vcf_uri="gs://b/x.vcf.gz", assembly="GRCh38")
+        job = Job(sample=sample, status=status)
+        s.add_all([sample, job, *(variant(job=job, pos=i + 1) for i in range(n_variants))])
+        await s.commit()
+        return job.id
+
+
+async def predictions(job_id: uuid.UUID) -> list[Prediction]:
+    async with SessionLocal() as s:
+        rows = await s.scalars(
+            select(Prediction).join(Variant).where(Variant.job_id == job_id)
+        )
+        return list(rows)
+
+
+@pytest.mark.usefixtures("db")
+async def test_scoring_twice_updates_instead_of_failing(
+    client: AsyncClient, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    job_id = await make_job(JobStatus.succeeded, n_variants=3)
+
+    monkeypatch.setattr(scoring, "load_model", lambda: (FakeModel(0.9), "7"))
+    r = await client.post(f"/api/predictions/score/{job_id}")
+    assert r.status_code == 200, r.text
+    assert r.json() == {"job_id": str(job_id), "scored": 3, "model_version": "7"}
+
+    monkeypatch.setattr(scoring, "load_model", lambda: (FakeModel(0.2), "8"))
+    r = await client.post(f"/api/predictions/score/{job_id}")
+    assert r.status_code == 200, r.text
+
+    preds = await predictions(job_id)
+    assert len(preds) == 3
+    assert {(p.score, p.model_version) for p in preds} == {(0.2, "8")}
+
+
+@pytest.mark.usefixtures("db")
+async def test_scoring_unknown_job_is_404(client: AsyncClient) -> None:
+    r = await client.post(f"/api/predictions/score/{uuid.uuid4()}")
+    assert r.status_code == 404
+
+
+@pytest.mark.usefixtures("db")
+async def test_scoring_unfinished_job_is_409(client: AsyncClient) -> None:
+    job_id = await make_job(JobStatus.running, n_variants=1)
+    r = await client.post(f"/api/predictions/score/{job_id}")
+    assert r.status_code == 409

api/tests/test_security.py

@@ -0,0 +1,51 @@
+import pytest
+from httpx import AsyncClient
+from pydantic import ValidationError
+
+from app.schemas import SampleCreate
+
+
+@pytest.mark.parametrize(
+    "uri",
+    [
+        "gs://bucket/dir/sample.vcf.gz",
+        "gs://my.bucket-1/a/b.bcf",
+        "gs://bucket/x.vcf",
+        "/data/example.vcf.gz",
+        "/data/giab/hg002.chr22.vcf.bgz",
+    ],
+)
+def test_vcf_uri_accepts_gcs_objects_and_files_under_the_data_root(uri: str) -> None:
+    assert SampleCreate(name="s", vcf_uri=uri).vcf_uri == uri
+
+
+@pytest.mark.parametrize(
+    "uri",
+    [
+        "-c/tmp/evil.config",          # would be parsed as a Nextflow option
+        "--outdir=/etc",
+        "/etc/passwd",                  # outside the data root
+        "/data/../etc/shadow.vcf",      # traversal out of it
+        "data/example.vcf.gz",          # relative: depends on the API's working directory
+        "gs://bucket/notes.txt",        # not a VCF/BCF
+        "https://example.com/x.vcf.gz",
+        "gs:///x.vcf.gz",
+        "/data/x.vcf.gz\n--foo",        # control characters
+        "",
+    ],
+)
+def test_vcf_uri_rejects_everything_else(uri: str) -> None:
+    with pytest.raises(ValidationError):
+        SampleCreate(name="s", vcf_uri=uri)
+
+
+async def test_bad_vcf_uri_is_422_at_the_api(client: AsyncClient) -> None:
+    r = await client.post("/api/samples", json={"name": "s", "vcf_uri": "/etc/passwd"})
+    assert r.status_code == 422
+
+
+async def test_cors_allows_only_configured_origins(client: AsyncClient) -> None:
+    allowed = await client.get("/health", headers={"Origin": "http://localhost:5173"})
+    assert allowed.headers.get("access-control-allow-origin") == "http://localhost:5173"
+    other = await client.get("/health", headers={"Origin": "https://evil.example"})
+    assert "access-control-allow-origin" not in other.headers

api/tests/test_variants.py

@@ -0,0 +1,46 @@
+import uuid
+
+import pytest
+from factories import seed_job
+from httpx import AsyncClient
+
+
+@pytest.mark.parametrize(
+    "paging", [{"limit": 0}, {"limit": -1}, {"limit": 501}, {"offset": -1}]
+)
+async def test_out_of_range_paging_is_422_not_500(client: AsyncClient, paging: dict) -> None:
+    r = await client.get("/api/variants", params={"job_id": str(uuid.uuid4()), **paging})
+    assert r.status_code == 422
+
+
+async def positions(client: AsyncClient, job_id: uuid.UUID) -> list[tuple[str, int, str]]:
+    r = await client.get("/api/variants", params={"job_id": str(job_id)})
+    assert r.status_code == 200, r.text
+    return [(v["chrom"], v["pos"], v["alt"]) for v in r.json()["items"]]
+
+
+@pytest.mark.usefixtures("db")
+async def test_chromosomes_sort_naturally(client: AsyncClient) -> None:
+    job_id = await seed_job([{"chrom": c} for c in ["10", "MT", "2", "X", "chr3", "1", "Y"]])
+    assert [c for c, _, _ in await positions(client, job_id)] == [
+        "1", "2", "chr3", "10", "X", "Y", "MT",
+    ]
+
+
+@pytest.mark.usefixtures("db")
+async def test_same_position_keeps_insertion_order_across_pages(client: AsyncClient) -> None:
+    # Split multiallelics share chrom/pos; without a tiebreak, pages can repeat or skip rows.
+    job_id = await seed_job([{"pos": 5, "alt": a} for a in "CGT"])
+    assert [a for _, _, a in await positions(client, job_id)] == ["C", "G", "T"]
+
+
+@pytest.mark.usefixtures("db")
+async def test_long_vep_strings_are_stored(client: AsyncClient) -> None:
+    clin_sig = ",".join(["conflicting_classifications_of_pathogenicity"] * 8)
+    consequence = (
+        "splice_region_variant&splice_polypyrimidine_tract_variant&intron_variant"
+        "&NMD_transcript_variant&non_coding_transcript_variant"
+    )
+    job_id = await seed_job([{"clinvar_sig": clin_sig, "consequence": consequence}])
+    [v] = (await client.get("/api/variants", params={"job_id": str(job_id)})).json()["items"]
+    assert (v["clinvar_sig"], v["consequence"]) == (clin_sig, consequence)