fix: overhaul the platform skeleton, add a serverless deployment track

An end-to-end audit found the repo could not build, test or run as shipped. This
fixes every finding, then adds a Cloud Run track so the demo costs about £1/month
idle instead of ~£150.

CI (red on its first run)
- api: setuptools could not build the package (flat layout with app/ and alembic/)
- web: missing @types/node; `vitest run` exited 1 with no test files
- pipeline: the stub run needed a gitignored VCF, and no process had a stub block
- ruff pinned, mypy configured, DB tests on real Postgres (pgserver locally, service in CI)

ML serving (scores were meaningless)
- the registered model now carries its own feature engineering and returns predict_proba,
  so serving sends raw columns and cannot drift from training
- resolve by registry alias (stages are deprecated in MLflow 3) and record the real
  version; re-scoring upserts instead of failing on the unique constraint
- ClinVar labels parsed from VEP's lowercase terms

Pipeline
- exact ref/alt recovered from a CHROM_POS_REF_ALT VCF ID; loading is idempotent
- job status reaches running/failed/succeeded, so the UI stops polling dead jobs
- DATABASE_URL travels in the environment or a Nextflow secret, never on a command line
- VEP cache and plugins staged as inputs; the gcp profile runs tasks on Google Batch

Deployment
- the API serves /api (matching the ingress); the web app reads its API URL at runtime
- migrations run in an init container under a Postgres advisory lock
- terraform: custom VPC shared with Batch, private Cloud SQL, API enablement, Workload
  Identity bindings, Secret Manager, deletion protection
- serverless track, now the default: Cloud Run services scaling to zero, a Cloud Run job
  for the Nextflow driver, and Neon or Cloud SQL behind one DATABASE_URL secret. GKE and
  Argo remain, behind -var deploy_kubernetes=true. See docs/cloud.md.

Correctness and security
- 409 on duplicate sample names, 422 on bad paging, natural chromosome ordering, wider
  VEP text columns, enum dropped on downgrade, the sample's assembly actually used
- vcf_uri restricted to gs:// objects or files under the data root, blocking option injection
- CORS restricted to configured origins; `make down` no longer deletes volumes

Data
- docs/data.md records the peer-reviewed, openly licensed sources (GIAB HG002, ClinVar,
  gnomAD) with citations and an honest evaluation plan; `make data` fetches a chr22 slice

Verified: api 50 tests, ml 18, loader 16, web 12; ruff, mypy, svelte-check, terraform
validate and both kustomize overlays clean.

.github/workflows/ci.yml

@@ -19,7 +19,17 @@ jobs:
       - run: uv pip install --system -e "api[dev]"
       - run: cd api && ruff check . && mypy app
       - run: cd api && pytest -q
-        env: { DATABASE_URL: postgresql+asyncpg://rarelens:rarelens@localhost:5432/rarelens }
+        env:
+          DATABASE_URL: postgresql+asyncpg://rarelens:rarelens@localhost:5432/rarelens
+          REQUIRE_DB: "1"
+
+  ml:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v3
+      - run: uv pip install --system -e "ml[dev]"
+      - run: cd ml && pytest -q
 
   web:
     runs-on: ubuntu-latest
@@ -31,10 +41,23 @@ jobs:
 
   pipeline:
     runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:16-alpine
+        env: { POSTGRES_USER: rarelens, POSTGRES_PASSWORD: rarelens, POSTGRES_DB: rarelens }
+        ports: ["5432:5432"]
+        options: --health-cmd "pg_isready -U rarelens" --health-interval 5s --health-retries 10
     steps:
       - uses: actions/checkout@v4
+      - uses: astral-sh/setup-uv@v3
+      - run: uv pip install --system -r pipeline/requirements.txt pytest
+      - run: cd pipeline && pytest -q tests
+        env:
+          DATABASE_URL: postgresql://rarelens:rarelens@localhost:5432/rarelens
+          REQUIRE_DB: "1"
       - uses: nf-core/setup-nextflow@v2
-      - run: cd pipeline && nextflow run main.nf -profile docker --vcf ../data/example.vcf.gz -stub-run
+      # Stub run checks wiring and channel shapes only; no containers or VEP cache needed.
+      - run: cd pipeline && nextflow run main.nf -stub-run --vcf tests/data/tiny.vcf
 
   terraform:
     runs-on: ubuntu-latest
@@ -45,11 +68,19 @@ jobs:
 
   images:
     if: github.ref == 'refs/heads/main'
-    needs: [api, web, pipeline]
+    needs: [api, ml, web, pipeline]
     runs-on: ubuntu-latest
     permissions: { contents: read, id-token: write }
     strategy:
-      matrix: { component: [api, web, pipeline, ml] }
+      matrix:
+        include:
+          - { component: api, context: api, dockerfile: api/Dockerfile }
+          - { component: web, context: web, dockerfile: web/Dockerfile }
+          - { component: ml, context: ml, dockerfile: ml/Dockerfile }
+          - { component: pipeline, context: pipeline, dockerfile: pipeline/Dockerfile }
+          - { component: loader, context: pipeline, dockerfile: pipeline/loader.Dockerfile }
+    env:
+      REGISTRY: europe-west2-docker.pkg.dev/${{ secrets.GCP_PROJECT }}/rarelens
     steps:
       - uses: actions/checkout@v4
       - uses: google-github-actions/auth@v2
@@ -59,6 +90,9 @@ jobs:
       - run: gcloud auth configure-docker europe-west2-docker.pkg.dev --quiet
       - uses: docker/build-push-action@v6
         with:
-          context: ${{ matrix.component }}
+          context: ${{ matrix.context }}
+          file: ${{ matrix.dockerfile }}
           push: true
-          tags: europe-west2-docker.pkg.dev/${{ secrets.GCP_PROJECT }}/rarelens/${{ matrix.component }}:${{ github.sha }}
+          # Only the pipeline driver uses it: pins the loader image from the same commit.
+          build-args: LOADER_IMAGE=${{ env.REGISTRY }}/loader:${{ github.sha }}
+          tags: ${{ env.REGISTRY }}/${{ matrix.component }}:${{ github.sha }}

.github/workflows/deploy.yml

@@ -10,12 +10,19 @@ jobs:
   bump:
     if: ${{ github.event.workflow_run.conclusion == 'success' }}
     runs-on: ubuntu-latest
+    permissions: { contents: write }
     steps:
       - uses: actions/checkout@v4
+      - name: Refuse to deploy an unconfigured overlay
+        run: |
+          if grep -rl __GCP_PROJECT__ infra/k8s/overlays/gcp infra/argo-workflows; then
+            echo "::error::run 'make gcp-configure PROJECT=<id>' and commit before deploying"
+            exit 1
+          fi
       - uses: imranismail/setup-kustomize@v2
       - run: |
           cd infra/k8s/overlays/gcp
-          for c in api web; do
+          for c in api web pipeline loader; do
             kustomize edit set image rarelens/$c=europe-west2-docker.pkg.dev/${{ secrets.GCP_PROJECT }}/rarelens/$c:${{ github.event.workflow_run.head_sha }}
           done
       - uses: stefanzweifel/git-auto-commit-action@v5

.gitignore

@@ -1,5 +1,7 @@
 __pycache__/
 *.pyc
+*.egg-info/
+build/
 .venv/
 .env
 !web/.env

Makefile

@@ -1,25 +1,77 @@
-.PHONY: up down migrate test pipeline kind lint
+.PHONY: up down clean migrate test lint data loader pipeline annotate images kind serverless-deploy serverless-destroy gcp-configure gcp-secrets
+
+VCF ?= data/example.vcf.gz
+TAG ?= latest
+# The loader container reaches docker-compose's Postgres through the host.
+HOST_DB_URL ?= postgresql://rarelens:[email protected]:5432/rarelens
 
 up:
 	docker compose up -d --build
 
 down:
+	docker compose down
+
+clean:  ## also deletes the Postgres and MLflow volumes
 	docker compose down -v
 
 migrate:
 	docker compose exec api alembic upgrade head
 
 test:
-	cd api && uv run pytest -q
+	cd api && uv run --extra dev pytest -q
+	cd ml && uv run --extra dev pytest -q
+	cd pipeline && uv run --no-project --with-requirements requirements.txt --with pytest --with pgserver pytest -q tests
 	cd web && npm test
 
 lint:
-	cd api && uv run ruff check . && uv run mypy app
+	cd api && uv run --extra dev ruff check . && uv run --extra dev mypy app
 	cd web && npm run check
 
-pipeline:
-	cd pipeline && nextflow run main.nf -profile docker --vcf ../data/example.vcf.gz --outdir results
+data:  ## download the public demo slice: GIAB HG002 + ClinVar, chr22 (see docs/data.md)
+	scripts/fetch-demo-data.sh
 
-kind:
-	kind create cluster --name rarelens || true
+loader:
+	docker build -t rarelens/loader:dev -f pipeline/loader.Dockerfile pipeline
+
+pipeline: loader  ## dry run: annotate $(VCF) without touching the database
+	cd pipeline && nextflow run main.nf -profile docker --vcf ../$(VCF)
+
+annotate: loader  ## make annotate JOB=<job id> [VCF=data/x.vcf.gz]
+	@test -n "$(JOB)" || (echo "usage: make annotate JOB=<job id> [VCF=...]"; exit 1)
+	cd pipeline && DATABASE_URL=$(HOST_DB_URL) nextflow run main.nf -profile docker \
+		--vcf ../$(VCF) --job_id $(JOB)
+
+images:
+	docker build -t rarelens-api:dev api
+	docker build -t rarelens-web:dev web
+
+kind: images
+	kind create cluster --name rarelens 2>/dev/null || true
+	kind load docker-image rarelens-api:dev rarelens-web:dev --name rarelens
 	kubectl apply -k infra/k8s/overlays/local
+	kubectl -n rarelens rollout status deploy/postgres deploy/api deploy/web
+	@echo "kubectl -n rarelens port-forward svc/web 8080:80  (UI)"
+	@echo "kubectl -n rarelens port-forward svc/api 8000:80  (API, used by the UI)"
+
+serverless-deploy:  ## deploy the Cloud Run track: make serverless-deploy PROJECT=<id> [TAG=<sha>]
+	@test -n "$(PROJECT)" || (echo "usage: make serverless-deploy PROJECT=<gcp project id> [TAG=<image tag>]"; exit 1)
+	@test -n "$$TF_VAR_database_url" || echo "note: TF_VAR_database_url is unset; add -var deploy_cloud_sql=true or export a Postgres URL"
+	cd infra/terraform && terraform apply -var project=$(PROJECT) -var image_tag=$(TAG)
+
+serverless-destroy:  ## tear it all down
+	cd infra/terraform && terraform destroy -var project=$(PROJECT) -var deletion_protection=false
+
+gcp-configure:  ## one-time: write your GCP project id into the gcp overlay and Argo manifests
+	@test -n "$(PROJECT)" || (echo "usage: make gcp-configure PROJECT=<gcp project id>"; exit 1)
+	grep -rl __GCP_PROJECT__ infra/k8s/overlays/gcp infra/argo-workflows \
+		| xargs sed -i.bak "s/__GCP_PROJECT__/$(PROJECT)/g"
+	find infra -name '*.bak' -delete
+
+gcp-secrets:  ## after terraform apply: copy DB URLs from Secret Manager into k8s secrets
+	@test -n "$(PROJECT)" || (echo "usage: make gcp-secrets PROJECT=<gcp project id>"; exit 1)
+	kubectl -n rarelens create secret generic api-secrets --dry-run=client -o yaml \
+		--from-literal=DATABASE_URL="$$(gcloud secrets versions access latest --project $(PROJECT) --secret rarelens-api-database-url)" \
+		| kubectl apply -f -
+	kubectl -n rarelens create secret generic pipeline-secrets --dry-run=client -o yaml \
+		--from-literal=DATABASE_URL="$$(gcloud secrets versions access latest --project $(PROJECT) --secret DATABASE_URL)" \
+		| kubectl apply -f -

README.md

@@ -13,31 +13,93 @@ clinical tool and makes no diagnostic claims.
 
 | Layer      | Technology                                             | Directory            |
 |------------|--------------------------------------------------------|----------------------|
-| Pipeline   | Nextflow DSL2, bcftools, Ensembl VEP, Docker           | `pipeline/`          |
+| Pipeline   | Nextflow DSL2, bcftools, Ensembl VEP, Docker, Google Batch | `pipeline/`      |
 | API        | FastAPI, Pydantic v2, SQLAlchemy 2.0 (async), Alembic  | `api/`               |
 | Database   | PostgreSQL 16                                          | `docker-compose.yml` |
 | Frontend   | SvelteKit, TypeScript                                  | `web/`               |
-| ML         | LightGBM pathogenicity scorer, MLflow tracking         | `ml/`                |
-| Orchestration | Argo Workflows (pipeline), Pub/Sub (events)         | `infra/argo-workflows/` |
+| ML         | LightGBM pathogenicity scorer, MLflow registry         | `ml/`                |
+| Orchestration | Argo Workflows + Argo Events (pipeline), Pub/Sub (events) | `infra/argo-workflows/` |
 | Platform   | Kubernetes (Kustomize), ArgoCD (GitOps)                | `infra/k8s/`, `infra/argocd/` |
-| Cloud      | GCP: GKE Autopilot, Cloud SQL, GCS, Artifact Registry  | `infra/terraform/`   |
+| Cloud      | GCP: GKE Autopilot, Cloud SQL, GCS, Batch, Secret Manager, Artifact Registry | `infra/terraform/` |
 | CI/CD      | GitHub Actions, Workload Identity Federation           | `.github/workflows/` |
 
 ## Quick start (local)
 
 ```bash
-make up          # postgres + api + web via docker-compose
+make up          # postgres + api + web + mlflow via docker-compose
 make migrate     # alembic upgrade head
-make pipeline    # nextflow run pipeline/main.nf -profile docker --vcf data/example.vcf.gz
-make kind        # spin up a local kind cluster and apply infra/k8s/overlays/local
+make data        # real public data: GIAB HG002 + ClinVar, chr22 (needs bcftools)
+make test        # api, ml, loader and web tests (no Docker needed for the DB tests)
 ```
 
 Then open http://localhost:5173.
 
+The docker-compose API has no Nextflow, so "Run VEP annotation" marks the job failed with the
+command to run instead. With Nextflow and Docker on the host, a VEP cache in `pipeline/cache/vep`
+and a VCF under `data/` (see [data/README.md](data/README.md)):
+
+```bash
+make annotate JOB=<job id from the UI> VCF=data/example.vcf.gz
+make pipeline VCF=data/example.vcf.gz   # dry run: annotate without touching the database
+```
+
+To train and register a model (the API scores with `models:/rarelens-pathogenicity@production`):
+
+```bash
+cd ml && MLFLOW_TRACKING_URI=http://localhost:5000 \
+  uv run python -m rarelens_ml.train --tsv ../pipeline/results/<sample>.vep.tsv --register
+```
+
+Local Kubernetes: `make kind` builds the images, loads them into a kind cluster and applies
+`infra/k8s/overlays/local`.
+
+## Deploying to GCP
+
+Two tracks, same code. The serverless one is the default because it costs about £1/month idle;
+[docs/cloud.md](docs/cloud.md) has the numbers.
+
+**Serverless (Cloud Run + Google Batch).** The API and the UI scale to zero, and the Nextflow
+driver runs as a Cloud Run job only while a pipeline is running.
+
+```bash
+cd infra/terraform
+terraform init -backend-config="bucket=<tfstate bucket>"
+export TF_VAR_database_url='postgresql+asyncpg://user:pass@host/db?sslmode=require'  # e.g. Neon's free tier
+terraform apply -var project=<project id>          # add -var deploy_cloud_sql=true to use Cloud SQL instead
+cd ../.. && make serverless-deploy PROJECT=<project id> TAG=<commit sha>   # redeploy a new build
+```
+
+`terraform output web_url` is the URL to share; it serves the UI and proxies `/api` to the API, so
+there is one public address and no CORS. Upload the VEP cache to
+`gs://<project>-rarelens-data/refs/vep` before running a real annotation, and set
+`-var model_uri=gs://<project>-rarelens-data/models/pathogenicity/1` to score without running an
+MLflow server. Set a billing budget first — the demo has no authentication.
+
+**Kubernetes (GKE + Argo + ArgoCD).** Off by default; turn it on to demonstrate the GitOps path,
+then destroy it.
+
+```bash
+terraform apply -var project=<project id> -var deploy_kubernetes=true -var deploy_cloud_sql=true
+make gcp-configure PROJECT=<project id>   # once; commit the result
+make gcp-secrets PROJECT=<project id>
+```
+
+Then install Argo Workflows, Argo Events and ArgoCD, and `kubectl apply -f infra/argocd/app.yaml`.
+Every green CI run on `main` bumps image tags in the gcp overlay and ArgoCD deploys them.
+`make serverless-destroy PROJECT=<project id>` tears everything down.
+
+## Data
+
+The demo runs on published, openly licensed human data: the NIST Genome in a Bottle HG002
+benchmark genome as the sample, ClinVar for labels, gnomAD for allele frequencies. Sources,
+licences, citations and how the model should be evaluated honestly are in
+[docs/data.md](docs/data.md).
+
 ## Architecture
 
 See [docs/architecture.md](docs/architecture.md) for the diagram and the reasoning behind
-each choice.
+each choice, and [docs/cloud.md](docs/cloud.md) for why this deploys to Google Cloud rather
+than AWS.
 
 ## Status
 

api/.dockerignore

@@ -0,0 +1,5 @@
+.venv/
+.env
+tests/
+**/__pycache__/
+*.egg-info/

api/Dockerfile

@@ -2,7 +2,8 @@ FROM python:3.12-slim
 WORKDIR /app
 RUN pip install --no-cache-dir uv
 COPY pyproject.toml .
-RUN uv pip install --system -e .
+# Dependencies only, so this layer is cached until pyproject.toml changes; `app` runs from WORKDIR.
+RUN uv pip install --system -r pyproject.toml --extra gcp
 COPY . .
 EXPOSE 8000
 CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]

api/alembic/env.py

@@ -2,8 +2,8 @@ import asyncio
 from logging.config import fileConfig
 
 from alembic import context
-from sqlalchemy.ext.asyncio import async_engine_from_config
 from sqlalchemy import pool
+from sqlalchemy.ext.asyncio import async_engine_from_config
 
 from app.config import settings
 from app.models import Base
@@ -14,10 +14,15 @@ if config.config_file_name:
     fileConfig(config.config_file_name)
 target_metadata = Base.metadata
 
+# Arbitrary constant shared by every `alembic upgrade` (one per API replica's init container).
+MIGRATION_LOCK_ID = 72150001
+
 
 def run_migrations(connection):
     context.configure(connection=connection, target_metadata=target_metadata)
     with context.begin_transaction():
+        # Held until this transaction commits; a concurrent run waits, then finds nothing to do.
+        connection.exec_driver_sql(f"SELECT pg_advisory_xact_lock({MIGRATION_LOCK_ID})")
         context.run_migrations()
 
 
@@ -27,6 +32,22 @@ async def run_async():
     )
     async with engine.connect() as conn:
         await conn.run_sync(run_migrations)
+    await engine.dispose()
 
 
+def run_offline():
+    """`alembic upgrade head --sql`: emit the SQL instead of running it."""
+    context.configure(
+        url=settings.database_url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_offline()
+else:
     asyncio.run(run_async())

api/alembic/versions/5f2c8e1b9d04_widen_vep_text_columns.py

@@ -0,0 +1,34 @@
+"""widen VEP text columns
+
+VEP joins multiple consequences with "&" and co-located ClinVar significances with ",", which
+overflowed VARCHAR(120) and aborted the whole load transaction.
+
+Revision ID: 5f2c8e1b9d04
+Revises: a3e9ead256a5
+Create Date: 2026-09-11 18:00:00.000000
+
+"""
+from collections.abc import Sequence
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = '5f2c8e1b9d04'
+down_revision: str | None = 'a3e9ead256a5'
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+COLUMNS = ('consequence', 'clinvar_sig')
+
+
+def upgrade() -> None:
+    for column in COLUMNS:
+        op.alter_column('variants', column, type_=sa.Text(), existing_type=sa.String(length=120))
+
+
+def downgrade() -> None:
+    for column in COLUMNS:
+        op.alter_column(
+            'variants', column, type_=sa.String(length=120), existing_type=sa.Text(),
+            postgresql_using=f'left({column}, 120)',
+        )

api/alembic/versions/a3e9ead256a5_initial_schema.py

@@ -5,17 +5,17 @@ Revises:
 Create Date: 2026-09-11 15:36:04.335440
 
 """
-from typing import Sequence, Union
+from collections.abc import Sequence
 
-from alembic import op
 import sqlalchemy as sa
+from alembic import op
 from sqlalchemy.dialects import postgresql
 
 # revision identifiers, used by Alembic.
 revision: str = 'a3e9ead256a5'
-down_revision: Union[str, None] = None
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
+down_revision: str | None = None
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
 
 
 def upgrade() -> None:
@@ -88,3 +88,6 @@ def downgrade() -> None:
     op.drop_table('jobs')
     op.drop_table('samples')
     # ### end Alembic commands ###
+    # Autogenerate does not drop the type created by sa.Enum; without this, upgrading again
+    # fails with "type jobstatus already exists".
+    sa.Enum(name='jobstatus').drop(op.get_bind(), checkfirst=True)

api/app/config.py

@@ -1,5 +1,10 @@
+from pathlib import Path
+
 from pydantic_settings import BaseSettings, SettingsConfigDict
 
+# api/app/config.py -> repo root locally; "/" in the API image, where compose mounts /pipeline.
+REPO_ROOT = Path(__file__).resolve().parents[2]
+
 
 class Settings(BaseSettings):
     model_config = SettingsConfigDict(env_file=".env", extra="ignore")
@@ -7,9 +12,22 @@ class Settings(BaseSettings):
     database_url: str = "postgresql+asyncpg://rarelens:rarelens@localhost:5432/rarelens"
     mlflow_tracking_uri: str = "http://localhost:5000"
     model_name: str = "rarelens-pathogenicity"
-    model_stage: str = "Production"
+    # Registry alias set by `rarelens_ml.train --register` (stages are deprecated in MLflow 3).
+    model_alias: str = "production"
+    # A model artifact URI (gs://...) scores without an MLflow server running; wins over the registry.
+    model_uri: str | None = None
     gcs_bucket: str | None = None          # set in GCP; local uses ./data
     pubsub_topic: str | None = None        # "vcf-uploaded" in GCP; local runs pipeline inline
+    # Serverless track: run the Nextflow driver as a Cloud Run job instead of Argo + Pub/Sub.
+    cloudrun_job: str | None = None
+    gcp_project: str | None = None         # required with pubsub_topic or cloudrun_job
+    gcp_region: str = "europe-west2"
+    pipeline_dir: Path = REPO_ROOT / "pipeline"
+    nextflow_profile: str = "docker"
+    # Local (non-gs://) VCFs must live under this directory.
+    local_data_root: Path = Path("/data")
+    # Browsers calling the API cross-origin; behind the ingress the UI is same-origin.
+    cors_origins: list[str] = ["http://localhost:5173"]
 
 
 settings = Settings()

api/app/db.py

@@ -1,13 +1,33 @@
 from collections.abc import AsyncIterator
+from typing import Annotated
 
+from fastapi import Depends
+from sqlalchemy.engine import make_url
 from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
 
 from app.config import settings
 
-engine = create_async_engine(settings.database_url, pool_pre_ping=True)
+
+def normalize_async_url(url: str) -> str:
+    """Translate libpq's `sslmode=` into the `ssl=` asyncpg understands.
+
+    Hosted Postgres (Neon, Supabase) hands out sslmode= URLs, which asyncpg rejects outright.
+    """
+    parsed = make_url(url)
+    if "asyncpg" not in parsed.drivername or "sslmode" not in parsed.query:
+        return url
+    query = dict(parsed.query)
+    query["ssl"] = query.pop("sslmode")
+    return parsed.set(query=query).render_as_string(hide_password=False)
+
+
+engine = create_async_engine(normalize_async_url(settings.database_url), pool_pre_ping=True)
 SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
 
 
 async def get_session() -> AsyncIterator[AsyncSession]:
     async with SessionLocal() as session:
         yield session
+
+
+SessionDep = Annotated[AsyncSession, Depends(get_session)]

api/app/main.py

@@ -1,8 +1,9 @@
 from contextlib import asynccontextmanager
 
-from fastapi import FastAPI
+from fastapi import APIRouter, FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 
+from app.config import settings
 from app.routers import jobs, predictions, samples, variants
 
 
@@ -14,13 +15,19 @@ async def lifespan(app: FastAPI):
 
 app = FastAPI(title="rarelens API", version="0.1.0", lifespan=lifespan)
 app.add_middleware(
-    CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"]
+    CORSMiddleware,
+    allow_origins=settings.cors_origins,
+    allow_methods=["GET", "POST"],
+    allow_headers=["content-type"],
 )
 
-app.include_router(samples.router, prefix="/samples", tags=["samples"])
-app.include_router(jobs.router, prefix="/jobs", tags=["jobs"])
-app.include_router(variants.router, prefix="/variants", tags=["variants"])
-app.include_router(predictions.router, prefix="/predictions", tags=["predictions"])
+# The ingress forwards /api/* to this service unchanged, and local dev uses the same prefix.
+api = APIRouter(prefix="/api")
+api.include_router(samples.router, prefix="/samples", tags=["samples"])
+api.include_router(jobs.router, prefix="/jobs", tags=["jobs"])
+api.include_router(variants.router, prefix="/variants", tags=["variants"])
+api.include_router(predictions.router, prefix="/predictions", tags=["predictions"])
+app.include_router(api)
 
 
 @app.get("/health", tags=["ops"])

api/app/models.py

@@ -2,9 +2,9 @@
 
 One sample -> many jobs; one job -> many variants; one variant -> one prediction (latest).
 """
-from datetime import datetime
 import enum
 import uuid
+from datetime import datetime
 
 from sqlalchemy import DateTime, Enum, Float, ForeignKey, Integer, String, Text, func
 from sqlalchemy.dialects.postgresql import JSONB, UUID
@@ -55,12 +55,12 @@ class Variant(Base):
     ref: Mapped[str] = mapped_column(Text)
     alt: Mapped[str] = mapped_column(Text)
     gene: Mapped[str | None] = mapped_column(String(60), index=True)
-    consequence: Mapped[str | None] = mapped_column(String(120))
+    consequence: Mapped[str | None] = mapped_column(Text)   # "&"-joined VEP terms
     impact: Mapped[str | None] = mapped_column(String(20))
     hgvsc: Mapped[str | None] = mapped_column(Text)
     hgvsp: Mapped[str | None] = mapped_column(Text)
     gnomad_af: Mapped[float | None] = mapped_column(Float)
-    clinvar_sig: Mapped[str | None] = mapped_column(String(120))
+    clinvar_sig: Mapped[str | None] = mapped_column(Text)   # ","-joined co-located ClinVar terms
     annotations: Mapped[dict] = mapped_column(JSONB, default=dict)   # full VEP CSQ record
     job: Mapped[Job] = relationship(back_populates="variants")
     prediction: Mapped["Prediction | None"] = relationship(back_populates="variant", uselist=False)

api/app/routers/jobs.py

@@ -1,9 +1,8 @@
 import uuid
 
-from fastapi import APIRouter, Depends, HTTPException
-from sqlalchemy.ext.asyncio import AsyncSession
+from fastapi import APIRouter, HTTPException
 
-from app.db import get_session
+from app.db import SessionDep
 from app.models import Job
 from app.schemas import JobOut
 
@@ -11,7 +10,7 @@ router = APIRouter()
 
 
 @router.get("/{job_id}", response_model=JobOut)
-async def get_job(job_id: uuid.UUID, session: AsyncSession = Depends(get_session)):
+async def get_job(job_id: uuid.UUID, session: SessionDep):
     job = await session.get(Job, job_id)
     if job is None:
         raise HTTPException(404, "job not found")

api/app/routers/predictions.py

@@ -1,16 +1,22 @@
 import uuid
 
-from fastapi import APIRouter, Depends
-from sqlalchemy.ext.asyncio import AsyncSession
+from fastapi import APIRouter, HTTPException
 
-from app.db import get_session
+from app.db import SessionDep
+from app.models import Job, JobStatus
+from app.schemas import ScoreOut
 from app.services.scoring import score_job
 
 router = APIRouter()
 
 
-@router.post("/score/{job_id}")
-async def score(job_id: uuid.UUID, session: AsyncSession = Depends(get_session)) -> dict:
-    """Load the registered MLflow model and score every variant of a job."""
-    n = await score_job(job_id, session)
-    return {"job_id": str(job_id), "scored": n}
+@router.post("/score/{job_id}", response_model=ScoreOut)
+async def score(job_id: uuid.UUID, session: SessionDep) -> ScoreOut:
+    """Score every variant of a finished job with the model behind the registry alias."""
+    job = await session.get(Job, job_id)
+    if job is None:
+        raise HTTPException(404, "job not found")
+    if job.status != JobStatus.succeeded:
+        raise HTTPException(409, f"job is {job.status.value}; only succeeded jobs can be scored")
+    n, version = await score_job(job_id, session)
+    return ScoreOut(job_id=job_id, scored=n, model_version=version)

api/app/routers/samples.py

@@ -1,34 +1,39 @@
 import uuid
+from datetime import UTC, datetime
 
-from fastapi import APIRouter, Depends, HTTPException, status
+from fastapi import APIRouter, HTTPException, status
 from sqlalchemy import select
-from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.exc import IntegrityError
 
-from app.db import get_session
-from app.models import Job, Sample
+from app.db import SessionDep
+from app.models import Job, JobStatus, Sample
 from app.schemas import JobOut, SampleCreate, SampleOut
-from app.services.events import publish_vcf_uploaded
+from app.services import events
 
 router = APIRouter()
 
 
 @router.get("", response_model=list[SampleOut])
-async def list_samples(session: AsyncSession = Depends(get_session)):
+async def list_samples(session: SessionDep):
     result = await session.scalars(select(Sample).order_by(Sample.created_at.desc()))
     return result.all()
 
 
 @router.post("", response_model=SampleOut, status_code=status.HTTP_201_CREATED)
-async def create_sample(payload: SampleCreate, session: AsyncSession = Depends(get_session)):
+async def create_sample(payload: SampleCreate, session: SessionDep):
     sample = Sample(**payload.model_dump())
     session.add(sample)
+    try:
         await session.commit()
+    except IntegrityError:  # samples.name is unique
+        await session.rollback()
+        raise HTTPException(409, f"a sample named {payload.name!r} already exists") from None
     await session.refresh(sample)
     return sample
 
 
 @router.get("/{sample_id}/jobs", response_model=list[JobOut])
-async def list_jobs(sample_id: uuid.UUID, session: AsyncSession = Depends(get_session)):
+async def list_jobs(sample_id: uuid.UUID, session: SessionDep):
     result = await session.scalars(
         select(Job).where(Job.sample_id == sample_id).order_by(Job.created_at.desc())
     )
@@ -36,14 +41,21 @@ async def list_jobs(sample_id: uuid.UUID, session: AsyncSession = Depends(get_se
 
 
 @router.post("/{sample_id}/annotate", response_model=JobOut, status_code=status.HTTP_202_ACCEPTED)
-async def annotate(sample_id: uuid.UUID, session: AsyncSession = Depends(get_session)):
+async def annotate(sample_id: uuid.UUID, session: SessionDep):
     sample = await session.get(Sample, sample_id)
     if sample is None:
         raise HTTPException(404, "sample not found")
-    job = Job(sample_id=sample.id)
+    # Commit `running` before launching: a local run that dies instantly is marked failed by its
+    # watcher, and a later status write here would overwrite that.
+    job = Job(sample_id=sample.id, status=JobStatus.running)
     session.add(job)
     await session.commit()
+    try:
+        job.workflow_ref = await events.launch(job.id, sample.vcf_uri, sample.assembly)
+    except events.LaunchError as e:
+        job.status = JobStatus.failed
+        job.log = str(e)
+        job.finished_at = datetime.now(UTC)
+    await session.commit()
     await session.refresh(job)
-    # Emits to Pub/Sub in GCP; runs the Nextflow pipeline inline for local dev.
-    await publish_vcf_uploaded(job_id=job.id, vcf_uri=sample.vcf_uri)
     return job

api/app/routers/variants.py

@@ -1,28 +1,37 @@
 import uuid
 
-from fastapi import APIRouter, Depends, Query
-from sqlalchemy import func, select
-from sqlalchemy.ext.asyncio import AsyncSession
+from fastapi import APIRouter, Query
+from sqlalchemy import Integer, case, cast, func, select
 from sqlalchemy.orm import selectinload
 
-from app.db import get_session
+from app.db import SessionDep
 from app.models import Prediction, Variant
-from app.schemas import VariantPage
+from app.schemas import VariantOut, VariantPage
 
 router = APIRouter()
 
+# Karyotype order (1..22, X, Y, MT) instead of text order, where "10" sorts before "2".
+_chrom = func.regexp_replace(Variant.chrom, "^chr", "", "i")
+CHROM_ORDER = case(
+    (_chrom.regexp_match("^[0-9]+$"), cast(_chrom, Integer)),
+    (_chrom == "X", 23),
+    (_chrom == "Y", 24),
+    (_chrom.in_(["M", "MT"]), 25),
+    else_=26,
+)
+
 
 @router.get("", response_model=VariantPage)
 async def list_variants(
     job_id: uuid.UUID,
+    session: SessionDep,
     gene: str | None = None,
     impact: str | None = Query(None, pattern="^(HIGH|MODERATE|LOW|MODIFIER)$"),
     max_af: float | None = Query(None, ge=0, le=1),
     min_score: float | None = Query(None, ge=0, le=1),
-    limit: int = Query(50, le=500),
-    offset: int = 0,
-    session: AsyncSession = Depends(get_session),
-):
+    limit: int = Query(50, ge=1, le=500),
+    offset: int = Query(0, ge=0),
+) -> VariantPage:
     stmt = select(Variant).where(Variant.job_id == job_id)
     if gene:
         stmt = stmt.where(Variant.gene == gene.upper())
@@ -31,13 +40,21 @@ async def list_variants(
     if max_af is not None:
         stmt = stmt.where((Variant.gnomad_af.is_(None)) | (Variant.gnomad_af <= max_af))
     if min_score is not None:
-        stmt = stmt.join(Prediction, Prediction.variant_id == Variant.id).where(Prediction.score >= min_score)
+        stmt = stmt.join(Prediction, Prediction.variant_id == Variant.id).where(
+            Prediction.score >= min_score
+        )
 
     total = await session.scalar(select(func.count()).select_from(stmt.subquery()))
     rows = await session.scalars(
         stmt.options(selectinload(Variant.prediction))
-        .order_by(Variant.chrom, Variant.pos)
+        # id breaks ties between split multiallelics at one position, keeping pages stable.
+        .order_by(CHROM_ORDER, Variant.chrom, Variant.pos, Variant.id)
         .limit(limit)
         .offset(offset)
     )
-    return VariantPage(items=rows.all(), total=total or 0, limit=limit, offset=offset)
+    return VariantPage(
+        items=[VariantOut.model_validate(v) for v in rows],
+        total=total or 0,
+        limit=limit,
+        offset=offset,
+    )

api/app/schemas.py

@@ -1,8 +1,12 @@
-from datetime import datetime
+import re
 import uuid
+from datetime import datetime
+from pathlib import PurePosixPath
+from typing import Literal
 
-from pydantic import BaseModel, ConfigDict, Field
+from pydantic import BaseModel, ConfigDict, Field, field_validator
 
+from app.config import settings
 from app.models import JobStatus
 
 
@@ -10,10 +14,34 @@ class ORMModel(BaseModel):
     model_config = ConfigDict(from_attributes=True)
 
 
+Assembly = Literal["GRCh38", "GRCh37"]
+VCF_SUFFIXES = (".vcf", ".vcf.gz", ".vcf.bgz", ".bcf")
+GCS_URI = re.compile(r"gs://[a-z0-9][a-z0-9._-]{1,220}[a-z0-9]/\S+")
+
+
 class SampleCreate(BaseModel):
     name: str = Field(min_length=1, max_length=120)
     vcf_uri: str
-    assembly: str = "GRCh38"
+    # Passed to VEP --assembly; the VEP cache must contain it.
+    assembly: Assembly = "GRCh38"
+
+    @field_validator("vcf_uri")
+    @classmethod
+    def _gcs_object_or_file_under_data_root(cls, v: str) -> str:
+        # The URI becomes a Nextflow argument and a path the pipeline reads: accept a GCS object or
+        # a file under the local data root, never something that parses as an option.
+        if not v or any(ord(c) < 32 for c in v):
+            raise ValueError("vcf_uri must be a non-empty single line")
+        if not v.lower().endswith(VCF_SUFFIXES):
+            raise ValueError(f"vcf_uri must end in one of {', '.join(VCF_SUFFIXES)}")
+        if v.startswith("gs://"):
+            if not GCS_URI.fullmatch(v):
+                raise ValueError("vcf_uri is not a valid gs://bucket/object URI")
+            return v
+        path, root = PurePosixPath(v), PurePosixPath(settings.local_data_root)
+        if not path.is_absolute() or ".." in path.parts or not path.is_relative_to(root):
+            raise ValueError(f"local VCFs must be absolute paths under {root}")
+        return v
 
 
 class SampleOut(ORMModel):
@@ -30,6 +58,7 @@ class JobOut(ORMModel):
     status: JobStatus
     workflow_ref: str | None
     vep_version: str | None
+    log: str | None
     created_at: datetime
     finished_at: datetime | None
 
@@ -56,6 +85,12 @@ class VariantOut(ORMModel):
     prediction: PredictionOut | None = None
 
 
+class ScoreOut(BaseModel):
+    job_id: uuid.UUID
+    scored: int
+    model_version: str
+
+
 class VariantPage(BaseModel):
     items: list[VariantOut]
     total: int

api/app/services/events.py

@@ -1,31 +1,142 @@
-"""Event publishing.
+"""Start the annotation pipeline for a job.
 
-GCP: publish a JSON message to Pub/Sub; an Argo Events sensor turns it into an Argo Workflow.
-Local: run Nextflow directly in a background task so `docker compose up` gives a working demo.
+Serverless (CLOUDRUN_JOB set): execute a Cloud Run job that runs the Nextflow driver, which
+submits the pipeline tasks to Google Batch. Nothing runs, or is billed, between annotations.
+Kubernetes (PUBSUB_TOPIC set): publish to Pub/Sub; an Argo Events sensor submits the annotate-vcf
+WorkflowTemplate, whose exit handler marks the job failed if the workflow does not succeed.
+Local: run Nextflow as a child process and watch it. The docker-compose API image has neither
+Nextflow nor Docker, so there the job fails at once with instructions to run it from the host.
 """
 import asyncio
 import json
+import logging
+import os
+import shutil
 import uuid
+from datetime import UTC, datetime
+from functools import cache
+from typing import Any
 
 from app.config import settings
+from app.db import SessionLocal
+from app.models import Job, JobStatus
+
+logger = logging.getLogger(__name__)
+
+PIPELINE_ENTRYPOINT = "/pipeline/main.nf"
+LOG_TAIL_CHARS = 4000
+PUBLISH_TIMEOUT_S = 30
+_watchers: set[asyncio.Task[None]] = set()
 
 
-async def publish_vcf_uploaded(job_id: uuid.UUID, vcf_uri: str) -> None:
-    message = {"job_id": str(job_id), "vcf_uri": vcf_uri}
+class LaunchError(Exception):
+    """The pipeline could not be started; the message is stored on the job for the user."""
+
+
+async def launch(job_id: uuid.UUID, vcf_uri: str, assembly: str) -> str:
+    """Start the pipeline and return a reference to the run (stored as jobs.workflow_ref)."""
+    if settings.cloudrun_job:
+        return await _run_cloud_run_job(job_id, vcf_uri, assembly)
     if settings.pubsub_topic:
-        from google.cloud import pubsub_v1  # optional dependency, installed in the GCP image
+        message = {"job_id": str(job_id), "vcf_uri": vcf_uri, "assembly": assembly}
+        return await _publish(settings.pubsub_topic, message)
+    return await _run_local(job_id, vcf_uri, assembly)
 
-        publisher = pubsub_v1.PublisherClient()
-        publisher.publish(settings.pubsub_topic, json.dumps(message).encode())
-        return
 
-    cmd = [
-        "nextflow", "run", "/pipeline/main.nf", "-profile", "docker",
-        "--vcf", vcf_uri, "--job_id", str(job_id), "--db_url", settings.database_url,
+async def drain() -> None:
+    """Wait for local pipeline watchers (tests, graceful shutdown)."""
+    await asyncio.gather(*_watchers)
+
+
+@cache
+def _jobs_client() -> Any:
+    from google.cloud import run_v2
+
+    return run_v2.JobsClient()
+
+
+async def _run_cloud_run_job(job_id: uuid.UUID, vcf_uri: str, assembly: str) -> str:
+    if not settings.gcp_project:
+        raise LaunchError("CLOUDRUN_JOB is set but GCP_PROJECT is not")
+    name = (
+        f"projects/{settings.gcp_project}/locations/{settings.gcp_region}"
+        f"/jobs/{settings.cloudrun_job}"
+    )
+    args = [
+        "run", PIPELINE_ENTRYPOINT, "-profile", "gcp",
+        "--vcf", vcf_uri, "--job_id", str(job_id), "--assembly", assembly,
     ]
+    request = {"name": name, "overrides": {"container_overrides": [{"args": args}]}}
     try:
-        await asyncio.create_subprocess_exec(*cmd)
-    except FileNotFoundError:
-        # Nextflow is not installed in this environment (no Java toolchain yet);
-        # leave the job queued instead of crashing the request.
-        print(f"[rarelens] nextflow not found; cannot start pipeline for job {job_id}", flush=True)
+        operation = await asyncio.to_thread(_jobs_client().run_job, request=request)
+    except Exception as e:
+        logger.exception("starting Cloud Run job %s failed", name)
+        raise LaunchError(f"could not start {settings.cloudrun_job}: {e}") from e
+    name_or_job = getattr(getattr(operation, "metadata", None), "name", "")
+    execution = name_or_job or settings.cloudrun_job or "started"
+    return f"cloudrun:{execution.rsplit('/', 1)[-1]}"
+
+
+@cache
+def _publisher() -> Any:
+    from google.cloud import pubsub_v1
+
+    return pubsub_v1.PublisherClient()
+
+
+async def _publish(topic_name: str, message: dict[str, str]) -> str:
+    if not settings.gcp_project:
+        raise LaunchError("PUBSUB_TOPIC is set but GCP_PROJECT is not")
+    publisher = _publisher()
+    topic = publisher.topic_path(settings.gcp_project, topic_name)
+    try:
+        future = publisher.publish(topic, json.dumps(message).encode())
+        message_id = await asyncio.to_thread(future.result, timeout=PUBLISH_TIMEOUT_S)
+    except Exception as e:
+        logger.exception("publishing to %s failed", topic)
+        raise LaunchError(f"could not publish to {topic}: {e}") from e
+    return f"pubsub:{message_id}"
+
+
+async def _run_local(job_id: uuid.UUID, vcf_uri: str, assembly: str) -> str:
+    nextflow = shutil.which("nextflow")
+    if nextflow is None:
+        raise LaunchError(
+            "Nextflow is not installed where the API runs. Start the pipeline from the host with "
+            f"`make annotate JOB={job_id} VCF=<path to the VCF>`."
+        )
+    cmd = [
+        nextflow, "run", str(settings.pipeline_dir / "main.nf"),
+        "-profile", settings.nextflow_profile,
+        "--vcf", vcf_uri, "--job_id", str(job_id), "--assembly", assembly,
+    ]
+    # The loader reads DATABASE_URL from its environment; keep it off the command line.
+    env = {**os.environ, "DATABASE_URL": settings.database_url}
+    try:
+        proc = await asyncio.create_subprocess_exec(
+            *cmd,
+            cwd=settings.pipeline_dir,
+            env=env,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.STDOUT,
+        )
+    except OSError as e:
+        raise LaunchError(f"could not start Nextflow: {e}") from e
+    task = asyncio.create_task(_watch(job_id, proc))
+    _watchers.add(task)
+    task.add_done_callback(_watchers.discard)
+    return f"nextflow:pid-{proc.pid}"
+
+
+async def _watch(job_id: uuid.UUID, proc: Any) -> None:
+    out, err = await proc.communicate()
+    if proc.returncode == 0:
+        return  # the loader marks the job succeeded
+    tail = ((out or b"") + (err or b"")).decode(errors="replace")[-LOG_TAIL_CHARS:]
+    async with SessionLocal() as session:
+        job = await session.get(Job, job_id)
+        if job is not None and job.status != JobStatus.succeeded:
+            job.status = JobStatus.failed
+            job.log = f"nextflow exited with code {proc.returncode}\n{tail}"
+            job.finished_at = datetime.now(UTC)
+            await session.commit()

api/app/services/scoring.py

@@ -1,45 +1,107 @@
+"""Score a job's variants with the registered MLflow model.
+
+The registered model is a pyfunc that owns its feature engineering (rarelens_ml.features travels
+with it as model code) and returns P(pathogenic). Serving therefore only sends the raw columns
+below and cannot drift from training.
+"""
+import asyncio
 import uuid
+from collections.abc import Sequence
+from typing import Any
 
 import mlflow
 import pandas as pd
-from sqlalchemy import select
+from mlflow import MlflowClient
+from sqlalchemy import func, select
+from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy.ext.asyncio import AsyncSession
 
 from app.config import settings
 from app.models import Prediction, Variant
 
-_model = None
+# Must match rarelens_ml.features.RAW_COLUMNS.
+RAW_COLUMNS = ["impact", "consequence", "gnomad_af", "cadd_phred", "am_pathogenicity"]
+CHUNK_SIZE = 5000
+
+_models: dict[str, Any] = {}  # model version -> loaded pyfunc
 
 
-def load_model():
-    global _model
-    if _model is None:
+def load_model() -> tuple[Any, str]:
+    """Return (model, version), from MODEL_URI if set, else from the registry alias."""
+    if settings.model_uri:
+        return _load_uri(settings.model_uri)
+    client = MlflowClient(tracking_uri=settings.mlflow_tracking_uri)
+    version = client.get_model_version_by_alias(settings.model_name, settings.model_alias).version
+    if version not in _models:
         mlflow.set_tracking_uri(settings.mlflow_tracking_uri)
-        _model = mlflow.pyfunc.load_model(f"models:/{settings.model_name}/{settings.model_stage}")
-    return _model
+        _models.clear()
+        _models[version] = mlflow.pyfunc.load_model(f"models:/{settings.model_name}/{version}")
+    return _models[version], version
 
 
-def featurise(variants: list[Variant]) -> pd.DataFrame:
-    # Mirror ml/rarelens_ml/features.py exactly; shared package later.
+def _load_uri(uri: str) -> tuple[Any, str]:
+    """Load a model artifact directly (gs://...): no tracking server, nothing running when idle."""
+    version = uri.rstrip("/").rsplit("/", 1)[-1][:40] or "uri"
+    if version not in _models:
+        _models.clear()
+        _models[version] = mlflow.pyfunc.load_model(uri)
+    return _models[version], version
+
+
+def raw_frame(variants: Sequence[Variant]) -> pd.DataFrame:
     return pd.DataFrame(
         {
             "impact": [v.impact for v in variants],
             "consequence": [v.consequence for v in variants],
-            "gnomad_af": [v.gnomad_af if v.gnomad_af is not None else 0.0 for v in variants],
+            "gnomad_af": [v.gnomad_af if v.gnomad_af is not None else float("nan") for v in variants],
             "cadd_phred": [v.annotations.get("CADD_PHRED") for v in variants],
             "am_pathogenicity": [v.annotations.get("am_pathogenicity") for v in variants],
-        }
+        },
+        columns=RAW_COLUMNS,
     )
 
 
-async def score_job(job_id: uuid.UUID, session: AsyncSession) -> int:
-    variants = (await session.scalars(select(Variant).where(Variant.job_id == job_id))).all()
+async def score_job(job_id: uuid.UUID, session: AsyncSession) -> tuple[int, str]:
+    model, version = await asyncio.to_thread(load_model)
+    scored = 0
+    last_id = 0
+    # Keyset pagination keeps memory flat for whole-genome jobs.
+    while True:
+        variants = (
+            await session.scalars(
+                select(Variant)
+                .where(Variant.job_id == job_id, Variant.id > last_id)
+                .order_by(Variant.id)
+                .limit(CHUNK_SIZE)
+            )
+        ).all()
         if not variants:
-        return 0
-    model = load_model()
-    scores = model.predict(featurise(variants))
-    for v, s in zip(variants, scores):
-        session.add(Prediction(variant_id=v.id, model_name=settings.model_name,
-                               model_version=settings.model_stage, score=float(s)))
+            break
+        scores = await asyncio.to_thread(model.predict, raw_frame(variants))
+        stmt = insert(Prediction).values(
+            [
+                {
+                    "variant_id": v.id,
+                    "model_name": settings.model_name,
+                    "model_version": version,
+                    "score": float(s),
+                }
+                for v, s in zip(variants, scores, strict=True)
+            ]
+        )
+        # Re-scoring (e.g. after a new model version) replaces the previous prediction.
+        await session.execute(
+            stmt.on_conflict_do_update(
+                index_elements=[Prediction.variant_id],
+                set_={
+                    "model_name": stmt.excluded.model_name,
+                    "model_version": stmt.excluded.model_version,
+                    "score": stmt.excluded.score,
+                    "created_at": func.now(),
+                },
+            )
+        )
+        scored += len(variants)
+        last_id = variants[-1].id
     await session.commit()
-    return len(variants)
+    return scored, version

api/pyproject.toml

@@ -1,3 +1,7 @@
+[build-system]
+requires = ["setuptools>=69"]
+build-backend = "setuptools.build_meta"
+
 [project]
 name = "rarelens-api"
 version = "0.1.0"
@@ -12,17 +16,50 @@ dependencies = [
   "pydantic>=2.8",
   "pydantic-settings>=2.4",
   "httpx>=0.27",
-  "mlflow-skinny>=2.16",
+  # mlflow major must match rarelens-ml and the tracking server image.
+  "mlflow-skinny>=3,<4",
   "lightgbm>=4.5",
+  "scikit-learn>=1.5",  # the pickled LGBMClassifier inside the pyfunc needs it to load
   "pandas>=2.2",
 ]
 
 [project.optional-dependencies]
-dev = ["pytest", "pytest-asyncio", "ruff", "mypy", "aiosqlite"]
+# pubsub: Kubernetes track; run: serverless track; storage: load a model from gs://
+gcp = ["google-cloud-pubsub>=2.23", "google-cloud-run>=0.10", "google-cloud-storage>=2.18"]
+dev = [
+  "pytest>=8",
+  "pytest-asyncio>=1.0",
+  "ruff==0.16.2",
+  "mypy>=1.11",
+  "pandas-stubs",
+  # Throwaway Postgres for `pytest` when DATABASE_URL is unset (no Docker needed).
+  "pgserver; sys_platform != 'win32'",
+]
+
+# The repo also has top-level `alembic/` and `tests/` dirs; only `app` is the package.
+[tool.setuptools.packages.find]
+include = ["app*"]
 
 [tool.ruff]
 line-length = 100
 target-version = "py312"
 
+[tool.ruff.lint.isort]
+# Without this the local `alembic/` migrations dir makes ruff treat the alembic library as first-party.
+known-third-party = ["alembic"]
+
+[tool.mypy]
+python_version = "3.12"
+
+[[tool.mypy.overrides]]
+module = ["google.cloud.*", "mlflow.*"]
+ignore_missing_imports = true
+
 [tool.pytest.ini_options]
 asyncio_mode = "auto"
+# asyncpg connections are bound to the loop that opened them; one loop for the whole run
+# lets the app's pooled engine be shared across tests.
+asyncio_default_fixture_loop_scope = "session"
+asyncio_default_test_loop_scope = "session"
+pythonpath = ["."]
+testpaths = ["tests"]

api/tests/conftest.py

@@ -0,0 +1,83 @@
+"""Test bootstrap.
+
+CI provides DATABASE_URL (a Postgres service container). Locally, when it is unset, a throwaway
+Postgres is started with `pgserver` so `pytest` needs no Docker. Set REQUIRE_DB=1 to turn a missing
+database into a failure instead of a skip (CI does this).
+
+DATABASE_URL must be settled before anything imports `app`, because `app.config.settings` and the
+engine in `app.db` are created at import time.
+"""
+import os
+import tempfile
+from collections.abc import AsyncIterator, Iterator
+from pathlib import Path
+from typing import TYPE_CHECKING
+
+import pytest
+from httpx import ASGITransport, AsyncClient
+
+if TYPE_CHECKING:
+    from alembic.config import Config
+
+API_DIR = Path(__file__).resolve().parents[1]
+_pg_server = None  # keeps the pgserver handle alive for the whole session
+
+
+def _resolve_database_url() -> str | None:
+    global _pg_server
+    if url := os.environ.get("DATABASE_URL"):
+        return url
+    try:
+        import pgserver
+    except ImportError:
+        return None
+    _pg_server = pgserver.get_server(tempfile.mkdtemp(prefix="rarelens-pg-"), cleanup_mode="delete")
+    socket_dir = _pg_server.get_uri().split("host=", 1)[1]
+    return f"postgresql+asyncpg://postgres@/postgres?host={socket_dir}"
+
+
+DATABASE_URL = _resolve_database_url()
+if DATABASE_URL:
+    os.environ["DATABASE_URL"] = DATABASE_URL
+
+
+@pytest.fixture(scope="session")
+def alembic_cfg() -> "Config":
+    if not DATABASE_URL:
+        if os.environ.get("REQUIRE_DB"):
+            pytest.fail("REQUIRE_DB is set but no database is available")
+        pytest.skip("no DATABASE_URL and pgserver is not installed")
+    from alembic.config import Config
+
+    cfg = Config(str(API_DIR / "alembic.ini"))
+    cfg.set_main_option("script_location", str(API_DIR / "alembic"))
+    return cfg
+
+
+@pytest.fixture(scope="session")
+def migrated_db(alembic_cfg: "Config") -> Iterator[None]:
+    from alembic import command
+
+    command.upgrade(alembic_cfg, "head")
+    yield
+
+
+@pytest.fixture
+async def db(migrated_db: None) -> AsyncIterator[None]:
+    from sqlalchemy import text
+
+    from app.db import engine
+
+    async with engine.begin() as conn:
+        await conn.execute(
+            text("TRUNCATE samples, jobs, variants, predictions RESTART IDENTITY CASCADE")
+        )
+    yield
+
+
+@pytest.fixture
+async def client() -> AsyncIterator[AsyncClient]:
+    from app.main import app
+
+    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
+        yield c

api/tests/factories.py

@@ -0,0 +1,21 @@
+import uuid
+from typing import Any
+
+from app.db import SessionLocal
+from app.models import Job, JobStatus, Sample, Variant
+
+
+async def seed_job(
+    variants: list[dict[str, Any]], status: JobStatus = JobStatus.succeeded
+) -> uuid.UUID:
+    """Insert a sample, a job and its variants; each variant dict overrides the defaults."""
+    async with SessionLocal() as s:
+        sample = Sample(name=f"s-{uuid.uuid4()}", vcf_uri="gs://b/x.vcf.gz", assembly="GRCh38")
+        job = Job(sample=sample, status=status)
+        rows = [
+            Variant(job=job, **{"chrom": "22", "pos": 1, "ref": "A", "alt": "G", "annotations": {}} | v)
+            for v in variants
+        ]
+        s.add_all([sample, job, *rows])
+        await s.commit()
+        return job.id

api/tests/test_annotate.py

@@ -0,0 +1,126 @@
+import uuid
+from typing import Any
+
+import pytest
+from httpx import AsyncClient
+
+from app.config import settings
+from app.services import events
+
+
+async def new_sample(client: AsyncClient) -> str:
+    r = await client.post(
+        "/api/samples",
+        json={"name": f"s-{uuid.uuid4()}", "vcf_uri": "gs://bucket/x.vcf.gz", "assembly": "GRCh37"},
+    )
+    assert r.status_code == 201, r.text
+    return r.json()["id"]
+
+
+@pytest.mark.usefixtures("db")
+async def test_local_without_nextflow_fails_fast_with_instructions(
+    client: AsyncClient, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    monkeypatch.setattr(settings, "pubsub_topic", None)
+    monkeypatch.setattr(events.shutil, "which", lambda _: None)
+    sample_id = await new_sample(client)
+
+    r = await client.post(f"/api/samples/{sample_id}/annotate")
+    assert r.status_code == 202
+    job = r.json()
+    assert job["status"] == "failed"
+    assert "make annotate" in job["log"]
+    assert job["finished_at"] is not None
+
+
+class FakeProcess:
+    def __init__(self, returncode: int, stderr: bytes) -> None:
+        self.returncode = returncode
+        self._stderr = stderr
+        self.pid = 4242
+
+    async def communicate(self) -> tuple[bytes, bytes]:
+        return b"", self._stderr
+
+
+@pytest.mark.usefixtures("db")
+async def test_local_run_is_watched_and_a_crash_marks_the_job_failed(
+    client: AsyncClient, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    monkeypatch.setattr(settings, "pubsub_topic", None)
+    monkeypatch.setattr(events.shutil, "which", lambda _: "/usr/bin/nextflow")
+    launched: dict[str, Any] = {}
+
+    async def fake_exec(*cmd: str, **kw: Any) -> FakeProcess:
+        launched["cmd"], launched["env"] = cmd, kw["env"]
+        return FakeProcess(1, b"ERROR ~ VEP cache not found")
+
+    monkeypatch.setattr(events.asyncio, "create_subprocess_exec", fake_exec)
+    sample_id = await new_sample(client)
+
+    r = await client.post(f"/api/samples/{sample_id}/annotate")
+    assert r.status_code == 202
+    await events.drain()
+
+    job = (await client.get(f"/api/jobs/{r.json()['id']}")).json()
+    assert job["status"] == "failed"
+    assert "VEP cache not found" in job["log"]
+    # The DB password travels in the environment, never on the command line.
+    assert settings.database_url not in " ".join(launched["cmd"])
+    assert launched["env"]["DATABASE_URL"] == settings.database_url
+    assert launched["cmd"][launched["cmd"].index("--assembly") + 1] == "GRCh37"
+
+
+class FakeFuture:
+    def __init__(self, result: str | Exception) -> None:
+        self._result = result
+
+    def result(self, timeout: float | None = None) -> str:
+        if isinstance(self._result, Exception):
+            raise self._result
+        return self._result
+
+
+class FakePublisher:
+    def __init__(self, result: str | Exception) -> None:
+        self.result = result
+        self.published: list[tuple[str, bytes]] = []
+
+    def topic_path(self, project: str, topic: str) -> str:
+        return f"projects/{project}/topics/{topic}"
+
+    def publish(self, topic: str, data: bytes) -> FakeFuture:
+        self.published.append((topic, data))
+        return FakeFuture(self.result)
+
+
+@pytest.mark.usefixtures("db")
+async def test_pubsub_publishes_to_the_full_topic_path(
+    client: AsyncClient, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    publisher = FakePublisher("msg-123")
+    monkeypatch.setattr(settings, "pubsub_topic", "vcf-uploaded")
+    monkeypatch.setattr(settings, "gcp_project", "my-proj")
+    monkeypatch.setattr(events, "_publisher", lambda: publisher)
+    sample_id = await new_sample(client)
+
+    r = await client.post(f"/api/samples/{sample_id}/annotate")
+    job = r.json()
+    assert (job["status"], job["workflow_ref"]) == ("running", "pubsub:msg-123")
+    [(topic, data)] = publisher.published
+    assert topic == "projects/my-proj/topics/vcf-uploaded"
+    assert b'"assembly": "GRCh37"' in data and job["id"].encode() in data
+
+
+@pytest.mark.usefixtures("db")
+async def test_pubsub_failure_marks_the_job_failed(
+    client: AsyncClient, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    monkeypatch.setattr(settings, "pubsub_topic", "vcf-uploaded")
+    monkeypatch.setattr(settings, "gcp_project", "my-proj")
+    monkeypatch.setattr(events, "_publisher", lambda: FakePublisher(RuntimeError("403 denied")))
+    sample_id = await new_sample(client)
+
+    job = (await client.post(f"/api/samples/{sample_id}/annotate")).json()
+    assert job["status"] == "failed"
+    assert "403 denied" in job["log"]

api/tests/test_cloudrun.py

@@ -0,0 +1,97 @@
+"""The serverless track: the API starts a Cloud Run job instead of publishing to Pub/Sub."""
+import uuid
+from typing import Any
+
+import pytest
+from httpx import AsyncClient
+
+from app.config import settings
+from app.services import events
+
+
+class FakeOperation:
+    def __init__(self, execution: str) -> None:
+        self.metadata = type("Meta", (), {"name": execution})()
+
+
+class FakeJobsClient:
+    def __init__(self, result: Any = None) -> None:
+        self.result = result
+        self.requests: list[dict] = []
+
+    def run_job(self, request: dict) -> FakeOperation:
+        self.requests.append(request)
+        if isinstance(self.result, Exception):
+            raise self.result
+        return self.result
+
+
+async def new_sample(client: AsyncClient) -> str:
+    r = await client.post(
+        "/api/samples",
+        json={"name": f"s-{uuid.uuid4()}", "vcf_uri": "gs://bucket/x.vcf.gz", "assembly": "GRCh37"},
+    )
+    assert r.status_code == 201, r.text
+    return r.json()["id"]
+
+
+@pytest.fixture
+def cloudrun(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(settings, "pubsub_topic", None)
+    monkeypatch.setattr(settings, "cloudrun_job", "rarelens-nextflow")
+    monkeypatch.setattr(settings, "gcp_project", "my-proj")
+    monkeypatch.setattr(settings, "gcp_region", "europe-west2")
+
+
+@pytest.mark.usefixtures("db", "cloudrun")
+async def test_annotate_executes_the_job_with_pipeline_arguments(
+    client: AsyncClient, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    jobs = FakeJobsClient(FakeOperation("projects/p/locations/l/executions/rarelens-nextflow-abc12"))
+    monkeypatch.setattr(events, "_jobs_client", lambda: jobs)
+    sample_id = await new_sample(client)
+
+    r = await client.post(f"/api/samples/{sample_id}/annotate")
+    assert r.status_code == 202
+    job = r.json()
+    assert job["status"] == "running"
+    assert job["workflow_ref"] == "cloudrun:rarelens-nextflow-abc12"
+
+    [request] = jobs.requests
+    assert request["name"] == "projects/my-proj/locations/europe-west2/jobs/rarelens-nextflow"
+    [override] = request["overrides"]["container_overrides"]
+    args = override["args"]
+    assert args[:4] == ["run", "/pipeline/main.nf", "-profile", "gcp"]
+    assert args[args.index("--job_id") + 1] == job["id"]
+    assert args[args.index("--assembly") + 1] == "GRCh37"
+    assert args[args.index("--vcf") + 1] == "gs://bucket/x.vcf.gz"
+
+
+@pytest.mark.usefixtures("db", "cloudrun")
+async def test_a_failed_execution_call_marks_the_job_failed(
+    client: AsyncClient, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    monkeypatch.setattr(
+        events, "_jobs_client", lambda: FakeJobsClient(RuntimeError("403 permission denied"))
+    )
+    sample_id = await new_sample(client)
+
+    job = (await client.post(f"/api/samples/{sample_id}/annotate")).json()
+    assert job["status"] == "failed"
+    assert "403 permission denied" in job["log"]
+
+
+@pytest.mark.usefixtures("db", "cloudrun")
+async def test_cloud_run_job_takes_precedence_over_pubsub(
+    client: AsyncClient, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    monkeypatch.setattr(settings, "pubsub_topic", "vcf-uploaded")
+    jobs = FakeJobsClient(FakeOperation("projects/p/locations/l/executions/x-1"))
+    monkeypatch.setattr(events, "_jobs_client", lambda: jobs)
+    monkeypatch.setattr(
+        events, "_publisher", lambda: pytest.fail("Pub/Sub must not be used in the serverless track")
+    )
+    sample_id = await new_sample(client)
+
+    assert (await client.post(f"/api/samples/{sample_id}/annotate")).json()["status"] == "running"
+    assert len(jobs.requests) == 1

api/tests/test_db_url.py

@@ -0,0 +1,34 @@
+"""Neon-style URLs use libpq's sslmode, which asyncpg does not understand."""
+import pytest
+
+from app.db import normalize_async_url
+
+
+@pytest.mark.parametrize(
+    ("raw", "expected"),
+    [
+        (
+            "postgresql+asyncpg://u:[email protected]/neondb?sslmode=require",
+            "postgresql+asyncpg://u:[email protected]/neondb?ssl=require",
+        ),
+        # Already asyncpg-shaped, or nothing to do.
+        (
+            "postgresql+asyncpg://u:p@h/db?ssl=require",
+            "postgresql+asyncpg://u:p@h/db?ssl=require",
+        ),
+        ("postgresql+asyncpg://u:p@h:5432/db", "postgresql+asyncpg://u:p@h:5432/db"),
+        # Only the asyncpg driver needs the rewrite.
+        ("postgresql+psycopg://u:p@h/db?sslmode=require", "postgresql+psycopg://u:p@h/db?sslmode=require"),
+    ],
+)
+def test_normalize_async_url(raw: str, expected: str) -> None:
+    assert normalize_async_url(raw) == expected
+
+
+def test_disable_is_preserved() -> None:
+    assert normalize_async_url("postgresql+asyncpg://u:p@h/db?sslmode=disable").endswith("ssl=disable")
+
+
+def test_unix_socket_url_is_untouched() -> None:
+    raw = "postgresql+asyncpg://postgres@/postgres?host=/tmp/pg"
+    assert normalize_async_url(raw) == raw

api/tests/test_health.py

@@ -1,12 +1,7 @@
-import pytest
-from httpx import ASGITransport, AsyncClient
-
-from app.main import app
+from httpx import AsyncClient
 
 
-@pytest.mark.asyncio
-async def test_health():
-    async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
-        r = await c.get("/health")
+async def test_health(client: AsyncClient) -> None:
+    r = await client.get("/health")
     assert r.status_code == 200
     assert r.json() == {"status": "ok"}

api/tests/test_migrations.py

@@ -0,0 +1,15 @@
+import asyncio
+
+from alembic import command
+from alembic.config import Config
+
+from app.db import engine
+
+
+async def test_downgrade_to_base_then_upgrade_again(alembic_cfg: Config) -> None:
+    # alembic's env.py runs its own event loop, so keep it off this one.
+    await asyncio.to_thread(command.downgrade, alembic_cfg, "base")
+    await asyncio.to_thread(command.upgrade, alembic_cfg, "head")
+    # The recreated enum type has a new OID that pooled asyncpg connections have not seen;
+    # reconnect so later tests (and the database left at head) are unaffected.
+    await engine.dispose()

api/tests/test_model_loading.py

@@ -0,0 +1,47 @@
+"""A model artifact URI (gs://...) lets the API score without an MLflow server running."""
+# Imported eagerly: mlflow loads .pyfunc lazily, so patching it by name can hit the proxy.
+import mlflow.pyfunc
+import pytest
+
+from app.config import settings
+from app.services import scoring
+
+
+@pytest.fixture(autouse=True)
+def clear_cache() -> None:
+    scoring._models.clear()
+
+
+def no_registry(**kwargs: object) -> None:
+    pytest.fail("the registry must not be contacted when MODEL_URI is set")
+
+
+def test_model_uri_skips_the_registry(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(settings, "model_uri", "gs://bucket/models/pathogenicity/3")
+    monkeypatch.setattr(scoring, "MlflowClient", no_registry)
+    monkeypatch.setattr(mlflow.pyfunc, "load_model", lambda uri: f"model@{uri}")
+
+    model, version = scoring.load_model()
+    assert model == "model@gs://bucket/models/pathogenicity/3"
+    assert version == "3"
+
+
+def test_model_uri_without_a_version_segment_still_labels_the_prediction(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.setattr(settings, "model_uri", "gs://bucket/models/pathogenicity/")
+    monkeypatch.setattr(scoring, "MlflowClient", no_registry)
+    monkeypatch.setattr(mlflow.pyfunc, "load_model", lambda uri: "model")
+
+    assert scoring.load_model()[1] == "pathogenicity"
+
+
+def test_model_uri_is_loaded_once(monkeypatch: pytest.MonkeyPatch) -> None:
+    monkeypatch.setattr(settings, "model_uri", "gs://bucket/models/pathogenicity/3")
+    monkeypatch.setattr(scoring, "MlflowClient", no_registry)
+    calls: list[str] = []
+    monkeypatch.setattr(mlflow.pyfunc, "load_model", lambda uri: calls.append(uri) or "model")
+
+    scoring.load_model()
+    scoring.load_model()
+    assert len(calls) == 1

api/tests/test_routes.py

@@ -0,0 +1,13 @@
+import pytest
+from httpx import AsyncClient
+
+
+@pytest.mark.usefixtures("db")
+async def test_resources_live_under_api_prefix(client: AsyncClient) -> None:
+    # The ingress forwards /api/* unchanged, so the app itself must serve that prefix.
+    assert (await client.get("/api/samples")).status_code == 200
+    assert (await client.get("/samples")).status_code == 404
+
+
+async def test_health_stays_at_root_for_probes(client: AsyncClient) -> None:
+    assert (await client.get("/health")).status_code == 200

api/tests/test_samples.py

@@ -0,0 +1,18 @@
+import pytest
+from httpx import AsyncClient
+
+
+@pytest.mark.usefixtures("db")
+async def test_duplicate_sample_name_is_409_not_500(client: AsyncClient) -> None:
+    body = {"name": "HG002", "vcf_uri": "gs://bucket/hg002.vcf.gz"}
+    assert (await client.post("/api/samples", json=body)).status_code == 201
+    r = await client.post("/api/samples", json=body)
+    assert r.status_code == 409
+    assert "HG002" in r.json()["detail"]
+
+
+async def test_unknown_assembly_is_rejected(client: AsyncClient) -> None:
+    r = await client.post(
+        "/api/samples", json={"name": "a", "vcf_uri": "gs://bucket/a.vcf.gz", "assembly": "hg19"}
+    )
+    assert r.status_code == 422

api/tests/test_scoring.py

@@ -0,0 +1,90 @@
+import math
+import uuid
+
+import numpy as np
+import pandas as pd
+import pytest
+from httpx import AsyncClient
+from sqlalchemy import select
+
+from app.db import SessionLocal
+from app.models import Job, JobStatus, Prediction, Sample, Variant
+from app.services import scoring
+
+
+def variant(**kw: object) -> Variant:
+    fields: dict = {"chrom": "22", "pos": 1, "ref": "A", "alt": "G", "annotations": {}}
+    fields.update(kw)
+    return Variant(**fields)
+
+
+def test_raw_frame_sends_the_model_contract_columns() -> None:
+    frame = scoring.raw_frame([
+        variant(impact="HIGH", consequence="stop_gained", gnomad_af=None,
+                annotations={"CADD_PHRED": "35", "am_pathogenicity": "0.98"}),
+        variant(impact="LOW", consequence="synonymous_variant", gnomad_af=0.2, annotations={}),
+    ])
+    assert list(frame.columns) == scoring.RAW_COLUMNS
+    assert frame["impact"].tolist() == ["HIGH", "LOW"]
+    assert frame["cadd_phred"].iloc[0] == "35"
+    assert pd.isna(frame["cadd_phred"].iloc[1])
+    assert math.isnan(frame["gnomad_af"].iloc[0])
+
+
+class FakeModel:
+    def __init__(self, score: float) -> None:
+        self.score = score
+
+    def predict(self, frame: pd.DataFrame) -> np.ndarray:
+        assert list(frame.columns) == scoring.RAW_COLUMNS
+        return np.full(len(frame), self.score)
+
+
+async def make_job(status: JobStatus, n_variants: int) -> uuid.UUID:
+    async with SessionLocal() as s:
+        sample = Sample(name=f"s-{uuid.uuid4()}", vcf_uri="gs://b/x.vcf.gz", assembly="GRCh38")
+        job = Job(sample=sample, status=status)
+        s.add_all([sample, job, *(variant(job=job, pos=i + 1) for i in range(n_variants))])
+        await s.commit()
+        return job.id
+
+
+async def predictions(job_id: uuid.UUID) -> list[Prediction]:
+    async with SessionLocal() as s:
+        rows = await s.scalars(
+            select(Prediction).join(Variant).where(Variant.job_id == job_id)
+        )
+        return list(rows)
+
+
+@pytest.mark.usefixtures("db")
+async def test_scoring_twice_updates_instead_of_failing(
+    client: AsyncClient, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    job_id = await make_job(JobStatus.succeeded, n_variants=3)
+
+    monkeypatch.setattr(scoring, "load_model", lambda: (FakeModel(0.9), "7"))
+    r = await client.post(f"/api/predictions/score/{job_id}")
+    assert r.status_code == 200, r.text
+    assert r.json() == {"job_id": str(job_id), "scored": 3, "model_version": "7"}
+
+    monkeypatch.setattr(scoring, "load_model", lambda: (FakeModel(0.2), "8"))
+    r = await client.post(f"/api/predictions/score/{job_id}")
+    assert r.status_code == 200, r.text
+
+    preds = await predictions(job_id)
+    assert len(preds) == 3
+    assert {(p.score, p.model_version) for p in preds} == {(0.2, "8")}
+
+
+@pytest.mark.usefixtures("db")
+async def test_scoring_unknown_job_is_404(client: AsyncClient) -> None:
+    r = await client.post(f"/api/predictions/score/{uuid.uuid4()}")
+    assert r.status_code == 404
+
+
+@pytest.mark.usefixtures("db")
+async def test_scoring_unfinished_job_is_409(client: AsyncClient) -> None:
+    job_id = await make_job(JobStatus.running, n_variants=1)
+    r = await client.post(f"/api/predictions/score/{job_id}")
+    assert r.status_code == 409

api/tests/test_security.py

@@ -0,0 +1,51 @@
+import pytest
+from httpx import AsyncClient
+from pydantic import ValidationError
+
+from app.schemas import SampleCreate
+
+
+@pytest.mark.parametrize(
+    "uri",
+    [
+        "gs://bucket/dir/sample.vcf.gz",
+        "gs://my.bucket-1/a/b.bcf",
+        "gs://bucket/x.vcf",
+        "/data/example.vcf.gz",
+        "/data/giab/hg002.chr22.vcf.bgz",
+    ],
+)
+def test_vcf_uri_accepts_gcs_objects_and_files_under_the_data_root(uri: str) -> None:
+    assert SampleCreate(name="s", vcf_uri=uri).vcf_uri == uri
+
+
+@pytest.mark.parametrize(
+    "uri",
+    [
+        "-c/tmp/evil.config",          # would be parsed as a Nextflow option
+        "--outdir=/etc",
+        "/etc/passwd",                  # outside the data root
+        "/data/../etc/shadow.vcf",      # traversal out of it
+        "data/example.vcf.gz",          # relative: depends on the API's working directory
+        "gs://bucket/notes.txt",        # not a VCF/BCF
+        "https://example.com/x.vcf.gz",
+        "gs:///x.vcf.gz",
+        "/data/x.vcf.gz\n--foo",        # control characters
+        "",
+    ],
+)
+def test_vcf_uri_rejects_everything_else(uri: str) -> None:
+    with pytest.raises(ValidationError):
+        SampleCreate(name="s", vcf_uri=uri)
+
+
+async def test_bad_vcf_uri_is_422_at_the_api(client: AsyncClient) -> None:
+    r = await client.post("/api/samples", json={"name": "s", "vcf_uri": "/etc/passwd"})
+    assert r.status_code == 422
+
+
+async def test_cors_allows_only_configured_origins(client: AsyncClient) -> None:
+    allowed = await client.get("/health", headers={"Origin": "http://localhost:5173"})
+    assert allowed.headers.get("access-control-allow-origin") == "http://localhost:5173"
+    other = await client.get("/health", headers={"Origin": "https://evil.example"})
+    assert "access-control-allow-origin" not in other.headers

api/tests/test_variants.py

@@ -0,0 +1,46 @@
+import uuid
+
+import pytest
+from factories import seed_job
+from httpx import AsyncClient
+
+
+@pytest.mark.parametrize(
+    "paging", [{"limit": 0}, {"limit": -1}, {"limit": 501}, {"offset": -1}]
+)
+async def test_out_of_range_paging_is_422_not_500(client: AsyncClient, paging: dict) -> None:
+    r = await client.get("/api/variants", params={"job_id": str(uuid.uuid4()), **paging})
+    assert r.status_code == 422
+
+
+async def positions(client: AsyncClient, job_id: uuid.UUID) -> list[tuple[str, int, str]]:
+    r = await client.get("/api/variants", params={"job_id": str(job_id)})
+    assert r.status_code == 200, r.text
+    return [(v["chrom"], v["pos"], v["alt"]) for v in r.json()["items"]]
+
+
+@pytest.mark.usefixtures("db")
+async def test_chromosomes_sort_naturally(client: AsyncClient) -> None:
+    job_id = await seed_job([{"chrom": c} for c in ["10", "MT", "2", "X", "chr3", "1", "Y"]])
+    assert [c for c, _, _ in await positions(client, job_id)] == [
+        "1", "2", "chr3", "10", "X", "Y", "MT",
+    ]
+
+
+@pytest.mark.usefixtures("db")
+async def test_same_position_keeps_insertion_order_across_pages(client: AsyncClient) -> None:
+    # Split multiallelics share chrom/pos; without a tiebreak, pages can repeat or skip rows.
+    job_id = await seed_job([{"pos": 5, "alt": a} for a in "CGT"])
+    assert [a for _, _, a in await positions(client, job_id)] == ["C", "G", "T"]
+
+
+@pytest.mark.usefixtures("db")
+async def test_long_vep_strings_are_stored(client: AsyncClient) -> None:
+    clin_sig = ",".join(["conflicting_classifications_of_pathogenicity"] * 8)
+    consequence = (
+        "splice_region_variant&splice_polypyrimidine_tract_variant&intron_variant"
+        "&NMD_transcript_variant&non_coding_transcript_variant"
+    )
+    job_id = await seed_job([{"clinvar_sig": clin_sig, "consequence": consequence}])
+    [v] = (await client.get("/api/variants", params={"job_id": str(job_id)})).json()["items"]
+    assert (v["clinvar_sig"], v["consequence"]) == (clin_sig, consequence)

data/README.md

@@ -1,5 +1,8 @@
 # Test data
 
+`make data` fetches the demo slice automatically. Provenance, licences, citations and the
+evaluation plan live in [../docs/data.md](../docs/data.md).
+
 No patient data. Use public sources only:
 
 - ClinVar VCF (GRCh38): https://ftp.ncbi.nlm.nih.gov/pub/clinvar/vcf_GRCh38/
@@ -14,3 +17,27 @@ bcftools view -r 22 clinvar.vcf.gz | bcftools view -H | head -2000 > body.vcf
 (bcftools view -h clinvar.vcf.gz; cat body.vcf) | bgzip > example.vcf.gz
 tabix -p vcf example.vcf.gz
 ```
+
+Samples added in the UI must point at a `gs://bucket/object` or at a file under `/data`
+(`LOCAL_DATA_ROOT`), ending in `.vcf`, `.vcf.gz`, `.vcf.bgz` or `.bcf`.
+
+## VEP cache and plugins
+
+The pipeline runs VEP offline. Install the cache once (about 25 GB for GRCh38):
+
+```bash
+docker run --rm -v $PWD/pipeline/cache/vep:/cache ensemblorg/ensembl-vep:release_113.0 \
+  INSTALL.pl -a cf -s homo_sapiens -y GRCh38 -c /cache
+```
+
+CADD and AlphaMissense are optional; the model treats their scores as missing without them. To
+enable them, put the plugin modules and data files in one directory and pass `--vep_plugin_data`:
+
+```bash
+docker run --rm -v $PWD/pipeline/cache/plugins:/plugins ensemblorg/ensembl-vep:release_113.0 \
+  INSTALL.pl -a p -g CADD,AlphaMissense -r /plugins
+# then add whole_genome_SNVs.tsv.gz, gnomad.genomes.r4.0.indel.tsv.gz (CADD) and
+# AlphaMissense_hg38.tsv.gz, each with its .tbi index, to pipeline/cache/plugins
+```
+
+`pipeline/tests/data/tiny.vcf` is a synthetic three-record fixture used by CI's stub run.

docker-compose.yml

@@ -28,15 +28,20 @@ services:
       context: ./web
       target: build
     environment:
-      PUBLIC_API_URL: http://localhost:8000
+      PUBLIC_API_URL: http://localhost:8000/api
     ports: ["5173:5173"]
     depends_on: [api]
     volumes: ["./web:/app", "/app/node_modules"]
     command: npm run dev -- --host
 
   mlflow:
-    image: ghcr.io/mlflow/mlflow:v2.16.0
-    command: mlflow server --host 0.0.0.0 --backend-store-uri sqlite:///mlflow.db --default-artifact-root /mlruns
+    image: ghcr.io/mlflow/mlflow:v3.16.0
+    # --artifacts-destination makes the server proxy artifacts over HTTP; a bare local
+    # --default-artifact-root would hand clients a /mlruns path only this container can see.
+    command: >
+      mlflow server --host 0.0.0.0
+      --backend-store-uri sqlite:////mlruns/mlflow.db
+      --artifacts-destination /mlruns
     ports: ["5000:5000"]
     volumes: [mlruns:/mlruns]
 

docs/architecture.md

@@ -3,30 +3,59 @@
 ```mermaid
 flowchart LR
   U[Scientist] -->|browser| W[SvelteKit web]
-  W -->|REST| A[FastAPI]
+  W -->|REST /api| A[FastAPI]
   A --> P[(PostgreSQL / Cloud SQL)]
   A -->|publish vcf-uploaded| Q[Pub/Sub]
   Q --> E[Argo Events sensor]
-  E --> AW[Argo Workflow]
-  AW --> NF[Nextflow: bcftools norm, VEP, load_db]
-  NF -->|reads VCF| G[(GCS bucket)]
-  NF -->|writes variants| P
-  A -->|models:/rarelens-pathogenicity| M[MLflow registry]
-  T[ml/train.py on GKE, optional GPU] --> M
+  E --> AW[Argo Workflow: Nextflow driver]
+  AW -->|tasks| B[Google Batch: bcftools norm, VEP, load_db]
+  B -->|reads VCF, VEP cache| G[(GCS bucket)]
+  B -->|writes variants, marks job succeeded| P
+  AW -.->|exit handler marks job failed| P
+  A -->|models:/rarelens-pathogenicity@production| M[MLflow registry]
+  T[ml/train.py] --> M
   GH[GitHub Actions] -->|images via WIF| AR[Artifact Registry]
   GH -->|bumps overlay tags| R[(git: infra/k8s/overlays/gcp)]
   R --> CD[ArgoCD] --> K[GKE Autopilot]
 ```
 
+## Two deployment tracks
+
+The same images and the same pipeline, deployed two ways (`infra/terraform/variables.tf`):
+
+| | Serverless (default) | Kubernetes (`-var deploy_kubernetes=true`) |
+|---|---|---|
+| api, web | Cloud Run, scale to zero | Deployments behind an ingress |
+| dispatch | the API executes a Cloud Run job | Pub/Sub -> Argo Events -> Argo Workflow |
+| pipeline tasks | Google Batch | Google Batch |
+| `/api` routing | the web service proxies it | the ingress routes it |
+| idle cost | ~£1/month | ~£130+/month |
+
+`app.services.events.launch()` picks the dispatch backend from configuration: a Cloud Run job when
+`CLOUDRUN_JOB` is set, Pub/Sub when `PUBSUB_TOPIC` is, and a local Nextflow process otherwise.
+See [cloud.md](cloud.md) for why the serverless one is the default.
+
 ## Why these choices
 
 **One monorepo.** The four components share a schema (`variants` table, feature columns) and the
 point of the exercise is to see them evolve together. Separate repos would hide the coupling.
 
 **Nextflow for the science, Argo Workflows for the trigger.** Nextflow is the lingua franca for
-bioinformatics pipelines and has a native Kubernetes executor. Argo is what the platform team
-already runs. So Argo owns *when* a pipeline runs; Nextflow owns *what* it does. The API never
-talks to Kubernetes directly; it publishes an event and gets on with its life.
+bioinformatics pipelines. Argo is what the platform team already runs. So Argo owns *when* a
+pipeline runs; Nextflow owns *what* it does. The API never talks to Kubernetes directly; it
+publishes an event and gets on with its life. The Nextflow driver runs in the Argo pod and sends
+each task to Google Batch: a `gs://` work directory needs an executor that stages through GCS
+(Nextflow's Kubernetes executor needs a shared ReadWriteMany volume instead).
+
+**Job lifecycle.** The API creates a job as `running` once the pipeline is dispatched, or `failed`
+with the reason in `jobs.log` when dispatch is impossible. The loader marks it `succeeded` in the
+same transaction that stores the variants. Anything else (Nextflow error, eviction) is caught by
+the Argo exit handler, or locally by the API watching the Nextflow process, and marked `failed`,
+so the UI never polls a dead job.
+
+**Variant identity.** NORMALISE sets each VCF ID to `CHROM_POS_REF_ALT`; VEP echoes it as
+`Uploaded_variation` and the loader takes exact VCF alleles from it, because VEP's own
+`Location`/`Allele` columns trim indel alleles.
 
 **FastAPI + Pydantic v2 + SQLAlchemy 2.0 async.** Typed at both boundaries: request/response models
 and ORM models are separate on purpose so the database can change without breaking the frontend
@@ -35,15 +64,21 @@ because the pipeline container should not import the API.
 
 **SvelteKit.** Small runtime, no virtual DOM, and Svelte 5 runes make server-driven state simple.
 The UI has exactly two pages; the goal is a table a scientist actually wants to filter, not a dashboard.
+`PUBLIC_API_URL` is read at runtime, so one image works behind the ingress (`/api`) and elsewhere.
 
 **GKE Autopilot + Cloud SQL, not self-managed.** The lab is about the platform patterns (Workload
-Identity, GitOps, Kustomize overlays, GPU node selection), not about running etcd.
+Identity, GitOps, Kustomize overlays, private networking), not about running etcd. Cloud SQL has only
+a private IP; the API reaches it through a Cloud SQL Proxy sidecar, pipeline tasks directly in the VPC.
+Database URLs live in Secret Manager.
 
 **GitOps.** CI builds and tests; it never runs `kubectl apply`. It edits image tags in the `gcp` overlay
-and ArgoCD reconciles. Rollback is `git revert`.
+and ArgoCD reconciles. Rollback is `git revert`. Migrations run in an init container under a Postgres
+advisory lock, so replicas starting together migrate once.
 
-**MLflow registry as the model contract.** The API loads `models:/rarelens-pathogenicity/Production`.
-Training writes there; serving reads there. Feature engineering lives in one module that both sides import.
+**MLflow registry as the model contract.** The API loads whichever version the `production` alias
+points at and records that version on every prediction. The registered model is a pyfunc that owns its
+feature engineering (`rarelens_ml.features` ships inside it) and returns P(pathogenic), so serving only
+sends raw columns and cannot drift from training.
 
 ## What is deliberately missing
 

docs/cloud.md

@@ -0,0 +1,112 @@
+# Cloud choice: Google Cloud, with a documented AWS escape hatch
+
+Decided 2026-09-12. Scope: `infra/terraform/`, `infra/k8s/`, `pipeline/nextflow.config`.
+
+## Decision
+
+rarelens deploys to **Google Cloud**. AWS was the serious alternative, and it is genuinely better
+on two points (below), but not by enough to justify rebuilding an estate that already works.
+
+## Why Google Cloud
+
+| Reason | Detail |
+|---|---|
+| One Kubernetes control plane is effectively free | GKE's free tier gives $74.40/month in credits per billing account, which covers one Autopilot or zonal cluster. EKS charges $0.10/hour per cluster (~$73/month) with no equivalent credit. For a self-funded lab this is the largest fixed monthly difference. |
+| The executor question is already settled here | Google retired Cloud Life Sciences on 8 July 2025; Batch is its successor, and Nextflow upstream moved to Google Batch in April 2025. `pipeline/nextflow.config` uses `google-batch`, which is the supported path rather than a legacy one. |
+| The estate exists and is verified | Terraform (custom VPC, private Cloud SQL, Workload Identity, Secret Manager, Batch IAM), Kustomize overlays, Argo Workflows/Events and CI all render, validate and pass tests today. Rebuilding this on AWS costs 1–2 weeks and mostly repeats learning already banked. |
+| No data lock-in | Every dataset the platform uses is readable from either cloud (see [data.md](data.md)): gnomAD publishes to GCP, AWS and Azure; GIAB and 1000 Genomes are open on AWS and NCBI; ClinVar is a plain NCBI download. |
+
+## What AWS is genuinely better at
+
+- **Managed Nextflow.** AWS HealthOmics runs Nextflow (up to 26.04), WDL and CWL as a managed
+  service, and is available in London (`eu-west-2`). GCP has no equivalent: you operate the
+  driver yourself, which is exactly what `infra/argo-workflows/annotate.yaml` does.
+- **UK life-sciences gravity.** The UK Biobank Research Analysis Platform is DNAnexus running on
+  AWS, hosted in the UK. If the aim is to mirror what Cambridge-area employers run day to day,
+  AWS is the more common answer.
+
+## When to revisit this
+
+Move to AWS if any of these becomes true:
+
+- The lab wants a managed pipeline runner instead of an Argo + Batch driver it maintains.
+- Matching an AWS-first employer's stack matters more than the two weeks it costs.
+- The shape changes: several clusters, or enough managed-service spend that one free control
+  plane stops being material.
+
+## Running this on a hobby budget
+
+The cloud is not the cost driver; the always-on shape is. Estimates below are list price, and
+rounded — treat them as orders of magnitude, not quotes.
+
+### What the Kubernetes estate costs at rest
+
+GKE Autopilot bills what pods *request*, not what they use, with a per-pod floor (250m vCPU /
+512 MiB). The free tier credit covers the cluster fee only, not pod-hours.
+
+| Always-on | Requests | ~Monthly (us-central1 rates: $0.0445/vCPU-h, $0.0049/GiB-h) |
+|---|---|---|
+| api + web (2 replicas each, incl. Cloud SQL proxy sidecar) | ~1.2 vCPU, ~2.3 GiB | ~$47 |
+| ArgoCD, Argo Workflows, Argo Events + NATS EventBus (~11 pods at the floor) | ~2.8 vCPU, ~5.5 GiB | ~$110 |
+| Cloud SQL `db-f1-micro` | — | ~$8–12 |
+| **Total** | | **~$165–170, London a bit more** |
+
+That is the wrong shape for a portfolio that is idle 99% of the time.
+
+### The shape that costs ~£1/month
+
+This is what `terraform apply` builds by default (`deploy_kubernetes` and `deploy_cloud_sql` are
+both `false`). Kubernetes becomes something you switch on to show, not something you rent:
+
+| Piece | Service | Idle cost |
+|---|---|---|
+| api, web | Cloud Run, `min-instances=0`, `max_instances` capped | £0 — Always Free covers 2M requests, 180k vCPU-s, 360k GiB-s per month |
+| Nextflow driver | Cloud Run **job**, started by the API through the Jobs API (`roles/run.jobsExecutorWithOverrides`, one job only) | £0 idle, pennies per run |
+| Pipeline tasks | Google Batch on **Spot** VMs | £0 idle; a chr22 VEP run is a few pence |
+| Database | Neon free tier (scale-to-zero, 0.5 GB) via `TF_VAR_database_url`, or `-var deploy_cloud_sql=true` | £0 (or ~$8–12) |
+| Model | pyfunc artifact loaded straight from GCS (`MODEL_URI`), no MLflow server running | £0 |
+| Storage | GCS + Artifact Registry | ~£1 (VEP cache dominates; Nearline halves it) |
+
+Trade-offs worth knowing: Cloud Run cold starts add 1–3 s to the first request after idle; the
+Cloud Run path drops Pub/Sub, Argo Events and Argo Workflows from the critical path (the API calls
+the Jobs API directly); and 0.5 GB of Neon does not fit a whole chromosome once `annotations`
+stores the full VEP record — demo a gene panel, or store only the annotation keys the UI uses.
+
+Only the web service needs to be public: it serves the UI and proxies `/api` to the API service
+(`web/src/routes/api/[...path]`), which is the same shape the ingress gives the Kubernetes track,
+so the frontend code is identical either way.
+
+### Keep the Kubernetes story, stop paying rent for it
+
+`infra/k8s/` and `infra/argo-workflows/` stay in the repo and stay deployable. Bring the estate up
+with `terraform apply` for an interview or a recording (roughly $0.25/hour while running, so a
+two-hour demo is small change), then `terraform destroy -var deletion_protection=false`. `make kind`
+runs the same manifests locally for free.
+
+### Guardrails
+
+- A billing budget with alerts at £5/£10, before anything else.
+- `max-instances` on every Cloud Run service: scale-to-zero protects the floor, a cap protects the ceiling.
+- Spot VMs for Batch, and the existing 30-day lifecycle rule on `work/` in the bucket.
+- New accounts get $300 of Google Cloud credit for 90 days, which covers the experimenting phase.
+
+### One more reason not to switch to AWS
+
+AWS replaced its 12-month free tier on 15 July 2025 with credits ($100, up to $200) on a Free plan
+that closes after six months or when the credits run out. Google's Always Free quotas, including
+Cloud Run's, are permanent. For a demo meant to stay reachable indefinitely at near-zero cost,
+that difference matters more than any feature comparison above.
+
+## The escape hatch
+
+Nextflow is the portability layer: executors are configuration, not code. An AWS run needs a new
+profile in `pipeline/nextflow.config` (`process.executor = 'awsbatch'`, an S3 work directory and a
+job queue), or a HealthOmics workflow definition. The processes themselves do not change. Keeping
+`pipeline/bin/` cloud-agnostic (the scripts read `DATABASE_URL` from the environment, never from a
+command line) is what keeps that true.
+
+Sources: [Migrate to Batch from Cloud Life Sciences](https://docs.cloud.google.com/batch/docs/migrate-to-batch-from-cloud-life-sciences),
+[GKE pricing](https://cloud.google.com/kubernetes-engine/pricing),
+[HealthOmics supported languages](https://docs.aws.amazon.com/omics/latest/dev/workflows-supported-languages.html),
+[HealthOmics Nextflow 26.04](https://aws.amazon.com/about-aws/whats-new/2026/06/aws-healthomics-nextflow-version-26-04/),
+[UK Biobank Research Analysis Platform](https://www.ukbiobank.ac.uk/use-our-data/research-analysis-platform/).

docs/data.md

@@ -0,0 +1,74 @@
+# Data: what rarelens actually runs on
+
+Everything below is public, peer-reviewed and consented for open redistribution. No patient data,
+no data access agreement, nothing that needs an application. These are the references to quote
+when showing the platform to someone.
+
+Citations were verified against [PubMed](https://pubmed.ncbi.nlm.nih.gov/); each row links its DOI.
+
+## The demo slice
+
+`make data` fetches two real files, chromosome 22 only (roughly 100 MB, minutes rather than hours):
+
+| File | What it is | Role |
+|---|---|---|
+| `data/example.vcf.gz` | GIAB HG002 (NA24385) v4.2.1 benchmark calls, GRCh38, chr22 | the sample a scientist annotates |
+| `data/clinvar.chr22.vcf.gz` | ClinVar, GRCh38, chr22 | training labels, and the ClinVar column in the UI |
+
+HG002 is the NIST Genome in a Bottle Ashkenazi son, recruited through the Personal Genome Project,
+which consents participants to unrestricted public release. It is the reference genome the field
+benchmarks variant callers against, so it is both realistic and unambiguously shareable.
+
+## Datasets
+
+| Dataset | Used for | Access | Terms | Citation |
+|---|---|---|---|---|
+| **ClinVar** (GRCh38) | pathogenic/benign labels, ClinVar column | `ftp.ncbi.nlm.nih.gov/pub/clinvar/vcf_GRCh38/` | NCBI public domain | Landrum et al., *Nucleic Acids Res* 48(D1):D835–D844, 2020. [10.1093/nar/gkz972](https://doi.org/10.1093/nar/gkz972) |
+| **Genome in a Bottle** HG002 v4.2.1 | the demo sample | `ftp-trace.ncbi.nlm.nih.gov/ReferenceSamples/giab/`, `s3://giab` | open, no use restriction | Zook et al., *Nat Biotechnol* 37:561–566, 2019. [10.1038/s41587-019-0074-6](https://doi.org/10.1038/s41587-019-0074-6) |
+| **gnomAD** v4 | allele frequency feature and filter | `gs://gcp-public-data--gnomad`, `s3://gnomad-public-us-east-1` | free use, no restriction | Chen et al., *Nature* 625:92–100, 2024. [10.1038/s41586-023-06045-0](https://doi.org/10.1038/s41586-023-06045-0); Karczewski et al., *Nature* 581:434–443, 2020. [10.1038/s41586-020-2308-7](https://doi.org/10.1038/s41586-020-2308-7) |
+| **1000 Genomes** 30x | optional cohort/trio data | EBI FTP, `s3://1000genomes` | fully open, no access restriction | Byrska-Bishop et al., *Cell* 185(18):3426–3440.e19, 2022. [10.1016/j.cell.2022.08.004](https://doi.org/10.1016/j.cell.2022.08.004) |
+| **MANE Select** | one transcript per gene, if transcript choice ever matters | Ensembl/RefSeq | open | Morales et al., *Nature* 604:310–315, 2022. [10.1038/s41586-022-04558-8](https://doi.org/10.1038/s41586-022-04558-8) |
+
+## Tools and scores
+
+| Tool | Role | Terms | Citation |
+|---|---|---|---|
+| **Ensembl VEP** 113 | annotation (`pipeline/modules/vep.nf`) | Apache 2.0 | McLaren et al., *Genome Biol* 17:122, 2016. [10.1186/s13059-016-0974-4](https://doi.org/10.1186/s13059-016-0974-4) |
+| **CADD** | `cadd_phred` feature | free for non-commercial use; commercial licence required | Rentzsch et al., *Nucleic Acids Res* 47(D1):D886–D894, 2019. [10.1093/nar/gky1016](https://doi.org/10.1093/nar/gky1016); Schubach et al., *Nucleic Acids Res* 52(D1), 2024. [10.1093/nar/gkad989](https://doi.org/10.1093/nar/gkad989) |
+| **AlphaMissense** | `am_pathogenicity` feature | predictions moved to CC BY 4.0 in March 2024 (originally CC BY-NC-SA) | Cheng et al., *Science* 381:eadg7492, 2023. [10.1126/science.adg7492](https://doi.org/10.1126/science.adg7492) |
+
+Neither score is required: `rarelens_ml.features` treats a missing CADD or AlphaMissense value as
+NaN and LightGBM handles it, so the pipeline runs without the plugin data.
+
+## Evaluating the model honestly
+
+The model trains on ClinVar labels and is scored on ClinVar-labelled variants, which is exactly
+where published benchmarks go wrong. What to do about it:
+
+1. **Never let the label into the features.** `CLIN_SIG` is excluded by construction; `clinvar_sig`
+   is stored for display only (`rarelens_ml/features.py` lists the five feature columns).
+2. **Split by gene, not by variant.** Random splits put variants from the same gene on both sides,
+   and a model can then score a gene rather than a variant. Grimm et al. showed this inflates
+   reported accuracy for exactly this class of tool: *Hum Mutat* 36:513–523, 2015.
+   [10.1002/humu.22768](https://doi.org/10.1002/humu.22768)
+3. **Prefer a time-based holdout.** Train on an older ClinVar release (monthly archives live under
+   `vcf_GRCh38/archive_2.0/`) and test only on variants classified after that date. This is the
+   closest thing to a prospective evaluation available without new patients.
+4. **Filter labels by review status.** ClinVar's `CLNREVSTAT` marks how much evidence backs a
+   classification; two-star and above ("multiple submitters, no conflicts") is the usual bar.
+   *Known gap*: VEP's `CLIN_SIG` does not carry review status, so this needs ClinVar annotated as a
+   custom field before it can be enforced.
+5. **Report against published baselines on the same rows.** CADD PHRED and AlphaMissense are
+   already columns in the variant table, so AUROC and AUPRC for the model next to those two, with
+   the variant count, is a fair comparison rather than a number with nothing to beat.
+6. **Report AUPRC, not just AUROC.** Pathogenic variants are the minority class; AUROC flatters.
+
+## What must not be claimed
+
+ACMG/AMP treats computational predictions as *supporting* evidence only, never sufficient on their
+own for classifying a variant (Richards et al., *Genet Med* 17:405–424, 2015.
+[10.1038/gim.2015.30](https://doi.org/10.1038/gim.2015.30)). rarelens is a learning platform on
+public data: it makes no diagnostic claim, and the UI shows a score next to the evidence rather
+than a verdict. For what a real diagnostic pipeline looks like end to end, see the 100,000 Genomes
+Project rare-disease pilot: Smedley et al., *N Engl J Med* 385:1868–1880, 2021.
+[10.1056/NEJMoa2035790](https://doi.org/10.1056/NEJMoa2035790)

infra/argo-workflows/annotate.yaml

@@ -1,37 +1,54 @@
-# Triggered by an Argo Events sensor listening on the Pub/Sub topic "vcf-uploaded".
+# Submitted by the Argo Events sensor in events.yaml for each "vcf-uploaded" Pub/Sub message.
+# The Nextflow driver runs here; its tasks run on Google Batch (see the gcp profile in
+# pipeline/nextflow.config). Image names are rewritten by the gcp overlay and bumped by CI.
 apiVersion: argoproj.io/v1alpha1
 kind: WorkflowTemplate
-metadata: { name: annotate-vcf, namespace: rarelens }
+metadata: { name: annotate-vcf }
 spec:
+  serviceAccountName: rarelens-pipeline
   entrypoint: nextflow
+  onExit: exit-handler
   arguments:
     parameters:
       - { name: job_id }
       - { name: vcf_uri }
+      - { name: assembly, value: GRCh38 }
   templates:
     - name: nextflow
-      inputs:
-        parameters: [{ name: job_id }, { name: vcf_uri }]
-      serviceAccountName: rarelens-pipeline
       container:
-        image: europe-west2-docker.pkg.dev/PROJECT/rarelens/pipeline:latest
-        command: [nextflow]
+        image: rarelens/pipeline
         args:
           - run
           - /pipeline/main.nf
           - -profile
           - gcp
           - --vcf
-          - "{{inputs.parameters.vcf_uri}}"
+          - "{{workflow.parameters.vcf_uri}}"
           - --job_id
-          - "{{inputs.parameters.job_id}}"
-          - --db_url
-          - "$(DATABASE_URL)"
-        envFrom: [{ secretRef: { name: api-secrets } }]
-        resources: { requests: { cpu: "2", memory: 4Gi } }
-    - name: score
-      # Optional GPU step for the deep-learning baseline; Autopilot schedules on an L4 node.
-      nodeSelector: { cloud.google.com/gke-accelerator: nvidia-l4 }
+          - "{{workflow.parameters.job_id}}"
+          - --assembly
+          - "{{workflow.parameters.assembly}}"
+        # GCP_PROJECT / GCS_BUCKET / GCP_REGION feed params in nextflow.config.
+        envFrom: [{ configMapRef: { name: pipeline-config } }]
+        resources: { requests: { cpu: "1", memory: 2Gi } }
+
+    # The loader marks success; anything else (Nextflow error, OOM, eviction) is marked here
+    # so the UI never polls a dead job forever.
+    - name: exit-handler
+      steps:
+        - - name: mark-failed
+            template: mark-failed
+            when: "{{workflow.status}} != Succeeded"
+    - name: mark-failed
       container:
-        image: europe-west2-docker.pkg.dev/PROJECT/rarelens/ml:latest
-        resources: { limits: { nvidia.com/gpu: 1 } }
+        image: rarelens/loader
+        command: [set_job_status.py]
+        args:
+          - --job-id
+          - "{{workflow.parameters.job_id}}"
+          - --status
+          - failed
+          - --log
+          - "Argo workflow {{workflow.name}} ended {{workflow.status}}"
+        envFrom: [{ secretRef: { name: pipeline-secrets } }]
+        resources: { requests: { cpu: 100m, memory: 256Mi } }

infra/argo-workflows/events.yaml

@@ -0,0 +1,46 @@
+# Pub/Sub "vcf-uploaded" (published by the API) -> Argo Workflow from the annotate-vcf template.
+apiVersion: argoproj.io/v1alpha1
+kind: EventSource
+metadata: { name: vcf-uploaded }
+spec:
+  template:
+    # Workload Identity: roles/pubsub.subscriber on the subscription (infra/terraform/iam.tf).
+    serviceAccountName: rarelens-pipeline
+  pubSub:
+    vcf-uploaded:
+      projectID: __GCP_PROJECT__
+      subscriptionID: vcf-uploaded-argo  # created by Terraform (infra/terraform/pubsub.tf)
+      jsonBody: true
+---
+apiVersion: argoproj.io/v1alpha1
+kind: Sensor
+metadata: { name: annotate-vcf }
+spec:
+  template:
+    serviceAccountName: argo-events-sensor
+  dependencies:
+    - { name: vcf, eventSourceName: vcf-uploaded, eventName: vcf-uploaded }
+  triggers:
+    - template:
+        name: submit-annotate-vcf
+        argoWorkflow:
+          operation: submit
+          source:
+            resource:
+              apiVersion: argoproj.io/v1alpha1
+              kind: Workflow
+              metadata: { generateName: annotate-vcf- }
+              spec:
+                workflowTemplateRef: { name: annotate-vcf }
+                arguments:
+                  parameters:
+                    - { name: job_id }
+                    - { name: vcf_uri }
+                    - { name: assembly }
+          parameters:
+            - src: { dependencyName: vcf, dataKey: body.job_id }
+              dest: spec.arguments.parameters.0.value
+            - src: { dependencyName: vcf, dataKey: body.vcf_uri }
+              dest: spec.arguments.parameters.1.value
+            - src: { dependencyName: vcf, dataKey: body.assembly, value: GRCh38 }
+              dest: spec.arguments.parameters.2.value

infra/argo-workflows/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+# Requires Argo Workflows and Argo Events (with the default EventBus) installed in the cluster.
+namespace: rarelens
+resources: [rbac.yaml, annotate.yaml, events.yaml]

infra/argo-workflows/rbac.yaml

@@ -0,0 +1,41 @@
+# Runs the annotate-vcf workflow pods and the Pub/Sub EventSource. Bound to the rarelens-pipeline
+# Google service account through Workload Identity (annotation added by the gcp overlay).
+apiVersion: v1
+kind: ServiceAccount
+metadata: { name: rarelens-pipeline }
+---
+# Argo's executor reports step outputs through WorkflowTaskResults.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata: { name: argo-executor }
+rules:
+  - apiGroups: [argoproj.io]
+    resources: [workflowtaskresults]
+    verbs: [create, patch]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata: { name: rarelens-pipeline-argo-executor }
+roleRef: { apiGroup: rbac.authorization.k8s.io, kind: Role, name: argo-executor }
+subjects: [{ kind: ServiceAccount, name: rarelens-pipeline }]
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata: { name: argo-events-sensor }
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata: { name: workflow-submitter }
+rules:
+  - apiGroups: [argoproj.io]
+    resources: [workflows]
+    verbs: [create]
+  - apiGroups: [argoproj.io]
+    resources: [workflowtemplates]
+    verbs: [get, list]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata: { name: argo-events-sensor-workflow-submitter }
+roleRef: { apiGroup: rbac.authorization.k8s.io, kind: Role, name: workflow-submitter }
+subjects: [{ kind: ServiceAccount, name: argo-events-sensor }]

infra/k8s/base/api.yaml

@@ -8,13 +8,24 @@ spec:
     metadata: { labels: { app: api } }
     spec:
       serviceAccountName: rarelens-api
+      initContainers:
+        # alembic/env.py holds a Postgres advisory lock, so replicas starting together migrate once.
+        - name: migrate
+          image: rarelens/api
+          command: [alembic, upgrade, head]
+          envFrom: [{ secretRef: { name: api-secrets } }]
+          resources: { requests: { cpu: 100m, memory: 256Mi }, limits: { cpu: 500m, memory: 512Mi } }
       containers:
         - name: api
           image: rarelens/api
           ports: [{ containerPort: 8000 }]
-          envFrom: [{ secretRef: { name: api-secrets } }]
+          envFrom:
+            - secretRef: { name: api-secrets }
+            - configMapRef: { name: api-config }
           readinessProbe: { httpGet: { path: /health, port: 8000 }, periodSeconds: 5 }
+          livenessProbe: { httpGet: { path: /health, port: 8000 }, periodSeconds: 10, failureThreshold: 6 }
           resources: { requests: { cpu: 250m, memory: 512Mi }, limits: { cpu: "1", memory: 1Gi } }
+          securityContext: { allowPrivilegeEscalation: false }
 ---
 apiVersion: v1
 kind: Service

infra/k8s/base/web.yaml

@@ -11,8 +11,12 @@ spec:
         - name: web
           image: rarelens/web
           ports: [{ containerPort: 3000 }]
+          # Read at runtime ($env/dynamic/public); the ingress routes /api to the API service.
           env: [{ name: PUBLIC_API_URL, value: /api }]
+          readinessProbe: { httpGet: { path: /, port: 3000 }, periodSeconds: 5 }
+          livenessProbe: { httpGet: { path: /, port: 3000 }, periodSeconds: 10, failureThreshold: 6 }
           resources: { requests: { cpu: 100m, memory: 128Mi }, limits: { cpu: 500m, memory: 256Mi } }
+          securityContext: { allowPrivilegeEscalation: false }
 ---
 apiVersion: v1
 kind: Service

infra/k8s/overlays/gcp/kustomization.yaml

@@ -1,21 +1,53 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
-resources: [../../base]
+# Project-specific values use the __GCP_PROJECT__ placeholder: run `make gcp-configure PROJECT=<id>`
+# once and commit. Image tags are then bumped by .github/workflows/deploy.yml after each green CI.
+# Generated ConfigMaps must share the Deployments' namespace, or their hashed names are not
+# propagated to envFrom references.
+namespace: rarelens
+resources: [../../base, ../../../argo-workflows]
+configurations: [kustomizeconfig.yaml]
 images:
-  - { name: rarelens/api, newName: europe-west2-docker.pkg.dev/PROJECT/rarelens/api, newTag: latest }
-  - { name: rarelens/web, newName: europe-west2-docker.pkg.dev/PROJECT/rarelens/web, newTag: latest }
+  - { name: rarelens/api, newName: europe-west2-docker.pkg.dev/__GCP_PROJECT__/rarelens/api, newTag: latest }
+  - { name: rarelens/web, newName: europe-west2-docker.pkg.dev/__GCP_PROJECT__/rarelens/web, newTag: latest }
+  - { name: rarelens/pipeline, newName: europe-west2-docker.pkg.dev/__GCP_PROJECT__/rarelens/pipeline, newTag: latest }
+  - { name: rarelens/loader, newName: europe-west2-docker.pkg.dev/__GCP_PROJECT__/rarelens/loader, newTag: latest }
+configMapGenerator:
+  - name: api-config
+    literals: [PUBSUB_TOPIC=vcf-uploaded, GCP_PROJECT=__GCP_PROJECT__]
+  - name: pipeline-config
+    # Referenced by name from the Argo WorkflowTemplate, which kustomize does not rewrite.
+    options: { disableNameSuffixHash: true }
+    literals: [GCP_PROJECT=__GCP_PROJECT__, GCP_REGION=europe-west2, GCS_BUCKET=__GCP_PROJECT__-rarelens-data]
+# api-secrets and pipeline-secrets come from Secret Manager: `make gcp-secrets PROJECT=<id>`.
 patches:
   - target: { kind: ServiceAccount, name: rarelens-api }
     patch: |
       - op: add
         path: /metadata/annotations
-        value: { iam.gke.io/gcp-service-account: [email protected] }
+        value: { iam.gke.io/gcp-service-account: rarelens-api@__GCP_PROJECT__.iam.gserviceaccount.com }
+  - target: { kind: ServiceAccount, name: rarelens-pipeline }
+    patch: |
+      - op: add
+        path: /metadata/annotations
+        value: { iam.gke.io/gcp-service-account: rarelens-pipeline@__GCP_PROJECT__.iam.gserviceaccount.com }
   - target: { kind: Deployment, name: api }
     patch: |
       - op: add
-        path: /spec/template/spec/containers/-
+        path: /spec/template/spec/initContainers/0
         value:
           name: cloud-sql-proxy
           image: gcr.io/cloud-sql-connectors/cloud-sql-proxy:2.13.0
-          args: ["--structured-logs", "--port=5432", "PROJECT:europe-west2:rarelens-pg"]
-          securityContext: { runAsNonRoot: true }
+          # Native sidecar: starts (and passes its startup probe) before the migrate init
+          # container, then keeps running alongside the API.
+          restartPolicy: Always
+          args:
+            - --structured-logs
+            - --private-ip
+            - --port=5432
+            - --health-check
+            - --http-address=0.0.0.0
+            - __GCP_PROJECT__:europe-west2:rarelens-pg
+          startupProbe: { httpGet: { path: /startup, port: 9090 }, periodSeconds: 1, failureThreshold: 60 }
+          securityContext: { runAsNonRoot: true, allowPrivilegeEscalation: false }
+          resources: { requests: { cpu: 100m, memory: 128Mi }, limits: { cpu: 500m, memory: 256Mi } }

infra/k8s/overlays/gcp/kustomizeconfig.yaml

@@ -0,0 +1,4 @@
+# Teach the `images:` transformer where Argo WorkflowTemplates keep their images.
+images:
+  - path: spec/templates/container/image
+    kind: WorkflowTemplate

infra/k8s/overlays/local/kustomization.yaml

@@ -1,5 +1,8 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
+# `make kind` builds and loads the dev images, then applies this overlay. There is no ingress in
+# kind: port-forward svc/web to 8080 and svc/api to 8000 (the UI calls the API directly).
+namespace: rarelens  # generated Secret/ConfigMap names only propagate within one namespace
 resources: [../../base, postgres.yaml]
 images:
   - { name: rarelens/api, newName: rarelens-api, newTag: dev }
@@ -7,3 +10,17 @@ images:
 secretGenerator:
   - name: api-secrets
     literals: [DATABASE_URL=postgresql+asyncpg://rarelens:rarelens@postgres:5432/rarelens]
+configMapGenerator:
+  - name: api-config
+    literals: ['CORS_ORIGINS=["http://localhost:8080"]']
+patches:
+  - patch: |
+      apiVersion: apps/v1
+      kind: Deployment
+      metadata: { name: web }
+      spec:
+        template:
+          spec:
+            containers:
+              - name: web
+                env: [{ name: PUBLIC_API_URL, value: "http://localhost:8000/api" }]

infra/k8s/overlays/local/postgres.yaml

@@ -13,6 +13,8 @@ spec:
             - { name: POSTGRES_USER, value: rarelens }
             - { name: POSTGRES_PASSWORD, value: rarelens }
             - { name: POSTGRES_DB, value: rarelens }
+          readinessProbe: { exec: { command: [pg_isready, -U, rarelens] }, periodSeconds: 5 }
+          resources: { requests: { cpu: 100m, memory: 256Mi }, limits: { cpu: 500m, memory: 512Mi } }
 ---
 apiVersion: v1
 kind: Service

infra/terraform/apis.tf

@@ -0,0 +1,18 @@
+resource "google_project_service" "enabled" {
+  for_each = toset([
+    "artifactregistry.googleapis.com",
+    "batch.googleapis.com",
+    "compute.googleapis.com",
+    "container.googleapis.com",
+    "iam.googleapis.com",
+    "iamcredentials.googleapis.com",
+    "logging.googleapis.com",
+    "pubsub.googleapis.com",
+    "secretmanager.googleapis.com",
+    "servicenetworking.googleapis.com",
+    "sqladmin.googleapis.com",
+    "sts.googleapis.com",
+  ])
+  service            = each.value
+  disable_on_destroy = false
+}

infra/terraform/cloudrun.tf

@@ -0,0 +1,178 @@
+# The serverless track: scale-to-zero services and an on-demand pipeline driver.
+# Idle cost is storage only; see docs/cloud.md.
+
+resource "google_cloud_run_v2_service" "api" {
+  name                = "rarelens-api"
+  location            = var.region
+  deletion_protection = false
+  ingress             = "INGRESS_TRAFFIC_ALL"
+
+  template {
+    service_account = google_service_account.api.email
+    scaling {
+      min_instance_count = 0 # nothing runs, and nothing is billed, between visits
+      max_instance_count = var.max_instances
+    }
+    containers {
+      image = "${local.registry}/api:${var.image_tag}"
+      ports { container_port = 8000 }
+      resources {
+        limits            = { cpu = "1", memory = "1Gi" }
+        cpu_idle          = true # bill CPU only while a request is in flight
+        startup_cpu_boost = true
+      }
+      env {
+        name = "DATABASE_URL"
+        value_source {
+          secret_key_ref {
+            secret  = google_secret_manager_secret.api_database_url.secret_id
+            version = "latest"
+          }
+        }
+      }
+      env {
+        name  = "CLOUDRUN_JOB"
+        value = google_cloud_run_v2_job.nextflow.name
+      }
+      env {
+        name  = "GCP_PROJECT"
+        value = var.project
+      }
+      env {
+        name  = "GCP_REGION"
+        value = var.region
+      }
+      env {
+        name  = "GCS_BUCKET"
+        value = google_storage_bucket.data.name
+      }
+      env {
+        name  = "MODEL_URI"
+        value = var.model_uri
+      }
+    }
+  }
+  depends_on = [google_secret_manager_secret_version.api_database_url]
+}
+
+resource "google_cloud_run_v2_service" "web" {
+  name                = "rarelens-web"
+  location            = var.region
+  deletion_protection = false
+  ingress             = "INGRESS_TRAFFIC_ALL"
+
+  template {
+    scaling {
+      min_instance_count = 0
+      max_instance_count = var.max_instances
+    }
+    containers {
+      image = "${local.registry}/web:${var.image_tag}"
+      ports { container_port = 3000 }
+      resources {
+        limits            = { cpu = "1", memory = "512Mi" }
+        cpu_idle          = true
+        startup_cpu_boost = true
+      }
+      # The browser calls /api on this origin; src/routes/api/[...path] forwards it, so there is
+      # one public URL and no CORS, exactly as the ingress arranges in the Kubernetes track.
+      env {
+        name  = "PUBLIC_API_URL"
+        value = "/api"
+      }
+      env {
+        name  = "API_INTERNAL_URL"
+        value = google_cloud_run_v2_service.api.uri
+      }
+      # adapter-node sits behind Cloud Run's proxy; derive the origin from the forwarded headers.
+      env {
+        name  = "PROTOCOL_HEADER"
+        value = "x-forwarded-proto"
+      }
+      env {
+        name  = "HOST_HEADER"
+        value = "x-forwarded-host"
+      }
+    }
+  }
+}
+
+# The Nextflow driver. Started per annotation by the API (overriding the container args); the
+# pipeline's own tasks then run on Google Batch (the gcp profile in pipeline/nextflow.config).
+resource "google_cloud_run_v2_job" "nextflow" {
+  name                = "rarelens-nextflow"
+  location            = var.region
+  deletion_protection = false
+
+  template {
+    task_count = 1
+    template {
+      service_account = google_service_account.pipeline.email
+      max_retries     = 0
+      timeout         = "7200s"
+      containers {
+        image = "${local.registry}/pipeline:${var.image_tag}"
+        args  = ["-version"] # replaced on every execution by the API's overrides
+        resources {
+          limits = { cpu = "1", memory = "2Gi" }
+        }
+        env {
+          name  = "GCP_PROJECT"
+          value = var.project
+        }
+        env {
+          name  = "GCP_REGION"
+          value = var.region
+        }
+        env {
+          name  = "GCS_BUCKET"
+          value = google_storage_bucket.data.name
+        }
+        env {
+          name  = "NXF_ANSI_LOG"
+          value = "false"
+        }
+      }
+    }
+  }
+}
+
+# Anyone can open the UI and the API. There is no authentication by design (docs/architecture.md);
+# max_instances and a billing budget are what bound the cost.
+resource "google_cloud_run_v2_service_iam_member" "web_public" {
+  project  = var.project
+  location = google_cloud_run_v2_service.web.location
+  name     = google_cloud_run_v2_service.web.name
+  role     = "roles/run.invoker"
+  member   = "allUsers"
+}
+
+resource "google_cloud_run_v2_service_iam_member" "api_public" {
+  project  = var.project
+  location = google_cloud_run_v2_service.api.location
+  name     = google_cloud_run_v2_service.api.name
+  role     = "roles/run.invoker"
+  member   = "allUsers"
+}
+
+# Least privilege: the API may execute this one job with argument overrides, nothing more.
+resource "google_cloud_run_v2_job_iam_member" "api_runs_nextflow" {
+  project  = var.project
+  location = google_cloud_run_v2_job.nextflow.location
+  name     = google_cloud_run_v2_job.nextflow.name
+  role     = "roles/run.jobsExecutorWithOverrides"
+  member   = "serviceAccount:${google_service_account.api.email}"
+}
+
+resource "google_secret_manager_secret_iam_member" "api_database_url" {
+  secret_id = google_secret_manager_secret.api_database_url.secret_id
+  role      = "roles/secretmanager.secretAccessor"
+  member    = "serviceAccount:${google_service_account.api.email}"
+}
+
+# Reading the model artifact from gs://<bucket>/models/... when MODEL_URI is set.
+resource "google_storage_bucket_iam_member" "api_reads_data" {
+  bucket = google_storage_bucket.data.name
+  role   = "roles/storage.objectViewer"
+  member = "serviceAccount:${google_service_account.api.email}"
+}

infra/terraform/cloudsql.tf

@@ -1,8 +1,10 @@
 resource "google_sql_database_instance" "pg" {
+  count               = var.deploy_cloud_sql ? 1 : 0
   name                = "rarelens-pg"
   database_version    = "POSTGRES_16"
   region              = var.region
-  deletion_protection = false
+  deletion_protection = var.deletion_protection
+  depends_on          = [google_service_networking_connection.private_services]
 
   settings {
     tier              = "db-f1-micro" # lab budget; bump for real use
@@ -16,14 +18,20 @@ resource "google_sql_database_instance" "pg" {
 }
 
 resource "google_sql_database" "rarelens" {
+  count    = var.deploy_cloud_sql ? 1 : 0
   name     = "rarelens"
-  instance = google_sql_database_instance.pg.name
+  instance = google_sql_database_instance.pg[0].name
 }
 
 resource "google_sql_user" "api" {
+  count    = var.deploy_cloud_sql ? 1 : 0
   name     = "rarelens"
-  instance = google_sql_database_instance.pg.name
-  password = random_password.pg.result
+  instance = google_sql_database_instance.pg[0].name
+  password = random_password.pg[0].result
 }
 
-resource "random_password" "pg" { length = 32 }
+resource "random_password" "pg" {
+  count   = var.deploy_cloud_sql ? 1 : 0
+  length  = 32
+  special = false # embedded in DATABASE_URL, where characters like @ / # % would break parsing
+}

infra/terraform/gke.tf

@@ -1,8 +1,13 @@
 resource "google_container_cluster" "rarelens" {
+  count               = var.deploy_kubernetes ? 1 : 0
   name                = "rarelens"
   location            = var.region
   enable_autopilot    = true
-  deletion_protection = false
+  deletion_protection = var.deletion_protection
+  # Same VPC as Cloud SQL's private IP; without this the cluster lands on the "default" network.
+  network    = google_compute_network.vpc.id
+  subnetwork = google_compute_subnetwork.gke.id
+  ip_allocation_policy {}
 
   workload_identity_config { workload_pool = "${var.project}.svc.id.goog" }
   release_channel { channel = "REGULAR" }

infra/terraform/iam.tf

@@ -1,6 +1,7 @@
 # Workload Identity Federation: GitHub Actions pushes images without long-lived keys.
 resource "google_iam_workload_identity_pool" "github" {
   workload_identity_pool_id = "github"
+  depends_on                = [google_project_service.enabled]
 }
 
 resource "google_iam_workload_identity_pool_provider" "github" {
@@ -29,22 +30,66 @@ resource "google_artifact_registry_repository_iam_member" "ci_push" {
   member     = "serviceAccount:${google_service_account.ci.email}"
 }
 
-# Runtime identities (bound to k8s ServiceAccounts via GKE Workload Identity)
+# Runtime identities, bound to Kubernetes ServiceAccounts in namespace "rarelens" via GKE
+# Workload Identity (the gcp overlay annotates the k8s side).
 resource "google_service_account" "api" { account_id = "rarelens-api" }
 resource "google_service_account" "pipeline" { account_id = "rarelens-pipeline" }
 
+resource "google_service_account_iam_member" "api_workload_identity" {
+  count              = var.deploy_kubernetes ? 1 : 0
+  service_account_id = google_service_account.api.name
+  role               = "roles/iam.workloadIdentityUser"
+  member             = "serviceAccount:${var.project}.svc.id.goog[rarelens/rarelens-api]"
+}
+
+resource "google_service_account_iam_member" "pipeline_workload_identity" {
+  count              = var.deploy_kubernetes ? 1 : 0
+  service_account_id = google_service_account.pipeline.name
+  role               = "roles/iam.workloadIdentityUser"
+  member             = "serviceAccount:${var.project}.svc.id.goog[rarelens/rarelens-pipeline]"
+}
+
 resource "google_project_iam_member" "api_sql" {
   project = var.project
   role    = "roles/cloudsql.client"
   member  = "serviceAccount:${google_service_account.api.email}"
 }
-resource "google_project_iam_member" "api_pubsub" {
-  project = var.project
+
+resource "google_pubsub_topic_iam_member" "api_publish" {
+  topic  = google_pubsub_topic.vcf_uploaded.name
   role   = "roles/pubsub.publisher"
   member = "serviceAccount:${google_service_account.api.email}"
 }
+
+resource "google_pubsub_subscription_iam_member" "pipeline_subscribe" {
+  subscription = google_pubsub_subscription.vcf_uploaded_argo.name
+  role         = "roles/pubsub.subscriber"
+  member       = local.pipeline_sa_member
+}
+
 resource "google_storage_bucket_iam_member" "pipeline_data" {
   bucket = google_storage_bucket.data.name
   role   = "roles/storage.objectAdmin"
-  member = "serviceAccount:${google_service_account.pipeline.email}"
+  member = local.pipeline_sa_member
+}
+
+resource "google_artifact_registry_repository_iam_member" "pipeline_pull" {
+  repository = google_artifact_registry_repository.images.name
+  location   = var.region
+  role       = "roles/artifactregistry.reader"
+  member     = local.pipeline_sa_member
+}
+
+# Nextflow's google-batch executor submits Batch jobs that run as the pipeline SA itself.
+resource "google_project_iam_member" "pipeline_batch" {
+  for_each = toset(["roles/batch.jobsEditor", "roles/batch.agentReporter", "roles/logging.logWriter"])
+  project  = var.project
+  role     = each.value
+  member   = local.pipeline_sa_member
+}
+
+resource "google_service_account_iam_member" "pipeline_act_as_self" {
+  service_account_id = google_service_account.pipeline.name
+  role               = "roles/iam.serviceAccountUser"
+  member             = local.pipeline_sa_member
 }

infra/terraform/locals.tf

@@ -0,0 +1,16 @@
+locals {
+  registry = "${var.region}-docker.pkg.dev/${var.project}/rarelens"
+
+  # join("", ...) rather than one(...): with count = 0 these collapse to "" instead of null.
+  db_credentials = "${join("", google_sql_user.api[*].name)}:${join("", random_password.pg[*].result)}"
+  db_name        = join("", google_sql_database.rarelens[*].name)
+  db_private_ip  = join("", google_sql_database_instance.pg[*].private_ip_address)
+
+  # The API reaches Cloud SQL through its cloud-sql-proxy sidecar on localhost; pipeline tasks
+  # (Google Batch VMs, Argo pods) use the private IP inside the VPC. With deploy_cloud_sql = false
+  # both use the URL you supplied, which is expected to be reachable over TLS.
+  api_database_url      = var.deploy_cloud_sql ? "postgresql+asyncpg://${local.db_credentials}@127.0.0.1:5432/${local.db_name}" : var.database_url
+  pipeline_database_url = var.deploy_cloud_sql ? "postgresql://${local.db_credentials}@${local.db_private_ip}:5432/${local.db_name}" : var.database_url
+
+  pipeline_sa_member = "serviceAccount:${google_service_account.pipeline.email}"
+}

infra/terraform/network.tf

@@ -1,4 +1,31 @@
 resource "google_compute_network" "vpc" {
   name                    = "rarelens-vpc"
-  auto_create_subnetworks = true
+  auto_create_subnetworks = false
+  depends_on              = [google_project_service.enabled]
+}
+
+# Shared by GKE and the Google Batch VMs that run pipeline tasks (pipeline/nextflow.config).
+resource "google_compute_subnetwork" "gke" {
+  name                     = "rarelens-gke"
+  region                   = var.region
+  network                  = google_compute_network.vpc.id
+  ip_cidr_range            = "10.10.0.0/20"
+  private_ip_google_access = true
+}
+
+# Private services access, so Cloud SQL gets a private IP inside this VPC. Only needed with it.
+resource "google_compute_global_address" "private_services" {
+  count         = var.deploy_cloud_sql ? 1 : 0
+  name          = "rarelens-private-services"
+  purpose       = "VPC_PEERING"
+  address_type  = "INTERNAL"
+  prefix_length = 16
+  network       = google_compute_network.vpc.id
+}
+
+resource "google_service_networking_connection" "private_services" {
+  count                   = var.deploy_cloud_sql ? 1 : 0
+  network                 = google_compute_network.vpc.id
+  service                 = "servicenetworking.googleapis.com"
+  reserved_peering_ranges = [google_compute_global_address.private_services[0].name]
 }

infra/terraform/outputs.tf

@@ -1,5 +1,14 @@
-output "cluster_name"   { value = google_container_cluster.rarelens.name }
-output "sql_connection" { value = google_sql_database_instance.pg.connection_name }
+output "web_url" {
+  description = "The one URL to share"
+  value       = google_cloud_run_v2_service.web.uri
+}
+output "api_url" { value = google_cloud_run_v2_service.api.uri }
+output "nextflow_job" { value = google_cloud_run_v2_job.nextflow.name }
 output "data_bucket" { value = google_storage_bucket.data.name }
+output "cluster_name" { value = one(google_container_cluster.rarelens[*].name) }
+output "sql_connection" { value = one(google_sql_database_instance.pg[*].connection_name) }
+output "sql_private_ip" { value = one(google_sql_database_instance.pg[*].private_ip_address) }
 output "wif_provider" { value = google_iam_workload_identity_pool_provider.github.name }
 output "ci_sa" { value = google_service_account.ci.email }
+output "api_sa" { value = google_service_account.api.email }
+output "pipeline_sa" { value = google_service_account.pipeline.email }

infra/terraform/pubsub.tf

@@ -0,0 +1,11 @@
+resource "google_pubsub_topic" "vcf_uploaded" {
+  name       = "vcf-uploaded"
+  depends_on = [google_project_service.enabled]
+}
+
+# Consumed by the Argo Events EventSource in infra/argo-workflows/events.yaml.
+resource "google_pubsub_subscription" "vcf_uploaded_argo" {
+  name                 = "vcf-uploaded-argo"
+  topic                = google_pubsub_topic.vcf_uploaded.id
+  ack_deadline_seconds = 60
+}

infra/terraform/secrets.tf

@@ -0,0 +1,41 @@
+# Copied into Kubernetes secrets by `make gcp-secrets PROJECT=<id>`.
+resource "google_secret_manager_secret" "api_database_url" {
+  secret_id = "rarelens-api-database-url"
+  replication {
+    auto {}
+  }
+  depends_on = [google_project_service.enabled]
+}
+
+resource "google_secret_manager_secret_version" "api_database_url" {
+  secret      = google_secret_manager_secret.api_database_url.id
+  secret_data = local.api_database_url
+
+  lifecycle {
+    precondition {
+      condition     = var.deploy_cloud_sql || var.database_url != ""
+      error_message = "Set database_url (e.g. a Neon URL), or deploy_cloud_sql = true."
+    }
+  }
+}
+
+# The id must match the Nextflow `secret = 'DATABASE_URL'` directive in pipeline/nextflow.config:
+# on Google Batch, Nextflow resolves secrets from Secret Manager by name.
+resource "google_secret_manager_secret" "pipeline_database_url" {
+  secret_id = "DATABASE_URL"
+  replication {
+    auto {}
+  }
+  depends_on = [google_project_service.enabled]
+}
+
+resource "google_secret_manager_secret_version" "pipeline_database_url" {
+  secret      = google_secret_manager_secret.pipeline_database_url.id
+  secret_data = local.pipeline_database_url
+}
+
+resource "google_secret_manager_secret_iam_member" "pipeline_database_url" {
+  secret_id = google_secret_manager_secret.pipeline_database_url.id
+  role      = "roles/secretmanager.secretAccessor"
+  member    = local.pipeline_sa_member
+}

infra/terraform/storage.tf

@@ -2,6 +2,8 @@ resource "google_storage_bucket" "data" {
   name                        = "${var.project}-rarelens-data"
   location                    = var.region
   uniform_bucket_level_access = true
+  public_access_prevention    = "enforced"
+  depends_on                  = [google_project_service.enabled]
   lifecycle_rule {
     condition {
       age            = 30
@@ -15,6 +17,5 @@ resource "google_artifact_registry_repository" "images" {
   repository_id = "rarelens"
   location      = var.region
   format        = "DOCKER"
+  depends_on    = [google_project_service.enabled]
 }
-
-resource "google_pubsub_topic" "vcf_uploaded" { name = "vcf-uploaded" }

infra/terraform/variables.tf

@@ -1,9 +1,59 @@
-variable "project" { type = string }
+variable "project" {
+  description = "GCP project id"
+  type        = string
+}
+
 variable "region" {
+  description = "GCP region for every regional resource"
   type        = string
   default     = "europe-west2" # London: keeps public genomic test data and the Cambridge team in one jurisdiction
 }
+
 variable "github_repo" {
+  description = "owner/name of the GitHub repo allowed to push images via Workload Identity Federation"
   type        = string
   default     = "lynchaos/rarelens"
 }
+
+variable "deletion_protection" {
+  description = "Protect the GKE cluster and Cloud SQL instance from `terraform destroy`; set false to tear the lab down"
+  type        = bool
+  default     = true
+}
+
+variable "deploy_kubernetes" {
+  description = "Create the GKE cluster (Argo/ArgoCD track). Off by default: it costs ~$150/month idle, while the serverless track costs ~£1 (docs/cloud.md)"
+  type        = bool
+  default     = false
+}
+
+variable "deploy_cloud_sql" {
+  description = "Create a Cloud SQL instance (~$10/month). Off by default: set database_url to a free scale-to-zero Postgres such as Neon"
+  type        = bool
+  default     = false
+}
+
+variable "database_url" {
+  description = "Postgres URL used when deploy_cloud_sql is false, e.g. postgresql+asyncpg://user:pass@host/db?sslmode=require"
+  type        = string
+  default     = ""
+  sensitive   = true
+}
+
+variable "image_tag" {
+  description = "Image tag deployed to Cloud Run; CI pushes the commit SHA"
+  type        = string
+  default     = "latest"
+}
+
+variable "model_uri" {
+  description = "Optional model artifact to score with, e.g. gs://<project>-rarelens-data/models/pathogenicity/1. Empty means use the MLflow registry"
+  type        = string
+  default     = ""
+}
+
+variable "max_instances" {
+  description = "Cloud Run instance ceiling per service: scale-to-zero bounds the floor, this bounds the bill"
+  type        = number
+  default     = 2
+}

infra/terraform/versions.tf

@@ -4,7 +4,10 @@ terraform {
     google = { source = "hashicorp/google", version = "~> 6.0" }
     random = { source = "hashicorp/random", version = "~> 3.6" }
   }
-  backend "gcs" { bucket = "REPLACE-tfstate", prefix = "rarelens" }
+  # Partial config: terraform init -backend-config="bucket=<your-tfstate-bucket>"
+  backend "gcs" {
+    prefix = "rarelens"
+  }
 }
 
 provider "google" {

ml/Dockerfile

@@ -2,6 +2,6 @@ FROM python:3.12-slim
 WORKDIR /ml
 RUN pip install --no-cache-dir uv
 COPY pyproject.toml .
-RUN uv pip install --system -e .
+RUN uv pip install --system -r pyproject.toml
 COPY rarelens_ml ./rarelens_ml
 ENTRYPOINT ["python", "-m", "rarelens_ml.train"]

ml/pyproject.toml

@@ -1,8 +1,21 @@
+[build-system]
+requires = ["setuptools>=69"]
+build-backend = "setuptools.build_meta"
+
 [project]
 name = "rarelens-ml"
 version = "0.1.0"
 requires-python = ">=3.12"
-dependencies = ["lightgbm>=4.5", "mlflow>=2.16", "pandas", "scikit-learn", "sqlalchemy", "psycopg[binary]"]
+# mlflow major must match the API's mlflow-skinny and the tracking server image.
+dependencies = ["lightgbm>=4.5", "mlflow>=3,<4", "pandas", "scikit-learn"]
 
 [project.optional-dependencies]
 gpu = ["torch"]   # for the optional deep-learning baseline on GPU
+dev = ["pytest>=8"]
+
+[tool.setuptools.packages.find]
+include = ["rarelens_ml*"]
+
+[tool.pytest.ini_options]
+pythonpath = ["."]
+testpaths = ["tests"]

ml/rarelens_ml/features.py

@@ -1,14 +1,20 @@
-"""Feature engineering shared by training and serving. Keep this identical to api/app/services/scoring.py."""
+"""Feature engineering: the only copy.
+
+Training imports it, and train.log_and_register ships this package inside the logged pyfunc
+(code_paths), so serving runs exactly this code on the raw columns below.
+"""
 import pandas as pd
 
+# What serving must send: raw values as stored in the variants table / its annotations.
+RAW_COLUMNS = ["impact", "consequence", "gnomad_af", "cadd_phred", "am_pathogenicity"]
+
 IMPACT_ORDER = {"MODIFIER": 0, "LOW": 1, "MODERATE": 2, "HIGH": 3}
-CATEGORICAL = ["consequence"]
-NUMERIC = ["impact_rank", "gnomad_af", "cadd_phred", "am_pathogenicity"]
 
 
 def build(df: pd.DataFrame) -> pd.DataFrame:
-    out = pd.DataFrame()
-    out["impact_rank"] = df["impact"].map(IMPACT_ORDER).fillna(0)
+    out = pd.DataFrame(index=df.index)
+    out["impact_rank"] = df["impact"].map(IMPACT_ORDER).fillna(0).astype(int)
+    # No gnomAD record means the variant was not observed: treat as AF 0.
     out["gnomad_af"] = pd.to_numeric(df["gnomad_af"], errors="coerce").fillna(0.0)
     out["cadd_phred"] = pd.to_numeric(df["cadd_phred"], errors="coerce")
     out["am_pathogenicity"] = pd.to_numeric(df["am_pathogenicity"], errors="coerce")

ml/rarelens_ml/model.py

@@ -0,0 +1,17 @@
+import mlflow
+
+from rarelens_ml.features import RAW_COLUMNS, build
+
+
+class PathogenicityModel(mlflow.pyfunc.PythonModel):
+    """Serving contract: raw VEP columns in, P(pathogenic) out.
+
+    The stock LightGBM pyfunc flavour calls `predict`, which returns class labels; wrapping the
+    classifier keeps feature engineering and `predict_proba` inside the registered artifact.
+    """
+
+    def __init__(self, classifier):
+        self.classifier = classifier
+
+    def predict(self, context, model_input, params=None):
+        return self.classifier.predict_proba(build(model_input[RAW_COLUMNS]))[:, 1]

ml/rarelens_ml/train.py

@@ -1,54 +1,130 @@
-"""Train a pathogenicity classifier on ClinVar labels (Pathogenic/Likely pathogenic vs Benign/Likely benign).
+"""Train a pathogenicity classifier on ClinVar labels ((likely) pathogenic vs (likely) benign).
 
 Label leakage warning: CLIN_SIG must never be a feature. This is a learning exercise, not a clinical model.
-Usage: python -m rarelens_ml.train --tsv results/clinvar.vep.tsv
+Usage: python -m rarelens_ml.train --tsv results/clinvar.vep.tsv --register
 """
 import argparse
+import re
+from pathlib import Path
 
 import lightgbm as lgb
 import mlflow
-import mlflow.lightgbm
 import pandas as pd
+import sklearn
+from mlflow import MlflowClient
 from sklearn.metrics import average_precision_score, roc_auc_score
 from sklearn.model_selection import train_test_split
 
-from rarelens_ml.features import build
+from rarelens_ml.features import RAW_COLUMNS, build
+from rarelens_ml.model import PathogenicityModel
 
-POS = {"Pathogenic", "Likely_pathogenic", "Pathogenic/Likely_pathogenic"}
-NEG = {"Benign", "Likely_benign", "Benign/Likely_benign"}
+PACKAGE_DIR = Path(__file__).resolve().parent
+MODEL_NAME = "rarelens-pathogenicity"
+PARAMS = {
+    "n_estimators": 400, "learning_rate": 0.05, "num_leaves": 31, "class_weight": "balanced",
+    "verbose": -1,
+}
+
+POS = {"pathogenic", "likely_pathogenic"}
+NEG = {"benign", "likely_benign"}
+# VEP --tab column -> raw feature column (am_pathogenicity already matches).
+VEP_TO_RAW = {
+    "IMPACT": "impact", "Consequence": "consequence", "gnomADe_AF": "gnomad_af",
+    "CADD_PHRED": "cadd_phred",
+}
+
+
+def label(clin_sig: object) -> int | None:
+    """1 / 0 when every ClinVar term agrees, None for VUS, conflicts and missing values.
+
+    Accepts VEP's lowercase comma-separated form ("pathogenic,likely_pathogenic") and ClinVar's
+    CLNSIG form ("Pathogenic/Likely_pathogenic").
+    """
+    if not isinstance(clin_sig, str):
+        return None
+    terms = {t for t in re.split(r"[,&/|]", clin_sig.strip().lower()) if t and t != "-"}
+    if terms and terms <= POS:
+        return 1
+    if terms and terms <= NEG:
+        return 0
+    return None
+
+
+def read_vep_tab(path: str | Path) -> pd.DataFrame:
+    """Read VEP --tab output as strings, keeping "-" (VEP's missing marker) verbatim.
+
+    Skips the "##" preamble by position instead of comment="#", which would also cut any value
+    containing "#".
+    """
+    with open(path) as fh:
+        for n, line in enumerate(fh):
+            if line.startswith("#Uploaded_variation"):
+                break
+        else:
+            raise ValueError(f"{path}: no #Uploaded_variation header; is this VEP --tab output?")
+    df = pd.read_csv(path, sep="\t", skiprows=n, dtype=str, keep_default_na=False)
+    return df.rename(columns={"#Uploaded_variation": "Uploaded_variation"})
 
 
 def load(tsv: str) -> tuple[pd.DataFrame, pd.Series]:
-    df = pd.read_csv(tsv, sep="\t", comment="#", header=None, dtype=str)
-    with open(tsv) as fh:
-        df.columns = next(l for l in fh if l.startswith("#Uploaded")).lstrip("#").rstrip().split("\t")
-    df = df.rename(columns={"IMPACT": "impact", "Consequence": "consequence", "gnomADe_AF": "gnomad_af",
-                            "CADD_PHRED": "cadd_phred"})
-    y = df["CLIN_SIG"].map(lambda s: 1 if s in POS else 0 if s in NEG else None)
+    df = read_vep_tab(tsv).rename(columns=VEP_TO_RAW)
+    for col in RAW_COLUMNS:  # plugin columns are absent when VEP ran without CADD/AlphaMissense
+        if col not in df:
+            df[col] = pd.NA
+    y = df["CLIN_SIG"].map(label)
     keep = y.notna()
-    return build(df[keep]), y[keep].astype(int)
+    return (
+        df.loc[keep, RAW_COLUMNS].reset_index(drop=True),
+        y[keep].astype(int).reset_index(drop=True),
+    )
+
+
+def fit(X: pd.DataFrame, y: pd.Series) -> lgb.LGBMClassifier:
+    return lgb.LGBMClassifier(**PARAMS).fit(build(X), y)
+
+
+def log_and_register(clf: lgb.LGBMClassifier, model_name: str, alias: str) -> str:
+    """Log the pyfunc, register it and point `alias` at the new version. Returns the version."""
+    info = mlflow.pyfunc.log_model(
+        name="model",
+        python_model=PathogenicityModel(clf),
+        code_paths=[str(PACKAGE_DIR)],
+        registered_model_name=model_name,
+        pip_requirements=[
+            f"lightgbm=={lgb.__version__}",
+            f"pandas=={pd.__version__}",
+            f"scikit-learn=={sklearn.__version__}",
+        ],
+    )
+    version = str(info.registered_model_version)
+    MlflowClient().set_registered_model_alias(model_name, alias, version)
+    return version
 
 
 def main() -> None:
     p = argparse.ArgumentParser()
     p.add_argument("--tsv", required=True)
-    p.add_argument("--register", action="store_true")
+    p.add_argument("--register", action="store_true",
+                   help="register the model and move the alias to the new version")
+    p.add_argument("--alias", default="production")
     a = p.parse_args()
 
     X, y = load(a.tsv)
     Xtr, Xte, ytr, yte = train_test_split(X, y, test_size=0.2, stratify=y, random_state=42)
 
-    mlflow.set_experiment("rarelens-pathogenicity")
+    mlflow.set_experiment(MODEL_NAME)
     with mlflow.start_run():
-        params = {"n_estimators": 400, "learning_rate": 0.05, "num_leaves": 31, "class_weight": "balanced"}
-        mlflow.log_params(params)
-        model = lgb.LGBMClassifier(**params).fit(Xtr, ytr)
-        proba = model.predict_proba(Xte)[:, 1]
-        mlflow.log_metrics({"auroc": roc_auc_score(yte, proba), "auprc": average_precision_score(yte, proba)})
-        mlflow.lightgbm.log_model(
-            model, "model",
-            registered_model_name="rarelens-pathogenicity" if a.register else None,
-        )
+        mlflow.log_params(PARAMS)
+        clf = fit(Xtr, ytr)
+        proba = clf.predict_proba(build(Xte))[:, 1]
+        mlflow.log_metrics({"auroc": roc_auc_score(yte, proba),
+                            "auprc": average_precision_score(yte, proba)})
+        if a.register:
+            version = log_and_register(clf, MODEL_NAME, a.alias)
+            print(f"registered {MODEL_NAME} v{version} as @{a.alias}")
+        else:
+            mlflow.pyfunc.log_model(name="model", python_model=PathogenicityModel(clf),
+                                    code_paths=[str(PACKAGE_DIR)])
 
 
 if __name__ == "__main__":

ml/tests/test_features.py

@@ -0,0 +1,38 @@
+import math
+
+import pandas as pd
+
+from rarelens_ml.features import RAW_COLUMNS, build
+
+
+def raw(**overrides: list) -> pd.DataFrame:
+    base = {
+        "impact": ["HIGH", "LOW", None],
+        "consequence": ["stop_gained", "synonymous_variant", None],
+        "gnomad_af": [None, "0.12", 0.001],
+        "cadd_phred": ["35", "2.1", "-"],
+        "am_pathogenicity": ["0.98", None, "-"],
+    }
+    base.update(overrides)
+    return pd.DataFrame(base, index=[10, 11, 12])
+
+
+def test_raw_columns_are_the_serving_contract() -> None:
+    assert RAW_COLUMNS == ["impact", "consequence", "gnomad_af", "cadd_phred", "am_pathogenicity"]
+
+
+def test_build_ranks_impact_and_coerces_numbers() -> None:
+    out = build(raw())
+    assert out["impact_rank"].tolist() == [3, 1, 0]
+    assert out["gnomad_af"].tolist() == [0.0, 0.12, 0.001]  # missing AF means absent from gnomAD
+    assert out["cadd_phred"].iloc[0] == 35.0
+    assert math.isnan(out["cadd_phred"].iloc[2])  # VEP writes "-" for missing
+    assert math.isnan(out["am_pathogenicity"].iloc[1])
+
+
+def test_build_keeps_the_input_index() -> None:
+    assert build(raw()).index.tolist() == [10, 11, 12]
+
+
+def test_build_makes_consequence_categorical() -> None:
+    assert isinstance(build(raw())["consequence"].dtype, pd.CategoricalDtype)

ml/tests/test_train.py

@@ -0,0 +1,104 @@
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import pytest
+
+from rarelens_ml.train import label, read_vep_tab
+
+HEADER = [
+    "Uploaded_variation", "Location", "Allele", "Consequence", "IMPACT", "SYMBOL",
+    "gnomADe_AF", "CLIN_SIG", "CADD_PHRED", "am_pathogenicity",
+]
+
+
+def write_vep_tab(path: Path, rows: list[list[str]]) -> Path:
+    lines = [
+        "## ENSEMBL VARIANT EFFECT PREDICTOR v113.0",
+        "## Column descriptions:",
+        "#" + "\t".join(HEADER),
+        *("\t".join(r) for r in rows),
+    ]
+    path.write_text("\n".join(lines) + "\n")
+    return path
+
+
+@pytest.mark.parametrize(
+    ("clin_sig", "expected"),
+    [
+        # VEP writes lowercase, comma-separated terms from co-located ClinVar records.
+        ("pathogenic", 1),
+        ("pathogenic,likely_pathogenic", 1),
+        ("likely_benign", 0),
+        ("benign,likely_benign", 0),
+        # ClinVar VCF CLNSIG spelling must keep working too.
+        ("Pathogenic/Likely_pathogenic", 1),
+        ("Benign", 0),
+        ("uncertain_significance", None),
+        ("pathogenic,benign", None),  # conflicting evidence is not a label
+        ("-", None),
+        ("", None),
+        (np.nan, None),
+    ],
+)
+def test_label(clin_sig: object, expected: int | None) -> None:
+    assert label(clin_sig) == expected
+
+
+def test_read_vep_tab_uses_the_hash_header_and_keeps_dashes(tmp_path: Path) -> None:
+    tsv = write_vep_tab(
+        tmp_path / "x.vep.tsv",
+        [["22_1_A_G", "22:1", "G", "missense_variant", "MODERATE", "TBX1", "-", "pathogenic", "28", "0.9"]],
+    )
+    df = read_vep_tab(tsv)
+    assert list(df.columns) == HEADER
+    assert df.loc[0, "gnomADe_AF"] == "-"
+    assert df.loc[0, "CLIN_SIG"] == "pathogenic"
+
+
+def test_load_returns_raw_serving_columns_and_labels(tmp_path: Path) -> None:
+    from rarelens_ml.features import RAW_COLUMNS
+    from rarelens_ml.train import load
+
+    tsv = write_vep_tab(
+        tmp_path / "x.vep.tsv",
+        [
+            ["a", "22:1", "G", "missense_variant", "MODERATE", "TBX1", "0.0001", "pathogenic", "28", "0.9"],
+            ["b", "22:2", "A", "synonymous_variant", "LOW", "CHEK2", "0.12", "benign", "3", "-"],
+            ["c", "22:3", "T", "intron_variant", "MODIFIER", "CHEK2", "0.3", "uncertain_significance", "1", "-"],
+        ],
+    )
+    X, y = load(str(tsv))
+    assert list(X.columns) == RAW_COLUMNS
+    assert y.tolist() == [1, 0]  # the VUS row is dropped
+
+
+def test_logged_model_returns_probabilities_from_raw_columns(tmp_path: Path) -> None:
+    """The registered model must take the raw columns serving sends and return P(pathogenic)."""
+    import mlflow
+
+    from rarelens_ml.train import fit, log_and_register
+
+    rng = np.random.default_rng(0)
+    n = 400
+    impact = rng.choice(["HIGH", "MODERATE", "LOW", "MODIFIER"], n)
+    y = pd.Series(((impact == "HIGH") | (rng.random(n) < 0.1)).astype(int))
+    X = pd.DataFrame({
+        "impact": impact,
+        "consequence": rng.choice(["stop_gained", "missense_variant", "intron_variant"], n),
+        "gnomad_af": rng.random(n).round(4).astype(str),   # strings, as read from the DB
+        "cadd_phred": (rng.random(n) * 40).round(1).astype(str),
+        "am_pathogenicity": "-",
+    })
+
+    mlflow.set_tracking_uri(f"sqlite:///{tmp_path}/mlflow.db")
+    mlflow.set_experiment("test")
+    clf = fit(X, y)
+    version = log_and_register(clf, model_name="rarelens-test", alias="production")
+
+    model = mlflow.pyfunc.load_model("models:/rarelens-test@production")
+    scores = np.asarray(model.predict(X.head(50)))
+    assert version == "1"
+    assert scores.shape == (50,)
+    assert ((scores >= 0) & (scores <= 1)).all()
+    assert not set(np.unique(scores)) <= {0.0, 1.0}, "got class labels, expected probabilities"

pipeline/.dockerignore

@@ -0,0 +1,5 @@
+work/
+results/
+cache/
+tests/
+.nextflow*

pipeline/Dockerfile

@@ -1,5 +1,19 @@
-# Loader image: pandas + psycopg for LOAD_DB
-FROM python:3.12-slim
-RUN pip install --no-cache-dir pandas sqlalchemy "psycopg[binary]"
-COPY bin/load_db.py /usr/local/bin/load_db.py
-RUN chmod +x /usr/local/bin/load_db.py
+# Nextflow driver for the Argo annotate-vcf workflow. Tasks run in their own containers.
+FROM eclipse-temurin:21-jre
+ARG NXF_VER=26.04.6
+ENV NXF_VER=${NXF_VER} NXF_HOME=/opt/nextflow
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends curl ca-certificates \
+ && rm -rf /var/lib/apt/lists/* \
+ && curl -fsSL https://get.nextflow.io | bash \
+ && mv nextflow /usr/local/bin/nextflow \
+ && nextflow plugin install nf-google
+COPY main.nf nextflow.config /pipeline/
+COPY modules /pipeline/modules
+COPY bin /pipeline/bin
+COPY assets /pipeline/assets
+# CI passes the loader image built from the same commit, so driver and loader never drift.
+ARG LOADER_IMAGE=rarelens/loader:dev
+ENV RARELENS_LOADER_IMAGE=${LOADER_IMAGE}
+WORKDIR /work
+ENTRYPOINT ["nextflow"]

pipeline/bin/load_db.py

@@ -1,59 +1,133 @@
-#!/usr/bin/env python
-"""Load a VEP --tab output into the rarelens Postgres schema and mark the job succeeded."""
+#!/usr/bin/env python3
+"""Load VEP --tab output into the rarelens Postgres schema and mark the job succeeded.
+
+Variant identity (chrom/pos/ref/alt) comes from the VCF ID, which NORMALISE sets to
+CHROM_POS_REF_ALT: VEP's own Location/Allele columns trim indel alleles and shift positions.
+The database URL is read from $DATABASE_URL so it never appears on a command line or in .command.sh.
+Loading replaces any rows already stored for the job, so a retried task cannot duplicate variants.
+"""
 import argparse
 import json
+import os
+import re
 import sys
+from pathlib import Path
 
 import pandas as pd
-from sqlalchemy import create_engine, text
+from sqlalchemy import Engine, create_engine, text
+from sqlalchemy.engine import make_url
+
+INSERT_CHUNK = 5000
+VEP_VERSION = re.compile(r"^## ENSEMBL VARIANT EFFECT PREDICTOR v(\S+)")
+
+
+def engine_for(url: str) -> Engine:
+    """Accept the API's asyncpg URL (or a plain postgresql:// one) and use psycopg."""
+    parsed = make_url(url).set(drivername="postgresql+psycopg")
+    query = dict(parsed.query)
+    if "ssl" in query:  # asyncpg's spelling; libpq (psycopg) wants sslmode
+        query["sslmode"] = query.pop("ssl")
+    return create_engine(parsed.set(query=query))
+
+
+def read_vep_tab(path: str | Path) -> pd.DataFrame:
+    """Read VEP --tab output as strings, keeping "-" (VEP's missing marker) verbatim."""
+    with open(path) as fh:
+        for n, line in enumerate(fh):
+            if line.startswith("#Uploaded_variation"):
+                break
+        else:
+            raise ValueError(f"{path}: no #Uploaded_variation header; is this VEP --tab output?")
+    df = pd.read_csv(path, sep="\t", skiprows=n, dtype=str, keep_default_na=False)
+    return df.rename(columns={"#Uploaded_variation": "Uploaded_variation"})
+
+
+def vep_version(path: str | Path) -> str | None:
+    with open(path) as fh:
+        for line in fh:
+            if not line.startswith("##"):
+                return None
+            if m := VEP_VERSION.match(line):
+                return m.group(1)
+    return None
+
+
+def parse_variant_id(uid: str) -> tuple[str, int, str, str]:
+    # rsplit: contig names may contain "_" (chrUn_KI270742v1); positions and alleles never do.
+    parts = uid.rsplit("_", 3)
+    if len(parts) != 4 or not parts[1].isdigit():
+        raise ValueError(
+            f"unexpected Uploaded_variation {uid!r}; NORMALISE must set VCF IDs to CHROM_POS_REF_ALT"
+        )
+    chrom, pos, ref, alt = parts
+    return chrom, int(pos), ref, alt
+
+
+def _value(v: str | None) -> str | None:
+    return None if v in (None, "", "-") else v
+
+
+def to_rows(df: pd.DataFrame, job_id: str) -> list[dict]:
+    rows = []
+    for rec in df.to_dict("records"):
+        chrom, pos, ref, alt = parse_variant_id(rec["Uploaded_variation"])
+        af = _value(rec.get("gnomADe_AF"))
+        rows.append({
+            "job_id": job_id,
+            "chrom": chrom,
+            "pos": pos,
+            "ref": ref,
+            "alt": alt,
+            "gene": _value(rec.get("SYMBOL")),
+            "consequence": _value(rec.get("Consequence")),
+            "impact": _value(rec.get("IMPACT")),
+            "hgvsc": _value(rec.get("HGVSc")),
+            "hgvsp": _value(rec.get("HGVSp")),
+            "gnomad_af": float(af) if af is not None else None,
+            "clinvar_sig": _value(rec.get("CLIN_SIG")),
+            "annotations": json.dumps({k: v for k, v in rec.items() if _value(v) is not None}),
+        })
+    return rows
+
+
+def load(engine: Engine, job_id: str, rows: list[dict], vep: str | None) -> None:
+    with engine.begin() as conn:
+        conn.execute(text("DELETE FROM variants WHERE job_id = :id"), {"id": job_id})
+        for start in range(0, len(rows), INSERT_CHUNK):
+            conn.execute(
+                text("""
+                    INSERT INTO variants (job_id, chrom, pos, ref, alt, gene, consequence, impact,
+                                          hgvsc, hgvsp, gnomad_af, clinvar_sig, annotations)
+                    VALUES (:job_id, :chrom, :pos, :ref, :alt, :gene, :consequence, :impact,
+                            :hgvsc, :hgvsp, :gnomad_af, :clinvar_sig, CAST(:annotations AS jsonb))
+                """),
+                rows[start:start + INSERT_CHUNK],
+            )
+        conn.execute(
+            text("""
+                UPDATE jobs SET status = 'succeeded', vep_version = :vep, log = NULL,
+                                finished_at = now()
+                WHERE id = :id
+            """),
+            {"id": job_id, "vep": vep},
+        )
 
 
 def main() -> None:
     p = argparse.ArgumentParser()
     p.add_argument("--tsv", required=True)
     p.add_argument("--job-id", required=True)
-    p.add_argument("--db-url", required=True)
+    p.add_argument("--dry-run", action="store_true", help="parse only; do not touch the database")
     a = p.parse_args()
 
-    df = pd.read_csv(a.tsv, sep="\t", comment="#", header=None, dtype=str)
-    with open(a.tsv) as fh:
-        header = next(l for l in fh if l.startswith("#Uploaded_variation")).lstrip("#").rstrip().split("\t")
-    df.columns = header
-
-    chrom_pos = df["Location"].str.split(":", expand=True)
-    rows = []
-    for i, r in df.iterrows():
-        ref, _, alt = r["Uploaded_variation"].partition("/") if "/" in r["Uploaded_variation"] else ("", "", r["Allele"])
-        rows.append({
-            "job_id": a.job_id,
-            "chrom": chrom_pos.iloc[i, 0],
-            "pos": int(chrom_pos.iloc[i, 1].split("-")[0]),
-            "ref": ref or "-",
-            "alt": r["Allele"],
-            "gene": r.get("SYMBOL") if r.get("SYMBOL") != "-" else None,
-            "consequence": r["Consequence"],
-            "impact": r["IMPACT"],
-            "hgvsc": None if r.get("HGVSc") == "-" else r.get("HGVSc"),
-            "hgvsp": None if r.get("HGVSp") == "-" else r.get("HGVSp"),
-            "gnomad_af": None if r.get("gnomADe_AF", "-") == "-" else float(r["gnomADe_AF"]),
-            "clinvar_sig": None if r.get("CLIN_SIG", "-") == "-" else r["CLIN_SIG"],
-            "annotations": json.dumps({k: v for k, v in r.items() if v != "-"}),
-        })
-
-    if a.db_url == "none":
+    rows = to_rows(read_vep_tab(a.tsv), job_id=a.job_id)
+    if a.dry_run:
         print(f"{len(rows)} variants parsed (dry run, no DB)")
         return
-
-    engine = create_engine(a.db_url.replace("+asyncpg", "+psycopg"))
-    with engine.begin() as conn:
-        conn.execute(text("""
-            INSERT INTO variants (job_id, chrom, pos, ref, alt, gene, consequence, impact,
-                                  hgvsc, hgvsp, gnomad_af, clinvar_sig, annotations)
-            VALUES (:job_id, :chrom, :pos, :ref, :alt, :gene, :consequence, :impact,
-                    :hgvsc, :hgvsp, :gnomad_af, :clinvar_sig, CAST(:annotations AS jsonb))
-        """), rows)
-        conn.execute(text("UPDATE jobs SET status='succeeded', finished_at=now() WHERE id=:id"),
-                     {"id": a.job_id})
+    url = os.environ.get("DATABASE_URL")
+    if not url:
+        sys.exit("DATABASE_URL is not set")
+    load(engine_for(url), a.job_id, rows, vep=vep_version(a.tsv))
     print(f"loaded {len(rows)} variants for job {a.job_id}", file=sys.stderr)
 
 

pipeline/bin/set_job_status.py

@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+"""Set a job's status, e.g. from the Argo exit handler when a workflow fails.
+
+A succeeded job is never overwritten: the loader's success is the source of truth.
+"""
+import argparse
+import os
+import sys
+
+from load_db import engine_for
+from sqlalchemy import Engine, text
+
+
+def set_status(engine: Engine, job_id: str, status: str, log: str | None) -> None:
+    with engine.begin() as conn:
+        conn.execute(
+            text("""
+                UPDATE jobs
+                SET status = CAST(:status AS jobstatus), log = :log,
+                    finished_at = CASE WHEN :status IN ('succeeded', 'failed') THEN now() END
+                WHERE id = :id AND status <> 'succeeded'
+            """),
+            {"id": job_id, "status": status, "log": log},
+        )
+
+
+def main() -> None:
+    p = argparse.ArgumentParser()
+    p.add_argument("--job-id", required=True)
+    p.add_argument("--status", required=True, choices=["running", "failed"])
+    p.add_argument("--log")
+    a = p.parse_args()
+    url = os.environ.get("DATABASE_URL")
+    if not url:
+        sys.exit("DATABASE_URL is not set")
+    set_status(engine_for(url), a.job_id, a.status, a.log)
+
+
+if __name__ == "__main__":
+    main()

pipeline/loader.Dockerfile

@@ -0,0 +1,8 @@
+# Task image for LOAD_DB and the Argo exit handler.
+FROM python:3.12-slim
+# procps: Nextflow uses `ps` inside task containers to collect metrics.
+RUN apt-get update && apt-get install -y --no-install-recommends procps && rm -rf /var/lib/apt/lists/*
+COPY requirements.txt /tmp/requirements.txt
+RUN pip install --no-cache-dir -r /tmp/requirements.txt
+COPY bin/load_db.py bin/set_job_status.py /usr/local/bin/
+RUN chmod +x /usr/local/bin/load_db.py /usr/local/bin/set_job_status.py

pipeline/main.nf

@@ -7,9 +7,17 @@ include { LOAD_DB }   from './modules/load_db'
 
 workflow {
     if (!params.vcf) error "Provide --vcf"
+    if (workflow.profile.tokenize(',').contains('gcp') && !(params.project && params.bucket)) {
+        error "The gcp profile needs --project and --bucket (or GCP_PROJECT and GCS_BUCKET)"
+    }
+    // Stub runs (CI) have no VEP cache or plugin data on disk.
+    def must_exist = !workflow.stubRun
+
     vcf_ch  = Channel.fromPath(params.vcf, checkIfExists: true)
+    cache   = file(params.vep_cache, checkIfExists: must_exist)
+    plugins = file(params.vep_plugin_data ?: "${projectDir}/assets/NO_FILE", checkIfExists: must_exist)
 
     NORMALISE(vcf_ch)
-    VEP(NORMALISE.out.vcf)
-    LOAD_DB(VEP.out.tsv, params.job_id ?: 'local', params.db_url ?: 'none')
+    VEP(NORMALISE.out.vcf, cache, plugins)
+    LOAD_DB(VEP.out.tsv, params.job_id ?: 'dry-run')
 }

pipeline/modules/load_db.nf

@@ -3,11 +3,18 @@ process LOAD_DB {
     input:
         path tsv
         val job_id
-        val db_url
     output: stdout
 
+    // DATABASE_URL reaches the task through its environment (docker profile) or a Nextflow
+    // secret (gcp profile); it never appears on a command line or in .command.sh.
     script:
+    def dry_run = job_id == 'dry-run' ? '--dry-run' : ''
     """
-    load_db.py --tsv $tsv --job-id $job_id --db-url '$db_url'
+    load_db.py --tsv $tsv --job-id $job_id $dry_run
+    """
+
+    stub:
+    """
+    echo "stub: would load $tsv for job $job_id"
     """
 }

pipeline/modules/normalise.nf

@@ -3,9 +3,16 @@ process NORMALISE {
     input:  path vcf
     output: path "${vcf.simpleName}.norm.vcf.gz", emit: vcf
 
+    // Split multiallelics, then set each ID to CHROM_POS_REF_ALT. VEP echoes the ID back as
+    // Uploaded_variation, and the loader takes exact VCF alleles from it (VEP trims indels).
     script:
     """
-    bcftools norm -m -both -Oz -o ${vcf.simpleName}.norm.vcf.gz $vcf
-    bcftools index -t ${vcf.simpleName}.norm.vcf.gz
+    bcftools norm -m -both -Ou $vcf \\
+        | bcftools annotate --set-id '%CHROM\\_%POS\\_%REF\\_%FIRST_ALT' -Oz -o ${vcf.simpleName}.norm.vcf.gz
+    """
+
+    stub:
+    """
+    touch ${vcf.simpleName}.norm.vcf.gz
     """
 }

pipeline/modules/vep.nf

@@ -1,16 +1,31 @@
 process VEP {
     tag "$vcf.simpleName"
     publishDir params.outdir, mode: 'copy'
-    input:  path vcf
+    input:
+        path vcf
+        path cache          // staged as an input so it is visible inside the container
+        path plugin_data    // assets/NO_FILE when plugins are not configured
     output:
         path "${vcf.simpleName}.vep.tsv",      emit: tsv
         path "${vcf.simpleName}.vep_summary.html"
 
     script:
+    // Plugins run only when --vep_plugin_data names a directory holding the plugin modules
+    // (INSTALL.pl -a p -g CADD,AlphaMissense -r <dir>) and the data files named in params.
+    def plugins = plugin_data.name == 'NO_FILE' ? '' : [
+        "--dir_plugins ${plugin_data}",
+        "--plugin CADD,snv=${plugin_data}/${params.cadd_snv},indels=${plugin_data}/${params.cadd_indels}",
+        "--plugin AlphaMissense,file=${plugin_data}/${params.alphamissense}",
+    ].join(' ')
     """
     vep -i $vcf -o ${vcf.simpleName}.vep.tsv --tab \\
-        --assembly ${params.assembly} --cache --dir_cache ${params.vep_cache} --offline \\
-        --everything --pick --af_gnomade --plugin CADD --plugin AlphaMissense \\
+        --assembly ${params.assembly} --cache --offline --dir_cache ${cache} \\
+        --everything --pick ${plugins} \\
         --stats_file ${vcf.simpleName}.vep_summary.html --fork ${task.cpus}
     """
+
+    stub:
+    """
+    touch ${vcf.simpleName}.vep.tsv ${vcf.simpleName}.vep_summary.html
+    """
 }

pipeline/nextflow.config

@@ -1,24 +1,52 @@
 params {
     vcf             = null
-    job_id     = null
-    db_url     = null
+    job_id          = null        // omit for a dry run that parses but does not load
     outdir          = "results"
     assembly        = "GRCh38"
-    vep_cache  = "${projectDir}/cache/vep"     // download once with `vep_install`; or use --offline false
-    vep_plugins = "CADD,AlphaMissense"
-}
+    vep_cache       = "${projectDir}/cache/vep"   // INSTALL.pl -a cf -s homo_sapiens -y GRCh38 -c <dir>
+    vep_plugin_data = null        // CADD + AlphaMissense modules and data; plugins skipped when null
+    cadd_snv        = "whole_genome_SNVs.tsv.gz"
+    cadd_indels     = "gnomad.genomes.r4.0.indel.tsv.gz"
+    alphamissense   = "AlphaMissense_hg38.tsv.gz"
+    // The driver image sets this to the loader image built from the same commit.
+    loader_image    = System.getenv('RARELENS_LOADER_IMAGE') ?: 'rarelens/loader:dev'
 
-profiles {
-    docker { docker.enabled = true }
-    gcp {
-        process.executor = 'k8s'               // runs inside GKE via Argo; Nextflow k8s executor
-        workDir = "gs://${params.bucket}/work"
-        google.project = params.project
-    }
+    // gcp profile; the Argo workflow provides these through the pipeline-config ConfigMap.
+    project         = System.getenv('GCP_PROJECT')
+    region          = System.getenv('GCP_REGION') ?: 'europe-west2'
+    bucket          = System.getenv('GCS_BUCKET')
 }
 
 process {
+    shell = ['/bin/bash', '-euo', 'pipefail']
     withName: VEP        { container = 'ensemblorg/ensembl-vep:release_113.0'; cpus = 4; memory = '8 GB' }
     withName: NORMALISE  { container = 'quay.io/biocontainers/bcftools:1.20--h8b25389_0' }
-    withName: LOAD_DB    { container = 'ghcr.io/lynchaos/rarelens-loader:latest' }
+    withName: LOAD_DB    { container = params.loader_image }
+}
+
+profiles {
+    docker {
+        docker.enabled      = true
+        docker.envWhitelist = ['DATABASE_URL']
+        // Lets the loader reach a Postgres published on the host (docker-compose's port 5432).
+        docker.runOptions   = '--add-host=host.docker.internal:host-gateway'
+    }
+    gcp {
+        // The driver runs in the Argo pod; each task runs as a Google Batch job, which is what a
+        // gs:// work directory requires (the k8s executor needs a shared ReadWriteMany volume).
+        workDir = "gs://${params.bucket}/work"
+        params.vep_cache = "gs://${params.bucket}/refs/vep"
+        google {
+            project  = params.project
+            location = params.region
+            batch.serviceAccountEmail = "rarelens-pipeline@${params.project}.iam.gserviceaccount.com"
+            batch.network    = "projects/${params.project}/global/networks/rarelens-vpc"
+            batch.subnetwork = "projects/${params.project}/regions/${params.region}/subnetworks/rarelens-gke"
+        }
+        process {
+            executor = 'google-batch'
+            // Google Secret Manager secret created by Terraform (infra/terraform/secrets.tf).
+            withName: LOAD_DB { secret = 'DATABASE_URL' }
+        }
+    }
 }

pipeline/requirements.txt

@@ -0,0 +1,3 @@
+pandas>=2.2
+sqlalchemy>=2.0
+psycopg[binary]>=3.2

pipeline/tests/conftest.py

@@ -0,0 +1,60 @@
+"""Loader tests. DB tests use DATABASE_URL (CI's Postgres service) or a throwaway pgserver."""
+import os
+import sys
+import tempfile
+from collections.abc import Iterator
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "bin"))
+
+# The subset of the Alembic schema (api/alembic/versions) that the loader writes to.
+SCHEMA = """
+CREATE TYPE jobstatus AS ENUM ('queued', 'running', 'succeeded', 'failed');
+CREATE TABLE jobs (
+    id uuid PRIMARY KEY,
+    status jobstatus NOT NULL,
+    vep_version text,
+    log text,
+    finished_at timestamptz
+);
+CREATE TABLE variants (
+    id serial PRIMARY KEY,
+    job_id uuid NOT NULL,
+    chrom text NOT NULL, pos integer NOT NULL, ref text NOT NULL, alt text NOT NULL,
+    gene text, consequence text, impact text, hgvsc text, hgvsp text,
+    gnomad_af double precision, clinvar_sig text, annotations jsonb NOT NULL
+);
+"""
+
+_server = None
+
+
+def _url() -> str | None:
+    global _server
+    if url := os.environ.get("DATABASE_URL"):
+        return url
+    try:
+        import pgserver
+    except ImportError:
+        return None
+    _server = pgserver.get_server(tempfile.mkdtemp(prefix="loader-pg-"), cleanup_mode="delete")
+    return "postgresql://postgres@/postgres?host=" + _server.get_uri().split("host=", 1)[1]
+
+
+@pytest.fixture
+def engine() -> Iterator:
+    url = _url()
+    if not url:
+        if os.environ.get("REQUIRE_DB"):
+            pytest.fail("REQUIRE_DB is set but no database is available")
+        pytest.skip("no DATABASE_URL and pgserver is not installed")
+    from load_db import engine_for
+
+    eng = engine_for(url)
+    with eng.begin() as conn:
+        conn.exec_driver_sql("DROP TABLE IF EXISTS variants, jobs; DROP TYPE IF EXISTS jobstatus")
+        conn.exec_driver_sql(SCHEMA)
+    yield eng
+    eng.dispose()

pipeline/tests/data/tiny.vcf

@@ -0,0 +1,7 @@
+##fileformat=VCFv4.2
+##source=rarelens synthetic smoke-test fixture (not real sample data)
+##contig=<ID=22,length=50818468>
+#CHROM	POS	ID	REF	ALT	QUAL	FILTER	INFO
+22	19710700	.	C	T	.	PASS	.
+22	29091857	.	G	A,C	.	PASS	.
+22	42126611	.	CT	C	.	PASS	.

pipeline/tests/test_load_db.py

@@ -0,0 +1,136 @@
+import json
+import uuid
+from pathlib import Path
+
+import pytest
+from load_db import engine_for, load, parse_variant_id, read_vep_tab, to_rows, vep_version
+from set_job_status import set_status
+from sqlalchemy import text
+
+HEADER = [
+    "Uploaded_variation", "Location", "Allele", "Consequence", "IMPACT", "SYMBOL",
+    "HGVSc", "HGVSp", "gnomADe_AF", "CLIN_SIG", "CADD_PHRED",
+]
+# Longer than the old VARCHAR(120) column.
+LONG_CLIN_SIG = (
+    "conflicting_classifications_of_pathogenicity,uncertain_significance,"
+    "likely_benign,benign,likely_pathogenic,pathogenic"
+)
+
+
+def vep_tab(tmp_path: Path, rows: list[list[str]]) -> Path:
+    path = tmp_path / "x.vep.tsv"
+    path.write_text("\n".join([
+        "## ENSEMBL VARIANT EFFECT PREDICTOR v113.0",
+        "## Output produced at 2026-09-11 12:00:00",
+        "#" + "\t".join(HEADER),
+        *("\t".join(r) for r in rows),
+    ]) + "\n")
+    return path
+
+
+ROWS = [
+    # SNV
+    ["22_19710700_C_T", "22:19710700", "T", "missense_variant", "MODERATE", "TBX1",
+     "ENST1:c.1C>T", "ENSP1:p.Arg1Trp", "0.0001", "pathogenic", "28.1"],
+    # Deletion: VEP reports a trimmed allele ("-") and a shifted Location; the ID keeps the VCF truth.
+    ["22_42126611_CT_C", "22:42126612", "-", "frameshift_variant", "HIGH", "CYP2D6",
+     "-", "-", "-", LONG_CLIN_SIG, "-"],
+]
+
+
+@pytest.mark.parametrize(
+    ("uid", "expected"),
+    [
+        ("22_19710700_C_T", ("22", 19710700, "C", "T")),
+        ("chrUn_KI270742v1_100_A_AT", ("chrUn_KI270742v1", 100, "A", "AT")),
+    ],
+)
+def test_parse_variant_id(uid: str, expected: tuple) -> None:
+    assert parse_variant_id(uid) == expected
+
+
+@pytest.mark.parametrize("uid", ["rs123", "12345", "22_x_A_G", "."])
+def test_parse_variant_id_rejects_ids_not_set_by_normalise(uid: str) -> None:
+    with pytest.raises(ValueError, match="CHROM_POS_REF_ALT"):
+        parse_variant_id(uid)
+
+
+def test_to_rows_takes_alleles_from_the_id_and_nulls_dashes(tmp_path: Path) -> None:
+    rows = to_rows(read_vep_tab(vep_tab(tmp_path, ROWS)), job_id="j")
+    snv, deletion = rows
+    assert (snv["chrom"], snv["pos"], snv["ref"], snv["alt"]) == ("22", 19710700, "C", "T")
+    assert (deletion["pos"], deletion["ref"], deletion["alt"]) == (42126611, "CT", "C")
+    assert snv["gnomad_af"] == 0.0001 and deletion["gnomad_af"] is None
+    assert deletion["hgvsc"] is None and deletion["gene"] == "CYP2D6"
+    assert deletion["clinvar_sig"] == LONG_CLIN_SIG
+    ann = json.loads(deletion["annotations"])
+    assert "-" not in ann.values() and "CADD_PHRED" not in ann
+    assert json.loads(snv["annotations"])["CADD_PHRED"] == "28.1"
+
+
+def test_vep_version(tmp_path: Path) -> None:
+    assert vep_version(vep_tab(tmp_path, ROWS)) == "113.0"
+
+
+def test_engine_for_uses_psycopg_for_any_postgres_url() -> None:
+    for url in ("postgresql+asyncpg://u:p@h/db", "postgresql://u:p@h/db"):
+        assert engine_for(url).url.drivername == "postgresql+psycopg"
+
+
+def new_job(engine, status: str = "running") -> str:
+    job_id = str(uuid.uuid4())
+    with engine.begin() as conn:
+        conn.execute(text("INSERT INTO jobs (id, status) VALUES (:id, :s)"), {"id": job_id, "s": status})
+    return job_id
+
+
+def job_and_count(engine, job_id: str) -> tuple:
+    with engine.connect() as conn:
+        job = conn.execute(text("SELECT status, vep_version, log FROM jobs WHERE id=:id"),
+                           {"id": job_id}).one()
+        n = conn.execute(text("SELECT count(*) FROM variants WHERE job_id=:id"), {"id": job_id}).scalar()
+    return (*job, n)
+
+
+def test_load_is_idempotent_and_marks_job_succeeded(engine, tmp_path: Path) -> None:
+    job_id = new_job(engine)
+    rows = to_rows(read_vep_tab(vep_tab(tmp_path, ROWS)), job_id=job_id)
+    load(engine, job_id, rows, vep="113.0")
+    load(engine, job_id, rows, vep="113.0")  # a retried task must not duplicate variants
+    assert job_and_count(engine, job_id) == ("succeeded", "113.0", None, 2)
+
+
+def test_load_with_no_variants_still_succeeds(engine, tmp_path: Path) -> None:
+    job_id = new_job(engine)
+    rows = to_rows(read_vep_tab(vep_tab(tmp_path, [])), job_id=job_id)
+    load(engine, job_id, rows, vep="113.0")
+    assert job_and_count(engine, job_id) == ("succeeded", "113.0", None, 0)
+
+
+def test_set_status_failed_records_the_reason(engine) -> None:
+    job_id = new_job(engine)
+    set_status(engine, job_id, "failed", "workflow annotate-abc Failed")
+    status, _, log, _ = job_and_count(engine, job_id)
+    assert (status, log) == ("failed", "workflow annotate-abc Failed")
+
+
+def test_set_status_never_overrides_a_succeeded_job(engine) -> None:
+    job_id = new_job(engine, status="succeeded")
+    set_status(engine, job_id, "failed", "late exit handler")
+    assert job_and_count(engine, job_id)[0] == "succeeded"
+
+
+@pytest.mark.parametrize(
+    ("raw", "expected_query"),
+    [
+        # asyncpg's ssl= becomes libpq's sslmode= for psycopg.
+        ("postgresql+asyncpg://u:p@h/db?ssl=require", {"sslmode": "require"}),
+        ("postgresql://u:p@h/db?sslmode=require", {"sslmode": "require"}),
+        ("postgresql+asyncpg://u:p@h/db", {}),
+    ],
+)
+def test_engine_for_translates_ssl_options(raw: str, expected_query: dict) -> None:
+    url = engine_for(raw).url
+    assert url.drivername == "postgresql+psycopg"
+    assert dict(url.query) == expected_query

scripts/fetch-demo-data.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# Fetch the public demo slice: a real GIAB genome and real ClinVar labels, chr22 only.
+# Provenance, licences and citations: docs/data.md
+set -euo pipefail
+
+OUT_DIR=${OUT_DIR:-data}
+CLINVAR_URL=${CLINVAR_URL:-https://ftp.ncbi.nlm.nih.gov/pub/clinvar/vcf_GRCh38/clinvar.vcf.gz}
+GIAB_URL=${GIAB_URL:-https://ftp-trace.ncbi.nlm.nih.gov/ReferenceSamples/giab/release/AshkenazimTrio/HG002_NA24385_son/NISTv4.2.1/GRCh38/HG002_GRCh38_1_22_v4.2.1_benchmark.vcf.gz}
+
+for tool in bcftools tabix; do
+  command -v "$tool" >/dev/null || {
+    echo "$tool is required (brew install bcftools, or apt install bcftools tabix)" >&2
+    exit 1
+  }
+done
+
+mkdir -p "$OUT_DIR"
+
+# Both sources are indexed, so bcftools streams one chromosome instead of downloading a whole
+# genome. ClinVar names contigs "22"; GIAB names them "chr22".
+echo "==> GIAB HG002 (NA24385) v4.2.1 benchmark, chr22 -> $OUT_DIR/example.vcf.gz"
+bcftools view -r chr22 "$GIAB_URL" -Oz -o "$OUT_DIR/example.vcf.gz"
+tabix -f -p vcf "$OUT_DIR/example.vcf.gz"
+
+echo "==> ClinVar GRCh38, chr22 -> $OUT_DIR/clinvar.chr22.vcf.gz"
+bcftools view -r 22 "$CLINVAR_URL" -Oz -o "$OUT_DIR/clinvar.chr22.vcf.gz"
+tabix -f -p vcf "$OUT_DIR/clinvar.chr22.vcf.gz"
+
+echo
+echo "Fetched:"
+ls -lh "$OUT_DIR/example.vcf.gz" "$OUT_DIR/clinvar.chr22.vcf.gz"
+cat <<'NEXT'
+
+Next:
+  make pipeline VCF=data/example.vcf.gz          # annotate the GIAB sample (needs a VEP cache)
+  make annotate JOB=<job id> VCF=data/example.vcf.gz
+NEXT

web/.dockerignore

@@ -0,0 +1,5 @@
+# .env holds the local dev API URL; the image reads PUBLIC_API_URL at runtime instead.
+.env
+node_modules/
+build/
+.svelte-kit/

web/.env

@@ -1 +1 @@
-PUBLIC_API_URL=http://localhost:8000
+PUBLIC_API_URL=http://localhost:8000/api

web/package-lock.json

@@ -11,6 +11,7 @@
         "@sveltejs/adapter-node": "^5.2.0",
         "@sveltejs/kit": "^2.5.0",
         "@sveltejs/vite-plugin-svelte": "^4.0.0",
+        "@types/node": "^22.20.2",
         "svelte": "^5.0.0",
         "svelte-check": "^4.0.0",
         "typescript": "^5.5.0",
@@ -1091,6 +1092,16 @@
       "dev": true,
       "license": "MIT"
     },
+    "node_modules/@types/node": {
+      "version": "22.20.2",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.2.tgz",
+      "integrity": "sha512-xlvWf4Vs9n1PEVYwP1n4vvG07M6y8WgvJ2t0vbrWTmijsIHp1cS+uJ2kMIRdY3nHZK0nCYKrPeD171+SzF4/zw==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "undici-types": "~6.21.0"
+      }
+    },
     "node_modules/@types/resolve": {
       "version": "1.20.2",
       "resolved": "https://registry.npmjs.org/@types/resolve/-/resolve-1.20.2.tgz",
@@ -2020,6 +2031,13 @@
         "node": ">=14.17"
       }
     },
+    "node_modules/undici-types": {
+      "version": "6.21.0",
+      "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz",
+      "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==",
+      "dev": true,
+      "license": "MIT"
+    },
     "node_modules/vite": {
       "version": "5.4.21",
       "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.21.tgz",

web/package.json

@@ -14,6 +14,7 @@
     "@sveltejs/adapter-node": "^5.2.0",
     "@sveltejs/kit": "^2.5.0",
     "@sveltejs/vite-plugin-svelte": "^4.0.0",
+    "@types/node": "^22.20.2",
     "svelte": "^5.0.0",
     "svelte-check": "^4.0.0",
     "typescript": "^5.5.0",

web/src/lib/api.ts

@@ -1,7 +1,13 @@
-import { PUBLIC_API_URL } from '$env/static/public';
+// Dynamic, not static: the same image serves /api behind the ingress and a full URL elsewhere.
+import { env } from '$env/dynamic/public';
+import { variantQuery, type VariantFilters } from './query';
 
-export type Sample = { id: string; name: string; vcf_uri: string; assembly: string; created_at: string };
-export type Job = { id: string; sample_id: string; status: 'queued' | 'running' | 'succeeded' | 'failed'; created_at: string; finished_at: string | null };
+export type Assembly = 'GRCh38' | 'GRCh37';
+export type Sample = { id: string; name: string; vcf_uri: string; assembly: Assembly; created_at: string };
+export type Job = {
+  id: string; sample_id: string; status: 'queued' | 'running' | 'succeeded' | 'failed';
+  log: string | null; created_at: string; finished_at: string | null;
+};
 export type Variant = {
   id: number; chrom: string; pos: number; ref: string; alt: string; gene: string | null;
   consequence: string | null; impact: string | null; hgvsc: string | null; hgvsp: string | null;
@@ -10,16 +16,30 @@ export type Variant = {
 export type VariantPage = { items: Variant[]; total: number; limit: number; offset: number };
 
 async function req<T>(path: string, init?: RequestInit): Promise<T> {
-  const r = await fetch(`${PUBLIC_API_URL}${path}`, { headers: { 'content-type': 'application/json' }, ...init });
-  if (!r.ok) throw new Error(`${r.status} ${r.statusText} on ${path}`);
+  const r = await fetch(`${env.PUBLIC_API_URL ?? '/api'}${path}`, { headers: { 'content-type': 'application/json' }, ...init });
+  if (!r.ok) throw new Error(await errorReason(r));
   return r.json() as Promise<T>;
 }
 
+// FastAPI puts the reason in `detail`: a string, or a list of validation errors.
+async function errorReason(r: Response): Promise<string> {
+  const fallback = `${r.status} ${r.statusText}`;
+  try {
+    const { detail } = await r.json();
+    if (typeof detail === 'string') return detail;
+    if (Array.isArray(detail)) return detail.map((d: { msg?: string }) => d.msg ?? '').join('; ') || fallback;
+  } catch {
+    // not JSON (e.g. a proxy error page)
+  }
+  return fallback;
+}
+
 export const api = {
   samples: () => req<Sample[]>('/samples'),
-  createSample: (body: Pick<Sample, 'name' | 'vcf_uri'>) => req<Sample>('/samples', { method: 'POST', body: JSON.stringify(body) }),
+  createSample: (body: Pick<Sample, 'name' | 'vcf_uri' | 'assembly'>) =>
+    req<Sample>('/samples', { method: 'POST', body: JSON.stringify(body) }),
   jobsForSample: (sampleId: string) => req<Job[]>(`/samples/${sampleId}/jobs`),
   annotate: (sampleId: string) => req<Job>(`/samples/${sampleId}/annotate`, { method: 'POST' }),
   job: (id: string) => req<Job>(`/jobs/${id}`),
-  variants: (q: Record<string, string | number>) => req<VariantPage>(`/variants?${new URLSearchParams(q as Record<string, string>)}`)
+  variants: (filters: VariantFilters) => req<VariantPage>(`/variants?${variantQuery(filters)}`)
 };

web/src/lib/poll.test.ts

@@ -0,0 +1,54 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { poll } from './poll';
+
+beforeEach(() => vi.useFakeTimers());
+afterEach(() => vi.useRealTimers());
+
+describe('poll', () => {
+  it('fetches until done, then stops', async () => {
+    const values = ['queued', 'running', 'succeeded', 'never'];
+    const fetch = vi.fn(async () => values.shift()!);
+    const seen: string[] = [];
+    poll(fetch, { intervalMs: 1000, done: (v) => v === 'succeeded', onValue: (v) => seen.push(v), onError: () => {} });
+
+    await vi.advanceTimersByTimeAsync(5000);
+    expect(seen).toEqual(['queued', 'running', 'succeeded']);
+    expect(fetch).toHaveBeenCalledTimes(3);
+  });
+
+  it('never has two requests in flight when the API is slower than the interval', async () => {
+    let inFlight = 0;
+    let maxInFlight = 0;
+    const fetch = async () => {
+      inFlight++;
+      maxInFlight = Math.max(maxInFlight, inFlight);
+      await new Promise((r) => setTimeout(r, 2500));
+      inFlight--;
+      return 'running';
+    };
+    poll(fetch, { intervalMs: 1000, done: () => false, onValue: () => {}, onError: () => {} });
+
+    await vi.advanceTimersByTimeAsync(10_000);
+    expect(maxInFlight).toBe(1);
+  });
+
+  it('reports an error once and stops', async () => {
+    const fetch = vi.fn(async () => { throw new Error('502'); });
+    const onError = vi.fn();
+    poll(fetch, { intervalMs: 1000, done: () => false, onValue: () => {}, onError });
+
+    await vi.advanceTimersByTimeAsync(5000);
+    expect(onError).toHaveBeenCalledOnce();
+    expect(fetch).toHaveBeenCalledOnce();
+  });
+
+  it('stop() cancels future fetches, e.g. when the page unmounts', async () => {
+    const fetch = vi.fn(async () => 'running');
+    const stop = poll(fetch, { intervalMs: 1000, done: () => false, onValue: () => {}, onError: () => {} });
+
+    await vi.advanceTimersByTimeAsync(1500);
+    stop();
+    await vi.advanceTimersByTimeAsync(10_000);
+    expect(fetch).toHaveBeenCalledTimes(1);
+  });
+});

web/src/lib/poll.ts

@@ -0,0 +1,35 @@
+export type PollOptions<T> = {
+  intervalMs: number;
+  done: (value: T) => boolean;
+  onValue: (value: T) => void;
+  onError: (error: unknown) => void;
+};
+
+/**
+ * Fetch every `intervalMs` until `done` returns true or a fetch fails. The next request is only
+ * scheduled once the previous one settles, so a slow API never has overlapping requests.
+ * Returns a stop function; call it when the component unmounts.
+ */
+export function poll<T>(fetch: () => Promise<T>, opts: PollOptions<T>): () => void {
+  let stopped = false;
+  let timer: ReturnType<typeof setTimeout> | undefined;
+
+  const tick = async () => {
+    let value: T;
+    try {
+      value = await fetch();
+    } catch (e) {
+      if (!stopped) opts.onError(e);
+      return;
+    }
+    if (stopped) return;
+    opts.onValue(value);
+    if (!opts.done(value)) timer = setTimeout(tick, opts.intervalMs);
+  };
+
+  timer = setTimeout(tick, opts.intervalMs);
+  return () => {
+    stopped = true;
+    clearTimeout(timer);
+  };
+}

web/src/lib/proxy.test.ts

@@ -0,0 +1,70 @@
+import { describe, expect, it, vi } from 'vitest';
+import { proxyToApi } from './proxy';
+
+// Same signature as fetch, so mock.calls stays typed.
+type FetchArgs = [input: RequestInfo | URL, init?: RequestInit];
+
+const upstreamOk = (body: unknown = { ok: true }) =>
+  vi.fn(async (..._args: FetchArgs) =>
+    new Response(JSON.stringify(body), { status: 200, headers: { 'content-type': 'application/json' } })
+  );
+
+describe('proxyToApi', () => {
+  it('forwards the path and query to the API', async () => {
+    const fetch = upstreamOk();
+    await proxyToApi({
+      request: new Request('https://web.example/api/variants?job_id=abc&limit=100'),
+      path: 'variants',
+      search: '?job_id=abc&limit=100',
+      base: 'https://api.internal',
+      fetch
+    });
+    expect(fetch.mock.calls[0][0]).toBe('https://api.internal/api/variants?job_id=abc&limit=100');
+  });
+
+  it('forwards POST bodies', async () => {
+    const fetch = upstreamOk();
+    await proxyToApi({
+      request: new Request('https://web.example/api/samples', {
+        method: 'POST',
+        body: JSON.stringify({ name: 'HG002' }),
+        headers: { 'content-type': 'application/json' }
+      }),
+      path: 'samples',
+      search: '',
+      base: 'https://api.internal',
+      fetch
+    });
+    const init = fetch.mock.calls[0][1]!;
+    expect(init.method).toBe('POST');
+    expect(init.body).toBe('{"name":"HG002"}');
+  });
+
+  it('passes the upstream status through, so the UI sees 404s and 409s', async () => {
+    const fetch = vi.fn(async (..._args: FetchArgs) =>
+      new Response(JSON.stringify({ detail: 'sample not found' }), { status: 404, headers: { 'content-type': 'application/json' } })
+    );
+    const res = await proxyToApi({
+      request: new Request('https://web.example/api/jobs/x'),
+      path: 'jobs/x',
+      search: '',
+      base: 'https://api.internal',
+      fetch
+    });
+    expect(res.status).toBe(404);
+    expect(await res.json()).toEqual({ detail: 'sample not found' });
+  });
+
+  it('reports a 502 when the API cannot be reached', async () => {
+    const fetch = vi.fn(async (..._args: FetchArgs): Promise<Response> => { throw new Error('ECONNREFUSED'); });
+    const res = await proxyToApi({
+      request: new Request('https://web.example/api/samples'),
+      path: 'samples',
+      search: '',
+      base: 'https://api.internal',
+      fetch
+    });
+    expect(res.status).toBe(502);
+    expect((await res.json()).detail).toContain('ECONNREFUSED');
+  });
+});

web/src/lib/proxy.ts

@@ -0,0 +1,35 @@
+export type ProxyOptions = {
+  request: Request;
+  path: string;
+  search: string;
+  base: string;
+  fetch: typeof globalThis.fetch;
+};
+
+/**
+ * Forward /api/* to the API service.
+ *
+ * Behind the Kubernetes ingress this never runs — the ingress routes /api itself. On Cloud Run it
+ * keeps the UI and the API on one origin, so there is one public URL and no CORS to configure.
+ */
+export async function proxyToApi({ request, path, search, base, fetch }: ProxyOptions): Promise<Response> {
+  const hasBody = request.method !== 'GET' && request.method !== 'HEAD';
+  const contentType = request.headers.get('content-type') ?? 'application/json';
+  try {
+    const upstream = await fetch(`${base}/api/${path}${search}`, {
+      method: request.method,
+      headers: { 'content-type': contentType },
+      body: hasBody ? await request.text() : undefined
+    });
+    return new Response(upstream.body, {
+      status: upstream.status,
+      headers: { 'content-type': upstream.headers.get('content-type') ?? 'application/json' }
+    });
+  } catch (e) {
+    // Shaped like FastAPI's errors so the UI reports it the same way.
+    return new Response(JSON.stringify({ detail: `API unreachable: ${(e as Error).message}` }), {
+      status: 502,
+      headers: { 'content-type': 'application/json' }
+    });
+  }
+}

web/src/lib/query.test.ts

@@ -0,0 +1,29 @@
+import { describe, expect, it } from 'vitest';
+import { variantQuery } from './query';
+
+describe('variantQuery', () => {
+  it('always includes the job id', () => {
+    expect(variantQuery({ jobId: 'abc' })).toBe('job_id=abc');
+  });
+
+  it('includes filters that have values', () => {
+    const q = new URLSearchParams(
+      variantQuery({ jobId: 'abc', gene: ' brca1 ', impact: 'HIGH', maxAf: 0.01, limit: 100 })
+    );
+    expect(q.get('gene')).toBe('brca1');
+    expect(q.get('impact')).toBe('HIGH');
+    expect(q.get('max_af')).toBe('0.01');
+    expect(q.get('limit')).toBe('100');
+  });
+
+  it('drops a cleared number input instead of sending "null"', () => {
+    // Svelte binds an emptied <input type="number"> to null.
+    for (const maxAf of [null, undefined, '', Number.NaN]) {
+      expect(variantQuery({ jobId: 'abc', maxAf })).toBe('job_id=abc');
+    }
+  });
+
+  it('drops blank gene and impact', () => {
+    expect(variantQuery({ jobId: 'abc', gene: '   ', impact: '' })).toBe('job_id=abc');
+  });
+});

web/src/lib/query.ts

@@ -0,0 +1,22 @@
+export type VariantFilters = {
+  jobId: string;
+  gene?: string;
+  impact?: string;
+  // An emptied <input type="number"> binds to null, so accept it and drop it.
+  maxAf?: number | string | null;
+  limit?: number;
+  offset?: number;
+};
+
+export function variantQuery(f: VariantFilters): string {
+  const q = new URLSearchParams({ job_id: f.jobId });
+  const gene = f.gene?.trim();
+  if (gene) q.set('gene', gene);
+  if (f.impact) q.set('impact', f.impact);
+  if (f.maxAf !== null && f.maxAf !== undefined && f.maxAf !== '' && Number.isFinite(Number(f.maxAf))) {
+    q.set('max_af', String(f.maxAf));
+  }
+  if (f.limit !== undefined) q.set('limit', String(f.limit));
+  if (f.offset !== undefined) q.set('offset', String(f.offset));
+  return q.toString();
+}

web/src/routes/+page.svelte

@@ -1,18 +1,30 @@
 <script lang="ts">
   import { onMount } from 'svelte';
-  import { api, type Sample } from '$lib/api';
+  import { api, type Assembly, type Sample } from '$lib/api';
 
   let samples = $state<Sample[]>([]);
+  let loaded = $state(false);
+  let loadError = $state<string | null>(null);
   let name = $state('');
   let vcfUri = $state('');
+  let assembly = $state<Assembly>('GRCh38');
   let error = $state<string | null>(null);
 
-  async function refresh() { samples = await api.samples(); }
+  async function refresh() {
+    try {
+      samples = await api.samples();
+      loadError = null;
+    } catch (e) {
+      loadError = (e as Error).message;
+    } finally {
+      loaded = true;
+    }
+  }
   onMount(refresh);
 
   async function add() {
     error = null;
-    try { await api.createSample({ name, vcf_uri: vcfUri }); name = ''; vcfUri = ''; await refresh(); }
+    try { await api.createSample({ name, vcf_uri: vcfUri, assembly }); name = ''; vcfUri = ''; await refresh(); }
     catch (e) { error = (e as Error).message; }
   }
 </script>
@@ -24,12 +36,19 @@
 <div style="display:flex; gap:0.5rem; flex-wrap:wrap">
   <input placeholder="Sample name" bind:value={name} aria-label="Sample name" />
   <input placeholder="gs://bucket/sample.vcf.gz or /data/example.vcf.gz" bind:value={vcfUri} aria-label="VCF path" style="flex:1; min-width: 20rem" />
+  <select bind:value={assembly} aria-label="Assembly">
+    <option>GRCh38</option><option>GRCh37</option>
+  </select>
   <button onclick={add} disabled={!name || !vcfUri}>Add sample</button>
 </div>
 {#if error}<p role="alert">Could not add the sample: {error}</p>{/if}
 
 <h2>Samples</h2>
-{#if samples.length === 0}
+{#if loadError}
+  <p role="alert">Could not load samples: {loadError}. <button class="quiet" onclick={refresh}>Retry</button></p>
+{:else if !loaded}
+  <p style="color:var(--ink-soft)">Loading samples…</p>
+{:else if samples.length === 0}
   <div class="empty">No samples yet. Add one above to run the annotation pipeline.</div>
 {:else}
   <table>

web/src/routes/api/[...path]/+server.ts

@@ -0,0 +1,15 @@
+import { env } from '$env/dynamic/private';
+import { proxyToApi } from '$lib/proxy';
+import type { RequestHandler } from './$types';
+
+const handle: RequestHandler = ({ request, params, url, fetch }) =>
+  proxyToApi({
+    request,
+    path: params.path,
+    search: url.search,
+    base: env.API_INTERNAL_URL ?? 'http://localhost:8000',
+    fetch
+  });
+
+export const GET = handle;
+export const POST = handle;

web/src/routes/samples/[id]/+page.svelte

@@ -1,38 +1,72 @@
 <script lang="ts">
-  import { onMount } from 'svelte';
+  import { onDestroy, onMount } from 'svelte';
   import { api, type Job, type VariantPage } from '$lib/api';
+  import { poll } from '$lib/poll';
+
+  const POLL_MS = 3000;
+  const finished = (j: Job) => j.status === 'succeeded' || j.status === 'failed';
 
   let { data } = $props();
   let job = $state<Job | null>(null);
   let page = $state<VariantPage | null>(null);
   let gene = $state('');
   let impact = $state('');
-  let maxAf = $state('0.01');
+  let maxAf = $state<number | null>(0.01);
   let busy = $state(false);
+  let error = $state<string | null>(null);
+  let stopPolling: (() => void) | null = null;
 
   onMount(async () => {
-    const jobs = await api.jobsForSample(data.sampleId);
-    const done = jobs.find(j => j.status === 'succeeded');
-    if (done) { job = done; await loadVariants(); }
-    else if (jobs.length) job = jobs[0];
+    try {
+      const jobs = await api.jobsForSample(data.sampleId); // newest first
+      const latest = jobs[0] ?? null;
+      const lastGood = jobs.find((j) => j.status === 'succeeded') ?? null;
+      // Follow a run in progress; otherwise show the latest results, else the latest failure.
+      job = latest && !finished(latest) ? latest : (lastGood ?? latest);
+      if (job) await follow(job);
+    } catch (e) {
+      error = (e as Error).message;
+    }
   });
+  onDestroy(() => stopPolling?.());
+
+  async function follow(j: Job) {
+    stopPolling?.();
+    if (j.status === 'succeeded') return loadVariants();
+    if (finished(j)) return;
+    stopPolling = poll(() => api.job(j.id), {
+      intervalMs: POLL_MS,
+      done: finished,
+      onValue: (next) => {
+        job = next;
+        if (next.status === 'succeeded') loadVariants();
+      },
+      onError: (e) => { error = `Lost track of the job: ${(e as Error).message}`; }
+    });
+  }
 
   async function runAnnotation() {
     busy = true;
+    error = null;
+    page = null;
+    try {
       job = await api.annotate(data.sampleId);
-    const poll = setInterval(async () => {
-      if (!job) return;
-      job = await api.job(job.id);
-      if (job.status === 'succeeded' || job.status === 'failed') { clearInterval(poll); busy = false; if (job.status === 'succeeded') await loadVariants(); }
-    }, 3000);
+      await follow(job);
+    } catch (e) {
+      error = (e as Error).message;
+    } finally {
+      busy = false;
+    }
   }
 
   async function loadVariants() {
     if (!job) return;
-    const q: Record<string, string> = { job_id: job.id, max_af: maxAf, limit: '100' };
-    if (gene) q.gene = gene;
-    if (impact) q.impact = impact;
-    page = await api.variants(q);
+    error = null;
+    try {
+      page = await api.variants({ jobId: job.id, gene, impact, maxAf, limit: 100 });
+    } catch (e) {
+      error = (e as Error).message;
+    }
   }
 
   const scoreClass = (s: number) => (s >= 0.8 ? 'score high' : s >= 0.5 ? 'score mid' : 'score');
@@ -41,10 +75,16 @@
 <a href="/">All samples</a>
 <h1>Sample {data.sampleId.slice(0, 8)}</h1>
 
-{#if !job}
-  <button onclick={runAnnotation} disabled={busy}>Run VEP annotation</button>
-{:else}
+{#if error}<p role="alert">{error}</p>{/if}
+
+{#if job}
   <p>Job <span class="hgvs">{job.id.slice(0, 8)}</span>: <span class="status {job.status}">{job.status}</span></p>
+  {#if job.status === 'failed' && job.log}
+    <pre class="hgvs" style="white-space:pre-wrap; background:white; border:1px solid var(--line); padding:0.75rem; max-height:16rem; overflow:auto">{job.log}</pre>
+  {/if}
+{/if}
+{#if !job || job.status === 'failed'}
+  <button onclick={runAnnotation} disabled={busy}>{job ? 'Run VEP annotation again' : 'Run VEP annotation'}</button>
 {/if}
 
 {#if job?.status === 'succeeded'}