fix: overhaul the platform skeleton, add a serverless deployment track

An end-to-end audit found the repo could not build, test or run as shipped. This fixes every finding, then adds a Cloud Run track so the demo costs about £1/month idle instead of ~£150. CI (red on its first run) - api: setuptools could not build the package (flat layout with app/ and alembic/) - web: missing @types/node; `vitest run` exited 1 with no test files - pipeline: the stub run needed a gitignored VCF, and no process had a stub block - ruff pinned, mypy configured, DB tests on real Postgres (pgserver locally, service in CI) ML serving (scores were meaningless) - the registered model now carries its own feature engineering and returns predict_proba, so serving sends raw columns and cannot drift from training - resolve by registry alias (stages are deprecated in MLflow 3) and record the real version; re-scoring upserts instead of failing on the unique constraint - ClinVar labels parsed from VEP's lowercase terms Pipeline - exact ref/alt recovered from a CHROM_POS_REF_ALT VCF ID; loading is idempotent - job status reaches running/failed/succeeded, so the UI stops polling dead jobs - DATABASE_URL travels in the environment or a Nextflow secret, never on a command line - VEP cache and plugins staged as inputs; the gcp profile runs tasks on Google Batch Deployment - the API serves /api (matching the ingress); the web app reads its API URL at runtime - migrations run in an init container under a Postgres advisory lock - terraform: custom VPC shared with Batch, private Cloud SQL, API enablement, Workload Identity bindings, Secret Manager, deletion protection - serverless track, now the default: Cloud Run services scaling to zero, a Cloud Run job for the Nextflow driver, and Neon or Cloud SQL behind one DATABASE_URL secret. GKE and Argo remain, behind -var deploy_kubernetes=true. See docs/cloud.md. Correctness and security - 409 on duplicate sample names, 422 on bad paging, natural chromosome ordering, wider VEP text columns, enum dropped on downgrade, the sample's assembly actually used - vcf_uri restricted to gs:// objects or files under the data root, blocking option injection - CORS restricted to configured origins; `make down` no longer deletes volumes Data - docs/data.md records the peer-reviewed, openly licensed sources (GIAB HG002, ClinVar, gnomAD) with citations and an honest evaluation plan; `make data` fetches a chr22 slice Verified: api 50 tests, ml 18, loader 16, web 12; ruff, mypy, svelte-check, terraform validate and both kustomize overlays clean.
2026-09-12 07:21:11 +01:00
parent 5463f489a3
commit 11fb6b3d73
100 changed files with 3431 additions and 340 deletions
@@ -0,0 +1,5 @@
+work/
+results/
+cache/
+tests/
+.nextflow*
@@ -1,5 +1,19 @@
-# Loader image: pandas + psycopg for LOAD_DB
-FROM python:3.12-slim
-RUN pip install --no-cache-dir pandas sqlalchemy "psycopg[binary]"
-COPY bin/load_db.py /usr/local/bin/load_db.py
-RUN chmod +x /usr/local/bin/load_db.py
+# Nextflow driver for the Argo annotate-vcf workflow. Tasks run in their own containers.
+FROM eclipse-temurin:21-jre
+ARG NXF_VER=26.04.6
+ENV NXF_VER=${NXF_VER} NXF_HOME=/opt/nextflow
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends curl ca-certificates \
+ && rm -rf /var/lib/apt/lists/* \
+ && curl -fsSL https://get.nextflow.io | bash \
+ && mv nextflow /usr/local/bin/nextflow \
+ && nextflow plugin install nf-google
+COPY main.nf nextflow.config /pipeline/
+COPY modules /pipeline/modules
+COPY bin /pipeline/bin
+COPY assets /pipeline/assets
+# CI passes the loader image built from the same commit, so driver and loader never drift.
+ARG LOADER_IMAGE=rarelens/loader:dev
+ENV RARELENS_LOADER_IMAGE=${LOADER_IMAGE}
+WORKDIR /work
+ENTRYPOINT ["nextflow"]
@@ -1,59 +1,133 @@
-#!/usr/bin/env python
-"""Load a VEP --tab output into the rarelens Postgres schema and mark the job succeeded."""
+#!/usr/bin/env python3
+"""Load VEP --tab output into the rarelens Postgres schema and mark the job succeeded.
+
+Variant identity (chrom/pos/ref/alt) comes from the VCF ID, which NORMALISE sets to
+CHROM_POS_REF_ALT: VEP's own Location/Allele columns trim indel alleles and shift positions.
+The database URL is read from $DATABASE_URL so it never appears on a command line or in .command.sh.
+Loading replaces any rows already stored for the job, so a retried task cannot duplicate variants.
+"""
 import argparse
 import json
+import os
+import re
 import sys
+from pathlib import Path

 import pandas as pd
-from sqlalchemy import create_engine, text
+from sqlalchemy import Engine, create_engine, text
+from sqlalchemy.engine import make_url
+
+INSERT_CHUNK = 5000
+VEP_VERSION = re.compile(r"^## ENSEMBL VARIANT EFFECT PREDICTOR v(\S+)")
+
+
+def engine_for(url: str) -> Engine:
+    """Accept the API's asyncpg URL (or a plain postgresql:// one) and use psycopg."""
+    parsed = make_url(url).set(drivername="postgresql+psycopg")
+    query = dict(parsed.query)
+    if "ssl" in query:  # asyncpg's spelling; libpq (psycopg) wants sslmode
+        query["sslmode"] = query.pop("ssl")
+    return create_engine(parsed.set(query=query))
+
+
+def read_vep_tab(path: str | Path) -> pd.DataFrame:
+    """Read VEP --tab output as strings, keeping "-" (VEP's missing marker) verbatim."""
+    with open(path) as fh:
+        for n, line in enumerate(fh):
+            if line.startswith("#Uploaded_variation"):
+                break
+        else:
+            raise ValueError(f"{path}: no #Uploaded_variation header; is this VEP --tab output?")
+    df = pd.read_csv(path, sep="\t", skiprows=n, dtype=str, keep_default_na=False)
+    return df.rename(columns={"#Uploaded_variation": "Uploaded_variation"})
+
+
+def vep_version(path: str | Path) -> str | None:
+    with open(path) as fh:
+        for line in fh:
+            if not line.startswith("##"):
+                return None
+            if m := VEP_VERSION.match(line):
+                return m.group(1)
+    return None
+
+
+def parse_variant_id(uid: str) -> tuple[str, int, str, str]:
+    # rsplit: contig names may contain "_" (chrUn_KI270742v1); positions and alleles never do.
+    parts = uid.rsplit("_", 3)
+    if len(parts) != 4 or not parts[1].isdigit():
+        raise ValueError(
+            f"unexpected Uploaded_variation {uid!r}; NORMALISE must set VCF IDs to CHROM_POS_REF_ALT"
+        )
+    chrom, pos, ref, alt = parts
+    return chrom, int(pos), ref, alt
+
+
+def _value(v: str | None) -> str | None:
+    return None if v in (None, "", "-") else v
+
+
+def to_rows(df: pd.DataFrame, job_id: str) -> list[dict]:
+    rows = []
+    for rec in df.to_dict("records"):
+        chrom, pos, ref, alt = parse_variant_id(rec["Uploaded_variation"])
+        af = _value(rec.get("gnomADe_AF"))
+        rows.append({
+            "job_id": job_id,
+            "chrom": chrom,
+            "pos": pos,
+            "ref": ref,
+            "alt": alt,
+            "gene": _value(rec.get("SYMBOL")),
+            "consequence": _value(rec.get("Consequence")),
+            "impact": _value(rec.get("IMPACT")),
+            "hgvsc": _value(rec.get("HGVSc")),
+            "hgvsp": _value(rec.get("HGVSp")),
+            "gnomad_af": float(af) if af is not None else None,
+            "clinvar_sig": _value(rec.get("CLIN_SIG")),
+            "annotations": json.dumps({k: v for k, v in rec.items() if _value(v) is not None}),
+        })
+    return rows
+
+
+def load(engine: Engine, job_id: str, rows: list[dict], vep: str | None) -> None:
+    with engine.begin() as conn:
+        conn.execute(text("DELETE FROM variants WHERE job_id = :id"), {"id": job_id})
+        for start in range(0, len(rows), INSERT_CHUNK):
+            conn.execute(
+                text("""
+                    INSERT INTO variants (job_id, chrom, pos, ref, alt, gene, consequence, impact,
+                                          hgvsc, hgvsp, gnomad_af, clinvar_sig, annotations)
+                    VALUES (:job_id, :chrom, :pos, :ref, :alt, :gene, :consequence, :impact,
+                            :hgvsc, :hgvsp, :gnomad_af, :clinvar_sig, CAST(:annotations AS jsonb))
+                """),
+                rows[start:start + INSERT_CHUNK],
+            )
+        conn.execute(
+            text("""
+                UPDATE jobs SET status = 'succeeded', vep_version = :vep, log = NULL,
+                                finished_at = now()
+                WHERE id = :id
+            """),
+            {"id": job_id, "vep": vep},
+        )


 def main() -> None:
    p = argparse.ArgumentParser()
    p.add_argument("--tsv", required=True)
    p.add_argument("--job-id", required=True)
-    p.add_argument("--db-url", required=True)
+    p.add_argument("--dry-run", action="store_true", help="parse only; do not touch the database")
    a = p.parse_args()

-    df = pd.read_csv(a.tsv, sep="\t", comment="#", header=None, dtype=str)
-    with open(a.tsv) as fh:
-        header = next(l for l in fh if l.startswith("#Uploaded_variation")).lstrip("#").rstrip().split("\t")
-    df.columns = header
-
-    chrom_pos = df["Location"].str.split(":", expand=True)
-    rows = []
-    for i, r in df.iterrows():
-        ref, _, alt = r["Uploaded_variation"].partition("/") if "/" in r["Uploaded_variation"] else ("", "", r["Allele"])
-        rows.append({
-            "job_id": a.job_id,
-            "chrom": chrom_pos.iloc[i, 0],
-            "pos": int(chrom_pos.iloc[i, 1].split("-")[0]),
-            "ref": ref or "-",
-            "alt": r["Allele"],
-            "gene": r.get("SYMBOL") if r.get("SYMBOL") != "-" else None,
-            "consequence": r["Consequence"],
-            "impact": r["IMPACT"],
-            "hgvsc": None if r.get("HGVSc") == "-" else r.get("HGVSc"),
-            "hgvsp": None if r.get("HGVSp") == "-" else r.get("HGVSp"),
-            "gnomad_af": None if r.get("gnomADe_AF", "-") == "-" else float(r["gnomADe_AF"]),
-            "clinvar_sig": None if r.get("CLIN_SIG", "-") == "-" else r["CLIN_SIG"],
-            "annotations": json.dumps({k: v for k, v in r.items() if v != "-"}),
-        })
-
-    if a.db_url == "none":
+    rows = to_rows(read_vep_tab(a.tsv), job_id=a.job_id)
+    if a.dry_run:
        print(f"{len(rows)} variants parsed (dry run, no DB)")
        return
-
-    engine = create_engine(a.db_url.replace("+asyncpg", "+psycopg"))
-    with engine.begin() as conn:
-        conn.execute(text("""
-            INSERT INTO variants (job_id, chrom, pos, ref, alt, gene, consequence, impact,
-                                  hgvsc, hgvsp, gnomad_af, clinvar_sig, annotations)
-            VALUES (:job_id, :chrom, :pos, :ref, :alt, :gene, :consequence, :impact,
-                    :hgvsc, :hgvsp, :gnomad_af, :clinvar_sig, CAST(:annotations AS jsonb))
-        """), rows)
-        conn.execute(text("UPDATE jobs SET status='succeeded', finished_at=now() WHERE id=:id"),
-                     {"id": a.job_id})
+    url = os.environ.get("DATABASE_URL")
+    if not url:
+        sys.exit("DATABASE_URL is not set")
+    load(engine_for(url), a.job_id, rows, vep=vep_version(a.tsv))
    print(f"loaded {len(rows)} variants for job {a.job_id}", file=sys.stderr)


@@ -0,0 +1,40 @@
+#!/usr/bin/env python3
+"""Set a job's status, e.g. from the Argo exit handler when a workflow fails.
+
+A succeeded job is never overwritten: the loader's success is the source of truth.
+"""
+import argparse
+import os
+import sys
+
+from load_db import engine_for
+from sqlalchemy import Engine, text
+
+
+def set_status(engine: Engine, job_id: str, status: str, log: str | None) -> None:
+    with engine.begin() as conn:
+        conn.execute(
+            text("""
+                UPDATE jobs
+                SET status = CAST(:status AS jobstatus), log = :log,
+                    finished_at = CASE WHEN :status IN ('succeeded', 'failed') THEN now() END
+                WHERE id = :id AND status <> 'succeeded'
+            """),
+            {"id": job_id, "status": status, "log": log},
+        )
+
+
+def main() -> None:
+    p = argparse.ArgumentParser()
+    p.add_argument("--job-id", required=True)
+    p.add_argument("--status", required=True, choices=["running", "failed"])
+    p.add_argument("--log")
+    a = p.parse_args()
+    url = os.environ.get("DATABASE_URL")
+    if not url:
+        sys.exit("DATABASE_URL is not set")
+    set_status(engine_for(url), a.job_id, a.status, a.log)
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,8 @@
+# Task image for LOAD_DB and the Argo exit handler.
+FROM python:3.12-slim
+# procps: Nextflow uses `ps` inside task containers to collect metrics.
+RUN apt-get update && apt-get install -y --no-install-recommends procps && rm -rf /var/lib/apt/lists/*
+COPY requirements.txt /tmp/requirements.txt
+RUN pip install --no-cache-dir -r /tmp/requirements.txt
+COPY bin/load_db.py bin/set_job_status.py /usr/local/bin/
+RUN chmod +x /usr/local/bin/load_db.py /usr/local/bin/set_job_status.py
@@ -7,9 +7,17 @@ include { LOAD_DB }   from './modules/load_db'

 workflow {
    if (!params.vcf) error "Provide --vcf"
-    vcf_ch = Channel.fromPath(params.vcf, checkIfExists: true)
+    if (workflow.profile.tokenize(',').contains('gcp') && !(params.project && params.bucket)) {
+        error "The gcp profile needs --project and --bucket (or GCP_PROJECT and GCS_BUCKET)"
+    }
+    // Stub runs (CI) have no VEP cache or plugin data on disk.
+    def must_exist = !workflow.stubRun
+
+    vcf_ch  = Channel.fromPath(params.vcf, checkIfExists: true)
+    cache   = file(params.vep_cache, checkIfExists: must_exist)
+    plugins = file(params.vep_plugin_data ?: "${projectDir}/assets/NO_FILE", checkIfExists: must_exist)

    NORMALISE(vcf_ch)
-    VEP(NORMALISE.out.vcf)
-    LOAD_DB(VEP.out.tsv, params.job_id ?: 'local', params.db_url ?: 'none')
+    VEP(NORMALISE.out.vcf, cache, plugins)
+    LOAD_DB(VEP.out.tsv, params.job_id ?: 'dry-run')
 }
@@ -3,11 +3,18 @@ process LOAD_DB {
    input:
        path tsv
        val job_id
-        val db_url
    output: stdout

+    // DATABASE_URL reaches the task through its environment (docker profile) or a Nextflow
+    // secret (gcp profile); it never appears on a command line or in .command.sh.
    script:
+    def dry_run = job_id == 'dry-run' ? '--dry-run' : ''
    """
-    load_db.py --tsv $tsv --job-id $job_id --db-url '$db_url'
+    load_db.py --tsv $tsv --job-id $job_id $dry_run
+    """
+
+    stub:
+    """
+    echo "stub: would load $tsv for job $job_id"
    """
 }
@@ -3,9 +3,16 @@ process NORMALISE {
    input:  path vcf
    output: path "${vcf.simpleName}.norm.vcf.gz", emit: vcf

+    // Split multiallelics, then set each ID to CHROM_POS_REF_ALT. VEP echoes the ID back as
+    // Uploaded_variation, and the loader takes exact VCF alleles from it (VEP trims indels).
    script:
    """
-    bcftools norm -m -both -Oz -o ${vcf.simpleName}.norm.vcf.gz $vcf
-    bcftools index -t ${vcf.simpleName}.norm.vcf.gz
+    bcftools norm -m -both -Ou $vcf \\
+        | bcftools annotate --set-id '%CHROM\\_%POS\\_%REF\\_%FIRST_ALT' -Oz -o ${vcf.simpleName}.norm.vcf.gz
+    """
+
+    stub:
+    """
+    touch ${vcf.simpleName}.norm.vcf.gz
    """
 }
@@ -1,16 +1,31 @@
 process VEP {
    tag "$vcf.simpleName"
    publishDir params.outdir, mode: 'copy'
-    input:  path vcf
+    input:
+        path vcf
+        path cache          // staged as an input so it is visible inside the container
+        path plugin_data    // assets/NO_FILE when plugins are not configured
    output:
        path "${vcf.simpleName}.vep.tsv",      emit: tsv
        path "${vcf.simpleName}.vep_summary.html"

    script:
+    // Plugins run only when --vep_plugin_data names a directory holding the plugin modules
+    // (INSTALL.pl -a p -g CADD,AlphaMissense -r <dir>) and the data files named in params.
+    def plugins = plugin_data.name == 'NO_FILE' ? '' : [
+        "--dir_plugins ${plugin_data}",
+        "--plugin CADD,snv=${plugin_data}/${params.cadd_snv},indels=${plugin_data}/${params.cadd_indels}",
+        "--plugin AlphaMissense,file=${plugin_data}/${params.alphamissense}",
+    ].join(' ')
    """
    vep -i $vcf -o ${vcf.simpleName}.vep.tsv --tab \\
-        --assembly ${params.assembly} --cache --dir_cache ${params.vep_cache} --offline \\
-        --everything --pick --af_gnomade --plugin CADD --plugin AlphaMissense \\
+        --assembly ${params.assembly} --cache --offline --dir_cache ${cache} \\
+        --everything --pick ${plugins} \\
        --stats_file ${vcf.simpleName}.vep_summary.html --fork ${task.cpus}
    """
+
+    stub:
+    """
+    touch ${vcf.simpleName}.vep.tsv ${vcf.simpleName}.vep_summary.html
+    """
 }
@@ -1,24 +1,52 @@
 params {
-    vcf        = null
-    job_id     = null
-    db_url     = null
-    outdir     = "results"
-    assembly   = "GRCh38"
-    vep_cache  = "${projectDir}/cache/vep"     // download once with `vep_install`; or use --offline false
-    vep_plugins = "CADD,AlphaMissense"
-}
+    vcf             = null
+    job_id          = null        // omit for a dry run that parses but does not load
+    outdir          = "results"
+    assembly        = "GRCh38"
+    vep_cache       = "${projectDir}/cache/vep"   // INSTALL.pl -a cf -s homo_sapiens -y GRCh38 -c <dir>
+    vep_plugin_data = null        // CADD + AlphaMissense modules and data; plugins skipped when null
+    cadd_snv        = "whole_genome_SNVs.tsv.gz"
+    cadd_indels     = "gnomad.genomes.r4.0.indel.tsv.gz"
+    alphamissense   = "AlphaMissense_hg38.tsv.gz"
+    // The driver image sets this to the loader image built from the same commit.
+    loader_image    = System.getenv('RARELENS_LOADER_IMAGE') ?: 'rarelens/loader:dev'

-profiles {
-    docker { docker.enabled = true }
-    gcp {
-        process.executor = 'k8s'               // runs inside GKE via Argo; Nextflow k8s executor
-        workDir = "gs://${params.bucket}/work"
-        google.project = params.project
-    }
+    // gcp profile; the Argo workflow provides these through the pipeline-config ConfigMap.
+    project         = System.getenv('GCP_PROJECT')
+    region          = System.getenv('GCP_REGION') ?: 'europe-west2'
+    bucket          = System.getenv('GCS_BUCKET')
 }

 process {
+    shell = ['/bin/bash', '-euo', 'pipefail']
    withName: VEP        { container = 'ensemblorg/ensembl-vep:release_113.0'; cpus = 4; memory = '8 GB' }
    withName: NORMALISE  { container = 'quay.io/biocontainers/bcftools:1.20--h8b25389_0' }
-    withName: LOAD_DB    { container = 'ghcr.io/lynchaos/rarelens-loader:latest' }
+    withName: LOAD_DB    { container = params.loader_image }
+}
+
+profiles {
+    docker {
+        docker.enabled      = true
+        docker.envWhitelist = ['DATABASE_URL']
+        // Lets the loader reach a Postgres published on the host (docker-compose's port 5432).
+        docker.runOptions   = '--add-host=host.docker.internal:host-gateway'
+    }
+    gcp {
+        // The driver runs in the Argo pod; each task runs as a Google Batch job, which is what a
+        // gs:// work directory requires (the k8s executor needs a shared ReadWriteMany volume).
+        workDir = "gs://${params.bucket}/work"
+        params.vep_cache = "gs://${params.bucket}/refs/vep"
+        google {
+            project  = params.project
+            location = params.region
+            batch.serviceAccountEmail = "rarelens-pipeline@${params.project}.iam.gserviceaccount.com"
+            batch.network    = "projects/${params.project}/global/networks/rarelens-vpc"
+            batch.subnetwork = "projects/${params.project}/regions/${params.region}/subnetworks/rarelens-gke"
+        }
+        process {
+            executor = 'google-batch'
+            // Google Secret Manager secret created by Terraform (infra/terraform/secrets.tf).
+            withName: LOAD_DB { secret = 'DATABASE_URL' }
+        }
+    }
 }
@@ -0,0 +1,3 @@
+pandas>=2.2
+sqlalchemy>=2.0
+psycopg[binary]>=3.2
@@ -0,0 +1,60 @@
+"""Loader tests. DB tests use DATABASE_URL (CI's Postgres service) or a throwaway pgserver."""
+import os
+import sys
+import tempfile
+from collections.abc import Iterator
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "bin"))
+
+# The subset of the Alembic schema (api/alembic/versions) that the loader writes to.
+SCHEMA = """
+CREATE TYPE jobstatus AS ENUM ('queued', 'running', 'succeeded', 'failed');
+CREATE TABLE jobs (
+    id uuid PRIMARY KEY,
+    status jobstatus NOT NULL,
+    vep_version text,
+    log text,
+    finished_at timestamptz
+);
+CREATE TABLE variants (
+    id serial PRIMARY KEY,
+    job_id uuid NOT NULL,
+    chrom text NOT NULL, pos integer NOT NULL, ref text NOT NULL, alt text NOT NULL,
+    gene text, consequence text, impact text, hgvsc text, hgvsp text,
+    gnomad_af double precision, clinvar_sig text, annotations jsonb NOT NULL
+);
+"""
+
+_server = None
+
+
+def _url() -> str | None:
+    global _server
+    if url := os.environ.get("DATABASE_URL"):
+        return url
+    try:
+        import pgserver
+    except ImportError:
+        return None
+    _server = pgserver.get_server(tempfile.mkdtemp(prefix="loader-pg-"), cleanup_mode="delete")
+    return "postgresql://postgres@/postgres?host=" + _server.get_uri().split("host=", 1)[1]
+
+
+@pytest.fixture
+def engine() -> Iterator:
+    url = _url()
+    if not url:
+        if os.environ.get("REQUIRE_DB"):
+            pytest.fail("REQUIRE_DB is set but no database is available")
+        pytest.skip("no DATABASE_URL and pgserver is not installed")
+    from load_db import engine_for
+
+    eng = engine_for(url)
+    with eng.begin() as conn:
+        conn.exec_driver_sql("DROP TABLE IF EXISTS variants, jobs; DROP TYPE IF EXISTS jobstatus")
+        conn.exec_driver_sql(SCHEMA)
+    yield eng
+    eng.dispose()
@@ -0,0 +1,7 @@
+##fileformat=VCFv4.2
+##source=rarelens synthetic smoke-test fixture (not real sample data)
+##contig=<ID=22,length=50818468>
+#CHROM	POS	ID	REF	ALT	QUAL	FILTER	INFO
+22	19710700	.	C	T	.	PASS	.
+22	29091857	.	G	A,C	.	PASS	.
+22	42126611	.	CT	C	.	PASS	.
@@ -0,0 +1,136 @@
+import json
+import uuid
+from pathlib import Path
+
+import pytest
+from load_db import engine_for, load, parse_variant_id, read_vep_tab, to_rows, vep_version
+from set_job_status import set_status
+from sqlalchemy import text
+
+HEADER = [
+    "Uploaded_variation", "Location", "Allele", "Consequence", "IMPACT", "SYMBOL",
+    "HGVSc", "HGVSp", "gnomADe_AF", "CLIN_SIG", "CADD_PHRED",
+]
+# Longer than the old VARCHAR(120) column.
+LONG_CLIN_SIG = (
+    "conflicting_classifications_of_pathogenicity,uncertain_significance,"
+    "likely_benign,benign,likely_pathogenic,pathogenic"
+)
+
+
+def vep_tab(tmp_path: Path, rows: list[list[str]]) -> Path:
+    path = tmp_path / "x.vep.tsv"
+    path.write_text("\n".join([
+        "## ENSEMBL VARIANT EFFECT PREDICTOR v113.0",
+        "## Output produced at 2026-09-11 12:00:00",
+        "#" + "\t".join(HEADER),
+        *("\t".join(r) for r in rows),
+    ]) + "\n")
+    return path
+
+
+ROWS = [
+    # SNV
+    ["22_19710700_C_T", "22:19710700", "T", "missense_variant", "MODERATE", "TBX1",
+     "ENST1:c.1C>T", "ENSP1:p.Arg1Trp", "0.0001", "pathogenic", "28.1"],
+    # Deletion: VEP reports a trimmed allele ("-") and a shifted Location; the ID keeps the VCF truth.
+    ["22_42126611_CT_C", "22:42126612", "-", "frameshift_variant", "HIGH", "CYP2D6",
+     "-", "-", "-", LONG_CLIN_SIG, "-"],
+]
+
+
+@pytest.mark.parametrize(
+    ("uid", "expected"),
+    [
+        ("22_19710700_C_T", ("22", 19710700, "C", "T")),
+        ("chrUn_KI270742v1_100_A_AT", ("chrUn_KI270742v1", 100, "A", "AT")),
+    ],
+)
+def test_parse_variant_id(uid: str, expected: tuple) -> None:
+    assert parse_variant_id(uid) == expected
+
+
+@pytest.mark.parametrize("uid", ["rs123", "12345", "22_x_A_G", "."])
+def test_parse_variant_id_rejects_ids_not_set_by_normalise(uid: str) -> None:
+    with pytest.raises(ValueError, match="CHROM_POS_REF_ALT"):
+        parse_variant_id(uid)
+
+
+def test_to_rows_takes_alleles_from_the_id_and_nulls_dashes(tmp_path: Path) -> None:
+    rows = to_rows(read_vep_tab(vep_tab(tmp_path, ROWS)), job_id="j")
+    snv, deletion = rows
+    assert (snv["chrom"], snv["pos"], snv["ref"], snv["alt"]) == ("22", 19710700, "C", "T")
+    assert (deletion["pos"], deletion["ref"], deletion["alt"]) == (42126611, "CT", "C")
+    assert snv["gnomad_af"] == 0.0001 and deletion["gnomad_af"] is None
+    assert deletion["hgvsc"] is None and deletion["gene"] == "CYP2D6"
+    assert deletion["clinvar_sig"] == LONG_CLIN_SIG
+    ann = json.loads(deletion["annotations"])
+    assert "-" not in ann.values() and "CADD_PHRED" not in ann
+    assert json.loads(snv["annotations"])["CADD_PHRED"] == "28.1"
+
+
+def test_vep_version(tmp_path: Path) -> None:
+    assert vep_version(vep_tab(tmp_path, ROWS)) == "113.0"
+
+
+def test_engine_for_uses_psycopg_for_any_postgres_url() -> None:
+    for url in ("postgresql+asyncpg://u:p@h/db", "postgresql://u:p@h/db"):
+        assert engine_for(url).url.drivername == "postgresql+psycopg"
+
+
+def new_job(engine, status: str = "running") -> str:
+    job_id = str(uuid.uuid4())
+    with engine.begin() as conn:
+        conn.execute(text("INSERT INTO jobs (id, status) VALUES (:id, :s)"), {"id": job_id, "s": status})
+    return job_id
+
+
+def job_and_count(engine, job_id: str) -> tuple:
+    with engine.connect() as conn:
+        job = conn.execute(text("SELECT status, vep_version, log FROM jobs WHERE id=:id"),
+                           {"id": job_id}).one()
+        n = conn.execute(text("SELECT count(*) FROM variants WHERE job_id=:id"), {"id": job_id}).scalar()
+    return (*job, n)
+
+
+def test_load_is_idempotent_and_marks_job_succeeded(engine, tmp_path: Path) -> None:
+    job_id = new_job(engine)
+    rows = to_rows(read_vep_tab(vep_tab(tmp_path, ROWS)), job_id=job_id)
+    load(engine, job_id, rows, vep="113.0")
+    load(engine, job_id, rows, vep="113.0")  # a retried task must not duplicate variants
+    assert job_and_count(engine, job_id) == ("succeeded", "113.0", None, 2)
+
+
+def test_load_with_no_variants_still_succeeds(engine, tmp_path: Path) -> None:
+    job_id = new_job(engine)
+    rows = to_rows(read_vep_tab(vep_tab(tmp_path, [])), job_id=job_id)
+    load(engine, job_id, rows, vep="113.0")
+    assert job_and_count(engine, job_id) == ("succeeded", "113.0", None, 0)
+
+
+def test_set_status_failed_records_the_reason(engine) -> None:
+    job_id = new_job(engine)
+    set_status(engine, job_id, "failed", "workflow annotate-abc Failed")
+    status, _, log, _ = job_and_count(engine, job_id)
+    assert (status, log) == ("failed", "workflow annotate-abc Failed")
+
+
+def test_set_status_never_overrides_a_succeeded_job(engine) -> None:
+    job_id = new_job(engine, status="succeeded")
+    set_status(engine, job_id, "failed", "late exit handler")
+    assert job_and_count(engine, job_id)[0] == "succeeded"
+
+
+@pytest.mark.parametrize(
+    ("raw", "expected_query"),
+    [
+        # asyncpg's ssl= becomes libpq's sslmode= for psycopg.
+        ("postgresql+asyncpg://u:p@h/db?ssl=require", {"sslmode": "require"}),
+        ("postgresql://u:p@h/db?sslmode=require", {"sslmode": "require"}),
+        ("postgresql+asyncpg://u:p@h/db", {}),
+    ],
+)
+def test_engine_for_translates_ssl_options(raw: str, expected_query: dict) -> None:
+    url = engine_for(raw).url
+    assert url.drivername == "postgresql+psycopg"
+    assert dict(url.query) == expected_query