feat(pipeline): VEP database mode, and a pipeline-specific database URL

Makes a real annotation runnable locally without the 25 GB VEP cache, which is what
the demo needs and what a reviewer can reproduce in minutes.

- params.vep_database (VEP_DATABASE=true) queries Ensembl's public database instead of
  a local cache. Slower per variant and fewer fields, so --everything is swapped for the
  flags the loader actually stores. Its cache placeholder is NO_CACHE, not NO_FILE:
  Nextflow rejects two staged inputs sharing a filename.
- PIPELINE_DATABASE_URL is handed to the pipeline when set. The loader runs inside a
  container, where the API's own localhost URL would point at the container itself.
- README: how to run the UI's annotate button locally against host Nextflow + Docker.

Verified end to end on pipeline/tests/data/tiny.vcf: bcftools norm split the multiallelic
record, VEP 113 annotated 4 variants live, the loader wrote them and marked the job
succeeded, and the UI shows them. The deletion came back as 22:42126611 CT>C with exact
VCF alleles, which is the case the audit's ID-tagging fix exists for.

Tests: api 51, loader 16, stub run 3/3; ruff, mypy clean.

README.md

@@ -43,6 +43,28 @@ make annotate JOB=<job id from the UI> VCF=data/example.vcf.gz
 make pipeline VCF=data/example.vcf.gz   # dry run: annotate without touching the database
 ```
 
+No cache? `VEP_DATABASE=true` queries Ensembl's public database instead. It is slow per variant
+and returns fewer fields, but it needs no 25 GB download, which is enough to demonstrate the
+pipeline on a handful of variants:
+
+```bash
+VEP_DATABASE=true make pipeline VCF=pipeline/tests/data/tiny.vcf
+```
+
+To make the UI's "Run VEP annotation" button work, run the API on the host (where Nextflow and
+Docker are) rather than in docker-compose:
+
+```bash
+docker compose up -d db
+cd api && DATABASE_URL=postgresql+asyncpg://rarelens:rarelens@localhost:5432/rarelens \
+  PIPELINE_DATABASE_URL=postgresql+asyncpg://rarelens:[email protected]:5432/rarelens \
+  LOCAL_DATA_ROOT=$PWD/.. VEP_DATABASE=true \
+  uv run --extra dev uvicorn app.main:app --port 8000
+```
+
+`PIPELINE_DATABASE_URL` is what the loader container gets: inside it, the API's own `localhost`
+would be the container itself. `LOCAL_DATA_ROOT` is the directory a sample's `vcf_uri` must sit under.
+
 To train and register a model (the API scores with `models:/rarelens-pathogenicity@production`):
 
 ```bash

api/app/config.py

@@ -23,6 +23,9 @@ class Settings(BaseSettings):
     gcp_project: str | None = None         # required with pubsub_topic or cloudrun_job
     gcp_region: str = "europe-west2"
     pipeline_dir: Path = REPO_ROOT / "pipeline"
+    # Handed to the pipeline when it differs from the API's own: the loader runs inside a
+    # container, where the API's localhost would be the container itself.
+    pipeline_database_url: str | None = None
     nextflow_profile: str = "docker"
     # Local (non-gs://) VCFs must live under this directory.
     local_data_root: Path = Path("/data")

api/app/services/events.py

@@ -111,7 +111,7 @@ async def _run_local(job_id: uuid.UUID, vcf_uri: str, assembly: str) -> str:
         "--vcf", vcf_uri, "--job_id", str(job_id), "--assembly", assembly,
     ]
     # The loader reads DATABASE_URL from its environment; keep it off the command line.
-    env = {**os.environ, "DATABASE_URL": settings.database_url}
+    env = {**os.environ, "DATABASE_URL": settings.pipeline_database_url or settings.database_url}
     try:
         proc = await asyncio.create_subprocess_exec(
             *cmd,

api/tests/test_annotate.py

@@ -124,3 +124,28 @@ async def test_pubsub_failure_marks_the_job_failed(
     job = (await client.post(f"/api/samples/{sample_id}/annotate")).json()
     assert job["status"] == "failed"
     assert "403 denied" in job["log"]
+
+
+@pytest.mark.usefixtures("db")
+async def test_the_pipeline_gets_its_own_database_url(
+    client: AsyncClient, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    """The loader runs in a container, where the API's own localhost URL would point at itself."""
+    monkeypatch.setattr(settings, "pubsub_topic", None)
+    monkeypatch.setattr(settings, "cloudrun_job", None)
+    monkeypatch.setattr(
+        settings, "pipeline_database_url", "postgresql+asyncpg://u:[email protected]:5432/db"
+    )
+    monkeypatch.setattr(events.shutil, "which", lambda _: "/usr/bin/nextflow")
+    launched: dict[str, Any] = {}
+
+    async def fake_exec(*cmd: str, **kw: Any) -> FakeProcess:
+        launched["env"] = kw["env"]
+        return FakeProcess(0, b"")
+
+    monkeypatch.setattr(events.asyncio, "create_subprocess_exec", fake_exec)
+    sample_id = await new_sample(client)
+
+    await client.post(f"/api/samples/{sample_id}/annotate")
+    await events.drain()
+    assert launched["env"]["DATABASE_URL"] == "postgresql+asyncpg://u:[email protected]:5432/db"

pipeline/main.nf

@@ -14,7 +14,10 @@ workflow {
     def must_exist = !workflow.stubRun
 
     vcf_ch  = Channel.fromPath(params.vcf, checkIfExists: true)
-    cache   = file(params.vep_cache, checkIfExists: must_exist)
+    // In database mode there is no cache to stage. Its placeholder differs from the plugin one:
+    // Nextflow rejects two staged inputs that share a filename.
+    cache   = file(params.vep_database ? "${projectDir}/assets/NO_CACHE" : params.vep_cache,
+                   checkIfExists: must_exist && !params.vep_database)
     plugins = file(params.vep_plugin_data ?: "${projectDir}/assets/NO_FILE", checkIfExists: must_exist)
 
     NORMALISE(vcf_ch)

pipeline/modules/vep.nf

@@ -17,10 +17,14 @@ process VEP {
         "--plugin CADD,snv=${plugin_data}/${params.cadd_snv},indels=${plugin_data}/${params.cadd_indels}",
         "--plugin AlphaMissense,file=${plugin_data}/${params.alphamissense}",
     ].join(' ')
+    // --everything needs the cache (it implies --af_gnomade and friends); the database offers a
+    // smaller set, but still the consequence, gene, HGVS and ClinVar fields the loader stores.
+    def source = params.vep_database ? "--database" : "--cache --offline --dir_cache ${cache}"
+    def fields = params.vep_database ? "--symbol --hgvs --canonical --biotype --variant_class --check_existing" : "--everything"
     """
     vep -i $vcf -o ${vcf.simpleName}.vep.tsv --tab \\
-        --assembly ${params.assembly} --cache --offline --dir_cache ${cache} \\
+        --assembly ${params.assembly} ${source} \\
-        --everything --pick ${plugins} \\
+        ${fields} --pick ${plugins} \\
         --stats_file ${vcf.simpleName}.vep_summary.html --fork ${task.cpus}
     """
 

pipeline/nextflow.config

@@ -5,6 +5,9 @@ params {
     assembly        = "GRCh38"
     vep_cache       = "${projectDir}/cache/vep"   // INSTALL.pl -a cf -s homo_sapiens -y GRCh38 -c <dir>
     vep_plugin_data = null        // CADD + AlphaMissense modules and data; plugins skipped when null
+    // Query Ensembl's public database instead of a local cache: no 25 GB download, but slow
+    // per variant and fewer fields. Fine for a handful of variants, wrong for a whole genome.
+    vep_database    = (System.getenv('VEP_DATABASE') ?: 'false').toBoolean()
     cadd_snv        = "whole_genome_SNVs.tsv.gz"
     cadd_indels     = "gnomad.genomes.r4.0.indel.tsv.gz"
     alphamissense   = "AlphaMissense_hg38.tsv.gz"