diff --git a/README.md b/README.md index e2622e3..df02be9 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,28 @@ make annotate JOB= VCF=data/example.vcf.gz make pipeline VCF=data/example.vcf.gz # dry run: annotate without touching the database ``` +No cache? `VEP_DATABASE=true` queries Ensembl's public database instead. It is slow per variant +and returns fewer fields, but it needs no 25 GB download, which is enough to demonstrate the +pipeline on a handful of variants: + +```bash +VEP_DATABASE=true make pipeline VCF=pipeline/tests/data/tiny.vcf +``` + +To make the UI's "Run VEP annotation" button work, run the API on the host (where Nextflow and +Docker are) rather than in docker-compose: + +```bash +docker compose up -d db +cd api && DATABASE_URL=postgresql+asyncpg://rarelens:rarelens@localhost:5432/rarelens \ + PIPELINE_DATABASE_URL=postgresql+asyncpg://rarelens:rarelens@host.docker.internal:5432/rarelens \ + LOCAL_DATA_ROOT=$PWD/.. VEP_DATABASE=true \ + uv run --extra dev uvicorn app.main:app --port 8000 +``` + +`PIPELINE_DATABASE_URL` is what the loader container gets: inside it, the API's own `localhost` +would be the container itself. `LOCAL_DATA_ROOT` is the directory a sample's `vcf_uri` must sit under. + To train and register a model (the API scores with `models:/rarelens-pathogenicity@production`): ```bash diff --git a/api/app/config.py b/api/app/config.py index 6f3eb0e..aaa7a9c 100644 --- a/api/app/config.py +++ b/api/app/config.py @@ -23,6 +23,9 @@ class Settings(BaseSettings): gcp_project: str | None = None # required with pubsub_topic or cloudrun_job gcp_region: str = "europe-west2" pipeline_dir: Path = REPO_ROOT / "pipeline" + # Handed to the pipeline when it differs from the API's own: the loader runs inside a + # container, where the API's localhost would be the container itself. + pipeline_database_url: str | None = None nextflow_profile: str = "docker" # Local (non-gs://) VCFs must live under this directory. local_data_root: Path = Path("/data") diff --git a/api/app/services/events.py b/api/app/services/events.py index 9b40cc6..f56252d 100644 --- a/api/app/services/events.py +++ b/api/app/services/events.py @@ -111,7 +111,7 @@ async def _run_local(job_id: uuid.UUID, vcf_uri: str, assembly: str) -> str: "--vcf", vcf_uri, "--job_id", str(job_id), "--assembly", assembly, ] # The loader reads DATABASE_URL from its environment; keep it off the command line. - env = {**os.environ, "DATABASE_URL": settings.database_url} + env = {**os.environ, "DATABASE_URL": settings.pipeline_database_url or settings.database_url} try: proc = await asyncio.create_subprocess_exec( *cmd, diff --git a/api/tests/test_annotate.py b/api/tests/test_annotate.py index 1cc2e2e..811edd0 100644 --- a/api/tests/test_annotate.py +++ b/api/tests/test_annotate.py @@ -124,3 +124,28 @@ async def test_pubsub_failure_marks_the_job_failed( job = (await client.post(f"/api/samples/{sample_id}/annotate")).json() assert job["status"] == "failed" assert "403 denied" in job["log"] + + +@pytest.mark.usefixtures("db") +async def test_the_pipeline_gets_its_own_database_url( + client: AsyncClient, monkeypatch: pytest.MonkeyPatch +) -> None: + """The loader runs in a container, where the API's own localhost URL would point at itself.""" + monkeypatch.setattr(settings, "pubsub_topic", None) + monkeypatch.setattr(settings, "cloudrun_job", None) + monkeypatch.setattr( + settings, "pipeline_database_url", "postgresql+asyncpg://u:p@host.docker.internal:5432/db" + ) + monkeypatch.setattr(events.shutil, "which", lambda _: "/usr/bin/nextflow") + launched: dict[str, Any] = {} + + async def fake_exec(*cmd: str, **kw: Any) -> FakeProcess: + launched["env"] = kw["env"] + return FakeProcess(0, b"") + + monkeypatch.setattr(events.asyncio, "create_subprocess_exec", fake_exec) + sample_id = await new_sample(client) + + await client.post(f"/api/samples/{sample_id}/annotate") + await events.drain() + assert launched["env"]["DATABASE_URL"] == "postgresql+asyncpg://u:p@host.docker.internal:5432/db" diff --git a/pipeline/assets/NO_CACHE b/pipeline/assets/NO_CACHE new file mode 100644 index 0000000..e69de29 diff --git a/pipeline/main.nf b/pipeline/main.nf index 9031261..3b9efcc 100644 --- a/pipeline/main.nf +++ b/pipeline/main.nf @@ -14,7 +14,10 @@ workflow { def must_exist = !workflow.stubRun vcf_ch = Channel.fromPath(params.vcf, checkIfExists: true) - cache = file(params.vep_cache, checkIfExists: must_exist) + // In database mode there is no cache to stage. Its placeholder differs from the plugin one: + // Nextflow rejects two staged inputs that share a filename. + cache = file(params.vep_database ? "${projectDir}/assets/NO_CACHE" : params.vep_cache, + checkIfExists: must_exist && !params.vep_database) plugins = file(params.vep_plugin_data ?: "${projectDir}/assets/NO_FILE", checkIfExists: must_exist) NORMALISE(vcf_ch) diff --git a/pipeline/modules/vep.nf b/pipeline/modules/vep.nf index 4d48b21..a42b2ce 100644 --- a/pipeline/modules/vep.nf +++ b/pipeline/modules/vep.nf @@ -17,10 +17,14 @@ process VEP { "--plugin CADD,snv=${plugin_data}/${params.cadd_snv},indels=${plugin_data}/${params.cadd_indels}", "--plugin AlphaMissense,file=${plugin_data}/${params.alphamissense}", ].join(' ') + // --everything needs the cache (it implies --af_gnomade and friends); the database offers a + // smaller set, but still the consequence, gene, HGVS and ClinVar fields the loader stores. + def source = params.vep_database ? "--database" : "--cache --offline --dir_cache ${cache}" + def fields = params.vep_database ? "--symbol --hgvs --canonical --biotype --variant_class --check_existing" : "--everything" """ vep -i $vcf -o ${vcf.simpleName}.vep.tsv --tab \\ - --assembly ${params.assembly} --cache --offline --dir_cache ${cache} \\ - --everything --pick ${plugins} \\ + --assembly ${params.assembly} ${source} \\ + ${fields} --pick ${plugins} \\ --stats_file ${vcf.simpleName}.vep_summary.html --fork ${task.cpus} """ diff --git a/pipeline/nextflow.config b/pipeline/nextflow.config index b5a4511..3eeb642 100644 --- a/pipeline/nextflow.config +++ b/pipeline/nextflow.config @@ -5,6 +5,9 @@ params { assembly = "GRCh38" vep_cache = "${projectDir}/cache/vep" // INSTALL.pl -a cf -s homo_sapiens -y GRCh38 -c vep_plugin_data = null // CADD + AlphaMissense modules and data; plugins skipped when null + // Query Ensembl's public database instead of a local cache: no 25 GB download, but slow + // per variant and fewer fields. Fine for a handful of variants, wrong for a whole genome. + vep_database = (System.getenv('VEP_DATABASE') ?: 'false').toBoolean() cadd_snv = "whole_genome_SNVs.tsv.gz" cadd_indels = "gnomad.genomes.r4.0.indel.tsv.gz" alphamissense = "AlphaMissense_hg38.tsv.gz"