diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3f61fe0..d7a259e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,17 @@ jobs: - run: uv pip install --system -e "api[dev]" - run: cd api && ruff check . && mypy app - run: cd api && pytest -q - env: { DATABASE_URL: postgresql+asyncpg://rarelens:rarelens@localhost:5432/rarelens } + env: + DATABASE_URL: postgresql+asyncpg://rarelens:rarelens@localhost:5432/rarelens + REQUIRE_DB: "1" + + ml: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v3 + - run: uv pip install --system -e "ml[dev]" + - run: cd ml && pytest -q web: runs-on: ubuntu-latest @@ -31,10 +41,23 @@ jobs: pipeline: runs-on: ubuntu-latest + services: + postgres: + image: postgres:16-alpine + env: { POSTGRES_USER: rarelens, POSTGRES_PASSWORD: rarelens, POSTGRES_DB: rarelens } + ports: ["5432:5432"] + options: --health-cmd "pg_isready -U rarelens" --health-interval 5s --health-retries 10 steps: - uses: actions/checkout@v4 + - uses: astral-sh/setup-uv@v3 + - run: uv pip install --system -r pipeline/requirements.txt pytest + - run: cd pipeline && pytest -q tests + env: + DATABASE_URL: postgresql://rarelens:rarelens@localhost:5432/rarelens + REQUIRE_DB: "1" - uses: nf-core/setup-nextflow@v2 - - run: cd pipeline && nextflow run main.nf -profile docker --vcf ../data/example.vcf.gz -stub-run + # Stub run checks wiring and channel shapes only; no containers or VEP cache needed. + - run: cd pipeline && nextflow run main.nf -stub-run --vcf tests/data/tiny.vcf terraform: runs-on: ubuntu-latest @@ -45,11 +68,19 @@ jobs: images: if: github.ref == 'refs/heads/main' - needs: [api, web, pipeline] + needs: [api, ml, web, pipeline] runs-on: ubuntu-latest permissions: { contents: read, id-token: write } strategy: - matrix: { component: [api, web, pipeline, ml] } + matrix: + include: + - { component: api, context: api, dockerfile: api/Dockerfile } + - { component: web, context: web, dockerfile: web/Dockerfile } + - { component: ml, context: ml, dockerfile: ml/Dockerfile } + - { component: pipeline, context: pipeline, dockerfile: pipeline/Dockerfile } + - { component: loader, context: pipeline, dockerfile: pipeline/loader.Dockerfile } + env: + REGISTRY: europe-west2-docker.pkg.dev/${{ secrets.GCP_PROJECT }}/rarelens steps: - uses: actions/checkout@v4 - uses: google-github-actions/auth@v2 @@ -59,6 +90,9 @@ jobs: - run: gcloud auth configure-docker europe-west2-docker.pkg.dev --quiet - uses: docker/build-push-action@v6 with: - context: ${{ matrix.component }} + context: ${{ matrix.context }} + file: ${{ matrix.dockerfile }} push: true - tags: europe-west2-docker.pkg.dev/${{ secrets.GCP_PROJECT }}/rarelens/${{ matrix.component }}:${{ github.sha }} + # Only the pipeline driver uses it: pins the loader image from the same commit. + build-args: LOADER_IMAGE=${{ env.REGISTRY }}/loader:${{ github.sha }} + tags: ${{ env.REGISTRY }}/${{ matrix.component }}:${{ github.sha }} diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index fae4c9b..1bd6f97 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -10,12 +10,19 @@ jobs: bump: if: ${{ github.event.workflow_run.conclusion == 'success' }} runs-on: ubuntu-latest + permissions: { contents: write } steps: - uses: actions/checkout@v4 + - name: Refuse to deploy an unconfigured overlay + run: | + if grep -rl __GCP_PROJECT__ infra/k8s/overlays/gcp infra/argo-workflows; then + echo "::error::run 'make gcp-configure PROJECT=' and commit before deploying" + exit 1 + fi - uses: imranismail/setup-kustomize@v2 - run: | cd infra/k8s/overlays/gcp - for c in api web; do + for c in api web pipeline loader; do kustomize edit set image rarelens/$c=europe-west2-docker.pkg.dev/${{ secrets.GCP_PROJECT }}/rarelens/$c:${{ github.event.workflow_run.head_sha }} done - uses: stefanzweifel/git-auto-commit-action@v5 diff --git a/.gitignore b/.gitignore index 7c226ad..64657b4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ __pycache__/ *.pyc +*.egg-info/ +build/ .venv/ .env !web/.env diff --git a/Makefile b/Makefile index dba90b6..9dfcfd5 100644 --- a/Makefile +++ b/Makefile @@ -1,25 +1,77 @@ -.PHONY: up down migrate test pipeline kind lint +.PHONY: up down clean migrate test lint data loader pipeline annotate images kind serverless-deploy serverless-destroy gcp-configure gcp-secrets + +VCF ?= data/example.vcf.gz +TAG ?= latest +# The loader container reaches docker-compose's Postgres through the host. +HOST_DB_URL ?= postgresql://rarelens:rarelens@host.docker.internal:5432/rarelens up: docker compose up -d --build down: + docker compose down + +clean: ## also deletes the Postgres and MLflow volumes docker compose down -v migrate: docker compose exec api alembic upgrade head test: - cd api && uv run pytest -q + cd api && uv run --extra dev pytest -q + cd ml && uv run --extra dev pytest -q + cd pipeline && uv run --no-project --with-requirements requirements.txt --with pytest --with pgserver pytest -q tests cd web && npm test lint: - cd api && uv run ruff check . && uv run mypy app + cd api && uv run --extra dev ruff check . && uv run --extra dev mypy app cd web && npm run check -pipeline: - cd pipeline && nextflow run main.nf -profile docker --vcf ../data/example.vcf.gz --outdir results +data: ## download the public demo slice: GIAB HG002 + ClinVar, chr22 (see docs/data.md) + scripts/fetch-demo-data.sh -kind: - kind create cluster --name rarelens || true +loader: + docker build -t rarelens/loader:dev -f pipeline/loader.Dockerfile pipeline + +pipeline: loader ## dry run: annotate $(VCF) without touching the database + cd pipeline && nextflow run main.nf -profile docker --vcf ../$(VCF) + +annotate: loader ## make annotate JOB= [VCF=data/x.vcf.gz] + @test -n "$(JOB)" || (echo "usage: make annotate JOB= [VCF=...]"; exit 1) + cd pipeline && DATABASE_URL=$(HOST_DB_URL) nextflow run main.nf -profile docker \ + --vcf ../$(VCF) --job_id $(JOB) + +images: + docker build -t rarelens-api:dev api + docker build -t rarelens-web:dev web + +kind: images + kind create cluster --name rarelens 2>/dev/null || true + kind load docker-image rarelens-api:dev rarelens-web:dev --name rarelens kubectl apply -k infra/k8s/overlays/local + kubectl -n rarelens rollout status deploy/postgres deploy/api deploy/web + @echo "kubectl -n rarelens port-forward svc/web 8080:80 (UI)" + @echo "kubectl -n rarelens port-forward svc/api 8000:80 (API, used by the UI)" + +serverless-deploy: ## deploy the Cloud Run track: make serverless-deploy PROJECT= [TAG=] + @test -n "$(PROJECT)" || (echo "usage: make serverless-deploy PROJECT= [TAG=]"; exit 1) + @test -n "$$TF_VAR_database_url" || echo "note: TF_VAR_database_url is unset; add -var deploy_cloud_sql=true or export a Postgres URL" + cd infra/terraform && terraform apply -var project=$(PROJECT) -var image_tag=$(TAG) + +serverless-destroy: ## tear it all down + cd infra/terraform && terraform destroy -var project=$(PROJECT) -var deletion_protection=false + +gcp-configure: ## one-time: write your GCP project id into the gcp overlay and Argo manifests + @test -n "$(PROJECT)" || (echo "usage: make gcp-configure PROJECT="; exit 1) + grep -rl __GCP_PROJECT__ infra/k8s/overlays/gcp infra/argo-workflows \ + | xargs sed -i.bak "s/__GCP_PROJECT__/$(PROJECT)/g" + find infra -name '*.bak' -delete + +gcp-secrets: ## after terraform apply: copy DB URLs from Secret Manager into k8s secrets + @test -n "$(PROJECT)" || (echo "usage: make gcp-secrets PROJECT="; exit 1) + kubectl -n rarelens create secret generic api-secrets --dry-run=client -o yaml \ + --from-literal=DATABASE_URL="$$(gcloud secrets versions access latest --project $(PROJECT) --secret rarelens-api-database-url)" \ + | kubectl apply -f - + kubectl -n rarelens create secret generic pipeline-secrets --dry-run=client -o yaml \ + --from-literal=DATABASE_URL="$$(gcloud secrets versions access latest --project $(PROJECT) --secret DATABASE_URL)" \ + | kubectl apply -f - diff --git a/README.md b/README.md index 8f0198d..e2622e3 100644 --- a/README.md +++ b/README.md @@ -13,31 +13,93 @@ clinical tool and makes no diagnostic claims. | Layer | Technology | Directory | |------------|--------------------------------------------------------|----------------------| -| Pipeline | Nextflow DSL2, bcftools, Ensembl VEP, Docker | `pipeline/` | +| Pipeline | Nextflow DSL2, bcftools, Ensembl VEP, Docker, Google Batch | `pipeline/` | | API | FastAPI, Pydantic v2, SQLAlchemy 2.0 (async), Alembic | `api/` | | Database | PostgreSQL 16 | `docker-compose.yml` | | Frontend | SvelteKit, TypeScript | `web/` | -| ML | LightGBM pathogenicity scorer, MLflow tracking | `ml/` | -| Orchestration | Argo Workflows (pipeline), Pub/Sub (events) | `infra/argo-workflows/` | +| ML | LightGBM pathogenicity scorer, MLflow registry | `ml/` | +| Orchestration | Argo Workflows + Argo Events (pipeline), Pub/Sub (events) | `infra/argo-workflows/` | | Platform | Kubernetes (Kustomize), ArgoCD (GitOps) | `infra/k8s/`, `infra/argocd/` | -| Cloud | GCP: GKE Autopilot, Cloud SQL, GCS, Artifact Registry | `infra/terraform/` | +| Cloud | GCP: GKE Autopilot, Cloud SQL, GCS, Batch, Secret Manager, Artifact Registry | `infra/terraform/` | | CI/CD | GitHub Actions, Workload Identity Federation | `.github/workflows/` | ## Quick start (local) ```bash -make up # postgres + api + web via docker-compose +make up # postgres + api + web + mlflow via docker-compose make migrate # alembic upgrade head -make pipeline # nextflow run pipeline/main.nf -profile docker --vcf data/example.vcf.gz -make kind # spin up a local kind cluster and apply infra/k8s/overlays/local +make data # real public data: GIAB HG002 + ClinVar, chr22 (needs bcftools) +make test # api, ml, loader and web tests (no Docker needed for the DB tests) ``` Then open http://localhost:5173. +The docker-compose API has no Nextflow, so "Run VEP annotation" marks the job failed with the +command to run instead. With Nextflow and Docker on the host, a VEP cache in `pipeline/cache/vep` +and a VCF under `data/` (see [data/README.md](data/README.md)): + +```bash +make annotate JOB= VCF=data/example.vcf.gz +make pipeline VCF=data/example.vcf.gz # dry run: annotate without touching the database +``` + +To train and register a model (the API scores with `models:/rarelens-pathogenicity@production`): + +```bash +cd ml && MLFLOW_TRACKING_URI=http://localhost:5000 \ + uv run python -m rarelens_ml.train --tsv ../pipeline/results/.vep.tsv --register +``` + +Local Kubernetes: `make kind` builds the images, loads them into a kind cluster and applies +`infra/k8s/overlays/local`. + +## Deploying to GCP + +Two tracks, same code. The serverless one is the default because it costs about £1/month idle; +[docs/cloud.md](docs/cloud.md) has the numbers. + +**Serverless (Cloud Run + Google Batch).** The API and the UI scale to zero, and the Nextflow +driver runs as a Cloud Run job only while a pipeline is running. + +```bash +cd infra/terraform +terraform init -backend-config="bucket=" +export TF_VAR_database_url='postgresql+asyncpg://user:pass@host/db?sslmode=require' # e.g. Neon's free tier +terraform apply -var project= # add -var deploy_cloud_sql=true to use Cloud SQL instead +cd ../.. && make serverless-deploy PROJECT= TAG= # redeploy a new build +``` + +`terraform output web_url` is the URL to share; it serves the UI and proxies `/api` to the API, so +there is one public address and no CORS. Upload the VEP cache to +`gs://-rarelens-data/refs/vep` before running a real annotation, and set +`-var model_uri=gs://-rarelens-data/models/pathogenicity/1` to score without running an +MLflow server. Set a billing budget first — the demo has no authentication. + +**Kubernetes (GKE + Argo + ArgoCD).** Off by default; turn it on to demonstrate the GitOps path, +then destroy it. + +```bash +terraform apply -var project= -var deploy_kubernetes=true -var deploy_cloud_sql=true +make gcp-configure PROJECT= # once; commit the result +make gcp-secrets PROJECT= +``` + +Then install Argo Workflows, Argo Events and ArgoCD, and `kubectl apply -f infra/argocd/app.yaml`. +Every green CI run on `main` bumps image tags in the gcp overlay and ArgoCD deploys them. +`make serverless-destroy PROJECT=` tears everything down. + +## Data + +The demo runs on published, openly licensed human data: the NIST Genome in a Bottle HG002 +benchmark genome as the sample, ClinVar for labels, gnomAD for allele frequencies. Sources, +licences, citations and how the model should be evaluated honestly are in +[docs/data.md](docs/data.md). + ## Architecture See [docs/architecture.md](docs/architecture.md) for the diagram and the reasoning behind -each choice. +each choice, and [docs/cloud.md](docs/cloud.md) for why this deploys to Google Cloud rather +than AWS. ## Status diff --git a/api/.dockerignore b/api/.dockerignore new file mode 100644 index 0000000..3d31df7 --- /dev/null +++ b/api/.dockerignore @@ -0,0 +1,5 @@ +.venv/ +.env +tests/ +**/__pycache__/ +*.egg-info/ diff --git a/api/Dockerfile b/api/Dockerfile index 58161d7..1d7914c 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -2,7 +2,8 @@ FROM python:3.12-slim WORKDIR /app RUN pip install --no-cache-dir uv COPY pyproject.toml . -RUN uv pip install --system -e . +# Dependencies only, so this layer is cached until pyproject.toml changes; `app` runs from WORKDIR. +RUN uv pip install --system -r pyproject.toml --extra gcp COPY . . EXPOSE 8000 CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/api/alembic/env.py b/api/alembic/env.py index 1c58b11..0536936 100644 --- a/api/alembic/env.py +++ b/api/alembic/env.py @@ -2,8 +2,8 @@ import asyncio from logging.config import fileConfig from alembic import context -from sqlalchemy.ext.asyncio import async_engine_from_config from sqlalchemy import pool +from sqlalchemy.ext.asyncio import async_engine_from_config from app.config import settings from app.models import Base @@ -14,10 +14,15 @@ if config.config_file_name: fileConfig(config.config_file_name) target_metadata = Base.metadata +# Arbitrary constant shared by every `alembic upgrade` (one per API replica's init container). +MIGRATION_LOCK_ID = 72150001 + def run_migrations(connection): context.configure(connection=connection, target_metadata=target_metadata) with context.begin_transaction(): + # Held until this transaction commits; a concurrent run waits, then finds nothing to do. + connection.exec_driver_sql(f"SELECT pg_advisory_xact_lock({MIGRATION_LOCK_ID})") context.run_migrations() @@ -27,6 +32,22 @@ async def run_async(): ) async with engine.connect() as conn: await conn.run_sync(run_migrations) + await engine.dispose() -asyncio.run(run_async()) +def run_offline(): + """`alembic upgrade head --sql`: emit the SQL instead of running it.""" + context.configure( + url=settings.database_url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_offline() +else: + asyncio.run(run_async()) diff --git a/api/alembic/versions/5f2c8e1b9d04_widen_vep_text_columns.py b/api/alembic/versions/5f2c8e1b9d04_widen_vep_text_columns.py new file mode 100644 index 0000000..a11e2ba --- /dev/null +++ b/api/alembic/versions/5f2c8e1b9d04_widen_vep_text_columns.py @@ -0,0 +1,34 @@ +"""widen VEP text columns + +VEP joins multiple consequences with "&" and co-located ClinVar significances with ",", which +overflowed VARCHAR(120) and aborted the whole load transaction. + +Revision ID: 5f2c8e1b9d04 +Revises: a3e9ead256a5 +Create Date: 2026-09-11 18:00:00.000000 + +""" +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +revision: str = '5f2c8e1b9d04' +down_revision: str | None = 'a3e9ead256a5' +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +COLUMNS = ('consequence', 'clinvar_sig') + + +def upgrade() -> None: + for column in COLUMNS: + op.alter_column('variants', column, type_=sa.Text(), existing_type=sa.String(length=120)) + + +def downgrade() -> None: + for column in COLUMNS: + op.alter_column( + 'variants', column, type_=sa.String(length=120), existing_type=sa.Text(), + postgresql_using=f'left({column}, 120)', + ) diff --git a/api/alembic/versions/a3e9ead256a5_initial_schema.py b/api/alembic/versions/a3e9ead256a5_initial_schema.py index 19e8f55..fe86279 100644 --- a/api/alembic/versions/a3e9ead256a5_initial_schema.py +++ b/api/alembic/versions/a3e9ead256a5_initial_schema.py @@ -5,17 +5,17 @@ Revises: Create Date: 2026-09-11 15:36:04.335440 """ -from typing import Sequence, Union +from collections.abc import Sequence -from alembic import op import sqlalchemy as sa +from alembic import op from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. revision: str = 'a3e9ead256a5' -down_revision: Union[str, None] = None -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None +down_revision: str | None = None +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None def upgrade() -> None: @@ -87,4 +87,7 @@ def downgrade() -> None: op.drop_table('variants') op.drop_table('jobs') op.drop_table('samples') - # ### end Alembic commands ### \ No newline at end of file + # ### end Alembic commands ### + # Autogenerate does not drop the type created by sa.Enum; without this, upgrading again + # fails with "type jobstatus already exists". + sa.Enum(name='jobstatus').drop(op.get_bind(), checkfirst=True) \ No newline at end of file diff --git a/api/app/config.py b/api/app/config.py index 02d09b9..6f3eb0e 100644 --- a/api/app/config.py +++ b/api/app/config.py @@ -1,5 +1,10 @@ +from pathlib import Path + from pydantic_settings import BaseSettings, SettingsConfigDict +# api/app/config.py -> repo root locally; "/" in the API image, where compose mounts /pipeline. +REPO_ROOT = Path(__file__).resolve().parents[2] + class Settings(BaseSettings): model_config = SettingsConfigDict(env_file=".env", extra="ignore") @@ -7,9 +12,22 @@ class Settings(BaseSettings): database_url: str = "postgresql+asyncpg://rarelens:rarelens@localhost:5432/rarelens" mlflow_tracking_uri: str = "http://localhost:5000" model_name: str = "rarelens-pathogenicity" - model_stage: str = "Production" + # Registry alias set by `rarelens_ml.train --register` (stages are deprecated in MLflow 3). + model_alias: str = "production" + # A model artifact URI (gs://...) scores without an MLflow server running; wins over the registry. + model_uri: str | None = None gcs_bucket: str | None = None # set in GCP; local uses ./data pubsub_topic: str | None = None # "vcf-uploaded" in GCP; local runs pipeline inline + # Serverless track: run the Nextflow driver as a Cloud Run job instead of Argo + Pub/Sub. + cloudrun_job: str | None = None + gcp_project: str | None = None # required with pubsub_topic or cloudrun_job + gcp_region: str = "europe-west2" + pipeline_dir: Path = REPO_ROOT / "pipeline" + nextflow_profile: str = "docker" + # Local (non-gs://) VCFs must live under this directory. + local_data_root: Path = Path("/data") + # Browsers calling the API cross-origin; behind the ingress the UI is same-origin. + cors_origins: list[str] = ["http://localhost:5173"] settings = Settings() diff --git a/api/app/db.py b/api/app/db.py index db06307..f2c21ef 100644 --- a/api/app/db.py +++ b/api/app/db.py @@ -1,13 +1,33 @@ from collections.abc import AsyncIterator +from typing import Annotated +from fastapi import Depends +from sqlalchemy.engine import make_url from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine from app.config import settings -engine = create_async_engine(settings.database_url, pool_pre_ping=True) + +def normalize_async_url(url: str) -> str: + """Translate libpq's `sslmode=` into the `ssl=` asyncpg understands. + + Hosted Postgres (Neon, Supabase) hands out sslmode= URLs, which asyncpg rejects outright. + """ + parsed = make_url(url) + if "asyncpg" not in parsed.drivername or "sslmode" not in parsed.query: + return url + query = dict(parsed.query) + query["ssl"] = query.pop("sslmode") + return parsed.set(query=query).render_as_string(hide_password=False) + + +engine = create_async_engine(normalize_async_url(settings.database_url), pool_pre_ping=True) SessionLocal = async_sessionmaker(engine, expire_on_commit=False) async def get_session() -> AsyncIterator[AsyncSession]: async with SessionLocal() as session: yield session + + +SessionDep = Annotated[AsyncSession, Depends(get_session)] diff --git a/api/app/main.py b/api/app/main.py index b4468d1..3ec70bb 100644 --- a/api/app/main.py +++ b/api/app/main.py @@ -1,8 +1,9 @@ from contextlib import asynccontextmanager -from fastapi import FastAPI +from fastapi import APIRouter, FastAPI from fastapi.middleware.cors import CORSMiddleware +from app.config import settings from app.routers import jobs, predictions, samples, variants @@ -14,13 +15,19 @@ async def lifespan(app: FastAPI): app = FastAPI(title="rarelens API", version="0.1.0", lifespan=lifespan) app.add_middleware( - CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"] + CORSMiddleware, + allow_origins=settings.cors_origins, + allow_methods=["GET", "POST"], + allow_headers=["content-type"], ) -app.include_router(samples.router, prefix="/samples", tags=["samples"]) -app.include_router(jobs.router, prefix="/jobs", tags=["jobs"]) -app.include_router(variants.router, prefix="/variants", tags=["variants"]) -app.include_router(predictions.router, prefix="/predictions", tags=["predictions"]) +# The ingress forwards /api/* to this service unchanged, and local dev uses the same prefix. +api = APIRouter(prefix="/api") +api.include_router(samples.router, prefix="/samples", tags=["samples"]) +api.include_router(jobs.router, prefix="/jobs", tags=["jobs"]) +api.include_router(variants.router, prefix="/variants", tags=["variants"]) +api.include_router(predictions.router, prefix="/predictions", tags=["predictions"]) +app.include_router(api) @app.get("/health", tags=["ops"]) diff --git a/api/app/models.py b/api/app/models.py index ad42308..5bed222 100644 --- a/api/app/models.py +++ b/api/app/models.py @@ -2,9 +2,9 @@ One sample -> many jobs; one job -> many variants; one variant -> one prediction (latest). """ -from datetime import datetime import enum import uuid +from datetime import datetime from sqlalchemy import DateTime, Enum, Float, ForeignKey, Integer, String, Text, func from sqlalchemy.dialects.postgresql import JSONB, UUID @@ -55,12 +55,12 @@ class Variant(Base): ref: Mapped[str] = mapped_column(Text) alt: Mapped[str] = mapped_column(Text) gene: Mapped[str | None] = mapped_column(String(60), index=True) - consequence: Mapped[str | None] = mapped_column(String(120)) + consequence: Mapped[str | None] = mapped_column(Text) # "&"-joined VEP terms impact: Mapped[str | None] = mapped_column(String(20)) hgvsc: Mapped[str | None] = mapped_column(Text) hgvsp: Mapped[str | None] = mapped_column(Text) gnomad_af: Mapped[float | None] = mapped_column(Float) - clinvar_sig: Mapped[str | None] = mapped_column(String(120)) + clinvar_sig: Mapped[str | None] = mapped_column(Text) # ","-joined co-located ClinVar terms annotations: Mapped[dict] = mapped_column(JSONB, default=dict) # full VEP CSQ record job: Mapped[Job] = relationship(back_populates="variants") prediction: Mapped["Prediction | None"] = relationship(back_populates="variant", uselist=False) diff --git a/api/app/routers/jobs.py b/api/app/routers/jobs.py index 2769b4b..1425841 100644 --- a/api/app/routers/jobs.py +++ b/api/app/routers/jobs.py @@ -1,9 +1,8 @@ import uuid -from fastapi import APIRouter, Depends, HTTPException -from sqlalchemy.ext.asyncio import AsyncSession +from fastapi import APIRouter, HTTPException -from app.db import get_session +from app.db import SessionDep from app.models import Job from app.schemas import JobOut @@ -11,7 +10,7 @@ router = APIRouter() @router.get("/{job_id}", response_model=JobOut) -async def get_job(job_id: uuid.UUID, session: AsyncSession = Depends(get_session)): +async def get_job(job_id: uuid.UUID, session: SessionDep): job = await session.get(Job, job_id) if job is None: raise HTTPException(404, "job not found") diff --git a/api/app/routers/predictions.py b/api/app/routers/predictions.py index d3f7e93..3732e10 100644 --- a/api/app/routers/predictions.py +++ b/api/app/routers/predictions.py @@ -1,16 +1,22 @@ import uuid -from fastapi import APIRouter, Depends -from sqlalchemy.ext.asyncio import AsyncSession +from fastapi import APIRouter, HTTPException -from app.db import get_session +from app.db import SessionDep +from app.models import Job, JobStatus +from app.schemas import ScoreOut from app.services.scoring import score_job router = APIRouter() -@router.post("/score/{job_id}") -async def score(job_id: uuid.UUID, session: AsyncSession = Depends(get_session)) -> dict: - """Load the registered MLflow model and score every variant of a job.""" - n = await score_job(job_id, session) - return {"job_id": str(job_id), "scored": n} +@router.post("/score/{job_id}", response_model=ScoreOut) +async def score(job_id: uuid.UUID, session: SessionDep) -> ScoreOut: + """Score every variant of a finished job with the model behind the registry alias.""" + job = await session.get(Job, job_id) + if job is None: + raise HTTPException(404, "job not found") + if job.status != JobStatus.succeeded: + raise HTTPException(409, f"job is {job.status.value}; only succeeded jobs can be scored") + n, version = await score_job(job_id, session) + return ScoreOut(job_id=job_id, scored=n, model_version=version) diff --git a/api/app/routers/samples.py b/api/app/routers/samples.py index 7b64f56..c4b774e 100644 --- a/api/app/routers/samples.py +++ b/api/app/routers/samples.py @@ -1,34 +1,39 @@ import uuid +from datetime import UTC, datetime -from fastapi import APIRouter, Depends, HTTPException, status +from fastapi import APIRouter, HTTPException, status from sqlalchemy import select -from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.exc import IntegrityError -from app.db import get_session -from app.models import Job, Sample +from app.db import SessionDep +from app.models import Job, JobStatus, Sample from app.schemas import JobOut, SampleCreate, SampleOut -from app.services.events import publish_vcf_uploaded +from app.services import events router = APIRouter() @router.get("", response_model=list[SampleOut]) -async def list_samples(session: AsyncSession = Depends(get_session)): +async def list_samples(session: SessionDep): result = await session.scalars(select(Sample).order_by(Sample.created_at.desc())) return result.all() @router.post("", response_model=SampleOut, status_code=status.HTTP_201_CREATED) -async def create_sample(payload: SampleCreate, session: AsyncSession = Depends(get_session)): +async def create_sample(payload: SampleCreate, session: SessionDep): sample = Sample(**payload.model_dump()) session.add(sample) - await session.commit() + try: + await session.commit() + except IntegrityError: # samples.name is unique + await session.rollback() + raise HTTPException(409, f"a sample named {payload.name!r} already exists") from None await session.refresh(sample) return sample @router.get("/{sample_id}/jobs", response_model=list[JobOut]) -async def list_jobs(sample_id: uuid.UUID, session: AsyncSession = Depends(get_session)): +async def list_jobs(sample_id: uuid.UUID, session: SessionDep): result = await session.scalars( select(Job).where(Job.sample_id == sample_id).order_by(Job.created_at.desc()) ) @@ -36,14 +41,21 @@ async def list_jobs(sample_id: uuid.UUID, session: AsyncSession = Depends(get_se @router.post("/{sample_id}/annotate", response_model=JobOut, status_code=status.HTTP_202_ACCEPTED) -async def annotate(sample_id: uuid.UUID, session: AsyncSession = Depends(get_session)): +async def annotate(sample_id: uuid.UUID, session: SessionDep): sample = await session.get(Sample, sample_id) if sample is None: raise HTTPException(404, "sample not found") - job = Job(sample_id=sample.id) + # Commit `running` before launching: a local run that dies instantly is marked failed by its + # watcher, and a later status write here would overwrite that. + job = Job(sample_id=sample.id, status=JobStatus.running) session.add(job) await session.commit() + try: + job.workflow_ref = await events.launch(job.id, sample.vcf_uri, sample.assembly) + except events.LaunchError as e: + job.status = JobStatus.failed + job.log = str(e) + job.finished_at = datetime.now(UTC) + await session.commit() await session.refresh(job) - # Emits to Pub/Sub in GCP; runs the Nextflow pipeline inline for local dev. - await publish_vcf_uploaded(job_id=job.id, vcf_uri=sample.vcf_uri) return job diff --git a/api/app/routers/variants.py b/api/app/routers/variants.py index 2506efb..94f6854 100644 --- a/api/app/routers/variants.py +++ b/api/app/routers/variants.py @@ -1,28 +1,37 @@ import uuid -from fastapi import APIRouter, Depends, Query -from sqlalchemy import func, select -from sqlalchemy.ext.asyncio import AsyncSession +from fastapi import APIRouter, Query +from sqlalchemy import Integer, case, cast, func, select from sqlalchemy.orm import selectinload -from app.db import get_session +from app.db import SessionDep from app.models import Prediction, Variant -from app.schemas import VariantPage +from app.schemas import VariantOut, VariantPage router = APIRouter() +# Karyotype order (1..22, X, Y, MT) instead of text order, where "10" sorts before "2". +_chrom = func.regexp_replace(Variant.chrom, "^chr", "", "i") +CHROM_ORDER = case( + (_chrom.regexp_match("^[0-9]+$"), cast(_chrom, Integer)), + (_chrom == "X", 23), + (_chrom == "Y", 24), + (_chrom.in_(["M", "MT"]), 25), + else_=26, +) + @router.get("", response_model=VariantPage) async def list_variants( job_id: uuid.UUID, + session: SessionDep, gene: str | None = None, impact: str | None = Query(None, pattern="^(HIGH|MODERATE|LOW|MODIFIER)$"), max_af: float | None = Query(None, ge=0, le=1), min_score: float | None = Query(None, ge=0, le=1), - limit: int = Query(50, le=500), - offset: int = 0, - session: AsyncSession = Depends(get_session), -): + limit: int = Query(50, ge=1, le=500), + offset: int = Query(0, ge=0), +) -> VariantPage: stmt = select(Variant).where(Variant.job_id == job_id) if gene: stmt = stmt.where(Variant.gene == gene.upper()) @@ -31,13 +40,21 @@ async def list_variants( if max_af is not None: stmt = stmt.where((Variant.gnomad_af.is_(None)) | (Variant.gnomad_af <= max_af)) if min_score is not None: - stmt = stmt.join(Prediction, Prediction.variant_id == Variant.id).where(Prediction.score >= min_score) + stmt = stmt.join(Prediction, Prediction.variant_id == Variant.id).where( + Prediction.score >= min_score + ) total = await session.scalar(select(func.count()).select_from(stmt.subquery())) rows = await session.scalars( stmt.options(selectinload(Variant.prediction)) - .order_by(Variant.chrom, Variant.pos) + # id breaks ties between split multiallelics at one position, keeping pages stable. + .order_by(CHROM_ORDER, Variant.chrom, Variant.pos, Variant.id) .limit(limit) .offset(offset) ) - return VariantPage(items=rows.all(), total=total or 0, limit=limit, offset=offset) + return VariantPage( + items=[VariantOut.model_validate(v) for v in rows], + total=total or 0, + limit=limit, + offset=offset, + ) diff --git a/api/app/schemas.py b/api/app/schemas.py index 974f6a6..dfc3896 100644 --- a/api/app/schemas.py +++ b/api/app/schemas.py @@ -1,8 +1,12 @@ -from datetime import datetime +import re import uuid +from datetime import datetime +from pathlib import PurePosixPath +from typing import Literal -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, field_validator +from app.config import settings from app.models import JobStatus @@ -10,10 +14,34 @@ class ORMModel(BaseModel): model_config = ConfigDict(from_attributes=True) +Assembly = Literal["GRCh38", "GRCh37"] +VCF_SUFFIXES = (".vcf", ".vcf.gz", ".vcf.bgz", ".bcf") +GCS_URI = re.compile(r"gs://[a-z0-9][a-z0-9._-]{1,220}[a-z0-9]/\S+") + + class SampleCreate(BaseModel): name: str = Field(min_length=1, max_length=120) vcf_uri: str - assembly: str = "GRCh38" + # Passed to VEP --assembly; the VEP cache must contain it. + assembly: Assembly = "GRCh38" + + @field_validator("vcf_uri") + @classmethod + def _gcs_object_or_file_under_data_root(cls, v: str) -> str: + # The URI becomes a Nextflow argument and a path the pipeline reads: accept a GCS object or + # a file under the local data root, never something that parses as an option. + if not v or any(ord(c) < 32 for c in v): + raise ValueError("vcf_uri must be a non-empty single line") + if not v.lower().endswith(VCF_SUFFIXES): + raise ValueError(f"vcf_uri must end in one of {', '.join(VCF_SUFFIXES)}") + if v.startswith("gs://"): + if not GCS_URI.fullmatch(v): + raise ValueError("vcf_uri is not a valid gs://bucket/object URI") + return v + path, root = PurePosixPath(v), PurePosixPath(settings.local_data_root) + if not path.is_absolute() or ".." in path.parts or not path.is_relative_to(root): + raise ValueError(f"local VCFs must be absolute paths under {root}") + return v class SampleOut(ORMModel): @@ -30,6 +58,7 @@ class JobOut(ORMModel): status: JobStatus workflow_ref: str | None vep_version: str | None + log: str | None created_at: datetime finished_at: datetime | None @@ -56,6 +85,12 @@ class VariantOut(ORMModel): prediction: PredictionOut | None = None +class ScoreOut(BaseModel): + job_id: uuid.UUID + scored: int + model_version: str + + class VariantPage(BaseModel): items: list[VariantOut] total: int diff --git a/api/app/services/events.py b/api/app/services/events.py index 20856ce..9b40cc6 100644 --- a/api/app/services/events.py +++ b/api/app/services/events.py @@ -1,31 +1,142 @@ -"""Event publishing. +"""Start the annotation pipeline for a job. -GCP: publish a JSON message to Pub/Sub; an Argo Events sensor turns it into an Argo Workflow. -Local: run Nextflow directly in a background task so `docker compose up` gives a working demo. +Serverless (CLOUDRUN_JOB set): execute a Cloud Run job that runs the Nextflow driver, which +submits the pipeline tasks to Google Batch. Nothing runs, or is billed, between annotations. +Kubernetes (PUBSUB_TOPIC set): publish to Pub/Sub; an Argo Events sensor submits the annotate-vcf +WorkflowTemplate, whose exit handler marks the job failed if the workflow does not succeed. +Local: run Nextflow as a child process and watch it. The docker-compose API image has neither +Nextflow nor Docker, so there the job fails at once with instructions to run it from the host. """ import asyncio import json +import logging +import os +import shutil import uuid +from datetime import UTC, datetime +from functools import cache +from typing import Any from app.config import settings +from app.db import SessionLocal +from app.models import Job, JobStatus + +logger = logging.getLogger(__name__) + +PIPELINE_ENTRYPOINT = "/pipeline/main.nf" +LOG_TAIL_CHARS = 4000 +PUBLISH_TIMEOUT_S = 30 +_watchers: set[asyncio.Task[None]] = set() -async def publish_vcf_uploaded(job_id: uuid.UUID, vcf_uri: str) -> None: - message = {"job_id": str(job_id), "vcf_uri": vcf_uri} +class LaunchError(Exception): + """The pipeline could not be started; the message is stored on the job for the user.""" + + +async def launch(job_id: uuid.UUID, vcf_uri: str, assembly: str) -> str: + """Start the pipeline and return a reference to the run (stored as jobs.workflow_ref).""" + if settings.cloudrun_job: + return await _run_cloud_run_job(job_id, vcf_uri, assembly) if settings.pubsub_topic: - from google.cloud import pubsub_v1 # optional dependency, installed in the GCP image + message = {"job_id": str(job_id), "vcf_uri": vcf_uri, "assembly": assembly} + return await _publish(settings.pubsub_topic, message) + return await _run_local(job_id, vcf_uri, assembly) - publisher = pubsub_v1.PublisherClient() - publisher.publish(settings.pubsub_topic, json.dumps(message).encode()) - return - cmd = [ - "nextflow", "run", "/pipeline/main.nf", "-profile", "docker", - "--vcf", vcf_uri, "--job_id", str(job_id), "--db_url", settings.database_url, +async def drain() -> None: + """Wait for local pipeline watchers (tests, graceful shutdown).""" + await asyncio.gather(*_watchers) + + +@cache +def _jobs_client() -> Any: + from google.cloud import run_v2 + + return run_v2.JobsClient() + + +async def _run_cloud_run_job(job_id: uuid.UUID, vcf_uri: str, assembly: str) -> str: + if not settings.gcp_project: + raise LaunchError("CLOUDRUN_JOB is set but GCP_PROJECT is not") + name = ( + f"projects/{settings.gcp_project}/locations/{settings.gcp_region}" + f"/jobs/{settings.cloudrun_job}" + ) + args = [ + "run", PIPELINE_ENTRYPOINT, "-profile", "gcp", + "--vcf", vcf_uri, "--job_id", str(job_id), "--assembly", assembly, ] + request = {"name": name, "overrides": {"container_overrides": [{"args": args}]}} try: - await asyncio.create_subprocess_exec(*cmd) - except FileNotFoundError: - # Nextflow is not installed in this environment (no Java toolchain yet); - # leave the job queued instead of crashing the request. - print(f"[rarelens] nextflow not found; cannot start pipeline for job {job_id}", flush=True) + operation = await asyncio.to_thread(_jobs_client().run_job, request=request) + except Exception as e: + logger.exception("starting Cloud Run job %s failed", name) + raise LaunchError(f"could not start {settings.cloudrun_job}: {e}") from e + name_or_job = getattr(getattr(operation, "metadata", None), "name", "") + execution = name_or_job or settings.cloudrun_job or "started" + return f"cloudrun:{execution.rsplit('/', 1)[-1]}" + + +@cache +def _publisher() -> Any: + from google.cloud import pubsub_v1 + + return pubsub_v1.PublisherClient() + + +async def _publish(topic_name: str, message: dict[str, str]) -> str: + if not settings.gcp_project: + raise LaunchError("PUBSUB_TOPIC is set but GCP_PROJECT is not") + publisher = _publisher() + topic = publisher.topic_path(settings.gcp_project, topic_name) + try: + future = publisher.publish(topic, json.dumps(message).encode()) + message_id = await asyncio.to_thread(future.result, timeout=PUBLISH_TIMEOUT_S) + except Exception as e: + logger.exception("publishing to %s failed", topic) + raise LaunchError(f"could not publish to {topic}: {e}") from e + return f"pubsub:{message_id}" + + +async def _run_local(job_id: uuid.UUID, vcf_uri: str, assembly: str) -> str: + nextflow = shutil.which("nextflow") + if nextflow is None: + raise LaunchError( + "Nextflow is not installed where the API runs. Start the pipeline from the host with " + f"`make annotate JOB={job_id} VCF=`." + ) + cmd = [ + nextflow, "run", str(settings.pipeline_dir / "main.nf"), + "-profile", settings.nextflow_profile, + "--vcf", vcf_uri, "--job_id", str(job_id), "--assembly", assembly, + ] + # The loader reads DATABASE_URL from its environment; keep it off the command line. + env = {**os.environ, "DATABASE_URL": settings.database_url} + try: + proc = await asyncio.create_subprocess_exec( + *cmd, + cwd=settings.pipeline_dir, + env=env, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.STDOUT, + ) + except OSError as e: + raise LaunchError(f"could not start Nextflow: {e}") from e + task = asyncio.create_task(_watch(job_id, proc)) + _watchers.add(task) + task.add_done_callback(_watchers.discard) + return f"nextflow:pid-{proc.pid}" + + +async def _watch(job_id: uuid.UUID, proc: Any) -> None: + out, err = await proc.communicate() + if proc.returncode == 0: + return # the loader marks the job succeeded + tail = ((out or b"") + (err or b"")).decode(errors="replace")[-LOG_TAIL_CHARS:] + async with SessionLocal() as session: + job = await session.get(Job, job_id) + if job is not None and job.status != JobStatus.succeeded: + job.status = JobStatus.failed + job.log = f"nextflow exited with code {proc.returncode}\n{tail}" + job.finished_at = datetime.now(UTC) + await session.commit() diff --git a/api/app/services/scoring.py b/api/app/services/scoring.py index e0bba0e..13dd28b 100644 --- a/api/app/services/scoring.py +++ b/api/app/services/scoring.py @@ -1,45 +1,107 @@ +"""Score a job's variants with the registered MLflow model. + +The registered model is a pyfunc that owns its feature engineering (rarelens_ml.features travels +with it as model code) and returns P(pathogenic). Serving therefore only sends the raw columns +below and cannot drift from training. +""" +import asyncio import uuid +from collections.abc import Sequence +from typing import Any import mlflow import pandas as pd -from sqlalchemy import select +from mlflow import MlflowClient +from sqlalchemy import func, select +from sqlalchemy.dialects.postgresql import insert from sqlalchemy.ext.asyncio import AsyncSession from app.config import settings from app.models import Prediction, Variant -_model = None +# Must match rarelens_ml.features.RAW_COLUMNS. +RAW_COLUMNS = ["impact", "consequence", "gnomad_af", "cadd_phred", "am_pathogenicity"] +CHUNK_SIZE = 5000 + +_models: dict[str, Any] = {} # model version -> loaded pyfunc -def load_model(): - global _model - if _model is None: +def load_model() -> tuple[Any, str]: + """Return (model, version), from MODEL_URI if set, else from the registry alias.""" + if settings.model_uri: + return _load_uri(settings.model_uri) + client = MlflowClient(tracking_uri=settings.mlflow_tracking_uri) + version = client.get_model_version_by_alias(settings.model_name, settings.model_alias).version + if version not in _models: mlflow.set_tracking_uri(settings.mlflow_tracking_uri) - _model = mlflow.pyfunc.load_model(f"models:/{settings.model_name}/{settings.model_stage}") - return _model + _models.clear() + _models[version] = mlflow.pyfunc.load_model(f"models:/{settings.model_name}/{version}") + return _models[version], version -def featurise(variants: list[Variant]) -> pd.DataFrame: - # Mirror ml/rarelens_ml/features.py exactly; shared package later. +def _load_uri(uri: str) -> tuple[Any, str]: + """Load a model artifact directly (gs://...): no tracking server, nothing running when idle.""" + version = uri.rstrip("/").rsplit("/", 1)[-1][:40] or "uri" + if version not in _models: + _models.clear() + _models[version] = mlflow.pyfunc.load_model(uri) + return _models[version], version + + +def raw_frame(variants: Sequence[Variant]) -> pd.DataFrame: return pd.DataFrame( { "impact": [v.impact for v in variants], "consequence": [v.consequence for v in variants], - "gnomad_af": [v.gnomad_af if v.gnomad_af is not None else 0.0 for v in variants], + "gnomad_af": [v.gnomad_af if v.gnomad_af is not None else float("nan") for v in variants], "cadd_phred": [v.annotations.get("CADD_PHRED") for v in variants], "am_pathogenicity": [v.annotations.get("am_pathogenicity") for v in variants], - } + }, + columns=RAW_COLUMNS, ) -async def score_job(job_id: uuid.UUID, session: AsyncSession) -> int: - variants = (await session.scalars(select(Variant).where(Variant.job_id == job_id))).all() - if not variants: - return 0 - model = load_model() - scores = model.predict(featurise(variants)) - for v, s in zip(variants, scores): - session.add(Prediction(variant_id=v.id, model_name=settings.model_name, - model_version=settings.model_stage, score=float(s))) +async def score_job(job_id: uuid.UUID, session: AsyncSession) -> tuple[int, str]: + model, version = await asyncio.to_thread(load_model) + scored = 0 + last_id = 0 + # Keyset pagination keeps memory flat for whole-genome jobs. + while True: + variants = ( + await session.scalars( + select(Variant) + .where(Variant.job_id == job_id, Variant.id > last_id) + .order_by(Variant.id) + .limit(CHUNK_SIZE) + ) + ).all() + if not variants: + break + scores = await asyncio.to_thread(model.predict, raw_frame(variants)) + stmt = insert(Prediction).values( + [ + { + "variant_id": v.id, + "model_name": settings.model_name, + "model_version": version, + "score": float(s), + } + for v, s in zip(variants, scores, strict=True) + ] + ) + # Re-scoring (e.g. after a new model version) replaces the previous prediction. + await session.execute( + stmt.on_conflict_do_update( + index_elements=[Prediction.variant_id], + set_={ + "model_name": stmt.excluded.model_name, + "model_version": stmt.excluded.model_version, + "score": stmt.excluded.score, + "created_at": func.now(), + }, + ) + ) + scored += len(variants) + last_id = variants[-1].id await session.commit() - return len(variants) + return scored, version diff --git a/api/pyproject.toml b/api/pyproject.toml index c668f33..777d607 100644 --- a/api/pyproject.toml +++ b/api/pyproject.toml @@ -1,3 +1,7 @@ +[build-system] +requires = ["setuptools>=69"] +build-backend = "setuptools.build_meta" + [project] name = "rarelens-api" version = "0.1.0" @@ -12,17 +16,50 @@ dependencies = [ "pydantic>=2.8", "pydantic-settings>=2.4", "httpx>=0.27", - "mlflow-skinny>=2.16", + # mlflow major must match rarelens-ml and the tracking server image. + "mlflow-skinny>=3,<4", "lightgbm>=4.5", + "scikit-learn>=1.5", # the pickled LGBMClassifier inside the pyfunc needs it to load "pandas>=2.2", ] [project.optional-dependencies] -dev = ["pytest", "pytest-asyncio", "ruff", "mypy", "aiosqlite"] +# pubsub: Kubernetes track; run: serverless track; storage: load a model from gs:// +gcp = ["google-cloud-pubsub>=2.23", "google-cloud-run>=0.10", "google-cloud-storage>=2.18"] +dev = [ + "pytest>=8", + "pytest-asyncio>=1.0", + "ruff==0.16.2", + "mypy>=1.11", + "pandas-stubs", + # Throwaway Postgres for `pytest` when DATABASE_URL is unset (no Docker needed). + "pgserver; sys_platform != 'win32'", +] + +# The repo also has top-level `alembic/` and `tests/` dirs; only `app` is the package. +[tool.setuptools.packages.find] +include = ["app*"] [tool.ruff] line-length = 100 target-version = "py312" +[tool.ruff.lint.isort] +# Without this the local `alembic/` migrations dir makes ruff treat the alembic library as first-party. +known-third-party = ["alembic"] + +[tool.mypy] +python_version = "3.12" + +[[tool.mypy.overrides]] +module = ["google.cloud.*", "mlflow.*"] +ignore_missing_imports = true + [tool.pytest.ini_options] asyncio_mode = "auto" +# asyncpg connections are bound to the loop that opened them; one loop for the whole run +# lets the app's pooled engine be shared across tests. +asyncio_default_fixture_loop_scope = "session" +asyncio_default_test_loop_scope = "session" +pythonpath = ["."] +testpaths = ["tests"] diff --git a/api/tests/conftest.py b/api/tests/conftest.py new file mode 100644 index 0000000..80692a5 --- /dev/null +++ b/api/tests/conftest.py @@ -0,0 +1,83 @@ +"""Test bootstrap. + +CI provides DATABASE_URL (a Postgres service container). Locally, when it is unset, a throwaway +Postgres is started with `pgserver` so `pytest` needs no Docker. Set REQUIRE_DB=1 to turn a missing +database into a failure instead of a skip (CI does this). + +DATABASE_URL must be settled before anything imports `app`, because `app.config.settings` and the +engine in `app.db` are created at import time. +""" +import os +import tempfile +from collections.abc import AsyncIterator, Iterator +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest +from httpx import ASGITransport, AsyncClient + +if TYPE_CHECKING: + from alembic.config import Config + +API_DIR = Path(__file__).resolve().parents[1] +_pg_server = None # keeps the pgserver handle alive for the whole session + + +def _resolve_database_url() -> str | None: + global _pg_server + if url := os.environ.get("DATABASE_URL"): + return url + try: + import pgserver + except ImportError: + return None + _pg_server = pgserver.get_server(tempfile.mkdtemp(prefix="rarelens-pg-"), cleanup_mode="delete") + socket_dir = _pg_server.get_uri().split("host=", 1)[1] + return f"postgresql+asyncpg://postgres@/postgres?host={socket_dir}" + + +DATABASE_URL = _resolve_database_url() +if DATABASE_URL: + os.environ["DATABASE_URL"] = DATABASE_URL + + +@pytest.fixture(scope="session") +def alembic_cfg() -> "Config": + if not DATABASE_URL: + if os.environ.get("REQUIRE_DB"): + pytest.fail("REQUIRE_DB is set but no database is available") + pytest.skip("no DATABASE_URL and pgserver is not installed") + from alembic.config import Config + + cfg = Config(str(API_DIR / "alembic.ini")) + cfg.set_main_option("script_location", str(API_DIR / "alembic")) + return cfg + + +@pytest.fixture(scope="session") +def migrated_db(alembic_cfg: "Config") -> Iterator[None]: + from alembic import command + + command.upgrade(alembic_cfg, "head") + yield + + +@pytest.fixture +async def db(migrated_db: None) -> AsyncIterator[None]: + from sqlalchemy import text + + from app.db import engine + + async with engine.begin() as conn: + await conn.execute( + text("TRUNCATE samples, jobs, variants, predictions RESTART IDENTITY CASCADE") + ) + yield + + +@pytest.fixture +async def client() -> AsyncIterator[AsyncClient]: + from app.main import app + + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c diff --git a/api/tests/factories.py b/api/tests/factories.py new file mode 100644 index 0000000..8fc481e --- /dev/null +++ b/api/tests/factories.py @@ -0,0 +1,21 @@ +import uuid +from typing import Any + +from app.db import SessionLocal +from app.models import Job, JobStatus, Sample, Variant + + +async def seed_job( + variants: list[dict[str, Any]], status: JobStatus = JobStatus.succeeded +) -> uuid.UUID: + """Insert a sample, a job and its variants; each variant dict overrides the defaults.""" + async with SessionLocal() as s: + sample = Sample(name=f"s-{uuid.uuid4()}", vcf_uri="gs://b/x.vcf.gz", assembly="GRCh38") + job = Job(sample=sample, status=status) + rows = [ + Variant(job=job, **{"chrom": "22", "pos": 1, "ref": "A", "alt": "G", "annotations": {}} | v) + for v in variants + ] + s.add_all([sample, job, *rows]) + await s.commit() + return job.id diff --git a/api/tests/test_annotate.py b/api/tests/test_annotate.py new file mode 100644 index 0000000..1cc2e2e --- /dev/null +++ b/api/tests/test_annotate.py @@ -0,0 +1,126 @@ +import uuid +from typing import Any + +import pytest +from httpx import AsyncClient + +from app.config import settings +from app.services import events + + +async def new_sample(client: AsyncClient) -> str: + r = await client.post( + "/api/samples", + json={"name": f"s-{uuid.uuid4()}", "vcf_uri": "gs://bucket/x.vcf.gz", "assembly": "GRCh37"}, + ) + assert r.status_code == 201, r.text + return r.json()["id"] + + +@pytest.mark.usefixtures("db") +async def test_local_without_nextflow_fails_fast_with_instructions( + client: AsyncClient, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(settings, "pubsub_topic", None) + monkeypatch.setattr(events.shutil, "which", lambda _: None) + sample_id = await new_sample(client) + + r = await client.post(f"/api/samples/{sample_id}/annotate") + assert r.status_code == 202 + job = r.json() + assert job["status"] == "failed" + assert "make annotate" in job["log"] + assert job["finished_at"] is not None + + +class FakeProcess: + def __init__(self, returncode: int, stderr: bytes) -> None: + self.returncode = returncode + self._stderr = stderr + self.pid = 4242 + + async def communicate(self) -> tuple[bytes, bytes]: + return b"", self._stderr + + +@pytest.mark.usefixtures("db") +async def test_local_run_is_watched_and_a_crash_marks_the_job_failed( + client: AsyncClient, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(settings, "pubsub_topic", None) + monkeypatch.setattr(events.shutil, "which", lambda _: "/usr/bin/nextflow") + launched: dict[str, Any] = {} + + async def fake_exec(*cmd: str, **kw: Any) -> FakeProcess: + launched["cmd"], launched["env"] = cmd, kw["env"] + return FakeProcess(1, b"ERROR ~ VEP cache not found") + + monkeypatch.setattr(events.asyncio, "create_subprocess_exec", fake_exec) + sample_id = await new_sample(client) + + r = await client.post(f"/api/samples/{sample_id}/annotate") + assert r.status_code == 202 + await events.drain() + + job = (await client.get(f"/api/jobs/{r.json()['id']}")).json() + assert job["status"] == "failed" + assert "VEP cache not found" in job["log"] + # The DB password travels in the environment, never on the command line. + assert settings.database_url not in " ".join(launched["cmd"]) + assert launched["env"]["DATABASE_URL"] == settings.database_url + assert launched["cmd"][launched["cmd"].index("--assembly") + 1] == "GRCh37" + + +class FakeFuture: + def __init__(self, result: str | Exception) -> None: + self._result = result + + def result(self, timeout: float | None = None) -> str: + if isinstance(self._result, Exception): + raise self._result + return self._result + + +class FakePublisher: + def __init__(self, result: str | Exception) -> None: + self.result = result + self.published: list[tuple[str, bytes]] = [] + + def topic_path(self, project: str, topic: str) -> str: + return f"projects/{project}/topics/{topic}" + + def publish(self, topic: str, data: bytes) -> FakeFuture: + self.published.append((topic, data)) + return FakeFuture(self.result) + + +@pytest.mark.usefixtures("db") +async def test_pubsub_publishes_to_the_full_topic_path( + client: AsyncClient, monkeypatch: pytest.MonkeyPatch +) -> None: + publisher = FakePublisher("msg-123") + monkeypatch.setattr(settings, "pubsub_topic", "vcf-uploaded") + monkeypatch.setattr(settings, "gcp_project", "my-proj") + monkeypatch.setattr(events, "_publisher", lambda: publisher) + sample_id = await new_sample(client) + + r = await client.post(f"/api/samples/{sample_id}/annotate") + job = r.json() + assert (job["status"], job["workflow_ref"]) == ("running", "pubsub:msg-123") + [(topic, data)] = publisher.published + assert topic == "projects/my-proj/topics/vcf-uploaded" + assert b'"assembly": "GRCh37"' in data and job["id"].encode() in data + + +@pytest.mark.usefixtures("db") +async def test_pubsub_failure_marks_the_job_failed( + client: AsyncClient, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(settings, "pubsub_topic", "vcf-uploaded") + monkeypatch.setattr(settings, "gcp_project", "my-proj") + monkeypatch.setattr(events, "_publisher", lambda: FakePublisher(RuntimeError("403 denied"))) + sample_id = await new_sample(client) + + job = (await client.post(f"/api/samples/{sample_id}/annotate")).json() + assert job["status"] == "failed" + assert "403 denied" in job["log"] diff --git a/api/tests/test_cloudrun.py b/api/tests/test_cloudrun.py new file mode 100644 index 0000000..7309844 --- /dev/null +++ b/api/tests/test_cloudrun.py @@ -0,0 +1,97 @@ +"""The serverless track: the API starts a Cloud Run job instead of publishing to Pub/Sub.""" +import uuid +from typing import Any + +import pytest +from httpx import AsyncClient + +from app.config import settings +from app.services import events + + +class FakeOperation: + def __init__(self, execution: str) -> None: + self.metadata = type("Meta", (), {"name": execution})() + + +class FakeJobsClient: + def __init__(self, result: Any = None) -> None: + self.result = result + self.requests: list[dict] = [] + + def run_job(self, request: dict) -> FakeOperation: + self.requests.append(request) + if isinstance(self.result, Exception): + raise self.result + return self.result + + +async def new_sample(client: AsyncClient) -> str: + r = await client.post( + "/api/samples", + json={"name": f"s-{uuid.uuid4()}", "vcf_uri": "gs://bucket/x.vcf.gz", "assembly": "GRCh37"}, + ) + assert r.status_code == 201, r.text + return r.json()["id"] + + +@pytest.fixture +def cloudrun(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(settings, "pubsub_topic", None) + monkeypatch.setattr(settings, "cloudrun_job", "rarelens-nextflow") + monkeypatch.setattr(settings, "gcp_project", "my-proj") + monkeypatch.setattr(settings, "gcp_region", "europe-west2") + + +@pytest.mark.usefixtures("db", "cloudrun") +async def test_annotate_executes_the_job_with_pipeline_arguments( + client: AsyncClient, monkeypatch: pytest.MonkeyPatch +) -> None: + jobs = FakeJobsClient(FakeOperation("projects/p/locations/l/executions/rarelens-nextflow-abc12")) + monkeypatch.setattr(events, "_jobs_client", lambda: jobs) + sample_id = await new_sample(client) + + r = await client.post(f"/api/samples/{sample_id}/annotate") + assert r.status_code == 202 + job = r.json() + assert job["status"] == "running" + assert job["workflow_ref"] == "cloudrun:rarelens-nextflow-abc12" + + [request] = jobs.requests + assert request["name"] == "projects/my-proj/locations/europe-west2/jobs/rarelens-nextflow" + [override] = request["overrides"]["container_overrides"] + args = override["args"] + assert args[:4] == ["run", "/pipeline/main.nf", "-profile", "gcp"] + assert args[args.index("--job_id") + 1] == job["id"] + assert args[args.index("--assembly") + 1] == "GRCh37" + assert args[args.index("--vcf") + 1] == "gs://bucket/x.vcf.gz" + + +@pytest.mark.usefixtures("db", "cloudrun") +async def test_a_failed_execution_call_marks_the_job_failed( + client: AsyncClient, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr( + events, "_jobs_client", lambda: FakeJobsClient(RuntimeError("403 permission denied")) + ) + sample_id = await new_sample(client) + + job = (await client.post(f"/api/samples/{sample_id}/annotate")).json() + assert job["status"] == "failed" + assert "403 permission denied" in job["log"] + + +@pytest.mark.usefixtures("db", "cloudrun") +async def test_cloud_run_job_takes_precedence_over_pubsub( + client: AsyncClient, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(settings, "pubsub_topic", "vcf-uploaded") + jobs = FakeJobsClient(FakeOperation("projects/p/locations/l/executions/x-1")) + monkeypatch.setattr(events, "_jobs_client", lambda: jobs) + monkeypatch.setattr( + events, "_publisher", lambda: pytest.fail("Pub/Sub must not be used in the serverless track") + ) + sample_id = await new_sample(client) + + assert (await client.post(f"/api/samples/{sample_id}/annotate")).json()["status"] == "running" + assert len(jobs.requests) == 1 diff --git a/api/tests/test_db_url.py b/api/tests/test_db_url.py new file mode 100644 index 0000000..4795e0d --- /dev/null +++ b/api/tests/test_db_url.py @@ -0,0 +1,34 @@ +"""Neon-style URLs use libpq's sslmode, which asyncpg does not understand.""" +import pytest + +from app.db import normalize_async_url + + +@pytest.mark.parametrize( + ("raw", "expected"), + [ + ( + "postgresql+asyncpg://u:p@ep-x.eu-central-1.aws.neon.tech/neondb?sslmode=require", + "postgresql+asyncpg://u:p@ep-x.eu-central-1.aws.neon.tech/neondb?ssl=require", + ), + # Already asyncpg-shaped, or nothing to do. + ( + "postgresql+asyncpg://u:p@h/db?ssl=require", + "postgresql+asyncpg://u:p@h/db?ssl=require", + ), + ("postgresql+asyncpg://u:p@h:5432/db", "postgresql+asyncpg://u:p@h:5432/db"), + # Only the asyncpg driver needs the rewrite. + ("postgresql+psycopg://u:p@h/db?sslmode=require", "postgresql+psycopg://u:p@h/db?sslmode=require"), + ], +) +def test_normalize_async_url(raw: str, expected: str) -> None: + assert normalize_async_url(raw) == expected + + +def test_disable_is_preserved() -> None: + assert normalize_async_url("postgresql+asyncpg://u:p@h/db?sslmode=disable").endswith("ssl=disable") + + +def test_unix_socket_url_is_untouched() -> None: + raw = "postgresql+asyncpg://postgres@/postgres?host=/tmp/pg" + assert normalize_async_url(raw) == raw diff --git a/api/tests/test_health.py b/api/tests/test_health.py index 336384a..73fc019 100644 --- a/api/tests/test_health.py +++ b/api/tests/test_health.py @@ -1,12 +1,7 @@ -import pytest -from httpx import ASGITransport, AsyncClient - -from app.main import app +from httpx import AsyncClient -@pytest.mark.asyncio -async def test_health(): - async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: - r = await c.get("/health") +async def test_health(client: AsyncClient) -> None: + r = await client.get("/health") assert r.status_code == 200 assert r.json() == {"status": "ok"} diff --git a/api/tests/test_migrations.py b/api/tests/test_migrations.py new file mode 100644 index 0000000..dc516ff --- /dev/null +++ b/api/tests/test_migrations.py @@ -0,0 +1,15 @@ +import asyncio + +from alembic import command +from alembic.config import Config + +from app.db import engine + + +async def test_downgrade_to_base_then_upgrade_again(alembic_cfg: Config) -> None: + # alembic's env.py runs its own event loop, so keep it off this one. + await asyncio.to_thread(command.downgrade, alembic_cfg, "base") + await asyncio.to_thread(command.upgrade, alembic_cfg, "head") + # The recreated enum type has a new OID that pooled asyncpg connections have not seen; + # reconnect so later tests (and the database left at head) are unaffected. + await engine.dispose() diff --git a/api/tests/test_model_loading.py b/api/tests/test_model_loading.py new file mode 100644 index 0000000..79d74ca --- /dev/null +++ b/api/tests/test_model_loading.py @@ -0,0 +1,47 @@ +"""A model artifact URI (gs://...) lets the API score without an MLflow server running.""" +# Imported eagerly: mlflow loads .pyfunc lazily, so patching it by name can hit the proxy. +import mlflow.pyfunc +import pytest + +from app.config import settings +from app.services import scoring + + +@pytest.fixture(autouse=True) +def clear_cache() -> None: + scoring._models.clear() + + +def no_registry(**kwargs: object) -> None: + pytest.fail("the registry must not be contacted when MODEL_URI is set") + + +def test_model_uri_skips_the_registry(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(settings, "model_uri", "gs://bucket/models/pathogenicity/3") + monkeypatch.setattr(scoring, "MlflowClient", no_registry) + monkeypatch.setattr(mlflow.pyfunc, "load_model", lambda uri: f"model@{uri}") + + model, version = scoring.load_model() + assert model == "model@gs://bucket/models/pathogenicity/3" + assert version == "3" + + +def test_model_uri_without_a_version_segment_still_labels_the_prediction( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(settings, "model_uri", "gs://bucket/models/pathogenicity/") + monkeypatch.setattr(scoring, "MlflowClient", no_registry) + monkeypatch.setattr(mlflow.pyfunc, "load_model", lambda uri: "model") + + assert scoring.load_model()[1] == "pathogenicity" + + +def test_model_uri_is_loaded_once(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(settings, "model_uri", "gs://bucket/models/pathogenicity/3") + monkeypatch.setattr(scoring, "MlflowClient", no_registry) + calls: list[str] = [] + monkeypatch.setattr(mlflow.pyfunc, "load_model", lambda uri: calls.append(uri) or "model") + + scoring.load_model() + scoring.load_model() + assert len(calls) == 1 diff --git a/api/tests/test_routes.py b/api/tests/test_routes.py new file mode 100644 index 0000000..b8d133b --- /dev/null +++ b/api/tests/test_routes.py @@ -0,0 +1,13 @@ +import pytest +from httpx import AsyncClient + + +@pytest.mark.usefixtures("db") +async def test_resources_live_under_api_prefix(client: AsyncClient) -> None: + # The ingress forwards /api/* unchanged, so the app itself must serve that prefix. + assert (await client.get("/api/samples")).status_code == 200 + assert (await client.get("/samples")).status_code == 404 + + +async def test_health_stays_at_root_for_probes(client: AsyncClient) -> None: + assert (await client.get("/health")).status_code == 200 diff --git a/api/tests/test_samples.py b/api/tests/test_samples.py new file mode 100644 index 0000000..0d4b8de --- /dev/null +++ b/api/tests/test_samples.py @@ -0,0 +1,18 @@ +import pytest +from httpx import AsyncClient + + +@pytest.mark.usefixtures("db") +async def test_duplicate_sample_name_is_409_not_500(client: AsyncClient) -> None: + body = {"name": "HG002", "vcf_uri": "gs://bucket/hg002.vcf.gz"} + assert (await client.post("/api/samples", json=body)).status_code == 201 + r = await client.post("/api/samples", json=body) + assert r.status_code == 409 + assert "HG002" in r.json()["detail"] + + +async def test_unknown_assembly_is_rejected(client: AsyncClient) -> None: + r = await client.post( + "/api/samples", json={"name": "a", "vcf_uri": "gs://bucket/a.vcf.gz", "assembly": "hg19"} + ) + assert r.status_code == 422 diff --git a/api/tests/test_scoring.py b/api/tests/test_scoring.py new file mode 100644 index 0000000..cb85d9a --- /dev/null +++ b/api/tests/test_scoring.py @@ -0,0 +1,90 @@ +import math +import uuid + +import numpy as np +import pandas as pd +import pytest +from httpx import AsyncClient +from sqlalchemy import select + +from app.db import SessionLocal +from app.models import Job, JobStatus, Prediction, Sample, Variant +from app.services import scoring + + +def variant(**kw: object) -> Variant: + fields: dict = {"chrom": "22", "pos": 1, "ref": "A", "alt": "G", "annotations": {}} + fields.update(kw) + return Variant(**fields) + + +def test_raw_frame_sends_the_model_contract_columns() -> None: + frame = scoring.raw_frame([ + variant(impact="HIGH", consequence="stop_gained", gnomad_af=None, + annotations={"CADD_PHRED": "35", "am_pathogenicity": "0.98"}), + variant(impact="LOW", consequence="synonymous_variant", gnomad_af=0.2, annotations={}), + ]) + assert list(frame.columns) == scoring.RAW_COLUMNS + assert frame["impact"].tolist() == ["HIGH", "LOW"] + assert frame["cadd_phred"].iloc[0] == "35" + assert pd.isna(frame["cadd_phred"].iloc[1]) + assert math.isnan(frame["gnomad_af"].iloc[0]) + + +class FakeModel: + def __init__(self, score: float) -> None: + self.score = score + + def predict(self, frame: pd.DataFrame) -> np.ndarray: + assert list(frame.columns) == scoring.RAW_COLUMNS + return np.full(len(frame), self.score) + + +async def make_job(status: JobStatus, n_variants: int) -> uuid.UUID: + async with SessionLocal() as s: + sample = Sample(name=f"s-{uuid.uuid4()}", vcf_uri="gs://b/x.vcf.gz", assembly="GRCh38") + job = Job(sample=sample, status=status) + s.add_all([sample, job, *(variant(job=job, pos=i + 1) for i in range(n_variants))]) + await s.commit() + return job.id + + +async def predictions(job_id: uuid.UUID) -> list[Prediction]: + async with SessionLocal() as s: + rows = await s.scalars( + select(Prediction).join(Variant).where(Variant.job_id == job_id) + ) + return list(rows) + + +@pytest.mark.usefixtures("db") +async def test_scoring_twice_updates_instead_of_failing( + client: AsyncClient, monkeypatch: pytest.MonkeyPatch +) -> None: + job_id = await make_job(JobStatus.succeeded, n_variants=3) + + monkeypatch.setattr(scoring, "load_model", lambda: (FakeModel(0.9), "7")) + r = await client.post(f"/api/predictions/score/{job_id}") + assert r.status_code == 200, r.text + assert r.json() == {"job_id": str(job_id), "scored": 3, "model_version": "7"} + + monkeypatch.setattr(scoring, "load_model", lambda: (FakeModel(0.2), "8")) + r = await client.post(f"/api/predictions/score/{job_id}") + assert r.status_code == 200, r.text + + preds = await predictions(job_id) + assert len(preds) == 3 + assert {(p.score, p.model_version) for p in preds} == {(0.2, "8")} + + +@pytest.mark.usefixtures("db") +async def test_scoring_unknown_job_is_404(client: AsyncClient) -> None: + r = await client.post(f"/api/predictions/score/{uuid.uuid4()}") + assert r.status_code == 404 + + +@pytest.mark.usefixtures("db") +async def test_scoring_unfinished_job_is_409(client: AsyncClient) -> None: + job_id = await make_job(JobStatus.running, n_variants=1) + r = await client.post(f"/api/predictions/score/{job_id}") + assert r.status_code == 409 diff --git a/api/tests/test_security.py b/api/tests/test_security.py new file mode 100644 index 0000000..86b6bdd --- /dev/null +++ b/api/tests/test_security.py @@ -0,0 +1,51 @@ +import pytest +from httpx import AsyncClient +from pydantic import ValidationError + +from app.schemas import SampleCreate + + +@pytest.mark.parametrize( + "uri", + [ + "gs://bucket/dir/sample.vcf.gz", + "gs://my.bucket-1/a/b.bcf", + "gs://bucket/x.vcf", + "/data/example.vcf.gz", + "/data/giab/hg002.chr22.vcf.bgz", + ], +) +def test_vcf_uri_accepts_gcs_objects_and_files_under_the_data_root(uri: str) -> None: + assert SampleCreate(name="s", vcf_uri=uri).vcf_uri == uri + + +@pytest.mark.parametrize( + "uri", + [ + "-c/tmp/evil.config", # would be parsed as a Nextflow option + "--outdir=/etc", + "/etc/passwd", # outside the data root + "/data/../etc/shadow.vcf", # traversal out of it + "data/example.vcf.gz", # relative: depends on the API's working directory + "gs://bucket/notes.txt", # not a VCF/BCF + "https://example.com/x.vcf.gz", + "gs:///x.vcf.gz", + "/data/x.vcf.gz\n--foo", # control characters + "", + ], +) +def test_vcf_uri_rejects_everything_else(uri: str) -> None: + with pytest.raises(ValidationError): + SampleCreate(name="s", vcf_uri=uri) + + +async def test_bad_vcf_uri_is_422_at_the_api(client: AsyncClient) -> None: + r = await client.post("/api/samples", json={"name": "s", "vcf_uri": "/etc/passwd"}) + assert r.status_code == 422 + + +async def test_cors_allows_only_configured_origins(client: AsyncClient) -> None: + allowed = await client.get("/health", headers={"Origin": "http://localhost:5173"}) + assert allowed.headers.get("access-control-allow-origin") == "http://localhost:5173" + other = await client.get("/health", headers={"Origin": "https://evil.example"}) + assert "access-control-allow-origin" not in other.headers diff --git a/api/tests/test_variants.py b/api/tests/test_variants.py new file mode 100644 index 0000000..386cadd --- /dev/null +++ b/api/tests/test_variants.py @@ -0,0 +1,46 @@ +import uuid + +import pytest +from factories import seed_job +from httpx import AsyncClient + + +@pytest.mark.parametrize( + "paging", [{"limit": 0}, {"limit": -1}, {"limit": 501}, {"offset": -1}] +) +async def test_out_of_range_paging_is_422_not_500(client: AsyncClient, paging: dict) -> None: + r = await client.get("/api/variants", params={"job_id": str(uuid.uuid4()), **paging}) + assert r.status_code == 422 + + +async def positions(client: AsyncClient, job_id: uuid.UUID) -> list[tuple[str, int, str]]: + r = await client.get("/api/variants", params={"job_id": str(job_id)}) + assert r.status_code == 200, r.text + return [(v["chrom"], v["pos"], v["alt"]) for v in r.json()["items"]] + + +@pytest.mark.usefixtures("db") +async def test_chromosomes_sort_naturally(client: AsyncClient) -> None: + job_id = await seed_job([{"chrom": c} for c in ["10", "MT", "2", "X", "chr3", "1", "Y"]]) + assert [c for c, _, _ in await positions(client, job_id)] == [ + "1", "2", "chr3", "10", "X", "Y", "MT", + ] + + +@pytest.mark.usefixtures("db") +async def test_same_position_keeps_insertion_order_across_pages(client: AsyncClient) -> None: + # Split multiallelics share chrom/pos; without a tiebreak, pages can repeat or skip rows. + job_id = await seed_job([{"pos": 5, "alt": a} for a in "CGT"]) + assert [a for _, _, a in await positions(client, job_id)] == ["C", "G", "T"] + + +@pytest.mark.usefixtures("db") +async def test_long_vep_strings_are_stored(client: AsyncClient) -> None: + clin_sig = ",".join(["conflicting_classifications_of_pathogenicity"] * 8) + consequence = ( + "splice_region_variant&splice_polypyrimidine_tract_variant&intron_variant" + "&NMD_transcript_variant&non_coding_transcript_variant" + ) + job_id = await seed_job([{"clinvar_sig": clin_sig, "consequence": consequence}]) + [v] = (await client.get("/api/variants", params={"job_id": str(job_id)})).json()["items"] + assert (v["clinvar_sig"], v["consequence"]) == (clin_sig, consequence) diff --git a/data/README.md b/data/README.md index 8d57f0e..c062cec 100644 --- a/data/README.md +++ b/data/README.md @@ -1,5 +1,8 @@ # Test data +`make data` fetches the demo slice automatically. Provenance, licences, citations and the +evaluation plan live in [../docs/data.md](../docs/data.md). + No patient data. Use public sources only: - ClinVar VCF (GRCh38): https://ftp.ncbi.nlm.nih.gov/pub/clinvar/vcf_GRCh38/ @@ -14,3 +17,27 @@ bcftools view -r 22 clinvar.vcf.gz | bcftools view -H | head -2000 > body.vcf (bcftools view -h clinvar.vcf.gz; cat body.vcf) | bgzip > example.vcf.gz tabix -p vcf example.vcf.gz ``` + +Samples added in the UI must point at a `gs://bucket/object` or at a file under `/data` +(`LOCAL_DATA_ROOT`), ending in `.vcf`, `.vcf.gz`, `.vcf.bgz` or `.bcf`. + +## VEP cache and plugins + +The pipeline runs VEP offline. Install the cache once (about 25 GB for GRCh38): + +```bash +docker run --rm -v $PWD/pipeline/cache/vep:/cache ensemblorg/ensembl-vep:release_113.0 \ + INSTALL.pl -a cf -s homo_sapiens -y GRCh38 -c /cache +``` + +CADD and AlphaMissense are optional; the model treats their scores as missing without them. To +enable them, put the plugin modules and data files in one directory and pass `--vep_plugin_data`: + +```bash +docker run --rm -v $PWD/pipeline/cache/plugins:/plugins ensemblorg/ensembl-vep:release_113.0 \ + INSTALL.pl -a p -g CADD,AlphaMissense -r /plugins +# then add whole_genome_SNVs.tsv.gz, gnomad.genomes.r4.0.indel.tsv.gz (CADD) and +# AlphaMissense_hg38.tsv.gz, each with its .tbi index, to pipeline/cache/plugins +``` + +`pipeline/tests/data/tiny.vcf` is a synthetic three-record fixture used by CI's stub run. diff --git a/docker-compose.yml b/docker-compose.yml index 159ccc2..3559bea 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,15 +28,20 @@ services: context: ./web target: build environment: - PUBLIC_API_URL: http://localhost:8000 + PUBLIC_API_URL: http://localhost:8000/api ports: ["5173:5173"] depends_on: [api] volumes: ["./web:/app", "/app/node_modules"] command: npm run dev -- --host mlflow: - image: ghcr.io/mlflow/mlflow:v2.16.0 - command: mlflow server --host 0.0.0.0 --backend-store-uri sqlite:///mlflow.db --default-artifact-root /mlruns + image: ghcr.io/mlflow/mlflow:v3.16.0 + # --artifacts-destination makes the server proxy artifacts over HTTP; a bare local + # --default-artifact-root would hand clients a /mlruns path only this container can see. + command: > + mlflow server --host 0.0.0.0 + --backend-store-uri sqlite:////mlruns/mlflow.db + --artifacts-destination /mlruns ports: ["5000:5000"] volumes: [mlruns:/mlruns] diff --git a/docs/architecture.md b/docs/architecture.md index ad015c0..c807654 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -3,30 +3,59 @@ ```mermaid flowchart LR U[Scientist] -->|browser| W[SvelteKit web] - W -->|REST| A[FastAPI] + W -->|REST /api| A[FastAPI] A --> P[(PostgreSQL / Cloud SQL)] A -->|publish vcf-uploaded| Q[Pub/Sub] Q --> E[Argo Events sensor] - E --> AW[Argo Workflow] - AW --> NF[Nextflow: bcftools norm, VEP, load_db] - NF -->|reads VCF| G[(GCS bucket)] - NF -->|writes variants| P - A -->|models:/rarelens-pathogenicity| M[MLflow registry] - T[ml/train.py on GKE, optional GPU] --> M + E --> AW[Argo Workflow: Nextflow driver] + AW -->|tasks| B[Google Batch: bcftools norm, VEP, load_db] + B -->|reads VCF, VEP cache| G[(GCS bucket)] + B -->|writes variants, marks job succeeded| P + AW -.->|exit handler marks job failed| P + A -->|models:/rarelens-pathogenicity@production| M[MLflow registry] + T[ml/train.py] --> M GH[GitHub Actions] -->|images via WIF| AR[Artifact Registry] GH -->|bumps overlay tags| R[(git: infra/k8s/overlays/gcp)] R --> CD[ArgoCD] --> K[GKE Autopilot] ``` +## Two deployment tracks + +The same images and the same pipeline, deployed two ways (`infra/terraform/variables.tf`): + +| | Serverless (default) | Kubernetes (`-var deploy_kubernetes=true`) | +|---|---|---| +| api, web | Cloud Run, scale to zero | Deployments behind an ingress | +| dispatch | the API executes a Cloud Run job | Pub/Sub -> Argo Events -> Argo Workflow | +| pipeline tasks | Google Batch | Google Batch | +| `/api` routing | the web service proxies it | the ingress routes it | +| idle cost | ~£1/month | ~£130+/month | + +`app.services.events.launch()` picks the dispatch backend from configuration: a Cloud Run job when +`CLOUDRUN_JOB` is set, Pub/Sub when `PUBSUB_TOPIC` is, and a local Nextflow process otherwise. +See [cloud.md](cloud.md) for why the serverless one is the default. + ## Why these choices **One monorepo.** The four components share a schema (`variants` table, feature columns) and the point of the exercise is to see them evolve together. Separate repos would hide the coupling. **Nextflow for the science, Argo Workflows for the trigger.** Nextflow is the lingua franca for -bioinformatics pipelines and has a native Kubernetes executor. Argo is what the platform team -already runs. So Argo owns *when* a pipeline runs; Nextflow owns *what* it does. The API never -talks to Kubernetes directly; it publishes an event and gets on with its life. +bioinformatics pipelines. Argo is what the platform team already runs. So Argo owns *when* a +pipeline runs; Nextflow owns *what* it does. The API never talks to Kubernetes directly; it +publishes an event and gets on with its life. The Nextflow driver runs in the Argo pod and sends +each task to Google Batch: a `gs://` work directory needs an executor that stages through GCS +(Nextflow's Kubernetes executor needs a shared ReadWriteMany volume instead). + +**Job lifecycle.** The API creates a job as `running` once the pipeline is dispatched, or `failed` +with the reason in `jobs.log` when dispatch is impossible. The loader marks it `succeeded` in the +same transaction that stores the variants. Anything else (Nextflow error, eviction) is caught by +the Argo exit handler, or locally by the API watching the Nextflow process, and marked `failed`, +so the UI never polls a dead job. + +**Variant identity.** NORMALISE sets each VCF ID to `CHROM_POS_REF_ALT`; VEP echoes it as +`Uploaded_variation` and the loader takes exact VCF alleles from it, because VEP's own +`Location`/`Allele` columns trim indel alleles. **FastAPI + Pydantic v2 + SQLAlchemy 2.0 async.** Typed at both boundaries: request/response models and ORM models are separate on purpose so the database can change without breaking the frontend @@ -35,15 +64,21 @@ because the pipeline container should not import the API. **SvelteKit.** Small runtime, no virtual DOM, and Svelte 5 runes make server-driven state simple. The UI has exactly two pages; the goal is a table a scientist actually wants to filter, not a dashboard. +`PUBLIC_API_URL` is read at runtime, so one image works behind the ingress (`/api`) and elsewhere. **GKE Autopilot + Cloud SQL, not self-managed.** The lab is about the platform patterns (Workload -Identity, GitOps, Kustomize overlays, GPU node selection), not about running etcd. +Identity, GitOps, Kustomize overlays, private networking), not about running etcd. Cloud SQL has only +a private IP; the API reaches it through a Cloud SQL Proxy sidecar, pipeline tasks directly in the VPC. +Database URLs live in Secret Manager. **GitOps.** CI builds and tests; it never runs `kubectl apply`. It edits image tags in the `gcp` overlay -and ArgoCD reconciles. Rollback is `git revert`. +and ArgoCD reconciles. Rollback is `git revert`. Migrations run in an init container under a Postgres +advisory lock, so replicas starting together migrate once. -**MLflow registry as the model contract.** The API loads `models:/rarelens-pathogenicity/Production`. -Training writes there; serving reads there. Feature engineering lives in one module that both sides import. +**MLflow registry as the model contract.** The API loads whichever version the `production` alias +points at and records that version on every prediction. The registered model is a pyfunc that owns its +feature engineering (`rarelens_ml.features` ships inside it) and returns P(pathogenic), so serving only +sends raw columns and cannot drift from training. ## What is deliberately missing diff --git a/docs/cloud.md b/docs/cloud.md new file mode 100644 index 0000000..d819560 --- /dev/null +++ b/docs/cloud.md @@ -0,0 +1,112 @@ +# Cloud choice: Google Cloud, with a documented AWS escape hatch + +Decided 2026-09-12. Scope: `infra/terraform/`, `infra/k8s/`, `pipeline/nextflow.config`. + +## Decision + +rarelens deploys to **Google Cloud**. AWS was the serious alternative, and it is genuinely better +on two points (below), but not by enough to justify rebuilding an estate that already works. + +## Why Google Cloud + +| Reason | Detail | +|---|---| +| One Kubernetes control plane is effectively free | GKE's free tier gives $74.40/month in credits per billing account, which covers one Autopilot or zonal cluster. EKS charges $0.10/hour per cluster (~$73/month) with no equivalent credit. For a self-funded lab this is the largest fixed monthly difference. | +| The executor question is already settled here | Google retired Cloud Life Sciences on 8 July 2025; Batch is its successor, and Nextflow upstream moved to Google Batch in April 2025. `pipeline/nextflow.config` uses `google-batch`, which is the supported path rather than a legacy one. | +| The estate exists and is verified | Terraform (custom VPC, private Cloud SQL, Workload Identity, Secret Manager, Batch IAM), Kustomize overlays, Argo Workflows/Events and CI all render, validate and pass tests today. Rebuilding this on AWS costs 1–2 weeks and mostly repeats learning already banked. | +| No data lock-in | Every dataset the platform uses is readable from either cloud (see [data.md](data.md)): gnomAD publishes to GCP, AWS and Azure; GIAB and 1000 Genomes are open on AWS and NCBI; ClinVar is a plain NCBI download. | + +## What AWS is genuinely better at + +- **Managed Nextflow.** AWS HealthOmics runs Nextflow (up to 26.04), WDL and CWL as a managed + service, and is available in London (`eu-west-2`). GCP has no equivalent: you operate the + driver yourself, which is exactly what `infra/argo-workflows/annotate.yaml` does. +- **UK life-sciences gravity.** The UK Biobank Research Analysis Platform is DNAnexus running on + AWS, hosted in the UK. If the aim is to mirror what Cambridge-area employers run day to day, + AWS is the more common answer. + +## When to revisit this + +Move to AWS if any of these becomes true: + +- The lab wants a managed pipeline runner instead of an Argo + Batch driver it maintains. +- Matching an AWS-first employer's stack matters more than the two weeks it costs. +- The shape changes: several clusters, or enough managed-service spend that one free control + plane stops being material. + +## Running this on a hobby budget + +The cloud is not the cost driver; the always-on shape is. Estimates below are list price, and +rounded — treat them as orders of magnitude, not quotes. + +### What the Kubernetes estate costs at rest + +GKE Autopilot bills what pods *request*, not what they use, with a per-pod floor (250m vCPU / +512 MiB). The free tier credit covers the cluster fee only, not pod-hours. + +| Always-on | Requests | ~Monthly (us-central1 rates: $0.0445/vCPU-h, $0.0049/GiB-h) | +|---|---|---| +| api + web (2 replicas each, incl. Cloud SQL proxy sidecar) | ~1.2 vCPU, ~2.3 GiB | ~$47 | +| ArgoCD, Argo Workflows, Argo Events + NATS EventBus (~11 pods at the floor) | ~2.8 vCPU, ~5.5 GiB | ~$110 | +| Cloud SQL `db-f1-micro` | — | ~$8–12 | +| **Total** | | **~$165–170, London a bit more** | + +That is the wrong shape for a portfolio that is idle 99% of the time. + +### The shape that costs ~£1/month + +This is what `terraform apply` builds by default (`deploy_kubernetes` and `deploy_cloud_sql` are +both `false`). Kubernetes becomes something you switch on to show, not something you rent: + +| Piece | Service | Idle cost | +|---|---|---| +| api, web | Cloud Run, `min-instances=0`, `max_instances` capped | £0 — Always Free covers 2M requests, 180k vCPU-s, 360k GiB-s per month | +| Nextflow driver | Cloud Run **job**, started by the API through the Jobs API (`roles/run.jobsExecutorWithOverrides`, one job only) | £0 idle, pennies per run | +| Pipeline tasks | Google Batch on **Spot** VMs | £0 idle; a chr22 VEP run is a few pence | +| Database | Neon free tier (scale-to-zero, 0.5 GB) via `TF_VAR_database_url`, or `-var deploy_cloud_sql=true` | £0 (or ~$8–12) | +| Model | pyfunc artifact loaded straight from GCS (`MODEL_URI`), no MLflow server running | £0 | +| Storage | GCS + Artifact Registry | ~£1 (VEP cache dominates; Nearline halves it) | + +Trade-offs worth knowing: Cloud Run cold starts add 1–3 s to the first request after idle; the +Cloud Run path drops Pub/Sub, Argo Events and Argo Workflows from the critical path (the API calls +the Jobs API directly); and 0.5 GB of Neon does not fit a whole chromosome once `annotations` +stores the full VEP record — demo a gene panel, or store only the annotation keys the UI uses. + +Only the web service needs to be public: it serves the UI and proxies `/api` to the API service +(`web/src/routes/api/[...path]`), which is the same shape the ingress gives the Kubernetes track, +so the frontend code is identical either way. + +### Keep the Kubernetes story, stop paying rent for it + +`infra/k8s/` and `infra/argo-workflows/` stay in the repo and stay deployable. Bring the estate up +with `terraform apply` for an interview or a recording (roughly $0.25/hour while running, so a +two-hour demo is small change), then `terraform destroy -var deletion_protection=false`. `make kind` +runs the same manifests locally for free. + +### Guardrails + +- A billing budget with alerts at £5/£10, before anything else. +- `max-instances` on every Cloud Run service: scale-to-zero protects the floor, a cap protects the ceiling. +- Spot VMs for Batch, and the existing 30-day lifecycle rule on `work/` in the bucket. +- New accounts get $300 of Google Cloud credit for 90 days, which covers the experimenting phase. + +### One more reason not to switch to AWS + +AWS replaced its 12-month free tier on 15 July 2025 with credits ($100, up to $200) on a Free plan +that closes after six months or when the credits run out. Google's Always Free quotas, including +Cloud Run's, are permanent. For a demo meant to stay reachable indefinitely at near-zero cost, +that difference matters more than any feature comparison above. + +## The escape hatch + +Nextflow is the portability layer: executors are configuration, not code. An AWS run needs a new +profile in `pipeline/nextflow.config` (`process.executor = 'awsbatch'`, an S3 work directory and a +job queue), or a HealthOmics workflow definition. The processes themselves do not change. Keeping +`pipeline/bin/` cloud-agnostic (the scripts read `DATABASE_URL` from the environment, never from a +command line) is what keeps that true. + +Sources: [Migrate to Batch from Cloud Life Sciences](https://docs.cloud.google.com/batch/docs/migrate-to-batch-from-cloud-life-sciences), +[GKE pricing](https://cloud.google.com/kubernetes-engine/pricing), +[HealthOmics supported languages](https://docs.aws.amazon.com/omics/latest/dev/workflows-supported-languages.html), +[HealthOmics Nextflow 26.04](https://aws.amazon.com/about-aws/whats-new/2026/06/aws-healthomics-nextflow-version-26-04/), +[UK Biobank Research Analysis Platform](https://www.ukbiobank.ac.uk/use-our-data/research-analysis-platform/). diff --git a/docs/data.md b/docs/data.md new file mode 100644 index 0000000..a9d0925 --- /dev/null +++ b/docs/data.md @@ -0,0 +1,74 @@ +# Data: what rarelens actually runs on + +Everything below is public, peer-reviewed and consented for open redistribution. No patient data, +no data access agreement, nothing that needs an application. These are the references to quote +when showing the platform to someone. + +Citations were verified against [PubMed](https://pubmed.ncbi.nlm.nih.gov/); each row links its DOI. + +## The demo slice + +`make data` fetches two real files, chromosome 22 only (roughly 100 MB, minutes rather than hours): + +| File | What it is | Role | +|---|---|---| +| `data/example.vcf.gz` | GIAB HG002 (NA24385) v4.2.1 benchmark calls, GRCh38, chr22 | the sample a scientist annotates | +| `data/clinvar.chr22.vcf.gz` | ClinVar, GRCh38, chr22 | training labels, and the ClinVar column in the UI | + +HG002 is the NIST Genome in a Bottle Ashkenazi son, recruited through the Personal Genome Project, +which consents participants to unrestricted public release. It is the reference genome the field +benchmarks variant callers against, so it is both realistic and unambiguously shareable. + +## Datasets + +| Dataset | Used for | Access | Terms | Citation | +|---|---|---|---|---| +| **ClinVar** (GRCh38) | pathogenic/benign labels, ClinVar column | `ftp.ncbi.nlm.nih.gov/pub/clinvar/vcf_GRCh38/` | NCBI public domain | Landrum et al., *Nucleic Acids Res* 48(D1):D835–D844, 2020. [10.1093/nar/gkz972](https://doi.org/10.1093/nar/gkz972) | +| **Genome in a Bottle** HG002 v4.2.1 | the demo sample | `ftp-trace.ncbi.nlm.nih.gov/ReferenceSamples/giab/`, `s3://giab` | open, no use restriction | Zook et al., *Nat Biotechnol* 37:561–566, 2019. [10.1038/s41587-019-0074-6](https://doi.org/10.1038/s41587-019-0074-6) | +| **gnomAD** v4 | allele frequency feature and filter | `gs://gcp-public-data--gnomad`, `s3://gnomad-public-us-east-1` | free use, no restriction | Chen et al., *Nature* 625:92–100, 2024. [10.1038/s41586-023-06045-0](https://doi.org/10.1038/s41586-023-06045-0); Karczewski et al., *Nature* 581:434–443, 2020. [10.1038/s41586-020-2308-7](https://doi.org/10.1038/s41586-020-2308-7) | +| **1000 Genomes** 30x | optional cohort/trio data | EBI FTP, `s3://1000genomes` | fully open, no access restriction | Byrska-Bishop et al., *Cell* 185(18):3426–3440.e19, 2022. [10.1016/j.cell.2022.08.004](https://doi.org/10.1016/j.cell.2022.08.004) | +| **MANE Select** | one transcript per gene, if transcript choice ever matters | Ensembl/RefSeq | open | Morales et al., *Nature* 604:310–315, 2022. [10.1038/s41586-022-04558-8](https://doi.org/10.1038/s41586-022-04558-8) | + +## Tools and scores + +| Tool | Role | Terms | Citation | +|---|---|---|---| +| **Ensembl VEP** 113 | annotation (`pipeline/modules/vep.nf`) | Apache 2.0 | McLaren et al., *Genome Biol* 17:122, 2016. [10.1186/s13059-016-0974-4](https://doi.org/10.1186/s13059-016-0974-4) | +| **CADD** | `cadd_phred` feature | free for non-commercial use; commercial licence required | Rentzsch et al., *Nucleic Acids Res* 47(D1):D886–D894, 2019. [10.1093/nar/gky1016](https://doi.org/10.1093/nar/gky1016); Schubach et al., *Nucleic Acids Res* 52(D1), 2024. [10.1093/nar/gkad989](https://doi.org/10.1093/nar/gkad989) | +| **AlphaMissense** | `am_pathogenicity` feature | predictions moved to CC BY 4.0 in March 2024 (originally CC BY-NC-SA) | Cheng et al., *Science* 381:eadg7492, 2023. [10.1126/science.adg7492](https://doi.org/10.1126/science.adg7492) | + +Neither score is required: `rarelens_ml.features` treats a missing CADD or AlphaMissense value as +NaN and LightGBM handles it, so the pipeline runs without the plugin data. + +## Evaluating the model honestly + +The model trains on ClinVar labels and is scored on ClinVar-labelled variants, which is exactly +where published benchmarks go wrong. What to do about it: + +1. **Never let the label into the features.** `CLIN_SIG` is excluded by construction; `clinvar_sig` + is stored for display only (`rarelens_ml/features.py` lists the five feature columns). +2. **Split by gene, not by variant.** Random splits put variants from the same gene on both sides, + and a model can then score a gene rather than a variant. Grimm et al. showed this inflates + reported accuracy for exactly this class of tool: *Hum Mutat* 36:513–523, 2015. + [10.1002/humu.22768](https://doi.org/10.1002/humu.22768) +3. **Prefer a time-based holdout.** Train on an older ClinVar release (monthly archives live under + `vcf_GRCh38/archive_2.0/`) and test only on variants classified after that date. This is the + closest thing to a prospective evaluation available without new patients. +4. **Filter labels by review status.** ClinVar's `CLNREVSTAT` marks how much evidence backs a + classification; two-star and above ("multiple submitters, no conflicts") is the usual bar. + *Known gap*: VEP's `CLIN_SIG` does not carry review status, so this needs ClinVar annotated as a + custom field before it can be enforced. +5. **Report against published baselines on the same rows.** CADD PHRED and AlphaMissense are + already columns in the variant table, so AUROC and AUPRC for the model next to those two, with + the variant count, is a fair comparison rather than a number with nothing to beat. +6. **Report AUPRC, not just AUROC.** Pathogenic variants are the minority class; AUROC flatters. + +## What must not be claimed + +ACMG/AMP treats computational predictions as *supporting* evidence only, never sufficient on their +own for classifying a variant (Richards et al., *Genet Med* 17:405–424, 2015. +[10.1038/gim.2015.30](https://doi.org/10.1038/gim.2015.30)). rarelens is a learning platform on +public data: it makes no diagnostic claim, and the UI shows a score next to the evidence rather +than a verdict. For what a real diagnostic pipeline looks like end to end, see the 100,000 Genomes +Project rare-disease pilot: Smedley et al., *N Engl J Med* 385:1868–1880, 2021. +[10.1056/NEJMoa2035790](https://doi.org/10.1056/NEJMoa2035790) diff --git a/infra/argo-workflows/annotate.yaml b/infra/argo-workflows/annotate.yaml index cf7840d..f3388c6 100644 --- a/infra/argo-workflows/annotate.yaml +++ b/infra/argo-workflows/annotate.yaml @@ -1,37 +1,54 @@ -# Triggered by an Argo Events sensor listening on the Pub/Sub topic "vcf-uploaded". +# Submitted by the Argo Events sensor in events.yaml for each "vcf-uploaded" Pub/Sub message. +# The Nextflow driver runs here; its tasks run on Google Batch (see the gcp profile in +# pipeline/nextflow.config). Image names are rewritten by the gcp overlay and bumped by CI. apiVersion: argoproj.io/v1alpha1 kind: WorkflowTemplate -metadata: { name: annotate-vcf, namespace: rarelens } +metadata: { name: annotate-vcf } spec: + serviceAccountName: rarelens-pipeline entrypoint: nextflow + onExit: exit-handler arguments: parameters: - { name: job_id } - { name: vcf_uri } + - { name: assembly, value: GRCh38 } templates: - name: nextflow - inputs: - parameters: [{ name: job_id }, { name: vcf_uri }] - serviceAccountName: rarelens-pipeline container: - image: europe-west2-docker.pkg.dev/PROJECT/rarelens/pipeline:latest - command: [nextflow] + image: rarelens/pipeline args: - run - /pipeline/main.nf - -profile - gcp - --vcf - - "{{inputs.parameters.vcf_uri}}" + - "{{workflow.parameters.vcf_uri}}" - --job_id - - "{{inputs.parameters.job_id}}" - - --db_url - - "$(DATABASE_URL)" - envFrom: [{ secretRef: { name: api-secrets } }] - resources: { requests: { cpu: "2", memory: 4Gi } } - - name: score - # Optional GPU step for the deep-learning baseline; Autopilot schedules on an L4 node. - nodeSelector: { cloud.google.com/gke-accelerator: nvidia-l4 } + - "{{workflow.parameters.job_id}}" + - --assembly + - "{{workflow.parameters.assembly}}" + # GCP_PROJECT / GCS_BUCKET / GCP_REGION feed params in nextflow.config. + envFrom: [{ configMapRef: { name: pipeline-config } }] + resources: { requests: { cpu: "1", memory: 2Gi } } + + # The loader marks success; anything else (Nextflow error, OOM, eviction) is marked here + # so the UI never polls a dead job forever. + - name: exit-handler + steps: + - - name: mark-failed + template: mark-failed + when: "{{workflow.status}} != Succeeded" + - name: mark-failed container: - image: europe-west2-docker.pkg.dev/PROJECT/rarelens/ml:latest - resources: { limits: { nvidia.com/gpu: 1 } } + image: rarelens/loader + command: [set_job_status.py] + args: + - --job-id + - "{{workflow.parameters.job_id}}" + - --status + - failed + - --log + - "Argo workflow {{workflow.name}} ended {{workflow.status}}" + envFrom: [{ secretRef: { name: pipeline-secrets } }] + resources: { requests: { cpu: 100m, memory: 256Mi } } diff --git a/infra/argo-workflows/events.yaml b/infra/argo-workflows/events.yaml new file mode 100644 index 0000000..252a11c --- /dev/null +++ b/infra/argo-workflows/events.yaml @@ -0,0 +1,46 @@ +# Pub/Sub "vcf-uploaded" (published by the API) -> Argo Workflow from the annotate-vcf template. +apiVersion: argoproj.io/v1alpha1 +kind: EventSource +metadata: { name: vcf-uploaded } +spec: + template: + # Workload Identity: roles/pubsub.subscriber on the subscription (infra/terraform/iam.tf). + serviceAccountName: rarelens-pipeline + pubSub: + vcf-uploaded: + projectID: __GCP_PROJECT__ + subscriptionID: vcf-uploaded-argo # created by Terraform (infra/terraform/pubsub.tf) + jsonBody: true +--- +apiVersion: argoproj.io/v1alpha1 +kind: Sensor +metadata: { name: annotate-vcf } +spec: + template: + serviceAccountName: argo-events-sensor + dependencies: + - { name: vcf, eventSourceName: vcf-uploaded, eventName: vcf-uploaded } + triggers: + - template: + name: submit-annotate-vcf + argoWorkflow: + operation: submit + source: + resource: + apiVersion: argoproj.io/v1alpha1 + kind: Workflow + metadata: { generateName: annotate-vcf- } + spec: + workflowTemplateRef: { name: annotate-vcf } + arguments: + parameters: + - { name: job_id } + - { name: vcf_uri } + - { name: assembly } + parameters: + - src: { dependencyName: vcf, dataKey: body.job_id } + dest: spec.arguments.parameters.0.value + - src: { dependencyName: vcf, dataKey: body.vcf_uri } + dest: spec.arguments.parameters.1.value + - src: { dependencyName: vcf, dataKey: body.assembly, value: GRCh38 } + dest: spec.arguments.parameters.2.value diff --git a/infra/argo-workflows/kustomization.yaml b/infra/argo-workflows/kustomization.yaml new file mode 100644 index 0000000..42ca345 --- /dev/null +++ b/infra/argo-workflows/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +# Requires Argo Workflows and Argo Events (with the default EventBus) installed in the cluster. +namespace: rarelens +resources: [rbac.yaml, annotate.yaml, events.yaml] diff --git a/infra/argo-workflows/rbac.yaml b/infra/argo-workflows/rbac.yaml new file mode 100644 index 0000000..3afe668 --- /dev/null +++ b/infra/argo-workflows/rbac.yaml @@ -0,0 +1,41 @@ +# Runs the annotate-vcf workflow pods and the Pub/Sub EventSource. Bound to the rarelens-pipeline +# Google service account through Workload Identity (annotation added by the gcp overlay). +apiVersion: v1 +kind: ServiceAccount +metadata: { name: rarelens-pipeline } +--- +# Argo's executor reports step outputs through WorkflowTaskResults. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: { name: argo-executor } +rules: + - apiGroups: [argoproj.io] + resources: [workflowtaskresults] + verbs: [create, patch] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: { name: rarelens-pipeline-argo-executor } +roleRef: { apiGroup: rbac.authorization.k8s.io, kind: Role, name: argo-executor } +subjects: [{ kind: ServiceAccount, name: rarelens-pipeline }] +--- +apiVersion: v1 +kind: ServiceAccount +metadata: { name: argo-events-sensor } +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: { name: workflow-submitter } +rules: + - apiGroups: [argoproj.io] + resources: [workflows] + verbs: [create] + - apiGroups: [argoproj.io] + resources: [workflowtemplates] + verbs: [get, list] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: { name: argo-events-sensor-workflow-submitter } +roleRef: { apiGroup: rbac.authorization.k8s.io, kind: Role, name: workflow-submitter } +subjects: [{ kind: ServiceAccount, name: argo-events-sensor }] diff --git a/infra/k8s/base/api.yaml b/infra/k8s/base/api.yaml index b7d4671..8efd745 100644 --- a/infra/k8s/base/api.yaml +++ b/infra/k8s/base/api.yaml @@ -8,13 +8,24 @@ spec: metadata: { labels: { app: api } } spec: serviceAccountName: rarelens-api + initContainers: + # alembic/env.py holds a Postgres advisory lock, so replicas starting together migrate once. + - name: migrate + image: rarelens/api + command: [alembic, upgrade, head] + envFrom: [{ secretRef: { name: api-secrets } }] + resources: { requests: { cpu: 100m, memory: 256Mi }, limits: { cpu: 500m, memory: 512Mi } } containers: - name: api image: rarelens/api ports: [{ containerPort: 8000 }] - envFrom: [{ secretRef: { name: api-secrets } }] + envFrom: + - secretRef: { name: api-secrets } + - configMapRef: { name: api-config } readinessProbe: { httpGet: { path: /health, port: 8000 }, periodSeconds: 5 } + livenessProbe: { httpGet: { path: /health, port: 8000 }, periodSeconds: 10, failureThreshold: 6 } resources: { requests: { cpu: 250m, memory: 512Mi }, limits: { cpu: "1", memory: 1Gi } } + securityContext: { allowPrivilegeEscalation: false } --- apiVersion: v1 kind: Service diff --git a/infra/k8s/base/web.yaml b/infra/k8s/base/web.yaml index b7ac555..8b7814e 100644 --- a/infra/k8s/base/web.yaml +++ b/infra/k8s/base/web.yaml @@ -11,8 +11,12 @@ spec: - name: web image: rarelens/web ports: [{ containerPort: 3000 }] + # Read at runtime ($env/dynamic/public); the ingress routes /api to the API service. env: [{ name: PUBLIC_API_URL, value: /api }] + readinessProbe: { httpGet: { path: /, port: 3000 }, periodSeconds: 5 } + livenessProbe: { httpGet: { path: /, port: 3000 }, periodSeconds: 10, failureThreshold: 6 } resources: { requests: { cpu: 100m, memory: 128Mi }, limits: { cpu: 500m, memory: 256Mi } } + securityContext: { allowPrivilegeEscalation: false } --- apiVersion: v1 kind: Service diff --git a/infra/k8s/overlays/gcp/kustomization.yaml b/infra/k8s/overlays/gcp/kustomization.yaml index 920593d..b3596de 100644 --- a/infra/k8s/overlays/gcp/kustomization.yaml +++ b/infra/k8s/overlays/gcp/kustomization.yaml @@ -1,21 +1,53 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization -resources: [../../base] +# Project-specific values use the __GCP_PROJECT__ placeholder: run `make gcp-configure PROJECT=` +# once and commit. Image tags are then bumped by .github/workflows/deploy.yml after each green CI. +# Generated ConfigMaps must share the Deployments' namespace, or their hashed names are not +# propagated to envFrom references. +namespace: rarelens +resources: [../../base, ../../../argo-workflows] +configurations: [kustomizeconfig.yaml] images: - - { name: rarelens/api, newName: europe-west2-docker.pkg.dev/PROJECT/rarelens/api, newTag: latest } - - { name: rarelens/web, newName: europe-west2-docker.pkg.dev/PROJECT/rarelens/web, newTag: latest } + - { name: rarelens/api, newName: europe-west2-docker.pkg.dev/__GCP_PROJECT__/rarelens/api, newTag: latest } + - { name: rarelens/web, newName: europe-west2-docker.pkg.dev/__GCP_PROJECT__/rarelens/web, newTag: latest } + - { name: rarelens/pipeline, newName: europe-west2-docker.pkg.dev/__GCP_PROJECT__/rarelens/pipeline, newTag: latest } + - { name: rarelens/loader, newName: europe-west2-docker.pkg.dev/__GCP_PROJECT__/rarelens/loader, newTag: latest } +configMapGenerator: + - name: api-config + literals: [PUBSUB_TOPIC=vcf-uploaded, GCP_PROJECT=__GCP_PROJECT__] + - name: pipeline-config + # Referenced by name from the Argo WorkflowTemplate, which kustomize does not rewrite. + options: { disableNameSuffixHash: true } + literals: [GCP_PROJECT=__GCP_PROJECT__, GCP_REGION=europe-west2, GCS_BUCKET=__GCP_PROJECT__-rarelens-data] +# api-secrets and pipeline-secrets come from Secret Manager: `make gcp-secrets PROJECT=`. patches: - target: { kind: ServiceAccount, name: rarelens-api } patch: | - op: add path: /metadata/annotations - value: { iam.gke.io/gcp-service-account: rarelens-api@PROJECT.iam.gserviceaccount.com } + value: { iam.gke.io/gcp-service-account: rarelens-api@__GCP_PROJECT__.iam.gserviceaccount.com } + - target: { kind: ServiceAccount, name: rarelens-pipeline } + patch: | + - op: add + path: /metadata/annotations + value: { iam.gke.io/gcp-service-account: rarelens-pipeline@__GCP_PROJECT__.iam.gserviceaccount.com } - target: { kind: Deployment, name: api } patch: | - op: add - path: /spec/template/spec/containers/- + path: /spec/template/spec/initContainers/0 value: name: cloud-sql-proxy image: gcr.io/cloud-sql-connectors/cloud-sql-proxy:2.13.0 - args: ["--structured-logs", "--port=5432", "PROJECT:europe-west2:rarelens-pg"] - securityContext: { runAsNonRoot: true } + # Native sidecar: starts (and passes its startup probe) before the migrate init + # container, then keeps running alongside the API. + restartPolicy: Always + args: + - --structured-logs + - --private-ip + - --port=5432 + - --health-check + - --http-address=0.0.0.0 + - __GCP_PROJECT__:europe-west2:rarelens-pg + startupProbe: { httpGet: { path: /startup, port: 9090 }, periodSeconds: 1, failureThreshold: 60 } + securityContext: { runAsNonRoot: true, allowPrivilegeEscalation: false } + resources: { requests: { cpu: 100m, memory: 128Mi }, limits: { cpu: 500m, memory: 256Mi } } diff --git a/infra/k8s/overlays/gcp/kustomizeconfig.yaml b/infra/k8s/overlays/gcp/kustomizeconfig.yaml new file mode 100644 index 0000000..97d71f8 --- /dev/null +++ b/infra/k8s/overlays/gcp/kustomizeconfig.yaml @@ -0,0 +1,4 @@ +# Teach the `images:` transformer where Argo WorkflowTemplates keep their images. +images: + - path: spec/templates/container/image + kind: WorkflowTemplate diff --git a/infra/k8s/overlays/local/kustomization.yaml b/infra/k8s/overlays/local/kustomization.yaml index 76e89f6..8302843 100644 --- a/infra/k8s/overlays/local/kustomization.yaml +++ b/infra/k8s/overlays/local/kustomization.yaml @@ -1,5 +1,8 @@ apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization +# `make kind` builds and loads the dev images, then applies this overlay. There is no ingress in +# kind: port-forward svc/web to 8080 and svc/api to 8000 (the UI calls the API directly). +namespace: rarelens # generated Secret/ConfigMap names only propagate within one namespace resources: [../../base, postgres.yaml] images: - { name: rarelens/api, newName: rarelens-api, newTag: dev } @@ -7,3 +10,17 @@ images: secretGenerator: - name: api-secrets literals: [DATABASE_URL=postgresql+asyncpg://rarelens:rarelens@postgres:5432/rarelens] +configMapGenerator: + - name: api-config + literals: ['CORS_ORIGINS=["http://localhost:8080"]'] +patches: + - patch: | + apiVersion: apps/v1 + kind: Deployment + metadata: { name: web } + spec: + template: + spec: + containers: + - name: web + env: [{ name: PUBLIC_API_URL, value: "http://localhost:8000/api" }] diff --git a/infra/k8s/overlays/local/postgres.yaml b/infra/k8s/overlays/local/postgres.yaml index 356fa68..2908bb5 100644 --- a/infra/k8s/overlays/local/postgres.yaml +++ b/infra/k8s/overlays/local/postgres.yaml @@ -13,6 +13,8 @@ spec: - { name: POSTGRES_USER, value: rarelens } - { name: POSTGRES_PASSWORD, value: rarelens } - { name: POSTGRES_DB, value: rarelens } + readinessProbe: { exec: { command: [pg_isready, -U, rarelens] }, periodSeconds: 5 } + resources: { requests: { cpu: 100m, memory: 256Mi }, limits: { cpu: 500m, memory: 512Mi } } --- apiVersion: v1 kind: Service diff --git a/infra/terraform/apis.tf b/infra/terraform/apis.tf new file mode 100644 index 0000000..236e050 --- /dev/null +++ b/infra/terraform/apis.tf @@ -0,0 +1,18 @@ +resource "google_project_service" "enabled" { + for_each = toset([ + "artifactregistry.googleapis.com", + "batch.googleapis.com", + "compute.googleapis.com", + "container.googleapis.com", + "iam.googleapis.com", + "iamcredentials.googleapis.com", + "logging.googleapis.com", + "pubsub.googleapis.com", + "secretmanager.googleapis.com", + "servicenetworking.googleapis.com", + "sqladmin.googleapis.com", + "sts.googleapis.com", + ]) + service = each.value + disable_on_destroy = false +} diff --git a/infra/terraform/cloudrun.tf b/infra/terraform/cloudrun.tf new file mode 100644 index 0000000..85808ae --- /dev/null +++ b/infra/terraform/cloudrun.tf @@ -0,0 +1,178 @@ +# The serverless track: scale-to-zero services and an on-demand pipeline driver. +# Idle cost is storage only; see docs/cloud.md. + +resource "google_cloud_run_v2_service" "api" { + name = "rarelens-api" + location = var.region + deletion_protection = false + ingress = "INGRESS_TRAFFIC_ALL" + + template { + service_account = google_service_account.api.email + scaling { + min_instance_count = 0 # nothing runs, and nothing is billed, between visits + max_instance_count = var.max_instances + } + containers { + image = "${local.registry}/api:${var.image_tag}" + ports { container_port = 8000 } + resources { + limits = { cpu = "1", memory = "1Gi" } + cpu_idle = true # bill CPU only while a request is in flight + startup_cpu_boost = true + } + env { + name = "DATABASE_URL" + value_source { + secret_key_ref { + secret = google_secret_manager_secret.api_database_url.secret_id + version = "latest" + } + } + } + env { + name = "CLOUDRUN_JOB" + value = google_cloud_run_v2_job.nextflow.name + } + env { + name = "GCP_PROJECT" + value = var.project + } + env { + name = "GCP_REGION" + value = var.region + } + env { + name = "GCS_BUCKET" + value = google_storage_bucket.data.name + } + env { + name = "MODEL_URI" + value = var.model_uri + } + } + } + depends_on = [google_secret_manager_secret_version.api_database_url] +} + +resource "google_cloud_run_v2_service" "web" { + name = "rarelens-web" + location = var.region + deletion_protection = false + ingress = "INGRESS_TRAFFIC_ALL" + + template { + scaling { + min_instance_count = 0 + max_instance_count = var.max_instances + } + containers { + image = "${local.registry}/web:${var.image_tag}" + ports { container_port = 3000 } + resources { + limits = { cpu = "1", memory = "512Mi" } + cpu_idle = true + startup_cpu_boost = true + } + # The browser calls /api on this origin; src/routes/api/[...path] forwards it, so there is + # one public URL and no CORS, exactly as the ingress arranges in the Kubernetes track. + env { + name = "PUBLIC_API_URL" + value = "/api" + } + env { + name = "API_INTERNAL_URL" + value = google_cloud_run_v2_service.api.uri + } + # adapter-node sits behind Cloud Run's proxy; derive the origin from the forwarded headers. + env { + name = "PROTOCOL_HEADER" + value = "x-forwarded-proto" + } + env { + name = "HOST_HEADER" + value = "x-forwarded-host" + } + } + } +} + +# The Nextflow driver. Started per annotation by the API (overriding the container args); the +# pipeline's own tasks then run on Google Batch (the gcp profile in pipeline/nextflow.config). +resource "google_cloud_run_v2_job" "nextflow" { + name = "rarelens-nextflow" + location = var.region + deletion_protection = false + + template { + task_count = 1 + template { + service_account = google_service_account.pipeline.email + max_retries = 0 + timeout = "7200s" + containers { + image = "${local.registry}/pipeline:${var.image_tag}" + args = ["-version"] # replaced on every execution by the API's overrides + resources { + limits = { cpu = "1", memory = "2Gi" } + } + env { + name = "GCP_PROJECT" + value = var.project + } + env { + name = "GCP_REGION" + value = var.region + } + env { + name = "GCS_BUCKET" + value = google_storage_bucket.data.name + } + env { + name = "NXF_ANSI_LOG" + value = "false" + } + } + } + } +} + +# Anyone can open the UI and the API. There is no authentication by design (docs/architecture.md); +# max_instances and a billing budget are what bound the cost. +resource "google_cloud_run_v2_service_iam_member" "web_public" { + project = var.project + location = google_cloud_run_v2_service.web.location + name = google_cloud_run_v2_service.web.name + role = "roles/run.invoker" + member = "allUsers" +} + +resource "google_cloud_run_v2_service_iam_member" "api_public" { + project = var.project + location = google_cloud_run_v2_service.api.location + name = google_cloud_run_v2_service.api.name + role = "roles/run.invoker" + member = "allUsers" +} + +# Least privilege: the API may execute this one job with argument overrides, nothing more. +resource "google_cloud_run_v2_job_iam_member" "api_runs_nextflow" { + project = var.project + location = google_cloud_run_v2_job.nextflow.location + name = google_cloud_run_v2_job.nextflow.name + role = "roles/run.jobsExecutorWithOverrides" + member = "serviceAccount:${google_service_account.api.email}" +} + +resource "google_secret_manager_secret_iam_member" "api_database_url" { + secret_id = google_secret_manager_secret.api_database_url.secret_id + role = "roles/secretmanager.secretAccessor" + member = "serviceAccount:${google_service_account.api.email}" +} + +# Reading the model artifact from gs:///models/... when MODEL_URI is set. +resource "google_storage_bucket_iam_member" "api_reads_data" { + bucket = google_storage_bucket.data.name + role = "roles/storage.objectViewer" + member = "serviceAccount:${google_service_account.api.email}" +} diff --git a/infra/terraform/cloudsql.tf b/infra/terraform/cloudsql.tf index e89a057..b9b1436 100644 --- a/infra/terraform/cloudsql.tf +++ b/infra/terraform/cloudsql.tf @@ -1,8 +1,10 @@ resource "google_sql_database_instance" "pg" { - name = "rarelens-pg" - database_version = "POSTGRES_16" - region = var.region - deletion_protection = false + count = var.deploy_cloud_sql ? 1 : 0 + name = "rarelens-pg" + database_version = "POSTGRES_16" + region = var.region + deletion_protection = var.deletion_protection + depends_on = [google_service_networking_connection.private_services] settings { tier = "db-f1-micro" # lab budget; bump for real use @@ -16,14 +18,20 @@ resource "google_sql_database_instance" "pg" { } resource "google_sql_database" "rarelens" { + count = var.deploy_cloud_sql ? 1 : 0 name = "rarelens" - instance = google_sql_database_instance.pg.name + instance = google_sql_database_instance.pg[0].name } resource "google_sql_user" "api" { + count = var.deploy_cloud_sql ? 1 : 0 name = "rarelens" - instance = google_sql_database_instance.pg.name - password = random_password.pg.result + instance = google_sql_database_instance.pg[0].name + password = random_password.pg[0].result } -resource "random_password" "pg" { length = 32 } +resource "random_password" "pg" { + count = var.deploy_cloud_sql ? 1 : 0 + length = 32 + special = false # embedded in DATABASE_URL, where characters like @ / # % would break parsing +} diff --git a/infra/terraform/gke.tf b/infra/terraform/gke.tf index 32951ac..2218f38 100644 --- a/infra/terraform/gke.tf +++ b/infra/terraform/gke.tf @@ -1,8 +1,13 @@ resource "google_container_cluster" "rarelens" { - name = "rarelens" - location = var.region - enable_autopilot = true - deletion_protection = false + count = var.deploy_kubernetes ? 1 : 0 + name = "rarelens" + location = var.region + enable_autopilot = true + deletion_protection = var.deletion_protection + # Same VPC as Cloud SQL's private IP; without this the cluster lands on the "default" network. + network = google_compute_network.vpc.id + subnetwork = google_compute_subnetwork.gke.id + ip_allocation_policy {} workload_identity_config { workload_pool = "${var.project}.svc.id.goog" } release_channel { channel = "REGULAR" } diff --git a/infra/terraform/iam.tf b/infra/terraform/iam.tf index 3fa4343..1a0cdea 100644 --- a/infra/terraform/iam.tf +++ b/infra/terraform/iam.tf @@ -1,6 +1,7 @@ # Workload Identity Federation: GitHub Actions pushes images without long-lived keys. resource "google_iam_workload_identity_pool" "github" { workload_identity_pool_id = "github" + depends_on = [google_project_service.enabled] } resource "google_iam_workload_identity_pool_provider" "github" { @@ -29,22 +30,66 @@ resource "google_artifact_registry_repository_iam_member" "ci_push" { member = "serviceAccount:${google_service_account.ci.email}" } -# Runtime identities (bound to k8s ServiceAccounts via GKE Workload Identity) -resource "google_service_account" "api" { account_id = "rarelens-api" } +# Runtime identities, bound to Kubernetes ServiceAccounts in namespace "rarelens" via GKE +# Workload Identity (the gcp overlay annotates the k8s side). +resource "google_service_account" "api" { account_id = "rarelens-api" } resource "google_service_account" "pipeline" { account_id = "rarelens-pipeline" } +resource "google_service_account_iam_member" "api_workload_identity" { + count = var.deploy_kubernetes ? 1 : 0 + service_account_id = google_service_account.api.name + role = "roles/iam.workloadIdentityUser" + member = "serviceAccount:${var.project}.svc.id.goog[rarelens/rarelens-api]" +} + +resource "google_service_account_iam_member" "pipeline_workload_identity" { + count = var.deploy_kubernetes ? 1 : 0 + service_account_id = google_service_account.pipeline.name + role = "roles/iam.workloadIdentityUser" + member = "serviceAccount:${var.project}.svc.id.goog[rarelens/rarelens-pipeline]" +} + resource "google_project_iam_member" "api_sql" { project = var.project role = "roles/cloudsql.client" member = "serviceAccount:${google_service_account.api.email}" } -resource "google_project_iam_member" "api_pubsub" { - project = var.project - role = "roles/pubsub.publisher" - member = "serviceAccount:${google_service_account.api.email}" + +resource "google_pubsub_topic_iam_member" "api_publish" { + topic = google_pubsub_topic.vcf_uploaded.name + role = "roles/pubsub.publisher" + member = "serviceAccount:${google_service_account.api.email}" } + +resource "google_pubsub_subscription_iam_member" "pipeline_subscribe" { + subscription = google_pubsub_subscription.vcf_uploaded_argo.name + role = "roles/pubsub.subscriber" + member = local.pipeline_sa_member +} + resource "google_storage_bucket_iam_member" "pipeline_data" { bucket = google_storage_bucket.data.name role = "roles/storage.objectAdmin" - member = "serviceAccount:${google_service_account.pipeline.email}" + member = local.pipeline_sa_member +} + +resource "google_artifact_registry_repository_iam_member" "pipeline_pull" { + repository = google_artifact_registry_repository.images.name + location = var.region + role = "roles/artifactregistry.reader" + member = local.pipeline_sa_member +} + +# Nextflow's google-batch executor submits Batch jobs that run as the pipeline SA itself. +resource "google_project_iam_member" "pipeline_batch" { + for_each = toset(["roles/batch.jobsEditor", "roles/batch.agentReporter", "roles/logging.logWriter"]) + project = var.project + role = each.value + member = local.pipeline_sa_member +} + +resource "google_service_account_iam_member" "pipeline_act_as_self" { + service_account_id = google_service_account.pipeline.name + role = "roles/iam.serviceAccountUser" + member = local.pipeline_sa_member } diff --git a/infra/terraform/locals.tf b/infra/terraform/locals.tf new file mode 100644 index 0000000..6459a88 --- /dev/null +++ b/infra/terraform/locals.tf @@ -0,0 +1,16 @@ +locals { + registry = "${var.region}-docker.pkg.dev/${var.project}/rarelens" + + # join("", ...) rather than one(...): with count = 0 these collapse to "" instead of null. + db_credentials = "${join("", google_sql_user.api[*].name)}:${join("", random_password.pg[*].result)}" + db_name = join("", google_sql_database.rarelens[*].name) + db_private_ip = join("", google_sql_database_instance.pg[*].private_ip_address) + + # The API reaches Cloud SQL through its cloud-sql-proxy sidecar on localhost; pipeline tasks + # (Google Batch VMs, Argo pods) use the private IP inside the VPC. With deploy_cloud_sql = false + # both use the URL you supplied, which is expected to be reachable over TLS. + api_database_url = var.deploy_cloud_sql ? "postgresql+asyncpg://${local.db_credentials}@127.0.0.1:5432/${local.db_name}" : var.database_url + pipeline_database_url = var.deploy_cloud_sql ? "postgresql://${local.db_credentials}@${local.db_private_ip}:5432/${local.db_name}" : var.database_url + + pipeline_sa_member = "serviceAccount:${google_service_account.pipeline.email}" +} diff --git a/infra/terraform/network.tf b/infra/terraform/network.tf index 224f31b..e2c4aac 100644 --- a/infra/terraform/network.tf +++ b/infra/terraform/network.tf @@ -1,4 +1,31 @@ resource "google_compute_network" "vpc" { name = "rarelens-vpc" - auto_create_subnetworks = true + auto_create_subnetworks = false + depends_on = [google_project_service.enabled] +} + +# Shared by GKE and the Google Batch VMs that run pipeline tasks (pipeline/nextflow.config). +resource "google_compute_subnetwork" "gke" { + name = "rarelens-gke" + region = var.region + network = google_compute_network.vpc.id + ip_cidr_range = "10.10.0.0/20" + private_ip_google_access = true +} + +# Private services access, so Cloud SQL gets a private IP inside this VPC. Only needed with it. +resource "google_compute_global_address" "private_services" { + count = var.deploy_cloud_sql ? 1 : 0 + name = "rarelens-private-services" + purpose = "VPC_PEERING" + address_type = "INTERNAL" + prefix_length = 16 + network = google_compute_network.vpc.id +} + +resource "google_service_networking_connection" "private_services" { + count = var.deploy_cloud_sql ? 1 : 0 + network = google_compute_network.vpc.id + service = "servicenetworking.googleapis.com" + reserved_peering_ranges = [google_compute_global_address.private_services[0].name] } diff --git a/infra/terraform/outputs.tf b/infra/terraform/outputs.tf index 79d32ac..630d3b8 100644 --- a/infra/terraform/outputs.tf +++ b/infra/terraform/outputs.tf @@ -1,5 +1,14 @@ -output "cluster_name" { value = google_container_cluster.rarelens.name } -output "sql_connection" { value = google_sql_database_instance.pg.connection_name } -output "data_bucket" { value = google_storage_bucket.data.name } -output "wif_provider" { value = google_iam_workload_identity_pool_provider.github.name } -output "ci_sa" { value = google_service_account.ci.email } +output "web_url" { + description = "The one URL to share" + value = google_cloud_run_v2_service.web.uri +} +output "api_url" { value = google_cloud_run_v2_service.api.uri } +output "nextflow_job" { value = google_cloud_run_v2_job.nextflow.name } +output "data_bucket" { value = google_storage_bucket.data.name } +output "cluster_name" { value = one(google_container_cluster.rarelens[*].name) } +output "sql_connection" { value = one(google_sql_database_instance.pg[*].connection_name) } +output "sql_private_ip" { value = one(google_sql_database_instance.pg[*].private_ip_address) } +output "wif_provider" { value = google_iam_workload_identity_pool_provider.github.name } +output "ci_sa" { value = google_service_account.ci.email } +output "api_sa" { value = google_service_account.api.email } +output "pipeline_sa" { value = google_service_account.pipeline.email } diff --git a/infra/terraform/pubsub.tf b/infra/terraform/pubsub.tf new file mode 100644 index 0000000..2c26449 --- /dev/null +++ b/infra/terraform/pubsub.tf @@ -0,0 +1,11 @@ +resource "google_pubsub_topic" "vcf_uploaded" { + name = "vcf-uploaded" + depends_on = [google_project_service.enabled] +} + +# Consumed by the Argo Events EventSource in infra/argo-workflows/events.yaml. +resource "google_pubsub_subscription" "vcf_uploaded_argo" { + name = "vcf-uploaded-argo" + topic = google_pubsub_topic.vcf_uploaded.id + ack_deadline_seconds = 60 +} diff --git a/infra/terraform/secrets.tf b/infra/terraform/secrets.tf new file mode 100644 index 0000000..96b6179 --- /dev/null +++ b/infra/terraform/secrets.tf @@ -0,0 +1,41 @@ +# Copied into Kubernetes secrets by `make gcp-secrets PROJECT=`. +resource "google_secret_manager_secret" "api_database_url" { + secret_id = "rarelens-api-database-url" + replication { + auto {} + } + depends_on = [google_project_service.enabled] +} + +resource "google_secret_manager_secret_version" "api_database_url" { + secret = google_secret_manager_secret.api_database_url.id + secret_data = local.api_database_url + + lifecycle { + precondition { + condition = var.deploy_cloud_sql || var.database_url != "" + error_message = "Set database_url (e.g. a Neon URL), or deploy_cloud_sql = true." + } + } +} + +# The id must match the Nextflow `secret = 'DATABASE_URL'` directive in pipeline/nextflow.config: +# on Google Batch, Nextflow resolves secrets from Secret Manager by name. +resource "google_secret_manager_secret" "pipeline_database_url" { + secret_id = "DATABASE_URL" + replication { + auto {} + } + depends_on = [google_project_service.enabled] +} + +resource "google_secret_manager_secret_version" "pipeline_database_url" { + secret = google_secret_manager_secret.pipeline_database_url.id + secret_data = local.pipeline_database_url +} + +resource "google_secret_manager_secret_iam_member" "pipeline_database_url" { + secret_id = google_secret_manager_secret.pipeline_database_url.id + role = "roles/secretmanager.secretAccessor" + member = local.pipeline_sa_member +} diff --git a/infra/terraform/storage.tf b/infra/terraform/storage.tf index e226157..44cf314 100644 --- a/infra/terraform/storage.tf +++ b/infra/terraform/storage.tf @@ -2,6 +2,8 @@ resource "google_storage_bucket" "data" { name = "${var.project}-rarelens-data" location = var.region uniform_bucket_level_access = true + public_access_prevention = "enforced" + depends_on = [google_project_service.enabled] lifecycle_rule { condition { age = 30 @@ -15,6 +17,5 @@ resource "google_artifact_registry_repository" "images" { repository_id = "rarelens" location = var.region format = "DOCKER" + depends_on = [google_project_service.enabled] } - -resource "google_pubsub_topic" "vcf_uploaded" { name = "vcf-uploaded" } diff --git a/infra/terraform/variables.tf b/infra/terraform/variables.tf index 490a316..d93cfda 100644 --- a/infra/terraform/variables.tf +++ b/infra/terraform/variables.tf @@ -1,9 +1,59 @@ -variable "project" { type = string } +variable "project" { + description = "GCP project id" + type = string +} + variable "region" { - type = string - default = "europe-west2" # London: keeps public genomic test data and the Cambridge team in one jurisdiction + description = "GCP region for every regional resource" + type = string + default = "europe-west2" # London: keeps public genomic test data and the Cambridge team in one jurisdiction } + variable "github_repo" { - type = string - default = "lynchaos/rarelens" + description = "owner/name of the GitHub repo allowed to push images via Workload Identity Federation" + type = string + default = "lynchaos/rarelens" +} + +variable "deletion_protection" { + description = "Protect the GKE cluster and Cloud SQL instance from `terraform destroy`; set false to tear the lab down" + type = bool + default = true +} + +variable "deploy_kubernetes" { + description = "Create the GKE cluster (Argo/ArgoCD track). Off by default: it costs ~$150/month idle, while the serverless track costs ~£1 (docs/cloud.md)" + type = bool + default = false +} + +variable "deploy_cloud_sql" { + description = "Create a Cloud SQL instance (~$10/month). Off by default: set database_url to a free scale-to-zero Postgres such as Neon" + type = bool + default = false +} + +variable "database_url" { + description = "Postgres URL used when deploy_cloud_sql is false, e.g. postgresql+asyncpg://user:pass@host/db?sslmode=require" + type = string + default = "" + sensitive = true +} + +variable "image_tag" { + description = "Image tag deployed to Cloud Run; CI pushes the commit SHA" + type = string + default = "latest" +} + +variable "model_uri" { + description = "Optional model artifact to score with, e.g. gs://-rarelens-data/models/pathogenicity/1. Empty means use the MLflow registry" + type = string + default = "" +} + +variable "max_instances" { + description = "Cloud Run instance ceiling per service: scale-to-zero bounds the floor, this bounds the bill" + type = number + default = 2 } diff --git a/infra/terraform/versions.tf b/infra/terraform/versions.tf index 449370e..3aebc68 100644 --- a/infra/terraform/versions.tf +++ b/infra/terraform/versions.tf @@ -4,7 +4,10 @@ terraform { google = { source = "hashicorp/google", version = "~> 6.0" } random = { source = "hashicorp/random", version = "~> 3.6" } } - backend "gcs" { bucket = "REPLACE-tfstate", prefix = "rarelens" } + # Partial config: terraform init -backend-config="bucket=" + backend "gcs" { + prefix = "rarelens" + } } provider "google" { diff --git a/ml/Dockerfile b/ml/Dockerfile index a2323df..f951933 100644 --- a/ml/Dockerfile +++ b/ml/Dockerfile @@ -2,6 +2,6 @@ FROM python:3.12-slim WORKDIR /ml RUN pip install --no-cache-dir uv COPY pyproject.toml . -RUN uv pip install --system -e . +RUN uv pip install --system -r pyproject.toml COPY rarelens_ml ./rarelens_ml ENTRYPOINT ["python", "-m", "rarelens_ml.train"] diff --git a/ml/pyproject.toml b/ml/pyproject.toml index af80f6f..e2dbb1a 100644 --- a/ml/pyproject.toml +++ b/ml/pyproject.toml @@ -1,8 +1,21 @@ +[build-system] +requires = ["setuptools>=69"] +build-backend = "setuptools.build_meta" + [project] name = "rarelens-ml" version = "0.1.0" requires-python = ">=3.12" -dependencies = ["lightgbm>=4.5", "mlflow>=2.16", "pandas", "scikit-learn", "sqlalchemy", "psycopg[binary]"] +# mlflow major must match the API's mlflow-skinny and the tracking server image. +dependencies = ["lightgbm>=4.5", "mlflow>=3,<4", "pandas", "scikit-learn"] [project.optional-dependencies] gpu = ["torch"] # for the optional deep-learning baseline on GPU +dev = ["pytest>=8"] + +[tool.setuptools.packages.find] +include = ["rarelens_ml*"] + +[tool.pytest.ini_options] +pythonpath = ["."] +testpaths = ["tests"] diff --git a/ml/rarelens_ml/features.py b/ml/rarelens_ml/features.py index 8a9c35e..7a851c9 100644 --- a/ml/rarelens_ml/features.py +++ b/ml/rarelens_ml/features.py @@ -1,14 +1,20 @@ -"""Feature engineering shared by training and serving. Keep this identical to api/app/services/scoring.py.""" +"""Feature engineering: the only copy. + +Training imports it, and train.log_and_register ships this package inside the logged pyfunc +(code_paths), so serving runs exactly this code on the raw columns below. +""" import pandas as pd +# What serving must send: raw values as stored in the variants table / its annotations. +RAW_COLUMNS = ["impact", "consequence", "gnomad_af", "cadd_phred", "am_pathogenicity"] + IMPACT_ORDER = {"MODIFIER": 0, "LOW": 1, "MODERATE": 2, "HIGH": 3} -CATEGORICAL = ["consequence"] -NUMERIC = ["impact_rank", "gnomad_af", "cadd_phred", "am_pathogenicity"] def build(df: pd.DataFrame) -> pd.DataFrame: - out = pd.DataFrame() - out["impact_rank"] = df["impact"].map(IMPACT_ORDER).fillna(0) + out = pd.DataFrame(index=df.index) + out["impact_rank"] = df["impact"].map(IMPACT_ORDER).fillna(0).astype(int) + # No gnomAD record means the variant was not observed: treat as AF 0. out["gnomad_af"] = pd.to_numeric(df["gnomad_af"], errors="coerce").fillna(0.0) out["cadd_phred"] = pd.to_numeric(df["cadd_phred"], errors="coerce") out["am_pathogenicity"] = pd.to_numeric(df["am_pathogenicity"], errors="coerce") diff --git a/ml/rarelens_ml/model.py b/ml/rarelens_ml/model.py new file mode 100644 index 0000000..02c64f3 --- /dev/null +++ b/ml/rarelens_ml/model.py @@ -0,0 +1,17 @@ +import mlflow + +from rarelens_ml.features import RAW_COLUMNS, build + + +class PathogenicityModel(mlflow.pyfunc.PythonModel): + """Serving contract: raw VEP columns in, P(pathogenic) out. + + The stock LightGBM pyfunc flavour calls `predict`, which returns class labels; wrapping the + classifier keeps feature engineering and `predict_proba` inside the registered artifact. + """ + + def __init__(self, classifier): + self.classifier = classifier + + def predict(self, context, model_input, params=None): + return self.classifier.predict_proba(build(model_input[RAW_COLUMNS]))[:, 1] diff --git a/ml/rarelens_ml/train.py b/ml/rarelens_ml/train.py index ed0d392..d1a8443 100644 --- a/ml/rarelens_ml/train.py +++ b/ml/rarelens_ml/train.py @@ -1,54 +1,130 @@ -"""Train a pathogenicity classifier on ClinVar labels (Pathogenic/Likely pathogenic vs Benign/Likely benign). +"""Train a pathogenicity classifier on ClinVar labels ((likely) pathogenic vs (likely) benign). Label leakage warning: CLIN_SIG must never be a feature. This is a learning exercise, not a clinical model. -Usage: python -m rarelens_ml.train --tsv results/clinvar.vep.tsv +Usage: python -m rarelens_ml.train --tsv results/clinvar.vep.tsv --register """ import argparse +import re +from pathlib import Path import lightgbm as lgb import mlflow -import mlflow.lightgbm import pandas as pd +import sklearn +from mlflow import MlflowClient from sklearn.metrics import average_precision_score, roc_auc_score from sklearn.model_selection import train_test_split -from rarelens_ml.features import build +from rarelens_ml.features import RAW_COLUMNS, build +from rarelens_ml.model import PathogenicityModel -POS = {"Pathogenic", "Likely_pathogenic", "Pathogenic/Likely_pathogenic"} -NEG = {"Benign", "Likely_benign", "Benign/Likely_benign"} +PACKAGE_DIR = Path(__file__).resolve().parent +MODEL_NAME = "rarelens-pathogenicity" +PARAMS = { + "n_estimators": 400, "learning_rate": 0.05, "num_leaves": 31, "class_weight": "balanced", + "verbose": -1, +} + +POS = {"pathogenic", "likely_pathogenic"} +NEG = {"benign", "likely_benign"} +# VEP --tab column -> raw feature column (am_pathogenicity already matches). +VEP_TO_RAW = { + "IMPACT": "impact", "Consequence": "consequence", "gnomADe_AF": "gnomad_af", + "CADD_PHRED": "cadd_phred", +} + + +def label(clin_sig: object) -> int | None: + """1 / 0 when every ClinVar term agrees, None for VUS, conflicts and missing values. + + Accepts VEP's lowercase comma-separated form ("pathogenic,likely_pathogenic") and ClinVar's + CLNSIG form ("Pathogenic/Likely_pathogenic"). + """ + if not isinstance(clin_sig, str): + return None + terms = {t for t in re.split(r"[,&/|]", clin_sig.strip().lower()) if t and t != "-"} + if terms and terms <= POS: + return 1 + if terms and terms <= NEG: + return 0 + return None + + +def read_vep_tab(path: str | Path) -> pd.DataFrame: + """Read VEP --tab output as strings, keeping "-" (VEP's missing marker) verbatim. + + Skips the "##" preamble by position instead of comment="#", which would also cut any value + containing "#". + """ + with open(path) as fh: + for n, line in enumerate(fh): + if line.startswith("#Uploaded_variation"): + break + else: + raise ValueError(f"{path}: no #Uploaded_variation header; is this VEP --tab output?") + df = pd.read_csv(path, sep="\t", skiprows=n, dtype=str, keep_default_na=False) + return df.rename(columns={"#Uploaded_variation": "Uploaded_variation"}) def load(tsv: str) -> tuple[pd.DataFrame, pd.Series]: - df = pd.read_csv(tsv, sep="\t", comment="#", header=None, dtype=str) - with open(tsv) as fh: - df.columns = next(l for l in fh if l.startswith("#Uploaded")).lstrip("#").rstrip().split("\t") - df = df.rename(columns={"IMPACT": "impact", "Consequence": "consequence", "gnomADe_AF": "gnomad_af", - "CADD_PHRED": "cadd_phred"}) - y = df["CLIN_SIG"].map(lambda s: 1 if s in POS else 0 if s in NEG else None) + df = read_vep_tab(tsv).rename(columns=VEP_TO_RAW) + for col in RAW_COLUMNS: # plugin columns are absent when VEP ran without CADD/AlphaMissense + if col not in df: + df[col] = pd.NA + y = df["CLIN_SIG"].map(label) keep = y.notna() - return build(df[keep]), y[keep].astype(int) + return ( + df.loc[keep, RAW_COLUMNS].reset_index(drop=True), + y[keep].astype(int).reset_index(drop=True), + ) + + +def fit(X: pd.DataFrame, y: pd.Series) -> lgb.LGBMClassifier: + return lgb.LGBMClassifier(**PARAMS).fit(build(X), y) + + +def log_and_register(clf: lgb.LGBMClassifier, model_name: str, alias: str) -> str: + """Log the pyfunc, register it and point `alias` at the new version. Returns the version.""" + info = mlflow.pyfunc.log_model( + name="model", + python_model=PathogenicityModel(clf), + code_paths=[str(PACKAGE_DIR)], + registered_model_name=model_name, + pip_requirements=[ + f"lightgbm=={lgb.__version__}", + f"pandas=={pd.__version__}", + f"scikit-learn=={sklearn.__version__}", + ], + ) + version = str(info.registered_model_version) + MlflowClient().set_registered_model_alias(model_name, alias, version) + return version def main() -> None: p = argparse.ArgumentParser() p.add_argument("--tsv", required=True) - p.add_argument("--register", action="store_true") + p.add_argument("--register", action="store_true", + help="register the model and move the alias to the new version") + p.add_argument("--alias", default="production") a = p.parse_args() X, y = load(a.tsv) Xtr, Xte, ytr, yte = train_test_split(X, y, test_size=0.2, stratify=y, random_state=42) - mlflow.set_experiment("rarelens-pathogenicity") + mlflow.set_experiment(MODEL_NAME) with mlflow.start_run(): - params = {"n_estimators": 400, "learning_rate": 0.05, "num_leaves": 31, "class_weight": "balanced"} - mlflow.log_params(params) - model = lgb.LGBMClassifier(**params).fit(Xtr, ytr) - proba = model.predict_proba(Xte)[:, 1] - mlflow.log_metrics({"auroc": roc_auc_score(yte, proba), "auprc": average_precision_score(yte, proba)}) - mlflow.lightgbm.log_model( - model, "model", - registered_model_name="rarelens-pathogenicity" if a.register else None, - ) + mlflow.log_params(PARAMS) + clf = fit(Xtr, ytr) + proba = clf.predict_proba(build(Xte))[:, 1] + mlflow.log_metrics({"auroc": roc_auc_score(yte, proba), + "auprc": average_precision_score(yte, proba)}) + if a.register: + version = log_and_register(clf, MODEL_NAME, a.alias) + print(f"registered {MODEL_NAME} v{version} as @{a.alias}") + else: + mlflow.pyfunc.log_model(name="model", python_model=PathogenicityModel(clf), + code_paths=[str(PACKAGE_DIR)]) if __name__ == "__main__": diff --git a/ml/tests/test_features.py b/ml/tests/test_features.py new file mode 100644 index 0000000..1836174 --- /dev/null +++ b/ml/tests/test_features.py @@ -0,0 +1,38 @@ +import math + +import pandas as pd + +from rarelens_ml.features import RAW_COLUMNS, build + + +def raw(**overrides: list) -> pd.DataFrame: + base = { + "impact": ["HIGH", "LOW", None], + "consequence": ["stop_gained", "synonymous_variant", None], + "gnomad_af": [None, "0.12", 0.001], + "cadd_phred": ["35", "2.1", "-"], + "am_pathogenicity": ["0.98", None, "-"], + } + base.update(overrides) + return pd.DataFrame(base, index=[10, 11, 12]) + + +def test_raw_columns_are_the_serving_contract() -> None: + assert RAW_COLUMNS == ["impact", "consequence", "gnomad_af", "cadd_phred", "am_pathogenicity"] + + +def test_build_ranks_impact_and_coerces_numbers() -> None: + out = build(raw()) + assert out["impact_rank"].tolist() == [3, 1, 0] + assert out["gnomad_af"].tolist() == [0.0, 0.12, 0.001] # missing AF means absent from gnomAD + assert out["cadd_phred"].iloc[0] == 35.0 + assert math.isnan(out["cadd_phred"].iloc[2]) # VEP writes "-" for missing + assert math.isnan(out["am_pathogenicity"].iloc[1]) + + +def test_build_keeps_the_input_index() -> None: + assert build(raw()).index.tolist() == [10, 11, 12] + + +def test_build_makes_consequence_categorical() -> None: + assert isinstance(build(raw())["consequence"].dtype, pd.CategoricalDtype) diff --git a/ml/tests/test_train.py b/ml/tests/test_train.py new file mode 100644 index 0000000..6945f28 --- /dev/null +++ b/ml/tests/test_train.py @@ -0,0 +1,104 @@ +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest + +from rarelens_ml.train import label, read_vep_tab + +HEADER = [ + "Uploaded_variation", "Location", "Allele", "Consequence", "IMPACT", "SYMBOL", + "gnomADe_AF", "CLIN_SIG", "CADD_PHRED", "am_pathogenicity", +] + + +def write_vep_tab(path: Path, rows: list[list[str]]) -> Path: + lines = [ + "## ENSEMBL VARIANT EFFECT PREDICTOR v113.0", + "## Column descriptions:", + "#" + "\t".join(HEADER), + *("\t".join(r) for r in rows), + ] + path.write_text("\n".join(lines) + "\n") + return path + + +@pytest.mark.parametrize( + ("clin_sig", "expected"), + [ + # VEP writes lowercase, comma-separated terms from co-located ClinVar records. + ("pathogenic", 1), + ("pathogenic,likely_pathogenic", 1), + ("likely_benign", 0), + ("benign,likely_benign", 0), + # ClinVar VCF CLNSIG spelling must keep working too. + ("Pathogenic/Likely_pathogenic", 1), + ("Benign", 0), + ("uncertain_significance", None), + ("pathogenic,benign", None), # conflicting evidence is not a label + ("-", None), + ("", None), + (np.nan, None), + ], +) +def test_label(clin_sig: object, expected: int | None) -> None: + assert label(clin_sig) == expected + + +def test_read_vep_tab_uses_the_hash_header_and_keeps_dashes(tmp_path: Path) -> None: + tsv = write_vep_tab( + tmp_path / "x.vep.tsv", + [["22_1_A_G", "22:1", "G", "missense_variant", "MODERATE", "TBX1", "-", "pathogenic", "28", "0.9"]], + ) + df = read_vep_tab(tsv) + assert list(df.columns) == HEADER + assert df.loc[0, "gnomADe_AF"] == "-" + assert df.loc[0, "CLIN_SIG"] == "pathogenic" + + +def test_load_returns_raw_serving_columns_and_labels(tmp_path: Path) -> None: + from rarelens_ml.features import RAW_COLUMNS + from rarelens_ml.train import load + + tsv = write_vep_tab( + tmp_path / "x.vep.tsv", + [ + ["a", "22:1", "G", "missense_variant", "MODERATE", "TBX1", "0.0001", "pathogenic", "28", "0.9"], + ["b", "22:2", "A", "synonymous_variant", "LOW", "CHEK2", "0.12", "benign", "3", "-"], + ["c", "22:3", "T", "intron_variant", "MODIFIER", "CHEK2", "0.3", "uncertain_significance", "1", "-"], + ], + ) + X, y = load(str(tsv)) + assert list(X.columns) == RAW_COLUMNS + assert y.tolist() == [1, 0] # the VUS row is dropped + + +def test_logged_model_returns_probabilities_from_raw_columns(tmp_path: Path) -> None: + """The registered model must take the raw columns serving sends and return P(pathogenic).""" + import mlflow + + from rarelens_ml.train import fit, log_and_register + + rng = np.random.default_rng(0) + n = 400 + impact = rng.choice(["HIGH", "MODERATE", "LOW", "MODIFIER"], n) + y = pd.Series(((impact == "HIGH") | (rng.random(n) < 0.1)).astype(int)) + X = pd.DataFrame({ + "impact": impact, + "consequence": rng.choice(["stop_gained", "missense_variant", "intron_variant"], n), + "gnomad_af": rng.random(n).round(4).astype(str), # strings, as read from the DB + "cadd_phred": (rng.random(n) * 40).round(1).astype(str), + "am_pathogenicity": "-", + }) + + mlflow.set_tracking_uri(f"sqlite:///{tmp_path}/mlflow.db") + mlflow.set_experiment("test") + clf = fit(X, y) + version = log_and_register(clf, model_name="rarelens-test", alias="production") + + model = mlflow.pyfunc.load_model("models:/rarelens-test@production") + scores = np.asarray(model.predict(X.head(50))) + assert version == "1" + assert scores.shape == (50,) + assert ((scores >= 0) & (scores <= 1)).all() + assert not set(np.unique(scores)) <= {0.0, 1.0}, "got class labels, expected probabilities" diff --git a/pipeline/.dockerignore b/pipeline/.dockerignore new file mode 100644 index 0000000..790b748 --- /dev/null +++ b/pipeline/.dockerignore @@ -0,0 +1,5 @@ +work/ +results/ +cache/ +tests/ +.nextflow* diff --git a/pipeline/Dockerfile b/pipeline/Dockerfile index 20f1b61..42b4507 100644 --- a/pipeline/Dockerfile +++ b/pipeline/Dockerfile @@ -1,5 +1,19 @@ -# Loader image: pandas + psycopg for LOAD_DB -FROM python:3.12-slim -RUN pip install --no-cache-dir pandas sqlalchemy "psycopg[binary]" -COPY bin/load_db.py /usr/local/bin/load_db.py -RUN chmod +x /usr/local/bin/load_db.py +# Nextflow driver for the Argo annotate-vcf workflow. Tasks run in their own containers. +FROM eclipse-temurin:21-jre +ARG NXF_VER=26.04.6 +ENV NXF_VER=${NXF_VER} NXF_HOME=/opt/nextflow +RUN apt-get update \ + && apt-get install -y --no-install-recommends curl ca-certificates \ + && rm -rf /var/lib/apt/lists/* \ + && curl -fsSL https://get.nextflow.io | bash \ + && mv nextflow /usr/local/bin/nextflow \ + && nextflow plugin install nf-google +COPY main.nf nextflow.config /pipeline/ +COPY modules /pipeline/modules +COPY bin /pipeline/bin +COPY assets /pipeline/assets +# CI passes the loader image built from the same commit, so driver and loader never drift. +ARG LOADER_IMAGE=rarelens/loader:dev +ENV RARELENS_LOADER_IMAGE=${LOADER_IMAGE} +WORKDIR /work +ENTRYPOINT ["nextflow"] diff --git a/pipeline/assets/NO_FILE b/pipeline/assets/NO_FILE new file mode 100644 index 0000000..e69de29 diff --git a/pipeline/bin/load_db.py b/pipeline/bin/load_db.py index 7aa62e4..70f0410 100755 --- a/pipeline/bin/load_db.py +++ b/pipeline/bin/load_db.py @@ -1,59 +1,133 @@ -#!/usr/bin/env python -"""Load a VEP --tab output into the rarelens Postgres schema and mark the job succeeded.""" +#!/usr/bin/env python3 +"""Load VEP --tab output into the rarelens Postgres schema and mark the job succeeded. + +Variant identity (chrom/pos/ref/alt) comes from the VCF ID, which NORMALISE sets to +CHROM_POS_REF_ALT: VEP's own Location/Allele columns trim indel alleles and shift positions. +The database URL is read from $DATABASE_URL so it never appears on a command line or in .command.sh. +Loading replaces any rows already stored for the job, so a retried task cannot duplicate variants. +""" import argparse import json +import os +import re import sys +from pathlib import Path import pandas as pd -from sqlalchemy import create_engine, text +from sqlalchemy import Engine, create_engine, text +from sqlalchemy.engine import make_url + +INSERT_CHUNK = 5000 +VEP_VERSION = re.compile(r"^## ENSEMBL VARIANT EFFECT PREDICTOR v(\S+)") + + +def engine_for(url: str) -> Engine: + """Accept the API's asyncpg URL (or a plain postgresql:// one) and use psycopg.""" + parsed = make_url(url).set(drivername="postgresql+psycopg") + query = dict(parsed.query) + if "ssl" in query: # asyncpg's spelling; libpq (psycopg) wants sslmode + query["sslmode"] = query.pop("ssl") + return create_engine(parsed.set(query=query)) + + +def read_vep_tab(path: str | Path) -> pd.DataFrame: + """Read VEP --tab output as strings, keeping "-" (VEP's missing marker) verbatim.""" + with open(path) as fh: + for n, line in enumerate(fh): + if line.startswith("#Uploaded_variation"): + break + else: + raise ValueError(f"{path}: no #Uploaded_variation header; is this VEP --tab output?") + df = pd.read_csv(path, sep="\t", skiprows=n, dtype=str, keep_default_na=False) + return df.rename(columns={"#Uploaded_variation": "Uploaded_variation"}) + + +def vep_version(path: str | Path) -> str | None: + with open(path) as fh: + for line in fh: + if not line.startswith("##"): + return None + if m := VEP_VERSION.match(line): + return m.group(1) + return None + + +def parse_variant_id(uid: str) -> tuple[str, int, str, str]: + # rsplit: contig names may contain "_" (chrUn_KI270742v1); positions and alleles never do. + parts = uid.rsplit("_", 3) + if len(parts) != 4 or not parts[1].isdigit(): + raise ValueError( + f"unexpected Uploaded_variation {uid!r}; NORMALISE must set VCF IDs to CHROM_POS_REF_ALT" + ) + chrom, pos, ref, alt = parts + return chrom, int(pos), ref, alt + + +def _value(v: str | None) -> str | None: + return None if v in (None, "", "-") else v + + +def to_rows(df: pd.DataFrame, job_id: str) -> list[dict]: + rows = [] + for rec in df.to_dict("records"): + chrom, pos, ref, alt = parse_variant_id(rec["Uploaded_variation"]) + af = _value(rec.get("gnomADe_AF")) + rows.append({ + "job_id": job_id, + "chrom": chrom, + "pos": pos, + "ref": ref, + "alt": alt, + "gene": _value(rec.get("SYMBOL")), + "consequence": _value(rec.get("Consequence")), + "impact": _value(rec.get("IMPACT")), + "hgvsc": _value(rec.get("HGVSc")), + "hgvsp": _value(rec.get("HGVSp")), + "gnomad_af": float(af) if af is not None else None, + "clinvar_sig": _value(rec.get("CLIN_SIG")), + "annotations": json.dumps({k: v for k, v in rec.items() if _value(v) is not None}), + }) + return rows + + +def load(engine: Engine, job_id: str, rows: list[dict], vep: str | None) -> None: + with engine.begin() as conn: + conn.execute(text("DELETE FROM variants WHERE job_id = :id"), {"id": job_id}) + for start in range(0, len(rows), INSERT_CHUNK): + conn.execute( + text(""" + INSERT INTO variants (job_id, chrom, pos, ref, alt, gene, consequence, impact, + hgvsc, hgvsp, gnomad_af, clinvar_sig, annotations) + VALUES (:job_id, :chrom, :pos, :ref, :alt, :gene, :consequence, :impact, + :hgvsc, :hgvsp, :gnomad_af, :clinvar_sig, CAST(:annotations AS jsonb)) + """), + rows[start:start + INSERT_CHUNK], + ) + conn.execute( + text(""" + UPDATE jobs SET status = 'succeeded', vep_version = :vep, log = NULL, + finished_at = now() + WHERE id = :id + """), + {"id": job_id, "vep": vep}, + ) def main() -> None: p = argparse.ArgumentParser() p.add_argument("--tsv", required=True) p.add_argument("--job-id", required=True) - p.add_argument("--db-url", required=True) + p.add_argument("--dry-run", action="store_true", help="parse only; do not touch the database") a = p.parse_args() - df = pd.read_csv(a.tsv, sep="\t", comment="#", header=None, dtype=str) - with open(a.tsv) as fh: - header = next(l for l in fh if l.startswith("#Uploaded_variation")).lstrip("#").rstrip().split("\t") - df.columns = header - - chrom_pos = df["Location"].str.split(":", expand=True) - rows = [] - for i, r in df.iterrows(): - ref, _, alt = r["Uploaded_variation"].partition("/") if "/" in r["Uploaded_variation"] else ("", "", r["Allele"]) - rows.append({ - "job_id": a.job_id, - "chrom": chrom_pos.iloc[i, 0], - "pos": int(chrom_pos.iloc[i, 1].split("-")[0]), - "ref": ref or "-", - "alt": r["Allele"], - "gene": r.get("SYMBOL") if r.get("SYMBOL") != "-" else None, - "consequence": r["Consequence"], - "impact": r["IMPACT"], - "hgvsc": None if r.get("HGVSc") == "-" else r.get("HGVSc"), - "hgvsp": None if r.get("HGVSp") == "-" else r.get("HGVSp"), - "gnomad_af": None if r.get("gnomADe_AF", "-") == "-" else float(r["gnomADe_AF"]), - "clinvar_sig": None if r.get("CLIN_SIG", "-") == "-" else r["CLIN_SIG"], - "annotations": json.dumps({k: v for k, v in r.items() if v != "-"}), - }) - - if a.db_url == "none": + rows = to_rows(read_vep_tab(a.tsv), job_id=a.job_id) + if a.dry_run: print(f"{len(rows)} variants parsed (dry run, no DB)") return - - engine = create_engine(a.db_url.replace("+asyncpg", "+psycopg")) - with engine.begin() as conn: - conn.execute(text(""" - INSERT INTO variants (job_id, chrom, pos, ref, alt, gene, consequence, impact, - hgvsc, hgvsp, gnomad_af, clinvar_sig, annotations) - VALUES (:job_id, :chrom, :pos, :ref, :alt, :gene, :consequence, :impact, - :hgvsc, :hgvsp, :gnomad_af, :clinvar_sig, CAST(:annotations AS jsonb)) - """), rows) - conn.execute(text("UPDATE jobs SET status='succeeded', finished_at=now() WHERE id=:id"), - {"id": a.job_id}) + url = os.environ.get("DATABASE_URL") + if not url: + sys.exit("DATABASE_URL is not set") + load(engine_for(url), a.job_id, rows, vep=vep_version(a.tsv)) print(f"loaded {len(rows)} variants for job {a.job_id}", file=sys.stderr) diff --git a/pipeline/bin/set_job_status.py b/pipeline/bin/set_job_status.py new file mode 100755 index 0000000..680b3ef --- /dev/null +++ b/pipeline/bin/set_job_status.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +"""Set a job's status, e.g. from the Argo exit handler when a workflow fails. + +A succeeded job is never overwritten: the loader's success is the source of truth. +""" +import argparse +import os +import sys + +from load_db import engine_for +from sqlalchemy import Engine, text + + +def set_status(engine: Engine, job_id: str, status: str, log: str | None) -> None: + with engine.begin() as conn: + conn.execute( + text(""" + UPDATE jobs + SET status = CAST(:status AS jobstatus), log = :log, + finished_at = CASE WHEN :status IN ('succeeded', 'failed') THEN now() END + WHERE id = :id AND status <> 'succeeded' + """), + {"id": job_id, "status": status, "log": log}, + ) + + +def main() -> None: + p = argparse.ArgumentParser() + p.add_argument("--job-id", required=True) + p.add_argument("--status", required=True, choices=["running", "failed"]) + p.add_argument("--log") + a = p.parse_args() + url = os.environ.get("DATABASE_URL") + if not url: + sys.exit("DATABASE_URL is not set") + set_status(engine_for(url), a.job_id, a.status, a.log) + + +if __name__ == "__main__": + main() diff --git a/pipeline/loader.Dockerfile b/pipeline/loader.Dockerfile new file mode 100644 index 0000000..812ff84 --- /dev/null +++ b/pipeline/loader.Dockerfile @@ -0,0 +1,8 @@ +# Task image for LOAD_DB and the Argo exit handler. +FROM python:3.12-slim +# procps: Nextflow uses `ps` inside task containers to collect metrics. +RUN apt-get update && apt-get install -y --no-install-recommends procps && rm -rf /var/lib/apt/lists/* +COPY requirements.txt /tmp/requirements.txt +RUN pip install --no-cache-dir -r /tmp/requirements.txt +COPY bin/load_db.py bin/set_job_status.py /usr/local/bin/ +RUN chmod +x /usr/local/bin/load_db.py /usr/local/bin/set_job_status.py diff --git a/pipeline/main.nf b/pipeline/main.nf index 1e6c874..9031261 100644 --- a/pipeline/main.nf +++ b/pipeline/main.nf @@ -7,9 +7,17 @@ include { LOAD_DB } from './modules/load_db' workflow { if (!params.vcf) error "Provide --vcf" - vcf_ch = Channel.fromPath(params.vcf, checkIfExists: true) + if (workflow.profile.tokenize(',').contains('gcp') && !(params.project && params.bucket)) { + error "The gcp profile needs --project and --bucket (or GCP_PROJECT and GCS_BUCKET)" + } + // Stub runs (CI) have no VEP cache or plugin data on disk. + def must_exist = !workflow.stubRun + + vcf_ch = Channel.fromPath(params.vcf, checkIfExists: true) + cache = file(params.vep_cache, checkIfExists: must_exist) + plugins = file(params.vep_plugin_data ?: "${projectDir}/assets/NO_FILE", checkIfExists: must_exist) NORMALISE(vcf_ch) - VEP(NORMALISE.out.vcf) - LOAD_DB(VEP.out.tsv, params.job_id ?: 'local', params.db_url ?: 'none') + VEP(NORMALISE.out.vcf, cache, plugins) + LOAD_DB(VEP.out.tsv, params.job_id ?: 'dry-run') } diff --git a/pipeline/modules/load_db.nf b/pipeline/modules/load_db.nf index fe3c778..d15909b 100644 --- a/pipeline/modules/load_db.nf +++ b/pipeline/modules/load_db.nf @@ -3,11 +3,18 @@ process LOAD_DB { input: path tsv val job_id - val db_url output: stdout + // DATABASE_URL reaches the task through its environment (docker profile) or a Nextflow + // secret (gcp profile); it never appears on a command line or in .command.sh. script: + def dry_run = job_id == 'dry-run' ? '--dry-run' : '' """ - load_db.py --tsv $tsv --job-id $job_id --db-url '$db_url' + load_db.py --tsv $tsv --job-id $job_id $dry_run + """ + + stub: + """ + echo "stub: would load $tsv for job $job_id" """ } diff --git a/pipeline/modules/normalise.nf b/pipeline/modules/normalise.nf index a566c0c..2a67be6 100644 --- a/pipeline/modules/normalise.nf +++ b/pipeline/modules/normalise.nf @@ -3,9 +3,16 @@ process NORMALISE { input: path vcf output: path "${vcf.simpleName}.norm.vcf.gz", emit: vcf + // Split multiallelics, then set each ID to CHROM_POS_REF_ALT. VEP echoes the ID back as + // Uploaded_variation, and the loader takes exact VCF alleles from it (VEP trims indels). script: """ - bcftools norm -m -both -Oz -o ${vcf.simpleName}.norm.vcf.gz $vcf - bcftools index -t ${vcf.simpleName}.norm.vcf.gz + bcftools norm -m -both -Ou $vcf \\ + | bcftools annotate --set-id '%CHROM\\_%POS\\_%REF\\_%FIRST_ALT' -Oz -o ${vcf.simpleName}.norm.vcf.gz + """ + + stub: + """ + touch ${vcf.simpleName}.norm.vcf.gz """ } diff --git a/pipeline/modules/vep.nf b/pipeline/modules/vep.nf index e12be3c..4d48b21 100644 --- a/pipeline/modules/vep.nf +++ b/pipeline/modules/vep.nf @@ -1,16 +1,31 @@ process VEP { tag "$vcf.simpleName" publishDir params.outdir, mode: 'copy' - input: path vcf + input: + path vcf + path cache // staged as an input so it is visible inside the container + path plugin_data // assets/NO_FILE when plugins are not configured output: path "${vcf.simpleName}.vep.tsv", emit: tsv path "${vcf.simpleName}.vep_summary.html" script: + // Plugins run only when --vep_plugin_data names a directory holding the plugin modules + // (INSTALL.pl -a p -g CADD,AlphaMissense -r ) and the data files named in params. + def plugins = plugin_data.name == 'NO_FILE' ? '' : [ + "--dir_plugins ${plugin_data}", + "--plugin CADD,snv=${plugin_data}/${params.cadd_snv},indels=${plugin_data}/${params.cadd_indels}", + "--plugin AlphaMissense,file=${plugin_data}/${params.alphamissense}", + ].join(' ') """ vep -i $vcf -o ${vcf.simpleName}.vep.tsv --tab \\ - --assembly ${params.assembly} --cache --dir_cache ${params.vep_cache} --offline \\ - --everything --pick --af_gnomade --plugin CADD --plugin AlphaMissense \\ + --assembly ${params.assembly} --cache --offline --dir_cache ${cache} \\ + --everything --pick ${plugins} \\ --stats_file ${vcf.simpleName}.vep_summary.html --fork ${task.cpus} """ + + stub: + """ + touch ${vcf.simpleName}.vep.tsv ${vcf.simpleName}.vep_summary.html + """ } diff --git a/pipeline/nextflow.config b/pipeline/nextflow.config index df697ab..b5a4511 100644 --- a/pipeline/nextflow.config +++ b/pipeline/nextflow.config @@ -1,24 +1,52 @@ params { - vcf = null - job_id = null - db_url = null - outdir = "results" - assembly = "GRCh38" - vep_cache = "${projectDir}/cache/vep" // download once with `vep_install`; or use --offline false - vep_plugins = "CADD,AlphaMissense" -} + vcf = null + job_id = null // omit for a dry run that parses but does not load + outdir = "results" + assembly = "GRCh38" + vep_cache = "${projectDir}/cache/vep" // INSTALL.pl -a cf -s homo_sapiens -y GRCh38 -c + vep_plugin_data = null // CADD + AlphaMissense modules and data; plugins skipped when null + cadd_snv = "whole_genome_SNVs.tsv.gz" + cadd_indels = "gnomad.genomes.r4.0.indel.tsv.gz" + alphamissense = "AlphaMissense_hg38.tsv.gz" + // The driver image sets this to the loader image built from the same commit. + loader_image = System.getenv('RARELENS_LOADER_IMAGE') ?: 'rarelens/loader:dev' -profiles { - docker { docker.enabled = true } - gcp { - process.executor = 'k8s' // runs inside GKE via Argo; Nextflow k8s executor - workDir = "gs://${params.bucket}/work" - google.project = params.project - } + // gcp profile; the Argo workflow provides these through the pipeline-config ConfigMap. + project = System.getenv('GCP_PROJECT') + region = System.getenv('GCP_REGION') ?: 'europe-west2' + bucket = System.getenv('GCS_BUCKET') } process { + shell = ['/bin/bash', '-euo', 'pipefail'] withName: VEP { container = 'ensemblorg/ensembl-vep:release_113.0'; cpus = 4; memory = '8 GB' } withName: NORMALISE { container = 'quay.io/biocontainers/bcftools:1.20--h8b25389_0' } - withName: LOAD_DB { container = 'ghcr.io/lynchaos/rarelens-loader:latest' } + withName: LOAD_DB { container = params.loader_image } +} + +profiles { + docker { + docker.enabled = true + docker.envWhitelist = ['DATABASE_URL'] + // Lets the loader reach a Postgres published on the host (docker-compose's port 5432). + docker.runOptions = '--add-host=host.docker.internal:host-gateway' + } + gcp { + // The driver runs in the Argo pod; each task runs as a Google Batch job, which is what a + // gs:// work directory requires (the k8s executor needs a shared ReadWriteMany volume). + workDir = "gs://${params.bucket}/work" + params.vep_cache = "gs://${params.bucket}/refs/vep" + google { + project = params.project + location = params.region + batch.serviceAccountEmail = "rarelens-pipeline@${params.project}.iam.gserviceaccount.com" + batch.network = "projects/${params.project}/global/networks/rarelens-vpc" + batch.subnetwork = "projects/${params.project}/regions/${params.region}/subnetworks/rarelens-gke" + } + process { + executor = 'google-batch' + // Google Secret Manager secret created by Terraform (infra/terraform/secrets.tf). + withName: LOAD_DB { secret = 'DATABASE_URL' } + } + } } diff --git a/pipeline/requirements.txt b/pipeline/requirements.txt new file mode 100644 index 0000000..1e522d0 --- /dev/null +++ b/pipeline/requirements.txt @@ -0,0 +1,3 @@ +pandas>=2.2 +sqlalchemy>=2.0 +psycopg[binary]>=3.2 diff --git a/pipeline/tests/conftest.py b/pipeline/tests/conftest.py new file mode 100644 index 0000000..0ece951 --- /dev/null +++ b/pipeline/tests/conftest.py @@ -0,0 +1,60 @@ +"""Loader tests. DB tests use DATABASE_URL (CI's Postgres service) or a throwaway pgserver.""" +import os +import sys +import tempfile +from collections.abc import Iterator +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "bin")) + +# The subset of the Alembic schema (api/alembic/versions) that the loader writes to. +SCHEMA = """ +CREATE TYPE jobstatus AS ENUM ('queued', 'running', 'succeeded', 'failed'); +CREATE TABLE jobs ( + id uuid PRIMARY KEY, + status jobstatus NOT NULL, + vep_version text, + log text, + finished_at timestamptz +); +CREATE TABLE variants ( + id serial PRIMARY KEY, + job_id uuid NOT NULL, + chrom text NOT NULL, pos integer NOT NULL, ref text NOT NULL, alt text NOT NULL, + gene text, consequence text, impact text, hgvsc text, hgvsp text, + gnomad_af double precision, clinvar_sig text, annotations jsonb NOT NULL +); +""" + +_server = None + + +def _url() -> str | None: + global _server + if url := os.environ.get("DATABASE_URL"): + return url + try: + import pgserver + except ImportError: + return None + _server = pgserver.get_server(tempfile.mkdtemp(prefix="loader-pg-"), cleanup_mode="delete") + return "postgresql://postgres@/postgres?host=" + _server.get_uri().split("host=", 1)[1] + + +@pytest.fixture +def engine() -> Iterator: + url = _url() + if not url: + if os.environ.get("REQUIRE_DB"): + pytest.fail("REQUIRE_DB is set but no database is available") + pytest.skip("no DATABASE_URL and pgserver is not installed") + from load_db import engine_for + + eng = engine_for(url) + with eng.begin() as conn: + conn.exec_driver_sql("DROP TABLE IF EXISTS variants, jobs; DROP TYPE IF EXISTS jobstatus") + conn.exec_driver_sql(SCHEMA) + yield eng + eng.dispose() diff --git a/pipeline/tests/data/tiny.vcf b/pipeline/tests/data/tiny.vcf new file mode 100644 index 0000000..e389d4f --- /dev/null +++ b/pipeline/tests/data/tiny.vcf @@ -0,0 +1,7 @@ +##fileformat=VCFv4.2 +##source=rarelens synthetic smoke-test fixture (not real sample data) +##contig= +#CHROM POS ID REF ALT QUAL FILTER INFO +22 19710700 . C T . PASS . +22 29091857 . G A,C . PASS . +22 42126611 . CT C . PASS . diff --git a/pipeline/tests/test_load_db.py b/pipeline/tests/test_load_db.py new file mode 100644 index 0000000..7f3e937 --- /dev/null +++ b/pipeline/tests/test_load_db.py @@ -0,0 +1,136 @@ +import json +import uuid +from pathlib import Path + +import pytest +from load_db import engine_for, load, parse_variant_id, read_vep_tab, to_rows, vep_version +from set_job_status import set_status +from sqlalchemy import text + +HEADER = [ + "Uploaded_variation", "Location", "Allele", "Consequence", "IMPACT", "SYMBOL", + "HGVSc", "HGVSp", "gnomADe_AF", "CLIN_SIG", "CADD_PHRED", +] +# Longer than the old VARCHAR(120) column. +LONG_CLIN_SIG = ( + "conflicting_classifications_of_pathogenicity,uncertain_significance," + "likely_benign,benign,likely_pathogenic,pathogenic" +) + + +def vep_tab(tmp_path: Path, rows: list[list[str]]) -> Path: + path = tmp_path / "x.vep.tsv" + path.write_text("\n".join([ + "## ENSEMBL VARIANT EFFECT PREDICTOR v113.0", + "## Output produced at 2026-09-11 12:00:00", + "#" + "\t".join(HEADER), + *("\t".join(r) for r in rows), + ]) + "\n") + return path + + +ROWS = [ + # SNV + ["22_19710700_C_T", "22:19710700", "T", "missense_variant", "MODERATE", "TBX1", + "ENST1:c.1C>T", "ENSP1:p.Arg1Trp", "0.0001", "pathogenic", "28.1"], + # Deletion: VEP reports a trimmed allele ("-") and a shifted Location; the ID keeps the VCF truth. + ["22_42126611_CT_C", "22:42126612", "-", "frameshift_variant", "HIGH", "CYP2D6", + "-", "-", "-", LONG_CLIN_SIG, "-"], +] + + +@pytest.mark.parametrize( + ("uid", "expected"), + [ + ("22_19710700_C_T", ("22", 19710700, "C", "T")), + ("chrUn_KI270742v1_100_A_AT", ("chrUn_KI270742v1", 100, "A", "AT")), + ], +) +def test_parse_variant_id(uid: str, expected: tuple) -> None: + assert parse_variant_id(uid) == expected + + +@pytest.mark.parametrize("uid", ["rs123", "12345", "22_x_A_G", "."]) +def test_parse_variant_id_rejects_ids_not_set_by_normalise(uid: str) -> None: + with pytest.raises(ValueError, match="CHROM_POS_REF_ALT"): + parse_variant_id(uid) + + +def test_to_rows_takes_alleles_from_the_id_and_nulls_dashes(tmp_path: Path) -> None: + rows = to_rows(read_vep_tab(vep_tab(tmp_path, ROWS)), job_id="j") + snv, deletion = rows + assert (snv["chrom"], snv["pos"], snv["ref"], snv["alt"]) == ("22", 19710700, "C", "T") + assert (deletion["pos"], deletion["ref"], deletion["alt"]) == (42126611, "CT", "C") + assert snv["gnomad_af"] == 0.0001 and deletion["gnomad_af"] is None + assert deletion["hgvsc"] is None and deletion["gene"] == "CYP2D6" + assert deletion["clinvar_sig"] == LONG_CLIN_SIG + ann = json.loads(deletion["annotations"]) + assert "-" not in ann.values() and "CADD_PHRED" not in ann + assert json.loads(snv["annotations"])["CADD_PHRED"] == "28.1" + + +def test_vep_version(tmp_path: Path) -> None: + assert vep_version(vep_tab(tmp_path, ROWS)) == "113.0" + + +def test_engine_for_uses_psycopg_for_any_postgres_url() -> None: + for url in ("postgresql+asyncpg://u:p@h/db", "postgresql://u:p@h/db"): + assert engine_for(url).url.drivername == "postgresql+psycopg" + + +def new_job(engine, status: str = "running") -> str: + job_id = str(uuid.uuid4()) + with engine.begin() as conn: + conn.execute(text("INSERT INTO jobs (id, status) VALUES (:id, :s)"), {"id": job_id, "s": status}) + return job_id + + +def job_and_count(engine, job_id: str) -> tuple: + with engine.connect() as conn: + job = conn.execute(text("SELECT status, vep_version, log FROM jobs WHERE id=:id"), + {"id": job_id}).one() + n = conn.execute(text("SELECT count(*) FROM variants WHERE job_id=:id"), {"id": job_id}).scalar() + return (*job, n) + + +def test_load_is_idempotent_and_marks_job_succeeded(engine, tmp_path: Path) -> None: + job_id = new_job(engine) + rows = to_rows(read_vep_tab(vep_tab(tmp_path, ROWS)), job_id=job_id) + load(engine, job_id, rows, vep="113.0") + load(engine, job_id, rows, vep="113.0") # a retried task must not duplicate variants + assert job_and_count(engine, job_id) == ("succeeded", "113.0", None, 2) + + +def test_load_with_no_variants_still_succeeds(engine, tmp_path: Path) -> None: + job_id = new_job(engine) + rows = to_rows(read_vep_tab(vep_tab(tmp_path, [])), job_id=job_id) + load(engine, job_id, rows, vep="113.0") + assert job_and_count(engine, job_id) == ("succeeded", "113.0", None, 0) + + +def test_set_status_failed_records_the_reason(engine) -> None: + job_id = new_job(engine) + set_status(engine, job_id, "failed", "workflow annotate-abc Failed") + status, _, log, _ = job_and_count(engine, job_id) + assert (status, log) == ("failed", "workflow annotate-abc Failed") + + +def test_set_status_never_overrides_a_succeeded_job(engine) -> None: + job_id = new_job(engine, status="succeeded") + set_status(engine, job_id, "failed", "late exit handler") + assert job_and_count(engine, job_id)[0] == "succeeded" + + +@pytest.mark.parametrize( + ("raw", "expected_query"), + [ + # asyncpg's ssl= becomes libpq's sslmode= for psycopg. + ("postgresql+asyncpg://u:p@h/db?ssl=require", {"sslmode": "require"}), + ("postgresql://u:p@h/db?sslmode=require", {"sslmode": "require"}), + ("postgresql+asyncpg://u:p@h/db", {}), + ], +) +def test_engine_for_translates_ssl_options(raw: str, expected_query: dict) -> None: + url = engine_for(raw).url + assert url.drivername == "postgresql+psycopg" + assert dict(url.query) == expected_query diff --git a/scripts/fetch-demo-data.sh b/scripts/fetch-demo-data.sh new file mode 100755 index 0000000..d30eeec --- /dev/null +++ b/scripts/fetch-demo-data.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Fetch the public demo slice: a real GIAB genome and real ClinVar labels, chr22 only. +# Provenance, licences and citations: docs/data.md +set -euo pipefail + +OUT_DIR=${OUT_DIR:-data} +CLINVAR_URL=${CLINVAR_URL:-https://ftp.ncbi.nlm.nih.gov/pub/clinvar/vcf_GRCh38/clinvar.vcf.gz} +GIAB_URL=${GIAB_URL:-https://ftp-trace.ncbi.nlm.nih.gov/ReferenceSamples/giab/release/AshkenazimTrio/HG002_NA24385_son/NISTv4.2.1/GRCh38/HG002_GRCh38_1_22_v4.2.1_benchmark.vcf.gz} + +for tool in bcftools tabix; do + command -v "$tool" >/dev/null || { + echo "$tool is required (brew install bcftools, or apt install bcftools tabix)" >&2 + exit 1 + } +done + +mkdir -p "$OUT_DIR" + +# Both sources are indexed, so bcftools streams one chromosome instead of downloading a whole +# genome. ClinVar names contigs "22"; GIAB names them "chr22". +echo "==> GIAB HG002 (NA24385) v4.2.1 benchmark, chr22 -> $OUT_DIR/example.vcf.gz" +bcftools view -r chr22 "$GIAB_URL" -Oz -o "$OUT_DIR/example.vcf.gz" +tabix -f -p vcf "$OUT_DIR/example.vcf.gz" + +echo "==> ClinVar GRCh38, chr22 -> $OUT_DIR/clinvar.chr22.vcf.gz" +bcftools view -r 22 "$CLINVAR_URL" -Oz -o "$OUT_DIR/clinvar.chr22.vcf.gz" +tabix -f -p vcf "$OUT_DIR/clinvar.chr22.vcf.gz" + +echo +echo "Fetched:" +ls -lh "$OUT_DIR/example.vcf.gz" "$OUT_DIR/clinvar.chr22.vcf.gz" +cat <<'NEXT' + +Next: + make pipeline VCF=data/example.vcf.gz # annotate the GIAB sample (needs a VEP cache) + make annotate JOB= VCF=data/example.vcf.gz +NEXT diff --git a/web/.dockerignore b/web/.dockerignore new file mode 100644 index 0000000..6ae7bf1 --- /dev/null +++ b/web/.dockerignore @@ -0,0 +1,5 @@ +# .env holds the local dev API URL; the image reads PUBLIC_API_URL at runtime instead. +.env +node_modules/ +build/ +.svelte-kit/ diff --git a/web/.env b/web/.env index 7f13e1b..5575f86 100644 --- a/web/.env +++ b/web/.env @@ -1 +1 @@ -PUBLIC_API_URL=http://localhost:8000 \ No newline at end of file +PUBLIC_API_URL=http://localhost:8000/api diff --git a/web/package-lock.json b/web/package-lock.json index e7951ee..b527a24 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -11,6 +11,7 @@ "@sveltejs/adapter-node": "^5.2.0", "@sveltejs/kit": "^2.5.0", "@sveltejs/vite-plugin-svelte": "^4.0.0", + "@types/node": "^22.20.2", "svelte": "^5.0.0", "svelte-check": "^4.0.0", "typescript": "^5.5.0", @@ -1091,6 +1092,16 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/node": { + "version": "22.20.2", + "resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.2.tgz", + "integrity": "sha512-xlvWf4Vs9n1PEVYwP1n4vvG07M6y8WgvJ2t0vbrWTmijsIHp1cS+uJ2kMIRdY3nHZK0nCYKrPeD171+SzF4/zw==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~6.21.0" + } + }, "node_modules/@types/resolve": { "version": "1.20.2", "resolved": "https://registry.npmjs.org/@types/resolve/-/resolve-1.20.2.tgz", @@ -2020,6 +2031,13 @@ "node": ">=14.17" } }, + "node_modules/undici-types": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz", + "integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==", + "dev": true, + "license": "MIT" + }, "node_modules/vite": { "version": "5.4.21", "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.21.tgz", diff --git a/web/package.json b/web/package.json index d79c683..ae5b020 100644 --- a/web/package.json +++ b/web/package.json @@ -14,6 +14,7 @@ "@sveltejs/adapter-node": "^5.2.0", "@sveltejs/kit": "^2.5.0", "@sveltejs/vite-plugin-svelte": "^4.0.0", + "@types/node": "^22.20.2", "svelte": "^5.0.0", "svelte-check": "^4.0.0", "typescript": "^5.5.0", diff --git a/web/src/lib/api.ts b/web/src/lib/api.ts index 4ffc62f..15fbcaa 100644 --- a/web/src/lib/api.ts +++ b/web/src/lib/api.ts @@ -1,7 +1,13 @@ -import { PUBLIC_API_URL } from '$env/static/public'; +// Dynamic, not static: the same image serves /api behind the ingress and a full URL elsewhere. +import { env } from '$env/dynamic/public'; +import { variantQuery, type VariantFilters } from './query'; -export type Sample = { id: string; name: string; vcf_uri: string; assembly: string; created_at: string }; -export type Job = { id: string; sample_id: string; status: 'queued' | 'running' | 'succeeded' | 'failed'; created_at: string; finished_at: string | null }; +export type Assembly = 'GRCh38' | 'GRCh37'; +export type Sample = { id: string; name: string; vcf_uri: string; assembly: Assembly; created_at: string }; +export type Job = { + id: string; sample_id: string; status: 'queued' | 'running' | 'succeeded' | 'failed'; + log: string | null; created_at: string; finished_at: string | null; +}; export type Variant = { id: number; chrom: string; pos: number; ref: string; alt: string; gene: string | null; consequence: string | null; impact: string | null; hgvsc: string | null; hgvsp: string | null; @@ -10,16 +16,30 @@ export type Variant = { export type VariantPage = { items: Variant[]; total: number; limit: number; offset: number }; async function req(path: string, init?: RequestInit): Promise { - const r = await fetch(`${PUBLIC_API_URL}${path}`, { headers: { 'content-type': 'application/json' }, ...init }); - if (!r.ok) throw new Error(`${r.status} ${r.statusText} on ${path}`); + const r = await fetch(`${env.PUBLIC_API_URL ?? '/api'}${path}`, { headers: { 'content-type': 'application/json' }, ...init }); + if (!r.ok) throw new Error(await errorReason(r)); return r.json() as Promise; } +// FastAPI puts the reason in `detail`: a string, or a list of validation errors. +async function errorReason(r: Response): Promise { + const fallback = `${r.status} ${r.statusText}`; + try { + const { detail } = await r.json(); + if (typeof detail === 'string') return detail; + if (Array.isArray(detail)) return detail.map((d: { msg?: string }) => d.msg ?? '').join('; ') || fallback; + } catch { + // not JSON (e.g. a proxy error page) + } + return fallback; +} + export const api = { samples: () => req('/samples'), - createSample: (body: Pick) => req('/samples', { method: 'POST', body: JSON.stringify(body) }), + createSample: (body: Pick) => + req('/samples', { method: 'POST', body: JSON.stringify(body) }), jobsForSample: (sampleId: string) => req(`/samples/${sampleId}/jobs`), annotate: (sampleId: string) => req(`/samples/${sampleId}/annotate`, { method: 'POST' }), job: (id: string) => req(`/jobs/${id}`), - variants: (q: Record) => req(`/variants?${new URLSearchParams(q as Record)}`) + variants: (filters: VariantFilters) => req(`/variants?${variantQuery(filters)}`) }; diff --git a/web/src/lib/poll.test.ts b/web/src/lib/poll.test.ts new file mode 100644 index 0000000..69f6b70 --- /dev/null +++ b/web/src/lib/poll.test.ts @@ -0,0 +1,54 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { poll } from './poll'; + +beforeEach(() => vi.useFakeTimers()); +afterEach(() => vi.useRealTimers()); + +describe('poll', () => { + it('fetches until done, then stops', async () => { + const values = ['queued', 'running', 'succeeded', 'never']; + const fetch = vi.fn(async () => values.shift()!); + const seen: string[] = []; + poll(fetch, { intervalMs: 1000, done: (v) => v === 'succeeded', onValue: (v) => seen.push(v), onError: () => {} }); + + await vi.advanceTimersByTimeAsync(5000); + expect(seen).toEqual(['queued', 'running', 'succeeded']); + expect(fetch).toHaveBeenCalledTimes(3); + }); + + it('never has two requests in flight when the API is slower than the interval', async () => { + let inFlight = 0; + let maxInFlight = 0; + const fetch = async () => { + inFlight++; + maxInFlight = Math.max(maxInFlight, inFlight); + await new Promise((r) => setTimeout(r, 2500)); + inFlight--; + return 'running'; + }; + poll(fetch, { intervalMs: 1000, done: () => false, onValue: () => {}, onError: () => {} }); + + await vi.advanceTimersByTimeAsync(10_000); + expect(maxInFlight).toBe(1); + }); + + it('reports an error once and stops', async () => { + const fetch = vi.fn(async () => { throw new Error('502'); }); + const onError = vi.fn(); + poll(fetch, { intervalMs: 1000, done: () => false, onValue: () => {}, onError }); + + await vi.advanceTimersByTimeAsync(5000); + expect(onError).toHaveBeenCalledOnce(); + expect(fetch).toHaveBeenCalledOnce(); + }); + + it('stop() cancels future fetches, e.g. when the page unmounts', async () => { + const fetch = vi.fn(async () => 'running'); + const stop = poll(fetch, { intervalMs: 1000, done: () => false, onValue: () => {}, onError: () => {} }); + + await vi.advanceTimersByTimeAsync(1500); + stop(); + await vi.advanceTimersByTimeAsync(10_000); + expect(fetch).toHaveBeenCalledTimes(1); + }); +}); diff --git a/web/src/lib/poll.ts b/web/src/lib/poll.ts new file mode 100644 index 0000000..f86ba6c --- /dev/null +++ b/web/src/lib/poll.ts @@ -0,0 +1,35 @@ +export type PollOptions = { + intervalMs: number; + done: (value: T) => boolean; + onValue: (value: T) => void; + onError: (error: unknown) => void; +}; + +/** + * Fetch every `intervalMs` until `done` returns true or a fetch fails. The next request is only + * scheduled once the previous one settles, so a slow API never has overlapping requests. + * Returns a stop function; call it when the component unmounts. + */ +export function poll(fetch: () => Promise, opts: PollOptions): () => void { + let stopped = false; + let timer: ReturnType | undefined; + + const tick = async () => { + let value: T; + try { + value = await fetch(); + } catch (e) { + if (!stopped) opts.onError(e); + return; + } + if (stopped) return; + opts.onValue(value); + if (!opts.done(value)) timer = setTimeout(tick, opts.intervalMs); + }; + + timer = setTimeout(tick, opts.intervalMs); + return () => { + stopped = true; + clearTimeout(timer); + }; +} diff --git a/web/src/lib/proxy.test.ts b/web/src/lib/proxy.test.ts new file mode 100644 index 0000000..fbf522f --- /dev/null +++ b/web/src/lib/proxy.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it, vi } from 'vitest'; +import { proxyToApi } from './proxy'; + +// Same signature as fetch, so mock.calls stays typed. +type FetchArgs = [input: RequestInfo | URL, init?: RequestInit]; + +const upstreamOk = (body: unknown = { ok: true }) => + vi.fn(async (..._args: FetchArgs) => + new Response(JSON.stringify(body), { status: 200, headers: { 'content-type': 'application/json' } }) + ); + +describe('proxyToApi', () => { + it('forwards the path and query to the API', async () => { + const fetch = upstreamOk(); + await proxyToApi({ + request: new Request('https://web.example/api/variants?job_id=abc&limit=100'), + path: 'variants', + search: '?job_id=abc&limit=100', + base: 'https://api.internal', + fetch + }); + expect(fetch.mock.calls[0][0]).toBe('https://api.internal/api/variants?job_id=abc&limit=100'); + }); + + it('forwards POST bodies', async () => { + const fetch = upstreamOk(); + await proxyToApi({ + request: new Request('https://web.example/api/samples', { + method: 'POST', + body: JSON.stringify({ name: 'HG002' }), + headers: { 'content-type': 'application/json' } + }), + path: 'samples', + search: '', + base: 'https://api.internal', + fetch + }); + const init = fetch.mock.calls[0][1]!; + expect(init.method).toBe('POST'); + expect(init.body).toBe('{"name":"HG002"}'); + }); + + it('passes the upstream status through, so the UI sees 404s and 409s', async () => { + const fetch = vi.fn(async (..._args: FetchArgs) => + new Response(JSON.stringify({ detail: 'sample not found' }), { status: 404, headers: { 'content-type': 'application/json' } }) + ); + const res = await proxyToApi({ + request: new Request('https://web.example/api/jobs/x'), + path: 'jobs/x', + search: '', + base: 'https://api.internal', + fetch + }); + expect(res.status).toBe(404); + expect(await res.json()).toEqual({ detail: 'sample not found' }); + }); + + it('reports a 502 when the API cannot be reached', async () => { + const fetch = vi.fn(async (..._args: FetchArgs): Promise => { throw new Error('ECONNREFUSED'); }); + const res = await proxyToApi({ + request: new Request('https://web.example/api/samples'), + path: 'samples', + search: '', + base: 'https://api.internal', + fetch + }); + expect(res.status).toBe(502); + expect((await res.json()).detail).toContain('ECONNREFUSED'); + }); +}); diff --git a/web/src/lib/proxy.ts b/web/src/lib/proxy.ts new file mode 100644 index 0000000..f779453 --- /dev/null +++ b/web/src/lib/proxy.ts @@ -0,0 +1,35 @@ +export type ProxyOptions = { + request: Request; + path: string; + search: string; + base: string; + fetch: typeof globalThis.fetch; +}; + +/** + * Forward /api/* to the API service. + * + * Behind the Kubernetes ingress this never runs — the ingress routes /api itself. On Cloud Run it + * keeps the UI and the API on one origin, so there is one public URL and no CORS to configure. + */ +export async function proxyToApi({ request, path, search, base, fetch }: ProxyOptions): Promise { + const hasBody = request.method !== 'GET' && request.method !== 'HEAD'; + const contentType = request.headers.get('content-type') ?? 'application/json'; + try { + const upstream = await fetch(`${base}/api/${path}${search}`, { + method: request.method, + headers: { 'content-type': contentType }, + body: hasBody ? await request.text() : undefined + }); + return new Response(upstream.body, { + status: upstream.status, + headers: { 'content-type': upstream.headers.get('content-type') ?? 'application/json' } + }); + } catch (e) { + // Shaped like FastAPI's errors so the UI reports it the same way. + return new Response(JSON.stringify({ detail: `API unreachable: ${(e as Error).message}` }), { + status: 502, + headers: { 'content-type': 'application/json' } + }); + } +} diff --git a/web/src/lib/query.test.ts b/web/src/lib/query.test.ts new file mode 100644 index 0000000..dd602d2 --- /dev/null +++ b/web/src/lib/query.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it } from 'vitest'; +import { variantQuery } from './query'; + +describe('variantQuery', () => { + it('always includes the job id', () => { + expect(variantQuery({ jobId: 'abc' })).toBe('job_id=abc'); + }); + + it('includes filters that have values', () => { + const q = new URLSearchParams( + variantQuery({ jobId: 'abc', gene: ' brca1 ', impact: 'HIGH', maxAf: 0.01, limit: 100 }) + ); + expect(q.get('gene')).toBe('brca1'); + expect(q.get('impact')).toBe('HIGH'); + expect(q.get('max_af')).toBe('0.01'); + expect(q.get('limit')).toBe('100'); + }); + + it('drops a cleared number input instead of sending "null"', () => { + // Svelte binds an emptied to null. + for (const maxAf of [null, undefined, '', Number.NaN]) { + expect(variantQuery({ jobId: 'abc', maxAf })).toBe('job_id=abc'); + } + }); + + it('drops blank gene and impact', () => { + expect(variantQuery({ jobId: 'abc', gene: ' ', impact: '' })).toBe('job_id=abc'); + }); +}); diff --git a/web/src/lib/query.ts b/web/src/lib/query.ts new file mode 100644 index 0000000..1d83f74 --- /dev/null +++ b/web/src/lib/query.ts @@ -0,0 +1,22 @@ +export type VariantFilters = { + jobId: string; + gene?: string; + impact?: string; + // An emptied binds to null, so accept it and drop it. + maxAf?: number | string | null; + limit?: number; + offset?: number; +}; + +export function variantQuery(f: VariantFilters): string { + const q = new URLSearchParams({ job_id: f.jobId }); + const gene = f.gene?.trim(); + if (gene) q.set('gene', gene); + if (f.impact) q.set('impact', f.impact); + if (f.maxAf !== null && f.maxAf !== undefined && f.maxAf !== '' && Number.isFinite(Number(f.maxAf))) { + q.set('max_af', String(f.maxAf)); + } + if (f.limit !== undefined) q.set('limit', String(f.limit)); + if (f.offset !== undefined) q.set('offset', String(f.offset)); + return q.toString(); +} diff --git a/web/src/routes/+page.svelte b/web/src/routes/+page.svelte index 026a4ba..7b258a0 100644 --- a/web/src/routes/+page.svelte +++ b/web/src/routes/+page.svelte @@ -1,18 +1,30 @@ @@ -24,12 +36,19 @@
+
{#if error}

Could not add the sample: {error}

{/if}

Samples

-{#if samples.length === 0} +{#if loadError} +

Could not load samples: {loadError}.

+{:else if !loaded} +

Loading samples…

+{:else if samples.length === 0}
No samples yet. Add one above to run the annotation pipeline.
{:else} diff --git a/web/src/routes/api/[...path]/+server.ts b/web/src/routes/api/[...path]/+server.ts new file mode 100644 index 0000000..83ab397 --- /dev/null +++ b/web/src/routes/api/[...path]/+server.ts @@ -0,0 +1,15 @@ +import { env } from '$env/dynamic/private'; +import { proxyToApi } from '$lib/proxy'; +import type { RequestHandler } from './$types'; + +const handle: RequestHandler = ({ request, params, url, fetch }) => + proxyToApi({ + request, + path: params.path, + search: url.search, + base: env.API_INTERNAL_URL ?? 'http://localhost:8000', + fetch + }); + +export const GET = handle; +export const POST = handle; diff --git a/web/src/routes/samples/[id]/+page.svelte b/web/src/routes/samples/[id]/+page.svelte index c8ef3aa..7402699 100644 --- a/web/src/routes/samples/[id]/+page.svelte +++ b/web/src/routes/samples/[id]/+page.svelte @@ -1,38 +1,72 @@