diff --git a/README.md b/README.md index 9aae36b..9da8943 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,43 @@ propagation does not. Local Kubernetes: `make kind` builds the images, loads them into a kind cluster and applies `infra/k8s/overlays/local`. +## Deploying to Railway + +The quickest way to put it in front of people. Three services — Postgres, the API, the UI — with +the API reachable only over Railway's private network, so the UI's `/api` proxy is the single +public entry point and there is no CORS. + +```bash +railway link --project --environment production --service api +cd api && railway up --service api # the repo is on Gitea, so deploy from the working copy +cd ../web && railway up --service web +``` + +Set on the API: `DATABASE_URL` (pointing at `postgres.railway.internal`) and +`DATABASE_IDLE_CONNECTIONS=false`. On the UI: `API_INTERNAL_URL=http://api.railway.internal:8000`, +`PUBLIC_PIPELINE_ENABLED=false`, and `BASIC_AUTH_USER` / `BASIC_AUTH_PASSWORD`. + +Three things are worth knowing before copying this: + +- **The pipeline cannot run there.** Nextflow shells out to `docker run` for VEP and bcftools, and + Railway gives you a container, not a Docker daemon. So the cases are annotated here and copied + up with `scripts/seed-remote.sh`, and `PUBLIC_PIPELINE_ENABLED=false` hides the buttons that + would otherwise be left to fail. Visitors explore real analysed cases; they do not run VEP. +- **`DATABASE_IDLE_CONNECTIONS=false` is what makes sleeping work.** Railway decides a service is + idle from its *outbound* traffic, and a pooled database connection is outbound traffic, so the + default pool would keep the API awake and billable for ever. It uses `NullPool` instead, which + costs a connection per request and is the wrong trade under real load. +- **Serverless must be enabled per service and only takes effect on the next deploy.** Leave it off + for Postgres, which holds the volume. + +`BASIC_AUTH_USER`/`BASIC_AUTH_PASSWORD` put one shared credential in front of the whole site +(`web/src/hooks.server.ts`); unset, the site is open, which is what local development wants. +Nothing here is patient data, so this stops the URL being wandered into rather than protecting +anyone's privacy. + +Cost: Railway's Hobby plan is $5/month flat including $5 of usage, so it costs that whether or not +anyone visits — more at rest than the GCP track below, and much less to operate. + ## Deploying to GCP Two tracks, same code. The serverless one is the default because it costs about £1/month idle; diff --git a/api/app/config.py b/api/app/config.py index 3b16569..d71eabc 100644 --- a/api/app/config.py +++ b/api/app/config.py @@ -32,6 +32,12 @@ class Settings(BaseSettings): local_data_root: Path = Path("/data") # Browsers calling the API cross-origin; behind the ingress the UI is same-origin. cors_origins: list[str] = ["http://localhost:5173"] + # Whether to keep pooled database connections open between requests. Set it false on a host + # that sleeps idle containers to save money -- Railway's serverless mode decides a service is + # idle from its *outbound* traffic, and a held connection is outbound traffic, so a pool keeps + # the service awake and billable forever. Costs a connection setup per request, which is + # nothing at demo traffic and the wrong trade under real load. + database_idle_connections: bool = True settings = Settings() diff --git a/api/app/db.py b/api/app/db.py index f2c21ef..312d5e7 100644 --- a/api/app/db.py +++ b/api/app/db.py @@ -4,6 +4,7 @@ from typing import Annotated from fastapi import Depends from sqlalchemy.engine import make_url from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine +from sqlalchemy.pool import NullPool from app.config import settings @@ -21,7 +22,12 @@ def normalize_async_url(url: str) -> str: return parsed.set(query=query).render_as_string(hide_password=False) -engine = create_async_engine(normalize_async_url(settings.database_url), pool_pre_ping=True) +# NullPool opens and closes a connection per checkout, so an idle API sends nothing and a host +# that sleeps idle containers can actually do so. See settings.database_idle_connections. +_pool = {} if settings.database_idle_connections else {"poolclass": NullPool} +engine = create_async_engine( + normalize_async_url(settings.database_url), pool_pre_ping=True, **_pool +) SessionLocal = async_sessionmaker(engine, expire_on_commit=False) diff --git a/scripts/seed-remote.sh b/scripts/seed-remote.sh new file mode 100755 index 0000000..1bc535f --- /dev/null +++ b/scripts/seed-remote.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# Copy the analysed demo cases and the HPO reference data into a remote database. +# +# The deployed demo has no Nextflow and no Docker daemon, so it cannot run VEP. Instead the cases +# are annotated here, where the pipeline works, and the results are copied up. Visitors get real +# analysed cases -- funnel, ranking, evidence, decisions, report -- against a database that is +# never asked to produce them. +# +# Everything copied is derived from public, openly licensed sources (docs/data.md). There is no +# patient data in this dump, and there must never be. +# +# scripts/seed-remote.sh "$(railway variables -s Postgres --kv | grep DATABASE_PUBLIC_URL | cut -d= -f2-)" +set -euo pipefail + +TARGET=${1:-${TARGET_DATABASE_URL:-}} +SOURCE=${SOURCE_DATABASE_URL:-postgresql://rarelens:rarelens@localhost:5432/rarelens} +IMAGE=${PG_IMAGE:-postgres:16-alpine} + +# Reference data first, then a case's own rows: variants reference jobs, jobs reference cases. +TABLES=(hpo_terms gene_phenotypes cases case_phenotypes jobs variants predictions variant_decisions) + +if [ -z "$TARGET" ]; then + echo "usage: $0 (or set TARGET_DATABASE_URL)" >&2 + echo "the target must already have the schema: run 'alembic upgrade head' against it first" >&2 + exit 2 +fi + +case "$TARGET" in + postgres://*|postgresql://*) ;; + *) echo "target must be a postgres:// URL, got ${TARGET%%:*}:..." >&2; exit 2 ;; +esac + +echo "==> source: ${SOURCE%%\?*}" >&2 +echo "==> target: ${TARGET%%:*}://…${TARGET##*@}" >&2 # host only: the password stays out of the log +echo "==> tables: ${TABLES[*]}" >&2 + +# --data-only: the target's schema comes from Alembic, so the two can never disagree about it. +# Truncating first makes the script repeatable; cascade because variants hang off jobs. +docker run --rm -i --add-host=host.docker.internal:host-gateway \ + -e PGPASSWORD_UNUSED=1 "$IMAGE" \ + pg_dump --data-only --no-owner --no-privileges \ + $(printf -- '--table=%s ' "${TABLES[@]}") \ + "${SOURCE/localhost/host.docker.internal}" \ + > /tmp/rarelens-seed.sql + +lines=$(wc -l < /tmp/rarelens-seed.sql) +echo "==> dumped $lines lines" >&2 + +docker run --rm -i "$IMAGE" psql "$TARGET" -v ON_ERROR_STOP=1 \ + -c "TRUNCATE $(IFS=,; echo "${TABLES[*]}") RESTART IDENTITY CASCADE" >&2 + +docker run --rm -i "$IMAGE" psql "$TARGET" -v ON_ERROR_STOP=1 -q < /tmp/rarelens-seed.sql +rm -f /tmp/rarelens-seed.sql + +echo "==> loaded. counts on the target:" >&2 +docker run --rm -i "$IMAGE" psql "$TARGET" -At -c " + select 'cases: '||count(*) from cases + union all select 'variants: '||count(*) from variants + union all select 'gene_phenotypes: '||count(*) from gene_phenotypes + union all select 'hpo_terms: '||count(*) from hpo_terms" diff --git a/web/.dockerignore b/web/.dockerignore index 6ae7bf1..7fd02f9 100644 --- a/web/.dockerignore +++ b/web/.dockerignore @@ -1,5 +1,6 @@ -# .env holds the local dev API URL; the image reads PUBLIC_API_URL at runtime instead. -.env node_modules/ build/ .svelte-kit/ +# .env sets PUBLIC_API_URL for local development. Baked into the image it would override the +# same-origin /api proxy the deployment relies on, so it must not travel. +.env diff --git a/web/src/hooks.server.test.ts b/web/src/hooks.server.test.ts new file mode 100644 index 0000000..a49ba1d --- /dev/null +++ b/web/src/hooks.server.test.ts @@ -0,0 +1,53 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +const env: Record = {}; +vi.mock('$env/dynamic/private', () => ({ env })); + +const { handle } = await import('./hooks.server'); + +const credential = (user: string, password: string) => + `Basic ${Buffer.from(`${user}:${password}`).toString('base64')}`; + +function run(authorization?: string) { + const event = { + request: new Request('https://demo.example/cases', { + headers: authorization ? { authorization } : {} + }) + }; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + return handle({ event, resolve: async () => new Response('ok') } as any); +} + +describe('basic auth', () => { + beforeEach(() => { + env.BASIC_AUTH_USER = 'demo'; + env.BASIC_AUTH_PASSWORD = 'letmein'; + }); + + it('is off when no credential is configured, so local development is unaffected', async () => { + env.BASIC_AUTH_USER = undefined; + env.BASIC_AUTH_PASSWORD = undefined; + expect((await run()).status).toBe(200); + }); + + it('challenges an anonymous request', async () => { + const res = await run(); + expect(res.status).toBe(401); + expect(res.headers.get('www-authenticate')).toContain('Basic realm="rarelens"'); + }); + + it('lets the right credential through', async () => { + expect((await run(credential('demo', 'letmein'))).status).toBe(200); + }); + + it.each([ + ['wrong password', credential('demo', 'nope')], + ['wrong user', credential('someone', 'letmein')], + ['password that is a prefix of the real one', credential('demo', 'letme')], + ['not basic at all', 'Bearer letmein'], + ['no colon in the decoded value', `Basic ${Buffer.from('demo').toString('base64')}`], + ['empty', ''] + ])('rejects %s', async (_label, header) => { + expect((await run(header)).status).toBe(401); + }); +}); diff --git a/web/src/hooks.server.ts b/web/src/hooks.server.ts new file mode 100644 index 0000000..023ed9e --- /dev/null +++ b/web/src/hooks.server.ts @@ -0,0 +1,45 @@ +import { env } from '$env/dynamic/private'; +import type { Handle } from '@sveltejs/kit'; +import { timingSafeEqual } from 'node:crypto'; + +/** + * One shared credential in front of the whole site, for the deployed demo. + * + * Everything here is public, openly licensed data and there is no patient information, so this is + * not protecting anyone's privacy. It keeps the URL from being wandered into or indexed while a + * handful of people are looking at it, which is the actual requirement. Unset the variables and + * the site is open, which is what local development wants. + */ +const REALM = 'rarelens'; + +/** Constant time, and safe when the two differ in length (timingSafeEqual throws on that). */ +function matches(given: string, expected: string): boolean { + const a = Buffer.from(given); + const b = Buffer.from(expected); + return a.length === b.length && timingSafeEqual(a, b); +} + +function authorised(header: string | null, user: string, password: string): boolean { + if (!header?.startsWith('Basic ')) return false; + const decoded = Buffer.from(header.slice(6), 'base64').toString('utf8'); + const separator = decoded.indexOf(':'); + if (separator < 0) return false; + // Compare both halves even when the first fails, so the reply time says nothing about which did. + const okUser = matches(decoded.slice(0, separator), user); + const okPassword = matches(decoded.slice(separator + 1), password); + return okUser && okPassword; +} + +export const handle: Handle = async ({ event, resolve }) => { + const user = env.BASIC_AUTH_USER; + const password = env.BASIC_AUTH_PASSWORD; + if (!user || !password) return resolve(event); + + if (!authorised(event.request.headers.get('authorization'), user, password)) { + return new Response('Authentication required', { + status: 401, + headers: { 'www-authenticate': `Basic realm="${REALM}", charset="UTF-8"` } + }); + } + return resolve(event); +}; diff --git a/web/src/routes/+page.svelte b/web/src/routes/+page.svelte index 4286529..44c4095 100644 --- a/web/src/routes/+page.svelte +++ b/web/src/routes/+page.svelte @@ -1,4 +1,5 @@

rarelens

@@ -46,6 +49,7 @@ candidate carrying the evidence for its rank. Research demo, not a diagnostic tool.

+{#if canAnalyse}

New case

@@ -63,6 +67,7 @@
{#if error}

Could not create the case: {error}

{/if} +{/if}

Cases

{#if loadError} @@ -70,7 +75,13 @@ {:else if !loaded}

Loading…

{:else if cases.length === 0} -
No cases yet. Create one above, then run the annotation pipeline on it.
+
+ {#if canAnalyse} + No cases yet. Create one above, then run the annotation pipeline on it. + {:else} + No cases loaded. + {/if} +
{:else}
    {#each cases as c (c.id)} diff --git a/web/src/routes/cases/[id]/+page.svelte b/web/src/routes/cases/[id]/+page.svelte index 60c147a..2a603f5 100644 --- a/web/src/routes/cases/[id]/+page.svelte +++ b/web/src/routes/cases/[id]/+page.svelte @@ -3,6 +3,7 @@ import { api, type Candidate, type CandidatePage, type Case, type Job, type VariantDetail } from '$lib/api'; import { poll } from '$lib/poll'; import { formatElapsed, latestStep } from '$lib/progress'; + import { env } from '$env/dynamic/public'; import { missingEvidenceNote, needsScoring, plural } from '$lib/candidates'; import CandidateRow from '$lib/components/CandidateRow.svelte'; import Funnel from '$lib/components/Funnel.svelte'; @@ -35,6 +36,9 @@ // not need a five-minute re-run of VEP to get its score. const unscored = $derived(!!page && page.items.length > 0 && needsScoring(page.items, page.evidence)); const evidenceNote = $derived(page ? missingEvidenceNote(page.evidence) : null); + // The deployed demo has no Nextflow and no Docker daemon to run VEP in, so the pipeline cannot + // run there. Hide the buttons rather than leave them to fail: the cases are already analysed. + const canAnalyse = env.PUBLIC_PIPELINE_ENABLED !== 'false'; // Tick the elapsed time while a run is in flight; polling refreshes the step itself. $effect(() => { @@ -160,9 +164,11 @@

    {:else}

    - - {#if unscored} - + {#if canAnalyse} + + {#if unscored} + + {/if} {/if} {#if page && page.funnel.total > 0} Case report → @@ -219,7 +225,12 @@

{:else if page && !running}
- No results yet. "Analyse case" runs VEP over the VCF, scores each variant, then ranks what is - left against the phenotype. + {#if canAnalyse} + No results yet. "Analyse case" runs VEP over the VCF, scores each variant, then ranks what is + left against the phenotype. + {:else} + No results for this case. This deployment shows cases that were analysed beforehand; running + the pipeline needs Nextflow and a Docker daemon, which it does not have. + {/if}
{/if}