fix: overhaul the platform skeleton, add a serverless deployment track

An end-to-end audit found the repo could not build, test or run as shipped. This
fixes every finding, then adds a Cloud Run track so the demo costs about £1/month
idle instead of ~£150.

CI (red on its first run)
- api: setuptools could not build the package (flat layout with app/ and alembic/)
- web: missing @types/node; `vitest run` exited 1 with no test files
- pipeline: the stub run needed a gitignored VCF, and no process had a stub block
- ruff pinned, mypy configured, DB tests on real Postgres (pgserver locally, service in CI)

ML serving (scores were meaningless)
- the registered model now carries its own feature engineering and returns predict_proba,
  so serving sends raw columns and cannot drift from training
- resolve by registry alias (stages are deprecated in MLflow 3) and record the real
  version; re-scoring upserts instead of failing on the unique constraint
- ClinVar labels parsed from VEP's lowercase terms

Pipeline
- exact ref/alt recovered from a CHROM_POS_REF_ALT VCF ID; loading is idempotent
- job status reaches running/failed/succeeded, so the UI stops polling dead jobs
- DATABASE_URL travels in the environment or a Nextflow secret, never on a command line
- VEP cache and plugins staged as inputs; the gcp profile runs tasks on Google Batch

Deployment
- the API serves /api (matching the ingress); the web app reads its API URL at runtime
- migrations run in an init container under a Postgres advisory lock
- terraform: custom VPC shared with Batch, private Cloud SQL, API enablement, Workload
  Identity bindings, Secret Manager, deletion protection
- serverless track, now the default: Cloud Run services scaling to zero, a Cloud Run job
  for the Nextflow driver, and Neon or Cloud SQL behind one DATABASE_URL secret. GKE and
  Argo remain, behind -var deploy_kubernetes=true. See docs/cloud.md.

Correctness and security
- 409 on duplicate sample names, 422 on bad paging, natural chromosome ordering, wider
  VEP text columns, enum dropped on downgrade, the sample's assembly actually used
- vcf_uri restricted to gs:// objects or files under the data root, blocking option injection
- CORS restricted to configured origins; `make down` no longer deletes volumes

Data
- docs/data.md records the peer-reviewed, openly licensed sources (GIAB HG002, ClinVar,
  gnomAD) with citations and an honest evaluation plan; `make data` fetches a chr22 slice

Verified: api 50 tests, ml 18, loader 16, web 12; ruff, mypy, svelte-check, terraform
validate and both kustomize overlays clean.
This commit is contained in:
Kemal Yaylali
2026-09-12 07:21:11 +01:00
parent 5463f489a3
commit 11fb6b3d73
100 changed files with 3431 additions and 340 deletions
+5
View File
@@ -0,0 +1,5 @@
# .env holds the local dev API URL; the image reads PUBLIC_API_URL at runtime instead.
.env
node_modules/
build/
.svelte-kit/
+1 -1
View File
@@ -1 +1 @@
PUBLIC_API_URL=http://localhost:8000
PUBLIC_API_URL=http://localhost:8000/api
+18
View File
@@ -11,6 +11,7 @@
"@sveltejs/adapter-node": "^5.2.0",
"@sveltejs/kit": "^2.5.0",
"@sveltejs/vite-plugin-svelte": "^4.0.0",
"@types/node": "^22.20.2",
"svelte": "^5.0.0",
"svelte-check": "^4.0.0",
"typescript": "^5.5.0",
@@ -1091,6 +1092,16 @@
"dev": true,
"license": "MIT"
},
"node_modules/@types/node": {
"version": "22.20.2",
"resolved": "https://registry.npmjs.org/@types/node/-/node-22.20.2.tgz",
"integrity": "sha512-xlvWf4Vs9n1PEVYwP1n4vvG07M6y8WgvJ2t0vbrWTmijsIHp1cS+uJ2kMIRdY3nHZK0nCYKrPeD171+SzF4/zw==",
"dev": true,
"license": "MIT",
"dependencies": {
"undici-types": "~6.21.0"
}
},
"node_modules/@types/resolve": {
"version": "1.20.2",
"resolved": "https://registry.npmjs.org/@types/resolve/-/resolve-1.20.2.tgz",
@@ -2020,6 +2031,13 @@
"node": ">=14.17"
}
},
"node_modules/undici-types": {
"version": "6.21.0",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz",
"integrity": "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==",
"dev": true,
"license": "MIT"
},
"node_modules/vite": {
"version": "5.4.21",
"resolved": "https://registry.npmjs.org/vite/-/vite-5.4.21.tgz",
+1
View File
@@ -14,6 +14,7 @@
"@sveltejs/adapter-node": "^5.2.0",
"@sveltejs/kit": "^2.5.0",
"@sveltejs/vite-plugin-svelte": "^4.0.0",
"@types/node": "^22.20.2",
"svelte": "^5.0.0",
"svelte-check": "^4.0.0",
"typescript": "^5.5.0",
+27 -7
View File
@@ -1,7 +1,13 @@
import { PUBLIC_API_URL } from '$env/static/public';
// Dynamic, not static: the same image serves /api behind the ingress and a full URL elsewhere.
import { env } from '$env/dynamic/public';
import { variantQuery, type VariantFilters } from './query';
export type Sample = { id: string; name: string; vcf_uri: string; assembly: string; created_at: string };
export type Job = { id: string; sample_id: string; status: 'queued' | 'running' | 'succeeded' | 'failed'; created_at: string; finished_at: string | null };
export type Assembly = 'GRCh38' | 'GRCh37';
export type Sample = { id: string; name: string; vcf_uri: string; assembly: Assembly; created_at: string };
export type Job = {
id: string; sample_id: string; status: 'queued' | 'running' | 'succeeded' | 'failed';
log: string | null; created_at: string; finished_at: string | null;
};
export type Variant = {
id: number; chrom: string; pos: number; ref: string; alt: string; gene: string | null;
consequence: string | null; impact: string | null; hgvsc: string | null; hgvsp: string | null;
@@ -10,16 +16,30 @@ export type Variant = {
export type VariantPage = { items: Variant[]; total: number; limit: number; offset: number };
async function req<T>(path: string, init?: RequestInit): Promise<T> {
const r = await fetch(`${PUBLIC_API_URL}${path}`, { headers: { 'content-type': 'application/json' }, ...init });
if (!r.ok) throw new Error(`${r.status} ${r.statusText} on ${path}`);
const r = await fetch(`${env.PUBLIC_API_URL ?? '/api'}${path}`, { headers: { 'content-type': 'application/json' }, ...init });
if (!r.ok) throw new Error(await errorReason(r));
return r.json() as Promise<T>;
}
// FastAPI puts the reason in `detail`: a string, or a list of validation errors.
async function errorReason(r: Response): Promise<string> {
const fallback = `${r.status} ${r.statusText}`;
try {
const { detail } = await r.json();
if (typeof detail === 'string') return detail;
if (Array.isArray(detail)) return detail.map((d: { msg?: string }) => d.msg ?? '').join('; ') || fallback;
} catch {
// not JSON (e.g. a proxy error page)
}
return fallback;
}
export const api = {
samples: () => req<Sample[]>('/samples'),
createSample: (body: Pick<Sample, 'name' | 'vcf_uri'>) => req<Sample>('/samples', { method: 'POST', body: JSON.stringify(body) }),
createSample: (body: Pick<Sample, 'name' | 'vcf_uri' | 'assembly'>) =>
req<Sample>('/samples', { method: 'POST', body: JSON.stringify(body) }),
jobsForSample: (sampleId: string) => req<Job[]>(`/samples/${sampleId}/jobs`),
annotate: (sampleId: string) => req<Job>(`/samples/${sampleId}/annotate`, { method: 'POST' }),
job: (id: string) => req<Job>(`/jobs/${id}`),
variants: (q: Record<string, string | number>) => req<VariantPage>(`/variants?${new URLSearchParams(q as Record<string, string>)}`)
variants: (filters: VariantFilters) => req<VariantPage>(`/variants?${variantQuery(filters)}`)
};
+54
View File
@@ -0,0 +1,54 @@
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import { poll } from './poll';
beforeEach(() => vi.useFakeTimers());
afterEach(() => vi.useRealTimers());
describe('poll', () => {
it('fetches until done, then stops', async () => {
const values = ['queued', 'running', 'succeeded', 'never'];
const fetch = vi.fn(async () => values.shift()!);
const seen: string[] = [];
poll(fetch, { intervalMs: 1000, done: (v) => v === 'succeeded', onValue: (v) => seen.push(v), onError: () => {} });
await vi.advanceTimersByTimeAsync(5000);
expect(seen).toEqual(['queued', 'running', 'succeeded']);
expect(fetch).toHaveBeenCalledTimes(3);
});
it('never has two requests in flight when the API is slower than the interval', async () => {
let inFlight = 0;
let maxInFlight = 0;
const fetch = async () => {
inFlight++;
maxInFlight = Math.max(maxInFlight, inFlight);
await new Promise((r) => setTimeout(r, 2500));
inFlight--;
return 'running';
};
poll(fetch, { intervalMs: 1000, done: () => false, onValue: () => {}, onError: () => {} });
await vi.advanceTimersByTimeAsync(10_000);
expect(maxInFlight).toBe(1);
});
it('reports an error once and stops', async () => {
const fetch = vi.fn(async () => { throw new Error('502'); });
const onError = vi.fn();
poll(fetch, { intervalMs: 1000, done: () => false, onValue: () => {}, onError });
await vi.advanceTimersByTimeAsync(5000);
expect(onError).toHaveBeenCalledOnce();
expect(fetch).toHaveBeenCalledOnce();
});
it('stop() cancels future fetches, e.g. when the page unmounts', async () => {
const fetch = vi.fn(async () => 'running');
const stop = poll(fetch, { intervalMs: 1000, done: () => false, onValue: () => {}, onError: () => {} });
await vi.advanceTimersByTimeAsync(1500);
stop();
await vi.advanceTimersByTimeAsync(10_000);
expect(fetch).toHaveBeenCalledTimes(1);
});
});
+35
View File
@@ -0,0 +1,35 @@
export type PollOptions<T> = {
intervalMs: number;
done: (value: T) => boolean;
onValue: (value: T) => void;
onError: (error: unknown) => void;
};
/**
* Fetch every `intervalMs` until `done` returns true or a fetch fails. The next request is only
* scheduled once the previous one settles, so a slow API never has overlapping requests.
* Returns a stop function; call it when the component unmounts.
*/
export function poll<T>(fetch: () => Promise<T>, opts: PollOptions<T>): () => void {
let stopped = false;
let timer: ReturnType<typeof setTimeout> | undefined;
const tick = async () => {
let value: T;
try {
value = await fetch();
} catch (e) {
if (!stopped) opts.onError(e);
return;
}
if (stopped) return;
opts.onValue(value);
if (!opts.done(value)) timer = setTimeout(tick, opts.intervalMs);
};
timer = setTimeout(tick, opts.intervalMs);
return () => {
stopped = true;
clearTimeout(timer);
};
}
+70
View File
@@ -0,0 +1,70 @@
import { describe, expect, it, vi } from 'vitest';
import { proxyToApi } from './proxy';
// Same signature as fetch, so mock.calls stays typed.
type FetchArgs = [input: RequestInfo | URL, init?: RequestInit];
const upstreamOk = (body: unknown = { ok: true }) =>
vi.fn(async (..._args: FetchArgs) =>
new Response(JSON.stringify(body), { status: 200, headers: { 'content-type': 'application/json' } })
);
describe('proxyToApi', () => {
it('forwards the path and query to the API', async () => {
const fetch = upstreamOk();
await proxyToApi({
request: new Request('https://web.example/api/variants?job_id=abc&limit=100'),
path: 'variants',
search: '?job_id=abc&limit=100',
base: 'https://api.internal',
fetch
});
expect(fetch.mock.calls[0][0]).toBe('https://api.internal/api/variants?job_id=abc&limit=100');
});
it('forwards POST bodies', async () => {
const fetch = upstreamOk();
await proxyToApi({
request: new Request('https://web.example/api/samples', {
method: 'POST',
body: JSON.stringify({ name: 'HG002' }),
headers: { 'content-type': 'application/json' }
}),
path: 'samples',
search: '',
base: 'https://api.internal',
fetch
});
const init = fetch.mock.calls[0][1]!;
expect(init.method).toBe('POST');
expect(init.body).toBe('{"name":"HG002"}');
});
it('passes the upstream status through, so the UI sees 404s and 409s', async () => {
const fetch = vi.fn(async (..._args: FetchArgs) =>
new Response(JSON.stringify({ detail: 'sample not found' }), { status: 404, headers: { 'content-type': 'application/json' } })
);
const res = await proxyToApi({
request: new Request('https://web.example/api/jobs/x'),
path: 'jobs/x',
search: '',
base: 'https://api.internal',
fetch
});
expect(res.status).toBe(404);
expect(await res.json()).toEqual({ detail: 'sample not found' });
});
it('reports a 502 when the API cannot be reached', async () => {
const fetch = vi.fn(async (..._args: FetchArgs): Promise<Response> => { throw new Error('ECONNREFUSED'); });
const res = await proxyToApi({
request: new Request('https://web.example/api/samples'),
path: 'samples',
search: '',
base: 'https://api.internal',
fetch
});
expect(res.status).toBe(502);
expect((await res.json()).detail).toContain('ECONNREFUSED');
});
});
+35
View File
@@ -0,0 +1,35 @@
export type ProxyOptions = {
request: Request;
path: string;
search: string;
base: string;
fetch: typeof globalThis.fetch;
};
/**
* Forward /api/* to the API service.
*
* Behind the Kubernetes ingress this never runs — the ingress routes /api itself. On Cloud Run it
* keeps the UI and the API on one origin, so there is one public URL and no CORS to configure.
*/
export async function proxyToApi({ request, path, search, base, fetch }: ProxyOptions): Promise<Response> {
const hasBody = request.method !== 'GET' && request.method !== 'HEAD';
const contentType = request.headers.get('content-type') ?? 'application/json';
try {
const upstream = await fetch(`${base}/api/${path}${search}`, {
method: request.method,
headers: { 'content-type': contentType },
body: hasBody ? await request.text() : undefined
});
return new Response(upstream.body, {
status: upstream.status,
headers: { 'content-type': upstream.headers.get('content-type') ?? 'application/json' }
});
} catch (e) {
// Shaped like FastAPI's errors so the UI reports it the same way.
return new Response(JSON.stringify({ detail: `API unreachable: ${(e as Error).message}` }), {
status: 502,
headers: { 'content-type': 'application/json' }
});
}
}
+29
View File
@@ -0,0 +1,29 @@
import { describe, expect, it } from 'vitest';
import { variantQuery } from './query';
describe('variantQuery', () => {
it('always includes the job id', () => {
expect(variantQuery({ jobId: 'abc' })).toBe('job_id=abc');
});
it('includes filters that have values', () => {
const q = new URLSearchParams(
variantQuery({ jobId: 'abc', gene: ' brca1 ', impact: 'HIGH', maxAf: 0.01, limit: 100 })
);
expect(q.get('gene')).toBe('brca1');
expect(q.get('impact')).toBe('HIGH');
expect(q.get('max_af')).toBe('0.01');
expect(q.get('limit')).toBe('100');
});
it('drops a cleared number input instead of sending "null"', () => {
// Svelte binds an emptied <input type="number"> to null.
for (const maxAf of [null, undefined, '', Number.NaN]) {
expect(variantQuery({ jobId: 'abc', maxAf })).toBe('job_id=abc');
}
});
it('drops blank gene and impact', () => {
expect(variantQuery({ jobId: 'abc', gene: ' ', impact: '' })).toBe('job_id=abc');
});
});
+22
View File
@@ -0,0 +1,22 @@
export type VariantFilters = {
jobId: string;
gene?: string;
impact?: string;
// An emptied <input type="number"> binds to null, so accept it and drop it.
maxAf?: number | string | null;
limit?: number;
offset?: number;
};
export function variantQuery(f: VariantFilters): string {
const q = new URLSearchParams({ job_id: f.jobId });
const gene = f.gene?.trim();
if (gene) q.set('gene', gene);
if (f.impact) q.set('impact', f.impact);
if (f.maxAf !== null && f.maxAf !== undefined && f.maxAf !== '' && Number.isFinite(Number(f.maxAf))) {
q.set('max_af', String(f.maxAf));
}
if (f.limit !== undefined) q.set('limit', String(f.limit));
if (f.offset !== undefined) q.set('offset', String(f.offset));
return q.toString();
}
+23 -4
View File
@@ -1,18 +1,30 @@
<script lang="ts">
import { onMount } from 'svelte';
import { api, type Sample } from '$lib/api';
import { api, type Assembly, type Sample } from '$lib/api';
let samples = $state<Sample[]>([]);
let loaded = $state(false);
let loadError = $state<string | null>(null);
let name = $state('');
let vcfUri = $state('');
let assembly = $state<Assembly>('GRCh38');
let error = $state<string | null>(null);
async function refresh() { samples = await api.samples(); }
async function refresh() {
try {
samples = await api.samples();
loadError = null;
} catch (e) {
loadError = (e as Error).message;
} finally {
loaded = true;
}
}
onMount(refresh);
async function add() {
error = null;
try { await api.createSample({ name, vcf_uri: vcfUri }); name = ''; vcfUri = ''; await refresh(); }
try { await api.createSample({ name, vcf_uri: vcfUri, assembly }); name = ''; vcfUri = ''; await refresh(); }
catch (e) { error = (e as Error).message; }
}
</script>
@@ -24,12 +36,19 @@
<div style="display:flex; gap:0.5rem; flex-wrap:wrap">
<input placeholder="Sample name" bind:value={name} aria-label="Sample name" />
<input placeholder="gs://bucket/sample.vcf.gz or /data/example.vcf.gz" bind:value={vcfUri} aria-label="VCF path" style="flex:1; min-width: 20rem" />
<select bind:value={assembly} aria-label="Assembly">
<option>GRCh38</option><option>GRCh37</option>
</select>
<button onclick={add} disabled={!name || !vcfUri}>Add sample</button>
</div>
{#if error}<p role="alert">Could not add the sample: {error}</p>{/if}
<h2>Samples</h2>
{#if samples.length === 0}
{#if loadError}
<p role="alert">Could not load samples: {loadError}. <button class="quiet" onclick={refresh}>Retry</button></p>
{:else if !loaded}
<p style="color:var(--ink-soft)">Loading samples…</p>
{:else if samples.length === 0}
<div class="empty">No samples yet. Add one above to run the annotation pipeline.</div>
{:else}
<table>
+15
View File
@@ -0,0 +1,15 @@
import { env } from '$env/dynamic/private';
import { proxyToApi } from '$lib/proxy';
import type { RequestHandler } from './$types';
const handle: RequestHandler = ({ request, params, url, fetch }) =>
proxyToApi({
request,
path: params.path,
search: url.search,
base: env.API_INTERNAL_URL ?? 'http://localhost:8000',
fetch
});
export const GET = handle;
export const POST = handle;
+59 -19
View File
@@ -1,38 +1,72 @@
<script lang="ts">
import { onMount } from 'svelte';
import { onDestroy, onMount } from 'svelte';
import { api, type Job, type VariantPage } from '$lib/api';
import { poll } from '$lib/poll';
const POLL_MS = 3000;
const finished = (j: Job) => j.status === 'succeeded' || j.status === 'failed';
let { data } = $props();
let job = $state<Job | null>(null);
let page = $state<VariantPage | null>(null);
let gene = $state('');
let impact = $state('');
let maxAf = $state('0.01');
let maxAf = $state<number | null>(0.01);
let busy = $state(false);
let error = $state<string | null>(null);
let stopPolling: (() => void) | null = null;
onMount(async () => {
const jobs = await api.jobsForSample(data.sampleId);
const done = jobs.find(j => j.status === 'succeeded');
if (done) { job = done; await loadVariants(); }
else if (jobs.length) job = jobs[0];
try {
const jobs = await api.jobsForSample(data.sampleId); // newest first
const latest = jobs[0] ?? null;
const lastGood = jobs.find((j) => j.status === 'succeeded') ?? null;
// Follow a run in progress; otherwise show the latest results, else the latest failure.
job = latest && !finished(latest) ? latest : (lastGood ?? latest);
if (job) await follow(job);
} catch (e) {
error = (e as Error).message;
}
});
onDestroy(() => stopPolling?.());
async function follow(j: Job) {
stopPolling?.();
if (j.status === 'succeeded') return loadVariants();
if (finished(j)) return;
stopPolling = poll(() => api.job(j.id), {
intervalMs: POLL_MS,
done: finished,
onValue: (next) => {
job = next;
if (next.status === 'succeeded') loadVariants();
},
onError: (e) => { error = `Lost track of the job: ${(e as Error).message}`; }
});
}
async function runAnnotation() {
busy = true;
job = await api.annotate(data.sampleId);
const poll = setInterval(async () => {
if (!job) return;
job = await api.job(job.id);
if (job.status === 'succeeded' || job.status === 'failed') { clearInterval(poll); busy = false; if (job.status === 'succeeded') await loadVariants(); }
}, 3000);
error = null;
page = null;
try {
job = await api.annotate(data.sampleId);
await follow(job);
} catch (e) {
error = (e as Error).message;
} finally {
busy = false;
}
}
async function loadVariants() {
if (!job) return;
const q: Record<string, string> = { job_id: job.id, max_af: maxAf, limit: '100' };
if (gene) q.gene = gene;
if (impact) q.impact = impact;
page = await api.variants(q);
error = null;
try {
page = await api.variants({ jobId: job.id, gene, impact, maxAf, limit: 100 });
} catch (e) {
error = (e as Error).message;
}
}
const scoreClass = (s: number) => (s >= 0.8 ? 'score high' : s >= 0.5 ? 'score mid' : 'score');
@@ -41,10 +75,16 @@
<a href="/">All samples</a>
<h1>Sample {data.sampleId.slice(0, 8)}</h1>
{#if !job}
<button onclick={runAnnotation} disabled={busy}>Run VEP annotation</button>
{:else}
{#if error}<p role="alert">{error}</p>{/if}
{#if job}
<p>Job <span class="hgvs">{job.id.slice(0, 8)}</span>: <span class="status {job.status}">{job.status}</span></p>
{#if job.status === 'failed' && job.log}
<pre class="hgvs" style="white-space:pre-wrap; background:white; border:1px solid var(--line); padding:0.75rem; max-height:16rem; overflow:auto">{job.log}</pre>
{/if}
{/if}
{#if !job || job.status === 'failed'}
<button onclick={runAnnotation} disabled={busy}>{job ? 'Run VEP annotation again' : 'Run VEP annotation'}</button>
{/if}
{#if job?.status === 'succeeded'}