An end-to-end audit found the repo could not build, test or run as shipped. This fixes every finding, then adds a Cloud Run track so the demo costs about £1/month idle instead of ~£150. CI (red on its first run) - api: setuptools could not build the package (flat layout with app/ and alembic/) - web: missing @types/node; `vitest run` exited 1 with no test files - pipeline: the stub run needed a gitignored VCF, and no process had a stub block - ruff pinned, mypy configured, DB tests on real Postgres (pgserver locally, service in CI) ML serving (scores were meaningless) - the registered model now carries its own feature engineering and returns predict_proba, so serving sends raw columns and cannot drift from training - resolve by registry alias (stages are deprecated in MLflow 3) and record the real version; re-scoring upserts instead of failing on the unique constraint - ClinVar labels parsed from VEP's lowercase terms Pipeline - exact ref/alt recovered from a CHROM_POS_REF_ALT VCF ID; loading is idempotent - job status reaches running/failed/succeeded, so the UI stops polling dead jobs - DATABASE_URL travels in the environment or a Nextflow secret, never on a command line - VEP cache and plugins staged as inputs; the gcp profile runs tasks on Google Batch Deployment - the API serves /api (matching the ingress); the web app reads its API URL at runtime - migrations run in an init container under a Postgres advisory lock - terraform: custom VPC shared with Batch, private Cloud SQL, API enablement, Workload Identity bindings, Secret Manager, deletion protection - serverless track, now the default: Cloud Run services scaling to zero, a Cloud Run job for the Nextflow driver, and Neon or Cloud SQL behind one DATABASE_URL secret. GKE and Argo remain, behind -var deploy_kubernetes=true. See docs/cloud.md. Correctness and security - 409 on duplicate sample names, 422 on bad paging, natural chromosome ordering, wider VEP text columns, enum dropped on downgrade, the sample's assembly actually used - vcf_uri restricted to gs:// objects or files under the data root, blocking option injection - CORS restricted to configured origins; `make down` no longer deletes volumes Data - docs/data.md records the peer-reviewed, openly licensed sources (GIAB HG002, ClinVar, gnomAD) with citations and an honest evaluation plan; `make data` fetches a chr22 slice Verified: api 50 tests, ml 18, loader 16, web 12; ruff, mypy, svelte-check, terraform validate and both kustomize overlays clean.
179 lines
5.5 KiB
Terraform
179 lines
5.5 KiB
Terraform
# The serverless track: scale-to-zero services and an on-demand pipeline driver.
|
|
# Idle cost is storage only; see docs/cloud.md.
|
|
|
|
resource "google_cloud_run_v2_service" "api" {
|
|
name = "rarelens-api"
|
|
location = var.region
|
|
deletion_protection = false
|
|
ingress = "INGRESS_TRAFFIC_ALL"
|
|
|
|
template {
|
|
service_account = google_service_account.api.email
|
|
scaling {
|
|
min_instance_count = 0 # nothing runs, and nothing is billed, between visits
|
|
max_instance_count = var.max_instances
|
|
}
|
|
containers {
|
|
image = "${local.registry}/api:${var.image_tag}"
|
|
ports { container_port = 8000 }
|
|
resources {
|
|
limits = { cpu = "1", memory = "1Gi" }
|
|
cpu_idle = true # bill CPU only while a request is in flight
|
|
startup_cpu_boost = true
|
|
}
|
|
env {
|
|
name = "DATABASE_URL"
|
|
value_source {
|
|
secret_key_ref {
|
|
secret = google_secret_manager_secret.api_database_url.secret_id
|
|
version = "latest"
|
|
}
|
|
}
|
|
}
|
|
env {
|
|
name = "CLOUDRUN_JOB"
|
|
value = google_cloud_run_v2_job.nextflow.name
|
|
}
|
|
env {
|
|
name = "GCP_PROJECT"
|
|
value = var.project
|
|
}
|
|
env {
|
|
name = "GCP_REGION"
|
|
value = var.region
|
|
}
|
|
env {
|
|
name = "GCS_BUCKET"
|
|
value = google_storage_bucket.data.name
|
|
}
|
|
env {
|
|
name = "MODEL_URI"
|
|
value = var.model_uri
|
|
}
|
|
}
|
|
}
|
|
depends_on = [google_secret_manager_secret_version.api_database_url]
|
|
}
|
|
|
|
resource "google_cloud_run_v2_service" "web" {
|
|
name = "rarelens-web"
|
|
location = var.region
|
|
deletion_protection = false
|
|
ingress = "INGRESS_TRAFFIC_ALL"
|
|
|
|
template {
|
|
scaling {
|
|
min_instance_count = 0
|
|
max_instance_count = var.max_instances
|
|
}
|
|
containers {
|
|
image = "${local.registry}/web:${var.image_tag}"
|
|
ports { container_port = 3000 }
|
|
resources {
|
|
limits = { cpu = "1", memory = "512Mi" }
|
|
cpu_idle = true
|
|
startup_cpu_boost = true
|
|
}
|
|
# The browser calls /api on this origin; src/routes/api/[...path] forwards it, so there is
|
|
# one public URL and no CORS, exactly as the ingress arranges in the Kubernetes track.
|
|
env {
|
|
name = "PUBLIC_API_URL"
|
|
value = "/api"
|
|
}
|
|
env {
|
|
name = "API_INTERNAL_URL"
|
|
value = google_cloud_run_v2_service.api.uri
|
|
}
|
|
# adapter-node sits behind Cloud Run's proxy; derive the origin from the forwarded headers.
|
|
env {
|
|
name = "PROTOCOL_HEADER"
|
|
value = "x-forwarded-proto"
|
|
}
|
|
env {
|
|
name = "HOST_HEADER"
|
|
value = "x-forwarded-host"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
# The Nextflow driver. Started per annotation by the API (overriding the container args); the
|
|
# pipeline's own tasks then run on Google Batch (the gcp profile in pipeline/nextflow.config).
|
|
resource "google_cloud_run_v2_job" "nextflow" {
|
|
name = "rarelens-nextflow"
|
|
location = var.region
|
|
deletion_protection = false
|
|
|
|
template {
|
|
task_count = 1
|
|
template {
|
|
service_account = google_service_account.pipeline.email
|
|
max_retries = 0
|
|
timeout = "7200s"
|
|
containers {
|
|
image = "${local.registry}/pipeline:${var.image_tag}"
|
|
args = ["-version"] # replaced on every execution by the API's overrides
|
|
resources {
|
|
limits = { cpu = "1", memory = "2Gi" }
|
|
}
|
|
env {
|
|
name = "GCP_PROJECT"
|
|
value = var.project
|
|
}
|
|
env {
|
|
name = "GCP_REGION"
|
|
value = var.region
|
|
}
|
|
env {
|
|
name = "GCS_BUCKET"
|
|
value = google_storage_bucket.data.name
|
|
}
|
|
env {
|
|
name = "NXF_ANSI_LOG"
|
|
value = "false"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
# Anyone can open the UI and the API. There is no authentication by design (docs/architecture.md);
|
|
# max_instances and a billing budget are what bound the cost.
|
|
resource "google_cloud_run_v2_service_iam_member" "web_public" {
|
|
project = var.project
|
|
location = google_cloud_run_v2_service.web.location
|
|
name = google_cloud_run_v2_service.web.name
|
|
role = "roles/run.invoker"
|
|
member = "allUsers"
|
|
}
|
|
|
|
resource "google_cloud_run_v2_service_iam_member" "api_public" {
|
|
project = var.project
|
|
location = google_cloud_run_v2_service.api.location
|
|
name = google_cloud_run_v2_service.api.name
|
|
role = "roles/run.invoker"
|
|
member = "allUsers"
|
|
}
|
|
|
|
# Least privilege: the API may execute this one job with argument overrides, nothing more.
|
|
resource "google_cloud_run_v2_job_iam_member" "api_runs_nextflow" {
|
|
project = var.project
|
|
location = google_cloud_run_v2_job.nextflow.location
|
|
name = google_cloud_run_v2_job.nextflow.name
|
|
role = "roles/run.jobsExecutorWithOverrides"
|
|
member = "serviceAccount:${google_service_account.api.email}"
|
|
}
|
|
|
|
resource "google_secret_manager_secret_iam_member" "api_database_url" {
|
|
secret_id = google_secret_manager_secret.api_database_url.secret_id
|
|
role = "roles/secretmanager.secretAccessor"
|
|
member = "serviceAccount:${google_service_account.api.email}"
|
|
}
|
|
|
|
# Reading the model artifact from gs://<bucket>/models/... when MODEL_URI is set.
|
|
resource "google_storage_bucket_iam_member" "api_reads_data" {
|
|
bucket = google_storage_bucket.data.name
|
|
role = "roles/storage.objectViewer"
|
|
member = "serviceAccount:${google_service_account.api.email}"
|
|
}
|