Kemal Yaylali
11fb6b3d73
An end-to-end audit found the repo could not build, test or run as shipped. This fixes every finding, then adds a Cloud Run track so the demo costs about £1/month idle instead of ~£150. CI (red on its first run) - api: setuptools could not build the package (flat layout with app/ and alembic/) - web: missing @types/node; `vitest run` exited 1 with no test files - pipeline: the stub run needed a gitignored VCF, and no process had a stub block - ruff pinned, mypy configured, DB tests on real Postgres (pgserver locally, service in CI) ML serving (scores were meaningless) - the registered model now carries its own feature engineering and returns predict_proba, so serving sends raw columns and cannot drift from training - resolve by registry alias (stages are deprecated in MLflow 3) and record the real version; re-scoring upserts instead of failing on the unique constraint - ClinVar labels parsed from VEP's lowercase terms Pipeline - exact ref/alt recovered from a CHROM_POS_REF_ALT VCF ID; loading is idempotent - job status reaches running/failed/succeeded, so the UI stops polling dead jobs - DATABASE_URL travels in the environment or a Nextflow secret, never on a command line - VEP cache and plugins staged as inputs; the gcp profile runs tasks on Google Batch Deployment - the API serves /api (matching the ingress); the web app reads its API URL at runtime - migrations run in an init container under a Postgres advisory lock - terraform: custom VPC shared with Batch, private Cloud SQL, API enablement, Workload Identity bindings, Secret Manager, deletion protection - serverless track, now the default: Cloud Run services scaling to zero, a Cloud Run job for the Nextflow driver, and Neon or Cloud SQL behind one DATABASE_URL secret. GKE and Argo remain, behind -var deploy_kubernetes=true. See docs/cloud.md. Correctness and security - 409 on duplicate sample names, 422 on bad paging, natural chromosome ordering, wider VEP text columns, enum dropped on downgrade, the sample's assembly actually used - vcf_uri restricted to gs:// objects or files under the data root, blocking option injection - CORS restricted to configured origins; `make down` no longer deletes volumes Data - docs/data.md records the peer-reviewed, openly licensed sources (GIAB HG002, ClinVar, gnomAD) with citations and an honest evaluation plan; `make data` fetches a chr22 slice Verified: api 50 tests, ml 18, loader 16, web 12; ruff, mypy, svelte-check, terraform validate and both kustomize overlays clean.
53 lines
2.5 KiB
Plaintext
53 lines
2.5 KiB
Plaintext
params {
|
|
vcf = null
|
|
job_id = null // omit for a dry run that parses but does not load
|
|
outdir = "results"
|
|
assembly = "GRCh38"
|
|
vep_cache = "${projectDir}/cache/vep" // INSTALL.pl -a cf -s homo_sapiens -y GRCh38 -c <dir>
|
|
vep_plugin_data = null // CADD + AlphaMissense modules and data; plugins skipped when null
|
|
cadd_snv = "whole_genome_SNVs.tsv.gz"
|
|
cadd_indels = "gnomad.genomes.r4.0.indel.tsv.gz"
|
|
alphamissense = "AlphaMissense_hg38.tsv.gz"
|
|
// The driver image sets this to the loader image built from the same commit.
|
|
loader_image = System.getenv('RARELENS_LOADER_IMAGE') ?: 'rarelens/loader:dev'
|
|
|
|
// gcp profile; the Argo workflow provides these through the pipeline-config ConfigMap.
|
|
project = System.getenv('GCP_PROJECT')
|
|
region = System.getenv('GCP_REGION') ?: 'europe-west2'
|
|
bucket = System.getenv('GCS_BUCKET')
|
|
}
|
|
|
|
process {
|
|
shell = ['/bin/bash', '-euo', 'pipefail']
|
|
withName: VEP { container = 'ensemblorg/ensembl-vep:release_113.0'; cpus = 4; memory = '8 GB' }
|
|
withName: NORMALISE { container = 'quay.io/biocontainers/bcftools:1.20--h8b25389_0' }
|
|
withName: LOAD_DB { container = params.loader_image }
|
|
}
|
|
|
|
profiles {
|
|
docker {
|
|
docker.enabled = true
|
|
docker.envWhitelist = ['DATABASE_URL']
|
|
// Lets the loader reach a Postgres published on the host (docker-compose's port 5432).
|
|
docker.runOptions = '--add-host=host.docker.internal:host-gateway'
|
|
}
|
|
gcp {
|
|
// The driver runs in the Argo pod; each task runs as a Google Batch job, which is what a
|
|
// gs:// work directory requires (the k8s executor needs a shared ReadWriteMany volume).
|
|
workDir = "gs://${params.bucket}/work"
|
|
params.vep_cache = "gs://${params.bucket}/refs/vep"
|
|
google {
|
|
project = params.project
|
|
location = params.region
|
|
batch.serviceAccountEmail = "rarelens-pipeline@${params.project}.iam.gserviceaccount.com"
|
|
batch.network = "projects/${params.project}/global/networks/rarelens-vpc"
|
|
batch.subnetwork = "projects/${params.project}/regions/${params.region}/subnetworks/rarelens-gke"
|
|
}
|
|
process {
|
|
executor = 'google-batch'
|
|
// Google Secret Manager secret created by Terraform (infra/terraform/secrets.tf).
|
|
withName: LOAD_DB { secret = 'DATABASE_URL' }
|
|
}
|
|
}
|
|
}
|