Kemal Yaylali
11fb6b3d73
An end-to-end audit found the repo could not build, test or run as shipped. This fixes every finding, then adds a Cloud Run track so the demo costs about £1/month idle instead of ~£150. CI (red on its first run) - api: setuptools could not build the package (flat layout with app/ and alembic/) - web: missing @types/node; `vitest run` exited 1 with no test files - pipeline: the stub run needed a gitignored VCF, and no process had a stub block - ruff pinned, mypy configured, DB tests on real Postgres (pgserver locally, service in CI) ML serving (scores were meaningless) - the registered model now carries its own feature engineering and returns predict_proba, so serving sends raw columns and cannot drift from training - resolve by registry alias (stages are deprecated in MLflow 3) and record the real version; re-scoring upserts instead of failing on the unique constraint - ClinVar labels parsed from VEP's lowercase terms Pipeline - exact ref/alt recovered from a CHROM_POS_REF_ALT VCF ID; loading is idempotent - job status reaches running/failed/succeeded, so the UI stops polling dead jobs - DATABASE_URL travels in the environment or a Nextflow secret, never on a command line - VEP cache and plugins staged as inputs; the gcp profile runs tasks on Google Batch Deployment - the API serves /api (matching the ingress); the web app reads its API URL at runtime - migrations run in an init container under a Postgres advisory lock - terraform: custom VPC shared with Batch, private Cloud SQL, API enablement, Workload Identity bindings, Secret Manager, deletion protection - serverless track, now the default: Cloud Run services scaling to zero, a Cloud Run job for the Nextflow driver, and Neon or Cloud SQL behind one DATABASE_URL secret. GKE and Argo remain, behind -var deploy_kubernetes=true. See docs/cloud.md. Correctness and security - 409 on duplicate sample names, 422 on bad paging, natural chromosome ordering, wider VEP text columns, enum dropped on downgrade, the sample's assembly actually used - vcf_uri restricted to gs:// objects or files under the data root, blocking option injection - CORS restricted to configured origins; `make down` no longer deletes volumes Data - docs/data.md records the peer-reviewed, openly licensed sources (GIAB HG002, ClinVar, gnomAD) with citations and an honest evaluation plan; `make data` fetches a chr22 slice Verified: api 50 tests, ml 18, loader 16, web 12; ruff, mypy, svelte-check, terraform validate and both kustomize overlays clean.
66 lines
1.8 KiB
TOML
66 lines
1.8 KiB
TOML
[build-system]
|
|
requires = ["setuptools>=69"]
|
|
build-backend = "setuptools.build_meta"
|
|
|
|
[project]
|
|
name = "rarelens-api"
|
|
version = "0.1.0"
|
|
description = "FastAPI backend for rarelens"
|
|
requires-python = ">=3.12"
|
|
dependencies = [
|
|
"fastapi>=0.115",
|
|
"uvicorn[standard]>=0.30",
|
|
"sqlalchemy[asyncio]>=2.0",
|
|
"asyncpg>=0.29",
|
|
"alembic>=1.13",
|
|
"pydantic>=2.8",
|
|
"pydantic-settings>=2.4",
|
|
"httpx>=0.27",
|
|
# mlflow major must match rarelens-ml and the tracking server image.
|
|
"mlflow-skinny>=3,<4",
|
|
"lightgbm>=4.5",
|
|
"scikit-learn>=1.5", # the pickled LGBMClassifier inside the pyfunc needs it to load
|
|
"pandas>=2.2",
|
|
]
|
|
|
|
[project.optional-dependencies]
|
|
# pubsub: Kubernetes track; run: serverless track; storage: load a model from gs://
|
|
gcp = ["google-cloud-pubsub>=2.23", "google-cloud-run>=0.10", "google-cloud-storage>=2.18"]
|
|
dev = [
|
|
"pytest>=8",
|
|
"pytest-asyncio>=1.0",
|
|
"ruff==0.16.2",
|
|
"mypy>=1.11",
|
|
"pandas-stubs",
|
|
# Throwaway Postgres for `pytest` when DATABASE_URL is unset (no Docker needed).
|
|
"pgserver; sys_platform != 'win32'",
|
|
]
|
|
|
|
# The repo also has top-level `alembic/` and `tests/` dirs; only `app` is the package.
|
|
[tool.setuptools.packages.find]
|
|
include = ["app*"]
|
|
|
|
[tool.ruff]
|
|
line-length = 100
|
|
target-version = "py312"
|
|
|
|
[tool.ruff.lint.isort]
|
|
# Without this the local `alembic/` migrations dir makes ruff treat the alembic library as first-party.
|
|
known-third-party = ["alembic"]
|
|
|
|
[tool.mypy]
|
|
python_version = "3.12"
|
|
|
|
[[tool.mypy.overrides]]
|
|
module = ["google.cloud.*", "mlflow.*"]
|
|
ignore_missing_imports = true
|
|
|
|
[tool.pytest.ini_options]
|
|
asyncio_mode = "auto"
|
|
# asyncpg connections are bound to the loop that opened them; one loop for the whole run
|
|
# lets the app's pooled engine be shared across tests.
|
|
asyncio_default_fixture_loop_scope = "session"
|
|
asyncio_default_test_loop_scope = "session"
|
|
pythonpath = ["."]
|
|
testpaths = ["tests"]
|