Kemal Yaylali
11fb6b3d73
An end-to-end audit found the repo could not build, test or run as shipped. This fixes every finding, then adds a Cloud Run track so the demo costs about £1/month idle instead of ~£150. CI (red on its first run) - api: setuptools could not build the package (flat layout with app/ and alembic/) - web: missing @types/node; `vitest run` exited 1 with no test files - pipeline: the stub run needed a gitignored VCF, and no process had a stub block - ruff pinned, mypy configured, DB tests on real Postgres (pgserver locally, service in CI) ML serving (scores were meaningless) - the registered model now carries its own feature engineering and returns predict_proba, so serving sends raw columns and cannot drift from training - resolve by registry alias (stages are deprecated in MLflow 3) and record the real version; re-scoring upserts instead of failing on the unique constraint - ClinVar labels parsed from VEP's lowercase terms Pipeline - exact ref/alt recovered from a CHROM_POS_REF_ALT VCF ID; loading is idempotent - job status reaches running/failed/succeeded, so the UI stops polling dead jobs - DATABASE_URL travels in the environment or a Nextflow secret, never on a command line - VEP cache and plugins staged as inputs; the gcp profile runs tasks on Google Batch Deployment - the API serves /api (matching the ingress); the web app reads its API URL at runtime - migrations run in an init container under a Postgres advisory lock - terraform: custom VPC shared with Batch, private Cloud SQL, API enablement, Workload Identity bindings, Secret Manager, deletion protection - serverless track, now the default: Cloud Run services scaling to zero, a Cloud Run job for the Nextflow driver, and Neon or Cloud SQL behind one DATABASE_URL secret. GKE and Argo remain, behind -var deploy_kubernetes=true. See docs/cloud.md. Correctness and security - 409 on duplicate sample names, 422 on bad paging, natural chromosome ordering, wider VEP text columns, enum dropped on downgrade, the sample's assembly actually used - vcf_uri restricted to gs:// objects or files under the data root, blocking option injection - CORS restricted to configured origins; `make down` no longer deletes volumes Data - docs/data.md records the peer-reviewed, openly licensed sources (GIAB HG002, ClinVar, gnomAD) with citations and an honest evaluation plan; `make data` fetches a chr22 slice Verified: api 50 tests, ml 18, loader 16, web 12; ruff, mypy, svelte-check, terraform validate and both kustomize overlays clean.
93 lines
4.3 KiB
Python
93 lines
4.3 KiB
Python
"""initial schema
|
|
|
|
Revision ID: a3e9ead256a5
|
|
Revises:
|
|
Create Date: 2026-09-11 15:36:04.335440
|
|
|
|
"""
|
|
from collections.abc import Sequence
|
|
|
|
import sqlalchemy as sa
|
|
from alembic import op
|
|
from sqlalchemy.dialects import postgresql
|
|
|
|
# revision identifiers, used by Alembic.
|
|
revision: str = 'a3e9ead256a5'
|
|
down_revision: str | None = None
|
|
branch_labels: str | Sequence[str] | None = None
|
|
depends_on: str | Sequence[str] | None = None
|
|
|
|
|
|
def upgrade() -> None:
|
|
# ### commands auto generated by Alembic - please adjust! ###
|
|
op.create_table('samples',
|
|
sa.Column('id', sa.UUID(), nullable=False),
|
|
sa.Column('name', sa.String(length=120), nullable=False),
|
|
sa.Column('vcf_uri', sa.Text(), nullable=False),
|
|
sa.Column('assembly', sa.String(length=10), nullable=False),
|
|
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
|
sa.PrimaryKeyConstraint('id'),
|
|
sa.UniqueConstraint('name')
|
|
)
|
|
op.create_table('jobs',
|
|
sa.Column('id', sa.UUID(), nullable=False),
|
|
sa.Column('sample_id', sa.UUID(), nullable=False),
|
|
sa.Column('status', sa.Enum('queued', 'running', 'succeeded', 'failed', name='jobstatus'), nullable=False),
|
|
sa.Column('workflow_ref', sa.String(length=200), nullable=True),
|
|
sa.Column('vep_version', sa.String(length=40), nullable=True),
|
|
sa.Column('log', sa.Text(), nullable=True),
|
|
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
|
sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True),
|
|
sa.ForeignKeyConstraint(['sample_id'], ['samples.id'], ondelete='CASCADE'),
|
|
sa.PrimaryKeyConstraint('id')
|
|
)
|
|
op.create_table('variants',
|
|
sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
|
|
sa.Column('job_id', sa.UUID(), nullable=False),
|
|
sa.Column('chrom', sa.String(length=10), nullable=False),
|
|
sa.Column('pos', sa.Integer(), nullable=False),
|
|
sa.Column('ref', sa.Text(), nullable=False),
|
|
sa.Column('alt', sa.Text(), nullable=False),
|
|
sa.Column('gene', sa.String(length=60), nullable=True),
|
|
sa.Column('consequence', sa.String(length=120), nullable=True),
|
|
sa.Column('impact', sa.String(length=20), nullable=True),
|
|
sa.Column('hgvsc', sa.Text(), nullable=True),
|
|
sa.Column('hgvsp', sa.Text(), nullable=True),
|
|
sa.Column('gnomad_af', sa.Float(), nullable=True),
|
|
sa.Column('clinvar_sig', sa.String(length=120), nullable=True),
|
|
sa.Column('annotations', postgresql.JSONB(astext_type=sa.Text()), nullable=False),
|
|
sa.ForeignKeyConstraint(['job_id'], ['jobs.id'], ondelete='CASCADE'),
|
|
sa.PrimaryKeyConstraint('id')
|
|
)
|
|
op.create_index(op.f('ix_variants_chrom'), 'variants', ['chrom'], unique=False)
|
|
op.create_index(op.f('ix_variants_gene'), 'variants', ['gene'], unique=False)
|
|
op.create_index(op.f('ix_variants_job_id'), 'variants', ['job_id'], unique=False)
|
|
op.create_index(op.f('ix_variants_pos'), 'variants', ['pos'], unique=False)
|
|
op.create_table('predictions',
|
|
sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
|
|
sa.Column('variant_id', sa.Integer(), nullable=False),
|
|
sa.Column('model_name', sa.String(length=80), nullable=False),
|
|
sa.Column('model_version', sa.String(length=40), nullable=False),
|
|
sa.Column('score', sa.Float(), nullable=False),
|
|
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
|
|
sa.ForeignKeyConstraint(['variant_id'], ['variants.id'], ondelete='CASCADE'),
|
|
sa.PrimaryKeyConstraint('id'),
|
|
sa.UniqueConstraint('variant_id')
|
|
)
|
|
# ### end Alembic commands ###
|
|
|
|
|
|
def downgrade() -> None:
|
|
# ### commands auto generated by Alembic - please adjust! ###
|
|
op.drop_table('predictions')
|
|
op.drop_index(op.f('ix_variants_pos'), table_name='variants')
|
|
op.drop_index(op.f('ix_variants_job_id'), table_name='variants')
|
|
op.drop_index(op.f('ix_variants_gene'), table_name='variants')
|
|
op.drop_index(op.f('ix_variants_chrom'), table_name='variants')
|
|
op.drop_table('variants')
|
|
op.drop_table('jobs')
|
|
op.drop_table('samples')
|
|
# ### end Alembic commands ###
|
|
# Autogenerate does not drop the type created by sa.Enum; without this, upgrading again
|
|
# fails with "type jobstatus already exists".
|
|
sa.Enum(name='jobstatus').drop(op.get_bind(), checkfirst=True) |