Initial release: rarelens platform skeleton (AGPL-3.0)
ci / api (push) Failing after 10s
ci / terraform (push) Failing after 11s
ci / web (push) Failing after 35s
ci / pipeline (push) Failing after 2m29s
ci / images (api) (push) Skipped
ci / images (ml) (push) Skipped
ci / images (pipeline) (push) Skipped
ci / images (web) (push) Skipped

End-to-end variant interpretation platform for rare genetic disease research:
SvelteKit UI, FastAPI + PostgreSQL API, Nextflow/Ensembl VEP pipeline,
LightGBM pathogenicity scoring with MLflow, K8s/ArgoCD/GCP infrastructure.
Public test data only; no clinical claims.
This commit is contained in:
2026-09-11 16:55:35 +01:00
commit 5463f489a3
74 changed files with 4597 additions and 0 deletions
+8
View File
@@ -0,0 +1,8 @@
FROM python:3.12-slim
WORKDIR /app
RUN pip install --no-cache-dir uv
COPY pyproject.toml .
RUN uv pip install --system -e .
COPY . .
EXPOSE 8000
CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000"]
+36
View File
@@ -0,0 +1,36 @@
[alembic]
script_location = alembic
sqlalchemy.url = postgresql+asyncpg://rarelens:rarelens@db:5432/rarelens
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S
+32
View File
@@ -0,0 +1,32 @@
import asyncio
from logging.config import fileConfig
from alembic import context
from sqlalchemy.ext.asyncio import async_engine_from_config
from sqlalchemy import pool
from app.config import settings
from app.models import Base
config = context.config
config.set_main_option("sqlalchemy.url", settings.database_url)
if config.config_file_name:
fileConfig(config.config_file_name)
target_metadata = Base.metadata
def run_migrations(connection):
context.configure(connection=connection, target_metadata=target_metadata)
with context.begin_transaction():
context.run_migrations()
async def run_async():
engine = async_engine_from_config(
config.get_section(config.config_ini_section, {}), poolclass=pool.NullPool
)
async with engine.connect() as conn:
await conn.run_sync(run_migrations)
asyncio.run(run_async())
+26
View File
@@ -0,0 +1,26 @@
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision | comma,n}
Create Date: ${create_date}
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
# revision identifiers, used by Alembic.
revision: str = ${repr(up_revision)}
down_revision: Union[str, None] = ${repr(down_revision)}
branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
def upgrade() -> None:
${upgrades if upgrades else "pass"}
def downgrade() -> None:
${downgrades if downgrades else "pass"}
@@ -0,0 +1,90 @@
"""initial schema
Revision ID: a3e9ead256a5
Revises:
Create Date: 2026-09-11 15:36:04.335440
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
# revision identifiers, used by Alembic.
revision: str = 'a3e9ead256a5'
down_revision: Union[str, None] = None
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.create_table('samples',
sa.Column('id', sa.UUID(), nullable=False),
sa.Column('name', sa.String(length=120), nullable=False),
sa.Column('vcf_uri', sa.Text(), nullable=False),
sa.Column('assembly', sa.String(length=10), nullable=False),
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('name')
)
op.create_table('jobs',
sa.Column('id', sa.UUID(), nullable=False),
sa.Column('sample_id', sa.UUID(), nullable=False),
sa.Column('status', sa.Enum('queued', 'running', 'succeeded', 'failed', name='jobstatus'), nullable=False),
sa.Column('workflow_ref', sa.String(length=200), nullable=True),
sa.Column('vep_version', sa.String(length=40), nullable=True),
sa.Column('log', sa.Text(), nullable=True),
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
sa.Column('finished_at', sa.DateTime(timezone=True), nullable=True),
sa.ForeignKeyConstraint(['sample_id'], ['samples.id'], ondelete='CASCADE'),
sa.PrimaryKeyConstraint('id')
)
op.create_table('variants',
sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
sa.Column('job_id', sa.UUID(), nullable=False),
sa.Column('chrom', sa.String(length=10), nullable=False),
sa.Column('pos', sa.Integer(), nullable=False),
sa.Column('ref', sa.Text(), nullable=False),
sa.Column('alt', sa.Text(), nullable=False),
sa.Column('gene', sa.String(length=60), nullable=True),
sa.Column('consequence', sa.String(length=120), nullable=True),
sa.Column('impact', sa.String(length=20), nullable=True),
sa.Column('hgvsc', sa.Text(), nullable=True),
sa.Column('hgvsp', sa.Text(), nullable=True),
sa.Column('gnomad_af', sa.Float(), nullable=True),
sa.Column('clinvar_sig', sa.String(length=120), nullable=True),
sa.Column('annotations', postgresql.JSONB(astext_type=sa.Text()), nullable=False),
sa.ForeignKeyConstraint(['job_id'], ['jobs.id'], ondelete='CASCADE'),
sa.PrimaryKeyConstraint('id')
)
op.create_index(op.f('ix_variants_chrom'), 'variants', ['chrom'], unique=False)
op.create_index(op.f('ix_variants_gene'), 'variants', ['gene'], unique=False)
op.create_index(op.f('ix_variants_job_id'), 'variants', ['job_id'], unique=False)
op.create_index(op.f('ix_variants_pos'), 'variants', ['pos'], unique=False)
op.create_table('predictions',
sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
sa.Column('variant_id', sa.Integer(), nullable=False),
sa.Column('model_name', sa.String(length=80), nullable=False),
sa.Column('model_version', sa.String(length=40), nullable=False),
sa.Column('score', sa.Float(), nullable=False),
sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=False),
sa.ForeignKeyConstraint(['variant_id'], ['variants.id'], ondelete='CASCADE'),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('variant_id')
)
# ### end Alembic commands ###
def downgrade() -> None:
# ### commands auto generated by Alembic - please adjust! ###
op.drop_table('predictions')
op.drop_index(op.f('ix_variants_pos'), table_name='variants')
op.drop_index(op.f('ix_variants_job_id'), table_name='variants')
op.drop_index(op.f('ix_variants_gene'), table_name='variants')
op.drop_index(op.f('ix_variants_chrom'), table_name='variants')
op.drop_table('variants')
op.drop_table('jobs')
op.drop_table('samples')
# ### end Alembic commands ###
View File
+15
View File
@@ -0,0 +1,15 @@
from pydantic_settings import BaseSettings, SettingsConfigDict
class Settings(BaseSettings):
model_config = SettingsConfigDict(env_file=".env", extra="ignore")
database_url: str = "postgresql+asyncpg://rarelens:rarelens@localhost:5432/rarelens"
mlflow_tracking_uri: str = "http://localhost:5000"
model_name: str = "rarelens-pathogenicity"
model_stage: str = "Production"
gcs_bucket: str | None = None # set in GCP; local uses ./data
pubsub_topic: str | None = None # "vcf-uploaded" in GCP; local runs pipeline inline
settings = Settings()
+13
View File
@@ -0,0 +1,13 @@
from collections.abc import AsyncIterator
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
from app.config import settings
engine = create_async_engine(settings.database_url, pool_pre_ping=True)
SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
async def get_session() -> AsyncIterator[AsyncSession]:
async with SessionLocal() as session:
yield session
+28
View File
@@ -0,0 +1,28 @@
from contextlib import asynccontextmanager
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from app.routers import jobs, predictions, samples, variants
@asynccontextmanager
async def lifespan(app: FastAPI):
# Warm the model cache here once ml/ is wired in.
yield
app = FastAPI(title="rarelens API", version="0.1.0", lifespan=lifespan)
app.add_middleware(
CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"]
)
app.include_router(samples.router, prefix="/samples", tags=["samples"])
app.include_router(jobs.router, prefix="/jobs", tags=["jobs"])
app.include_router(variants.router, prefix="/variants", tags=["variants"])
app.include_router(predictions.router, prefix="/predictions", tags=["predictions"])
@app.get("/health", tags=["ops"])
async def health() -> dict[str, str]:
return {"status": "ok"}
+77
View File
@@ -0,0 +1,77 @@
"""SQLAlchemy 2.0 declarative models.
One sample -> many jobs; one job -> many variants; one variant -> one prediction (latest).
"""
from datetime import datetime
import enum
import uuid
from sqlalchemy import DateTime, Enum, Float, ForeignKey, Integer, String, Text, func
from sqlalchemy.dialects.postgresql import JSONB, UUID
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
class Base(DeclarativeBase):
pass
class JobStatus(str, enum.Enum):
queued = "queued"
running = "running"
succeeded = "succeeded"
failed = "failed"
class Sample(Base):
__tablename__ = "samples"
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
name: Mapped[str] = mapped_column(String(120), unique=True)
vcf_uri: Mapped[str] = mapped_column(Text)
assembly: Mapped[str] = mapped_column(String(10), default="GRCh38")
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
jobs: Mapped[list["Job"]] = relationship(back_populates="sample")
class Job(Base):
__tablename__ = "jobs"
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
sample_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("samples.id", ondelete="CASCADE"))
status: Mapped[JobStatus] = mapped_column(Enum(JobStatus), default=JobStatus.queued)
workflow_ref: Mapped[str | None] = mapped_column(String(200)) # Argo workflow name / nf run id
vep_version: Mapped[str | None] = mapped_column(String(40))
log: Mapped[str | None] = mapped_column(Text)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
sample: Mapped[Sample] = relationship(back_populates="jobs")
variants: Mapped[list["Variant"]] = relationship(back_populates="job")
class Variant(Base):
__tablename__ = "variants"
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
job_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("jobs.id", ondelete="CASCADE"), index=True)
chrom: Mapped[str] = mapped_column(String(10), index=True)
pos: Mapped[int] = mapped_column(Integer, index=True)
ref: Mapped[str] = mapped_column(Text)
alt: Mapped[str] = mapped_column(Text)
gene: Mapped[str | None] = mapped_column(String(60), index=True)
consequence: Mapped[str | None] = mapped_column(String(120))
impact: Mapped[str | None] = mapped_column(String(20))
hgvsc: Mapped[str | None] = mapped_column(Text)
hgvsp: Mapped[str | None] = mapped_column(Text)
gnomad_af: Mapped[float | None] = mapped_column(Float)
clinvar_sig: Mapped[str | None] = mapped_column(String(120))
annotations: Mapped[dict] = mapped_column(JSONB, default=dict) # full VEP CSQ record
job: Mapped[Job] = relationship(back_populates="variants")
prediction: Mapped["Prediction | None"] = relationship(back_populates="variant", uselist=False)
class Prediction(Base):
__tablename__ = "predictions"
id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
variant_id: Mapped[int] = mapped_column(ForeignKey("variants.id", ondelete="CASCADE"), unique=True)
model_name: Mapped[str] = mapped_column(String(80))
model_version: Mapped[str] = mapped_column(String(40))
score: Mapped[float] = mapped_column(Float) # P(pathogenic)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
variant: Mapped[Variant] = relationship(back_populates="prediction")
View File
+18
View File
@@ -0,0 +1,18 @@
import uuid
from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy.ext.asyncio import AsyncSession
from app.db import get_session
from app.models import Job
from app.schemas import JobOut
router = APIRouter()
@router.get("/{job_id}", response_model=JobOut)
async def get_job(job_id: uuid.UUID, session: AsyncSession = Depends(get_session)):
job = await session.get(Job, job_id)
if job is None:
raise HTTPException(404, "job not found")
return job
+16
View File
@@ -0,0 +1,16 @@
import uuid
from fastapi import APIRouter, Depends
from sqlalchemy.ext.asyncio import AsyncSession
from app.db import get_session
from app.services.scoring import score_job
router = APIRouter()
@router.post("/score/{job_id}")
async def score(job_id: uuid.UUID, session: AsyncSession = Depends(get_session)) -> dict:
"""Load the registered MLflow model and score every variant of a job."""
n = await score_job(job_id, session)
return {"job_id": str(job_id), "scored": n}
+49
View File
@@ -0,0 +1,49 @@
import uuid
from fastapi import APIRouter, Depends, HTTPException, status
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.db import get_session
from app.models import Job, Sample
from app.schemas import JobOut, SampleCreate, SampleOut
from app.services.events import publish_vcf_uploaded
router = APIRouter()
@router.get("", response_model=list[SampleOut])
async def list_samples(session: AsyncSession = Depends(get_session)):
result = await session.scalars(select(Sample).order_by(Sample.created_at.desc()))
return result.all()
@router.post("", response_model=SampleOut, status_code=status.HTTP_201_CREATED)
async def create_sample(payload: SampleCreate, session: AsyncSession = Depends(get_session)):
sample = Sample(**payload.model_dump())
session.add(sample)
await session.commit()
await session.refresh(sample)
return sample
@router.get("/{sample_id}/jobs", response_model=list[JobOut])
async def list_jobs(sample_id: uuid.UUID, session: AsyncSession = Depends(get_session)):
result = await session.scalars(
select(Job).where(Job.sample_id == sample_id).order_by(Job.created_at.desc())
)
return result.all()
@router.post("/{sample_id}/annotate", response_model=JobOut, status_code=status.HTTP_202_ACCEPTED)
async def annotate(sample_id: uuid.UUID, session: AsyncSession = Depends(get_session)):
sample = await session.get(Sample, sample_id)
if sample is None:
raise HTTPException(404, "sample not found")
job = Job(sample_id=sample.id)
session.add(job)
await session.commit()
await session.refresh(job)
# Emits to Pub/Sub in GCP; runs the Nextflow pipeline inline for local dev.
await publish_vcf_uploaded(job_id=job.id, vcf_uri=sample.vcf_uri)
return job
+43
View File
@@ -0,0 +1,43 @@
import uuid
from fastapi import APIRouter, Depends, Query
from sqlalchemy import func, select
from sqlalchemy.ext.asyncio import AsyncSession
from sqlalchemy.orm import selectinload
from app.db import get_session
from app.models import Prediction, Variant
from app.schemas import VariantPage
router = APIRouter()
@router.get("", response_model=VariantPage)
async def list_variants(
job_id: uuid.UUID,
gene: str | None = None,
impact: str | None = Query(None, pattern="^(HIGH|MODERATE|LOW|MODIFIER)$"),
max_af: float | None = Query(None, ge=0, le=1),
min_score: float | None = Query(None, ge=0, le=1),
limit: int = Query(50, le=500),
offset: int = 0,
session: AsyncSession = Depends(get_session),
):
stmt = select(Variant).where(Variant.job_id == job_id)
if gene:
stmt = stmt.where(Variant.gene == gene.upper())
if impact:
stmt = stmt.where(Variant.impact == impact)
if max_af is not None:
stmt = stmt.where((Variant.gnomad_af.is_(None)) | (Variant.gnomad_af <= max_af))
if min_score is not None:
stmt = stmt.join(Prediction, Prediction.variant_id == Variant.id).where(Prediction.score >= min_score)
total = await session.scalar(select(func.count()).select_from(stmt.subquery()))
rows = await session.scalars(
stmt.options(selectinload(Variant.prediction))
.order_by(Variant.chrom, Variant.pos)
.limit(limit)
.offset(offset)
)
return VariantPage(items=rows.all(), total=total or 0, limit=limit, offset=offset)
+63
View File
@@ -0,0 +1,63 @@
from datetime import datetime
import uuid
from pydantic import BaseModel, ConfigDict, Field
from app.models import JobStatus
class ORMModel(BaseModel):
model_config = ConfigDict(from_attributes=True)
class SampleCreate(BaseModel):
name: str = Field(min_length=1, max_length=120)
vcf_uri: str
assembly: str = "GRCh38"
class SampleOut(ORMModel):
id: uuid.UUID
name: str
vcf_uri: str
assembly: str
created_at: datetime
class JobOut(ORMModel):
id: uuid.UUID
sample_id: uuid.UUID
status: JobStatus
workflow_ref: str | None
vep_version: str | None
created_at: datetime
finished_at: datetime | None
class PredictionOut(ORMModel):
model_name: str
model_version: str
score: float
class VariantOut(ORMModel):
id: int
chrom: str
pos: int
ref: str
alt: str
gene: str | None
consequence: str | None
impact: str | None
hgvsc: str | None
hgvsp: str | None
gnomad_af: float | None
clinvar_sig: str | None
prediction: PredictionOut | None = None
class VariantPage(BaseModel):
items: list[VariantOut]
total: int
limit: int
offset: int
View File
+31
View File
@@ -0,0 +1,31 @@
"""Event publishing.
GCP: publish a JSON message to Pub/Sub; an Argo Events sensor turns it into an Argo Workflow.
Local: run Nextflow directly in a background task so `docker compose up` gives a working demo.
"""
import asyncio
import json
import uuid
from app.config import settings
async def publish_vcf_uploaded(job_id: uuid.UUID, vcf_uri: str) -> None:
message = {"job_id": str(job_id), "vcf_uri": vcf_uri}
if settings.pubsub_topic:
from google.cloud import pubsub_v1 # optional dependency, installed in the GCP image
publisher = pubsub_v1.PublisherClient()
publisher.publish(settings.pubsub_topic, json.dumps(message).encode())
return
cmd = [
"nextflow", "run", "/pipeline/main.nf", "-profile", "docker",
"--vcf", vcf_uri, "--job_id", str(job_id), "--db_url", settings.database_url,
]
try:
await asyncio.create_subprocess_exec(*cmd)
except FileNotFoundError:
# Nextflow is not installed in this environment (no Java toolchain yet);
# leave the job queued instead of crashing the request.
print(f"[rarelens] nextflow not found; cannot start pipeline for job {job_id}", flush=True)
+45
View File
@@ -0,0 +1,45 @@
import uuid
import mlflow
import pandas as pd
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession
from app.config import settings
from app.models import Prediction, Variant
_model = None
def load_model():
global _model
if _model is None:
mlflow.set_tracking_uri(settings.mlflow_tracking_uri)
_model = mlflow.pyfunc.load_model(f"models:/{settings.model_name}/{settings.model_stage}")
return _model
def featurise(variants: list[Variant]) -> pd.DataFrame:
# Mirror ml/rarelens_ml/features.py exactly; shared package later.
return pd.DataFrame(
{
"impact": [v.impact for v in variants],
"consequence": [v.consequence for v in variants],
"gnomad_af": [v.gnomad_af if v.gnomad_af is not None else 0.0 for v in variants],
"cadd_phred": [v.annotations.get("CADD_PHRED") for v in variants],
"am_pathogenicity": [v.annotations.get("am_pathogenicity") for v in variants],
}
)
async def score_job(job_id: uuid.UUID, session: AsyncSession) -> int:
variants = (await session.scalars(select(Variant).where(Variant.job_id == job_id))).all()
if not variants:
return 0
model = load_model()
scores = model.predict(featurise(variants))
for v, s in zip(variants, scores):
session.add(Prediction(variant_id=v.id, model_name=settings.model_name,
model_version=settings.model_stage, score=float(s)))
await session.commit()
return len(variants)
+28
View File
@@ -0,0 +1,28 @@
[project]
name = "rarelens-api"
version = "0.1.0"
description = "FastAPI backend for rarelens"
requires-python = ">=3.12"
dependencies = [
"fastapi>=0.115",
"uvicorn[standard]>=0.30",
"sqlalchemy[asyncio]>=2.0",
"asyncpg>=0.29",
"alembic>=1.13",
"pydantic>=2.8",
"pydantic-settings>=2.4",
"httpx>=0.27",
"mlflow-skinny>=2.16",
"lightgbm>=4.5",
"pandas>=2.2",
]
[project.optional-dependencies]
dev = ["pytest", "pytest-asyncio", "ruff", "mypy", "aiosqlite"]
[tool.ruff]
line-length = 100
target-version = "py312"
[tool.pytest.ini_options]
asyncio_mode = "auto"
+12
View File
@@ -0,0 +1,12 @@
import pytest
from httpx import ASGITransport, AsyncClient
from app.main import app
@pytest.mark.asyncio
async def test_health():
async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c:
r = await c.get("/health")
assert r.status_code == 200
assert r.json() == {"status": "ok"}