Initial release: rarelens platform skeleton (AGPL-3.0)

End-to-end variant interpretation platform for rare genetic disease research:
SvelteKit UI, FastAPI + PostgreSQL API, Nextflow/Ensembl VEP pipeline,
LightGBM pathogenicity scoring with MLflow, K8s/ArgoCD/GCP infrastructure.
Public test data only; no clinical claims.

api/app/config.py

@@ -0,0 +1,15 @@
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(env_file=".env", extra="ignore")
+
+    database_url: str = "postgresql+asyncpg://rarelens:rarelens@localhost:5432/rarelens"
+    mlflow_tracking_uri: str = "http://localhost:5000"
+    model_name: str = "rarelens-pathogenicity"
+    model_stage: str = "Production"
+    gcs_bucket: str | None = None          # set in GCP; local uses ./data
+    pubsub_topic: str | None = None        # "vcf-uploaded" in GCP; local runs pipeline inline
+
+
+settings = Settings()

api/app/db.py

@@ -0,0 +1,13 @@
+from collections.abc import AsyncIterator
+
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
+
+from app.config import settings
+
+engine = create_async_engine(settings.database_url, pool_pre_ping=True)
+SessionLocal = async_sessionmaker(engine, expire_on_commit=False)
+
+
+async def get_session() -> AsyncIterator[AsyncSession]:
+    async with SessionLocal() as session:
+        yield session

api/app/main.py

@@ -0,0 +1,28 @@
+from contextlib import asynccontextmanager
+
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+
+from app.routers import jobs, predictions, samples, variants
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    # Warm the model cache here once ml/ is wired in.
+    yield
+
+
+app = FastAPI(title="rarelens API", version="0.1.0", lifespan=lifespan)
+app.add_middleware(
+    CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"]
+)
+
+app.include_router(samples.router, prefix="/samples", tags=["samples"])
+app.include_router(jobs.router, prefix="/jobs", tags=["jobs"])
+app.include_router(variants.router, prefix="/variants", tags=["variants"])
+app.include_router(predictions.router, prefix="/predictions", tags=["predictions"])
+
+
+@app.get("/health", tags=["ops"])
+async def health() -> dict[str, str]:
+    return {"status": "ok"}

api/app/models.py

@@ -0,0 +1,77 @@
+"""SQLAlchemy 2.0 declarative models.
+
+One sample -> many jobs; one job -> many variants; one variant -> one prediction (latest).
+"""
+from datetime import datetime
+import enum
+import uuid
+
+from sqlalchemy import DateTime, Enum, Float, ForeignKey, Integer, String, Text, func
+from sqlalchemy.dialects.postgresql import JSONB, UUID
+from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
+
+
+class Base(DeclarativeBase):
+    pass
+
+
+class JobStatus(str, enum.Enum):
+    queued = "queued"
+    running = "running"
+    succeeded = "succeeded"
+    failed = "failed"
+
+
+class Sample(Base):
+    __tablename__ = "samples"
+    id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    name: Mapped[str] = mapped_column(String(120), unique=True)
+    vcf_uri: Mapped[str] = mapped_column(Text)
+    assembly: Mapped[str] = mapped_column(String(10), default="GRCh38")
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+    jobs: Mapped[list["Job"]] = relationship(back_populates="sample")
+
+
+class Job(Base):
+    __tablename__ = "jobs"
+    id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
+    sample_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("samples.id", ondelete="CASCADE"))
+    status: Mapped[JobStatus] = mapped_column(Enum(JobStatus), default=JobStatus.queued)
+    workflow_ref: Mapped[str | None] = mapped_column(String(200))   # Argo workflow name / nf run id
+    vep_version: Mapped[str | None] = mapped_column(String(40))
+    log: Mapped[str | None] = mapped_column(Text)
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+    finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+    sample: Mapped[Sample] = relationship(back_populates="jobs")
+    variants: Mapped[list["Variant"]] = relationship(back_populates="job")
+
+
+class Variant(Base):
+    __tablename__ = "variants"
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
+    job_id: Mapped[uuid.UUID] = mapped_column(ForeignKey("jobs.id", ondelete="CASCADE"), index=True)
+    chrom: Mapped[str] = mapped_column(String(10), index=True)
+    pos: Mapped[int] = mapped_column(Integer, index=True)
+    ref: Mapped[str] = mapped_column(Text)
+    alt: Mapped[str] = mapped_column(Text)
+    gene: Mapped[str | None] = mapped_column(String(60), index=True)
+    consequence: Mapped[str | None] = mapped_column(String(120))
+    impact: Mapped[str | None] = mapped_column(String(20))
+    hgvsc: Mapped[str | None] = mapped_column(Text)
+    hgvsp: Mapped[str | None] = mapped_column(Text)
+    gnomad_af: Mapped[float | None] = mapped_column(Float)
+    clinvar_sig: Mapped[str | None] = mapped_column(String(120))
+    annotations: Mapped[dict] = mapped_column(JSONB, default=dict)   # full VEP CSQ record
+    job: Mapped[Job] = relationship(back_populates="variants")
+    prediction: Mapped["Prediction | None"] = relationship(back_populates="variant", uselist=False)
+
+
+class Prediction(Base):
+    __tablename__ = "predictions"
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
+    variant_id: Mapped[int] = mapped_column(ForeignKey("variants.id", ondelete="CASCADE"), unique=True)
+    model_name: Mapped[str] = mapped_column(String(80))
+    model_version: Mapped[str] = mapped_column(String(40))
+    score: Mapped[float] = mapped_column(Float)         # P(pathogenic)
+    created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
+    variant: Mapped[Variant] = relationship(back_populates="prediction")

api/app/routers/jobs.py

@@ -0,0 +1,18 @@
+import uuid
+
+from fastapi import APIRouter, Depends, HTTPException
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db import get_session
+from app.models import Job
+from app.schemas import JobOut
+
+router = APIRouter()
+
+
+@router.get("/{job_id}", response_model=JobOut)
+async def get_job(job_id: uuid.UUID, session: AsyncSession = Depends(get_session)):
+    job = await session.get(Job, job_id)
+    if job is None:
+        raise HTTPException(404, "job not found")
+    return job

api/app/routers/predictions.py

@@ -0,0 +1,16 @@
+import uuid
+
+from fastapi import APIRouter, Depends
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db import get_session
+from app.services.scoring import score_job
+
+router = APIRouter()
+
+
+@router.post("/score/{job_id}")
+async def score(job_id: uuid.UUID, session: AsyncSession = Depends(get_session)) -> dict:
+    """Load the registered MLflow model and score every variant of a job."""
+    n = await score_job(job_id, session)
+    return {"job_id": str(job_id), "scored": n}

api/app/routers/samples.py

@@ -0,0 +1,49 @@
+import uuid
+
+from fastapi import APIRouter, Depends, HTTPException, status
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.db import get_session
+from app.models import Job, Sample
+from app.schemas import JobOut, SampleCreate, SampleOut
+from app.services.events import publish_vcf_uploaded
+
+router = APIRouter()
+
+
+@router.get("", response_model=list[SampleOut])
+async def list_samples(session: AsyncSession = Depends(get_session)):
+    result = await session.scalars(select(Sample).order_by(Sample.created_at.desc()))
+    return result.all()
+
+
+@router.post("", response_model=SampleOut, status_code=status.HTTP_201_CREATED)
+async def create_sample(payload: SampleCreate, session: AsyncSession = Depends(get_session)):
+    sample = Sample(**payload.model_dump())
+    session.add(sample)
+    await session.commit()
+    await session.refresh(sample)
+    return sample
+
+
+@router.get("/{sample_id}/jobs", response_model=list[JobOut])
+async def list_jobs(sample_id: uuid.UUID, session: AsyncSession = Depends(get_session)):
+    result = await session.scalars(
+        select(Job).where(Job.sample_id == sample_id).order_by(Job.created_at.desc())
+    )
+    return result.all()
+
+
+@router.post("/{sample_id}/annotate", response_model=JobOut, status_code=status.HTTP_202_ACCEPTED)
+async def annotate(sample_id: uuid.UUID, session: AsyncSession = Depends(get_session)):
+    sample = await session.get(Sample, sample_id)
+    if sample is None:
+        raise HTTPException(404, "sample not found")
+    job = Job(sample_id=sample.id)
+    session.add(job)
+    await session.commit()
+    await session.refresh(job)
+    # Emits to Pub/Sub in GCP; runs the Nextflow pipeline inline for local dev.
+    await publish_vcf_uploaded(job_id=job.id, vcf_uri=sample.vcf_uri)
+    return job

api/app/routers/variants.py

@@ -0,0 +1,43 @@
+import uuid
+
+from fastapi import APIRouter, Depends, Query
+from sqlalchemy import func, select
+from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.orm import selectinload
+
+from app.db import get_session
+from app.models import Prediction, Variant
+from app.schemas import VariantPage
+
+router = APIRouter()
+
+
+@router.get("", response_model=VariantPage)
+async def list_variants(
+    job_id: uuid.UUID,
+    gene: str | None = None,
+    impact: str | None = Query(None, pattern="^(HIGH|MODERATE|LOW|MODIFIER)$"),
+    max_af: float | None = Query(None, ge=0, le=1),
+    min_score: float | None = Query(None, ge=0, le=1),
+    limit: int = Query(50, le=500),
+    offset: int = 0,
+    session: AsyncSession = Depends(get_session),
+):
+    stmt = select(Variant).where(Variant.job_id == job_id)
+    if gene:
+        stmt = stmt.where(Variant.gene == gene.upper())
+    if impact:
+        stmt = stmt.where(Variant.impact == impact)
+    if max_af is not None:
+        stmt = stmt.where((Variant.gnomad_af.is_(None)) | (Variant.gnomad_af <= max_af))
+    if min_score is not None:
+        stmt = stmt.join(Prediction, Prediction.variant_id == Variant.id).where(Prediction.score >= min_score)
+
+    total = await session.scalar(select(func.count()).select_from(stmt.subquery()))
+    rows = await session.scalars(
+        stmt.options(selectinload(Variant.prediction))
+        .order_by(Variant.chrom, Variant.pos)
+        .limit(limit)
+        .offset(offset)
+    )
+    return VariantPage(items=rows.all(), total=total or 0, limit=limit, offset=offset)

api/app/schemas.py

@@ -0,0 +1,63 @@
+from datetime import datetime
+import uuid
+
+from pydantic import BaseModel, ConfigDict, Field
+
+from app.models import JobStatus
+
+
+class ORMModel(BaseModel):
+    model_config = ConfigDict(from_attributes=True)
+
+
+class SampleCreate(BaseModel):
+    name: str = Field(min_length=1, max_length=120)
+    vcf_uri: str
+    assembly: str = "GRCh38"
+
+
+class SampleOut(ORMModel):
+    id: uuid.UUID
+    name: str
+    vcf_uri: str
+    assembly: str
+    created_at: datetime
+
+
+class JobOut(ORMModel):
+    id: uuid.UUID
+    sample_id: uuid.UUID
+    status: JobStatus
+    workflow_ref: str | None
+    vep_version: str | None
+    created_at: datetime
+    finished_at: datetime | None
+
+
+class PredictionOut(ORMModel):
+    model_name: str
+    model_version: str
+    score: float
+
+
+class VariantOut(ORMModel):
+    id: int
+    chrom: str
+    pos: int
+    ref: str
+    alt: str
+    gene: str | None
+    consequence: str | None
+    impact: str | None
+    hgvsc: str | None
+    hgvsp: str | None
+    gnomad_af: float | None
+    clinvar_sig: str | None
+    prediction: PredictionOut | None = None
+
+
+class VariantPage(BaseModel):
+    items: list[VariantOut]
+    total: int
+    limit: int
+    offset: int

api/app/services/events.py

@@ -0,0 +1,31 @@
+"""Event publishing.
+
+GCP: publish a JSON message to Pub/Sub; an Argo Events sensor turns it into an Argo Workflow.
+Local: run Nextflow directly in a background task so `docker compose up` gives a working demo.
+"""
+import asyncio
+import json
+import uuid
+
+from app.config import settings
+
+
+async def publish_vcf_uploaded(job_id: uuid.UUID, vcf_uri: str) -> None:
+    message = {"job_id": str(job_id), "vcf_uri": vcf_uri}
+    if settings.pubsub_topic:
+        from google.cloud import pubsub_v1  # optional dependency, installed in the GCP image
+
+        publisher = pubsub_v1.PublisherClient()
+        publisher.publish(settings.pubsub_topic, json.dumps(message).encode())
+        return
+
+    cmd = [
+        "nextflow", "run", "/pipeline/main.nf", "-profile", "docker",
+        "--vcf", vcf_uri, "--job_id", str(job_id), "--db_url", settings.database_url,
+    ]
+    try:
+        await asyncio.create_subprocess_exec(*cmd)
+    except FileNotFoundError:
+        # Nextflow is not installed in this environment (no Java toolchain yet);
+        # leave the job queued instead of crashing the request.
+        print(f"[rarelens] nextflow not found; cannot start pipeline for job {job_id}", flush=True)

api/app/services/scoring.py

@@ -0,0 +1,45 @@
+import uuid
+
+import mlflow
+import pandas as pd
+from sqlalchemy import select
+from sqlalchemy.ext.asyncio import AsyncSession
+
+from app.config import settings
+from app.models import Prediction, Variant
+
+_model = None
+
+
+def load_model():
+    global _model
+    if _model is None:
+        mlflow.set_tracking_uri(settings.mlflow_tracking_uri)
+        _model = mlflow.pyfunc.load_model(f"models:/{settings.model_name}/{settings.model_stage}")
+    return _model
+
+
+def featurise(variants: list[Variant]) -> pd.DataFrame:
+    # Mirror ml/rarelens_ml/features.py exactly; shared package later.
+    return pd.DataFrame(
+        {
+            "impact": [v.impact for v in variants],
+            "consequence": [v.consequence for v in variants],
+            "gnomad_af": [v.gnomad_af if v.gnomad_af is not None else 0.0 for v in variants],
+            "cadd_phred": [v.annotations.get("CADD_PHRED") for v in variants],
+            "am_pathogenicity": [v.annotations.get("am_pathogenicity") for v in variants],
+        }
+    )
+
+
+async def score_job(job_id: uuid.UUID, session: AsyncSession) -> int:
+    variants = (await session.scalars(select(Variant).where(Variant.job_id == job_id))).all()
+    if not variants:
+        return 0
+    model = load_model()
+    scores = model.predict(featurise(variants))
+    for v, s in zip(variants, scores):
+        session.add(Prediction(variant_id=v.id, model_name=settings.model_name,
+                               model_version=settings.model_stage, score=float(s)))
+    await session.commit()
+    return len(variants)