Initial release: rarelens platform skeleton (AGPL-3.0)

End-to-end variant interpretation platform for rare genetic disease research:
SvelteKit UI, FastAPI + PostgreSQL API, Nextflow/Ensembl VEP pipeline,
LightGBM pathogenicity scoring with MLflow, K8s/ArgoCD/GCP infrastructure.
Public test data only; no clinical claims.

pipeline/Dockerfile

@@ -0,0 +1,5 @@
+# Loader image: pandas + psycopg for LOAD_DB
+FROM python:3.12-slim
+RUN pip install --no-cache-dir pandas sqlalchemy "psycopg[binary]"
+COPY bin/load_db.py /usr/local/bin/load_db.py
+RUN chmod +x /usr/local/bin/load_db.py

pipeline/bin/load_db.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python
+"""Load a VEP --tab output into the rarelens Postgres schema and mark the job succeeded."""
+import argparse
+import json
+import sys
+
+import pandas as pd
+from sqlalchemy import create_engine, text
+
+
+def main() -> None:
+    p = argparse.ArgumentParser()
+    p.add_argument("--tsv", required=True)
+    p.add_argument("--job-id", required=True)
+    p.add_argument("--db-url", required=True)
+    a = p.parse_args()
+
+    df = pd.read_csv(a.tsv, sep="\t", comment="#", header=None, dtype=str)
+    with open(a.tsv) as fh:
+        header = next(l for l in fh if l.startswith("#Uploaded_variation")).lstrip("#").rstrip().split("\t")
+    df.columns = header
+
+    chrom_pos = df["Location"].str.split(":", expand=True)
+    rows = []
+    for i, r in df.iterrows():
+        ref, _, alt = r["Uploaded_variation"].partition("/") if "/" in r["Uploaded_variation"] else ("", "", r["Allele"])
+        rows.append({
+            "job_id": a.job_id,
+            "chrom": chrom_pos.iloc[i, 0],
+            "pos": int(chrom_pos.iloc[i, 1].split("-")[0]),
+            "ref": ref or "-",
+            "alt": r["Allele"],
+            "gene": r.get("SYMBOL") if r.get("SYMBOL") != "-" else None,
+            "consequence": r["Consequence"],
+            "impact": r["IMPACT"],
+            "hgvsc": None if r.get("HGVSc") == "-" else r.get("HGVSc"),
+            "hgvsp": None if r.get("HGVSp") == "-" else r.get("HGVSp"),
+            "gnomad_af": None if r.get("gnomADe_AF", "-") == "-" else float(r["gnomADe_AF"]),
+            "clinvar_sig": None if r.get("CLIN_SIG", "-") == "-" else r["CLIN_SIG"],
+            "annotations": json.dumps({k: v for k, v in r.items() if v != "-"}),
+        })
+
+    if a.db_url == "none":
+        print(f"{len(rows)} variants parsed (dry run, no DB)")
+        return
+
+    engine = create_engine(a.db_url.replace("+asyncpg", "+psycopg"))
+    with engine.begin() as conn:
+        conn.execute(text("""
+            INSERT INTO variants (job_id, chrom, pos, ref, alt, gene, consequence, impact,
+                                  hgvsc, hgvsp, gnomad_af, clinvar_sig, annotations)
+            VALUES (:job_id, :chrom, :pos, :ref, :alt, :gene, :consequence, :impact,
+                    :hgvsc, :hgvsp, :gnomad_af, :clinvar_sig, CAST(:annotations AS jsonb))
+        """), rows)
+        conn.execute(text("UPDATE jobs SET status='succeeded', finished_at=now() WHERE id=:id"),
+                     {"id": a.job_id})
+    print(f"loaded {len(rows)} variants for job {a.job_id}", file=sys.stderr)
+
+
+if __name__ == "__main__":
+    main()

pipeline/main.nf

@@ -0,0 +1,15 @@
+#!/usr/bin/env nextflow
+nextflow.enable.dsl = 2
+
+include { NORMALISE } from './modules/normalise'
+include { VEP }       from './modules/vep'
+include { LOAD_DB }   from './modules/load_db'
+
+workflow {
+    if (!params.vcf) error "Provide --vcf"
+    vcf_ch = Channel.fromPath(params.vcf, checkIfExists: true)
+
+    NORMALISE(vcf_ch)
+    VEP(NORMALISE.out.vcf)
+    LOAD_DB(VEP.out.tsv, params.job_id ?: 'local', params.db_url ?: 'none')
+}

pipeline/modules/load_db.nf

@@ -0,0 +1,13 @@
+process LOAD_DB {
+    tag "$job_id"
+    input:
+        path tsv
+        val job_id
+        val db_url
+    output: stdout
+
+    script:
+    """
+    load_db.py --tsv $tsv --job-id $job_id --db-url '$db_url'
+    """
+}

pipeline/modules/normalise.nf

@@ -0,0 +1,11 @@
+process NORMALISE {
+    tag "$vcf.simpleName"
+    input:  path vcf
+    output: path "${vcf.simpleName}.norm.vcf.gz", emit: vcf
+
+    script:
+    """
+    bcftools norm -m -both -Oz -o ${vcf.simpleName}.norm.vcf.gz $vcf
+    bcftools index -t ${vcf.simpleName}.norm.vcf.gz
+    """
+}

pipeline/modules/vep.nf

@@ -0,0 +1,16 @@
+process VEP {
+    tag "$vcf.simpleName"
+    publishDir params.outdir, mode: 'copy'
+    input:  path vcf
+    output:
+        path "${vcf.simpleName}.vep.tsv",      emit: tsv
+        path "${vcf.simpleName}.vep_summary.html"
+
+    script:
+    """
+    vep -i $vcf -o ${vcf.simpleName}.vep.tsv --tab \\
+        --assembly ${params.assembly} --cache --dir_cache ${params.vep_cache} --offline \\
+        --everything --pick --af_gnomade --plugin CADD --plugin AlphaMissense \\
+        --stats_file ${vcf.simpleName}.vep_summary.html --fork ${task.cpus}
+    """
+}

pipeline/nextflow.config

@@ -0,0 +1,24 @@
+params {
+    vcf        = null
+    job_id     = null
+    db_url     = null
+    outdir     = "results"
+    assembly   = "GRCh38"
+    vep_cache  = "${projectDir}/cache/vep"     // download once with `vep_install`; or use --offline false
+    vep_plugins = "CADD,AlphaMissense"
+}
+
+profiles {
+    docker { docker.enabled = true }
+    gcp {
+        process.executor = 'k8s'               // runs inside GKE via Argo; Nextflow k8s executor
+        workDir = "gs://${params.bucket}/work"
+        google.project = params.project
+    }
+}
+
+process {
+    withName: VEP        { container = 'ensemblorg/ensembl-vep:release_113.0'; cpus = 4; memory = '8 GB' }
+    withName: NORMALISE  { container = 'quay.io/biocontainers/bcftools:1.20--h8b25389_0' }
+    withName: LOAD_DB    { container = 'ghcr.io/lynchaos/rarelens-loader:latest' }
+}