feat(web): show what the pipeline is doing while a job runs

"running" for two and a half minutes tells the user nothing. The job page now shows a
spinner, the elapsed time, and the pipeline step Nextflow is actually on.

- the API streams the Nextflow output into jobs.log as it arrives, instead of keeping it
  only when the run dies. Writes are throttled to one every 3s, or immediately when a new
  process starts, and are skipped for a job that has already finished, so a late line
  cannot overwrite the loader's result.
- web/src/lib/progress.ts formats the elapsed time and picks the latest [PROCESS] line.
  No percentage: the pipeline cannot honestly estimate one.
- the spinner animates only under prefers-reduced-motion: no-preference.

Verified on a live run: the page showed "VEP (tiny)" for the duration, then the variant
table replaced it on success.

Tests: api 53, web 20; ruff, mypy, svelte-check clean.
This commit is contained in:
Kemal Yaylali
2026-09-12 07:46:44 +01:00
parent ae58e33fe2
commit 33e788122b
14 changed files with 638 additions and 13 deletions
+32 -5
View File
@@ -12,11 +12,15 @@ import json
import logging
import os
import shutil
import time
import uuid
from collections import deque
from datetime import UTC, datetime
from functools import cache
from typing import Any
from sqlalchemy import update
from app.config import settings
from app.db import SessionLocal
from app.models import Job, JobStatus
@@ -24,7 +28,8 @@ from app.models import Job, JobStatus
logger = logging.getLogger(__name__)
PIPELINE_ENTRYPOINT = "/pipeline/main.nf"
LOG_TAIL_CHARS = 4000
PROGRESS_LINES = 12 # rolling window of pipeline output kept on the job
PROGRESS_INTERVAL_S = 3 # between writes, unless a new process starts
PUBLISH_TIMEOUT_S = 30
_watchers: set[asyncio.Task[None]] = set()
@@ -129,14 +134,36 @@ async def _run_local(job_id: uuid.UUID, vcf_uri: str, assembly: str) -> str:
async def _watch(job_id: uuid.UUID, proc: Any) -> None:
out, err = await proc.communicate()
if proc.returncode == 0:
"""Follow the pipeline: record progress while it runs, and the reason if it dies."""
tail: deque[str] = deque(maxlen=PROGRESS_LINES)
last_write = 0.0
if proc.stdout is not None: # stderr is merged into stdout
async for raw in proc.stdout:
line = raw.decode(errors="replace").rstrip()
if not line:
continue
tail.append(line)
now = time.monotonic()
# A new pipeline step is worth showing at once; otherwise throttle the writes.
if line.startswith("[PROCESS") or now - last_write >= PROGRESS_INTERVAL_S:
last_write = now
await _record_progress(job_id, "\n".join(tail))
returncode = await proc.wait()
if returncode == 0:
return # the loader marks the job succeeded
tail = ((out or b"") + (err or b"")).decode(errors="replace")[-LOG_TAIL_CHARS:]
async with SessionLocal() as session:
job = await session.get(Job, job_id)
if job is not None and job.status != JobStatus.succeeded:
job.status = JobStatus.failed
job.log = f"nextflow exited with code {proc.returncode}\n{tail}"
job.log = f"nextflow exited with code {returncode}\n" + "\n".join(tail)
job.finished_at = datetime.now(UTC)
await session.commit()
async def _record_progress(job_id: uuid.UUID, text: str) -> None:
"""Put the latest output on the job, leaving a job that has already finished alone."""
async with SessionLocal() as session:
await session.execute(
update(Job).where(Job.id == job_id, Job.status == JobStatus.running).values(log=text)
)
await session.commit()