feat(web): show what the pipeline is doing while a job runs
"running" for two and a half minutes tells the user nothing. The job page now shows a spinner, the elapsed time, and the pipeline step Nextflow is actually on. - the API streams the Nextflow output into jobs.log as it arrives, instead of keeping it only when the run dies. Writes are throttled to one every 3s, or immediately when a new process starts, and are skipped for a job that has already finished, so a late line cannot overwrite the loader's result. - web/src/lib/progress.ts formats the elapsed time and picks the latest [PROCESS] line. No percentage: the pipeline cannot honestly estimate one. - the spinner animates only under prefers-reduced-motion: no-preference. Verified on a live run: the page showed "VEP (tiny)" for the duration, then the variant table replaced it on success. Tests: api 53, web 20; ruff, mypy, svelte-check clean.
This commit is contained in:
@@ -12,11 +12,15 @@ import json
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import time
|
||||
import uuid
|
||||
from collections import deque
|
||||
from datetime import UTC, datetime
|
||||
from functools import cache
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import update
|
||||
|
||||
from app.config import settings
|
||||
from app.db import SessionLocal
|
||||
from app.models import Job, JobStatus
|
||||
@@ -24,7 +28,8 @@ from app.models import Job, JobStatus
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
PIPELINE_ENTRYPOINT = "/pipeline/main.nf"
|
||||
LOG_TAIL_CHARS = 4000
|
||||
PROGRESS_LINES = 12 # rolling window of pipeline output kept on the job
|
||||
PROGRESS_INTERVAL_S = 3 # between writes, unless a new process starts
|
||||
PUBLISH_TIMEOUT_S = 30
|
||||
_watchers: set[asyncio.Task[None]] = set()
|
||||
|
||||
@@ -129,14 +134,36 @@ async def _run_local(job_id: uuid.UUID, vcf_uri: str, assembly: str) -> str:
|
||||
|
||||
|
||||
async def _watch(job_id: uuid.UUID, proc: Any) -> None:
|
||||
out, err = await proc.communicate()
|
||||
if proc.returncode == 0:
|
||||
"""Follow the pipeline: record progress while it runs, and the reason if it dies."""
|
||||
tail: deque[str] = deque(maxlen=PROGRESS_LINES)
|
||||
last_write = 0.0
|
||||
if proc.stdout is not None: # stderr is merged into stdout
|
||||
async for raw in proc.stdout:
|
||||
line = raw.decode(errors="replace").rstrip()
|
||||
if not line:
|
||||
continue
|
||||
tail.append(line)
|
||||
now = time.monotonic()
|
||||
# A new pipeline step is worth showing at once; otherwise throttle the writes.
|
||||
if line.startswith("[PROCESS") or now - last_write >= PROGRESS_INTERVAL_S:
|
||||
last_write = now
|
||||
await _record_progress(job_id, "\n".join(tail))
|
||||
returncode = await proc.wait()
|
||||
if returncode == 0:
|
||||
return # the loader marks the job succeeded
|
||||
tail = ((out or b"") + (err or b"")).decode(errors="replace")[-LOG_TAIL_CHARS:]
|
||||
async with SessionLocal() as session:
|
||||
job = await session.get(Job, job_id)
|
||||
if job is not None and job.status != JobStatus.succeeded:
|
||||
job.status = JobStatus.failed
|
||||
job.log = f"nextflow exited with code {proc.returncode}\n{tail}"
|
||||
job.log = f"nextflow exited with code {returncode}\n" + "\n".join(tail)
|
||||
job.finished_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def _record_progress(job_id: uuid.UUID, text: str) -> None:
|
||||
"""Put the latest output on the job, leaving a job that has already finished alone."""
|
||||
async with SessionLocal() as session:
|
||||
await session.execute(
|
||||
update(Job).where(Job.id == job_id, Job.status == JobStatus.running).values(log=text)
|
||||
)
|
||||
await session.commit()
|
||||
|
||||
Reference in New Issue
Block a user