P2 (WIP): raw X/Y assembly, canonical Parquet model, simurg ingest

- signal/raw_xy.py: vectorised Infinium I/II raw X/Y and control assembly
  (plan §7.1) with a per-chip address-map cache; IN-027 coverage gate
- core/schemas.py: Arrow schemas (loci, sample_intensity, controls),
  deterministic atomic Parquet writing
- pipeline/ingest.py + CLI: discovery, sheet, IN-010 naming, overwrite
  check before any work, spawn-based workers, per-sample isolation,
  --resume by input hashes, run manifest, exit codes 0/3/4
- matches gtc2vcf raw X/Y and controls exactly on 18 GSA samples;
  outputs byte-identical across thread counts and layouts
This commit is contained in:
kemal committed 2026-09-27 20:54:06 +01:00
1 parent a7a235beb5
commit 87f9b3499c
9 files changed
+970 -2

No files matched your search

+50 -1
View File
@@ -1,6 +1,6 @@
"""``simurg`` command-line interface (spec §2, plan §14).
Phase P1 provides ``simurg version`` and ``simurg inspect``. Later phases add the other commands of CLI-001.
Available: ``version``, ``inspect`` (P1), ``ingest`` (P2). Later phases add the other commands of CLI-001.
"""
from __future__ import annotations
@@ -203,5 +203,54 @@ def inspect_cmd(
run_guarded(rep, body)
# --------------------------------------------------------------------------------------------------------
@app.command("ingest")
def ingest_cmd(
bpm: Annotated[Path, typer.Option("--bpm", help="BPM manifest (required).")],
out: Annotated[Path, typer.Option("--out", help="Output folder (required).")],
idats: Annotated[Path | None, typer.Option("--idats", help="Folder searched recursively for IDAT pairs.")] = None,
sheet: Annotated[Path | None, typer.Option("--sheet", help="Sample sheet (CSV).")] = None,
csv: Annotated[Path | None, typer.Option("--csv", help="CSV manifest, checked against the BPM.")] = None,
threads: Annotated[int, typer.Option("--threads", min=0, max=1024, help="Worker processes (0 = auto).")] = 0,
force: Annotated[bool, typer.Option("--force", help="Replace existing outputs.")] = False,
resume: Annotated[bool, typer.Option("--resume", help="Keep outputs whose inputs are unchanged.")] = False,
layout: Annotated[str, typer.Option("--layout", help="nested|flat")] = "nested",
on_duplicate_idat: Annotated[str, typer.Option("--on-duplicate-idat", help="error|first|newest")] = "error",
log_json: Annotated[bool, typer.Option("--log-json", "--json-log", help="JSON Lines log on stderr.")] = False,
debug: Annotated[bool, typer.Option("--debug", help="Debug logging with tracebacks.")] = False,
log_level: Annotated[str, typer.Option("--log-level", help="debug|info|warning|error")] = "info",
quiet: Annotated[bool, typer.Option("--quiet", "--no-progress", help="Suppress info messages.")] = False,
) -> None:
"""IDAT pairs + BPM → canonical per-sample raw intensities (Parquet). Phase P2."""
from simurg.pipeline.ingest import IngestOptions, run_ingest
if layout not in ("nested", "flat"):
raise typer.BadParameter("--layout must be nested or flat")
if on_duplicate_idat not in ("error", "first", "newest"):
raise typer.BadParameter("--on-duplicate-idat must be error, first or newest")
rep = Reporter(
command="ingest",
level="debug" if debug else log_level,
json_stderr=log_json,
debug=debug,
quiet=quiet,
log_file=out / "logs" / "ingest.log",
)
opts = IngestOptions(
out=out,
bpm=bpm,
idats=idats,
sheet=sheet,
csv=csv,
threads=threads,
force=force,
resume=resume,
layout=layout,
on_duplicate_idat=on_duplicate_idat,
argv=sys.argv[1:],
)
run_guarded(rep, lambda: run_ingest(opts, rep))
if __name__ == "__main__": # pragma: no cover
app()