P1: file parsers, sample sheet, discovery, CLI inspect

Readers for IDAT v3, BPM (locus entries v4-v8), CSV manifest, EGT
(data blocks 5-9) and GTC v3-v5 (plus a layout-preserving GTC writer);
sample-sheet validation (IN-001..IN-014), IDAT discovery and pairing
(IN-020..IN-022), message catalogue and exit codes (CLI-014/016),
JSON Lines logging (CLI-007/008), atomic writes (CLI-011), run manifest,
and `simurg inspect`.

Validation (docs/validation/P1.md): GTC, BPM and EGT readers agree
exactly with Illumina BeadArrayFiles on GSA v4 / GDA v1 files; GTCs
round-trip byte-identically; BPM and CSV manifests are consistent.

Co-Authored-By: Claude Opus 5.5 <[email protected]>
Claude-Session: https://claude.ai/code/session_01814GKVzZVL3FuF3DeiZXhv

src/simurg/core/__init__.py

@@ -0,0 +1 @@
+"""SimurgArray core package."""

src/simurg/core/atomic.py

@@ -0,0 +1,86 @@
+"""Atomic output writing and overwrite policy (spec CLI-010, CLI-011).
+
+Every output is written to ``<name>.part`` in the same directory, fsynced, then renamed, so an interrupted run
+never leaves a truncated file under its final name.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import os
+from collections.abc import Iterable, Iterator
+from pathlib import Path
+from typing import IO
+
+from simurg.core.errors import SimurgError
+
+PART_SUFFIX = ".part"
+
+
+def _fsync_dir(d: Path) -> None:
+    with contextlib.suppress(OSError):
+        fd = os.open(d, os.O_RDONLY)
+        try:
+            os.fsync(fd)
+        finally:
+            os.close(fd)
+
+
+@contextlib.contextmanager
+def atomic_open(path: Path, mode: str = "wb", encoding: str | None = None) -> Iterator[IO[bytes] | IO[str]]:
+    """Open ``path.part`` for writing; on success fsync and rename to ``path``; on error remove the part file."""
+    path = Path(path)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    tmp = path.with_name(path.name + PART_SUFFIX)
+    fh: IO[bytes] | IO[str] = (
+        open(tmp, mode)  # noqa: SIM115 - closed below
+        if "b" in mode
+        else open(tmp, mode, encoding=encoding or "utf-8", newline="")  # noqa: SIM115
+    )
+    try:
+        yield fh
+        fh.flush()
+        os.fsync(fh.fileno())
+        fh.close()
+        os.replace(tmp, path)
+        _fsync_dir(path.parent)
+    except BaseException:
+        fh.close()
+        with contextlib.suppress(FileNotFoundError):
+            tmp.unlink()
+        raise
+
+
+def atomic_write_bytes(path: Path, data: bytes) -> None:
+    with atomic_open(path, "wb") as fh:
+        fh.write(data)  # type: ignore[arg-type]
+
+
+def atomic_write_text(path: Path, text: str) -> None:
+    with atomic_open(path, "w", encoding="utf-8") as fh:
+        fh.write(text)  # type: ignore[call-overload]
+
+
+def check_outputs_absent(paths: Iterable[Path], *, force: bool) -> None:
+    """Pre-flight overwrite check (CLI-010): raise E_OUTPUT_EXISTS listing up to 10 existing paths."""
+    if force:
+        return
+    existing = [str(p) for p in paths if Path(p).exists()]
+    if existing:
+        shown = "\n  ".join(existing[:10])
+        more = f"\n  … and {len(existing) - 10} more" if len(existing) > 10 else ""
+        raise SimurgError(
+            "E_OUTPUT_EXISTS",
+            f"{len(existing)} output file(s) already exist:\n  {shown}{more}",
+            hint="use --force to replace them or --resume to keep completed samples",
+        )
+
+
+def remove_stale_parts(directory: Path) -> list[Path]:
+    """Delete leftover ``*.part`` files (e.g. after a crash); returns what was removed."""
+    removed = []
+    for p in sorted(Path(directory).rglob(f"*{PART_SUFFIX}")):
+        with contextlib.suppress(OSError):
+            p.unlink()
+            removed.append(p)
+    return removed

src/simurg/core/errors.py

@@ -0,0 +1,134 @@
+"""Stable message catalogue and exception types (spec CLI-014, CLI-016).
+
+Every error or warning SimurgArray emits has a stable ``code``. The human-readable text may change between
+versions; the code may not. ``docs/messages.md`` is generated from :data:`CATALOGUE`
+(``python -m simurg.core.errors``).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import IntEnum
+from typing import Literal
+
+Level = Literal["error", "warning", "info"]
+
+
+class ExitCode(IntEnum):
+    """Process exit codes (spec CLI-014)."""
+
+    OK = 0
+    INTERNAL = 1
+    USAGE = 2  # usage or pre-flight validation error; no per-sample work started
+    PARTIAL = 3  # >= 1 sample failed and >= 1 succeeded
+    ALL_FAILED = 4  # run completed but no sample succeeded
+    SIGINT = 130
+    SIGTERM = 143
+
+
+@dataclass(frozen=True)
+class MessageDef:
+    code: str
+    level: Level
+    summary: str
+    spec: str
+
+
+def _m(code: str, level: Level, summary: str, spec: str) -> tuple[str, MessageDef]:
+    return code, MessageDef(code, level, summary, spec)
+
+
+CATALOGUE: dict[str, MessageDef] = dict(
+    [
+        # --- CLI / run control -------------------------------------------------------------------
+        _m("E_NO_INPUT", "error", "Neither an input folder nor a sample sheet was given", "CLI-006"),
+        _m("E_NO_INPUT_FOLDER", "error", "Sample sheet has no RootFolder and no folder was given", "CLI-006"),
+        _m("E_OUTPUT_EXISTS", "error", "Outputs already exist; use --force or --resume", "CLI-010"),
+        _m("E_CONFIG_KEY", "error", "Unknown key in --config YAML", "CLI-019"),
+        _m("E_CONFIG_PARSE", "error", "--config file is not valid YAML", "CLI-019"),
+        _m("E_FILE_NOT_FOUND", "error", "Input file or folder does not exist", "CLI-016"),
+        _m("E_FILE_UNREADABLE", "error", "Input file cannot be read", "CLI-016"),
+        _m("E_INTERNAL", "error", "Internal error (bug); rerun with --debug and report", "CLI-014"),
+        # --- sample sheet (IN-001 .. IN-017) -------------------------------------------------------
+        _m("E_SHEET_DELIMITER", "error", "Sample sheet is not comma-delimited CSV", "IN-001"),
+        _m("E_SHEET_EMPTY", "error", "Sample sheet has no data rows", "IN-001"),
+        _m("E_SHEET_NO_KEY_COLUMNS", "error", "Sample sheet lacks SentrixBarcode_A/SentrixPosition_A", "IN-002"),
+        _m("E_SHEET_DUP_COLUMN", "error", "Duplicate column (column names are case-insensitive)", "IN-003"),
+        _m("W_SHEET_EMPTY_COLUMN", "warning", "Columns with empty headers are ignored", "IN-004"),
+        _m("E_SHEET_RESERVED", "error", "Sample sheet column collides with an output column", "IN-005"),
+        _m("E_SHEET_TOO_LONG", "error", "Sample sheet name or value longer than 500 characters", "IN-006"),
+        _m("E_SHEET_BARCODE_FORMAT", "error", "Invalid SentrixBarcode_A / SentrixPosition_A value", "IN-007"),
+        _m("W_SHEET_BARCODE_REPAIRED", "warning", "Barcode written as a decimal was repaired", "IN-007"),
+        _m("W_SHEET_ROOTFOLDER_OVERRIDDEN", "warning", "CLI folder overrides the sheet RootFolder", "IN-008"),
+        _m("E_SHEET_DUP_SAMPLE", "error", "The same barcode/position appears in several rows", "IN-009"),
+        _m("E_SHEET_NAME_EMPTY", "error", "Sample_Name is empty in at least one row", "IN-010"),
+        _m("E_SHEET_NAME_COLLISION", "error", "Sample names are not unique or clash with sample IDs", "IN-010"),
+        _m("W_SHEET_SEX_CONFLICT", "warning", "Reported-sex columns disagree", "IN-014"),
+        # --- IDAT / GTC discovery & parsing (IN-020 .. IN-033) --------------------------------------
+        _m("E_DUP_IDAT", "error", "Several IDAT pairs resolve to the same sample", "IN-022"),
+        _m("W_DUP_IDAT_RESOLVED", "warning", "Duplicate IDAT pair resolved by --on-duplicate-idat", "IN-022"),
+        _m("W_IDAT_UNPAIRED", "warning", "IDAT channel file without its partner", "IN-021"),
+        _m("E_IDAT_FORMAT", "error", "IDAT file is malformed", "IN-023"),
+        _m("E_IDAT_ENCRYPTED", "error", "Encrypted (version 1) IDATs are not supported", "IN-023"),
+        _m("E_IDAT_PAIR_MISMATCH", "error", "Red and Grn IDATs of a sample do not match", "IN-023"),
+        _m("W_IDAT_NAME_MISMATCH", "warning", "IDAT barcode/position disagrees with its file name", "IN-023"),
+        _m("W_IDAT_NO_SCAN_TIME", "warning", "IDAT has no scan entry in its run info", "IN-024"),
+        _m("I_UNLISTED_IDATS", "info", "IDAT pairs not listed in the sample sheet are ignored", "IN-016"),
+        _m("E_DUP_GTC", "error", "Several GTC files resolve to the same sample", "IN-030"),
+        _m("E_GTC_CORRUPT", "warning", "GTC file failed validation; sample skipped", "IN-032"),
+        _m("W_GTC_FOREIGN", "warning", "GTC was produced by another tool", "IN-031"),
+        _m("E_GTC_UNSUITABLE", "error", "GTC lacks fields required by this command", "IN-031"),
+        # --- manifests & cluster files (IN-040 .. IN-047) -------------------------------------------
+        _m("E_BPM_FORMAT", "error", "BPM manifest is malformed or unsupported", "IN-040"),
+        _m("E_CSV_MANIFEST_FORMAT", "error", "CSV manifest is malformed", "IN-041"),
+        _m("E_BPM_CSV_MISMATCH", "error", "BPM and CSV manifests describe different loci", "IN-042"),
+        _m("E_EGT_LOAD", "error", "Cluster (EGT) file cannot be loaded", "IN-044"),
+        _m("W_EGT_NAME", "warning", "Cluster file name is not portable", "IN-045"),
+        _m("W_MANIFEST_NO_REFSTRAND", "warning", "Manifest lacks RefStrand annotation", "IN-040"),
+    ]
+)
+
+
+class SimurgError(Exception):
+    """An error with a stable catalogue code and an exit code."""
+
+    def __init__(
+        self,
+        code: str,
+        message: str,
+        *,
+        exit_code: ExitCode = ExitCode.USAGE,
+        hint: str | None = None,
+        path: str | None = None,
+    ) -> None:
+        if code not in CATALOGUE:
+            raise ValueError(f"unknown message code {code!r}")
+        self.code = code
+        self.message = message
+        self.exit_code = exit_code
+        self.hint = hint
+        self.path = path
+        super().__init__(f"[{code}] {message}" + (f" (hint: {hint})" if hint else ""))
+
+
+class FormatError(SimurgError):
+    """Malformed binary/text input. ``code`` names the file type (E_IDAT_FORMAT, E_GTC_CORRUPT, ...)."""
+
+
+def render_markdown() -> str:
+    """Render the catalogue as ``docs/messages.md``."""
+    lines = [
+        "# SimurgArray message catalogue",
+        "",
+        "Generated from `src/simurg/core/errors.py` — do not edit by hand.",
+        "",
+        "| Code | Level | Meaning | Spec |",
+        "|---|---|---|---|",
+    ]
+    for d in sorted(CATALOGUE.values(), key=lambda m: m.code):
+        lines.append(f"| `{d.code}` | {d.level} | {d.summary} | {d.spec} |")
+    return "\n".join(lines) + "\n"
+
+
+if __name__ == "__main__":  # pragma: no cover
+    print(render_markdown(), end="")

src/simurg/core/log.py

@@ -0,0 +1,128 @@
+"""Structured message reporting — spec CLI-007 (logging), CLI-008 (rate limiting), CLI-009 (run summary).
+
+Messages carry a stable catalogue ``code``. Human-readable lines go to stderr; with ``--log-json`` each record is
+one JSON object per line (schema ``simurg.log/1``). A log file, when configured, always receives JSON Lines.
+"""
+
+from __future__ import annotations
+
+import datetime as _dt
+import json
+import sys
+import traceback
+from collections import Counter
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import IO, Any
+
+from simurg.core.errors import CATALOGUE
+
+LOG_SCHEMA = "simurg.log/1"
+LEVELS = {"debug": 10, "info": 20, "warning": 30, "error": 40}
+MAX_PER_SAMPLE = 20
+
+
+def _now() -> str:
+    return _dt.datetime.now(_dt.UTC).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
+
+
+@dataclass
+class Reporter:
+    command: str = ""
+    run_id: str = ""
+    level: str = "info"
+    json_stderr: bool = False
+    debug: bool = False
+    quiet: bool = False
+    log_file: Path | None = None
+    stream: IO[str] = field(default_factory=lambda: sys.stderr)
+    counts: Counter[str] = field(default_factory=Counter)
+    _per_sample: Counter[tuple[str, str]] = field(default_factory=Counter)
+    _suppressed: Counter[tuple[str, str]] = field(default_factory=Counter)
+    _fh: IO[str] | None = None
+
+    def open(self) -> None:
+        if self.log_file is not None:
+            self.log_file.parent.mkdir(parents=True, exist_ok=True)
+            self._fh = self.log_file.open("a", encoding="utf-8")
+
+    def close(self) -> None:
+        self._flush_suppressed()
+        if self._fh is not None:
+            self._fh.close()
+            self._fh = None
+
+    # -- core ---------------------------------------------------------------------------------------------
+    def emit(self, level: str, code: str | None, msg: str, *, exc: BaseException | None = None, **ctx: Any) -> None:
+        if code:
+            self.counts[code] += 1
+        sample = ctx.get("sample_id") or ""
+        if code and level == "warning":
+            key = (code, sample)
+            self._per_sample[key] += 1
+            if self._per_sample[key] > MAX_PER_SAMPLE:
+                self._suppressed[key] += 1
+                self._write_file(level, code, msg, exc, ctx)
+                return
+        record_exc = "".join(traceback.format_exception(exc)) if (exc is not None and self.debug) else None
+        self._write_file(level, code, msg, exc, ctx)
+        if LEVELS.get(level, 20) < LEVELS.get(self.level, 20):
+            return
+        if self.quiet and level in ("info", "debug"):
+            return
+        if self.json_stderr:
+            self.stream.write(json.dumps(self._record(level, code, msg, record_exc, ctx), ensure_ascii=False) + "\n")
+        else:
+            prefix = {"error": "ERROR", "warning": "WARNING", "info": "INFO", "debug": "DEBUG"}.get(level, level)
+            code_s = f" [{code}]" if code else ""
+            self.stream.write(f"{prefix}{code_s}: {msg}\n")
+            if record_exc:
+                self.stream.write(record_exc)
+        self.stream.flush()
+
+    def _record(self, level: str, code: str | None, msg: str, exc: str | None, ctx: dict[str, Any]) -> dict[str, Any]:
+        rec: dict[str, Any] = {
+            "schema": LOG_SCHEMA,
+            "ts": _now(),
+            "level": level,
+            "code": code,
+            "msg": msg,
+            "command": self.command,
+            "run_id": self.run_id,
+            "step": ctx.pop("step", None),
+            "sample_id": ctx.pop("sample_id", None),
+            "locus": ctx.pop("locus", None),
+            "file": ctx.pop("file", None),
+            "count": ctx.pop("count", 1),
+            "elapsed_ms": ctx.pop("elapsed_ms", None),
+            "exc": exc,
+        }
+        rec.update(ctx)
+        return rec
+
+    def _write_file(
+        self, level: str, code: str | None, msg: str, exc: BaseException | None, ctx: dict[str, Any]
+    ) -> None:
+        if self._fh is None:
+            return
+        ex = "".join(traceback.format_exception(exc)) if exc is not None else None
+        self._fh.write(json.dumps(self._record(level, code, msg, ex, dict(ctx)), ensure_ascii=False) + "\n")
+
+    def _flush_suppressed(self) -> None:
+        for (code, sample), n in sorted(self._suppressed.items()):
+            where = f" for {sample}" if sample else ""
+            self.emit("info", None, f"{code} repeated {n} more times{where}")
+        self._suppressed.clear()
+
+    # -- helpers ------------------------------------------------------------------------------------------
+    def warn(self, code: str, msg: str, **ctx: Any) -> None:
+        self.emit(CATALOGUE[code].level if code in CATALOGUE else "warning", code, msg, **ctx)
+
+    def info(self, msg: str, code: str | None = None, **ctx: Any) -> None:
+        self.emit("info", code, msg, **ctx)
+
+    def error(self, code: str, msg: str, *, exc: BaseException | None = None, **ctx: Any) -> None:
+        self.emit("error", code, msg, exc=exc, **ctx)
+
+    def warning_counts(self) -> dict[str, int]:
+        return dict(sorted(self.counts.items()))

src/simurg/core/provenance.py

@@ -0,0 +1,127 @@
+"""Provenance and the run manifest — plan §15, spec IN-047, OUT-073, CLI-005.
+
+Timestamps and the run ID live only in ``run_manifest.json`` so that data outputs stay byte-identical across runs.
+"""
+
+from __future__ import annotations
+
+import datetime as _dt
+import hashlib
+import json
+import os
+import platform
+import secrets
+import sys
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+
+from simurg import PRODUCT_NAME, __version__
+from simurg.core.atomic import atomic_write_text
+
+MANIFEST_SCHEMA = "simurg.run_manifest/1"
+
+
+def sha256_file(path: str | Path, chunk: int = 1 << 20) -> str:
+    h = hashlib.sha256()
+    with Path(path).open("rb") as fh:
+        while b := fh.read(chunk):
+            h.update(b)
+    return h.hexdigest()
+
+
+def new_run_id() -> str:
+    """Sortable, unique run ID: UTC timestamp + 8 random hex digits."""
+    return _dt.datetime.now(_dt.UTC).strftime("%Y%m%dT%H%M%SZ") + "-" + secrets.token_hex(4)
+
+
+def git_revision() -> str | None:
+    here = Path(__file__).resolve()
+    for parent in here.parents:
+        head = parent / ".git" / "HEAD"
+        if head.is_file():
+            ref = head.read_text().strip()
+            if ref.startswith("ref: "):
+                ref_file = parent / ".git" / ref[5:]
+                return ref_file.read_text().strip()[:12] if ref_file.is_file() else None
+            return ref[:12]
+    return None
+
+
+def environment() -> dict[str, Any]:
+    return {
+        "python": sys.version.split()[0],
+        "numpy": np.__version__,
+        "platform": platform.platform(),
+        "machine": platform.machine(),
+        "cpu_count": os.cpu_count(),
+    }
+
+
+@dataclass
+class RunManifest:
+    command: str
+    argv: list[str]
+    run_id: str = field(default_factory=new_run_id)
+    started: str = field(default_factory=lambda: _dt.datetime.now(_dt.UTC).isoformat())
+    effective_options: dict[str, dict[str, Any]] = field(default_factory=dict)
+    inputs: list[dict[str, Any]] = field(default_factory=list)
+    outputs: list[dict[str, Any]] = field(default_factory=list)
+    sheet_header: dict[str, str] = field(default_factory=dict)
+    failed_samples: list[dict[str, str]] = field(default_factory=list)
+    summary: dict[str, Any] = field(default_factory=dict)
+    schema_versions: dict[str, str] = field(default_factory=dict)
+    status: str = "running"
+    seed: int = 42
+
+    def set_option(self, name: str, value: Any, source: str) -> None:
+        """Record an effective option with its source ``cli|yaml:<path>|profile:<name>|default`` (CLI-005)."""
+        self.effective_options[name] = {"value": value, "source": source}
+
+    def add_input(self, role: str, path: str | Path, *, hash_file: bool = True) -> None:
+        p = Path(path)
+        entry: dict[str, Any] = {"role": role, "path": str(p.resolve()), "name": p.name}
+        if p.is_file():
+            entry["size"] = p.stat().st_size
+            if hash_file:
+                entry["sha256"] = sha256_file(p)
+        self.inputs.append(entry)
+
+    def add_output(self, path: str | Path, schema: str | None = None) -> None:
+        p = Path(path)
+        entry: dict[str, Any] = {"path": str(p.resolve())}
+        if p.is_file():
+            entry["size"] = p.stat().st_size
+            entry["sha256"] = sha256_file(p)
+        if schema:
+            entry["schema"] = schema
+            self.schema_versions[p.name] = schema
+        self.outputs.append(entry)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "schema": MANIFEST_SCHEMA,
+            "product": PRODUCT_NAME,
+            "version": __version__,
+            "git_revision": git_revision(),
+            "run_id": self.run_id,
+            "command": self.command,
+            "argv": self.argv,
+            "status": self.status,
+            "started": self.started,
+            "finished": _dt.datetime.now(_dt.UTC).isoformat(),
+            "seed": self.seed,
+            "environment": environment(),
+            "effective_options": self.effective_options,
+            "inputs": self.inputs,
+            "outputs": self.outputs,
+            "sheet_header": self.sheet_header,
+            "failed_samples": self.failed_samples,
+            "summary": self.summary,
+            "schema_versions": self.schema_versions,
+        }
+
+    def write(self, path: str | Path) -> None:
+        atomic_write_text(Path(path), json.dumps(self.to_dict(), indent=2, ensure_ascii=False) + "\n")