P1: file parsers, sample sheet, discovery, CLI inspect

Readers for IDAT v3, BPM (locus entries v4-v8), CSV manifest, EGT
(data blocks 5-9) and GTC v3-v5 (plus a layout-preserving GTC writer);
sample-sheet validation (IN-001..IN-014), IDAT discovery and pairing
(IN-020..IN-022), message catalogue and exit codes (CLI-014/016),
JSON Lines logging (CLI-007/008), atomic writes (CLI-011), run manifest,
and `simurg inspect`.

Validation (docs/validation/P1.md): GTC, BPM and EGT readers agree
exactly with Illumina BeadArrayFiles on GSA v4 / GDA v1 files; GTCs
round-trip byte-identically; BPM and CSV manifests are consistent.
This commit is contained in:
kemal committed 2026-09-27 16:56:39 +01:00
1 parent 368c97b9aa
commit 8cd092460f
40 files changed
+4742

No files matched your search

+127
View File
@@ -0,0 +1,127 @@
"""Provenance and the run manifest — plan §15, spec IN-047, OUT-073, CLI-005.
Timestamps and the run ID live only in ``run_manifest.json`` so that data outputs stay byte-identical across runs.
"""
from __future__ import annotations
import datetime as _dt
import hashlib
import json
import os
import platform
import secrets
import sys
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
import numpy as np
from simurg import PRODUCT_NAME, __version__
from simurg.core.atomic import atomic_write_text
MANIFEST_SCHEMA = "simurg.run_manifest/1"
def sha256_file(path: str | Path, chunk: int = 1 << 20) -> str:
h = hashlib.sha256()
with Path(path).open("rb") as fh:
while b := fh.read(chunk):
h.update(b)
return h.hexdigest()
def new_run_id() -> str:
"""Sortable, unique run ID: UTC timestamp + 8 random hex digits."""
return _dt.datetime.now(_dt.UTC).strftime("%Y%m%dT%H%M%SZ") + "-" + secrets.token_hex(4)
def git_revision() -> str | None:
here = Path(__file__).resolve()
for parent in here.parents:
head = parent / ".git" / "HEAD"
if head.is_file():
ref = head.read_text().strip()
if ref.startswith("ref: "):
ref_file = parent / ".git" / ref[5:]
return ref_file.read_text().strip()[:12] if ref_file.is_file() else None
return ref[:12]
return None
def environment() -> dict[str, Any]:
return {
"python": sys.version.split()[0],
"numpy": np.__version__,
"platform": platform.platform(),
"machine": platform.machine(),
"cpu_count": os.cpu_count(),
}
@dataclass
class RunManifest:
command: str
argv: list[str]
run_id: str = field(default_factory=new_run_id)
started: str = field(default_factory=lambda: _dt.datetime.now(_dt.UTC).isoformat())
effective_options: dict[str, dict[str, Any]] = field(default_factory=dict)
inputs: list[dict[str, Any]] = field(default_factory=list)
outputs: list[dict[str, Any]] = field(default_factory=list)
sheet_header: dict[str, str] = field(default_factory=dict)
failed_samples: list[dict[str, str]] = field(default_factory=list)
summary: dict[str, Any] = field(default_factory=dict)
schema_versions: dict[str, str] = field(default_factory=dict)
status: str = "running"
seed: int = 42
def set_option(self, name: str, value: Any, source: str) -> None:
"""Record an effective option with its source ``cli|yaml:<path>|profile:<name>|default`` (CLI-005)."""
self.effective_options[name] = {"value": value, "source": source}
def add_input(self, role: str, path: str | Path, *, hash_file: bool = True) -> None:
p = Path(path)
entry: dict[str, Any] = {"role": role, "path": str(p.resolve()), "name": p.name}
if p.is_file():
entry["size"] = p.stat().st_size
if hash_file:
entry["sha256"] = sha256_file(p)
self.inputs.append(entry)
def add_output(self, path: str | Path, schema: str | None = None) -> None:
p = Path(path)
entry: dict[str, Any] = {"path": str(p.resolve())}
if p.is_file():
entry["size"] = p.stat().st_size
entry["sha256"] = sha256_file(p)
if schema:
entry["schema"] = schema
self.schema_versions[p.name] = schema
self.outputs.append(entry)
def to_dict(self) -> dict[str, Any]:
return {
"schema": MANIFEST_SCHEMA,
"product": PRODUCT_NAME,
"version": __version__,
"git_revision": git_revision(),
"run_id": self.run_id,
"command": self.command,
"argv": self.argv,
"status": self.status,
"started": self.started,
"finished": _dt.datetime.now(_dt.UTC).isoformat(),
"seed": self.seed,
"environment": environment(),
"effective_options": self.effective_options,
"inputs": self.inputs,
"outputs": self.outputs,
"sheet_header": self.sheet_header,
"failed_samples": self.failed_samples,
"summary": self.summary,
"schema_versions": self.schema_versions,
}
def write(self, path: str | Path) -> None:
atomic_write_text(Path(path), json.dumps(self.to_dict(), indent=2, ensure_ascii=False) + "\n")