P1: file parsers, sample sheet, discovery, CLI inspect
Readers for IDAT v3, BPM (locus entries v4-v8), CSV manifest, EGT (data blocks 5-9) and GTC v3-v5 (plus a layout-preserving GTC writer); sample-sheet validation (IN-001..IN-014), IDAT discovery and pairing (IN-020..IN-022), message catalogue and exit codes (CLI-014/016), JSON Lines logging (CLI-007/008), atomic writes (CLI-011), run manifest, and `simurg inspect`. Validation (docs/validation/P1.md): GTC, BPM and EGT readers agree exactly with Illumina BeadArrayFiles on GSA v4 / GDA v1 files; GTCs round-trip byte-identically; BPM and CSV manifests are consistent.
This commit is contained in:
1 parent
368c97b9aa
commit
8cd092460f
40 files changed
+4742
No files matched your search
@@ -0,0 +1,127 @@
|
||||
"""Provenance and the run manifest — plan §15, spec IN-047, OUT-073, CLI-005.
|
||||
|
||||
Timestamps and the run ID live only in ``run_manifest.json`` so that data outputs stay byte-identical across runs.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as _dt
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import secrets
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
from simurg import PRODUCT_NAME, __version__
|
||||
from simurg.core.atomic import atomic_write_text
|
||||
|
||||
MANIFEST_SCHEMA = "simurg.run_manifest/1"
|
||||
|
||||
|
||||
def sha256_file(path: str | Path, chunk: int = 1 << 20) -> str:
|
||||
h = hashlib.sha256()
|
||||
with Path(path).open("rb") as fh:
|
||||
while b := fh.read(chunk):
|
||||
h.update(b)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def new_run_id() -> str:
|
||||
"""Sortable, unique run ID: UTC timestamp + 8 random hex digits."""
|
||||
return _dt.datetime.now(_dt.UTC).strftime("%Y%m%dT%H%M%SZ") + "-" + secrets.token_hex(4)
|
||||
|
||||
|
||||
def git_revision() -> str | None:
|
||||
here = Path(__file__).resolve()
|
||||
for parent in here.parents:
|
||||
head = parent / ".git" / "HEAD"
|
||||
if head.is_file():
|
||||
ref = head.read_text().strip()
|
||||
if ref.startswith("ref: "):
|
||||
ref_file = parent / ".git" / ref[5:]
|
||||
return ref_file.read_text().strip()[:12] if ref_file.is_file() else None
|
||||
return ref[:12]
|
||||
return None
|
||||
|
||||
|
||||
def environment() -> dict[str, Any]:
|
||||
return {
|
||||
"python": sys.version.split()[0],
|
||||
"numpy": np.__version__,
|
||||
"platform": platform.platform(),
|
||||
"machine": platform.machine(),
|
||||
"cpu_count": os.cpu_count(),
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class RunManifest:
|
||||
command: str
|
||||
argv: list[str]
|
||||
run_id: str = field(default_factory=new_run_id)
|
||||
started: str = field(default_factory=lambda: _dt.datetime.now(_dt.UTC).isoformat())
|
||||
effective_options: dict[str, dict[str, Any]] = field(default_factory=dict)
|
||||
inputs: list[dict[str, Any]] = field(default_factory=list)
|
||||
outputs: list[dict[str, Any]] = field(default_factory=list)
|
||||
sheet_header: dict[str, str] = field(default_factory=dict)
|
||||
failed_samples: list[dict[str, str]] = field(default_factory=list)
|
||||
summary: dict[str, Any] = field(default_factory=dict)
|
||||
schema_versions: dict[str, str] = field(default_factory=dict)
|
||||
status: str = "running"
|
||||
seed: int = 42
|
||||
|
||||
def set_option(self, name: str, value: Any, source: str) -> None:
|
||||
"""Record an effective option with its source ``cli|yaml:<path>|profile:<name>|default`` (CLI-005)."""
|
||||
self.effective_options[name] = {"value": value, "source": source}
|
||||
|
||||
def add_input(self, role: str, path: str | Path, *, hash_file: bool = True) -> None:
|
||||
p = Path(path)
|
||||
entry: dict[str, Any] = {"role": role, "path": str(p.resolve()), "name": p.name}
|
||||
if p.is_file():
|
||||
entry["size"] = p.stat().st_size
|
||||
if hash_file:
|
||||
entry["sha256"] = sha256_file(p)
|
||||
self.inputs.append(entry)
|
||||
|
||||
def add_output(self, path: str | Path, schema: str | None = None) -> None:
|
||||
p = Path(path)
|
||||
entry: dict[str, Any] = {"path": str(p.resolve())}
|
||||
if p.is_file():
|
||||
entry["size"] = p.stat().st_size
|
||||
entry["sha256"] = sha256_file(p)
|
||||
if schema:
|
||||
entry["schema"] = schema
|
||||
self.schema_versions[p.name] = schema
|
||||
self.outputs.append(entry)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"schema": MANIFEST_SCHEMA,
|
||||
"product": PRODUCT_NAME,
|
||||
"version": __version__,
|
||||
"git_revision": git_revision(),
|
||||
"run_id": self.run_id,
|
||||
"command": self.command,
|
||||
"argv": self.argv,
|
||||
"status": self.status,
|
||||
"started": self.started,
|
||||
"finished": _dt.datetime.now(_dt.UTC).isoformat(),
|
||||
"seed": self.seed,
|
||||
"environment": environment(),
|
||||
"effective_options": self.effective_options,
|
||||
"inputs": self.inputs,
|
||||
"outputs": self.outputs,
|
||||
"sheet_header": self.sheet_header,
|
||||
"failed_samples": self.failed_samples,
|
||||
"summary": self.summary,
|
||||
"schema_versions": self.schema_versions,
|
||||
}
|
||||
|
||||
def write(self, path: str | Path) -> None:
|
||||
atomic_write_text(Path(path), json.dumps(self.to_dict(), indent=2, ensure_ascii=False) + "\n")
|
||||
Reference in new issue
Block a user