kemal
8cd092460f
Readers for IDAT v3, BPM (locus entries v4-v8), CSV manifest, EGT (data blocks 5-9) and GTC v3-v5 (plus a layout-preserving GTC writer); sample-sheet validation (IN-001..IN-014), IDAT discovery and pairing (IN-020..IN-022), message catalogue and exit codes (CLI-014/016), JSON Lines logging (CLI-007/008), atomic writes (CLI-011), run manifest, and `simurg inspect`. Validation (docs/validation/P1.md): GTC, BPM and EGT readers agree exactly with Illumina BeadArrayFiles on GSA v4 / GDA v1 files; GTCs round-trip byte-identically; BPM and CSV manifests are consistent.
128 lines
4.4 KiB
Python
128 lines
4.4 KiB
Python
"""Provenance and the run manifest — plan §15, spec IN-047, OUT-073, CLI-005.
|
|
|
|
Timestamps and the run ID live only in ``run_manifest.json`` so that data outputs stay byte-identical across runs.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import datetime as _dt
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import platform
|
|
import secrets
|
|
import sys
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import numpy as np
|
|
|
|
from simurg import PRODUCT_NAME, __version__
|
|
from simurg.core.atomic import atomic_write_text
|
|
|
|
MANIFEST_SCHEMA = "simurg.run_manifest/1"
|
|
|
|
|
|
def sha256_file(path: str | Path, chunk: int = 1 << 20) -> str:
|
|
h = hashlib.sha256()
|
|
with Path(path).open("rb") as fh:
|
|
while b := fh.read(chunk):
|
|
h.update(b)
|
|
return h.hexdigest()
|
|
|
|
|
|
def new_run_id() -> str:
|
|
"""Sortable, unique run ID: UTC timestamp + 8 random hex digits."""
|
|
return _dt.datetime.now(_dt.UTC).strftime("%Y%m%dT%H%M%SZ") + "-" + secrets.token_hex(4)
|
|
|
|
|
|
def git_revision() -> str | None:
|
|
here = Path(__file__).resolve()
|
|
for parent in here.parents:
|
|
head = parent / ".git" / "HEAD"
|
|
if head.is_file():
|
|
ref = head.read_text().strip()
|
|
if ref.startswith("ref: "):
|
|
ref_file = parent / ".git" / ref[5:]
|
|
return ref_file.read_text().strip()[:12] if ref_file.is_file() else None
|
|
return ref[:12]
|
|
return None
|
|
|
|
|
|
def environment() -> dict[str, Any]:
|
|
return {
|
|
"python": sys.version.split()[0],
|
|
"numpy": np.__version__,
|
|
"platform": platform.platform(),
|
|
"machine": platform.machine(),
|
|
"cpu_count": os.cpu_count(),
|
|
}
|
|
|
|
|
|
@dataclass
|
|
class RunManifest:
|
|
command: str
|
|
argv: list[str]
|
|
run_id: str = field(default_factory=new_run_id)
|
|
started: str = field(default_factory=lambda: _dt.datetime.now(_dt.UTC).isoformat())
|
|
effective_options: dict[str, dict[str, Any]] = field(default_factory=dict)
|
|
inputs: list[dict[str, Any]] = field(default_factory=list)
|
|
outputs: list[dict[str, Any]] = field(default_factory=list)
|
|
sheet_header: dict[str, str] = field(default_factory=dict)
|
|
failed_samples: list[dict[str, str]] = field(default_factory=list)
|
|
summary: dict[str, Any] = field(default_factory=dict)
|
|
schema_versions: dict[str, str] = field(default_factory=dict)
|
|
status: str = "running"
|
|
seed: int = 42
|
|
|
|
def set_option(self, name: str, value: Any, source: str) -> None:
|
|
"""Record an effective option with its source ``cli|yaml:<path>|profile:<name>|default`` (CLI-005)."""
|
|
self.effective_options[name] = {"value": value, "source": source}
|
|
|
|
def add_input(self, role: str, path: str | Path, *, hash_file: bool = True) -> None:
|
|
p = Path(path)
|
|
entry: dict[str, Any] = {"role": role, "path": str(p.resolve()), "name": p.name}
|
|
if p.is_file():
|
|
entry["size"] = p.stat().st_size
|
|
if hash_file:
|
|
entry["sha256"] = sha256_file(p)
|
|
self.inputs.append(entry)
|
|
|
|
def add_output(self, path: str | Path, schema: str | None = None) -> None:
|
|
p = Path(path)
|
|
entry: dict[str, Any] = {"path": str(p.resolve())}
|
|
if p.is_file():
|
|
entry["size"] = p.stat().st_size
|
|
entry["sha256"] = sha256_file(p)
|
|
if schema:
|
|
entry["schema"] = schema
|
|
self.schema_versions[p.name] = schema
|
|
self.outputs.append(entry)
|
|
|
|
def to_dict(self) -> dict[str, Any]:
|
|
return {
|
|
"schema": MANIFEST_SCHEMA,
|
|
"product": PRODUCT_NAME,
|
|
"version": __version__,
|
|
"git_revision": git_revision(),
|
|
"run_id": self.run_id,
|
|
"command": self.command,
|
|
"argv": self.argv,
|
|
"status": self.status,
|
|
"started": self.started,
|
|
"finished": _dt.datetime.now(_dt.UTC).isoformat(),
|
|
"seed": self.seed,
|
|
"environment": environment(),
|
|
"effective_options": self.effective_options,
|
|
"inputs": self.inputs,
|
|
"outputs": self.outputs,
|
|
"sheet_header": self.sheet_header,
|
|
"failed_samples": self.failed_samples,
|
|
"summary": self.summary,
|
|
"schema_versions": self.schema_versions,
|
|
}
|
|
|
|
def write(self, path: str | Path) -> None:
|
|
atomic_write_text(Path(path), json.dumps(self.to_dict(), indent=2, ensure_ascii=False) + "\n")
|