P1: file parsers, sample sheet, discovery, CLI inspect
Readers for IDAT v3, BPM (locus entries v4-v8), CSV manifest, EGT (data blocks 5-9) and GTC v3-v5 (plus a layout-preserving GTC writer); sample-sheet validation (IN-001..IN-014), IDAT discovery and pairing (IN-020..IN-022), message catalogue and exit codes (CLI-014/016), JSON Lines logging (CLI-007/008), atomic writes (CLI-011), run manifest, and `simurg inspect`. Validation (docs/validation/P1.md): GTC, BPM and EGT readers agree exactly with Illumina BeadArrayFiles on GSA v4 / GDA v1 files; GTCs round-trip byte-identically; BPM and CSV manifests are consistent.
This commit is contained in:
1 parent
0a421dffc9
commit
5a20e7131c
40 files changed
+4742
No files matched your search
@@ -0,0 +1,86 @@
|
||||
"""Atomic output writing and overwrite policy (spec CLI-010, CLI-011).
|
||||
|
||||
Every output is written to ``<name>.part`` in the same directory, fsynced, then renamed, so an interrupted run
|
||||
never leaves a truncated file under its final name.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import os
|
||||
from collections.abc import Iterable, Iterator
|
||||
from pathlib import Path
|
||||
from typing import IO
|
||||
|
||||
from simurg.core.errors import SimurgError
|
||||
|
||||
PART_SUFFIX = ".part"
|
||||
|
||||
|
||||
def _fsync_dir(d: Path) -> None:
|
||||
with contextlib.suppress(OSError):
|
||||
fd = os.open(d, os.O_RDONLY)
|
||||
try:
|
||||
os.fsync(fd)
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def atomic_open(path: Path, mode: str = "wb", encoding: str | None = None) -> Iterator[IO[bytes] | IO[str]]:
|
||||
"""Open ``path.part`` for writing; on success fsync and rename to ``path``; on error remove the part file."""
|
||||
path = Path(path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = path.with_name(path.name + PART_SUFFIX)
|
||||
fh: IO[bytes] | IO[str] = (
|
||||
open(tmp, mode) # noqa: SIM115 - closed below
|
||||
if "b" in mode
|
||||
else open(tmp, mode, encoding=encoding or "utf-8", newline="") # noqa: SIM115
|
||||
)
|
||||
try:
|
||||
yield fh
|
||||
fh.flush()
|
||||
os.fsync(fh.fileno())
|
||||
fh.close()
|
||||
os.replace(tmp, path)
|
||||
_fsync_dir(path.parent)
|
||||
except BaseException:
|
||||
fh.close()
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
tmp.unlink()
|
||||
raise
|
||||
|
||||
|
||||
def atomic_write_bytes(path: Path, data: bytes) -> None:
|
||||
with atomic_open(path, "wb") as fh:
|
||||
fh.write(data) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def atomic_write_text(path: Path, text: str) -> None:
|
||||
with atomic_open(path, "w", encoding="utf-8") as fh:
|
||||
fh.write(text) # type: ignore[call-overload]
|
||||
|
||||
|
||||
def check_outputs_absent(paths: Iterable[Path], *, force: bool) -> None:
|
||||
"""Pre-flight overwrite check (CLI-010): raise E_OUTPUT_EXISTS listing up to 10 existing paths."""
|
||||
if force:
|
||||
return
|
||||
existing = [str(p) for p in paths if Path(p).exists()]
|
||||
if existing:
|
||||
shown = "\n ".join(existing[:10])
|
||||
more = f"\n … and {len(existing) - 10} more" if len(existing) > 10 else ""
|
||||
raise SimurgError(
|
||||
"E_OUTPUT_EXISTS",
|
||||
f"{len(existing)} output file(s) already exist:\n {shown}{more}",
|
||||
hint="use --force to replace them or --resume to keep completed samples",
|
||||
)
|
||||
|
||||
|
||||
def remove_stale_parts(directory: Path) -> list[Path]:
|
||||
"""Delete leftover ``*.part`` files (e.g. after a crash); returns what was removed."""
|
||||
removed = []
|
||||
for p in sorted(Path(directory).rglob(f"*{PART_SUFFIX}")):
|
||||
with contextlib.suppress(OSError):
|
||||
p.unlink()
|
||||
removed.append(p)
|
||||
return removed
|
||||
Reference in new issue
Block a user