P1: file parsers, sample sheet, discovery, CLI inspect

Readers for IDAT v3, BPM (locus entries v4-v8), CSV manifest, EGT
(data blocks 5-9) and GTC v3-v5 (plus a layout-preserving GTC writer);
sample-sheet validation (IN-001..IN-014), IDAT discovery and pairing
(IN-020..IN-022), message catalogue and exit codes (CLI-014/016),
JSON Lines logging (CLI-007/008), atomic writes (CLI-011), run manifest,
and `simurg inspect`.

Validation (docs/validation/P1.md): GTC, BPM and EGT readers agree
exactly with Illumina BeadArrayFiles on GSA v4 / GDA v1 files; GTCs
round-trip byte-identically; BPM and CSV manifests are consistent.
This commit is contained in:
kemal committed 2026-09-27 16:56:39 +01:00
1 parent 0a421dffc9
commit 5a20e7131c
40 files changed
+4742

No files matched your search

+120
View File
@@ -0,0 +1,120 @@
"""IDAT discovery (IN-020 … IN-022, EDGE-005) and atomic writes / overwrite policy (CLI-010, CLI-011)."""
from __future__ import annotations
import os
from pathlib import Path
import numpy as np
import pytest
from simurg.core.atomic import atomic_open, atomic_write_bytes, check_outputs_absent
from simurg.core.errors import SimurgError
from simurg.formats import discovery as disc
from simurg.formats.samplesheet import parse_samplesheet
from simurg.synth.files import SynthIdat, encode_idat
def write_pair(
folder: Path,
barcode: str,
pos: str,
scan: str = "12/27/2023 5:52:21 PM",
channels: tuple[str, ...] = ("Grn", "Red"),
) -> None:
folder.mkdir(parents=True, exist_ok=True)
s = SynthIdat(
address_ids=np.arange(1, 11, dtype=np.int32),
mean=np.arange(10, dtype=np.uint16),
barcode=barcode,
position=pos,
run_info=[(scan, "Scan", "", "iScan", "1")],
)
for ch in channels:
(folder / f"{barcode}_{pos}_{ch}.idat").write_bytes(encode_idat(s))
def test_pairs_and_unpaired(tmp_path) -> None: # type: ignore[no-untyped-def]
write_pair(tmp_path / "a", "200000000001", "R01C01")
write_pair(tmp_path / "b" / "deep", "200000000001", "R02C01", channels=("Grn",))
d = disc.discover_idats(tmp_path)
assert sorted(d.pairs) == ["200000000001_R01C01", "200000000001_R02C01"]
assert d.pairs["200000000001_R01C01"].complete
assert not d.pairs["200000000001_R02C01"].complete
assert [c for c, _ in d.warnings] == ["W_IDAT_UNPAIRED"]
def test_duplicate_pairs_error_lists_all_paths(tmp_path) -> None: # type: ignore[no-untyped-def]
write_pair(tmp_path / "run1", "200000000001", "R01C01")
write_pair(tmp_path / "run2", "200000000001", "R01C01")
with pytest.raises(SimurgError) as e:
disc.discover_idats(tmp_path)
assert e.value.code == "E_DUP_IDAT"
assert "run1" in e.value.message and "run2" in e.value.message
def test_duplicate_first_and_newest_deterministic(tmp_path, monkeypatch) -> None: # type: ignore[no-untyped-def]
write_pair(tmp_path / "z_old", "200000000001", "R01C01", scan="1/1/2020 1:00:00 PM")
write_pair(tmp_path / "a_new", "200000000001", "R01C01", scan="1/1/2024 1:00:00 PM")
first = disc.discover_idats(tmp_path, on_duplicate="first").pairs["200000000001_R01C01"]
newest = disc.discover_idats(tmp_path, on_duplicate="newest").pairs["200000000001_R01C01"]
assert first.folder.name == "a_new" # bytewise smallest path
assert newest.folder.name == "a_new"
# shuffled directory listing must not change anything
real_scandir = os.scandir
def shuffled(path): # type: ignore[no-untyped-def]
return list(reversed(list(real_scandir(path))))
monkeypatch.setattr(os, "scandir", shuffled)
assert disc.discover_idats(tmp_path, on_duplicate="first").pairs["200000000001_R01C01"].folder.name == "a_new"
def test_symlink_loop(tmp_path) -> None: # type: ignore[no-untyped-def]
write_pair(tmp_path / "a", "200000000001", "R01C01")
(tmp_path / "a" / "loop").symlink_to(tmp_path, target_is_directory=True)
assert list(disc.discover_idats(tmp_path).pairs) == ["200000000001_R01C01"]
def test_resolve_with_sheet(tmp_path) -> None: # type: ignore[no-untyped-def]
write_pair(tmp_path / "idats", "200000000001", "R01C01")
write_pair(tmp_path / "idats", "200000000001", "R03C01")
sh = tmp_path / "s.csv"
sh.write_text(
"[Header]\nRootFolder,idats\n[Data]\nSentrixBarcode_A,SentrixPosition_A,Sample_Name\n"
"200000000001,R01C01,alpha\n200000000001,R02C01,beta\n"
)
sheet = parse_samplesheet(sh)
folder, notes = disc.resolve_input_folder(None, sheet)
assert folder == tmp_path / "idats" and notes == []
samples, notes = disc.resolve_samples(disc.discover_idats(folder), sheet)
assert [(s.sample_name, s.status) for s in samples] == [("alpha", "READY"), ("beta", "MISSING_INPUT")]
assert notes[0][0] == "I_UNLISTED_IDATS"
_, notes = disc.resolve_input_folder(tmp_path / "other", sheet)
assert notes[0][0] == "W_SHEET_ROOTFOLDER_OVERRIDDEN"
def test_input_rules() -> None:
with pytest.raises(SimurgError) as e:
disc.resolve_input_folder(None, None)
assert e.value.code == "E_NO_INPUT"
def test_atomic_write_and_failure(tmp_path) -> None: # type: ignore[no-untyped-def]
p = tmp_path / "out" / "x.bin"
atomic_write_bytes(p, b"abc")
assert p.read_bytes() == b"abc"
with pytest.raises(RuntimeError), atomic_open(tmp_path / "y.bin") as fh:
fh.write(b"partial") # type: ignore[arg-type]
raise RuntimeError("boom")
assert not (tmp_path / "y.bin").exists()
assert not (tmp_path / "y.bin.part").exists()
def test_overwrite_policy(tmp_path) -> None: # type: ignore[no-untyped-def]
p = tmp_path / "x"
p.write_text("1")
with pytest.raises(SimurgError) as e:
check_outputs_absent([p, tmp_path / "missing"], force=False)
assert e.value.code == "E_OUTPUT_EXISTS"
check_outputs_absent([p], force=True)