P1: file parsers, sample sheet, discovery, CLI inspect
Readers for IDAT v3, BPM (locus entries v4-v8), CSV manifest, EGT (data blocks 5-9) and GTC v3-v5 (plus a layout-preserving GTC writer); sample-sheet validation (IN-001..IN-014), IDAT discovery and pairing (IN-020..IN-022), message catalogue and exit codes (CLI-014/016), JSON Lines logging (CLI-007/008), atomic writes (CLI-011), run manifest, and `simurg inspect`. Validation (docs/validation/P1.md): GTC, BPM and EGT readers agree exactly with Illumina BeadArrayFiles on GSA v4 / GDA v1 files; GTCs round-trip byte-identically; BPM and CSV manifests are consistent.
This commit is contained in:
kemal committed
1 parent
0a421dffc9
commit
5a20e7131c
40 files changed
+4742
No files matched your search
@@ -0,0 +1,343 @@
|
||||
"""IDAT (binary, version 3) reader — plan §6.1, spec IN-023/IN-024/IN-025.
|
||||
|
||||
Layout (little-endian)::
|
||||
|
||||
char[4] "IDAT"; int64 version (3); int32 n_fields; n_fields × {uint16 code; int64 offset}
|
||||
|
||||
Field codes follow illuminaio (GPL, consulted as documentation only), gtc2vcf ``idat2gtc.c`` (MIT) and
|
||||
BeadArrayFiles (BSD-2). The element type of the MidBlock (code 200) differs between those readers (int32 in
|
||||
illuminaio, uint8 in gtc2vcf); we infer it from the space available before the next section (plan §20 U2).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime as _dt
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import numpy.typing as npt
|
||||
|
||||
from simurg.core.errors import FormatError
|
||||
from simurg.formats.binutil import BinaryReader, load_bytes
|
||||
|
||||
MAGIC = b"IDAT"
|
||||
|
||||
# field codes
|
||||
N_SNPS = 1000
|
||||
ILLUMINA_ID = 102
|
||||
SD = 103
|
||||
MEAN = 104
|
||||
NBEADS = 107
|
||||
MID_BLOCK = 200
|
||||
RUN_INFO = 300
|
||||
RED_GREEN = 400
|
||||
MANIFEST = 401
|
||||
BARCODE = 402
|
||||
CHIP_TYPE = 403
|
||||
POSITION = 404
|
||||
BEADSET = 405
|
||||
SAMPLE_ID = 406
|
||||
DESCRIPTION = 407
|
||||
PLATE = 408
|
||||
WELL = 409
|
||||
SAMPLE_COUNT = 410
|
||||
VLN = 510
|
||||
|
||||
FIELD_NAMES: dict[int, str] = {
|
||||
N_SNPS: "n_addresses",
|
||||
ILLUMINA_ID: "address_ids",
|
||||
SD: "sd",
|
||||
MEAN: "mean",
|
||||
NBEADS: "n_beads",
|
||||
MID_BLOCK: "mid_block",
|
||||
RUN_INFO: "run_info",
|
||||
RED_GREEN: "red_green",
|
||||
MANIFEST: "manifest",
|
||||
BARCODE: "barcode",
|
||||
CHIP_TYPE: "chip_type",
|
||||
POSITION: "position",
|
||||
BEADSET: "beadset",
|
||||
SAMPLE_ID: "sample_id",
|
||||
DESCRIPTION: "description",
|
||||
PLATE: "plate",
|
||||
WELL: "well",
|
||||
SAMPLE_COUNT: "sample_count",
|
||||
VLN: "vln",
|
||||
}
|
||||
|
||||
_STRING_FIELDS = {
|
||||
MANIFEST: "manifest",
|
||||
BARCODE: "barcode",
|
||||
CHIP_TYPE: "chip_type",
|
||||
POSITION: "position",
|
||||
BEADSET: "beadset",
|
||||
SAMPLE_ID: "sample_id",
|
||||
DESCRIPTION: "description",
|
||||
PLATE: "plate",
|
||||
WELL: "well",
|
||||
VLN: "vln",
|
||||
}
|
||||
|
||||
_NAME_RE = re.compile(
|
||||
r"^(?P<barcode>[^_/\\]+)_(?P<position>R\d{2}C\d{2})_(?P<channel>Grn|Red)\.idat(\.gz)?$", re.IGNORECASE
|
||||
)
|
||||
|
||||
_DATE_FORMATS = (
|
||||
"%m/%d/%Y %I:%M:%S %p",
|
||||
"%m/%d/%Y %H:%M:%S",
|
||||
"%Y-%m-%dT%H:%M:%S",
|
||||
"%Y-%m-%d %H:%M:%S",
|
||||
"%d.%m.%Y %H:%M:%S",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RunInfoEntry:
|
||||
run_time: str
|
||||
block_type: str
|
||||
block_pars: str
|
||||
block_code: str
|
||||
code_version: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class Idat:
|
||||
"""One colour channel of one BeadChip section."""
|
||||
|
||||
path: str
|
||||
version: int
|
||||
toc: dict[int, int]
|
||||
n_addresses: int
|
||||
address_ids: npt.NDArray[np.int32]
|
||||
mean: npt.NDArray[np.uint16]
|
||||
sd: npt.NDArray[np.uint16]
|
||||
n_beads: npt.NDArray[np.uint8]
|
||||
mid_block: npt.NDArray[Any] | None = None
|
||||
mid_block_itemsize: int | None = None
|
||||
run_info: list[RunInfoEntry] = field(default_factory=list)
|
||||
red_green: bytes = b""
|
||||
manifest: str = ""
|
||||
barcode: str = ""
|
||||
chip_type: str = ""
|
||||
position: str = ""
|
||||
beadset: str = ""
|
||||
sample_id: str = ""
|
||||
description: str = ""
|
||||
plate: str = ""
|
||||
well: str = ""
|
||||
vln: str = ""
|
||||
sample_count: int | None = None
|
||||
unknown_fields: dict[int, int] = field(default_factory=dict)
|
||||
|
||||
# -- derived metadata (IN-024) --------------------------------------------------------------------
|
||||
def scan_entry(self) -> RunInfoEntry | None:
|
||||
scans = [r for r in self.run_info if r.block_type.strip().lower() == "scan"]
|
||||
return scans[-1] if scans else None
|
||||
|
||||
@property
|
||||
def scan_time_raw(self) -> str:
|
||||
e = self.scan_entry()
|
||||
return e.run_time if e else ""
|
||||
|
||||
@property
|
||||
def scan_time_iso(self) -> str:
|
||||
return parse_run_time(self.scan_time_raw)
|
||||
|
||||
@property
|
||||
def scanner_software(self) -> str:
|
||||
e = self.scan_entry()
|
||||
return e.code_version if e else ""
|
||||
|
||||
@property
|
||||
def scanner_pars(self) -> str:
|
||||
e = self.scan_entry()
|
||||
return e.block_pars if e else ""
|
||||
|
||||
def metadata(self) -> dict[str, Any]:
|
||||
return {
|
||||
"path": self.path,
|
||||
"version": self.version,
|
||||
"n_addresses": self.n_addresses,
|
||||
"barcode": self.barcode,
|
||||
"position": self.position,
|
||||
"chip_type": self.chip_type,
|
||||
"manifest": self.manifest,
|
||||
"beadset": self.beadset,
|
||||
"sample_id": self.sample_id,
|
||||
"description": self.description,
|
||||
"plate": self.plate,
|
||||
"well": self.well,
|
||||
"sample_count": self.sample_count,
|
||||
"vln": self.vln,
|
||||
"red_green": self.red_green.hex(" "),
|
||||
"mid_block_count": None if self.mid_block is None else len(self.mid_block),
|
||||
"mid_block_itemsize": self.mid_block_itemsize,
|
||||
"scan_time": self.scan_time_iso,
|
||||
"scan_time_raw": self.scan_time_raw,
|
||||
"scanner_software": self.scanner_software,
|
||||
"toc_codes": sorted(self.toc),
|
||||
"unknown_codes": sorted(self.unknown_fields),
|
||||
"run_info": [r.__dict__ for r in self.run_info],
|
||||
}
|
||||
|
||||
|
||||
def parse_run_time(raw: str) -> str:
|
||||
"""Convert a RunInfo timestamp to ISO-8601 without consulting the process locale (EDGE-001).
|
||||
|
||||
Returns ``""`` if the value is empty or cannot be parsed.
|
||||
"""
|
||||
s = raw.strip()
|
||||
if not s:
|
||||
return ""
|
||||
for fmt in _DATE_FORMATS:
|
||||
try:
|
||||
return _dt.datetime.strptime(s, fmt).isoformat()
|
||||
except ValueError:
|
||||
continue
|
||||
return ""
|
||||
|
||||
|
||||
def parse_idat_filename(path: str | Path) -> tuple[str, str, str] | None:
|
||||
"""``<barcode>_<RxxCxx>_<Grn|Red>.idat[.gz]`` → (barcode, POSITION, 'Grn'|'Red'); None if not an IDAT name."""
|
||||
m = _NAME_RE.match(Path(path).name)
|
||||
if not m:
|
||||
return None
|
||||
ch = "Grn" if m.group("channel").lower() == "grn" else "Red"
|
||||
return m.group("barcode"), m.group("position").upper(), ch
|
||||
|
||||
|
||||
def read_idat(path: str | Path, *, load_arrays: bool = True) -> Idat:
|
||||
"""Parse an IDAT file. Raises FormatError (E_IDAT_FORMAT / E_IDAT_ENCRYPTED)."""
|
||||
spath = str(path)
|
||||
buf = load_bytes(path)
|
||||
r = BinaryReader(buf, "E_IDAT_FORMAT", spath)
|
||||
if r.size < 16 or r.raw(4, "magic") != MAGIC:
|
||||
raise r.fail("bad magic (not an IDAT file)")
|
||||
version = r.i64("version")
|
||||
# encrypted files carry version 1 in the low 32 bits followed by ciphertext (illuminaio reads int32)
|
||||
if version & 0xFFFFFFFF == 1:
|
||||
raise FormatError(
|
||||
"E_IDAT_ENCRYPTED",
|
||||
f"{spath}: encrypted (version 1) IDAT files are not supported",
|
||||
path=spath,
|
||||
hint="convert with an external tool first",
|
||||
)
|
||||
if version != 3:
|
||||
raise r.fail(f"unsupported IDAT version {version}")
|
||||
n_fields = r.i32("TOC size")
|
||||
if n_fields <= 0 or n_fields > 4096:
|
||||
raise r.fail(f"implausible TOC size {n_fields}")
|
||||
toc: dict[int, int] = {}
|
||||
for _ in range(n_fields):
|
||||
code = r.u16("TOC code")
|
||||
off = r.i64("TOC offset")
|
||||
if off < 0 or off >= r.size:
|
||||
raise r.fail(f"TOC offset {off} for field {code} outside file (size {r.size})")
|
||||
if code in toc:
|
||||
raise r.fail(f"duplicate TOC field {code}")
|
||||
toc[code] = off
|
||||
|
||||
for required in (N_SNPS, ILLUMINA_ID, MEAN, SD, NBEADS):
|
||||
if required not in toc:
|
||||
raise r.fail(f"missing required field {required} ({FIELD_NAMES[required]})")
|
||||
|
||||
r.seek(toc[N_SNPS])
|
||||
n = r.i32("number of addresses")
|
||||
if n < 0:
|
||||
raise r.fail(f"negative address count {n}")
|
||||
|
||||
empty_i32 = np.empty(0, dtype="<i4")
|
||||
idat = Idat(
|
||||
path=spath,
|
||||
version=version,
|
||||
toc=dict(toc),
|
||||
n_addresses=n,
|
||||
address_ids=empty_i32,
|
||||
mean=np.empty(0, dtype="<u2"),
|
||||
sd=np.empty(0, dtype="<u2"),
|
||||
n_beads=np.empty(0, dtype="u1"),
|
||||
)
|
||||
|
||||
if load_arrays:
|
||||
r.seek(toc[ILLUMINA_ID])
|
||||
idat.address_ids = r.array("<i4", n, "address ids")
|
||||
r.seek(toc[SD])
|
||||
idat.sd = r.array("<u2", n, "SD")
|
||||
r.seek(toc[MEAN])
|
||||
idat.mean = r.array("<u2", n, "Mean")
|
||||
r.seek(toc[NBEADS])
|
||||
idat.n_beads = r.array("u1", n, "NBeads")
|
||||
if has_duplicates(idat.address_ids):
|
||||
raise r.fail("duplicate address IDs")
|
||||
|
||||
offsets_sorted = sorted(set(toc.values()) | {r.size})
|
||||
|
||||
def section_end(off: int) -> int:
|
||||
return next(o for o in offsets_sorted if o > off)
|
||||
|
||||
for code, off in sorted(toc.items(), key=lambda kv: kv[1]):
|
||||
r.seek(off)
|
||||
if code in _STRING_FIELDS:
|
||||
setattr(idat, _STRING_FIELDS[code], r.string(FIELD_NAMES[code]))
|
||||
elif code == RUN_INFO:
|
||||
m = r.i32("run info count")
|
||||
if m < 0 or m > 100_000:
|
||||
raise r.fail(f"implausible run info count {m}")
|
||||
idat.run_info = [RunInfoEntry(*(r.string("run info") for _ in range(5))) for _ in range(m)]
|
||||
elif code == RED_GREEN:
|
||||
idat.red_green = r.raw(4, "RedGreen")
|
||||
elif code == SAMPLE_COUNT:
|
||||
idat.sample_count = r.i32("sample count")
|
||||
elif code == MID_BLOCK:
|
||||
if not load_arrays:
|
||||
continue
|
||||
count = r.i32("mid block count")
|
||||
space = section_end(off) - off - 4
|
||||
if count < 0:
|
||||
raise r.fail(f"negative mid block count {count}")
|
||||
# element size inferred from the space before the next section (int32 vs uint8, plan §20 U2)
|
||||
if count == space and count * 4 != space:
|
||||
itemsize = 1
|
||||
elif count * 4 <= space:
|
||||
itemsize = 4
|
||||
elif count <= space:
|
||||
itemsize = 1
|
||||
else:
|
||||
raise r.fail(f"mid block of {count} entries does not fit in {space} bytes")
|
||||
idat.mid_block_itemsize = itemsize
|
||||
idat.mid_block = r.array("<i4" if itemsize == 4 else "u1", count, "mid block")
|
||||
elif code in (N_SNPS, ILLUMINA_ID, SD, MEAN, NBEADS):
|
||||
continue
|
||||
else:
|
||||
idat.unknown_fields[code] = off
|
||||
return idat
|
||||
|
||||
|
||||
def has_duplicates(ids: npt.NDArray[np.int32]) -> bool:
|
||||
"""Duplicate check; O(n) when IDs are already strictly increasing (the usual case)."""
|
||||
if len(ids) < 2:
|
||||
return False
|
||||
d = np.diff(ids)
|
||||
if (d > 0).all():
|
||||
return False
|
||||
s = np.sort(ids)
|
||||
return bool((s[1:] == s[:-1]).any())
|
||||
|
||||
|
||||
def channel_of(idat: Idat) -> str | None:
|
||||
parsed = parse_idat_filename(idat.path)
|
||||
return parsed[2] if parsed else None
|
||||
|
||||
|
||||
def check_pair(grn: Idat, red: Idat) -> None:
|
||||
"""Red/Grn consistency (IN-023): same N and the same address set. Raises E_IDAT_PAIR_MISMATCH."""
|
||||
if grn.n_addresses != red.n_addresses:
|
||||
raise FormatError(
|
||||
"E_IDAT_PAIR_MISMATCH",
|
||||
f"Grn has {grn.n_addresses} addresses, Red has {red.n_addresses}: {grn.path}, {red.path}",
|
||||
)
|
||||
if len(grn.address_ids) and not np.array_equal(np.sort(grn.address_ids), np.sort(red.address_ids)):
|
||||
raise FormatError("E_IDAT_PAIR_MISMATCH", f"address sets differ: {grn.path}, {red.path}")
|
||||
Reference in new issue
Block a user