"""IDAT (binary, version 3) reader — plan §6.1, spec IN-023/IN-024/IN-025. Layout (little-endian):: char[4] "IDAT"; int64 version (3); int32 n_fields; n_fields × {uint16 code; int64 offset} Field codes follow illuminaio (GPL, consulted as documentation only), gtc2vcf ``idat2gtc.c`` (MIT) and BeadArrayFiles (BSD-2). The element type of the MidBlock (code 200) differs between those readers (int32 in illuminaio, uint8 in gtc2vcf); we infer it from the space available before the next section (plan §20 U2). """ from __future__ import annotations import datetime as _dt import re from dataclasses import dataclass, field from pathlib import Path from typing import Any import numpy as np import numpy.typing as npt from simurg.core.errors import FormatError from simurg.formats.binutil import BinaryReader, load_bytes MAGIC = b"IDAT" # field codes N_SNPS = 1000 ILLUMINA_ID = 102 SD = 103 MEAN = 104 NBEADS = 107 MID_BLOCK = 200 RUN_INFO = 300 RED_GREEN = 400 MANIFEST = 401 BARCODE = 402 CHIP_TYPE = 403 POSITION = 404 BEADSET = 405 SAMPLE_ID = 406 DESCRIPTION = 407 PLATE = 408 WELL = 409 SAMPLE_COUNT = 410 VLN = 510 FIELD_NAMES: dict[int, str] = { N_SNPS: "n_addresses", ILLUMINA_ID: "address_ids", SD: "sd", MEAN: "mean", NBEADS: "n_beads", MID_BLOCK: "mid_block", RUN_INFO: "run_info", RED_GREEN: "red_green", MANIFEST: "manifest", BARCODE: "barcode", CHIP_TYPE: "chip_type", POSITION: "position", BEADSET: "beadset", SAMPLE_ID: "sample_id", DESCRIPTION: "description", PLATE: "plate", WELL: "well", SAMPLE_COUNT: "sample_count", VLN: "vln", } _STRING_FIELDS = { MANIFEST: "manifest", BARCODE: "barcode", CHIP_TYPE: "chip_type", POSITION: "position", BEADSET: "beadset", SAMPLE_ID: "sample_id", DESCRIPTION: "description", PLATE: "plate", WELL: "well", VLN: "vln", } _NAME_RE = re.compile( r"^(?P[^_/\\]+)_(?PR\d{2}C\d{2})_(?PGrn|Red)\.idat(\.gz)?$", re.IGNORECASE ) _DATE_FORMATS = ( "%m/%d/%Y %I:%M:%S %p", "%m/%d/%Y %H:%M:%S", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d %H:%M:%S", "%d.%m.%Y %H:%M:%S", ) @dataclass(frozen=True) class RunInfoEntry: run_time: str block_type: str block_pars: str block_code: str code_version: str @dataclass class Idat: """One colour channel of one BeadChip section.""" path: str version: int toc: dict[int, int] n_addresses: int address_ids: npt.NDArray[np.int32] mean: npt.NDArray[np.uint16] sd: npt.NDArray[np.uint16] n_beads: npt.NDArray[np.uint8] mid_block: npt.NDArray[Any] | None = None mid_block_itemsize: int | None = None run_info: list[RunInfoEntry] = field(default_factory=list) red_green: bytes = b"" manifest: str = "" barcode: str = "" chip_type: str = "" position: str = "" beadset: str = "" sample_id: str = "" description: str = "" plate: str = "" well: str = "" vln: str = "" sample_count: int | None = None unknown_fields: dict[int, int] = field(default_factory=dict) # -- derived metadata (IN-024) -------------------------------------------------------------------- def scan_entry(self) -> RunInfoEntry | None: scans = [r for r in self.run_info if r.block_type.strip().lower() == "scan"] return scans[-1] if scans else None @property def scan_time_raw(self) -> str: e = self.scan_entry() return e.run_time if e else "" @property def scan_time_iso(self) -> str: return parse_run_time(self.scan_time_raw) @property def scanner_software(self) -> str: e = self.scan_entry() return e.code_version if e else "" @property def scanner_pars(self) -> str: e = self.scan_entry() return e.block_pars if e else "" def metadata(self) -> dict[str, Any]: return { "path": self.path, "version": self.version, "n_addresses": self.n_addresses, "barcode": self.barcode, "position": self.position, "chip_type": self.chip_type, "manifest": self.manifest, "beadset": self.beadset, "sample_id": self.sample_id, "description": self.description, "plate": self.plate, "well": self.well, "sample_count": self.sample_count, "vln": self.vln, "red_green": self.red_green.hex(" "), "mid_block_count": None if self.mid_block is None else len(self.mid_block), "mid_block_itemsize": self.mid_block_itemsize, "scan_time": self.scan_time_iso, "scan_time_raw": self.scan_time_raw, "scanner_software": self.scanner_software, "toc_codes": sorted(self.toc), "unknown_codes": sorted(self.unknown_fields), "run_info": [r.__dict__ for r in self.run_info], } def parse_run_time(raw: str) -> str: """Convert a RunInfo timestamp to ISO-8601 without consulting the process locale (EDGE-001). Returns ``""`` if the value is empty or cannot be parsed. """ s = raw.strip() if not s: return "" for fmt in _DATE_FORMATS: try: return _dt.datetime.strptime(s, fmt).isoformat() except ValueError: continue return "" def parse_idat_filename(path: str | Path) -> tuple[str, str, str] | None: """``__.idat[.gz]`` → (barcode, POSITION, 'Grn'|'Red'); None if not an IDAT name.""" m = _NAME_RE.match(Path(path).name) if not m: return None ch = "Grn" if m.group("channel").lower() == "grn" else "Red" return m.group("barcode"), m.group("position").upper(), ch def read_idat(path: str | Path, *, load_arrays: bool = True) -> Idat: """Parse an IDAT file. Raises FormatError (E_IDAT_FORMAT / E_IDAT_ENCRYPTED).""" spath = str(path) buf = load_bytes(path) r = BinaryReader(buf, "E_IDAT_FORMAT", spath) if r.size < 16 or r.raw(4, "magic") != MAGIC: raise r.fail("bad magic (not an IDAT file)") version = r.i64("version") # encrypted files carry version 1 in the low 32 bits followed by ciphertext (illuminaio reads int32) if version & 0xFFFFFFFF == 1: raise FormatError( "E_IDAT_ENCRYPTED", f"{spath}: encrypted (version 1) IDAT files are not supported", path=spath, hint="convert with an external tool first", ) if version != 3: raise r.fail(f"unsupported IDAT version {version}") n_fields = r.i32("TOC size") if n_fields <= 0 or n_fields > 4096: raise r.fail(f"implausible TOC size {n_fields}") toc: dict[int, int] = {} for _ in range(n_fields): code = r.u16("TOC code") off = r.i64("TOC offset") if off < 0 or off >= r.size: raise r.fail(f"TOC offset {off} for field {code} outside file (size {r.size})") if code in toc: raise r.fail(f"duplicate TOC field {code}") toc[code] = off for required in (N_SNPS, ILLUMINA_ID, MEAN, SD, NBEADS): if required not in toc: raise r.fail(f"missing required field {required} ({FIELD_NAMES[required]})") r.seek(toc[N_SNPS]) n = r.i32("number of addresses") if n < 0: raise r.fail(f"negative address count {n}") empty_i32 = np.empty(0, dtype=" int: return next(o for o in offsets_sorted if o > off) for code, off in sorted(toc.items(), key=lambda kv: kv[1]): r.seek(off) if code in _STRING_FIELDS: setattr(idat, _STRING_FIELDS[code], r.string(FIELD_NAMES[code])) elif code == RUN_INFO: m = r.i32("run info count") if m < 0 or m > 100_000: raise r.fail(f"implausible run info count {m}") idat.run_info = [RunInfoEntry(*(r.string("run info") for _ in range(5))) for _ in range(m)] elif code == RED_GREEN: idat.red_green = r.raw(4, "RedGreen") elif code == SAMPLE_COUNT: idat.sample_count = r.i32("sample count") elif code == MID_BLOCK: if not load_arrays: continue count = r.i32("mid block count") space = section_end(off) - off - 4 if count < 0: raise r.fail(f"negative mid block count {count}") # element size inferred from the space before the next section (int32 vs uint8, plan §20 U2) if count == space and count * 4 != space: itemsize = 1 elif count * 4 <= space: itemsize = 4 elif count <= space: itemsize = 1 else: raise r.fail(f"mid block of {count} entries does not fit in {space} bytes") idat.mid_block_itemsize = itemsize idat.mid_block = r.array(" bool: """Duplicate check; O(n) when IDs are already strictly increasing (the usual case).""" if len(ids) < 2: return False d = np.diff(ids) if (d > 0).all(): return False s = np.sort(ids) return bool((s[1:] == s[:-1]).any()) def channel_of(idat: Idat) -> str | None: parsed = parse_idat_filename(idat.path) return parsed[2] if parsed else None def check_pair(grn: Idat, red: Idat) -> None: """Red/Grn consistency (IN-023): same N and the same address set. Raises E_IDAT_PAIR_MISMATCH.""" if grn.n_addresses != red.n_addresses: raise FormatError( "E_IDAT_PAIR_MISMATCH", f"Grn has {grn.n_addresses} addresses, Red has {red.n_addresses}: {grn.path}, {red.path}", ) if len(grn.address_ids) and not np.array_equal(np.sort(grn.address_ids), np.sort(red.address_ids)): raise FormatError("E_IDAT_PAIR_MISMATCH", f"address sets differ: {grn.path}, {red.path}")