kemal
5a20e7131c
Readers for IDAT v3, BPM (locus entries v4-v8), CSV manifest, EGT (data blocks 5-9) and GTC v3-v5 (plus a layout-preserving GTC writer); sample-sheet validation (IN-001..IN-014), IDAT discovery and pairing (IN-020..IN-022), message catalogue and exit codes (CLI-014/016), JSON Lines logging (CLI-007/008), atomic writes (CLI-011), run manifest, and `simurg inspect`. Validation (docs/validation/P1.md): GTC, BPM and EGT readers agree exactly with Illumina BeadArrayFiles on GSA v4 / GDA v1 files; GTCs round-trip byte-identically; BPM and CSV manifests are consistent.
344 lines
11 KiB
Python
344 lines
11 KiB
Python
"""IDAT (binary, version 3) reader — plan §6.1, spec IN-023/IN-024/IN-025.
|
||
|
||
Layout (little-endian)::
|
||
|
||
char[4] "IDAT"; int64 version (3); int32 n_fields; n_fields × {uint16 code; int64 offset}
|
||
|
||
Field codes follow illuminaio (GPL, consulted as documentation only), gtc2vcf ``idat2gtc.c`` (MIT) and
|
||
BeadArrayFiles (BSD-2). The element type of the MidBlock (code 200) differs between those readers (int32 in
|
||
illuminaio, uint8 in gtc2vcf); we infer it from the space available before the next section (plan §20 U2).
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import datetime as _dt
|
||
import re
|
||
from dataclasses import dataclass, field
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
import numpy as np
|
||
import numpy.typing as npt
|
||
|
||
from simurg.core.errors import FormatError
|
||
from simurg.formats.binutil import BinaryReader, load_bytes
|
||
|
||
MAGIC = b"IDAT"
|
||
|
||
# field codes
|
||
N_SNPS = 1000
|
||
ILLUMINA_ID = 102
|
||
SD = 103
|
||
MEAN = 104
|
||
NBEADS = 107
|
||
MID_BLOCK = 200
|
||
RUN_INFO = 300
|
||
RED_GREEN = 400
|
||
MANIFEST = 401
|
||
BARCODE = 402
|
||
CHIP_TYPE = 403
|
||
POSITION = 404
|
||
BEADSET = 405
|
||
SAMPLE_ID = 406
|
||
DESCRIPTION = 407
|
||
PLATE = 408
|
||
WELL = 409
|
||
SAMPLE_COUNT = 410
|
||
VLN = 510
|
||
|
||
FIELD_NAMES: dict[int, str] = {
|
||
N_SNPS: "n_addresses",
|
||
ILLUMINA_ID: "address_ids",
|
||
SD: "sd",
|
||
MEAN: "mean",
|
||
NBEADS: "n_beads",
|
||
MID_BLOCK: "mid_block",
|
||
RUN_INFO: "run_info",
|
||
RED_GREEN: "red_green",
|
||
MANIFEST: "manifest",
|
||
BARCODE: "barcode",
|
||
CHIP_TYPE: "chip_type",
|
||
POSITION: "position",
|
||
BEADSET: "beadset",
|
||
SAMPLE_ID: "sample_id",
|
||
DESCRIPTION: "description",
|
||
PLATE: "plate",
|
||
WELL: "well",
|
||
SAMPLE_COUNT: "sample_count",
|
||
VLN: "vln",
|
||
}
|
||
|
||
_STRING_FIELDS = {
|
||
MANIFEST: "manifest",
|
||
BARCODE: "barcode",
|
||
CHIP_TYPE: "chip_type",
|
||
POSITION: "position",
|
||
BEADSET: "beadset",
|
||
SAMPLE_ID: "sample_id",
|
||
DESCRIPTION: "description",
|
||
PLATE: "plate",
|
||
WELL: "well",
|
||
VLN: "vln",
|
||
}
|
||
|
||
_NAME_RE = re.compile(
|
||
r"^(?P<barcode>[^_/\\]+)_(?P<position>R\d{2}C\d{2})_(?P<channel>Grn|Red)\.idat(\.gz)?$", re.IGNORECASE
|
||
)
|
||
|
||
_DATE_FORMATS = (
|
||
"%m/%d/%Y %I:%M:%S %p",
|
||
"%m/%d/%Y %H:%M:%S",
|
||
"%Y-%m-%dT%H:%M:%S",
|
||
"%Y-%m-%d %H:%M:%S",
|
||
"%d.%m.%Y %H:%M:%S",
|
||
)
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class RunInfoEntry:
|
||
run_time: str
|
||
block_type: str
|
||
block_pars: str
|
||
block_code: str
|
||
code_version: str
|
||
|
||
|
||
@dataclass
|
||
class Idat:
|
||
"""One colour channel of one BeadChip section."""
|
||
|
||
path: str
|
||
version: int
|
||
toc: dict[int, int]
|
||
n_addresses: int
|
||
address_ids: npt.NDArray[np.int32]
|
||
mean: npt.NDArray[np.uint16]
|
||
sd: npt.NDArray[np.uint16]
|
||
n_beads: npt.NDArray[np.uint8]
|
||
mid_block: npt.NDArray[Any] | None = None
|
||
mid_block_itemsize: int | None = None
|
||
run_info: list[RunInfoEntry] = field(default_factory=list)
|
||
red_green: bytes = b""
|
||
manifest: str = ""
|
||
barcode: str = ""
|
||
chip_type: str = ""
|
||
position: str = ""
|
||
beadset: str = ""
|
||
sample_id: str = ""
|
||
description: str = ""
|
||
plate: str = ""
|
||
well: str = ""
|
||
vln: str = ""
|
||
sample_count: int | None = None
|
||
unknown_fields: dict[int, int] = field(default_factory=dict)
|
||
|
||
# -- derived metadata (IN-024) --------------------------------------------------------------------
|
||
def scan_entry(self) -> RunInfoEntry | None:
|
||
scans = [r for r in self.run_info if r.block_type.strip().lower() == "scan"]
|
||
return scans[-1] if scans else None
|
||
|
||
@property
|
||
def scan_time_raw(self) -> str:
|
||
e = self.scan_entry()
|
||
return e.run_time if e else ""
|
||
|
||
@property
|
||
def scan_time_iso(self) -> str:
|
||
return parse_run_time(self.scan_time_raw)
|
||
|
||
@property
|
||
def scanner_software(self) -> str:
|
||
e = self.scan_entry()
|
||
return e.code_version if e else ""
|
||
|
||
@property
|
||
def scanner_pars(self) -> str:
|
||
e = self.scan_entry()
|
||
return e.block_pars if e else ""
|
||
|
||
def metadata(self) -> dict[str, Any]:
|
||
return {
|
||
"path": self.path,
|
||
"version": self.version,
|
||
"n_addresses": self.n_addresses,
|
||
"barcode": self.barcode,
|
||
"position": self.position,
|
||
"chip_type": self.chip_type,
|
||
"manifest": self.manifest,
|
||
"beadset": self.beadset,
|
||
"sample_id": self.sample_id,
|
||
"description": self.description,
|
||
"plate": self.plate,
|
||
"well": self.well,
|
||
"sample_count": self.sample_count,
|
||
"vln": self.vln,
|
||
"red_green": self.red_green.hex(" "),
|
||
"mid_block_count": None if self.mid_block is None else len(self.mid_block),
|
||
"mid_block_itemsize": self.mid_block_itemsize,
|
||
"scan_time": self.scan_time_iso,
|
||
"scan_time_raw": self.scan_time_raw,
|
||
"scanner_software": self.scanner_software,
|
||
"toc_codes": sorted(self.toc),
|
||
"unknown_codes": sorted(self.unknown_fields),
|
||
"run_info": [r.__dict__ for r in self.run_info],
|
||
}
|
||
|
||
|
||
def parse_run_time(raw: str) -> str:
|
||
"""Convert a RunInfo timestamp to ISO-8601 without consulting the process locale (EDGE-001).
|
||
|
||
Returns ``""`` if the value is empty or cannot be parsed.
|
||
"""
|
||
s = raw.strip()
|
||
if not s:
|
||
return ""
|
||
for fmt in _DATE_FORMATS:
|
||
try:
|
||
return _dt.datetime.strptime(s, fmt).isoformat()
|
||
except ValueError:
|
||
continue
|
||
return ""
|
||
|
||
|
||
def parse_idat_filename(path: str | Path) -> tuple[str, str, str] | None:
|
||
"""``<barcode>_<RxxCxx>_<Grn|Red>.idat[.gz]`` → (barcode, POSITION, 'Grn'|'Red'); None if not an IDAT name."""
|
||
m = _NAME_RE.match(Path(path).name)
|
||
if not m:
|
||
return None
|
||
ch = "Grn" if m.group("channel").lower() == "grn" else "Red"
|
||
return m.group("barcode"), m.group("position").upper(), ch
|
||
|
||
|
||
def read_idat(path: str | Path, *, load_arrays: bool = True) -> Idat:
|
||
"""Parse an IDAT file. Raises FormatError (E_IDAT_FORMAT / E_IDAT_ENCRYPTED)."""
|
||
spath = str(path)
|
||
buf = load_bytes(path)
|
||
r = BinaryReader(buf, "E_IDAT_FORMAT", spath)
|
||
if r.size < 16 or r.raw(4, "magic") != MAGIC:
|
||
raise r.fail("bad magic (not an IDAT file)")
|
||
version = r.i64("version")
|
||
# encrypted files carry version 1 in the low 32 bits followed by ciphertext (illuminaio reads int32)
|
||
if version & 0xFFFFFFFF == 1:
|
||
raise FormatError(
|
||
"E_IDAT_ENCRYPTED",
|
||
f"{spath}: encrypted (version 1) IDAT files are not supported",
|
||
path=spath,
|
||
hint="convert with an external tool first",
|
||
)
|
||
if version != 3:
|
||
raise r.fail(f"unsupported IDAT version {version}")
|
||
n_fields = r.i32("TOC size")
|
||
if n_fields <= 0 or n_fields > 4096:
|
||
raise r.fail(f"implausible TOC size {n_fields}")
|
||
toc: dict[int, int] = {}
|
||
for _ in range(n_fields):
|
||
code = r.u16("TOC code")
|
||
off = r.i64("TOC offset")
|
||
if off < 0 or off >= r.size:
|
||
raise r.fail(f"TOC offset {off} for field {code} outside file (size {r.size})")
|
||
if code in toc:
|
||
raise r.fail(f"duplicate TOC field {code}")
|
||
toc[code] = off
|
||
|
||
for required in (N_SNPS, ILLUMINA_ID, MEAN, SD, NBEADS):
|
||
if required not in toc:
|
||
raise r.fail(f"missing required field {required} ({FIELD_NAMES[required]})")
|
||
|
||
r.seek(toc[N_SNPS])
|
||
n = r.i32("number of addresses")
|
||
if n < 0:
|
||
raise r.fail(f"negative address count {n}")
|
||
|
||
empty_i32 = np.empty(0, dtype="<i4")
|
||
idat = Idat(
|
||
path=spath,
|
||
version=version,
|
||
toc=dict(toc),
|
||
n_addresses=n,
|
||
address_ids=empty_i32,
|
||
mean=np.empty(0, dtype="<u2"),
|
||
sd=np.empty(0, dtype="<u2"),
|
||
n_beads=np.empty(0, dtype="u1"),
|
||
)
|
||
|
||
if load_arrays:
|
||
r.seek(toc[ILLUMINA_ID])
|
||
idat.address_ids = r.array("<i4", n, "address ids")
|
||
r.seek(toc[SD])
|
||
idat.sd = r.array("<u2", n, "SD")
|
||
r.seek(toc[MEAN])
|
||
idat.mean = r.array("<u2", n, "Mean")
|
||
r.seek(toc[NBEADS])
|
||
idat.n_beads = r.array("u1", n, "NBeads")
|
||
if has_duplicates(idat.address_ids):
|
||
raise r.fail("duplicate address IDs")
|
||
|
||
offsets_sorted = sorted(set(toc.values()) | {r.size})
|
||
|
||
def section_end(off: int) -> int:
|
||
return next(o for o in offsets_sorted if o > off)
|
||
|
||
for code, off in sorted(toc.items(), key=lambda kv: kv[1]):
|
||
r.seek(off)
|
||
if code in _STRING_FIELDS:
|
||
setattr(idat, _STRING_FIELDS[code], r.string(FIELD_NAMES[code]))
|
||
elif code == RUN_INFO:
|
||
m = r.i32("run info count")
|
||
if m < 0 or m > 100_000:
|
||
raise r.fail(f"implausible run info count {m}")
|
||
idat.run_info = [RunInfoEntry(*(r.string("run info") for _ in range(5))) for _ in range(m)]
|
||
elif code == RED_GREEN:
|
||
idat.red_green = r.raw(4, "RedGreen")
|
||
elif code == SAMPLE_COUNT:
|
||
idat.sample_count = r.i32("sample count")
|
||
elif code == MID_BLOCK:
|
||
if not load_arrays:
|
||
continue
|
||
count = r.i32("mid block count")
|
||
space = section_end(off) - off - 4
|
||
if count < 0:
|
||
raise r.fail(f"negative mid block count {count}")
|
||
# element size inferred from the space before the next section (int32 vs uint8, plan §20 U2)
|
||
if count == space and count * 4 != space:
|
||
itemsize = 1
|
||
elif count * 4 <= space:
|
||
itemsize = 4
|
||
elif count <= space:
|
||
itemsize = 1
|
||
else:
|
||
raise r.fail(f"mid block of {count} entries does not fit in {space} bytes")
|
||
idat.mid_block_itemsize = itemsize
|
||
idat.mid_block = r.array("<i4" if itemsize == 4 else "u1", count, "mid block")
|
||
elif code in (N_SNPS, ILLUMINA_ID, SD, MEAN, NBEADS):
|
||
continue
|
||
else:
|
||
idat.unknown_fields[code] = off
|
||
return idat
|
||
|
||
|
||
def has_duplicates(ids: npt.NDArray[np.int32]) -> bool:
|
||
"""Duplicate check; O(n) when IDs are already strictly increasing (the usual case)."""
|
||
if len(ids) < 2:
|
||
return False
|
||
d = np.diff(ids)
|
||
if (d > 0).all():
|
||
return False
|
||
s = np.sort(ids)
|
||
return bool((s[1:] == s[:-1]).any())
|
||
|
||
|
||
def channel_of(idat: Idat) -> str | None:
|
||
parsed = parse_idat_filename(idat.path)
|
||
return parsed[2] if parsed else None
|
||
|
||
|
||
def check_pair(grn: Idat, red: Idat) -> None:
|
||
"""Red/Grn consistency (IN-023): same N and the same address set. Raises E_IDAT_PAIR_MISMATCH."""
|
||
if grn.n_addresses != red.n_addresses:
|
||
raise FormatError(
|
||
"E_IDAT_PAIR_MISMATCH",
|
||
f"Grn has {grn.n_addresses} addresses, Red has {red.n_addresses}: {grn.path}, {red.path}",
|
||
)
|
||
if len(grn.address_ids) and not np.array_equal(np.sort(grn.address_ids), np.sort(red.address_ids)):
|
||
raise FormatError("E_IDAT_PAIR_MISMATCH", f"address sets differ: {grn.path}, {red.path}")
|