P2 (WIP): raw X/Y assembly, canonical Parquet model, simurg ingest

- signal/raw_xy.py: vectorised Infinium I/II raw X/Y and control assembly
  (plan §7.1) with a per-chip address-map cache; IN-027 coverage gate
- core/schemas.py: Arrow schemas (loci, sample_intensity, controls),
  deterministic atomic Parquet writing
- pipeline/ingest.py + CLI: discovery, sheet, IN-010 naming, overwrite
  check before any work, spawn-based workers, per-sample isolation,
  --resume by input hashes, run manifest, exit codes 0/3/4
- matches gtc2vcf raw X/Y and controls exactly on 18 GSA samples;
  outputs byte-identical across thread counts and layouts
This commit is contained in:
kemal committed 2026-09-27 20:54:06 +01:00
1 parent 7bd73700b2
commit 82bb3637b4
9 files changed
+970 -2

No files matched your search

+32 -1
View File
@@ -25,7 +25,7 @@ Normalization-bin rule (plan §6.2)::
from __future__ import annotations
import struct
from dataclasses import dataclass, field
from dataclasses import dataclass, field, replace
from pathlib import Path
from typing import Any
@@ -103,6 +103,37 @@ class Manifest:
def name_index(self) -> dict[str, int]:
return {n: i for i, n in enumerate(self.names)}
def lite(self) -> Manifest:
"""Copy keeping only what intensity assembly needs (cheap to pickle to worker processes)."""
n = self.num_loci
empty = [""] * 0
return replace(
self,
names=list(self.names),
ilmn_ids=empty,
snps=empty,
chroms=empty,
ilmn_strands=empty,
source_strands=empty,
ref_strands=empty,
ploidies=empty,
species=empty,
genome_builds=empty,
sources=empty,
source_versions=empty,
customer_strands=empty,
source_seqs=empty,
top_genomic_seqs=empty,
allele_a_probe_seqs=empty,
allele_b_probe_seqs=empty,
map_infos=np.zeros(0, np.int64),
base_fractions=None,
header_lines=[],
heading={},
exp_clusters=self.exp_clusters[:n],
intensity_only=self.intensity_only[:n],
)
def summary(self) -> dict[str, Any]:
at = self.assay_types
return {