P3: normalization fit (1.2.0 and 1.1.2), apply, polar, simurg normalize

- signal/normalization/numerics.py: vendor percentile, trimmean, MAD sigma
  (old/new), bisquare robust fits, float32/float64 typing and sequential
  sums reproduced exactly (ported from gtc2vcf, MIT)
- signal/normalization/fit.py: outlier removal, offset, rotation, shear,
  scale, nn12/rr12, single-channel model for small Infinium I bins,
  exact and binned nearest-neighbour sweeps; vendor quirks marked QUIRK
- signal/normalization/apply.py: affine transform and theta/R
- pipeline/normalize.py + CLI `simurg normalize`
- oracle GTCs rebuilt with -ffp-contract=off (aarch64 gcc fuses a*b+c by
  default, shifting the last bits); the old FMA oracle differed by <=4 ULP

Result: transforms bit-identical to the oracle for all 36 GSA samples x
48 bins, for both normalization versions (3456/3456 bins, 0 ULP).
This commit is contained in:
kemal committed 2026-09-27 22:51:06 +01:00
1 parent c8cd2f741c
commit c0d556d4af
12 files changed
+1097 -14

No files matched your search

+1 -1
View File
@@ -6,12 +6,12 @@ from __future__ import annotations
import numpy as np
import pytest
from tests.conftest import GSA_BPM, GSA_IDATS, ORACLE_GTCS, need
from simurg.formats.bpm import read_bpm
from simurg.formats.gtc import read_gtc
from simurg.formats.idat import read_idat
from simurg.signal.raw_xy import LocusMapCache, assemble_raw_xy, check_coverage
from tests.conftest import GSA_BPM, GSA_IDATS, ORACLE_GTCS, need
pytestmark = pytest.mark.data
+45
View File
@@ -0,0 +1,45 @@
"""Phase P3 acceptance on P0 data: normalization transforms fitted from raw intensities are bit-identical to the
transforms stored in oracle GTCs (gtc2vcf built without FMA), for normalization 1.2.0 and 1.1.2.
The same check against vendor GTCs runs in scripts/validate_p1_large.py --p3.
"""
from __future__ import annotations
import numpy as np
import pytest
from tests.conftest import GSA_BPM, ORACLE_GTCS, ORACLE_GTCS_GT2, need
from simurg.formats.bpm import read_bpm
from simurg.formats.gtc import read_gtc
from simurg.signal.normalization.fit import fit_transforms
pytestmark = pytest.mark.data
FIELDS = ("offset_x", "offset_y", "scale_x", "scale_y", "shear", "theta")
def as_bits(xf) -> np.ndarray: # type: ignore[no-untyped-def,type-arg]
v = [getattr(xf, f) for f in FIELDS] + [xf.nn12, xf.rr12, xf.taa, xf.tbb]
return np.array(v, dtype=np.float32).view(np.int32)
def ref_bits(rec) -> np.ndarray: # type: ignore[no-untyped-def,type-arg]
v = [rec[f] for f in FIELDS] + list(rec["reserved"][2:6])
return np.array(v, dtype=np.float32).view(np.int32)
@pytest.mark.parametrize(
("folder", "version"), [(ORACLE_GTCS, "1.2.0"), (ORACLE_GTCS_GT2, "1.1.2")], ids=["1.2.0", "1.1.2"]
)
def test_transforms_bit_identical(folder, version) -> None: # type: ignore[no-untyped-def]
need(GSA_BPM, folder)
m = read_bpm(GSA_BPM)
bins = m.norm_bin_ids()
gtcs = sorted(folder.glob("*.gtc"))
assert len(gtcs) >= 18
for g in gtcs:
ref = read_gtc(g)
mine = fit_transforms(ref.raw_x, ref.raw_y, bins, version)
assert len(mine) == len(ref.normalization_transforms)
for k, (a, b) in enumerate(zip(mine, ref.normalization_transforms, strict=True)):
assert np.array_equal(as_bits(a), ref_bits(b)), f"{g.name} bin {k}"