P3: normalization fit (1.2.0 and 1.1.2), apply, polar, simurg normalize
- signal/normalization/numerics.py: vendor percentile, trimmean, MAD sigma (old/new), bisquare robust fits, float32/float64 typing and sequential sums reproduced exactly (ported from gtc2vcf, MIT) - signal/normalization/fit.py: outlier removal, offset, rotation, shear, scale, nn12/rr12, single-channel model for small Infinium I bins, exact and binned nearest-neighbour sweeps; vendor quirks marked QUIRK - signal/normalization/apply.py: affine transform and theta/R - pipeline/normalize.py + CLI `simurg normalize` - oracle GTCs rebuilt with -ffp-contract=off (aarch64 gcc fuses a*b+c by default, shifting the last bits); the old FMA oracle differed by <=4 ULP Result: transforms bit-identical to the oracle for all 36 GSA samples x 48 bins, for both normalization versions (3456/3456 bins, 0 ULP). Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01814GKVzZVL3FuF3DeiZXhv
This commit is contained in:
1 parent
5c93e194e0
commit
314fdc40d1
12 files changed
+1097
-14
No files matched your search
@@ -20,6 +20,8 @@ from simurg.formats.bpm import Manifest
|
||||
LOCI_SCHEMA_ID = "simurg.loci/1"
|
||||
INTENSITY_SCHEMA_ID = "simurg.sample_intensity/1"
|
||||
CONTROLS_SCHEMA_ID = "simurg.sample_controls/1"
|
||||
NORMALIZED_SCHEMA_ID = "simurg.sample_normalized/1"
|
||||
TRANSFORMS_SCHEMA_ID = "simurg.sample_transforms/1"
|
||||
|
||||
LOCI_SCHEMA = pa.schema(
|
||||
[
|
||||
@@ -56,6 +58,33 @@ CONTROLS_SCHEMA = pa.schema(
|
||||
metadata={"simurg.schema": CONTROLS_SCHEMA_ID},
|
||||
)
|
||||
|
||||
NORMALIZED_SCHEMA = pa.schema(
|
||||
[("norm_x", pa.float32()), ("norm_y", pa.float32()), ("theta", pa.float32()), ("r", pa.float32())],
|
||||
metadata={"simurg.schema": NORMALIZED_SCHEMA_ID},
|
||||
)
|
||||
|
||||
TRANSFORMS_SCHEMA = pa.schema(
|
||||
[
|
||||
("bin_index", pa.int32()),
|
||||
("norm_bin_id", pa.uint8()),
|
||||
("n_loci", pa.int32()),
|
||||
("version", pa.int32()),
|
||||
("offset_x", pa.float32()),
|
||||
("offset_y", pa.float32()),
|
||||
("scale_x", pa.float32()),
|
||||
("scale_y", pa.float32()),
|
||||
("shear", pa.float32()),
|
||||
("theta", pa.float32()),
|
||||
("cvx", pa.float32()),
|
||||
("cvy", pa.float32()),
|
||||
("nn12", pa.float32()),
|
||||
("rr12", pa.float32()),
|
||||
("taa", pa.float32()),
|
||||
("tbb", pa.float32()),
|
||||
],
|
||||
metadata={"simurg.schema": TRANSFORMS_SCHEMA_ID},
|
||||
)
|
||||
|
||||
_WRITE_OPTS: dict[str, Any] = {"compression": "zstd", "use_dictionary": True, "write_statistics": True}
|
||||
|
||||
|
||||
@@ -122,6 +151,26 @@ def controls_table(addresses: np.ndarray, red: np.ndarray, green: np.ndarray, me
|
||||
return with_metadata(t, meta)
|
||||
|
||||
|
||||
def normalized_table(
|
||||
nx: np.ndarray, ny: np.ndarray, theta: np.ndarray, r: np.ndarray, meta: dict[str, Any]
|
||||
) -> pa.Table:
|
||||
t = pa.table(
|
||||
{
|
||||
"norm_x": pa.array(nx, pa.float32()),
|
||||
"norm_y": pa.array(ny, pa.float32()),
|
||||
"theta": pa.array(theta, pa.float32()),
|
||||
"r": pa.array(r, pa.float32()),
|
||||
},
|
||||
schema=NORMALIZED_SCHEMA,
|
||||
)
|
||||
return with_metadata(t, meta)
|
||||
|
||||
|
||||
def transforms_table(rows: list[dict[str, Any]], meta: dict[str, Any]) -> pa.Table:
|
||||
t = pa.Table.from_pylist(rows, schema=TRANSFORMS_SCHEMA)
|
||||
return with_metadata(t, meta)
|
||||
|
||||
|
||||
def write_parquet(table: pa.Table, path: Path) -> None:
|
||||
"""Atomic, deterministic Parquet write."""
|
||||
with atomic_open(Path(path), "wb") as fh:
|
||||
|
||||
Reference in new issue
Block a user