From 92416397fec675bfad2a1907417895198862d3db Mon Sep 17 00:00:00 2001 From: Kemal Yaylali Date: Mon, 28 Sep 2026 22:41:56 +0100 Subject: [PATCH] P10 (WIP): release build script, Dockerfile, conda recipes, fresh-VM acceptance task scripts/build_dist.sh: sdist + pure wheel + manylinux_2_28 kernel wheels (maturin --zig, both architectures). Dockerfile (multi-stage, kernels included, non-root user). packaging/conda/{simurg,simurg-core}. scripts/p10/digest.py: platform-independent content digests of GTC/VCF/summary outputs. Cloud task p10 installs the artifacts on a fresh Ubuntu VM and runs the tutorial with pip, Docker and conda installs. trial.sh picks the arm64/amd64 image from the instance type and can run instances in parallel (PARALLEL=1). Licence files ship in both wheels; Rust crate notices added. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01814GKVzZVL3FuF3DeiZXhv --- .dockerignore | 14 +++ Dockerfile | 28 ++++++ THIRD_PARTY_NOTICES.md | 1 + packaging/conda/simurg-core/meta.yaml | 40 +++++++++ packaging/conda/simurg/meta.yaml | 48 +++++++++++ pyproject.toml | 15 +++- rust/simurg-core/LICENSE | 21 +++++ rust/simurg-core/NOTICE | 6 ++ rust/simurg-core/pyproject.toml | 3 +- scripts/build_dist.sh | 22 +++++ scripts/cloud/tasks/p10.sh | 118 ++++++++++++++++++++++++++ scripts/cloud/trial.sh | 6 +- scripts/p10/digest.py | 86 +++++++++++++++++++ 13 files changed, 403 insertions(+), 5 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile create mode 100644 packaging/conda/simurg-core/meta.yaml create mode 100644 packaging/conda/simurg/meta.yaml create mode 100644 rust/simurg-core/LICENSE create mode 100644 rust/simurg-core/NOTICE create mode 100755 scripts/build_dist.sh create mode 100644 scripts/cloud/tasks/p10.sh create mode 100644 scripts/p10/digest.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..66eceb7 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,14 @@ +.git +.venv +data +refs +dist +results +_phase0_archive +**/__pycache__ +**/target +*.idat +*.idat.gz +*.bpm +*.egt +*.gtc diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..1e5ca1c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,28 @@ +# SimurgArray container: the `simurg` CLI with the compiled kernels. +# docker build -t simurg:1.0.0 . +# docker run --rm -v "$PWD":/work simurg:1.0.0 genotype --bpm M.bpm --egt C.egt --idats idats --out run/gtc +# Inputs and outputs live in the mounted /work folder; the image holds no data. +ARG PYTHON=3.12 + +FROM python:${PYTHON}-slim-bookworm AS build +RUN apt-get update -q && apt-get install -y -q --no-install-recommends curl gcc libc6-dev \ + && rm -rf /var/lib/apt/lists/* +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal +ENV PATH=/root/.cargo/bin:$PATH +WORKDIR /src +COPY pyproject.toml README.md LICENSE THIRD_PARTY_NOTICES.md ./ +COPY src ./src +COPY rust/simurg-core ./rust/simurg-core +RUN pip wheel --no-cache-dir --no-deps -w /wheels . ./rust/simurg-core + +FROM python:${PYTHON}-slim-bookworm +LABEL org.opencontainers.image.title="SimurgArray" \ + org.opencontainers.image.description="Research toolkit for Illumina Infinium genotyping BeadChip data" \ + org.opencontainers.image.licenses="MIT" +COPY --from=build /wheels /wheels +RUN pip install --no-cache-dir /wheels/*.whl && rm -rf /wheels \ + && useradd --create-home --uid 1000 simurg +USER simurg +WORKDIR /work +ENTRYPOINT ["simurg"] +CMD ["--help"] diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 85a9699..86aea9c 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -12,6 +12,7 @@ used only as external subprocess oracles and are not copied or linked. | [klib `ksort.h`](https://github.com/attractivechaos/klib) as shipped in htslib — Copyright (c) 2008, 2012-2013, 2017-2019 Genome Research Ltd | MIT | Quickselect `ks_ksmall` partition order, ported to `simurg/qc/contamination.py` (`ks_ksmall`) to reproduce BAFregress medians | | [Illumina GTCtoVCF](https://github.com/Illumina/GTCtoVCF) — Copyright Illumina, Inc. | Apache-2.0 | Indel source-sequence handling ported to `simurg/harmonize/sites.py` (design-strand orientation, left shift, IUPAC flank matching, deletion/insertion context score); multi-probe consistency idea in `simurg/harmonize/combine.py`; test oracle (run under Python 3 with two local compatibility patches, never shipped) | | [Picard `picard.arrays`](https://github.com/broadinstitute/picard) | MIT | Second format reference | +| [PyO3](https://github.com/PyO3/pyo3), [rust-numpy](https://github.com/PyO3/rust-numpy), [ndarray](https://github.com/rust-ndarray/ndarray) and their Rust dependencies | MIT / Apache-2.0 / BSD-2-Clause | Compiled into the optional `simurg_core` extension (`rust/simurg-core`); list with `cargo tree` | | [illuminaio](https://github.com/HenrikBengtsson/illuminaio) | GPL-2 | Documentation of IDAT field codes only; oracle via subprocess | ## BSD-2-Clause (BeadArrayFiles, verbatim from its README) diff --git a/packaging/conda/simurg-core/meta.yaml b/packaging/conda/simurg-core/meta.yaml new file mode 100644 index 0000000..ac3824b --- /dev/null +++ b/packaging/conda/simurg-core/meta.yaml @@ -0,0 +1,40 @@ +# conda recipe for the optional compiled kernels of SimurgArray (Rust, PyO3; abi3 for Python >= 3.12). +# conda build packaging/conda/simurg-core -c conda-forge +{% set version = load_file_regex(load_file="Cargo.toml", regex_pattern='version = "([^"]+)"', from_recipe_dir=False).group(1) %} + +package: + name: simurg-core + version: {{ version }} + +source: + path: ../../../rust/simurg-core + +build: + number: 0 + skip: true # [win] + script: {{ PYTHON }} -m pip install . --no-deps -vv + +requirements: + build: + - {{ compiler('c') }} + - {{ compiler('rust') }} + host: + - python >=3.12 + - pip + - maturin >=1.7,<2 + run: + - python >=3.12 + - numpy >=1.26 + +test: + imports: + - simurg_core + commands: + - python -c "import simurg_core, numpy as np; assert simurg_core.log10(np.array([0.0]))[0] == -np.inf" + +about: + summary: Optional compiled kernels of SimurgArray (results identical to the pure-Python paths) + license: MIT + license_file: + - LICENSE + - NOTICE diff --git a/packaging/conda/simurg/meta.yaml b/packaging/conda/simurg/meta.yaml new file mode 100644 index 0000000..b627427 --- /dev/null +++ b/packaging/conda/simurg/meta.yaml @@ -0,0 +1,48 @@ +# conda recipe for SimurgArray (pure Python). Build from a checkout: +# conda build packaging/conda/simurg -c conda-forge +{% set version = load_file_regex(load_file="src/simurg/__init__.py", regex_pattern='__version__ = "([^"]+)"', from_recipe_dir=False).group(1) %} + +package: + name: simurg + version: {{ version }} + +source: + path: ../../.. + +build: + noarch: python + number: 0 + script: {{ PYTHON }} -m pip install . --no-deps --no-build-isolation -vv + entry_points: + - simurg = simurg.cli.main:app + +requirements: + host: + - python >=3.12 + - pip + - hatchling >=1.27 + run: + - python >=3.12 + - numpy >=1.26 + - typer >=0.12 + - rich >=13 + - pyyaml >=6 + - pyarrow >=15 + - openpyxl >=3.1 + run_constrained: + - simurg-core =={{ version }} + +test: + imports: + - simurg + - simurg.cli.main + commands: + - simurg --version + - simurg --help + +about: + summary: Research toolkit for Illumina Infinium genotyping BeadChip data (IDAT to GTC, QC, VCF, PLINK) + license: MIT + license_file: + - LICENSE + - THIRD_PARTY_NOTICES.md diff --git a/pyproject.toml b/pyproject.toml index 174d9fb..49f772c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,14 +1,25 @@ [build-system] -requires = ["hatchling>=1.24"] +requires = ["hatchling>=1.27"] build-backend = "hatchling.build" [project] name = "simurg" description = "SimurgArray: open research toolkit for Illumina Infinium genotyping BeadChip data" readme = "README.md" -license = { text = "MIT" } +license = "MIT" +license-files = ["LICENSE", "THIRD_PARTY_NOTICES.md"] requires-python = ">=3.12" authors = [{ name = "Kemal Yaylali" }] +keywords = ["genotyping", "microarray", "infinium", "idat", "gtc", "vcf", "bioinformatics"] +classifiers = [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Science/Research", + "Operating System :: POSIX :: Linux", + "Operating System :: MacOS", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Topic :: Scientific/Engineering :: Bio-Informatics", +] dynamic = ["version"] dependencies = [ "numpy>=1.26", # all numerics diff --git a/rust/simurg-core/LICENSE b/rust/simurg-core/LICENSE new file mode 100644 index 0000000..33a742d --- /dev/null +++ b/rust/simurg-core/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Kemal Yaylali + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/rust/simurg-core/NOTICE b/rust/simurg-core/NOTICE new file mode 100644 index 0000000..a2fcc3a --- /dev/null +++ b/rust/simurg-core/NOTICE @@ -0,0 +1,6 @@ +simurg-core: optional compiled kernels of SimurgArray (MIT, see LICENSE). + +The kernels port numerics that SimurgArray's pure-Python modules port from gtc2vcf (MIT, (C) Giulio Genovese) and +Illumina BeadArrayFiles (BSD-2-Clause, (C) 2020 Illumina); full notices: THIRD_PARTY_NOTICES.md of SimurgArray. +The binary statically links PyO3, rust-numpy, ndarray and their dependencies (MIT / Apache-2.0 / BSD-2-Clause); +`cargo tree` lists them and each crate's licence ships with its source on crates.io. diff --git a/rust/simurg-core/pyproject.toml b/rust/simurg-core/pyproject.toml index f9148eb..2429209 100644 --- a/rust/simurg-core/pyproject.toml +++ b/rust/simurg-core/pyproject.toml @@ -6,7 +6,8 @@ build-backend = "maturin" name = "simurg-core" description = "Compiled kernels for SimurgArray (optional; the pure-Python paths give identical results)" requires-python = ">=3.12" -license = { text = "MIT" } +license = "MIT" +license-files = ["LICENSE", "NOTICE"] dependencies = ["numpy>=1.26"] dynamic = ["version"] diff --git a/scripts/build_dist.sh b/scripts/build_dist.sh new file mode 100755 index 0000000..8db7091 --- /dev/null +++ b/scripts/build_dist.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Build the release artifacts into dist/: +# simurg--py3-none-any.whl + simurg-.tar.gz (pure Python) +# simurg_core--cp312-abi3-manylinux_2_28_{aarch64,x86_64}.whl (compiled kernels, Python >= 3.12) +# Portable Linux wheels are linked against glibc 2.28 with `maturin --zig` (cross-compiles the other architecture). +# Needs: uv, a Rust toolchain (rustup), ~60 MB for zig on first use. +# bash scripts/build_dist.sh [aarch64|x86_64|all] (default: all) +set -euo pipefail +cd "$(dirname "$0")/.." +export PATH="$HOME/.cargo/bin:$PATH" +WHICH=${1:-all} +mkdir -p dist +uv build -o dist # sdist + pure wheel of simurg +MATURIN="uvx --from maturin[zig]>=1.7,<2 maturin" +for arch in aarch64 x86_64; do + [[ $WHICH == all || $WHICH == "$arch" ]] || continue + rustup target add "$arch-unknown-linux-gnu" > /dev/null + $MATURIN build --release --zig --compatibility manylinux_2_28 --target "$arch-unknown-linux-gnu" \ + -m rust/simurg-core/Cargo.toml -o dist +done +( cd dist && sha256sum ./*.whl ./*.tar.gz > SHA256SUMS ) +ls -la dist diff --git a/scripts/cloud/tasks/p10.sh b/scripts/cloud/tasks/p10.sh new file mode 100644 index 0000000..18c2da3 --- /dev/null +++ b/scripts/cloud/tasks/p10.sh @@ -0,0 +1,118 @@ +#!/bin/bash +# Phase P10 acceptance on a fresh Ubuntu 24.04 VM (plan §18: "clean install + tutorial works on fresh VM"): +# 1. build the release artifacts (scripts/build_dist.sh: sdist, pure wheel, manylinux_2_28 kernel wheels for both +# architectures); +# 2. install them with plain pip into new venvs of the system Python (with and without the kernels) and run the +# unit tests against the installed package; +# 3. fetch the public demo data and run the tutorial (examples/gsa_demo.sh, 6 samples) with each install; +# 4. build the Docker image and the conda packages, run genotype + gtc-to-vcf with them; +# 5. content digests of every output (scripts/p10/digest.py) must agree across all installs. +# Args: . Run with OMNI=0. +set -uo pipefail +OUT=$2 +R=$OUT/p10; W=/work/p10; mkdir -p $R $W +N=$(nproc) +ARCH=$(uname -m) +declare -A STEP +step() { STEP[$1]=$2; echo "--- $1: $2"; } + +export DEBIAN_FRONTEND=noninteractive +apt-get install -y -q gcc docker.io bsdextrautils > /dev/null +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal > /dev/null 2>&1 +export PATH="$HOME/.cargo/bin:$PATH" + +echo "=== 1. build artifacts $(date -u +%T) ===" +if bash scripts/build_dist.sh all > $R/build_dist.log 2>&1; then step build ok; else step build FAILED; tail -30 $R/build_dist.log; fi +cp dist/*.whl dist/*.tar.gz dist/SHA256SUMS $R/ 2>/dev/null +ls -la dist + +echo "=== 2. pip install into fresh venvs $(date -u +%T) ===" +PY=/usr/bin/python3 +$PY --version +CORE=$(ls dist/simurg_core-*manylinux_2_28_$ARCH.whl 2>/dev/null | head -1) +$PY -m venv $W/fresh && $W/fresh/bin/pip install -q dist/simurg-*-py3-none-any.whl "$CORE" \ + && step pip_install ok || step pip_install FAILED +$PY -m venv $W/pure && $W/pure/bin/pip install -q dist/simurg-*-py3-none-any.whl \ + && step pip_install_pure ok || step pip_install_pure FAILED +$W/fresh/bin/python -c "from simurg._core import backend; print('fresh:', backend())" +$W/pure/bin/python -c "from simurg._core import backend; print('pure:', backend())" +$W/fresh/bin/simurg --version +$W/fresh/bin/pip install -q pytest hypothesis +# tests import the installed package (the checkout keeps its sources under src/, not on sys.path) +( cd $W && cp -r /work/repo/tests /work/repo/docs . && $W/fresh/bin/python -m pytest tests/unit -q -p no:cacheprovider \ + -k "not selftest_in_browser" > $R/unit_tests_installed.txt 2>&1 ) && step unit_tests_installed ok \ + || { step unit_tests_installed FAILED; tail -20 $R/unit_tests_installed.txt; } +tail -3 $R/unit_tests_installed.txt + +echo "=== 3. tutorial $(date -u +%T) ===" +bash scripts/fetch_data.sh p0 > $W/fetch_p0.log 2>&1 && bash scripts/fetch_data.sh p6 > $W/fetch_p6.log 2>&1 \ + && step fetch_data ok || { step fetch_data FAILED; tail -5 $W/fetch_p0.log $W/fetch_p6.log; } +for v in fresh pure; do + t0=$SECONDS + SIMURG=$W/$v/bin/simurg THREADS=$N bash examples/gsa_demo.sh $W/tut_$v 6 > $R/tutorial_$v.log 2>&1 \ + && step tutorial_$v "ok ($((SECONDS - t0)) s)" || { step tutorial_$v FAILED; tail -20 $R/tutorial_$v.log; } + $W/fresh/bin/python scripts/p10/digest.py $W/tut_$v/gtc --out $R/digest_gtc_$v.json + $W/fresh/bin/python scripts/p10/digest.py $W/tut_$v/vcf --out $R/digest_vcf_$v.json +done + +echo "=== 4a. docker $(date -u +%T) ===" +D=data/p0/gsa_v4_pgx +BPM=$D/GSA-PGx-48v4-0_20079540_E2_bpm/GSA-PGx-48v4-0_20079540_E2.bpm +CSV=$D/GSA-PGx-48v4-0_20079540_E2_csv/GSA-PGx-48v4-0_20079540_E2.csv +EGT=$D/GSA-PGx-48v4-0_20079540_E2_ClusterFile/GSA-PGx-48v4-0_20079540_E2_ClusterFile.egt +FASTA=data/p6/GRCh38.primary.fa +if docker build -q -t simurg:test . > $R/docker_build.log 2>&1; then + step docker_build ok + docker image ls simurg:test --format '{{.Size}}' | sed 's/^/image size: /' + DR="docker run --rm --user 0:0 -v /work:/work -w /work/repo simurg:test" + $DR --version + $DR genotype --bpm $BPM --egt $EGT --csv $CSV --idats $W/tut_fresh/idats --out $W/docker/gtc \ + --qc-profile genotyping --threads $N --quiet --force \ + && $DR gtc-to-vcf --bpm $BPM --csv $CSV --fasta $FASTA --gtcs $W/docker/gtc --out $W/docker/vcf --threads $N \ + --quiet --force && step docker_run ok || step docker_run FAILED + $W/fresh/bin/python scripts/p10/digest.py $W/docker/gtc --out $R/digest_gtc_docker.json + $W/fresh/bin/python scripts/p10/digest.py $W/docker/vcf --out $R/digest_vcf_docker.json +else + step docker_build FAILED; tail -30 $R/docker_build.log +fi + +echo "=== 4b. conda $(date -u +%T) ===" +curl -sSfL -o $W/miniforge.sh "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-$ARCH.sh" \ + && bash $W/miniforge.sh -b -p $W/conda > /dev/null +C=$W/conda/bin +if $C/conda install -y -q -n base conda-build > $R/conda.log 2>&1 \ + && $C/conda build -c conda-forge --output-folder $W/condabld packaging/conda/simurg-core packaging/conda/simurg \ + >> $R/conda.log 2>&1; then + step conda_build ok + $C/conda create -y -q -p $W/cenv -c $W/condabld -c conda-forge simurg simurg-core >> $R/conda.log 2>&1 \ + && $W/cenv/bin/simurg --version \ + && $W/cenv/bin/python -c "from simurg._core import backend; print('conda:', backend())" \ + && $W/cenv/bin/simurg genotype --bpm $BPM --egt $EGT --idats $W/tut_fresh/idats --out $W/conda_run/gtc \ + --threads $N --quiet --force \ + && step conda_run ok || { step conda_run FAILED; tail -20 $R/conda.log; } + $W/fresh/bin/python scripts/p10/digest.py $W/conda_run/gtc --out $R/digest_gtc_conda.json +else + step conda_build FAILED; tail -40 $R/conda.log +fi + +echo "=== 5. digests $(date -u +%T) ===" +$W/fresh/bin/python - "$R" <<'PY' | tee $R/agreement.txt +import json, sys +from pathlib import Path +r = Path(sys.argv[1]) +for kind in ("gtc", "vcf"): + runs = {p.stem.split("_", 2)[2]: json.loads(p.read_text())[kind] for p in sorted(r.glob(f"digest_{kind}_*.json"))} + if not runs: + continue + ref_name, ref = "fresh", runs.get("fresh", {}) + for name, d in runs.items(): + same = d == ref and len(d) > 0 + print(f"{kind}: {name} vs {ref_name}: {'IDENTICAL' if same else 'DIFFERENT'} ({len(d)} files)") +PY +{ + echo "{"; echo " \"arch\": \"$ARCH\"," + for k in "${!STEP[@]}"; do echo " \"$k\": \"${STEP[$k]}\","; done + echo " \"python\": \"$($PY --version)\""; echo "}" +} > $R/steps.json +cat $R/steps.json +echo "=== task done $(date -u +%T) ===" diff --git a/scripts/cloud/trial.sh b/scripts/cloud/trial.sh index 3ac307e..f48fd50 100755 --- a/scripts/cloud/trial.sh +++ b/scripts/cloud/trial.sh @@ -14,6 +14,8 @@ set -euo pipefail cd "$(dirname "$0")/../.." REGION=${REGION:-us-east-1} TYPE=${TYPE:-m7g.4xlarge} +# Graviton families carry a "g" after the generation number (m7g, c7gn, r8g): arm64 image; everything else amd64 +if [[ $TYPE =~ ^[a-z]+[0-9]+g ]]; then ARCH=${ARCH:-arm64}; else ARCH=${ARCH:-amd64}; fi MAX_PRICE=${MAX_PRICE:-0.30} # USD/hour spot cap (on-demand is 0.6528) MAX_MINUTES=${MAX_MINUTES:-360} # hard cap: 6 hours DISK_GB=${DISK_GB:-250} @@ -59,7 +61,7 @@ EOF launch) running=$($AWS ec2 describe-instances --filters Name=tag:Project,Values=$TAG \ Name=instance-state-name,Values=pending,running --query 'length(Reservations[].Instances[])') - [[ "$running" == "0" ]] || { echo "a trial instance is already running; refusing to start another"; exit 1; } + [[ "$running" == "0" || "${PARALLEL:-0}" == "1" ]] || { echo "a trial instance is already running; refusing to start another (PARALLEL=1 overrides)"; exit 1; } RUN=run-$(date -u +%Y%m%dT%H%M%SZ) echo "-> packing repository at $(git rev-parse --short HEAD) -> s3://$BUCKET/$RUN/" git archive --format=tar.gz -o /tmp/simurg-repo.tar.gz HEAD @@ -69,7 +71,7 @@ launch) sed -e "s/__BUCKET__/$BUCKET/" -e "s/__RUN__/$RUN/" -e "s/__MAX_MINUTES__/$MAX_MINUTES/" -e "s/__TASK__/$TASK/g" \ -e "s/__OMNI__/$OMNI/" \ scripts/cloud/userdata.sh > /tmp/simurg-userdata.sh - AMI=$($AWS ssm get-parameter --name /aws/service/canonical/ubuntu/server/24.04/stable/current/arm64/hvm/ebs-gp3/ami-id \ + AMI=$($AWS ssm get-parameter --name /aws/service/canonical/ubuntu/server/24.04/stable/current/$ARCH/hvm/ebs-gp3/ami-id \ --query Parameter.Value --output text) echo "-> task: $TASK" echo "-> launching $TYPE spot (cap \$$MAX_PRICE/h, hard limit ${MAX_MINUTES} min, ${DISK_GB} GB disk deleted on exit)" diff --git a/scripts/p10/digest.py b/scripts/p10/digest.py new file mode 100644 index 0000000..42b2d14 --- /dev/null +++ b/scripts/p10/digest.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +"""Content digests of SimurgArray outputs, for comparing runs across machines, architectures and installs (P10). + +GTC: every numeric field and sample statistic, NaNs canonicalised (their sign/payload is platform-specific, K-014); +free-text metadata (dates, software version) is left out. VCF(.gz): all data lines, header lines excluded. +TSV: the listed columns of sample_summary.tsv. + + python scripts/p10/digest.py RUN_DIR [--out digest.json] +""" + +from __future__ import annotations + +import argparse +import gzip +import hashlib +import json +import sys +from pathlib import Path + +import numpy as np + +from simurg.formats.gtc import read_gtc + +GTC_FIELDS = ("normalization_transforms", "controls_x", "controls_y", "raw_x", "raw_y", "genotypes", "base_calls", + "genotype_scores", "b_allele_freqs", "logr_ratios", "percentiles_x", "percentiles_y", "sample_data", + "call_rate", "gender", "logr_dev", "gc10") # fmt: skip +SUMMARY_COLUMNS = ("sample_id", "status", "autosomal_call_rate", "call_rate", "logr_dev", "sex_est", "gc10", + "gc50", "n_calls", "n_nocalls", "het_rate", "contamination", "functional_status", "control_status", + "flags") # fmt: skip + + +def _canon(a: np.ndarray) -> bytes: + if a.dtype.names: # structured: canonicalise float members + a = a.copy() + for n in a.dtype.names: + if np.issubdtype(a.dtype[n], np.floating): + v = a[n] + v[np.isnan(v)] = np.nan + return a.tobytes() + if np.issubdtype(a.dtype, np.floating): + a = np.where(np.isnan(a), np.array(np.nan, a.dtype), a) + return np.ascontiguousarray(a).tobytes() + + +def gtc_digest(path: Path) -> dict[str, str]: + g = read_gtc(path) + out = {} + for f in GTC_FIELDS: + v = getattr(g, f) + b = _canon(np.asarray(v)) if not isinstance(v, bytes) else v + out[f] = hashlib.sha256(b).hexdigest()[:16] + return out + + +def vcf_digest(path: Path) -> str: + h = hashlib.sha256() + with gzip.open(path, "rt") if path.suffix == ".gz" else open(path) as fh: + for line in fh: + if not line.startswith("#"): + h.update(line.encode()) + return h.hexdigest()[:16] + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("run", type=Path) + ap.add_argument("--out", type=Path) + a = ap.parse_args() + d: dict[str, object] = {} + d["gtc"] = {p.stem: gtc_digest(p) for p in sorted(a.run.rglob("*.gtc"))} + d["vcf"] = {p.name: vcf_digest(p) for p in sorted(a.run.rglob("*.vcf.gz"))} + summ = {} + for p in sorted(a.run.rglob("sample_summary.tsv")): + rows = [ln.rstrip("\n").split("\t") for ln in p.open()] + idx = [rows[0].index(c) for c in SUMMARY_COLUMNS if c in rows[0]] + summ[str(p.relative_to(a.run))] = hashlib.sha256( + "\n".join("\t".join(r[i] for i in idx) for r in rows).encode() + ).hexdigest()[:16] + d["summary"] = summ + text = json.dumps(d, indent=1, sort_keys=True) + (a.out.write_text(text) if a.out else sys.stdout.write(text + "\n")) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())