P10 (WIP): release build script, Dockerfile, conda recipes, fresh-VM acceptance task
scripts/build_dist.sh: sdist + pure wheel + manylinux_2_28 kernel wheels (maturin --zig,
both architectures). Dockerfile (multi-stage, kernels included, non-root user).
packaging/conda/{simurg,simurg-core}. scripts/p10/digest.py: platform-independent content
digests of GTC/VCF/summary outputs. Cloud task p10 installs the artifacts on a fresh
Ubuntu VM and runs the tutorial with pip, Docker and conda installs. trial.sh picks the
arm64/amd64 image from the instance type and can run instances in parallel (PARALLEL=1).
Licence files ship in both wheels; Rust crate notices added.
This commit is contained in:
1 parent
7d2dbe939e
commit
1f2abcbdba
13 files changed
+403
-5
No files matched your search
@@ -0,0 +1,14 @@
|
|||||||
|
.git
|
||||||
|
.venv
|
||||||
|
data
|
||||||
|
refs
|
||||||
|
dist
|
||||||
|
results
|
||||||
|
_phase0_archive
|
||||||
|
**/__pycache__
|
||||||
|
**/target
|
||||||
|
*.idat
|
||||||
|
*.idat.gz
|
||||||
|
*.bpm
|
||||||
|
*.egt
|
||||||
|
*.gtc
|
||||||
+28
@@ -0,0 +1,28 @@
|
|||||||
|
# SimurgArray container: the `simurg` CLI with the compiled kernels.
|
||||||
|
# docker build -t simurg:1.0.0 .
|
||||||
|
# docker run --rm -v "$PWD":/work simurg:1.0.0 genotype --bpm M.bpm --egt C.egt --idats idats --out run/gtc
|
||||||
|
# Inputs and outputs live in the mounted /work folder; the image holds no data.
|
||||||
|
ARG PYTHON=3.12
|
||||||
|
|
||||||
|
FROM python:${PYTHON}-slim-bookworm AS build
|
||||||
|
RUN apt-get update -q && apt-get install -y -q --no-install-recommends curl gcc libc6-dev \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal
|
||||||
|
ENV PATH=/root/.cargo/bin:$PATH
|
||||||
|
WORKDIR /src
|
||||||
|
COPY pyproject.toml README.md LICENSE THIRD_PARTY_NOTICES.md ./
|
||||||
|
COPY src ./src
|
||||||
|
COPY rust/simurg-core ./rust/simurg-core
|
||||||
|
RUN pip wheel --no-cache-dir --no-deps -w /wheels . ./rust/simurg-core
|
||||||
|
|
||||||
|
FROM python:${PYTHON}-slim-bookworm
|
||||||
|
LABEL org.opencontainers.image.title="SimurgArray" \
|
||||||
|
org.opencontainers.image.description="Research toolkit for Illumina Infinium genotyping BeadChip data" \
|
||||||
|
org.opencontainers.image.licenses="MIT"
|
||||||
|
COPY --from=build /wheels /wheels
|
||||||
|
RUN pip install --no-cache-dir /wheels/*.whl && rm -rf /wheels \
|
||||||
|
&& useradd --create-home --uid 1000 simurg
|
||||||
|
USER simurg
|
||||||
|
WORKDIR /work
|
||||||
|
ENTRYPOINT ["simurg"]
|
||||||
|
CMD ["--help"]
|
||||||
@@ -12,6 +12,7 @@ used only as external subprocess oracles and are not copied or linked.
|
|||||||
| [klib `ksort.h`](https://github.com/attractivechaos/klib) as shipped in htslib — Copyright (c) 2008, 2012-2013, 2017-2019 Genome Research Ltd | MIT | Quickselect `ks_ksmall` partition order, ported to `simurg/qc/contamination.py` (`ks_ksmall`) to reproduce BAFregress medians |
|
| [klib `ksort.h`](https://github.com/attractivechaos/klib) as shipped in htslib — Copyright (c) 2008, 2012-2013, 2017-2019 Genome Research Ltd | MIT | Quickselect `ks_ksmall` partition order, ported to `simurg/qc/contamination.py` (`ks_ksmall`) to reproduce BAFregress medians |
|
||||||
| [Illumina GTCtoVCF](https://github.com/Illumina/GTCtoVCF) — Copyright Illumina, Inc. | Apache-2.0 | Indel source-sequence handling ported to `simurg/harmonize/sites.py` (design-strand orientation, left shift, IUPAC flank matching, deletion/insertion context score); multi-probe consistency idea in `simurg/harmonize/combine.py`; test oracle (run under Python 3 with two local compatibility patches, never shipped) |
|
| [Illumina GTCtoVCF](https://github.com/Illumina/GTCtoVCF) — Copyright Illumina, Inc. | Apache-2.0 | Indel source-sequence handling ported to `simurg/harmonize/sites.py` (design-strand orientation, left shift, IUPAC flank matching, deletion/insertion context score); multi-probe consistency idea in `simurg/harmonize/combine.py`; test oracle (run under Python 3 with two local compatibility patches, never shipped) |
|
||||||
| [Picard `picard.arrays`](https://github.com/broadinstitute/picard) | MIT | Second format reference |
|
| [Picard `picard.arrays`](https://github.com/broadinstitute/picard) | MIT | Second format reference |
|
||||||
|
| [PyO3](https://github.com/PyO3/pyo3), [rust-numpy](https://github.com/PyO3/rust-numpy), [ndarray](https://github.com/rust-ndarray/ndarray) and their Rust dependencies | MIT / Apache-2.0 / BSD-2-Clause | Compiled into the optional `simurg_core` extension (`rust/simurg-core`); list with `cargo tree` |
|
||||||
| [illuminaio](https://github.com/HenrikBengtsson/illuminaio) | GPL-2 | Documentation of IDAT field codes only; oracle via subprocess |
|
| [illuminaio](https://github.com/HenrikBengtsson/illuminaio) | GPL-2 | Documentation of IDAT field codes only; oracle via subprocess |
|
||||||
|
|
||||||
## BSD-2-Clause (BeadArrayFiles, verbatim from its README)
|
## BSD-2-Clause (BeadArrayFiles, verbatim from its README)
|
||||||
|
|||||||
@@ -0,0 +1,40 @@
|
|||||||
|
# conda recipe for the optional compiled kernels of SimurgArray (Rust, PyO3; abi3 for Python >= 3.12).
|
||||||
|
# conda build packaging/conda/simurg-core -c conda-forge
|
||||||
|
{% set version = load_file_regex(load_file="Cargo.toml", regex_pattern='version = "([^"]+)"', from_recipe_dir=False).group(1) %}
|
||||||
|
|
||||||
|
package:
|
||||||
|
name: simurg-core
|
||||||
|
version: {{ version }}
|
||||||
|
|
||||||
|
source:
|
||||||
|
path: ../../../rust/simurg-core
|
||||||
|
|
||||||
|
build:
|
||||||
|
number: 0
|
||||||
|
skip: true # [win]
|
||||||
|
script: {{ PYTHON }} -m pip install . --no-deps -vv
|
||||||
|
|
||||||
|
requirements:
|
||||||
|
build:
|
||||||
|
- {{ compiler('c') }}
|
||||||
|
- {{ compiler('rust') }}
|
||||||
|
host:
|
||||||
|
- python >=3.12
|
||||||
|
- pip
|
||||||
|
- maturin >=1.7,<2
|
||||||
|
run:
|
||||||
|
- python >=3.12
|
||||||
|
- numpy >=1.26
|
||||||
|
|
||||||
|
test:
|
||||||
|
imports:
|
||||||
|
- simurg_core
|
||||||
|
commands:
|
||||||
|
- python -c "import simurg_core, numpy as np; assert simurg_core.log10(np.array([0.0]))[0] == -np.inf"
|
||||||
|
|
||||||
|
about:
|
||||||
|
summary: Optional compiled kernels of SimurgArray (results identical to the pure-Python paths)
|
||||||
|
license: MIT
|
||||||
|
license_file:
|
||||||
|
- LICENSE
|
||||||
|
- NOTICE
|
||||||
@@ -0,0 +1,48 @@
|
|||||||
|
# conda recipe for SimurgArray (pure Python). Build from a checkout:
|
||||||
|
# conda build packaging/conda/simurg -c conda-forge
|
||||||
|
{% set version = load_file_regex(load_file="src/simurg/__init__.py", regex_pattern='__version__ = "([^"]+)"', from_recipe_dir=False).group(1) %}
|
||||||
|
|
||||||
|
package:
|
||||||
|
name: simurg
|
||||||
|
version: {{ version }}
|
||||||
|
|
||||||
|
source:
|
||||||
|
path: ../../..
|
||||||
|
|
||||||
|
build:
|
||||||
|
noarch: python
|
||||||
|
number: 0
|
||||||
|
script: {{ PYTHON }} -m pip install . --no-deps --no-build-isolation -vv
|
||||||
|
entry_points:
|
||||||
|
- simurg = simurg.cli.main:app
|
||||||
|
|
||||||
|
requirements:
|
||||||
|
host:
|
||||||
|
- python >=3.12
|
||||||
|
- pip
|
||||||
|
- hatchling >=1.27
|
||||||
|
run:
|
||||||
|
- python >=3.12
|
||||||
|
- numpy >=1.26
|
||||||
|
- typer >=0.12
|
||||||
|
- rich >=13
|
||||||
|
- pyyaml >=6
|
||||||
|
- pyarrow >=15
|
||||||
|
- openpyxl >=3.1
|
||||||
|
run_constrained:
|
||||||
|
- simurg-core =={{ version }}
|
||||||
|
|
||||||
|
test:
|
||||||
|
imports:
|
||||||
|
- simurg
|
||||||
|
- simurg.cli.main
|
||||||
|
commands:
|
||||||
|
- simurg --version
|
||||||
|
- simurg --help
|
||||||
|
|
||||||
|
about:
|
||||||
|
summary: Research toolkit for Illumina Infinium genotyping BeadChip data (IDAT to GTC, QC, VCF, PLINK)
|
||||||
|
license: MIT
|
||||||
|
license_file:
|
||||||
|
- LICENSE
|
||||||
|
- THIRD_PARTY_NOTICES.md
|
||||||
+13
-2
@@ -1,14 +1,25 @@
|
|||||||
[build-system]
|
[build-system]
|
||||||
requires = ["hatchling>=1.24"]
|
requires = ["hatchling>=1.27"]
|
||||||
build-backend = "hatchling.build"
|
build-backend = "hatchling.build"
|
||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "simurg"
|
name = "simurg"
|
||||||
description = "SimurgArray: open research toolkit for Illumina Infinium genotyping BeadChip data"
|
description = "SimurgArray: open research toolkit for Illumina Infinium genotyping BeadChip data"
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
license = { text = "MIT" }
|
license = "MIT"
|
||||||
|
license-files = ["LICENSE", "THIRD_PARTY_NOTICES.md"]
|
||||||
requires-python = ">=3.12"
|
requires-python = ">=3.12"
|
||||||
authors = [{ name = "Kemal Yaylali" }]
|
authors = [{ name = "Kemal Yaylali" }]
|
||||||
|
keywords = ["genotyping", "microarray", "infinium", "idat", "gtc", "vcf", "bioinformatics"]
|
||||||
|
classifiers = [
|
||||||
|
"Development Status :: 5 - Production/Stable",
|
||||||
|
"Intended Audience :: Science/Research",
|
||||||
|
"Operating System :: POSIX :: Linux",
|
||||||
|
"Operating System :: MacOS",
|
||||||
|
"Programming Language :: Python :: 3.12",
|
||||||
|
"Programming Language :: Python :: 3.13",
|
||||||
|
"Topic :: Scientific/Engineering :: Bio-Informatics",
|
||||||
|
]
|
||||||
dynamic = ["version"]
|
dynamic = ["version"]
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"numpy>=1.26", # all numerics
|
"numpy>=1.26", # all numerics
|
||||||
|
|||||||
@@ -0,0 +1,21 @@
|
|||||||
|
MIT License
|
||||||
|
|
||||||
|
Copyright (c) 2026 Kemal Yaylali
|
||||||
|
|
||||||
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||||
|
of this software and associated documentation files (the "Software"), to deal
|
||||||
|
in the Software without restriction, including without limitation the rights
|
||||||
|
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||||
|
copies of the Software, and to permit persons to whom the Software is
|
||||||
|
furnished to do so, subject to the following conditions:
|
||||||
|
|
||||||
|
The above copyright notice and this permission notice shall be included in all
|
||||||
|
copies or substantial portions of the Software.
|
||||||
|
|
||||||
|
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||||
|
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||||
|
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||||
|
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||||
|
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||||
|
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||||
|
SOFTWARE.
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
simurg-core: optional compiled kernels of SimurgArray (MIT, see LICENSE).
|
||||||
|
|
||||||
|
The kernels port numerics that SimurgArray's pure-Python modules port from gtc2vcf (MIT, (C) Giulio Genovese) and
|
||||||
|
Illumina BeadArrayFiles (BSD-2-Clause, (C) 2020 Illumina); full notices: THIRD_PARTY_NOTICES.md of SimurgArray.
|
||||||
|
The binary statically links PyO3, rust-numpy, ndarray and their dependencies (MIT / Apache-2.0 / BSD-2-Clause);
|
||||||
|
`cargo tree` lists them and each crate's licence ships with its source on crates.io.
|
||||||
@@ -6,7 +6,8 @@ build-backend = "maturin"
|
|||||||
name = "simurg-core"
|
name = "simurg-core"
|
||||||
description = "Compiled kernels for SimurgArray (optional; the pure-Python paths give identical results)"
|
description = "Compiled kernels for SimurgArray (optional; the pure-Python paths give identical results)"
|
||||||
requires-python = ">=3.12"
|
requires-python = ">=3.12"
|
||||||
license = { text = "MIT" }
|
license = "MIT"
|
||||||
|
license-files = ["LICENSE", "NOTICE"]
|
||||||
dependencies = ["numpy>=1.26"]
|
dependencies = ["numpy>=1.26"]
|
||||||
dynamic = ["version"]
|
dynamic = ["version"]
|
||||||
|
|
||||||
|
|||||||
Executable
+22
@@ -0,0 +1,22 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Build the release artifacts into dist/:
|
||||||
|
# simurg-<v>-py3-none-any.whl + simurg-<v>.tar.gz (pure Python)
|
||||||
|
# simurg_core-<v>-cp312-abi3-manylinux_2_28_{aarch64,x86_64}.whl (compiled kernels, Python >= 3.12)
|
||||||
|
# Portable Linux wheels are linked against glibc 2.28 with `maturin --zig` (cross-compiles the other architecture).
|
||||||
|
# Needs: uv, a Rust toolchain (rustup), ~60 MB for zig on first use.
|
||||||
|
# bash scripts/build_dist.sh [aarch64|x86_64|all] (default: all)
|
||||||
|
set -euo pipefail
|
||||||
|
cd "$(dirname "$0")/.."
|
||||||
|
export PATH="$HOME/.cargo/bin:$PATH"
|
||||||
|
WHICH=${1:-all}
|
||||||
|
mkdir -p dist
|
||||||
|
uv build -o dist # sdist + pure wheel of simurg
|
||||||
|
MATURIN="uvx --from maturin[zig]>=1.7,<2 maturin"
|
||||||
|
for arch in aarch64 x86_64; do
|
||||||
|
[[ $WHICH == all || $WHICH == "$arch" ]] || continue
|
||||||
|
rustup target add "$arch-unknown-linux-gnu" > /dev/null
|
||||||
|
$MATURIN build --release --zig --compatibility manylinux_2_28 --target "$arch-unknown-linux-gnu" \
|
||||||
|
-m rust/simurg-core/Cargo.toml -o dist
|
||||||
|
done
|
||||||
|
( cd dist && sha256sum ./*.whl ./*.tar.gz > SHA256SUMS )
|
||||||
|
ls -la dist
|
||||||
@@ -0,0 +1,118 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# Phase P10 acceptance on a fresh Ubuntu 24.04 VM (plan §18: "clean install + tutorial works on fresh VM"):
|
||||||
|
# 1. build the release artifacts (scripts/build_dist.sh: sdist, pure wheel, manylinux_2_28 kernel wheels for both
|
||||||
|
# architectures);
|
||||||
|
# 2. install them with plain pip into new venvs of the system Python (with and without the kernels) and run the
|
||||||
|
# unit tests against the installed package;
|
||||||
|
# 3. fetch the public demo data and run the tutorial (examples/gsa_demo.sh, 6 samples) with each install;
|
||||||
|
# 4. build the Docker image and the conda packages, run genotype + gtc-to-vcf with them;
|
||||||
|
# 5. content digests of every output (scripts/p10/digest.py) must agree across all installs.
|
||||||
|
# Args: <unused data dir> <results dir>. Run with OMNI=0.
|
||||||
|
set -uo pipefail
|
||||||
|
OUT=$2
|
||||||
|
R=$OUT/p10; W=/work/p10; mkdir -p $R $W
|
||||||
|
N=$(nproc)
|
||||||
|
ARCH=$(uname -m)
|
||||||
|
declare -A STEP
|
||||||
|
step() { STEP[$1]=$2; echo "--- $1: $2"; }
|
||||||
|
|
||||||
|
export DEBIAN_FRONTEND=noninteractive
|
||||||
|
apt-get install -y -q gcc docker.io bsdextrautils > /dev/null
|
||||||
|
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal > /dev/null 2>&1
|
||||||
|
export PATH="$HOME/.cargo/bin:$PATH"
|
||||||
|
|
||||||
|
echo "=== 1. build artifacts $(date -u +%T) ==="
|
||||||
|
if bash scripts/build_dist.sh all > $R/build_dist.log 2>&1; then step build ok; else step build FAILED; tail -30 $R/build_dist.log; fi
|
||||||
|
cp dist/*.whl dist/*.tar.gz dist/SHA256SUMS $R/ 2>/dev/null
|
||||||
|
ls -la dist
|
||||||
|
|
||||||
|
echo "=== 2. pip install into fresh venvs $(date -u +%T) ==="
|
||||||
|
PY=/usr/bin/python3
|
||||||
|
$PY --version
|
||||||
|
CORE=$(ls dist/simurg_core-*manylinux_2_28_$ARCH.whl 2>/dev/null | head -1)
|
||||||
|
$PY -m venv $W/fresh && $W/fresh/bin/pip install -q dist/simurg-*-py3-none-any.whl "$CORE" \
|
||||||
|
&& step pip_install ok || step pip_install FAILED
|
||||||
|
$PY -m venv $W/pure && $W/pure/bin/pip install -q dist/simurg-*-py3-none-any.whl \
|
||||||
|
&& step pip_install_pure ok || step pip_install_pure FAILED
|
||||||
|
$W/fresh/bin/python -c "from simurg._core import backend; print('fresh:', backend())"
|
||||||
|
$W/pure/bin/python -c "from simurg._core import backend; print('pure:', backend())"
|
||||||
|
$W/fresh/bin/simurg --version
|
||||||
|
$W/fresh/bin/pip install -q pytest hypothesis
|
||||||
|
# tests import the installed package (the checkout keeps its sources under src/, not on sys.path)
|
||||||
|
( cd $W && cp -r /work/repo/tests /work/repo/docs . && $W/fresh/bin/python -m pytest tests/unit -q -p no:cacheprovider \
|
||||||
|
-k "not selftest_in_browser" > $R/unit_tests_installed.txt 2>&1 ) && step unit_tests_installed ok \
|
||||||
|
|| { step unit_tests_installed FAILED; tail -20 $R/unit_tests_installed.txt; }
|
||||||
|
tail -3 $R/unit_tests_installed.txt
|
||||||
|
|
||||||
|
echo "=== 3. tutorial $(date -u +%T) ==="
|
||||||
|
bash scripts/fetch_data.sh p0 > $W/fetch_p0.log 2>&1 && bash scripts/fetch_data.sh p6 > $W/fetch_p6.log 2>&1 \
|
||||||
|
&& step fetch_data ok || { step fetch_data FAILED; tail -5 $W/fetch_p0.log $W/fetch_p6.log; }
|
||||||
|
for v in fresh pure; do
|
||||||
|
t0=$SECONDS
|
||||||
|
SIMURG=$W/$v/bin/simurg THREADS=$N bash examples/gsa_demo.sh $W/tut_$v 6 > $R/tutorial_$v.log 2>&1 \
|
||||||
|
&& step tutorial_$v "ok ($((SECONDS - t0)) s)" || { step tutorial_$v FAILED; tail -20 $R/tutorial_$v.log; }
|
||||||
|
$W/fresh/bin/python scripts/p10/digest.py $W/tut_$v/gtc --out $R/digest_gtc_$v.json
|
||||||
|
$W/fresh/bin/python scripts/p10/digest.py $W/tut_$v/vcf --out $R/digest_vcf_$v.json
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "=== 4a. docker $(date -u +%T) ==="
|
||||||
|
D=data/p0/gsa_v4_pgx
|
||||||
|
BPM=$D/GSA-PGx-48v4-0_20079540_E2_bpm/GSA-PGx-48v4-0_20079540_E2.bpm
|
||||||
|
CSV=$D/GSA-PGx-48v4-0_20079540_E2_csv/GSA-PGx-48v4-0_20079540_E2.csv
|
||||||
|
EGT=$D/GSA-PGx-48v4-0_20079540_E2_ClusterFile/GSA-PGx-48v4-0_20079540_E2_ClusterFile.egt
|
||||||
|
FASTA=data/p6/GRCh38.primary.fa
|
||||||
|
if docker build -q -t simurg:test . > $R/docker_build.log 2>&1; then
|
||||||
|
step docker_build ok
|
||||||
|
docker image ls simurg:test --format '{{.Size}}' | sed 's/^/image size: /'
|
||||||
|
DR="docker run --rm --user 0:0 -v /work:/work -w /work/repo simurg:test"
|
||||||
|
$DR --version
|
||||||
|
$DR genotype --bpm $BPM --egt $EGT --csv $CSV --idats $W/tut_fresh/idats --out $W/docker/gtc \
|
||||||
|
--qc-profile genotyping --threads $N --quiet --force \
|
||||||
|
&& $DR gtc-to-vcf --bpm $BPM --csv $CSV --fasta $FASTA --gtcs $W/docker/gtc --out $W/docker/vcf --threads $N \
|
||||||
|
--quiet --force && step docker_run ok || step docker_run FAILED
|
||||||
|
$W/fresh/bin/python scripts/p10/digest.py $W/docker/gtc --out $R/digest_gtc_docker.json
|
||||||
|
$W/fresh/bin/python scripts/p10/digest.py $W/docker/vcf --out $R/digest_vcf_docker.json
|
||||||
|
else
|
||||||
|
step docker_build FAILED; tail -30 $R/docker_build.log
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "=== 4b. conda $(date -u +%T) ==="
|
||||||
|
curl -sSfL -o $W/miniforge.sh "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-$ARCH.sh" \
|
||||||
|
&& bash $W/miniforge.sh -b -p $W/conda > /dev/null
|
||||||
|
C=$W/conda/bin
|
||||||
|
if $C/conda install -y -q -n base conda-build > $R/conda.log 2>&1 \
|
||||||
|
&& $C/conda build -c conda-forge --output-folder $W/condabld packaging/conda/simurg-core packaging/conda/simurg \
|
||||||
|
>> $R/conda.log 2>&1; then
|
||||||
|
step conda_build ok
|
||||||
|
$C/conda create -y -q -p $W/cenv -c $W/condabld -c conda-forge simurg simurg-core >> $R/conda.log 2>&1 \
|
||||||
|
&& $W/cenv/bin/simurg --version \
|
||||||
|
&& $W/cenv/bin/python -c "from simurg._core import backend; print('conda:', backend())" \
|
||||||
|
&& $W/cenv/bin/simurg genotype --bpm $BPM --egt $EGT --idats $W/tut_fresh/idats --out $W/conda_run/gtc \
|
||||||
|
--threads $N --quiet --force \
|
||||||
|
&& step conda_run ok || { step conda_run FAILED; tail -20 $R/conda.log; }
|
||||||
|
$W/fresh/bin/python scripts/p10/digest.py $W/conda_run/gtc --out $R/digest_gtc_conda.json
|
||||||
|
else
|
||||||
|
step conda_build FAILED; tail -40 $R/conda.log
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "=== 5. digests $(date -u +%T) ==="
|
||||||
|
$W/fresh/bin/python - "$R" <<'PY' | tee $R/agreement.txt
|
||||||
|
import json, sys
|
||||||
|
from pathlib import Path
|
||||||
|
r = Path(sys.argv[1])
|
||||||
|
for kind in ("gtc", "vcf"):
|
||||||
|
runs = {p.stem.split("_", 2)[2]: json.loads(p.read_text())[kind] for p in sorted(r.glob(f"digest_{kind}_*.json"))}
|
||||||
|
if not runs:
|
||||||
|
continue
|
||||||
|
ref_name, ref = "fresh", runs.get("fresh", {})
|
||||||
|
for name, d in runs.items():
|
||||||
|
same = d == ref and len(d) > 0
|
||||||
|
print(f"{kind}: {name} vs {ref_name}: {'IDENTICAL' if same else 'DIFFERENT'} ({len(d)} files)")
|
||||||
|
PY
|
||||||
|
{
|
||||||
|
echo "{"; echo " \"arch\": \"$ARCH\","
|
||||||
|
for k in "${!STEP[@]}"; do echo " \"$k\": \"${STEP[$k]}\","; done
|
||||||
|
echo " \"python\": \"$($PY --version)\""; echo "}"
|
||||||
|
} > $R/steps.json
|
||||||
|
cat $R/steps.json
|
||||||
|
echo "=== task done $(date -u +%T) ==="
|
||||||
@@ -14,6 +14,8 @@ set -euo pipefail
|
|||||||
cd "$(dirname "$0")/../.."
|
cd "$(dirname "$0")/../.."
|
||||||
REGION=${REGION:-us-east-1}
|
REGION=${REGION:-us-east-1}
|
||||||
TYPE=${TYPE:-m7g.4xlarge}
|
TYPE=${TYPE:-m7g.4xlarge}
|
||||||
|
# Graviton families carry a "g" after the generation number (m7g, c7gn, r8g): arm64 image; everything else amd64
|
||||||
|
if [[ $TYPE =~ ^[a-z]+[0-9]+g ]]; then ARCH=${ARCH:-arm64}; else ARCH=${ARCH:-amd64}; fi
|
||||||
MAX_PRICE=${MAX_PRICE:-0.30} # USD/hour spot cap (on-demand is 0.6528)
|
MAX_PRICE=${MAX_PRICE:-0.30} # USD/hour spot cap (on-demand is 0.6528)
|
||||||
MAX_MINUTES=${MAX_MINUTES:-360} # hard cap: 6 hours
|
MAX_MINUTES=${MAX_MINUTES:-360} # hard cap: 6 hours
|
||||||
DISK_GB=${DISK_GB:-250}
|
DISK_GB=${DISK_GB:-250}
|
||||||
@@ -59,7 +61,7 @@ EOF
|
|||||||
launch)
|
launch)
|
||||||
running=$($AWS ec2 describe-instances --filters Name=tag:Project,Values=$TAG \
|
running=$($AWS ec2 describe-instances --filters Name=tag:Project,Values=$TAG \
|
||||||
Name=instance-state-name,Values=pending,running --query 'length(Reservations[].Instances[])')
|
Name=instance-state-name,Values=pending,running --query 'length(Reservations[].Instances[])')
|
||||||
[[ "$running" == "0" ]] || { echo "a trial instance is already running; refusing to start another"; exit 1; }
|
[[ "$running" == "0" || "${PARALLEL:-0}" == "1" ]] || { echo "a trial instance is already running; refusing to start another (PARALLEL=1 overrides)"; exit 1; }
|
||||||
RUN=run-$(date -u +%Y%m%dT%H%M%SZ)
|
RUN=run-$(date -u +%Y%m%dT%H%M%SZ)
|
||||||
echo "-> packing repository at $(git rev-parse --short HEAD) -> s3://$BUCKET/$RUN/"
|
echo "-> packing repository at $(git rev-parse --short HEAD) -> s3://$BUCKET/$RUN/"
|
||||||
git archive --format=tar.gz -o /tmp/simurg-repo.tar.gz HEAD
|
git archive --format=tar.gz -o /tmp/simurg-repo.tar.gz HEAD
|
||||||
@@ -69,7 +71,7 @@ launch)
|
|||||||
sed -e "s/__BUCKET__/$BUCKET/" -e "s/__RUN__/$RUN/" -e "s/__MAX_MINUTES__/$MAX_MINUTES/" -e "s/__TASK__/$TASK/g" \
|
sed -e "s/__BUCKET__/$BUCKET/" -e "s/__RUN__/$RUN/" -e "s/__MAX_MINUTES__/$MAX_MINUTES/" -e "s/__TASK__/$TASK/g" \
|
||||||
-e "s/__OMNI__/$OMNI/" \
|
-e "s/__OMNI__/$OMNI/" \
|
||||||
scripts/cloud/userdata.sh > /tmp/simurg-userdata.sh
|
scripts/cloud/userdata.sh > /tmp/simurg-userdata.sh
|
||||||
AMI=$($AWS ssm get-parameter --name /aws/service/canonical/ubuntu/server/24.04/stable/current/arm64/hvm/ebs-gp3/ami-id \
|
AMI=$($AWS ssm get-parameter --name /aws/service/canonical/ubuntu/server/24.04/stable/current/$ARCH/hvm/ebs-gp3/ami-id \
|
||||||
--query Parameter.Value --output text)
|
--query Parameter.Value --output text)
|
||||||
echo "-> task: $TASK"
|
echo "-> task: $TASK"
|
||||||
echo "-> launching $TYPE spot (cap \$$MAX_PRICE/h, hard limit ${MAX_MINUTES} min, ${DISK_GB} GB disk deleted on exit)"
|
echo "-> launching $TYPE spot (cap \$$MAX_PRICE/h, hard limit ${MAX_MINUTES} min, ${DISK_GB} GB disk deleted on exit)"
|
||||||
|
|||||||
@@ -0,0 +1,86 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Content digests of SimurgArray outputs, for comparing runs across machines, architectures and installs (P10).
|
||||||
|
|
||||||
|
GTC: every numeric field and sample statistic, NaNs canonicalised (their sign/payload is platform-specific, K-014);
|
||||||
|
free-text metadata (dates, software version) is left out. VCF(.gz): all data lines, header lines excluded.
|
||||||
|
TSV: the listed columns of sample_summary.tsv.
|
||||||
|
|
||||||
|
python scripts/p10/digest.py RUN_DIR [--out digest.json]
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
|
||||||
|
from simurg.formats.gtc import read_gtc
|
||||||
|
|
||||||
|
GTC_FIELDS = ("normalization_transforms", "controls_x", "controls_y", "raw_x", "raw_y", "genotypes", "base_calls",
|
||||||
|
"genotype_scores", "b_allele_freqs", "logr_ratios", "percentiles_x", "percentiles_y", "sample_data",
|
||||||
|
"call_rate", "gender", "logr_dev", "gc10") # fmt: skip
|
||||||
|
SUMMARY_COLUMNS = ("sample_id", "status", "autosomal_call_rate", "call_rate", "logr_dev", "sex_est", "gc10",
|
||||||
|
"gc50", "n_calls", "n_nocalls", "het_rate", "contamination", "functional_status", "control_status",
|
||||||
|
"flags") # fmt: skip
|
||||||
|
|
||||||
|
|
||||||
|
def _canon(a: np.ndarray) -> bytes:
|
||||||
|
if a.dtype.names: # structured: canonicalise float members
|
||||||
|
a = a.copy()
|
||||||
|
for n in a.dtype.names:
|
||||||
|
if np.issubdtype(a.dtype[n], np.floating):
|
||||||
|
v = a[n]
|
||||||
|
v[np.isnan(v)] = np.nan
|
||||||
|
return a.tobytes()
|
||||||
|
if np.issubdtype(a.dtype, np.floating):
|
||||||
|
a = np.where(np.isnan(a), np.array(np.nan, a.dtype), a)
|
||||||
|
return np.ascontiguousarray(a).tobytes()
|
||||||
|
|
||||||
|
|
||||||
|
def gtc_digest(path: Path) -> dict[str, str]:
|
||||||
|
g = read_gtc(path)
|
||||||
|
out = {}
|
||||||
|
for f in GTC_FIELDS:
|
||||||
|
v = getattr(g, f)
|
||||||
|
b = _canon(np.asarray(v)) if not isinstance(v, bytes) else v
|
||||||
|
out[f] = hashlib.sha256(b).hexdigest()[:16]
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def vcf_digest(path: Path) -> str:
|
||||||
|
h = hashlib.sha256()
|
||||||
|
with gzip.open(path, "rt") if path.suffix == ".gz" else open(path) as fh:
|
||||||
|
for line in fh:
|
||||||
|
if not line.startswith("#"):
|
||||||
|
h.update(line.encode())
|
||||||
|
return h.hexdigest()[:16]
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||||
|
ap.add_argument("run", type=Path)
|
||||||
|
ap.add_argument("--out", type=Path)
|
||||||
|
a = ap.parse_args()
|
||||||
|
d: dict[str, object] = {}
|
||||||
|
d["gtc"] = {p.stem: gtc_digest(p) for p in sorted(a.run.rglob("*.gtc"))}
|
||||||
|
d["vcf"] = {p.name: vcf_digest(p) for p in sorted(a.run.rglob("*.vcf.gz"))}
|
||||||
|
summ = {}
|
||||||
|
for p in sorted(a.run.rglob("sample_summary.tsv")):
|
||||||
|
rows = [ln.rstrip("\n").split("\t") for ln in p.open()]
|
||||||
|
idx = [rows[0].index(c) for c in SUMMARY_COLUMNS if c in rows[0]]
|
||||||
|
summ[str(p.relative_to(a.run))] = hashlib.sha256(
|
||||||
|
"\n".join("\t".join(r[i] for i in idx) for r in rows).encode()
|
||||||
|
).hexdigest()[:16]
|
||||||
|
d["summary"] = summ
|
||||||
|
text = json.dumps(d, indent=1, sort_keys=True)
|
||||||
|
(a.out.write_text(text) if a.out else sys.stdout.write(text + "\n"))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
Reference in new issue
Block a user