P10 (WIP): release build script, Dockerfile, conda recipes, fresh-VM acceptance task

scripts/build_dist.sh: sdist + pure wheel + manylinux_2_28 kernel wheels (maturin --zig, both architectures). Dockerfile (multi-stage, kernels included, non-root user). packaging/conda/{simurg,simurg-core}. scripts/p10/digest.py: platform-independent content digests of GTC/VCF/summary outputs. Cloud task p10 installs the artifacts on a fresh Ubuntu VM and runs the tutorial with pip, Docker and conda installs. trial.sh picks the arm64/amd64 image from the instance type and can run instances in parallel (PARALLEL=1). Licence files ship in both wheels; Rust crate notices added.
2026-09-28 22:41:56 +01:00
1 parent 7d2dbe939e
commit 1f2abcbdba
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+# Build the release artifacts into dist/:
+#   simurg-<v>-py3-none-any.whl + simurg-<v>.tar.gz           (pure Python)
+#   simurg_core-<v>-cp312-abi3-manylinux_2_28_{aarch64,x86_64}.whl   (compiled kernels, Python >= 3.12)
+# Portable Linux wheels are linked against glibc 2.28 with `maturin --zig` (cross-compiles the other architecture).
+# Needs: uv, a Rust toolchain (rustup), ~60 MB for zig on first use.
+#   bash scripts/build_dist.sh [aarch64|x86_64|all]      (default: all)
+set -euo pipefail
+cd "$(dirname "$0")/.."
+export PATH="$HOME/.cargo/bin:$PATH"
+WHICH=${1:-all}
+mkdir -p dist
+uv build -o dist                                   # sdist + pure wheel of simurg
+MATURIN="uvx --from maturin[zig]>=1.7,<2 maturin"
+for arch in aarch64 x86_64; do
+  [[ $WHICH == all || $WHICH == "$arch" ]] || continue
+  rustup target add "$arch-unknown-linux-gnu" > /dev/null
+  $MATURIN build --release --zig --compatibility manylinux_2_28 --target "$arch-unknown-linux-gnu" \
+    -m rust/simurg-core/Cargo.toml -o dist
+done
+( cd dist && sha256sum ./*.whl ./*.tar.gz > SHA256SUMS )
+ls -la dist
@@ -0,0 +1,118 @@
+#!/bin/bash
+# Phase P10 acceptance on a fresh Ubuntu 24.04 VM (plan §18: "clean install + tutorial works on fresh VM"):
+#   1. build the release artifacts (scripts/build_dist.sh: sdist, pure wheel, manylinux_2_28 kernel wheels for both
+#      architectures);
+#   2. install them with plain pip into new venvs of the system Python (with and without the kernels) and run the
+#      unit tests against the installed package;
+#   3. fetch the public demo data and run the tutorial (examples/gsa_demo.sh, 6 samples) with each install;
+#   4. build the Docker image and the conda packages, run genotype + gtc-to-vcf with them;
+#   5. content digests of every output (scripts/p10/digest.py) must agree across all installs.
+# Args: <unused data dir>  <results dir>. Run with OMNI=0.
+set -uo pipefail
+OUT=$2
+R=$OUT/p10; W=/work/p10; mkdir -p $R $W
+N=$(nproc)
+ARCH=$(uname -m)
+declare -A STEP
+step() { STEP[$1]=$2; echo "--- $1: $2"; }
+
+export DEBIAN_FRONTEND=noninteractive
+apt-get install -y -q gcc docker.io bsdextrautils > /dev/null
+curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal > /dev/null 2>&1
+export PATH="$HOME/.cargo/bin:$PATH"
+
+echo "=== 1. build artifacts $(date -u +%T) ==="
+if bash scripts/build_dist.sh all > $R/build_dist.log 2>&1; then step build ok; else step build FAILED; tail -30 $R/build_dist.log; fi
+cp dist/*.whl dist/*.tar.gz dist/SHA256SUMS $R/ 2>/dev/null
+ls -la dist
+
+echo "=== 2. pip install into fresh venvs $(date -u +%T) ==="
+PY=/usr/bin/python3
+$PY --version
+CORE=$(ls dist/simurg_core-*manylinux_2_28_$ARCH.whl 2>/dev/null | head -1)
+$PY -m venv $W/fresh && $W/fresh/bin/pip install -q dist/simurg-*-py3-none-any.whl "$CORE" \
+  && step pip_install ok || step pip_install FAILED
+$PY -m venv $W/pure && $W/pure/bin/pip install -q dist/simurg-*-py3-none-any.whl \
+  && step pip_install_pure ok || step pip_install_pure FAILED
+$W/fresh/bin/python -c "from simurg._core import backend; print('fresh:', backend())"
+$W/pure/bin/python -c "from simurg._core import backend; print('pure:', backend())"
+$W/fresh/bin/simurg --version
+$W/fresh/bin/pip install -q pytest hypothesis
+# tests import the installed package (the checkout keeps its sources under src/, not on sys.path)
+( cd $W && cp -r /work/repo/tests /work/repo/docs . && $W/fresh/bin/python -m pytest tests/unit -q -p no:cacheprovider \
+    -k "not selftest_in_browser" > $R/unit_tests_installed.txt 2>&1 ) && step unit_tests_installed ok \
+  || { step unit_tests_installed FAILED; tail -20 $R/unit_tests_installed.txt; }
+tail -3 $R/unit_tests_installed.txt
+
+echo "=== 3. tutorial $(date -u +%T) ==="
+bash scripts/fetch_data.sh p0 > $W/fetch_p0.log 2>&1 && bash scripts/fetch_data.sh p6 > $W/fetch_p6.log 2>&1 \
+  && step fetch_data ok || { step fetch_data FAILED; tail -5 $W/fetch_p0.log $W/fetch_p6.log; }
+for v in fresh pure; do
+  t0=$SECONDS
+  SIMURG=$W/$v/bin/simurg THREADS=$N bash examples/gsa_demo.sh $W/tut_$v 6 > $R/tutorial_$v.log 2>&1 \
+    && step tutorial_$v "ok ($((SECONDS - t0)) s)" || { step tutorial_$v FAILED; tail -20 $R/tutorial_$v.log; }
+  $W/fresh/bin/python scripts/p10/digest.py $W/tut_$v/gtc --out $R/digest_gtc_$v.json
+  $W/fresh/bin/python scripts/p10/digest.py $W/tut_$v/vcf --out $R/digest_vcf_$v.json
+done
+
+echo "=== 4a. docker $(date -u +%T) ==="
+D=data/p0/gsa_v4_pgx
+BPM=$D/GSA-PGx-48v4-0_20079540_E2_bpm/GSA-PGx-48v4-0_20079540_E2.bpm
+CSV=$D/GSA-PGx-48v4-0_20079540_E2_csv/GSA-PGx-48v4-0_20079540_E2.csv
+EGT=$D/GSA-PGx-48v4-0_20079540_E2_ClusterFile/GSA-PGx-48v4-0_20079540_E2_ClusterFile.egt
+FASTA=data/p6/GRCh38.primary.fa
+if docker build -q -t simurg:test . > $R/docker_build.log 2>&1; then
+  step docker_build ok
+  docker image ls simurg:test --format '{{.Size}}' | sed 's/^/image size: /'
+  DR="docker run --rm --user 0:0 -v /work:/work -w /work/repo simurg:test"
+  $DR --version
+  $DR genotype --bpm $BPM --egt $EGT --csv $CSV --idats $W/tut_fresh/idats --out $W/docker/gtc \
+      --qc-profile genotyping --threads $N --quiet --force \
+    && $DR gtc-to-vcf --bpm $BPM --csv $CSV --fasta $FASTA --gtcs $W/docker/gtc --out $W/docker/vcf --threads $N \
+      --quiet --force && step docker_run ok || step docker_run FAILED
+  $W/fresh/bin/python scripts/p10/digest.py $W/docker/gtc --out $R/digest_gtc_docker.json
+  $W/fresh/bin/python scripts/p10/digest.py $W/docker/vcf --out $R/digest_vcf_docker.json
+else
+  step docker_build FAILED; tail -30 $R/docker_build.log
+fi
+
+echo "=== 4b. conda $(date -u +%T) ==="
+curl -sSfL -o $W/miniforge.sh "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-$ARCH.sh" \
+  && bash $W/miniforge.sh -b -p $W/conda > /dev/null
+C=$W/conda/bin
+if $C/conda install -y -q -n base conda-build > $R/conda.log 2>&1 \
+   && $C/conda build -c conda-forge --output-folder $W/condabld packaging/conda/simurg-core packaging/conda/simurg \
+      >> $R/conda.log 2>&1; then
+  step conda_build ok
+  $C/conda create -y -q -p $W/cenv -c $W/condabld -c conda-forge simurg simurg-core >> $R/conda.log 2>&1 \
+    && $W/cenv/bin/simurg --version \
+    && $W/cenv/bin/python -c "from simurg._core import backend; print('conda:', backend())" \
+    && $W/cenv/bin/simurg genotype --bpm $BPM --egt $EGT --idats $W/tut_fresh/idats --out $W/conda_run/gtc \
+         --threads $N --quiet --force \
+    && step conda_run ok || { step conda_run FAILED; tail -20 $R/conda.log; }
+  $W/fresh/bin/python scripts/p10/digest.py $W/conda_run/gtc --out $R/digest_gtc_conda.json
+else
+  step conda_build FAILED; tail -40 $R/conda.log
+fi
+
+echo "=== 5. digests $(date -u +%T) ==="
+$W/fresh/bin/python - "$R" <<'PY' | tee $R/agreement.txt
+import json, sys
+from pathlib import Path
+r = Path(sys.argv[1])
+for kind in ("gtc", "vcf"):
+    runs = {p.stem.split("_", 2)[2]: json.loads(p.read_text())[kind] for p in sorted(r.glob(f"digest_{kind}_*.json"))}
+    if not runs:
+        continue
+    ref_name, ref = "fresh", runs.get("fresh", {})
+    for name, d in runs.items():
+        same = d == ref and len(d) > 0
+        print(f"{kind}: {name} vs {ref_name}: {'IDENTICAL' if same else 'DIFFERENT'} ({len(d)} files)")
+PY
+{
+  echo "{"; echo "  \"arch\": \"$ARCH\","
+  for k in "${!STEP[@]}"; do echo "  \"$k\": \"${STEP[$k]}\","; done
+  echo "  \"python\": \"$($PY --version)\""; echo "}"
+} > $R/steps.json
+cat $R/steps.json
+echo "=== task done $(date -u +%T) ==="
@@ -14,6 +14,8 @@ set -euo pipefail
 cd "$(dirname "$0")/../.."
 REGION=${REGION:-us-east-1}
 TYPE=${TYPE:-m7g.4xlarge}
+# Graviton families carry a "g" after the generation number (m7g, c7gn, r8g): arm64 image; everything else amd64
+if [[ $TYPE =~ ^[a-z]+[0-9]+g ]]; then ARCH=${ARCH:-arm64}; else ARCH=${ARCH:-amd64}; fi
 MAX_PRICE=${MAX_PRICE:-0.30}        # USD/hour spot cap (on-demand is 0.6528)
 MAX_MINUTES=${MAX_MINUTES:-360}     # hard cap: 6 hours
 DISK_GB=${DISK_GB:-250}
@@ -59,7 +61,7 @@ EOF
 launch)
  running=$($AWS ec2 describe-instances --filters Name=tag:Project,Values=$TAG \
    Name=instance-state-name,Values=pending,running --query 'length(Reservations[].Instances[])')
-  [[ "$running" == "0" ]] || { echo "a trial instance is already running; refusing to start another"; exit 1; }
+  [[ "$running" == "0" || "${PARALLEL:-0}" == "1" ]] || { echo "a trial instance is already running; refusing to start another (PARALLEL=1 overrides)"; exit 1; }
  RUN=run-$(date -u +%Y%m%dT%H%M%SZ)
  echo "-> packing repository at $(git rev-parse --short HEAD) -> s3://$BUCKET/$RUN/"
  git archive --format=tar.gz -o /tmp/simurg-repo.tar.gz HEAD
@@ -69,7 +71,7 @@ launch)
  sed -e "s/__BUCKET__/$BUCKET/" -e "s/__RUN__/$RUN/" -e "s/__MAX_MINUTES__/$MAX_MINUTES/" -e "s/__TASK__/$TASK/g" \
    -e "s/__OMNI__/$OMNI/" \
    scripts/cloud/userdata.sh > /tmp/simurg-userdata.sh
-  AMI=$($AWS ssm get-parameter --name /aws/service/canonical/ubuntu/server/24.04/stable/current/arm64/hvm/ebs-gp3/ami-id \
+  AMI=$($AWS ssm get-parameter --name /aws/service/canonical/ubuntu/server/24.04/stable/current/$ARCH/hvm/ebs-gp3/ami-id \
    --query Parameter.Value --output text)
  echo "-> task: $TASK"
  echo "-> launching $TYPE spot (cap \$$MAX_PRICE/h, hard limit ${MAX_MINUTES} min, ${DISK_GB} GB disk deleted on exit)"
@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+"""Content digests of SimurgArray outputs, for comparing runs across machines, architectures and installs (P10).
+
+GTC: every numeric field and sample statistic, NaNs canonicalised (their sign/payload is platform-specific, K-014);
+free-text metadata (dates, software version) is left out. VCF(.gz): all data lines, header lines excluded.
+TSV: the listed columns of sample_summary.tsv.
+
+    python scripts/p10/digest.py RUN_DIR [--out digest.json]
+"""
+
+from __future__ import annotations
+
+import argparse
+import gzip
+import hashlib
+import json
+import sys
+from pathlib import Path
+
+import numpy as np
+
+from simurg.formats.gtc import read_gtc
+
+GTC_FIELDS = ("normalization_transforms", "controls_x", "controls_y", "raw_x", "raw_y", "genotypes", "base_calls",
+              "genotype_scores", "b_allele_freqs", "logr_ratios", "percentiles_x", "percentiles_y", "sample_data",
+              "call_rate", "gender", "logr_dev", "gc10")  # fmt: skip
+SUMMARY_COLUMNS = ("sample_id", "status", "autosomal_call_rate", "call_rate", "logr_dev", "sex_est", "gc10",
+                   "gc50", "n_calls", "n_nocalls", "het_rate", "contamination", "functional_status", "control_status",
+                   "flags")  # fmt: skip
+
+
+def _canon(a: np.ndarray) -> bytes:
+    if a.dtype.names:  # structured: canonicalise float members
+        a = a.copy()
+        for n in a.dtype.names:
+            if np.issubdtype(a.dtype[n], np.floating):
+                v = a[n]
+                v[np.isnan(v)] = np.nan
+        return a.tobytes()
+    if np.issubdtype(a.dtype, np.floating):
+        a = np.where(np.isnan(a), np.array(np.nan, a.dtype), a)
+    return np.ascontiguousarray(a).tobytes()
+
+
+def gtc_digest(path: Path) -> dict[str, str]:
+    g = read_gtc(path)
+    out = {}
+    for f in GTC_FIELDS:
+        v = getattr(g, f)
+        b = _canon(np.asarray(v)) if not isinstance(v, bytes) else v
+        out[f] = hashlib.sha256(b).hexdigest()[:16]
+    return out
+
+
+def vcf_digest(path: Path) -> str:
+    h = hashlib.sha256()
+    with gzip.open(path, "rt") if path.suffix == ".gz" else open(path) as fh:
+        for line in fh:
+            if not line.startswith("#"):
+                h.update(line.encode())
+    return h.hexdigest()[:16]
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
+    ap.add_argument("run", type=Path)
+    ap.add_argument("--out", type=Path)
+    a = ap.parse_args()
+    d: dict[str, object] = {}
+    d["gtc"] = {p.stem: gtc_digest(p) for p in sorted(a.run.rglob("*.gtc"))}
+    d["vcf"] = {p.name: vcf_digest(p) for p in sorted(a.run.rglob("*.vcf.gz"))}
+    summ = {}
+    for p in sorted(a.run.rglob("sample_summary.tsv")):
+        rows = [ln.rstrip("\n").split("\t") for ln in p.open()]
+        idx = [rows[0].index(c) for c in SUMMARY_COLUMNS if c in rows[0]]
+        summ[str(p.relative_to(a.run))] = hashlib.sha256(
+            "\n".join("\t".join(r[i] for i in idx) for r in rows).encode()
+        ).hexdigest()[:16]
+    d["summary"] = summ
+    text = json.dumps(d, indent=1, sort_keys=True)
+    (a.out.write_text(text) if a.out else sys.stdout.write(text + "\n"))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())