P10 (WIP): release build script, Dockerfile, conda recipes, fresh-VM acceptance task
scripts/build_dist.sh: sdist + pure wheel + manylinux_2_28 kernel wheels (maturin --zig,
both architectures). Dockerfile (multi-stage, kernels included, non-root user).
packaging/conda/{simurg,simurg-core}. scripts/p10/digest.py: platform-independent content
digests of GTC/VCF/summary outputs. Cloud task p10 installs the artifacts on a fresh
Ubuntu VM and runs the tutorial with pip, Docker and conda installs. trial.sh picks the
arm64/amd64 image from the instance type and can run instances in parallel (PARALLEL=1).
Licence files ship in both wheels; Rust crate notices added.
This commit is contained in:
kemal committed
1 parent
7d2dbe939e
commit
1f2abcbdba
13 files changed
+403
-5
No files matched your search
@@ -0,0 +1,118 @@
|
||||
#!/bin/bash
|
||||
# Phase P10 acceptance on a fresh Ubuntu 24.04 VM (plan §18: "clean install + tutorial works on fresh VM"):
|
||||
# 1. build the release artifacts (scripts/build_dist.sh: sdist, pure wheel, manylinux_2_28 kernel wheels for both
|
||||
# architectures);
|
||||
# 2. install them with plain pip into new venvs of the system Python (with and without the kernels) and run the
|
||||
# unit tests against the installed package;
|
||||
# 3. fetch the public demo data and run the tutorial (examples/gsa_demo.sh, 6 samples) with each install;
|
||||
# 4. build the Docker image and the conda packages, run genotype + gtc-to-vcf with them;
|
||||
# 5. content digests of every output (scripts/p10/digest.py) must agree across all installs.
|
||||
# Args: <unused data dir> <results dir>. Run with OMNI=0.
|
||||
set -uo pipefail
|
||||
OUT=$2
|
||||
R=$OUT/p10; W=/work/p10; mkdir -p $R $W
|
||||
N=$(nproc)
|
||||
ARCH=$(uname -m)
|
||||
declare -A STEP
|
||||
step() { STEP[$1]=$2; echo "--- $1: $2"; }
|
||||
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get install -y -q gcc docker.io bsdextrautils > /dev/null
|
||||
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal > /dev/null 2>&1
|
||||
export PATH="$HOME/.cargo/bin:$PATH"
|
||||
|
||||
echo "=== 1. build artifacts $(date -u +%T) ==="
|
||||
if bash scripts/build_dist.sh all > $R/build_dist.log 2>&1; then step build ok; else step build FAILED; tail -30 $R/build_dist.log; fi
|
||||
cp dist/*.whl dist/*.tar.gz dist/SHA256SUMS $R/ 2>/dev/null
|
||||
ls -la dist
|
||||
|
||||
echo "=== 2. pip install into fresh venvs $(date -u +%T) ==="
|
||||
PY=/usr/bin/python3
|
||||
$PY --version
|
||||
CORE=$(ls dist/simurg_core-*manylinux_2_28_$ARCH.whl 2>/dev/null | head -1)
|
||||
$PY -m venv $W/fresh && $W/fresh/bin/pip install -q dist/simurg-*-py3-none-any.whl "$CORE" \
|
||||
&& step pip_install ok || step pip_install FAILED
|
||||
$PY -m venv $W/pure && $W/pure/bin/pip install -q dist/simurg-*-py3-none-any.whl \
|
||||
&& step pip_install_pure ok || step pip_install_pure FAILED
|
||||
$W/fresh/bin/python -c "from simurg._core import backend; print('fresh:', backend())"
|
||||
$W/pure/bin/python -c "from simurg._core import backend; print('pure:', backend())"
|
||||
$W/fresh/bin/simurg --version
|
||||
$W/fresh/bin/pip install -q pytest hypothesis
|
||||
# tests import the installed package (the checkout keeps its sources under src/, not on sys.path)
|
||||
( cd $W && cp -r /work/repo/tests /work/repo/docs . && $W/fresh/bin/python -m pytest tests/unit -q -p no:cacheprovider \
|
||||
-k "not selftest_in_browser" > $R/unit_tests_installed.txt 2>&1 ) && step unit_tests_installed ok \
|
||||
|| { step unit_tests_installed FAILED; tail -20 $R/unit_tests_installed.txt; }
|
||||
tail -3 $R/unit_tests_installed.txt
|
||||
|
||||
echo "=== 3. tutorial $(date -u +%T) ==="
|
||||
bash scripts/fetch_data.sh p0 > $W/fetch_p0.log 2>&1 && bash scripts/fetch_data.sh p6 > $W/fetch_p6.log 2>&1 \
|
||||
&& step fetch_data ok || { step fetch_data FAILED; tail -5 $W/fetch_p0.log $W/fetch_p6.log; }
|
||||
for v in fresh pure; do
|
||||
t0=$SECONDS
|
||||
SIMURG=$W/$v/bin/simurg THREADS=$N bash examples/gsa_demo.sh $W/tut_$v 6 > $R/tutorial_$v.log 2>&1 \
|
||||
&& step tutorial_$v "ok ($((SECONDS - t0)) s)" || { step tutorial_$v FAILED; tail -20 $R/tutorial_$v.log; }
|
||||
$W/fresh/bin/python scripts/p10/digest.py $W/tut_$v/gtc --out $R/digest_gtc_$v.json
|
||||
$W/fresh/bin/python scripts/p10/digest.py $W/tut_$v/vcf --out $R/digest_vcf_$v.json
|
||||
done
|
||||
|
||||
echo "=== 4a. docker $(date -u +%T) ==="
|
||||
D=data/p0/gsa_v4_pgx
|
||||
BPM=$D/GSA-PGx-48v4-0_20079540_E2_bpm/GSA-PGx-48v4-0_20079540_E2.bpm
|
||||
CSV=$D/GSA-PGx-48v4-0_20079540_E2_csv/GSA-PGx-48v4-0_20079540_E2.csv
|
||||
EGT=$D/GSA-PGx-48v4-0_20079540_E2_ClusterFile/GSA-PGx-48v4-0_20079540_E2_ClusterFile.egt
|
||||
FASTA=data/p6/GRCh38.primary.fa
|
||||
if docker build -q -t simurg:test . > $R/docker_build.log 2>&1; then
|
||||
step docker_build ok
|
||||
docker image ls simurg:test --format '{{.Size}}' | sed 's/^/image size: /'
|
||||
DR="docker run --rm --user 0:0 -v /work:/work -w /work/repo simurg:test"
|
||||
$DR --version
|
||||
$DR genotype --bpm $BPM --egt $EGT --csv $CSV --idats $W/tut_fresh/idats --out $W/docker/gtc \
|
||||
--qc-profile genotyping --threads $N --quiet --force \
|
||||
&& $DR gtc-to-vcf --bpm $BPM --csv $CSV --fasta $FASTA --gtcs $W/docker/gtc --out $W/docker/vcf --threads $N \
|
||||
--quiet --force && step docker_run ok || step docker_run FAILED
|
||||
$W/fresh/bin/python scripts/p10/digest.py $W/docker/gtc --out $R/digest_gtc_docker.json
|
||||
$W/fresh/bin/python scripts/p10/digest.py $W/docker/vcf --out $R/digest_vcf_docker.json
|
||||
else
|
||||
step docker_build FAILED; tail -30 $R/docker_build.log
|
||||
fi
|
||||
|
||||
echo "=== 4b. conda $(date -u +%T) ==="
|
||||
curl -sSfL -o $W/miniforge.sh "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-$ARCH.sh" \
|
||||
&& bash $W/miniforge.sh -b -p $W/conda > /dev/null
|
||||
C=$W/conda/bin
|
||||
if $C/conda install -y -q -n base conda-build > $R/conda.log 2>&1 \
|
||||
&& $C/conda build -c conda-forge --output-folder $W/condabld packaging/conda/simurg-core packaging/conda/simurg \
|
||||
>> $R/conda.log 2>&1; then
|
||||
step conda_build ok
|
||||
$C/conda create -y -q -p $W/cenv -c $W/condabld -c conda-forge simurg simurg-core >> $R/conda.log 2>&1 \
|
||||
&& $W/cenv/bin/simurg --version \
|
||||
&& $W/cenv/bin/python -c "from simurg._core import backend; print('conda:', backend())" \
|
||||
&& $W/cenv/bin/simurg genotype --bpm $BPM --egt $EGT --idats $W/tut_fresh/idats --out $W/conda_run/gtc \
|
||||
--threads $N --quiet --force \
|
||||
&& step conda_run ok || { step conda_run FAILED; tail -20 $R/conda.log; }
|
||||
$W/fresh/bin/python scripts/p10/digest.py $W/conda_run/gtc --out $R/digest_gtc_conda.json
|
||||
else
|
||||
step conda_build FAILED; tail -40 $R/conda.log
|
||||
fi
|
||||
|
||||
echo "=== 5. digests $(date -u +%T) ==="
|
||||
$W/fresh/bin/python - "$R" <<'PY' | tee $R/agreement.txt
|
||||
import json, sys
|
||||
from pathlib import Path
|
||||
r = Path(sys.argv[1])
|
||||
for kind in ("gtc", "vcf"):
|
||||
runs = {p.stem.split("_", 2)[2]: json.loads(p.read_text())[kind] for p in sorted(r.glob(f"digest_{kind}_*.json"))}
|
||||
if not runs:
|
||||
continue
|
||||
ref_name, ref = "fresh", runs.get("fresh", {})
|
||||
for name, d in runs.items():
|
||||
same = d == ref and len(d) > 0
|
||||
print(f"{kind}: {name} vs {ref_name}: {'IDENTICAL' if same else 'DIFFERENT'} ({len(d)} files)")
|
||||
PY
|
||||
{
|
||||
echo "{"; echo " \"arch\": \"$ARCH\","
|
||||
for k in "${!STEP[@]}"; do echo " \"$k\": \"${STEP[$k]}\","; done
|
||||
echo " \"python\": \"$($PY --version)\""; echo "}"
|
||||
} > $R/steps.json
|
||||
cat $R/steps.json
|
||||
echo "=== task done $(date -u +%T) ==="
|
||||
Reference in new issue
Block a user