SimurgArray/scripts/cloud/tasks/p10.sh

#!/bin/bash
# Phase P10 acceptance on a fresh Ubuntu 24.04 VM (plan §18: "clean install + tutorial works on fresh VM"):
#   1. build the release artifacts (scripts/build_dist.sh: sdist, pure wheel, manylinux_2_28 kernel wheels for both
#      architectures);
#   2. install them with plain pip into new venvs of the system Python (with and without the kernels) and run the
#      unit tests against the installed package;
#   3. fetch the public demo data and run the tutorial (examples/gsa_demo.sh, 6 samples) with each install;
#   4. build the Docker image and the conda packages, run genotype + gtc-to-vcf with them;
#   5. content digests of every output (scripts/p10/digest.py) must agree across all installs.
# Args: <unused data dir>  <results dir>. Run with OMNI=0.
set -uo pipefail
OUT=$2
R=$OUT/p10; W=/work/p10; mkdir -p $R $W
N=$(nproc)
ARCH=$(uname -m)
declare -A STEP
step() { STEP[$1]=$2; echo "--- $1: $2"; }

export DEBIAN_FRONTEND=noninteractive
apt-get install -y -q gcc docker.io bsdextrautils > /dev/null
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal > /dev/null 2>&1
export PATH="$HOME/.cargo/bin:$PATH"

echo "=== 1. build artifacts $(date -u +%T) ==="
if bash scripts/build_dist.sh all > $R/build_dist.log 2>&1; then step build ok; else step build FAILED; tail -30 $R/build_dist.log; fi
cp dist/*.whl dist/*.tar.gz dist/SHA256SUMS $R/ 2>/dev/null
ls -la dist

echo "=== 2. pip install into fresh venvs $(date -u +%T) ==="
PY=/usr/bin/python3
$PY --version
CORE=$(ls dist/simurg_core-*manylinux_2_28_$ARCH.whl 2>/dev/null | head -1)
$PY -m venv $W/fresh && $W/fresh/bin/pip install -q dist/simurg-*-py3-none-any.whl "$CORE" \
  && step pip_install ok || step pip_install FAILED
$PY -m venv $W/pure && $W/pure/bin/pip install -q dist/simurg-*-py3-none-any.whl \
  && step pip_install_pure ok || step pip_install_pure FAILED
$W/fresh/bin/python -c "from simurg._core import backend; print('fresh:', backend())"
$W/pure/bin/python -c "from simurg._core import backend; print('pure:', backend())"
$W/fresh/bin/simurg --version
$W/fresh/bin/pip install -q pytest hypothesis
# tests import the installed package (the checkout keeps its sources under src/, not on sys.path)
( cd $W && cp -r /work/repo/tests /work/repo/docs . && $W/fresh/bin/python -m pytest tests/unit -q -p no:cacheprovider \
    -k "not selftest_in_browser" > $R/unit_tests_installed.txt 2>&1 ) && step unit_tests_installed ok \
  || { step unit_tests_installed FAILED; tail -20 $R/unit_tests_installed.txt; }
tail -3 $R/unit_tests_installed.txt

echo "=== 3. tutorial $(date -u +%T) ==="
bash scripts/fetch_data.sh p0 > $W/fetch_p0.log 2>&1 && bash scripts/fetch_data.sh p6 > $W/fetch_p6.log 2>&1 \
  && step fetch_data ok || { step fetch_data FAILED; tail -5 $W/fetch_p0.log $W/fetch_p6.log; }
for v in fresh pure; do
  t0=$SECONDS
  SIMURG=$W/$v/bin/simurg THREADS=$N bash examples/gsa_demo.sh $W/tut_$v 6 > $R/tutorial_$v.log 2>&1 \
    && step tutorial_$v "ok ($((SECONDS - t0)) s)" || { step tutorial_$v FAILED; tail -20 $R/tutorial_$v.log; }
  $W/fresh/bin/python scripts/p10/digest.py $W/tut_$v/gtc --out $R/digest_gtc_$v.json
  $W/fresh/bin/python scripts/p10/digest.py $W/tut_$v/vcf --out $R/digest_vcf_$v.json
done

echo "=== 4a. docker $(date -u +%T) ==="
if [[ "${SKIP_DOCKER:-0}" != "1" ]]; then
D=data/p0/gsa_v4_pgx
BPM=$D/GSA-PGx-48v4-0_20079540_E2_bpm/GSA-PGx-48v4-0_20079540_E2.bpm
CSV=$D/GSA-PGx-48v4-0_20079540_E2_csv/GSA-PGx-48v4-0_20079540_E2.csv
EGT=$D/GSA-PGx-48v4-0_20079540_E2_ClusterFile/GSA-PGx-48v4-0_20079540_E2_ClusterFile.egt
FASTA=data/p6/GRCh38.primary.fa
if docker build -q -t simurg:test . > $R/docker_build.log 2>&1; then
  step docker_build ok
  docker image ls simurg:test --format '{{.Size}}' | sed 's/^/image size: /'
  DR="docker run --rm --user 0:0 -v /work:/work -w /work/repo simurg:test"
  $DR --version
  $DR genotype --bpm $BPM --egt $EGT --csv $CSV --idats $W/tut_fresh/idats --out $W/docker/gtc \
      --qc-profile genotyping --threads $N --quiet --force \
    && $DR gtc-to-vcf --bpm $BPM --csv $CSV --fasta $FASTA --gtcs $W/docker/gtc --out $W/docker/vcf --threads $N \
      --quiet --force && step docker_run ok || step docker_run FAILED
  $W/fresh/bin/python scripts/p10/digest.py $W/docker/gtc --out $R/digest_gtc_docker.json
  $W/fresh/bin/python scripts/p10/digest.py $W/docker/vcf --out $R/digest_vcf_docker.json
else
  step docker_build FAILED; tail -30 $R/docker_build.log
fi
fi

echo "=== 4b. conda $(date -u +%T) ==="
if [[ "${SKIP_CONDA:-0}" != "1" ]]; then
curl -sSfL -o $W/miniforge.sh "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-$ARCH.sh" \
  && bash $W/miniforge.sh -b -p $W/conda > /dev/null
C=$W/conda/bin
if $C/conda install -y -q -n base conda-build > $R/conda.log 2>&1 \
   && $C/conda build -c conda-forge --output-folder $W/condabld packaging/conda/simurg-core packaging/conda/simurg \
      >> $R/conda.log 2>&1; then
  step conda_build ok
  $C/conda create -y -q -p $W/cenv -c $W/condabld -c conda-forge simurg simurg-core >> $R/conda.log 2>&1 \
    && $W/cenv/bin/simurg --version \
    && $W/cenv/bin/python -c "from simurg._core import backend; print('conda:', backend())" \
    && $W/cenv/bin/simurg genotype --bpm $BPM --egt $EGT --idats $W/tut_fresh/idats --out $W/conda_run/gtc \
         --threads $N --quiet --force \
    && step conda_run ok || { step conda_run FAILED; tail -20 $R/conda.log; }
  $W/fresh/bin/python scripts/p10/digest.py $W/conda_run/gtc --out $R/digest_gtc_conda.json
else
  step conda_build FAILED; tail -40 $R/conda.log
fi
fi

echo "=== 5. digests $(date -u +%T) ==="
$W/fresh/bin/python - "$R" <<'PY' | tee $R/agreement.txt
import json, sys
from pathlib import Path
r = Path(sys.argv[1])
for kind in ("gtc", "vcf"):
    runs = {p.stem.split("_", 2)[2]: json.loads(p.read_text())[kind] for p in sorted(r.glob(f"digest_{kind}_*.json"))}
    if not runs:
        continue
    ref = runs.get("fresh", {})
    if not ref:
        print(f"{kind}: no reference (the pip-installed tutorial produced no outputs)")
        continue
    for name, d in runs.items():
        if name != "fresh":
            print(f"{kind}: {name} vs fresh: {'IDENTICAL' if d == ref else 'DIFFERENT'} ({len(d)} vs {len(ref)} files)")
PY
{
  echo "{"; echo "  \"arch\": \"$ARCH\","
  for k in "${!STEP[@]}"; do echo "  \"$k\": \"${STEP[$k]}\","; done
  echo "  \"python\": \"$($PY --version)\""; echo "}"
} > $R/steps.json
cat $R/steps.json
echo "=== task done $(date -u +%T) ==="