CLI: a closed stdout pipe exits 141 silently (was E_INTERNAL); example avoids head under pipefail
Found by the P10 fresh-VM run: `simurg inspect … | head` raised BrokenPipeError, reported as an internal error, and stopped examples/gsa_demo.sh (set -o pipefail). run_guarded now flushes stdout inside the guard and maps BrokenPipeError to ExitCode.SIGPIPE (141) without a message. p10.sh: SKIP_DOCKER/SKIP_CONDA, clearer digest report; p10t.sh runs build+pip+tutorial only.
This commit is contained in:
1 parent
349be7a43c
commit
cbcb99f8d2
6 files changed
+38
-6
No files matched your search
@@ -30,7 +30,7 @@ mkdir -p "$OUT"
|
|||||||
# Optional subset: symlink the first N IDAT pairs into $OUT/idats (same <barcode>/<barcode>_<pos>_<ch>.idat layout).
|
# Optional subset: symlink the first N IDAT pairs into $OUT/idats (same <barcode>/<barcode>_<pos>_<ch>.idat layout).
|
||||||
if [[ $N -gt 0 ]]; then
|
if [[ $N -gt 0 ]]; then
|
||||||
rm -rf "$OUT/idats"
|
rm -rf "$OUT/idats"
|
||||||
find "$IDATS" -name '*_Grn.idat' | sort | head -n "$N" | while read -r g; do
|
find "$IDATS" -name '*_Grn.idat' | sort | sed -n "1,${N}p" | while read -r g; do
|
||||||
b=$(basename "$(dirname "$g")"); mkdir -p "$OUT/idats/$b"
|
b=$(basename "$(dirname "$g")"); mkdir -p "$OUT/idats/$b"
|
||||||
ln -s "$(realpath "$g")" "$OUT/idats/$b/"; ln -s "$(realpath "${g%_Grn.idat}_Red.idat")" "$OUT/idats/$b/"
|
ln -s "$(realpath "$g")" "$OUT/idats/$b/"; ln -s "$(realpath "${g%_Grn.idat}_Red.idat")" "$OUT/idats/$b/"
|
||||||
done
|
done
|
||||||
@@ -38,12 +38,12 @@ if [[ $N -gt 0 ]]; then
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
echo "== 1. inspect the inputs"
|
echo "== 1. inspect the inputs"
|
||||||
$SIMURG inspect "$BPM" "$EGT" | head -20
|
$SIMURG inspect "$BPM" "$EGT" | sed -n 1,20p # sed reads to the end: no SIGPIPE under pipefail
|
||||||
|
|
||||||
echo "== 2. genotype: GTC v5 per sample, sample_summary.tsv, control-probe QC, contamination, relatedness"
|
echo "== 2. genotype: GTC v5 per sample, sample_summary.tsv, control-probe QC, contamination, relatedness"
|
||||||
$SIMURG genotype --bpm "$BPM" --egt "$EGT" --csv "$CSV" --idats "$IDATS" --out "$OUT/gtc" \
|
$SIMURG genotype --bpm "$BPM" --egt "$EGT" --csv "$CSV" --idats "$IDATS" --out "$OUT/gtc" \
|
||||||
--qc-profile genotyping --threads "$THREADS" --force
|
--qc-profile genotyping --threads "$THREADS" --force
|
||||||
column -t -s$'\t' "$OUT/gtc/sample_summary.tsv" | cut -c1-150 | head -8
|
column -t -s$'\t' "$OUT/gtc/sample_summary.tsv" | cut -c1-150 | sed -n 1,8p
|
||||||
|
|
||||||
echo "== 3. HTML QC report + QC table"
|
echo "== 3. HTML QC report + QC table"
|
||||||
$SIMURG qc report --data "$OUT/gtc" --out "$OUT/qc_report" --output-format both
|
$SIMURG qc report --data "$OUT/gtc" --out "$OUT/qc_report" --output-format both
|
||||||
|
|||||||
@@ -56,6 +56,7 @@ for v in fresh pure; do
|
|||||||
done
|
done
|
||||||
|
|
||||||
echo "=== 4a. docker $(date -u +%T) ==="
|
echo "=== 4a. docker $(date -u +%T) ==="
|
||||||
|
if [[ "${SKIP_DOCKER:-0}" != "1" ]]; then
|
||||||
D=data/p0/gsa_v4_pgx
|
D=data/p0/gsa_v4_pgx
|
||||||
BPM=$D/GSA-PGx-48v4-0_20079540_E2_bpm/GSA-PGx-48v4-0_20079540_E2.bpm
|
BPM=$D/GSA-PGx-48v4-0_20079540_E2_bpm/GSA-PGx-48v4-0_20079540_E2.bpm
|
||||||
CSV=$D/GSA-PGx-48v4-0_20079540_E2_csv/GSA-PGx-48v4-0_20079540_E2.csv
|
CSV=$D/GSA-PGx-48v4-0_20079540_E2_csv/GSA-PGx-48v4-0_20079540_E2.csv
|
||||||
@@ -75,8 +76,10 @@ if docker build -q -t simurg:test . > $R/docker_build.log 2>&1; then
|
|||||||
else
|
else
|
||||||
step docker_build FAILED; tail -30 $R/docker_build.log
|
step docker_build FAILED; tail -30 $R/docker_build.log
|
||||||
fi
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
echo "=== 4b. conda $(date -u +%T) ==="
|
echo "=== 4b. conda $(date -u +%T) ==="
|
||||||
|
if [[ "${SKIP_CONDA:-0}" != "1" ]]; then
|
||||||
curl -sSfL -o $W/miniforge.sh "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-$ARCH.sh" \
|
curl -sSfL -o $W/miniforge.sh "https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-$ARCH.sh" \
|
||||||
&& bash $W/miniforge.sh -b -p $W/conda > /dev/null
|
&& bash $W/miniforge.sh -b -p $W/conda > /dev/null
|
||||||
C=$W/conda/bin
|
C=$W/conda/bin
|
||||||
@@ -94,6 +97,7 @@ if $C/conda install -y -q -n base conda-build > $R/conda.log 2>&1 \
|
|||||||
else
|
else
|
||||||
step conda_build FAILED; tail -40 $R/conda.log
|
step conda_build FAILED; tail -40 $R/conda.log
|
||||||
fi
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
echo "=== 5. digests $(date -u +%T) ==="
|
echo "=== 5. digests $(date -u +%T) ==="
|
||||||
$W/fresh/bin/python - "$R" <<'PY' | tee $R/agreement.txt
|
$W/fresh/bin/python - "$R" <<'PY' | tee $R/agreement.txt
|
||||||
@@ -104,10 +108,13 @@ for kind in ("gtc", "vcf"):
|
|||||||
runs = {p.stem.split("_", 2)[2]: json.loads(p.read_text())[kind] for p in sorted(r.glob(f"digest_{kind}_*.json"))}
|
runs = {p.stem.split("_", 2)[2]: json.loads(p.read_text())[kind] for p in sorted(r.glob(f"digest_{kind}_*.json"))}
|
||||||
if not runs:
|
if not runs:
|
||||||
continue
|
continue
|
||||||
ref_name, ref = "fresh", runs.get("fresh", {})
|
ref = runs.get("fresh", {})
|
||||||
|
if not ref:
|
||||||
|
print(f"{kind}: no reference (the pip-installed tutorial produced no outputs)")
|
||||||
|
continue
|
||||||
for name, d in runs.items():
|
for name, d in runs.items():
|
||||||
same = d == ref and len(d) > 0
|
if name != "fresh":
|
||||||
print(f"{kind}: {name} vs {ref_name}: {'IDENTICAL' if same else 'DIFFERENT'} ({len(d)} files)")
|
print(f"{kind}: {name} vs fresh: {'IDENTICAL' if d == ref else 'DIFFERENT'} ({len(d)} vs {len(ref)} files)")
|
||||||
PY
|
PY
|
||||||
{
|
{
|
||||||
echo "{"; echo " \"arch\": \"$ARCH\","
|
echo "{"; echo " \"arch\": \"$ARCH\","
|
||||||
|
|||||||
@@ -0,0 +1,3 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
# P10 acceptance without the Docker and conda parts (already verified): build, pip installs, tutorial, digests.
|
||||||
|
SKIP_DOCKER=1 SKIP_CONDA=1 bash "$(dirname "$0")/p10.sh" "$@"
|
||||||
@@ -7,6 +7,7 @@ Later phases add the other commands of CLI-001.
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
import os
|
||||||
import sys
|
import sys
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -58,6 +59,11 @@ def run_guarded(reporter: Reporter, fn: Callable[[], int]) -> None:
|
|||||||
code = ExitCode.OK
|
code = ExitCode.OK
|
||||||
try:
|
try:
|
||||||
code = ExitCode(fn())
|
code = ExitCode(fn())
|
||||||
|
sys.stdout.flush() # surface a closed pipe here, not at interpreter exit
|
||||||
|
except BrokenPipeError:
|
||||||
|
# the consumer of stdout stopped reading (`| head`): silence further writes, exit like a SIGPIPE'd tool
|
||||||
|
os.dup2(os.open(os.devnull, os.O_WRONLY), sys.stdout.fileno())
|
||||||
|
code = ExitCode.SIGPIPE
|
||||||
except SimurgError as e:
|
except SimurgError as e:
|
||||||
reporter.error(e.code, e.message + (f"\n hint: {e.hint}" if e.hint else ""), exc=e, file=e.path)
|
reporter.error(e.code, e.message + (f"\n hint: {e.hint}" if e.hint else ""), exc=e, file=e.path)
|
||||||
code = e.exit_code
|
code = e.exit_code
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ class ExitCode(IntEnum):
|
|||||||
PARTIAL = 3 # >= 1 sample failed and >= 1 succeeded
|
PARTIAL = 3 # >= 1 sample failed and >= 1 succeeded
|
||||||
ALL_FAILED = 4 # run completed but no sample succeeded
|
ALL_FAILED = 4 # run completed but no sample succeeded
|
||||||
SIGINT = 130
|
SIGINT = 130
|
||||||
|
SIGPIPE = 141 # the reader of stdout went away (e.g. `simurg inspect … | head`); nothing is reported
|
||||||
SIGTERM = 143
|
SIGTERM = 143
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -93,3 +93,18 @@ def test_locale_independent_output(tmp_path) -> None: # type: ignore[no-untyped
|
|||||||
assert r.returncode == 0, r.stderr
|
assert r.returncode == 0, r.stderr
|
||||||
outs.add(r.stdout)
|
outs.add(r.stdout)
|
||||||
assert len(outs) == 1
|
assert len(outs) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_closed_stdout_is_not_an_internal_error(tmp_path) -> None: # type: ignore[no-untyped-def]
|
||||||
|
# `simurg inspect … | head`: the reader goes away; the CLI must exit like a SIGPIPE'd tool (141), silently
|
||||||
|
f = _files(tmp_path)
|
||||||
|
p = subprocess.Popen(
|
||||||
|
[sys.executable, "-m", "simurg.cli.main", "inspect", "--json", str(f["bpm"]), str(f["egt"]), str(f["idat"])],
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.PIPE,
|
||||||
|
)
|
||||||
|
assert p.stdout is not None and p.stderr is not None
|
||||||
|
p.stdout.close() # nobody reads stdout
|
||||||
|
err = p.stderr.read().decode()
|
||||||
|
assert p.wait() == 141, err
|
||||||
|
assert "E_INTERNAL" not in err and "Traceback" not in err and "Exception ignored" not in err
|
||||||
Reference in new issue
Block a user