P3: vectorised nearest-neighbour search; task-driven cloud trials

- fit.py: exact and binned nearest-neighbour searches use one distance
  matrix per searched bin (first-minimum rule kept); fit 2.6 s -> 1.4 s
  per GSA sample, still bit-identical to the oracle
- scripts/validate_p3_large.py: refits 1.2.0 and 1.1.2 transforms for
  every vendor GTC and reports ULP distances per field
- cloud trial runs scripts/cloud/tasks/<TASK>.sh (TASK=p3 default;
  p1p2 = previous validation + ingest)
This commit is contained in:
kemal committed 2026-09-27 22:57:03 +01:00
1 parent 95e4e41560
commit bf00b0e453
6 files changed
+197 -44

No files matched your search

+5 -15
View File
@@ -54,20 +54,10 @@ echo "=== streaming 107 GB IDAT+GTC archive (no copy of the .tgz is kept) $(date
$S3 cp --no-sign-request "$SRC/Omni25_idats_gtcs_2141_samples.tgz" - | pigz -dc | tar -x -C $D/data
echo "=== extracted $(du -sh $D/data | cut -f1) at $(date -u +%T) ==="
echo "=== validation $(date -u +%T) ==="
echo "=== unit tests $(date -u +%T) ==="
.venv/bin/python -m pytest tests/unit -q -p no:cacheprovider || echo "unit tests reported failures (continuing)"
.venv/bin/python scripts/validate_p1_large.py --root $D/data --bpm $D/HumanOmni2.5-4v1-Multi_B.bpm \
--egt $D/HumanOmni2.5-4v1-Multi_B.egt --csv $D/HumanOmni2.5-4v1_B.csv --out /work/summary.json \
--workers "$(nproc)" || echo "validation reported differences (see summary.json)"
$S3 cp /work/summary.json "s3://$BUCKET/$RUN/summary.json" --only-show-errors
echo "=== simurg ingest at scale $(date -u +%T) ==="
/usr/bin/time -v .venv/bin/simurg ingest --bpm $D/HumanOmni2.5-4v1-Multi_B.bpm --idats $D/data \
--out /work/ingest --threads "$(nproc)" --quiet 2> /work/ingest_time.txt || echo "ingest exit code $?"
tail -25 /work/ingest_time.txt
for f in ingest_summary.tsv run_manifest.json; do
$S3 cp "/work/ingest/$f" "s3://$BUCKET/$RUN/ingest_$f" --only-show-errors || true
done
$S3 cp /work/ingest_time.txt "s3://$BUCKET/$RUN/ingest_time.txt" --only-show-errors || true
echo "ingest output size: $(du -sh /work/ingest | cut -f1)"
echo "=== task __TASK__ $(date -u +%T) ==="
mkdir -p /work/results
bash scripts/cloud/tasks/__TASK__.sh $D /work/results || echo "task exit code $?"
$S3 cp /work/results "s3://$BUCKET/$RUN/" --recursive --only-show-errors || true
echo "=== done $(date -u +%T) ==="