P5: cloud task for cohort QC validation; configurable gp3 throughput; batched contamination map
- scripts/cloud/tasks/p5.sh: P4 preset re-check, genotype (v5) with QC, validate_p5_large.py - scripts/validate_p5_large.py: identify samples via het pattern vs 1000G VCF; sex, parent-offspring, Mendel, contamination/control distributions - trial.sh: DISK_MBPS / DISK_IOPS - qc/contamination: bounded map batches (executor.map submits eagerly)
This commit is contained in:
1 parent
a8caa26aa4
commit
cea710f7b7
4 files changed
+357
-3
No files matched your search
Executable
+38
@@ -0,0 +1,38 @@
|
||||
#!/bin/bash
|
||||
# Phase P5 bundle on the 1000 Genomes Omni2.5 set (2141 samples):
|
||||
# 1. P4 re-check with --preset autocall-1.6.2 (expect byte-identical GTCs except the K-009 samples);
|
||||
# 2. `simurg genotype` (legacy preset, GTC v5 so BAF exists) with control tables and cohort QC;
|
||||
# 3. P5 validation against the 1000G pedigree and VCF (scripts/validate_p5_large.py).
|
||||
# Args: <data dir with bpm/egt and data/> <results dir>
|
||||
set -uo pipefail
|
||||
D=$1; OUT=$2
|
||||
SRC=s3://1000genomes/release/20130502/supporting/hd_genotype_chip
|
||||
mkdir -p $OUT/p4 $OUT/p5 $D/egt
|
||||
cp $D/HumanOmni2.5-4v1-Multi_B.egt $D/egt/
|
||||
|
||||
echo "=== P4 preset re-check $(date -u +%T) ==="
|
||||
/usr/bin/time -v .venv/bin/python scripts/validate_p4_large.py --root $D/data --bpm $D/HumanOmni2.5-4v1-Multi_B.bpm \
|
||||
--egt-dir $D/egt --out $OUT/p4/summary.json --workers "$(nproc)" 2> $OUT/p4/time.txt \
|
||||
|| echo "P4 validation reported differences"
|
||||
tail -22 $OUT/p4/time.txt
|
||||
|
||||
echo "=== P5 genotype + QC $(date -u +%T) ==="
|
||||
GT=/work/gt
|
||||
/usr/bin/time -v .venv/bin/simurg genotype --bpm $D/HumanOmni2.5-4v1-Multi_B.bpm --egt $D/HumanOmni2.5-4v1-Multi_B.egt \
|
||||
--idats $D/data --out $GT --preset autocall-1.6.2 --gtc-version 5 --autocall-date "1/1/2026 12:00:00 AM" \
|
||||
--qc-profile genotyping --threads "$(nproc)" --quiet 2> $OUT/p5/genotype_time.txt || echo "genotype exit $?"
|
||||
tail -22 $OUT/p5/genotype_time.txt
|
||||
cp $GT/sample_summary.tsv $GT/controls_qc.tsv $GT/relatedness.tsv $GT/run_manifest.json $OUT/p5/ 2>/dev/null
|
||||
cp $GT/mendel.tsv $OUT/p5/ 2>/dev/null
|
||||
cp -r $GT/logs $OUT/p5/ 2>/dev/null
|
||||
gzip -c $GT/controls_raw.tsv > $OUT/p5/controls_raw.tsv.gz 2>/dev/null
|
||||
df -h /work | tail -1
|
||||
|
||||
echo "=== P5 validation $(date -u +%T) ==="
|
||||
aws s3 cp --no-sign-request "$SRC/broad_intensities/Omni25_genotypes_2141_samples.b37.v2.vcf.gz" /work/ --only-show-errors
|
||||
aws s3 cp --no-sign-request "s3://1000genomes/release/20130502/integrated_call_samples.20130502.ALL.ped" /work/ --only-show-errors
|
||||
/usr/bin/time -v .venv/bin/python scripts/validate_p5_large.py --gt-dir $GT --bpm $D/HumanOmni2.5-4v1-Multi_B.bpm \
|
||||
--vcf /work/Omni25_genotypes_2141_samples.b37.v2.vcf.gz --ped /work/integrated_call_samples.20130502.ALL.ped \
|
||||
--out $OUT/p5/summary.json --workers "$(nproc)" 2> $OUT/p5/validate_time.txt || echo "P5 validation reported differences"
|
||||
tail -22 $OUT/p5/validate_time.txt
|
||||
echo "=== task done $(date -u +%T) ==="
|
||||
@@ -17,6 +17,8 @@ TYPE=${TYPE:-m7g.4xlarge}
|
||||
MAX_PRICE=${MAX_PRICE:-0.30} # USD/hour spot cap (on-demand is 0.6528)
|
||||
MAX_MINUTES=${MAX_MINUTES:-360} # hard cap: 6 hours
|
||||
DISK_GB=${DISK_GB:-250}
|
||||
DISK_MBPS=${DISK_MBPS:-125} # gp3 throughput (125 = free baseline; each extra MB/s ~ $0.00005/h)
|
||||
DISK_IOPS=${DISK_IOPS:-3000} # gp3 IOPS (3000 = free baseline)
|
||||
BUDGET_USD=${BUDGET_USD:-10}
|
||||
TASK=${TASK:-p3} # scripts/cloud/tasks/<TASK>.sh
|
||||
TAG=simurg-trial
|
||||
@@ -72,7 +74,7 @@ launch)
|
||||
--iam-instance-profile Name=$ROLE \
|
||||
--instance-initiated-shutdown-behavior terminate \
|
||||
--instance-market-options "MarketType=spot,SpotOptions={MaxPrice=$MAX_PRICE,SpotInstanceType=one-time,InstanceInterruptionBehavior=terminate}" \
|
||||
--block-device-mappings "DeviceName=/dev/sda1,Ebs={VolumeSize=$DISK_GB,VolumeType=gp3,DeleteOnTermination=true}" \
|
||||
--block-device-mappings "DeviceName=/dev/sda1,Ebs={VolumeSize=$DISK_GB,VolumeType=gp3,Throughput=$DISK_MBPS,Iops=$DISK_IOPS,DeleteOnTermination=true}" \
|
||||
--metadata-options HttpTokens=required \
|
||||
--user-data file:///tmp/simurg-userdata.sh \
|
||||
--tag-specifications "ResourceType=instance,Tags=[{Key=Project,Value=$TAG},{Key=Run,Value=$RUN}]" \
|
||||
|
||||
Reference in new issue
Block a user