P6: gtc-to-vcf (VCF/BCF + indexes), harmonisation, exports (plink, bedgraph, finalreport)
- formats/fasta: plain/BGZF FASTA, .fai/.gzi built identical to htslib, index check, build detection, contig aliases, mmap genome cache (IN-060..064) - formats/bgzf, formats/bcf: pure-Python BGZF, tabix .tbi, BCF 2.2 and CSI (OUT-010, OUT-038, OUT-040) - harmonize/sites: plus-strand mapping with audit rows, indel resolution from SourceSeq (port of GTCtoVCF, Apache-2.0) and left normalisation (OUT-014..021, OUT-030, OUT-042) - harmonize/combine: OUT-031 genotype combination (ploidy-aware, Infinium I/II signal classes, OUT-037) - pipeline/vcf: `simurg gtc-to-vcf` (per-sample VCF.gz+tbi, --merge cohort BCF+csi, audit parquet) - pipeline/export: `simurg export bedgraph|finalreport|plink|locus-summary` (OUT-043, OUT-050..052, OUT-071) - validation: 36 GSA samples vs gtc2vcf: 23.66 M identical calls, 0 unexplained differences; PLINK identical to plink2 on 637,397 variants; bcftools validates VCF/BCF and our indexes answer region queries correctly - fetch_data.sh p6 (Ensembl GRCh38 with checksum), make_oracle_vcf.sh, validate_p6_vcf.py; K-022/K-023 Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_01814GKVzZVL3FuF3DeiZXhv
This commit is contained in:
kemalandClaude Opus 5.5 committed
1 parent
60a05d5048
commit
b56fdebaaf
19 files changed
+3181
-9
No files matched your search
@@ -74,6 +74,18 @@ p2)
|
||||
get "https://ftp.ncbi.nlm.nih.gov/geo/series/GSE96nnn/GSE96909/suppl/GSE96909_RAW.tar" "$d/gse96909"
|
||||
get "https://s3.amazonaws.com/webdata.illumina.com/downloads/productfiles/global-diversity-array-pgx/GDAPGx_Demo_idats.zip" "$d/gda_pgx480"
|
||||
;;
|
||||
p6)
|
||||
# GRCh38 primary assembly (Ensembl release 110; contigs named 1..22, X, Y, MT like the GSA/GDA manifests)
|
||||
d=data/p6; mkdir -p "$d"
|
||||
ENS=https://ftp.ensembl.org/pub/release-110/fasta/homo_sapiens/dna
|
||||
f=Homo_sapiens.GRCh38.dna.primary_assembly.fa.gz
|
||||
[[ -f "$d/$f" ]] || curl -sSfL -o "$d/$f" "$ENS/$f"
|
||||
want=$(curl -sSfL "$ENS/CHECKSUMS" | awk -v f="$f" '$3==f {print $1" "$2}')
|
||||
got=$(sum "$d/$f" | awk '{print $1" "$2}')
|
||||
[[ "$want" == "$got" ]] || { echo "!! checksum mismatch for $f ($got vs $want)"; exit 1; }
|
||||
[[ -f "$d/GRCh38.primary.fa" ]] || gzip -dc "$d/$f" > "$d/GRCh38.primary.fa"
|
||||
echo "FASTA ready: $d/GRCh38.primary.fa (the .fai is built on first use)"
|
||||
;;
|
||||
*) echo "unknown tier $tier"; exit 2;;
|
||||
esac
|
||||
echo "checksums in $sums"
|
||||
Executable
+20
@@ -0,0 +1,20 @@
|
||||
#!/usr/bin/env bash
|
||||
# Oracle VCFs for phase P6 on the 36 GSA oracle GTCs (needs make_oracle_gtcs.sh and fetch_data.sh p6):
|
||||
# * gtc2vcf (MIT) → data/p6/oracle/gsa36_gtc2vcf.vcf.gz (one multi-sample VCF, one record per probe)
|
||||
# * optional: GTCtoVCF (Apache-2.0) — python2-era code run under python3 with two local compatibility patches,
|
||||
# see docs/validation/P6.md; enable with GTCTOVCF=1.
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
B=refs/tools/bcftools-1.22
|
||||
export BCFTOOLS_PLUGINS=$B/plugins
|
||||
D=data/p0/gsa_v4_pgx
|
||||
OUT=data/p6/oracle
|
||||
mkdir -p $OUT data/p6/work
|
||||
$B/bcftools +gtc2vcf --no-version -t GT,BAF,LRR \
|
||||
-b $D/GSA-PGx-48v4-0_20079540_E2_bpm/GSA-PGx-48v4-0_20079540_E2.bpm \
|
||||
-c $D/GSA-PGx-48v4-0_20079540_E2_csv/GSA-PGx-48v4-0_20079540_E2.csv \
|
||||
-e $D/GSA-PGx-48v4-0_20079540_E2_ClusterFile/GSA-PGx-48v4-0_20079540_E2_ClusterFile.egt \
|
||||
-f data/p6/GRCh38.primary.fa -g data/p0/oracle/gsa_gt3_nofma -O u 2> $OUT/gtc2vcf.log \
|
||||
| $B/bcftools sort -Oz -o $OUT/gsa36_gtc2vcf.vcf.gz -T data/p6/work/sorttmp
|
||||
$B/bcftools index -f -t $OUT/gsa36_gtc2vcf.vcf.gz
|
||||
echo "wrote $OUT/gsa36_gtc2vcf.vcf.gz"
|
||||
@@ -0,0 +1,129 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Phase P6 acceptance: SimurgArray VCFs vs a reference converter's multi-sample VCF (gtc2vcf or GTCtoVCF).
|
||||
|
||||
Both sides are left-aligned with ``bcftools norm -f`` and compared per probe (ID) and sample as unordered allele
|
||||
sets (a haploid call ``a`` equals ``a/a``). Every difference is put into a category; the acceptance criterion
|
||||
(plan §18 P6) is zero *unexplained* genotype differences.
|
||||
|
||||
Usage: validate_p6_vcf.py --ours DIR --oracle F.vcf.gz --fasta F --bcftools PATH --out summary.json
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from collections import Counter, defaultdict
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def query(bcftools: str, fasta: str, path: Path) -> list[list[str]]:
|
||||
norm = subprocess.Popen([bcftools, "norm", "-f", fasta, "-c", "w", "-Ou", str(path)], stdout=subprocess.PIPE,
|
||||
stderr=subprocess.DEVNULL) # fmt: skip
|
||||
out = subprocess.run(
|
||||
[bcftools, "query", "-f", "%ID\t%CHROM\t%POS\t%REF\t%ALT[\t%GT]\n", "-"],
|
||||
stdin=norm.stdout, capture_output=True, text=True, check=True,
|
||||
).stdout # fmt: skip
|
||||
norm.wait()
|
||||
return [ln.split("\t") for ln in out.splitlines()]
|
||||
|
||||
|
||||
def alleles_of(gt: str, ref: str, alt: str) -> tuple[str, ...] | None:
|
||||
if gt in (".", "./.", ".|."):
|
||||
return None
|
||||
al = [ref, *alt.split(",")]
|
||||
idx = gt.replace("|", "/").split("/")
|
||||
if any(i == "." for i in idx):
|
||||
return None
|
||||
got = tuple(sorted(al[int(i)] for i in idx))
|
||||
return got * 2 if len(got) == 1 else got
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--ours", required=True, type=Path)
|
||||
ap.add_argument("--oracle", required=True, type=Path)
|
||||
ap.add_argument("--fasta", required=True)
|
||||
ap.add_argument("--bcftools", required=True)
|
||||
ap.add_argument("--out", required=True, type=Path)
|
||||
ap.add_argument("--name", default="oracle")
|
||||
a = ap.parse_args()
|
||||
|
||||
samples = subprocess.run([a.bcftools, "query", "-l", str(a.oracle)], capture_output=True, text=True,
|
||||
check=True).stdout.split() # fmt: skip
|
||||
oracle: dict[str, tuple[str, str, str, str, list[str]]] = {}
|
||||
for f in query(a.bcftools, a.fasta, a.oracle):
|
||||
for pid in f[0].replace(";", ",").split(","):
|
||||
oracle[pid] = (f[1], f[2], f[3], f[4], f[5:])
|
||||
cats: Counter[str] = Counter()
|
||||
examples: dict[str, list[Any]] = defaultdict(list)
|
||||
ours_ids: set[str] = set()
|
||||
probes_in_merged = 0
|
||||
for si, sample in enumerate(samples):
|
||||
vcf = a.ours / f"{sample}.snv.vcf.gz"
|
||||
if not vcf.exists():
|
||||
cats["sample_missing"] += 1
|
||||
continue
|
||||
for f in query(a.bcftools, a.fasta, vcf):
|
||||
ids = f[0].replace(";", ",").split(",")
|
||||
merged = len(ids) > 1
|
||||
ours_al = alleles_of(f[5], f[3], f[4])
|
||||
haploid = "/" not in f[5] and f[5] != "."
|
||||
for pid in ids:
|
||||
if si == 0:
|
||||
ours_ids.add(pid)
|
||||
probes_in_merged += merged
|
||||
o = oracle.get(pid)
|
||||
if o is None:
|
||||
cats["only_ours"] += 1
|
||||
continue
|
||||
if (o[0], o[1], o[2]) != (f[1], f[2], f[3]):
|
||||
cats["representation_diff"] += 1
|
||||
if len(examples["representation_diff"]) < 20:
|
||||
examples["representation_diff"].append([pid, f[1:5], list(o[:4])])
|
||||
continue
|
||||
theirs = alleles_of(o[4][si], o[2], o[3])
|
||||
if ours_al == theirs:
|
||||
cats["match_called" if ours_al else "match_missing"] += 1
|
||||
continue
|
||||
if ours_al is None:
|
||||
if merged:
|
||||
key = "ours_missing_merged_site"
|
||||
elif f[1] in ("Y", "chrY") and f[5] == ".":
|
||||
key = "ours_missing_haploid_Y"
|
||||
elif theirs is not None and theirs[0] != theirs[1] and (f[5] == "." or haploid):
|
||||
key = "ours_missing_het_in_haploid"
|
||||
else:
|
||||
key = "ours_missing_UNEXPLAINED"
|
||||
elif theirs is None:
|
||||
key = "oracle_missing_ours_called_merged" if merged else "oracle_missing_ours_called"
|
||||
else:
|
||||
key = "diff_merged_site" if merged else "diff_UNEXPLAINED"
|
||||
cats[key] += 1
|
||||
if len(examples[key]) < 20:
|
||||
examples[key].append([sample, pid, f[1:6], o[4][si], list(o[:4])])
|
||||
only_oracle = sorted(set(oracle) - ours_ids)
|
||||
summary = {
|
||||
"oracle": a.name,
|
||||
"samples": len(samples),
|
||||
"probes_oracle": len(oracle),
|
||||
"probes_ours": len(ours_ids),
|
||||
"probes_in_merged_records": probes_in_merged,
|
||||
"probes_only_in_oracle": len(only_oracle),
|
||||
"only_in_oracle_examples": only_oracle[:20],
|
||||
"categories": dict(sorted(cats.items())),
|
||||
"examples": examples,
|
||||
}
|
||||
unexplained = sum(v for k, v in cats.items() if "UNEXPLAINED" in k)
|
||||
summary["unexplained"] = unexplained
|
||||
a.out.parent.mkdir(parents=True, exist_ok=True)
|
||||
a.out.write_text(json.dumps(summary, indent=2) + "\n")
|
||||
print(json.dumps({k: summary[k] for k in ("probes_oracle", "probes_ours", "probes_only_in_oracle", "categories",
|
||||
"unexplained")})) # fmt: skip
|
||||
return 0 if unexplained == 0 else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user