P6: gtc-to-vcf (VCF/BCF + indexes), harmonisation, exports (plink, bedgraph, finalreport)
- formats/fasta: plain/BGZF FASTA, .fai/.gzi built identical to htslib, index check, build detection, contig aliases, mmap genome cache (IN-060..064) - formats/bgzf, formats/bcf: pure-Python BGZF, tabix .tbi, BCF 2.2 and CSI (OUT-010, OUT-038, OUT-040) - harmonize/sites: plus-strand mapping with audit rows, indel resolution from SourceSeq (port of GTCtoVCF, Apache-2.0) and left normalisation (OUT-014..021, OUT-030, OUT-042) - harmonize/combine: OUT-031 genotype combination (ploidy-aware, Infinium I/II signal classes, OUT-037) - pipeline/vcf: `simurg gtc-to-vcf` (per-sample VCF.gz+tbi, --merge cohort BCF+csi, audit parquet) - pipeline/export: `simurg export bedgraph|finalreport|plink|locus-summary` (OUT-043, OUT-050..052, OUT-071) - validation: 36 GSA samples vs gtc2vcf: 23.66 M identical calls, 0 unexplained differences; PLINK identical to plink2 on 637,397 variants; bcftools validates VCF/BCF and our indexes answer region queries correctly - fetch_data.sh p6 (Ensembl GRCh38 with checksum), make_oracle_vcf.sh, validate_p6_vcf.py; K-022/K-023
This commit is contained in:
kemal committed
1 parent
80b93e9f45
commit
0b97462565
19 files changed
+3181
-9
No files matched your search
@@ -370,6 +370,153 @@ def genotype_cmd(
|
||||
run_guarded(rep, lambda: run_genotype(opts, rep))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------------------------------------
|
||||
@app.command("gtc-to-vcf")
|
||||
def gtc_to_vcf_cmd(
|
||||
bpm: Annotated[Path, typer.Option("--bpm", help="BPM manifest (required).")],
|
||||
fasta: Annotated[Path, typer.Option("--fasta", help="Reference FASTA (plain or bgzip; required).")],
|
||||
out: Annotated[Path, typer.Option("--out", help="Output folder (required).")],
|
||||
gtcs: Annotated[Path | None, typer.Option("--gtcs", help="Folder searched recursively for GTC files.")] = None,
|
||||
sheet: Annotated[Path | None, typer.Option("--sheet", help="Sample sheet (CSV).")] = None,
|
||||
csv: Annotated[Path | None, typer.Option("--csv", help="CSV manifest (SourceSeq; required for indels).")] = None,
|
||||
no_bgzip: Annotated[bool, typer.Option("--no-bgzip", help="Plain VCF, no index.")] = False,
|
||||
genome_cache: Annotated[bool, typer.Option("--genome-cache/--no-genome-cache", help="mmap genome cache.")] = True,
|
||||
exclude_loci: Annotated[
|
||||
Path | None, typer.Option("--exclude-loci", "--filter-loci", help="Probe names to drop (one per line).")
|
||||
] = None,
|
||||
include_loci: Annotated[Path | None, typer.Option("--include-loci", help="Only these probe names.")] = None,
|
||||
unsquash_duplicates: Annotated[bool, typer.Option("--unsquash-duplicates", help="One record per probe.")] = False,
|
||||
inf1_nocall_informative: Annotated[
|
||||
bool, typer.Option("--inf1-nocall-informative", help="Read Infinium I no-calls as allele absence.")
|
||||
] = False,
|
||||
drop_ambiguous: Annotated[bool, typer.Option("--drop-ambiguous", help="Drop low-confidence strand loci.")] = False,
|
||||
filter_gs: Annotated[float | None, typer.Option("--filter-gs", help="FILTER LowGS when GS < X.")] = None,
|
||||
contig_alias: Annotated[str, typer.Option("--contig-alias", help="strict|auto|FILE")] = "strict",
|
||||
ploidy_policy: Annotated[str, typer.Option("--ploidy-policy", help="sex-aware|diploid")] = "sex-aware",
|
||||
threads: Annotated[int, typer.Option("--threads", min=0, max=1024, help="Worker processes (0 = auto).")] = 0,
|
||||
merge: Annotated[
|
||||
bool, typer.Option("--merge", help="One multi-sample BCF (+ .csi) instead of per-sample VCFs.")
|
||||
] = False,
|
||||
force: Annotated[bool, typer.Option("--force", help="Replace existing outputs.")] = False,
|
||||
log_json: Annotated[bool, typer.Option("--log-json", "--json-log", help="JSON Lines log on stderr.")] = False,
|
||||
debug: Annotated[bool, typer.Option("--debug", help="Debug logging with tracebacks.")] = False,
|
||||
quiet: Annotated[bool, typer.Option("--quiet", "--no-progress", help="Suppress info messages.")] = False,
|
||||
) -> None:
|
||||
"""GTC files → one bgzipped, indexed SNV VCF per sample (plus-strand alleles, merged multi-probe sites)."""
|
||||
from simurg.pipeline.vcf import VcfOptions, run_gtc_to_vcf
|
||||
|
||||
rep = Reporter(
|
||||
command="gtc-to-vcf",
|
||||
level="debug" if debug else "info",
|
||||
json_stderr=log_json,
|
||||
debug=debug,
|
||||
quiet=quiet,
|
||||
log_file=out / "logs" / "gtc-to-vcf.log",
|
||||
)
|
||||
opts = VcfOptions(
|
||||
out=out, bpm=bpm, fasta=fasta, gtcs=gtcs, sheet=sheet, csv=csv, bgzip=not no_bgzip, genome_cache=genome_cache,
|
||||
exclude_loci=exclude_loci, include_loci=include_loci, unsquash_duplicates=unsquash_duplicates,
|
||||
inf1_nocall_informative=inf1_nocall_informative, drop_ambiguous=drop_ambiguous, filter_gs=filter_gs,
|
||||
contig_alias=contig_alias, ploidy_policy=ploidy_policy, threads=threads, merge=merge, force=force,
|
||||
argv=sys.argv[1:],
|
||||
) # fmt: skip
|
||||
run_guarded(rep, lambda: run_gtc_to_vcf(opts, rep))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------------------------------------
|
||||
export_app = typer.Typer(help="Exports from GTCs or VCFs: bedgraph, finalreport, plink, locus-summary.")
|
||||
app.add_typer(export_app, name="export")
|
||||
|
||||
|
||||
def _plain_reporter(command: str, log_json: bool, debug: bool, quiet: bool) -> Reporter:
|
||||
return Reporter(command=command, level="debug" if debug else "info", json_stderr=log_json, debug=debug, quiet=quiet)
|
||||
|
||||
|
||||
@export_app.command("bedgraph")
|
||||
def export_bedgraph_cmd(
|
||||
bpm: Annotated[Path, typer.Option("--bpm", help="BPM manifest (required).")],
|
||||
out: Annotated[Path, typer.Option("--out", help="Output folder (required).")],
|
||||
gtcs: Annotated[Path | None, typer.Option("--gtcs", help="Folder searched recursively for GTC files.")] = None,
|
||||
sheet: Annotated[Path | None, typer.Option("--sheet", help="Sample sheet (CSV).")] = None,
|
||||
no_gzip: Annotated[bool, typer.Option("--no-gzip", help="Plain .bedgraph files.")] = False,
|
||||
chrom_style: Annotated[str, typer.Option("--chrom-style", help="manifest|ucsc")] = "manifest",
|
||||
smoothed_lrr: Annotated[int, typer.Option("--smoothed-lrr", min=0, help="Extra moving-median LRR track, K.")] = 0,
|
||||
force: Annotated[bool, typer.Option("--force", help="Replace existing outputs.")] = False,
|
||||
log_json: Annotated[bool, typer.Option("--log-json", "--json-log")] = False,
|
||||
debug: Annotated[bool, typer.Option("--debug")] = False,
|
||||
quiet: Annotated[bool, typer.Option("--quiet", "--no-progress")] = False,
|
||||
) -> None:
|
||||
"""BAF and LRR bedGraph tracks per sample (OUT-050 … OUT-052)."""
|
||||
from simurg.pipeline.export import BedgraphOptions, run_export_bedgraph
|
||||
|
||||
rep = _plain_reporter("export bedgraph", log_json, debug, quiet)
|
||||
opts = BedgraphOptions(out=out, bpm=bpm, gtcs=gtcs, sheet=sheet, gzip=not no_gzip, chrom_style=chrom_style,
|
||||
smoothed_lrr=smoothed_lrr, force=force, argv=sys.argv[1:]) # fmt: skip
|
||||
run_guarded(rep, lambda: run_export_bedgraph(opts, rep))
|
||||
|
||||
|
||||
@export_app.command("finalreport")
|
||||
def export_finalreport_cmd(
|
||||
bpm: Annotated[Path, typer.Option("--bpm", help="BPM manifest (required).")],
|
||||
out: Annotated[Path, typer.Option("--out", help="Output folder (required).")],
|
||||
gtcs: Annotated[Path | None, typer.Option("--gtcs", help="Folder searched recursively for GTC files.")] = None,
|
||||
sheet: Annotated[Path | None, typer.Option("--sheet", help="Sample sheet (CSV).")] = None,
|
||||
csv: Annotated[Path | None, typer.Option("--csv", help="CSV manifest (source/reference strands).")] = None,
|
||||
egt: Annotated[Path | None, typer.Option("--egt", help="Cluster file (GT Score column).")] = None,
|
||||
matrix: Annotated[bool, typer.Option("--matrix", help="Loci × samples genotype matrix.")] = False,
|
||||
force: Annotated[bool, typer.Option("--force", help="Replace existing outputs.")] = False,
|
||||
log_json: Annotated[bool, typer.Option("--log-json", "--json-log")] = False,
|
||||
debug: Annotated[bool, typer.Option("--debug")] = False,
|
||||
quiet: Annotated[bool, typer.Option("--quiet", "--no-progress")] = False,
|
||||
) -> None:
|
||||
"""Long-format final report (one row per locus × sample), or a genotype matrix (OUT-071)."""
|
||||
from simurg.pipeline.export import FinalReportOptions, run_export_finalreport
|
||||
|
||||
rep = _plain_reporter("export finalreport", log_json, debug, quiet)
|
||||
opts = FinalReportOptions(out=out, bpm=bpm, gtcs=gtcs, sheet=sheet, csv=csv, egt=egt, matrix=matrix, force=force,
|
||||
argv=sys.argv[1:]) # fmt: skip
|
||||
run_guarded(rep, lambda: run_export_finalreport(opts, rep))
|
||||
|
||||
|
||||
@export_app.command("plink")
|
||||
def export_plink_cmd(
|
||||
vcfs: Annotated[Path, typer.Option("--vcfs", help="Folder with SimurgArray per-sample *.snv.vcf.gz.")],
|
||||
out: Annotated[Path, typer.Option("--out", help="Output folder (required).")],
|
||||
sheet: Annotated[Path | None, typer.Option("--sheet", help="Sample sheet (reported sex).")] = None,
|
||||
gtcs: Annotated[Path | None, typer.Option("--gtcs", help="GTC folder (estimated sex).")] = None,
|
||||
multiallelic: Annotated[str, typer.Option("--multiallelic", help="split|drop")] = "split",
|
||||
prefix: Annotated[str, typer.Option("--prefix", help="Output file prefix.")] = "cohort",
|
||||
force: Annotated[bool, typer.Option("--force", help="Replace existing outputs.")] = False,
|
||||
log_json: Annotated[bool, typer.Option("--log-json", "--json-log")] = False,
|
||||
debug: Annotated[bool, typer.Option("--debug")] = False,
|
||||
quiet: Annotated[bool, typer.Option("--quiet", "--no-progress")] = False,
|
||||
) -> None:
|
||||
"""PLINK 1 .bed/.bim/.fam from SimurgArray VCFs (OUT-043)."""
|
||||
from simurg.pipeline.export import PlinkOptions, run_export_plink
|
||||
|
||||
rep = _plain_reporter("export plink", log_json, debug, quiet)
|
||||
opts = PlinkOptions(out=out, vcfs=vcfs, sheet=sheet, gtcs=gtcs, multiallelic=multiallelic, prefix=prefix,
|
||||
force=force, argv=sys.argv[1:]) # fmt: skip
|
||||
run_guarded(rep, lambda: run_export_plink(opts, rep))
|
||||
|
||||
|
||||
@export_app.command("locus-summary")
|
||||
def export_locus_summary_cmd(
|
||||
bpm: Annotated[Path, typer.Option("--bpm", help="BPM manifest (required).")],
|
||||
egt: Annotated[Path, typer.Option("--egt", help="EGT cluster file (required).")],
|
||||
out: Annotated[Path, typer.Option("--out", help="Output folder (required).")],
|
||||
gtcs: Annotated[Path | None, typer.Option("--gtcs", help="Folder searched recursively for GTC files.")] = None,
|
||||
sheet: Annotated[Path | None, typer.Option("--sheet", help="Sample sheet (CSV).")] = None,
|
||||
threads: Annotated[int, typer.Option("--threads", min=0, max=1024)] = 0,
|
||||
force: Annotated[bool, typer.Option("--force", help="Replace existing outputs.")] = False,
|
||||
log_json: Annotated[bool, typer.Option("--log-json", "--json-log")] = False,
|
||||
debug: Annotated[bool, typer.Option("--debug")] = False,
|
||||
quiet: Annotated[bool, typer.Option("--quiet", "--no-progress")] = False,
|
||||
) -> None:
|
||||
"""Per-locus summary statistics (OUT-072); same as ``simurg qc loci``."""
|
||||
_qc_cohort("loci", out, bpm, gtcs, sheet, egt, threads, force, log_json, debug, quiet)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------------------------------------
|
||||
qc_app = typer.Typer(help="Quality control: controls, samples, contamination, relatedness, loci, thresholds.")
|
||||
app.add_typer(qc_app, name="qc")
|
||||
|
||||
Reference in new issue
Block a user