P5a: QC — control metrics, thresholds, status, contamination, relatedness, locus summary
- qc/controls: canonical control names (QC-002), bkg/offset (QC-003), 13 genotyping metrics (QC-004/005), missing/zero-divisor handling (QC-007/008); controls_raw.tsv / controls_qc.tsv (QC-009/011) incl. compat CSVs - qc/thresholds + profiles: YAML schema, validation codes, precedence, shipped profiles, auto (QC-020..025) - qc/status: functional PASS/FAIL/NA, control CLEAR/FLAGGED, flags (QC-030..033) - qc/contamination: BAF regression; bit-compatible with gtc2vcf BAFregress (36/36, K-017), exact mode too - qc/relatedness: KING-robust, duplicates, Mendelian errors, pedigree from sheet (IN-018) - qc/locus_summary: OUT-072, bit-identical to BeadArrayFiles except ChiTest (K-018) - qc/suggest: suggest-thresholds (QC-026) - pipeline/qc: disk-backed cohort store; CLI qc controls|samples|contamination|relatedness|loci|suggest-thresholds - genotype: control tables (OUT-070), het_rate, QC columns, cohort QC - tests: unit + oracle (BAFregress plugin, BeadArrayFiles locus summary)
This commit is contained in:
kemal committed
1 parent
f469c97917
commit
a4c9c2d7d8
28 files changed
+3355
-22
No files matched your search
@@ -300,6 +300,18 @@ def genotype_cmd(
|
||||
str | None, typer.Option("--autocall-date", help="Fixed value for GTC field 201 (reproducible output).")
|
||||
] = None,
|
||||
threads: Annotated[int, typer.Option("--threads", min=0, max=1024, help="Worker processes (0 = auto).")] = 0,
|
||||
controls: Annotated[
|
||||
bool, typer.Option("--controls/--no-controls", help="Write controls_raw.tsv and controls_qc.tsv.")
|
||||
] = True,
|
||||
cohort_qc: Annotated[
|
||||
bool, typer.Option("--cohort-qc/--no-cohort-qc", help="Contamination and relatedness after genotyping.")
|
||||
] = True,
|
||||
qc_config: Annotated[Path | None, typer.Option("--qc-config", help="QC threshold YAML (QC-020).")] = None,
|
||||
qc_profile: Annotated[str | None, typer.Option("--qc-profile", help="QC profile name or 'auto'.")] = None,
|
||||
threshold_call_rate: Annotated[float | None, typer.Option("--threshold-call-rate")] = None,
|
||||
threshold_lrr_sd: Annotated[float | None, typer.Option("--threshold-lrr-sd")] = None,
|
||||
threshold: Annotated[list[str] | None, typer.Option("--threshold", help="KEY=VALUE (repeatable).")] = None,
|
||||
bg_offset: Annotated[float | None, typer.Option("--bg-offset", help="Control background offset (3000).")] = None,
|
||||
force: Annotated[bool, typer.Option("--force", help="Replace existing outputs.")] = False,
|
||||
layout: Annotated[str, typer.Option("--layout", help="nested|flat")] = "nested",
|
||||
on_duplicate_idat: Annotated[str, typer.Option("--on-duplicate-idat", help="error|first|newest")] = "error",
|
||||
@@ -345,10 +357,276 @@ def genotype_cmd(
|
||||
sex_version=sex_version,
|
||||
gtc_version=gtc_version,
|
||||
autocall_date=autocall_date,
|
||||
controls=controls,
|
||||
cohort_qc=cohort_qc,
|
||||
qc_config=qc_config,
|
||||
qc_profile=qc_profile,
|
||||
threshold_call_rate=threshold_call_rate,
|
||||
threshold_lrr_sd=threshold_lrr_sd,
|
||||
thresholds=list(threshold or []),
|
||||
bg_offset=bg_offset,
|
||||
argv=sys.argv[1:],
|
||||
)
|
||||
run_guarded(rep, lambda: run_genotype(opts, rep))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------------------------------------
|
||||
qc_app = typer.Typer(help="Quality control: controls, samples, contamination, relatedness, loci, thresholds.")
|
||||
app.add_typer(qc_app, name="qc")
|
||||
|
||||
|
||||
def _reporter(command: str, out: Path | None, log_json: bool, debug: bool, quiet: bool) -> Reporter:
|
||||
return Reporter(
|
||||
command=command,
|
||||
level="debug" if debug else "info",
|
||||
json_stderr=log_json,
|
||||
debug=debug,
|
||||
quiet=quiet,
|
||||
log_file=(out / "logs" / f"{command.replace(' ', '_')}.log") if out is not None else None,
|
||||
)
|
||||
|
||||
|
||||
LogJson = Annotated[bool, typer.Option("--log-json", "--json-log", help="JSON Lines log on stderr.")]
|
||||
Debug = Annotated[bool, typer.Option("--debug", help="Debug logging with tracebacks.")]
|
||||
Quiet = Annotated[bool, typer.Option("--quiet", "--no-progress", help="Suppress info messages.")]
|
||||
|
||||
|
||||
@qc_app.command("controls")
|
||||
def qc_controls_cmd(
|
||||
out: Annotated[Path, typer.Option("--out", help="Output folder (required).")],
|
||||
array_type: Annotated[str, typer.Option("--array-type", help="genotyping|methylation (required).")],
|
||||
bpm: Annotated[Path | None, typer.Option("--bpm", help="BPM manifest (control definitions).")] = None,
|
||||
csv: Annotated[Path | None, typer.Option("--csv", help="CSV manifest (control definitions).")] = None,
|
||||
idats: Annotated[Path | None, typer.Option("--idats", help="Folder searched recursively for IDAT pairs.")] = None,
|
||||
sheet: Annotated[Path | None, typer.Option("--sheet", help="Sample sheet (CSV).")] = None,
|
||||
bg_offset: Annotated[float, typer.Option("--bg-offset", help="Background offset x (QC-003).")] = 3000.0,
|
||||
compat: Annotated[bool, typer.Option("--compat-column-names", help="Reference-style CSV names.")] = False,
|
||||
force: Annotated[bool, typer.Option("--force", help="Replace existing control tables.")] = False,
|
||||
log_json: LogJson = False,
|
||||
debug: Debug = False,
|
||||
quiet: Quiet = False,
|
||||
) -> None:
|
||||
"""Control-probe QC from IDATs without genotyping (QC-012)."""
|
||||
from simurg.pipeline.qc import QcControlsOptions, run_qc_controls
|
||||
|
||||
if array_type not in ("genotyping", "methylation"):
|
||||
raise typer.BadParameter("--array-type must be genotyping or methylation")
|
||||
rep = _reporter("qc controls", out, log_json, debug, quiet)
|
||||
opts = QcControlsOptions(
|
||||
out=out,
|
||||
bpm=bpm,
|
||||
csv=csv,
|
||||
idats=idats,
|
||||
sheet=sheet,
|
||||
array_type=array_type,
|
||||
bg_offset=bg_offset,
|
||||
compat=compat,
|
||||
force=force,
|
||||
argv=sys.argv[1:],
|
||||
)
|
||||
run_guarded(rep, lambda: run_qc_controls(opts, rep))
|
||||
|
||||
|
||||
def _qc_cohort(
|
||||
command: str,
|
||||
out: Path,
|
||||
bpm: Path,
|
||||
gtcs: Path | None,
|
||||
sheet: Path | None,
|
||||
egt: Path | None,
|
||||
threads: int,
|
||||
force: bool,
|
||||
log_json: bool,
|
||||
debug: bool,
|
||||
quiet: bool,
|
||||
*,
|
||||
all_pairs: bool = False,
|
||||
pedigree_from_sheet: bool = True,
|
||||
median: str = "reference",
|
||||
config: Path | None = None,
|
||||
profile: str | None = None,
|
||||
threshold_call_rate: float | None = None,
|
||||
threshold_lrr_sd: float | None = None,
|
||||
threshold: list[str] | None = None,
|
||||
) -> None:
|
||||
from simurg.pipeline.qc import QcCohortOptions, run_qc_cohort
|
||||
from simurg.qc.thresholds import parse_cli_pairs, resolve_thresholds
|
||||
|
||||
if median not in ("reference", "exact"):
|
||||
raise typer.BadParameter("--median must be reference or exact")
|
||||
rep = _reporter(f"qc {command}", out, log_json, debug, quiet)
|
||||
|
||||
def body() -> int:
|
||||
cli: dict[str, Any] = parse_cli_pairs(list(threshold or []))
|
||||
if threshold_call_rate is not None:
|
||||
cli["callRate"] = threshold_call_rate
|
||||
if threshold_lrr_sd is not None:
|
||||
cli["logRDev"] = threshold_lrr_sd
|
||||
th = resolve_thresholds(config=config, profile=profile, cli=cli)
|
||||
opts = QcCohortOptions(
|
||||
command=command,
|
||||
out=out,
|
||||
bpm=bpm,
|
||||
gtcs=gtcs,
|
||||
sheet=sheet,
|
||||
egt=egt,
|
||||
threads=threads,
|
||||
force=force,
|
||||
all_pairs=all_pairs,
|
||||
pedigree_from_sheet=pedigree_from_sheet,
|
||||
median=median,
|
||||
thresholds=th,
|
||||
argv=sys.argv[1:],
|
||||
)
|
||||
return run_qc_cohort(opts, rep)
|
||||
|
||||
run_guarded(rep, body)
|
||||
|
||||
|
||||
GtcsOpt = Annotated[Path | None, typer.Option("--gtcs", help="Folder searched recursively for GTC files.")]
|
||||
SheetOpt = Annotated[Path | None, typer.Option("--sheet", help="Sample sheet (CSV).")]
|
||||
BpmOpt = Annotated[Path, typer.Option("--bpm", help="BPM manifest (required).")]
|
||||
OutOpt = Annotated[Path, typer.Option("--out", help="Output folder (required).")]
|
||||
ThreadsOpt = Annotated[int, typer.Option("--threads", min=0, max=1024, help="Worker processes (0 = auto).")]
|
||||
ForceOpt = Annotated[bool, typer.Option("--force", help="Replace existing outputs.")]
|
||||
MedianOpt = Annotated[str, typer.Option("--median", help="reference (default, bit-compatible) or exact.")]
|
||||
|
||||
|
||||
@qc_app.command("samples")
|
||||
def qc_samples_cmd(
|
||||
out: OutOpt,
|
||||
bpm: BpmOpt,
|
||||
gtcs: GtcsOpt = None,
|
||||
sheet: SheetOpt = None,
|
||||
config: Annotated[Path | None, typer.Option("--config", help="QC threshold YAML.")] = None,
|
||||
profile: Annotated[str | None, typer.Option("--profile", help="QC profile name or 'auto'.")] = None,
|
||||
threshold_call_rate: Annotated[float | None, typer.Option("--threshold-call-rate")] = None,
|
||||
threshold_lrr_sd: Annotated[float | None, typer.Option("--threshold-lrr-sd")] = None,
|
||||
threshold: Annotated[list[str] | None, typer.Option("--threshold", help="KEY=VALUE (repeatable).")] = None,
|
||||
all_pairs: Annotated[bool, typer.Option("--all-pairs", help="Write every sample pair.")] = False,
|
||||
median: MedianOpt = "reference",
|
||||
threads: ThreadsOpt = 0,
|
||||
force: ForceOpt = False,
|
||||
log_json: LogJson = False,
|
||||
debug: Debug = False,
|
||||
quiet: Quiet = False,
|
||||
) -> None:
|
||||
"""Sample QC from GTCs: het rate, contamination, relatedness, flags (sample_qc.tsv)."""
|
||||
_qc_cohort(
|
||||
"samples",
|
||||
out,
|
||||
bpm,
|
||||
gtcs,
|
||||
sheet,
|
||||
None,
|
||||
threads,
|
||||
force,
|
||||
log_json,
|
||||
debug,
|
||||
quiet,
|
||||
all_pairs=all_pairs,
|
||||
median=median,
|
||||
config=config,
|
||||
profile=profile,
|
||||
threshold_call_rate=threshold_call_rate,
|
||||
threshold_lrr_sd=threshold_lrr_sd,
|
||||
threshold=threshold,
|
||||
)
|
||||
|
||||
|
||||
@qc_app.command("contamination")
|
||||
def qc_contamination_cmd(
|
||||
out: OutOpt,
|
||||
bpm: BpmOpt,
|
||||
gtcs: GtcsOpt = None,
|
||||
sheet: SheetOpt = None,
|
||||
median: MedianOpt = "reference",
|
||||
threads: ThreadsOpt = 0,
|
||||
force: ForceOpt = False,
|
||||
log_json: LogJson = False,
|
||||
debug: Debug = False,
|
||||
quiet: Quiet = False,
|
||||
) -> None:
|
||||
"""BAF-regression contamination estimate per sample (contamination.tsv)."""
|
||||
_qc_cohort("contamination", out, bpm, gtcs, sheet, None, threads, force, log_json, debug, quiet, median=median)
|
||||
|
||||
|
||||
@qc_app.command("relatedness")
|
||||
def qc_relatedness_cmd(
|
||||
out: OutOpt,
|
||||
bpm: BpmOpt,
|
||||
gtcs: GtcsOpt = None,
|
||||
sheet: SheetOpt = None,
|
||||
pedigree_from_sheet: Annotated[
|
||||
bool, typer.Option("--pedigree-from-sheet/--no-pedigree", help="Use Parent1/Parent2/Replicate.")
|
||||
] = True,
|
||||
all_pairs: Annotated[bool, typer.Option("--all-pairs", help="Write every sample pair.")] = False,
|
||||
threads: ThreadsOpt = 0,
|
||||
force: ForceOpt = False,
|
||||
log_json: LogJson = False,
|
||||
debug: Debug = False,
|
||||
quiet: Quiet = False,
|
||||
) -> None:
|
||||
"""KING-robust kinship, duplicates and Mendelian errors (relatedness.tsv, mendel.tsv)."""
|
||||
_qc_cohort(
|
||||
"relatedness",
|
||||
out,
|
||||
bpm,
|
||||
gtcs,
|
||||
sheet,
|
||||
None,
|
||||
threads,
|
||||
force,
|
||||
log_json,
|
||||
debug,
|
||||
quiet,
|
||||
all_pairs=all_pairs,
|
||||
pedigree_from_sheet=pedigree_from_sheet,
|
||||
)
|
||||
|
||||
|
||||
@qc_app.command("loci")
|
||||
def qc_loci_cmd(
|
||||
out: OutOpt,
|
||||
bpm: BpmOpt,
|
||||
egt: Annotated[Path, typer.Option("--egt", help="EGT cluster file (required).")],
|
||||
gtcs: GtcsOpt = None,
|
||||
sheet: SheetOpt = None,
|
||||
threads: ThreadsOpt = 0,
|
||||
force: ForceOpt = False,
|
||||
log_json: LogJson = False,
|
||||
debug: Debug = False,
|
||||
quiet: Quiet = False,
|
||||
) -> None:
|
||||
"""Per-locus summary statistics across samples (locus_summary.tsv, OUT-072)."""
|
||||
_qc_cohort("loci", out, bpm, gtcs, sheet, egt, threads, force, log_json, debug, quiet)
|
||||
|
||||
|
||||
@qc_app.command("suggest-thresholds")
|
||||
def qc_suggest_cmd(
|
||||
data: Annotated[list[Path], typer.Option("--data", help="Dataset folder(s) with controls_qc.tsv.")],
|
||||
out: Annotated[Path, typer.Option("--out", help="Output YAML file (required).")],
|
||||
k: Annotated[float, typer.Option("--k", min=0.0, help="Spread multiplier.")] = 4.0,
|
||||
min_samples: Annotated[int, typer.Option("--min-samples", min=1, help="Minimum samples per metric.")] = 30,
|
||||
config: Annotated[Path | None, typer.Option("--config", help="Current thresholds (YAML).")] = None,
|
||||
profile: Annotated[str | None, typer.Option("--profile", help="Current profile.")] = None,
|
||||
log_json: LogJson = False,
|
||||
debug: Debug = False,
|
||||
quiet: Quiet = False,
|
||||
) -> None:
|
||||
"""Data-driven QC threshold suggestion (QC-026)."""
|
||||
from simurg.pipeline.qc import run_suggest_thresholds
|
||||
from simurg.qc.thresholds import resolve_thresholds
|
||||
|
||||
rep = _reporter("qc suggest-thresholds", None, log_json, debug, quiet)
|
||||
folders = [Path(p) for d in data for p in str(d).split(",") if p]
|
||||
|
||||
def body() -> int:
|
||||
th = resolve_thresholds(config=config, profile=profile)
|
||||
return run_suggest_thresholds(folders, out, th, rep, k=k, min_samples=min_samples)
|
||||
|
||||
run_guarded(rep, body)
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
app()
|
||||
Reference in new issue
Block a user