161 lines
6.8 KiB
Python
161 lines
6.8 KiB
Python
"""Weekly reading list: new bioRxiv preprints and PubMed papers ranked by relevance to SimurgArray.
|
|
|
|
Code fetches the last N days from the bioRxiv API (bioinformatics, genomics, genetics, pharmacology) and PubMed
|
|
E-utilities, and keeps the abstracts that mention array, copy number, pharmacogenomics or methylation array terms.
|
|
For each kept paper Jev scores its relevance on four concrete levels (Score) and says whether it releases open code
|
|
or data (Noul). Only public titles and abstracts are sent. Output: ``data/literature/<date>.md`` (not committed).
|
|
|
|
python scripts/jev/literature_watch.py # last 7 days
|
|
python scripts/jev/literature_watch.py --days 14 --min-score 1.5
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import datetime as dt
|
|
import json
|
|
import re
|
|
import sys
|
|
import time
|
|
import urllib.parse
|
|
import urllib.request
|
|
import xml.etree.ElementTree as ET
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
from client import ROOT, noul, score, system_one
|
|
|
|
OUT = ROOT / "data" / "literature"
|
|
BIORXIV_CATEGORIES = {"bioinformatics", "genomics", "genetics", "pharmacology and toxicology"}
|
|
PUBMED_TERM = (
|
|
'("genotyping array" OR "SNP array" OR microarray OR BeadChip OR Infinium OR "copy number variation" OR '
|
|
'"mosaic chromosomal" OR pharmacogenomic* OR "star allele" OR CYP2D6 OR "methylation array" OR "EPIC array") '
|
|
"AND (genotyp* OR calling OR pipeline OR software OR algorithm OR benchmark)"
|
|
)
|
|
KEYWORDS = re.compile(
|
|
r"\b(genotyping arrays?|snp arrays?|microarray|beadchip|infinium|idat|genotype calling|cluster(ing)? file|"
|
|
r"copy number|cnv|mosaic|loss of heterozygosity|loh|pharmacogen\w*|star allele|cyp2d6|methylation arrays?|"
|
|
r"epic array|biobank genotyp\w*|imputation|baf|log r ratio)\b",
|
|
re.I,
|
|
)
|
|
LEVELS = [
|
|
"Unrelated to genotyping arrays, copy number, pharmacogenomics or methylation arrays",
|
|
"Related field, but not about the methods or data SimurgArray handles; for example an association study that only "
|
|
"used an array",
|
|
"Processes or evaluates array data the way SimurgArray does: genotype calling, QC, CNV or LOH calling, star allele "
|
|
"calling, cluster training or methylation QC",
|
|
"Proposes or benchmarks a method, tool, dataset or reference resource that SimurgArray could adopt or validate "
|
|
"against",
|
|
]
|
|
QUESTIONS = {
|
|
"relevance": score(
|
|
"How relevant is `paper` to SimurgArray, an open toolkit for genotyping array analysis?", LEVELS
|
|
),
|
|
"open": noul("Does `paper` release open source code or a public dataset that others can download?"),
|
|
}
|
|
|
|
|
|
@dataclass
|
|
class Paper:
|
|
source: str
|
|
title: str
|
|
abstract: str
|
|
url: str
|
|
date: str
|
|
|
|
|
|
def _get(url: str) -> bytes:
|
|
req = urllib.request.Request(url, headers={"User-Agent": "simurgarray-literature/1 ([email protected])"})
|
|
for attempt in range(4):
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=60) as r:
|
|
return r.read()
|
|
except OSError:
|
|
if attempt == 3:
|
|
raise
|
|
time.sleep(2**attempt)
|
|
raise RuntimeError("unreachable")
|
|
|
|
|
|
def biorxiv(start: dt.date, end: dt.date) -> list[Paper]:
|
|
out, cursor = [], 0
|
|
while True:
|
|
data = json.loads(_get(f"https://api.biorxiv.org/details/biorxiv/{start}/{end}/{cursor}"))
|
|
coll = data.get("collection", [])
|
|
for p in coll:
|
|
if p.get("category", "").lower() in BIORXIV_CATEGORIES:
|
|
out.append(Paper("bioRxiv", p["title"], p.get("abstract", ""), f"https://doi.org/{p['doi']}",
|
|
p.get("date", ""))) # fmt: skip
|
|
total = int(data.get("messages", [{}])[0].get("total", 0) or 0)
|
|
cursor += len(coll)
|
|
if not coll or cursor >= total:
|
|
return out
|
|
|
|
|
|
def pubmed(days: int) -> list[Paper]:
|
|
base = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils"
|
|
q = urllib.parse.urlencode({"db": "pubmed", "term": PUBMED_TERM, "reldate": days, "datetype": "edat",
|
|
"retmax": 300, "retmode": "json"}) # fmt: skip
|
|
ids = json.loads(_get(f"{base}/esearch.fcgi?{q}"))["esearchresult"]["idlist"]
|
|
out: list[Paper] = []
|
|
for i in range(0, len(ids), 100):
|
|
xml = _get(f"{base}/efetch.fcgi?db=pubmed&retmode=xml&id={','.join(ids[i : i + 100])}")
|
|
for art in ET.fromstring(xml).iter("PubmedArticle"):
|
|
pmid = art.findtext(".//PMID") or ""
|
|
title = "".join(art.find(".//ArticleTitle").itertext()) if art.find(".//ArticleTitle") is not None else ""
|
|
abstract = " ".join("".join(t.itertext()) for t in art.iter("AbstractText"))
|
|
year = art.findtext(".//PubDate/Year") or ""
|
|
out.append(Paper("PubMed", title, abstract, f"https://pubmed.ncbi.nlm.nih.gov/{pmid}/", year))
|
|
time.sleep(0.4) # NCBI asks for at most three requests a second without a key
|
|
return out
|
|
|
|
|
|
def prefilter(papers: list[Paper]) -> list[Paper]:
|
|
seen, keep = set(), []
|
|
for p in papers:
|
|
key = re.sub(r"\W+", "", p.title.lower())[:80]
|
|
if key in seen or not p.abstract:
|
|
continue
|
|
seen.add(key)
|
|
if KEYWORDS.search(p.title + " " + p.abstract):
|
|
keep.append(p)
|
|
return keep
|
|
|
|
|
|
def main(argv: list[str] | None = None) -> int:
|
|
ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
|
ap.add_argument("--days", type=int, default=7)
|
|
ap.add_argument("--min-score", type=float, default=1.8)
|
|
ap.add_argument("--dry-run", action="store_true", help="fetch and prefilter only, no API calls")
|
|
a = ap.parse_args(argv)
|
|
end = dt.date.today()
|
|
start = end - dt.timedelta(days=a.days)
|
|
papers = prefilter(biorxiv(start, end) + pubmed(a.days))
|
|
print(f"{len(papers)} candidate papers after the keyword filter")
|
|
if a.dry_run:
|
|
for p in papers[:20]:
|
|
print(f" {p.source}: {p.title[:110]}")
|
|
return 0
|
|
rows = []
|
|
for p in papers:
|
|
ans = system_one({"paper": {"title": p.title, "abstract": p.abstract[:4000]}}, QUESTIONS)
|
|
rows.append((float(ans["relevance"]["score"]), float(ans["open"]["noul"]), p))
|
|
rows.sort(key=lambda r: -r[0])
|
|
picked = [r for r in rows if r[0] >= a.min_score]
|
|
OUT.mkdir(parents=True, exist_ok=True)
|
|
path = OUT / f"{end}.md"
|
|
summary = f"{len(papers)} papers matched the keywords; {len(picked)} scored at least {a.min_score} of 3."
|
|
lines = [f"# Reading list {start} to {end}", "", summary, ""]
|
|
for s, o, p in picked:
|
|
tag = " · open code or data" if o >= 0.6 else ""
|
|
lines += [f"* **{s:.1f}** [{p.title}]({p.url}) ({p.source}, {p.date}){tag}",
|
|
f" > {p.abstract[:500]}"] # fmt: skip
|
|
path.write_text("\n".join(lines) + "\n")
|
|
print(f"{len(picked)} papers worth reading → {path}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|