#!/usr/bin/env python3
"""
Open-access full-text PDF batch retrieval.

Pipeline: arXiv (for 10.48550/arXiv.* DOIs) → Unpaywall →
          PMC (Europe PMC REST / OA FTP / web) → OpenAlex → Crossref →
          landing-page scrape.

Usage:
    python fetch_oa.py dois.txt --output pdfs/ --email user@example.com
    python fetch_oa.py worklist.tsv -o pdfs/ -e user@example.com --verbose
    python fetch_oa.py worklist.csv -o pdfs/ -e user@example.com --report pdfs/retrieval_report.json

Worklist formats: plain DOI-per-line, or TSV/CSV/Markdown-table with a DOI
column (optional PMID, Title and FirstAuthor columns). A separate source-identity
report compares first-page title and identifiers via `pdftotext` if installed.
"""

import argparse
import csv
import hashlib
import io
import json
import logging
import os
import re
import shutil
import subprocess
import time
import unicodedata
import urllib.error
import urllib.parse
import urllib.request
import xml.etree.ElementTree as ET
from pathlib import Path

MIN_PDF_BYTES = 10 * 1024
USER_AGENT = "medsci-skills/1.0"
REPORT_SCHEMA_VERSION = 2
TITLE_MATCH_THRESHOLD = 0.6
RETRIEVED_STATUSES = ("oa", "pmc", "arxiv", "skip")

log = logging.getLogger("fetch_oa")


# ============================================================
# Helpers
# ============================================================

def _ua(email: str) -> str:
    """Build a polite User-Agent string with contact email."""
    return f"{USER_AGENT} (mailto:{email})"


def safe_doi_name(doi: str) -> str:
    """Filesystem-safe filename stem for a DOI."""
    return re.sub(r"[^\w\-.]", "_", doi)


def is_valid_pdf(data: bytes) -> bool:
    return data.startswith(b"%PDF-") and len(data) >= MIN_PDF_BYTES


def fetch_bytes(url: str, email: str, accept: str = "*/*",
                timeout: int = 30) -> tuple[bytes, str, str]:
    req = urllib.request.Request(url, headers={
        "User-Agent": _ua(email),
        "Accept": accept,
    })
    with urllib.request.urlopen(req, timeout=timeout) as resp:
        return resp.read(), resp.geturl(), resp.headers.get("Content-Type", "")


def save_pdf(data: bytes, path: Path) -> bool:
    if not is_valid_pdf(data):
        return False
    path.write_bytes(data)
    return True


def existing_pdf_ok(path: Path) -> bool:
    if not path.exists():
        return False
    try:
        return is_valid_pdf(path.read_bytes())
    except OSError:
        return False


def pdf_sha256(path: Path) -> str:
    """Hash the PDF bytes without loading a potentially large file into memory."""
    digest = hashlib.sha256()
    with path.open("rb") as stream:
        for block in iter(lambda: stream.read(1024 * 1024), b""):
            digest.update(block)
    return digest.hexdigest()


# ============================================================
# Source-identity evidence (pure, offline-testable; advisory, not verification)
# ============================================================

def normalize_title(text: str) -> str:
    """Normalize Unicode, punctuation and PDF line-end hyphenation."""
    text = unicodedata.normalize("NFKD", text or "").casefold()
    text = "".join(c for c in text if not unicodedata.combining(c))
    text = text.replace("\u00ad", "")
    text = re.sub(r"(?<=\w)[-\u2010]\s*\n\s*(?=\w)", "", text)
    text = "".join(c if c.isalnum() or c.isspace() else " " for c in text)
    return " ".join(text.split())


def title_overlap(expected_title: str, extracted_text: str) -> float:
    """Fraction of meaningful expected-title tokens present in extracted text.

    Tokens of length <= 2 are dropped as near-stopwords. Returns 0.0 when the
    expected title has no usable tokens.
    """
    expected = {t for t in normalize_title(expected_title).split() if len(t) > 2}
    if not expected:
        return 0.0
    got = set(normalize_title(extracted_text).split())
    return len(expected & got) / len(expected)


def classify_title_match(expected_title: str, extracted_text,
                         threshold: float = TITLE_MATCH_THRESHOLD) -> str:
    """Tri-state title check: 'match' | 'mismatch' | 'unavailable'.

    A match requires the complete normalized title on up to six consecutive
    front-matter lines. Token overlap can only mark ambiguous text unavailable;
    it can never establish a match. Mismatch is advisory, never auto-rejection.
    """
    expected = normalize_title(expected_title)
    front = first_page_front_matter(extracted_text)
    if not expected or not front:
        return "unavailable"
    lines = [re.sub(r"^title\s*:\s*", "", ln.strip(), flags=re.I)
             for ln in front.splitlines() if ln.strip()]
    for start in range(len(lines)):
        for length in range(1, min(6, len(lines) - start) + 1):
            if normalize_title("\n".join(lines[start:start + length])) == expected:
                return "match"
    return "unavailable" if title_overlap(expected_title, front) >= threshold else "mismatch"


_BODY_START_RE = re.compile(
    r"^\s*(?:(?:\d+|[IVX]+)[.\s]+)?"
    r"(?:abstract|summary|introduction|background|references|bibliography|literature cited)"
    r"(?:\s*[:.\u2014\u2013-]|\s*$)", re.I,
)


def first_page_front_matter(extracted_text: str | None) -> str:
    """Bound evidence to page 1 before a body/reference heading, at most 40 lines.

    This is a conservative text-layout heuristic, not a PDF title-zone parser.
    Cover sheets, unusual layouts and unrecognized headings need human review.
    """
    if not extracted_text:
        return ""
    lines = []
    for line in extracted_text.split("\f", 1)[0][:4000].splitlines()[:40]:
        if _BODY_START_RE.match(line):
            break
        lines.append(line)
    return "\n".join(lines).strip()


def normalize_identifier(value: str) -> str:
    """Normalize DOI URL/prefix and equivalent arXiv DOI/id spellings."""
    value = re.sub(r"^(?:https?://(?:dx\.)?doi\.org/|doi:\s*)", "",
                   (value or "").strip(), flags=re.I).rstrip(".,;")
    aid = arxiv_id_from_doi(value)
    return (f"10.48550/arxiv.{aid}" if aid else value).casefold()


def front_matter_identifiers(front: str) -> list[str]:
    """Find DOI/arXiv evidence only in the bounded front matter, never PDF metadata."""
    values = re.findall(r"\b10\.\d{4,9}/[^\s<>\"\[\]]+", front, re.I)
    # Drop a citation's closing parenthesis, but retain balanced DOI suffixes.
    cleaned = []
    for value in values:
        value = value.rstrip(".,;")
        while value.endswith(")") and value.count(")") > value.count("("):
            value = value[:-1]
        cleaned.append(normalize_identifier(value))
    for aid in re.findall(r"\barxiv\s*:\s*((?:\d{4}\.\d{4,5}|[a-z-]+/\d{7})(?:v\d+)?)\b",
                          front, re.I):
        cleaned.append(normalize_identifier("arXiv:" + aid))
    return sorted(set(cleaned))


def identifier_matches(expected: str, observed: str) -> bool:
    """Unversioned arXiv requests accept a version; explicit versions must agree."""
    if expected == observed:
        return True
    aid = arxiv_id_from_doi(expected)
    return bool(aid and not re.search(r"v\d+$", aid, re.I)
                and re.sub(r"v\d+$", "", observed, flags=re.I) == expected)


def assess_source_identity(record: dict, extracted_text: str | None,
                           threshold: float = TITLE_MATCH_THRESHOLD) -> dict:
    """Report corroboration and uncertainty; never certify a source or delete it.

    'consistent' needs a complete title and only compatible identifiers in the
    same bounded first-page area. A supplied first-author name must also occur.
    Matching titles with a different DOI may be another version: unresolved.
    """
    front = first_page_front_matter(extracted_text)
    title_match = classify_title_match(record.get("title", ""), extracted_text, threshold)
    expected = normalize_identifier(record["doi"])
    observed = front_matter_identifiers(front)
    doi_match = ("unavailable" if not observed else
                 "match" if any(identifier_matches(expected, v) for v in observed)
                 else "mismatch")
    author = normalize_title(record.get("first_author", ""))
    author_text = normalize_title(front).replace(normalize_title(record.get("title", "")), "", 1)
    author_match = ("unavailable" if not author or not front else
                    "match" if f" {author} " in f" {author_text} " else "mismatch")
    status, reason = "unresolved", "title_not_matched"
    if not extracted_text or not extracted_text.strip():
        status, reason = "unavailable", "text_unavailable"
    elif not front:
        reason = "front_matter_unavailable"
    elif not normalize_title(record.get("title", "")):
        reason = "expected_title_missing"
    elif title_match == "match":
        if doi_match == "unavailable":
            reason = "identifier_not_found"
        elif doi_match == "mismatch":
            reason = "title_matches_other_identifier"
        elif not all(identifier_matches(expected, v) for v in observed):
            reason = "multiple_identifiers"
        elif author_match == "mismatch":
            reason = "first_author_not_found"
        else:
            status, reason = "consistent", "title_and_identifier_agree"
    elif title_match == "mismatch" and doi_match == "mismatch":
        status, reason = "conflict", "title_and_identifier_differ"
    return {"status": status, "reason": reason, "text_scope": "first_page_front_matter",
            "title_match": title_match, "doi_match": doi_match,
            "observed_identifiers": observed, "first_author_match": author_match}


def extract_pdf_text(path: Path, max_pages: int = 1) -> str | None:
    """Best-effort first-page text via `pdftotext` (poppler). None if unavailable."""
    if not shutil.which("pdftotext"):
        return None
    try:
        out = subprocess.run(
            ["pdftotext", "-f", "1", "-l", str(max_pages), str(path), "-"],
            capture_output=True, timeout=20,
        )
        if out.returncode == 0:
            return out.stdout.decode("utf-8", errors="ignore")
    except (OSError, subprocess.SubprocessError):
        pass
    return None


# ============================================================
# arXiv (direct, for 10.48550/arXiv.* DOIs)
# ============================================================

_ARXIV_DOI_RE = re.compile(r"^10\.48550/arxiv\.(.+)$", re.IGNORECASE)
_ARXIV_ID_RE = re.compile(r"^arxiv:(.+)$", re.IGNORECASE)


def arxiv_id_from_doi(doi: str) -> str | None:
    """Extract an arXiv ID from a DataCite arXiv DOI or a bare arXiv: id.

    Handles new-style (2401.01234, 2401.01234v2) and old-style
    (hep-th/9901001) identifiers; version suffix preserved when present.
    """
    s = (doi or "").strip()
    m = _ARXIV_DOI_RE.match(s) or _ARXIV_ID_RE.match(s)
    return m.group(1).strip() if m else None


def arxiv_pdf_url(doi: str) -> str | None:
    """Direct arXiv PDF URL for an arXiv DOI/ID (None if not an arXiv id)."""
    aid = arxiv_id_from_doi(doi)
    return f"https://arxiv.org/pdf/{aid}" if aid else None


# ============================================================
# 1. Unpaywall
# ============================================================

def unpaywall_lookup(doi: str, email: str) -> str | None:
    url = f"https://api.unpaywall.org/v2/{urllib.parse.quote(doi, safe='/')}" \
          f"?email={urllib.parse.quote(email)}"
    try:
        req = urllib.request.Request(url, headers={"User-Agent": _ua(email)})
        with urllib.request.urlopen(req, timeout=15) as resp:
            data = json.loads(resp.read())
        best = data.get("best_oa_location")
        if best and best.get("url_for_pdf"):
            return best["url_for_pdf"]
        for loc in data.get("oa_locations", []):
            if loc.get("url_for_pdf"):
                return loc["url_for_pdf"]
        if best and best.get("url"):
            return best["url"]
    except urllib.error.HTTPError as e:
        if e.code == 422:
            log.warning("Unpaywall rejected email '%s' (HTTP 422). "
                        "Use a real email address, not example.com.", email)
        else:
            log.debug("Unpaywall error for %s: %s", doi, e)
    except (urllib.error.URLError, json.JSONDecodeError) as e:
        log.debug("Unpaywall error for %s: %s", doi, e)
    return None


# ============================================================
# 2. PMC (3-method fallback, JS-challenge resistant)
# ============================================================

def id_to_pmcid(identifier: str, email: str) -> str | None:
    """Convert PMID or DOI to PMCID via NCBI ID converter."""
    if not identifier:
        return None
    url = (f"https://www.ncbi.nlm.nih.gov/pmc/utils/idconv/v1.0/"
           f"?ids={urllib.parse.quote(identifier, safe='/')}&format=json")
    try:
        req = urllib.request.Request(url, headers={"User-Agent": _ua(email)})
        with urllib.request.urlopen(req, timeout=15) as resp:
            data = json.loads(resp.read())
        records = data.get("records", [])
        if records and records[0].get("pmcid"):
            return records[0]["pmcid"]
    except (urllib.error.URLError, urllib.error.HTTPError, json.JSONDecodeError) as e:
        log.debug("NCBI ID converter error for %s: %s", identifier, e)
    return None


def download_pmc_pdf(pmcid: str, outpath: Path, email: str) -> bool:
    """Download PDF from PMC via Europe PMC → OA FTP → web fallback."""

    # Method A: Europe PMC REST API (most reliable, no JS)
    try:
        url = (f"https://europepmc.org/backend/ptpmcrender.fcgi"
               f"?accid={pmcid}&blobtype=pdf")
        data, _, _ = fetch_bytes(url, email, accept="application/pdf,*/*", timeout=30)
        if save_pdf(data, outpath):
            log.debug("PMC Method A (Europe PMC) succeeded for %s", pmcid)
            return True
    except (urllib.error.URLError, urllib.error.HTTPError, OSError) as e:
        log.debug("PMC Method A failed for %s: %s", pmcid, e)

    # Method B: PMC OA FTP service (XML with direct PDF link)
    try:
        url = f"https://www.ncbi.nlm.nih.gov/pmc/utils/oa/oa.fcgi?id={pmcid}"
        xml_data, _, _ = fetch_bytes(url, email, timeout=15)
        root = ET.fromstring(xml_data)
        # Check for error response (non-OA articles)
        if root.find(".//error") is not None:
            log.debug("PMC Method B: %s is not in OA subset", pmcid)
        else:
            for link in root.iter("link"):
                href = link.get("href", "")
                if href.endswith(".pdf"):
                    if href.startswith("ftp://"):
                        href = href.replace(
                            "ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/",
                            "https://ftp.ncbi.nlm.nih.gov/pub/pmc/", 1)
                    data, _, _ = fetch_bytes(
                        href, email, accept="application/pdf,*/*", timeout=30)
                    if save_pdf(data, outpath):
                        log.debug("PMC Method B (OA FTP) succeeded for %s", pmcid)
                        return True
    except (urllib.error.URLError, urllib.error.HTTPError,
            ET.ParseError, OSError) as e:
        log.debug("PMC Method B failed for %s: %s", pmcid, e)

    # Method C: Direct PMC web URL (may hit JS PoW challenge)
    try:
        url = f"https://www.ncbi.nlm.nih.gov/pmc/articles/{pmcid}/pdf/"
        data, final_url, ct = fetch_bytes(
            url, email, accept="application/pdf,*/*")
        if "pdf" in ct.lower() or final_url.endswith(".pdf"):
            if save_pdf(data, outpath):
                log.debug("PMC Method C (web) succeeded for %s", pmcid)
                return True
    except (urllib.error.URLError, urllib.error.HTTPError, OSError) as e:
        log.debug("PMC Method C failed for %s: %s", pmcid, e)

    return False


# ============================================================
# 3. OpenAlex + Crossref
# ============================================================

def openalex_lookup(doi: str, email: str) -> list[str]:
    url = (f"https://api.openalex.org/works/"
           f"https://doi.org/{urllib.parse.quote(doi, safe='/')}")
    candidates = []
    try:
        req = urllib.request.Request(url, headers={"User-Agent": _ua(email)})
        with urllib.request.urlopen(req, timeout=15) as resp:
            data = json.loads(resp.read())
        oa = data.get("open_access", {}) or {}
        primary = data.get("primary_location", {}) or {}
        for v in [primary.get("pdf_url"), oa.get("oa_url"),
                  primary.get("landing_page_url")]:
            if v and v not in candidates:
                candidates.append(v)
    except (urllib.error.URLError, urllib.error.HTTPError,
            json.JSONDecodeError) as e:
        log.debug("OpenAlex error for %s: %s", doi, e)
    return candidates


def crossref_lookup(doi: str, email: str) -> list[str]:
    url = f"https://api.crossref.org/works/{urllib.parse.quote(doi, safe='/')}"
    candidates = []
    try:
        req = urllib.request.Request(url, headers={"User-Agent": _ua(email)})
        with urllib.request.urlopen(req, timeout=15) as resp:
            data = json.loads(resp.read())
        msg = data.get("message", {}) or {}
        for link in msg.get("link", []) or []:
            v = link.get("URL")
            if v and v not in candidates:
                candidates.append(v)
        primary = ((msg.get("resource") or {}).get("primary") or {}).get("URL")
        if primary and primary not in candidates:
            candidates.append(primary)
    except (urllib.error.URLError, urllib.error.HTTPError,
            json.JSONDecodeError) as e:
        log.debug("Crossref error for %s: %s", doi, e)
    return candidates


# ============================================================
# 4. Landing page scraper
# ============================================================

def scrape_pdf_candidates(html: str) -> list[str]:
    patterns = [
        r'citation_pdf_url"\s+content="([^"]+)"',
        r"name=\"citation_pdf_url\"\s+content=\"([^\"]+)\"",
        r'href="([^"]+\.pdf[^"]*)"',
    ]
    found = []
    for pat in patterns:
        for m in re.findall(pat, html, flags=re.IGNORECASE):
            if m not in found:
                found.append(m)
    return found


def download_from_landing(url: str, outpath: Path, email: str) -> bool:
    try:
        raw, final_url, ct = fetch_bytes(url, email, accept="text/html,*/*")
        if "pdf" in ct.lower():
            return save_pdf(raw, outpath)
        html = raw.decode("utf-8", errors="ignore")
        for candidate in scrape_pdf_candidates(html):
            absolute = urllib.parse.urljoin(final_url, candidate)
            try:
                data, _, _ = fetch_bytes(
                    absolute, email, accept="application/pdf,*/*")
                if save_pdf(data, outpath):
                    return True
            except (urllib.error.URLError, urllib.error.HTTPError, OSError):
                continue
    except (urllib.error.URLError, urllib.error.HTTPError, OSError) as e:
        log.debug("Landing page error for %s: %s", url, e)
    return False


def download_pdf(url: str, outpath: Path, email: str) -> bool:
    try:
        data, _, _ = fetch_bytes(url, email, accept="application/pdf,*/*")
        return save_pdf(data, outpath)
    except (urllib.error.URLError, urllib.error.HTTPError, OSError) as e:
        log.debug("Direct download error for %s: %s", url, e)
    return False


# ============================================================
# 5. Main pipeline
# ============================================================

def process_doi(doi: str, outdir: Path, email: str,
                pmid: str = "") -> tuple[str, str]:
    """Try to download a PDF for one DOI.

    Returns (status, source):
      status ∈ {"arxiv", "oa", "pmc", "skip", "fail"}
      source identifies the resolver that succeeded (e.g. "unpaywall", "pmc",
      "openalex", "crossref", "landing", "arxiv", "existing", "").
    """
    outpath = outdir / f"{safe_doi_name(doi)}.pdf"

    if existing_pdf_ok(outpath):
        return ("skip", "existing")

    # Remove stale stub
    if outpath.exists():
        outpath.unlink(missing_ok=True)

    # Step 0: arXiv direct (for 10.48550/arXiv.* DOIs)
    ax_url = arxiv_pdf_url(doi)
    if ax_url and download_pdf(ax_url, outpath, email):
        return ("arxiv", "arxiv")

    # Step 1: Unpaywall direct PDF URL (fastest path)
    uw_url = unpaywall_lookup(doi, email)
    if uw_url and ".pdf" in uw_url.lower():
        if download_pdf(uw_url, outpath, email):
            return ("oa", "unpaywall")
        time.sleep(0.3)

    # Step 2: PMC (try before slow landing-page scraping)
    pmcid = id_to_pmcid(pmid, email) if pmid else None
    if not pmcid:
        pmcid = id_to_pmcid(doi, email)
    if pmcid and download_pmc_pdf(pmcid, outpath, email):
        return ("pmc", "pmc")

    # Step 3: OA candidates from OpenAlex, Crossref, landing pages
    candidates: list[tuple[str, str]] = []
    seen: set[str] = set()

    def add(source: str, url: str | None):
        if url and url not in seen:
            seen.add(url)
            candidates.append((source, url))

    add("unpaywall", uw_url)
    for v in openalex_lookup(doi, email):
        add("openalex", v)
    for v in crossref_lookup(doi, email):
        add("crossref", v)
    add("landing", f"https://doi.org/{doi}")

    for source, url in candidates:
        if ".pdf" in url.lower():
            ok = download_pdf(url, outpath, email)
        else:
            ok = download_from_landing(url, outpath, email)
        if ok:
            return ("oa", source)
        time.sleep(0.3)

    return ("fail", "")


def build_report(records: list[dict], results: dict[str, tuple[str, str]],
                 outdir: Path, extracted_text_by_doi: dict[str, str] | None = None,
                 threshold: float = TITLE_MATCH_THRESHOLD, *,
                 extracted_sha256_by_doi: dict[str, str] | None = None) -> dict:
    """Assemble a deterministic retrieval report (no network, no I/O writes).

    records: list of {"doi", "pmid", "title"}, optionally "first_author".
    results: doi -> (status, source) as returned by process_doi.
    outdir:  directory where PDFs were written (used for file/size lookup).
    extracted_text_by_doi: optional doi -> first-page text for title cross-check.
    extracted_sha256_by_doi: hashes captured before extraction by the CLI. When
        supplied, missing/different hashes invalidate that text's assessment.
    """
    extracted_text_by_doi = extracted_text_by_doi or {}
    items = []
    for rec in records:
        doi = rec["doi"]
        status, source = results.get(doi, ("fail", ""))
        path = outdir / f"{safe_doi_name(doi)}.pdf"
        have_file = status in RETRIEVED_STATUSES and path.exists()
        size = path.stat().st_size if have_file else 0
        digest = pdf_sha256(path) if have_file else ""
        text = extracted_text_by_doi.get(doi) if have_file else None
        changed = bool(text and extracted_sha256_by_doi is not None
                       and extracted_sha256_by_doi.get(doi) != digest)
        identity = assess_source_identity(rec, None if changed else text, threshold)
        if not have_file:
            identity["reason"] = "pdf_not_available"
        elif changed:
            identity["reason"] = "pdf_changed_during_assessment"
        items.append({
            "doi": doi,
            "pmid": rec.get("pmid", ""),
            "title": rec.get("title", ""),
            "first_author": rec.get("first_author", ""),
            "status": status,
            "source": source,
            "file": path.name if have_file else "",
            "size_bytes": size,
            "file_sha256": digest,
            "title_match": identity["title_match"],
            "source_identity": identity,
        })

    retrieved = [i for i in items if i["status"] in RETRIEVED_STATUSES]
    not_retrieved = [i for i in items if i["status"] == "fail"]
    return {
        "schema_version": REPORT_SCHEMA_VERSION,
        "generated_by": "fetch_oa.py",
        "counts": {
            "total": len(items),
            "retrieved": len(retrieved),
            "not_retrieved": len(not_retrieved),
            "title_mismatch": sum(1 for i in items if i["title_match"] == "mismatch"),
            "source_identity": {
                state: sum(i["source_identity"]["status"] == state for i in items)
                for state in ("consistent", "conflict", "unresolved", "unavailable")
            },
        },
        "items": items,
    }


def _norm_key(key: str) -> str:
    return (key or "").strip().lstrip("#").strip().lower()


def _records_from_dictrows(rows) -> list[dict]:
    records = []
    for row in rows:
        rec = {"doi": "", "pmid": "", "title": ""}
        for k, v in row.items():
            nk = _norm_key(k)
            if nk in ("firstauthor", "first_author", "first author"):
                rec["first_author"] = (v or "").strip()
            elif nk in rec:
                rec[nk] = (v or "").strip()
        if rec["doi"]:
            records.append(rec)
    return records


def _records_from_markdown(lines: list[str]) -> list[dict]:
    pipe_rows = [ln for ln in lines if ln.strip().startswith("|")]
    if not pipe_rows:
        return []

    def cells(line: str) -> list[str]:
        return [c.strip() for c in line.strip().strip("|").split("|")]

    header = [_norm_key(c) for c in cells(pipe_rows[0])]
    records = []
    for line in pipe_rows[1:]:
        c = cells(line)
        # Skip the |---|---| separator row
        if c and all(set(x) <= set("-: ") for x in c):
            continue
        row = dict(zip(header, c))
        doi = (row.get("doi") or "").strip()
        if doi:
            rec = {
                "doi": doi,
                "pmid": (row.get("pmid") or "").strip(),
                "title": (row.get("title") or "").strip(),
            }
            for key in ("firstauthor", "first_author", "first author"):
                if key in row:
                    rec["first_author"] = (row[key] or "").strip()
            records.append(rec)
    return records


def read_doi_file(path: Path) -> list[dict]:
    """Read a worklist of DOIs.

    Supports: plain DOI-per-line; TSV/CSV with a DOI header (optional PMID,
    Title and FirstAuthor columns); and a Markdown pipe table with a DOI column.
    Each record is {"doi", "pmid", "title"}, optionally "first_author".
    """
    text = Path(path).read_text(encoding="utf-8")
    lines = text.splitlines()
    first = next((ln for ln in lines
                  if ln.strip() and not ln.strip().startswith("#")), "")
    low = first.lower()

    # Markdown pipe table with a DOI column
    if first.strip().startswith("|") and "doi" in low:
        return _records_from_markdown(lines)

    # Delimited (TSV or CSV) with a DOI header
    if "doi" in low and ("\t" in first or "," in first):
        delimiter = "\t" if "\t" in first else ","
        body = "\n".join(ln for ln in lines if not ln.strip().startswith("#"))
        reader = csv.DictReader(io.StringIO(body), delimiter=delimiter)
        return _records_from_dictrows(reader)

    # Plain text: one DOI per line
    records = []
    for line in lines:
        line = line.strip()
        if line and not line.startswith("#"):
            records.append({"doi": line, "pmid": "", "title": ""})
    return records


def main():
    parser = argparse.ArgumentParser(
        description="Batch download open-access PDFs by DOI.")
    parser.add_argument("input", type=Path,
                        help="Worklist: DOIs (one per line) or TSV/CSV/Markdown "
                             "with a DOI column (optional PMID, Title, FirstAuthor)")
    parser.add_argument("-o", "--output", type=Path, default=Path("pdfs"),
                        help="Output directory (default: pdfs/)")
    parser.add_argument("-e", "--email", default=os.environ.get("MEDSCI_CONTACT_EMAIL"),
                        help="Contact email (required by Unpaywall TOS). "
                             "Falls back to the MEDSCI_CONTACT_EMAIL environment variable.")
    parser.add_argument("--report", type=Path, default=None,
                        help="Path for the JSON retrieval report "
                             "(default: <output>/retrieval_report.json)")
    parser.add_argument("-v", "--verbose", action="store_true",
                        help="Show debug messages")
    args = parser.parse_args()

    if not args.email:
        parser.error("a contact email is required (Unpaywall TOS): pass --email you@lab.org "
                     "or set MEDSCI_CONTACT_EMAIL")

    logging.basicConfig(
        level=logging.DEBUG if args.verbose else logging.WARNING,
        format="%(levelname)s: %(message)s",
    )

    args.output.mkdir(parents=True, exist_ok=True)
    report_path = args.report or (args.output / "retrieval_report.json")
    records = read_doi_file(args.input)
    print(f"Loaded {len(records)} DOIs from {args.input}")

    stats = {"arxiv": 0, "oa": 0, "pmc": 0, "fail": 0, "skip": 0}
    results: dict[str, tuple[str, str]] = {}

    for i, rec in enumerate(records, 1):
        doi = rec["doi"]
        pmid = rec.get("pmid", "")
        print(f"  [{i}/{len(records)}] {doi}", end=" … ", flush=True)

        status, source = process_doi(doi, args.output, args.email, pmid)
        results[doi] = (status, source)
        stats[status] += 1

        labels = {"arxiv": "DOWNLOADED (arXiv)", "oa": "DOWNLOADED (OA)",
                  "pmc": "DOWNLOADED (PMC)", "fail": "FAIL", "skip": "EXISTS"}
        print(labels[status])
        time.sleep(0.5)

    # Identity evidence is separate from retrieval, including DOI-only worklists.
    extracted: dict[str, str] = {}
    extracted_hashes: dict[str, str] = {}
    if shutil.which("pdftotext"):
        for rec in records:
            doi = rec["doi"]
            status, _ = results.get(doi, ("fail", ""))
            if status not in RETRIEVED_STATUSES:
                continue
            path = args.output / f"{safe_doi_name(doi)}.pdf"
            if path.exists():
                extracted_hashes[doi] = pdf_sha256(path)
                text = extract_pdf_text(path)
                if text:
                    extracted[doi] = text

    report = build_report(records, results, args.output, extracted,
                          extracted_sha256_by_doi=extracted_hashes)
    report_path.parent.mkdir(parents=True, exist_ok=True)
    report_path.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")

    print(f"\n--- Summary ---")
    print(f"  arXiv:   {stats['arxiv']}")
    print(f"  OA:      {stats['oa']}")
    print(f"  PMC:     {stats['pmc']}")
    print(f"  Failed:  {stats['fail']}")
    print(f"  Skipped: {stats['skip']}")
    total = stats["arxiv"] + stats["oa"] + stats["pmc"] + stats["fail"]
    if total > 0:
        pct = (stats["arxiv"] + stats["oa"] + stats["pmc"]) / total * 100
        print(f"  Download success: {pct:.0f}% (excludes existing files; not identity verification)")
    mismatches = report["counts"]["title_mismatch"]
    if mismatches:
        print(f"  Title mismatches flagged: {mismatches} (see report)")
    identity_counts = report["counts"]["source_identity"]
    print("  Source identity (advisory): " + ", ".join(
        f"{state}={count}" for state, count in identity_counts.items()))
    print("  Review source_identity and file_sha256 before using a PDF as evidence.")
    print(f"  Report:  {report_path}")

    # Write failed DOIs for manual retrieval
    if stats["fail"] > 0:
        fail_path = args.output / "manual_needed.txt"
        with open(fail_path, "w") as f:
            f.write("# DOIs needing manual retrieval\n")
            f.write("# Options: institutional access, ILL\n\n")
            for rec in records:
                doi = rec["doi"]
                pdf = args.output / f"{safe_doi_name(doi)}.pdf"
                if not existing_pdf_ok(pdf):
                    f.write(f"{doi}\n")
        print(f"  Manual list: {fail_path}")


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""
Convert research paper PDFs to LLM-friendly Markdown.

Uses pymupdf4llm for high-quality extraction optimized for academic papers:
two-column layout handling, table preservation, header/footer removal.

Requires: pip install pymupdf4llm

Usage:
    python pdf_to_md.py pdfs/                   # convert all PDFs in directory
    python pdf_to_md.py paper.pdf               # convert single file
    python pdf_to_md.py pdfs/ -o markdown/       # custom output directory
    python pdf_to_md.py pdfs/ --pages 0-9        # first 10 pages only
    python pdf_to_md.py pdfs/ --force            # overwrite existing .md files
"""

import argparse
import re
import sys
from pathlib import Path

try:
    import pymupdf4llm
except ImportError:
    print("Error: pymupdf4llm is not installed.", file=sys.stderr)
    print("Install with: pip install pymupdf4llm", file=sys.stderr)
    sys.exit(1)


def parse_page_range(spec: str) -> list[int]:
    """Parse page range string like '0-9' or '0,2,5-7' into list of ints."""
    pages = []
    for part in spec.split(","):
        part = part.strip()
        if "-" in part:
            start, end = part.split("-", 1)
            pages.extend(range(int(start), int(end) + 1))
        else:
            pages.append(int(part))
    return pages


def clean_markdown(text: str) -> str:
    """Post-process pymupdf4llm output for cleaner LLM consumption."""
    # Collapse excessive blank lines (3+ → 2)
    text = re.sub(r"\n{4,}", "\n\n\n", text)
    # Strip trailing whitespace per line
    text = "\n".join(line.rstrip() for line in text.splitlines())
    return text.strip() + "\n"


def convert_pdf(pdf_path: Path, out_dir: Path, *,
                pages: list[int] | None = None,
                force: bool = False,
                verbose: bool = False) -> bool:
    """Convert a single PDF to Markdown. Returns True on success."""
    md_path = out_dir / pdf_path.with_suffix(".md").name

    if md_path.exists() and md_path.stat().st_size > 0 and not force:
        if verbose:
            print(f"  SKIP: {md_path.name} (exists, use --force to overwrite)")
        return True

    try:
        kwargs = {
            "show_progress": False,
            # Academic paper defaults: skip images (saves tokens),
            # strict table detection for grid-line tables
            "write_images": False,
            "ignore_images": True,
            "table_strategy": "lines_strict",
        }
        if pages is not None:
            kwargs["pages"] = pages

        # Suppress pymupdf's C-level OCR/parser messages (stdout + stderr)
        if not verbose:
            import os as _os
            _devnull = _os.open(_os.devnull, _os.O_WRONLY)
            _old_stdout = _os.dup(1)
            _old_stderr = _os.dup(2)
            _os.dup2(_devnull, 1)
            _os.dup2(_devnull, 2)
        try:
            md_text = pymupdf4llm.to_markdown(str(pdf_path), **kwargs)
        finally:
            if not verbose:
                _os.dup2(_old_stdout, 1)
                _os.dup2(_old_stderr, 2)
                _os.close(_devnull)
                _os.close(_old_stdout)
                _os.close(_old_stderr)
        md_text = clean_markdown(md_text)

        md_path.write_text(md_text, encoding="utf-8")
        if verbose:
            kb = len(md_text.encode("utf-8")) / 1024
            print(f"  OK: {md_path.name} ({kb:.1f} KB)")
        return True
    except Exception as e:
        print(f"  FAIL: {pdf_path.name}: {e}", file=sys.stderr)
        return False


def main():
    parser = argparse.ArgumentParser(
        description="Convert research PDFs to LLM-friendly Markdown "
                    "(via pymupdf4llm).")
    parser.add_argument("input", type=Path,
                        help="PDF file or directory containing PDFs")
    parser.add_argument("-o", "--output", type=Path, default=None,
                        help="Output directory (default: same as input)")
    parser.add_argument("--pages", type=str, default=None,
                        help="Page range, e.g. '0-9' for first 10 pages")
    parser.add_argument("--force", action="store_true",
                        help="Overwrite existing .md files")
    parser.add_argument("-v", "--verbose", action="store_true",
                        help="Show per-file progress")
    args = parser.parse_args()

    # Resolve input
    if args.input.is_file():
        pdfs = [args.input]
        default_out = args.input.parent
    elif args.input.is_dir():
        pdfs = sorted(args.input.glob("*.pdf"))
        default_out = args.input
    else:
        print(f"Error: {args.input} not found", file=sys.stderr)
        sys.exit(1)

    out_dir = args.output or default_out
    out_dir.mkdir(parents=True, exist_ok=True)

    if not pdfs:
        print("No PDF files found.")
        return

    # Parse pages
    pages = parse_page_range(args.pages) if args.pages else None

    print(f"Converting {len(pdfs)} PDF(s) → Markdown", flush=True)
    ok = 0
    fail = 0
    for pdf in pdfs:
        if convert_pdf(pdf, out_dir, pages=pages, force=args.force,
                       verbose=args.verbose):
            ok += 1
        else:
            fail += 1

    print(f"\n--- Summary ---")
    print(f"  Converted: {ok}")
    print(f"  Failed:    {fail}")
    print(f"  Total:     {len(pdfs)}")


if __name__ == "__main__":
    main()

---
name: fulltext-retrieval
description: Batch download open-access PDFs by DOI using legitimate OA APIs (Unpaywall, PMC, OpenAlex, Crossref). Optional PDF→Markdown conversion for token-efficient LLM analysis.
triggers: PDF download, fulltext retrieval, open access PDF, batch download papers, meta-analysis PDF, PDF to markdown, convert PDF
tools: Read, Write, Edit, Bash, Grep, Glob
model: inherit
---

# Fulltext Retrieval Skill

Batch download open-access full-text PDFs from a DOI list using legitimate OA APIs only.

## Pipeline

```
DOI → arXiv (10.48550/arXiv.* DOIs) → Unpaywall → PMC (Europe PMC / OA FTP / web) → OpenAlex → Crossref → landing page
```

Each DOI goes through these sources in order until a valid PDF (≥10 KB, `%PDF-` header) is found. arXiv DOIs (`10.48550/arXiv.2401.01234`, version suffixes, old-style `hep-th/9901001`, or a bare `arXiv:` id) resolve directly to the arXiv PDF first.

## Quick Start

```bash
# Prepare a DOI list (one per line)
cat > dois.txt << 'EOF'
10.1007/s00330-010-1783-x
10.1002/mp.12524
10.1148/radiol.13131265
EOF

# Run
python fetch_oa.py dois.txt --output pdfs/ --email your@email.com

# Verbose mode for debugging
python fetch_oa.py dois.txt -o pdfs/ -e your@email.com --verbose
```

## Input Formats

**Plain text** — one DOI per line:
```
10.1007/s00330-010-1783-x
10.1002/mp.12524
```

**TSV / CSV with header** — must contain a `DOI` column; optional `PMID`, `Title`, and
`FirstAuthor` columns (first author's surname or full name for corroboration):
```tsv
ID	Title	DOI	PMID	Year
1	Some paper	10.1007/s00330-010-1783-x	20628747	2010
```

**Markdown table** — a pipe table with a `DOI` column also works:
```markdown
| DOI | PMID | Title |
|-----|------|-------|
| 10.1007/s00330-010-1783-x | 20628747 | Some paper |
```

When a PMID is available, the PMC lookup is more reliable (PMID → PMCID conversion).
Supply `Title` where available: a DOI-only worklist can download a PDF but cannot
establish title agreement. `FirstAuthor` is optional additional evidence.

## PMC Download (JS-Challenge Resistant)

PMC web pages may block automated downloads with JavaScript proof-of-work challenges. This tool uses three fallback methods:

### Method A: Europe PMC REST API (most reliable)

```bash
PMCID="PMC9733600"
curl -sLo output.pdf \
  "https://europepmc.org/backend/ptpmcrender.fcgi?accid=${PMCID}&blobtype=pdf"
```

### Method B: PMC OA FTP Service

```bash
curl -s "https://www.ncbi.nlm.nih.gov/pmc/utils/oa/oa.fcgi?id=${PMCID}" | \
    grep -oE 'href="[^"]*\.pdf"' | head -1 | \
    sed 's/href="//;s/"//' | xargs curl -sLo output.pdf
```

### DOI/PMID → PMCID Conversion

```bash
# Works with both DOI and PMID
curl -s "https://www.ncbi.nlm.nih.gov/pmc/utils/idconv/v1.0/?ids=${DOI}&format=json" | \
    python3 -c "import sys,json; print(json.load(sys.stdin)['records'][0].get('pmcid',''))"
```

## Output

- PDFs saved as `{DOI_safe}.pdf` (slashes replaced with underscores)
- `pdfs/retrieval_report.json` — structured per-DOI report (see below)
- `manual_needed.txt` — DOIs that could not be retrieved via OA
- Summary with arXiv/OA/PMC/fail/skip counts

## Retrieval report (`--report`)

Every run writes a structured report (default `<output>/retrieval_report.json`,
override with `--report PATH`):

```json
{
  "schema_version": 2,
  "generated_by": "fetch_oa.py",
  "counts": {"total": 4, "retrieved": 3, "not_retrieved": 1, "title_mismatch": 1,
             "source_identity": {"consistent": 1, "conflict": 1, "unresolved": 1, "unavailable": 1}},
  "items": [
    {"doi": "10.1000/synthetic.example", "pmid": "", "title": "Example title",
     "first_author": "", "status": "oa", "source": "unpaywall",
     "file": "10.1000_synthetic.example.pdf", "size_bytes": 482113,
     "file_sha256": "<SHA-256 of the downloaded file>", "title_match": "match",
     "source_identity": {"status": "consistent", "reason": "title_and_identifier_agree",
                         "text_scope": "first_page_front_matter", "title_match": "match",
                         "doi_match": "match", "observed_identifiers": ["10.1000/synthetic.example"],
                         "first_author_match": "unavailable"}}
  ]
}
```

The example abbreviates `items`. Legacy `status` (`arxiv | oa | pmc | skip | fail`),
`source`, and `counts.retrieved` retain their resolver-result meaning, including existing
files (`skip`). **They do not count identity-verified papers.** Report schema 2 adds the
file hash and separate identity evidence; no PDF is automatically deleted or rejected.

| `source_identity.status` | Meaning / action |
|---|---|
| `consistent` | Complete normalized title and a compatible DOI/arXiv identifier occur in the bounded first-page front matter; an optional supplied author must also match. Evidence agrees, but this is not independent source verification or claim validation. |
| `conflict` | Both the title and observed identifier differ. Inspect the PDF and requested record. |
| `unresolved` | Evidence is incomplete or ambiguous: title-only, DOI-only, missing author, multiple identifiers, or a matching title with another DOI/version. Inspect before using as evidence. |
| `unavailable` | No usable extracted text, Poppler unavailable, no output PDF, or the PDF changed during assessment. No current identity assessment was possible. |

`title_match` keeps its tri-state shape. A `match` now requires the complete normalized
title on up to six consecutive front-matter lines. Case, punctuation and line wrapping
are normalized. Scattered matching words cannot establish a match; partial overlap is
`unavailable`, and low overlap is an advisory `mismatch`.

Evidence is limited to the first page, before a recognized abstract/body/reference
heading, at most 40 lines / 4,000 characters. Thus a title cited in the body or references
does not establish a title match. These are conservative layout heuristics: cover sheets,
unrecognized headings, short or changed titles, unusual reading order and DOI footers
outside that area can remain unresolved. PDF metadata and the filename alone are not
identity evidence. The CLI compares hashes before extraction and when reporting;
changed files cannot inherit the previous text's assessment. Explicit arXiv versions
must agree; preprint/published-version DOI
differences require review rather than automatic rejection.

Downstream reports must preserve `source_identity` and `file_sha256`, keep unresolved
items visible, and check the hash still identifies the file being used. Older reports
without identity evidence remain **unassessed**; do not infer identity from `retrieved`
or `title_match=match`. Full-text conversion does not resolve an identity warning.

## Attach PDFs into Zotero ("Find Available PDF")

OA-only resolvers miss paywalled-but-licensed papers. To attach full text **inside
Zotero** at a much higher yield, use `references/find_available_pdf.js` — a user-run
snippet for Zotero's *Tools → Developer → Run JavaScript*. It triggers Zotero's own
`addAvailablePDF` / `addAvailablePDFs` and therefore reuses **your** OpenURL resolver /
institutional proxy config; **no credentials, proxy hosts, or institutional identifiers
are hard-coded or leave your Zotero client**. The no-code equivalent is right-click →
"Find Available PDF".

This path is **user-initiated** and depends on your live Zotero session, so its results
are recorded manually (not reproducible CI evidence). `/lit-sync` Phase 2.7 orchestrates
both routes (disk OA via this script + in-library via the snippet) and reconciles them in
a report.

## Requirements

- Python 3.10+ (stdlib only, no pip dependencies)
- Contact email (required by Unpaywall Terms of Service)

## API Policies

| Source | Rate Limit | Notes |
|--------|-----------|-------|
| Unpaywall | 100 req/sec | Email required |
| NCBI PMC | 3 req/sec without API key | Add `&api_key=` for higher limits |
| OpenAlex | 100k req/day | Polite pool with email in User-Agent |
| Crossref | 50 req/sec with email | Plus service with `mailto:` in UA |
| Europe PMC | No documented limit | Be polite, ≤1 req/sec recommended |

The script uses 0.3–0.5 second delays between requests.

## PDF → Markdown Conversion (Optional)

After downloading PDFs, convert them to LLM-friendly Markdown for token-efficient repeated analysis. Uses [pymupdf4llm](https://github.com/pymupdf/RAG) — optimized for academic papers with two-column layout handling and table preservation.

### Quick Start

```bash
# Install (one-time)
pip install pymupdf4llm

# Convert all PDFs in a directory
python pdf_to_md.py pdfs/

# Convert with verbose output
python pdf_to_md.py pdfs/ -v

# Custom output directory
python pdf_to_md.py pdfs/ -o markdown/

# First 10 pages only (useful for long supplements)
python pdf_to_md.py pdfs/ --pages 0-9

# Overwrite existing conversions
python pdf_to_md.py pdfs/ --force
```

### Combined Workflow

```bash
# Step 1: Download PDFs
python fetch_oa.py dois.txt -o pdfs/ -e your@email.com

# Step 2: Convert to Markdown (only successful downloads)
python pdf_to_md.py pdfs/ -v
```

After conversion, `.md` files sit alongside `.pdf` files. Claude Code can then use `Read` for full content or `Grep` for targeted extraction — significantly more token-efficient than re-reading PDFs.

### When to Convert

| Scenario | Recommendation |
|----------|---------------|
| Screening/triage (read once) | Skip — read PDF directly |
| Data extraction from k≥5 studies | Convert — repeated reads save tokens |
| Meta-analysis full pipeline | Convert — papers referenced across multiple phases |
| Single paper deep review | Optional — marginal benefit |

### Academic Paper Defaults

- **Images**: Skipped (saves tokens; figures referenced by caption text)
- **Tables**: `lines_strict` strategy (preserves grid-line tables accurately)
- **Layout**: Two-column academic layout handled automatically
- **Headers/footers**: Removed by pymupdf4llm

### Dependency Note

`pdf_to_md.py` requires [pymupdf4llm](https://pypi.org/project/pymupdf4llm/) (AGPL-3.0). This is an **optional** dependency — `fetch_oa.py` remains stdlib-only with zero external dependencies. The AGPL license applies to pymupdf4llm itself, not to this skill.

## Limitations

- Only retrieves **open-access** articles. Paywalled articles require institutional access.
- Landing page scraping may fail on publisher-specific JavaScript-heavy pages.
- Some recent articles may not yet be indexed by OA sources.
- PDF→Markdown quality depends on the PDF's text layer. Scanned-only PDFs may produce poor output.

## Anti-Hallucination

- **Never fabricate file paths, URLs, DOIs, or package names.** Verify existence before recommending.
- **Never invent journal metadata, impact factors, or submission policies** without verification at the journal's website.
- If a tool, package, or resource does not exist or you are unsure, say so explicitly rather than guessing.

schema_version: 2
name: fulltext-retrieval
layer: A
owner_domain: literature_discovery
maturity: official

when_to_use: "Batch-download open-access full-text PDFs from a DOI list; optionally convert them to Markdown for token-efficient analysis."
when_NOT_to_use: "Finding or verifying citations (use search-lit / verify-refs). Retrieving paywalled or non-open-access content."

inputs:
  - path: "DOI worklist (.txt one-per-line, or .tsv/.csv/.md with a DOI column; optional PMID, Title, FirstAuthor)"
    schema: csv
    required: true
outputs:
  - path: "downloaded open-access PDFs (pdfs/)"
  - path: "pdfs/retrieval_report.json (schema 2: retrieval status, source_identity evidence, file_sha256)"
  - path: "optional PDF-to-Markdown conversions"
  - path: "references/find_available_pdf.js (user-run Zotero 'Find Available PDF' batch snippet)"

deterministic_scripts:
  - fetch_oa.py
  - pdf_to_md.py
side_effects:
  - downloads_files
  - network_access_oa_apis
downstream_consumers:
  - meta-analysis
  - obsidian-paper-vault
forbidden_actions:
  - download_paywalled_content
  - bypass_publisher_access_controls

# v2.1 quality card
purpose: "Resolve a DOI list to open-access full-text PDFs via legitimate OA APIs, with optional Markdown conversion."
safety_boundaries:
  - "Uses legitimate open-access sources only (Unpaywall, PMC / Europe PMC, OpenAlex, Crossref); never circumvents paywalls or access controls."
  - "Validates each download (>=10 KB and a %PDF- header) before accepting it."
known_limitations:
  - "Only open-access content is retrievable; non-OA DOIs fail by design rather than fetching from unauthorized sources."
  - "Higher-yield in-library retrieval (find_available_pdf.js) is user-initiated inside Zotero and uses the user's own proxy/OpenURL config; it is not reproducible CI evidence."
  - "Retrieval success is not source verification. Source identity uses bounded first-page title/DOI evidence plus optional FirstAuthor; ambiguous or unavailable evidence stays visible, and no PDF is auto-rejected."
  - "Identity is advisory: unusual layouts, cover sheets, changed titles, and preprint/publication DOI differences need review. Downstream consumers must preserve source_identity and file_sha256; old reports are unassessed."
  - "PDF-to-Markdown conversion requires the optional pymupdf4llm dependency (AGPL-3.0 or commercial license)."
validation_commands:
  - "bash fetch_oa_report_challenge/verify.sh   # offline report projection (CI-wired)"
  - "python3 tests/test_source_identity.py   # synthetic identity cases + real PDF CLI round trips with Poppler (CI-wired)"
  - "python fetch_oa.py dois.txt -o pdfs/ -e <email> --report pdfs/retrieval_report.json --verbose   # per-DOI source trace"
  - "verify each output begins with %PDF- and is at least 10 KB"
evidence_surface: bundled_script