# Embase search via browser automation

Embase has no public API. Search and export it through Chrome browser automation (MCP):

1. Navigate to `embase.com` — institutional SSO authenticates automatically.
   If cookie error (`login?error#`), clear Elsevier/Embase cookies and retry.
2. Go to the **Advanced Search** tab.
3. Enter an Embase-syntax query (Emtree `/exp` + `:ab,ti` field tags).
   Uncheck "Map to preferred term in Emtree" when using explicit `/exp` terms.
4. After results appear, use the "Select number of items" dropdown → select the total count.
5. Click **Export** (in the Results section) → choose **CSV** format → check fields:
   Title, Author names, Source, Publication year, Publication type, DOI, Abstract,
   Language of article, Medline PMID.
6. Click Export → the Download tab opens → click Download.
7. The CSV is in **row format** (records separated by blank rows). Parse it as:
   ```python
   # Each record = consecutive rows until blank row
   # Row format: [FIELD_NAME, value1, value2, ...]
   # AUTHOR NAMES row has multiple values (one per author)
   ```

## PubMed → Embase query translation

- MeSH `[Mesh]` → Emtree `/exp`
- `[tiab]` → `:ab,ti`
- `[Title/Abstract]` → `:ab,ti`
- Boolean operators stay the same (AND, OR)
- Phrase search: use single quotes in Embase (`'artificial ascites'`)

#!/usr/bin/env python3
"""
Parse PubMed E-utilities responses into structured data.

Usage:
    # Parse esearch JSON → list of PMIDs
    echo '<json>' | python3 parse_pubmed.py esearch

    # Parse esummary JSON → markdown table
    echo '<json>' | python3 parse_pubmed.py esummary

    # Parse efetch XML → detailed metadata (for BibTeX generation)
    echo '<xml>' | python3 parse_pubmed.py efetch

    # Parse efetch XML → BibTeX entries
    echo '<xml>' | python3 parse_pubmed.py bibtex
"""

import sys
import json
import re
import xml.etree.ElementTree as ET
from datetime import date
from textwrap import shorten


# Heuristic for East Asian name reverse-encoding in PubMed XML.
# Cases observed: <LastName>Qiaoling</LastName><ForeName>Fu</ForeName> where
# Fu is the actual family name. Pattern: LastName looks like a long given
# name (≥3 alpha chars, no spaces) AND ForeName looks like a short surname
# fragment (1-2 chars, no period). The naive test catches the common reverse
# encoding without flagging legitimate short-surname authors.
_EAST_ASIAN_REVERSE_THRESHOLD = 3  # LastName length lower bound for suspicion


def _full_text(parent, path: str, default: str = "") -> str:
    """All text inside the element at `path`, including text inside inline children.

    PubMed marks up titles and abstracts inline (`<i>Helicobacter pylori</i>`, `CO<sub>2</sub>`).
    `findtext()` / `.text` return only the text before the first child element, which cut
    "Eradication of <i>Helicobacter pylori</i> infection." down to "Eradication of " while the
    BibTeX entry was still stamped verified. Absent element -> `default`.
    """
    el = parent.find(path) if parent is not None else None
    if el is None:
        return default
    return "".join(el.itertext())


def _looks_east_asian_reversed(last: str, fore: str) -> bool:
    """Return True if (LastName, ForeName) look swapped per PubMed encoding bug."""
    if not last or not fore:
        return False
    # ForeName should look like a surname (1-2 chars, no spaces, no period)
    # AND LastName should look like a multi-char given name.
    return (
        1 <= len(fore) <= 2
        and fore.isalpha()
        and "." not in fore
        and len(last) >= _EAST_ASIAN_REVERSE_THRESHOLD
        and last.isalpha()
    )


def _extract_authors(author_list_el):
    """Walk an <AuthorList> element. Return (bib_authors, display_authors,
    first_author_last, suspicions, has_collective_only).

    - bib_authors: list of "Family, Given" strings for BibTeX `author = {...}`.
      Corporate (<CollectiveName>) authors are double-braced.
    - display_authors: list of "Last First" strings for human-readable output.
    - first_author_last: surname of the first listed author (used for cite key).
    - suspicions: list of human-readable warning strings (East Asian reverse
      encoding, missing LastName, etc.).
    - has_collective_only: True if AuthorList contains only <CollectiveName>
      entries (no individual <LastName>). Caller should consider emitting
      `@misc` instead of `@article` (guideline / consortium pattern).
    """
    bib_authors: list[str] = []
    display_authors: list[str] = []
    first_author_last = ""
    suspicions: list[str] = []
    individual_count = 0
    collective_count = 0

    if author_list_el is None:
        return bib_authors, display_authors, "", suspicions, False

    for au in author_list_el.findall("Author"):
        last = au.findtext("LastName", "") or ""
        fore = au.findtext("ForeName", "") or ""
        collective = au.findtext("CollectiveName", "") or ""

        if collective:
            collective_count += 1
            # Double-brace to prevent BibTeX from splitting on the comma /
            # spaces inside the corporate name.
            bib_authors.append("{" + collective + "}")
            display_authors.append(collective)
            if not first_author_last:
                first_author_last = re.sub(r"[^A-Za-z]+", "", collective.split()[0]) or "Group"
            continue

        if last:
            individual_count += 1
            if _looks_east_asian_reversed(last, fore):
                suspicions.append(
                    f"East Asian name order suspected for '{last} {fore}' — "
                    "PubMed XML may have LastName/ForeName swapped"
                )
            bib_authors.append(f"{last}, {fore}")
            display_authors.append(f"{last} {fore}".strip())
            if not first_author_last:
                first_author_last = last
            continue

        # Author element with neither <LastName> nor <CollectiveName>: rare
        # but possible. Record as suspicion, otherwise skip.
        suspicions.append("Author element with no LastName and no CollectiveName")

    has_collective_only = collective_count > 0 and individual_count == 0
    return bib_authors, display_authors, first_author_last, suspicions, has_collective_only


def parse_esearch(data: str) -> None:
    """Parse esearch JSON response, print PMIDs and count."""
    # A body without `esearchresult.count`, or with an `ERROR` field, is not a search that
    # found nothing: it is a search that did not run. Defaulting the count to "0" printed
    # "Total results: 0" with exit 0 for an error body, which a caller reads as "no papers"
    # (ma-scout's "MA = 0" verdict). Name the input and exit 2 instead.
    result = json.loads(data)
    esearch = result.get("esearchresult") if isinstance(result, dict) else None
    problem = None
    if not isinstance(esearch, dict):
        problem = "no 'esearchresult' object"
        if isinstance(result, dict) and result.get("error"):
            problem += f" (error: {result.get('error')})"
    elif esearch.get("ERROR"):
        problem = f"esearchresult.ERROR: {esearch.get('ERROR')}"
    elif "count" not in esearch:
        problem = "esearchresult has no 'count'"
    if problem is not None:
        print(
            f"ERROR: esearch response is not a search result ({problem}); "
            "the count is unknown, not 0.",
            file=sys.stderr,
        )
        sys.exit(2)
    count = esearch["count"]
    ids = esearch.get("idlist", [])
    print(f"Total results: {count}")
    print(f"Returned: {len(ids)}")
    print(f"PMIDs: {','.join(ids)}")


def parse_esummary(data: str) -> None:
    """Parse esummary JSON response into a markdown table."""
    result = json.loads(data)
    docs = result.get("result", {})
    uids = docs.get("uids", [])

    if not uids:
        print("No results found.")
        return

    print("| # | PMID | Year | Journal | Title | Authors |")
    print("|---|------|------|---------|-------|---------|")

    for i, uid in enumerate(uids, 1):
        doc = docs.get(uid, {})
        title = shorten(doc.get("title", "N/A"), width=80, placeholder="...")
        authors_raw = doc.get("authors", [])
        if authors_raw:
            first = authors_raw[0].get("name", "")
            last = authors_raw[-1].get("name", "") if len(authors_raw) > 1 else ""
            authors = f"{first}, ... {last}" if last and last != first else first
        else:
            authors = "N/A"
        journal = shorten(doc.get("fulljournalname", doc.get("source", "N/A")),
                          width=40, placeholder="...")
        pubdate = doc.get("pubdate", "N/A")
        year = pubdate[:4] if pubdate else "N/A"
        doi_list = doc.get("articleids", [])
        doi = next((d["value"] for d in doi_list if d.get("idtype") == "doi"), "")

        print(f"| {i} | {uid} | {year} | {journal} | {title} | {authors} |")

    print(f"\n*{len(uids)} articles retrieved*")


def parse_efetch(data: str) -> None:
    """Parse efetch XML response into structured metadata."""
    root = ET.fromstring(data)
    articles = root.findall(".//PubmedArticle")

    for article in articles:
        medline = article.find("MedlineCitation")
        if medline is None:
            continue

        pmid = medline.findtext("PMID", "N/A")
        art = medline.find("Article")
        if art is None:
            continue

        title = _full_text(art, "ArticleTitle", "N/A")
        journal_el = art.find("Journal")
        journal = journal_el.findtext("Title", "N/A") if journal_el is not None else "N/A"
        journal_abbrev = journal_el.findtext("ISOAbbreviation", "") if journal_el is not None else ""

        # Year
        ji = journal_el.find("JournalIssue") if journal_el is not None else None
        pd = ji.find("PubDate") if ji is not None else None
        year = pd.findtext("Year", "") if pd is not None else ""
        if not year:
            medline_date = pd.findtext("MedlineDate", "") if pd is not None else ""
            year = medline_date[:4] if medline_date else "N/A"

        volume = ji.findtext("Volume", "") if ji is not None else ""
        issue = ji.findtext("Issue", "") if ji is not None else ""

        # Pages
        pages = art.findtext("Pagination/MedlinePgn", "")

        # Authors (handles East Asian reverse encoding + CollectiveName)
        author_list = art.find("AuthorList")
        _, authors, _, suspicions, _ = _extract_authors(author_list)

        # DOI
        doi = ""
        for aid in art.findall("ELocationID"):
            if aid.get("EIdType") == "doi":
                doi = aid.text or ""

        # Abstract
        abstract_el = art.find("Abstract")
        abstract = ""
        if abstract_el is not None:
            parts = abstract_el.findall("AbstractText")
            abstract = " ".join(
                (p.get("Label", "") + ": " if p.get("Label") else "") + "".join(p.itertext())
                for p in parts
            )

        print(f"## PMID: {pmid}")
        print(f"**Title**: {title}")
        print(f"**Authors**: {'; '.join(authors)}")
        print(f"**Journal**: {journal} ({journal_abbrev})")
        print(f"**Year**: {year}  **Volume**: {volume}  **Issue**: {issue}  **Pages**: {pages}")
        print(f"**DOI**: {doi}")
        if abstract:
            print(f"**Abstract**: {shorten(abstract, width=500, placeholder='...')}")
        for note in suspicions:
            print(f"> ⚠ {note}")
        print()


def generate_bibtex(data: str) -> None:
    """Parse efetch XML and generate BibTeX entries."""
    root = ET.fromstring(data)
    articles = root.findall(".//PubmedArticle")

    for article in articles:
        medline = article.find("MedlineCitation")
        if medline is None:
            continue

        pmid = medline.findtext("PMID", "")
        art = medline.find("Article")
        if art is None:
            continue

        title = _full_text(art, "ArticleTitle", "")
        journal_el = art.find("Journal")
        journal_abbrev = journal_el.findtext("ISOAbbreviation", "") if journal_el is not None else ""
        journal_full = journal_el.findtext("Title", "") if journal_el is not None else ""

        ji = journal_el.find("JournalIssue") if journal_el is not None else None
        pd = ji.find("PubDate") if ji is not None else None
        year = pd.findtext("Year", "") if pd is not None else ""
        if not year:
            md = pd.findtext("MedlineDate", "") if pd is not None else ""
            year = md[:4] if md else ""

        volume = ji.findtext("Volume", "") if ji is not None else ""
        issue = ji.findtext("Issue", "") if ji is not None else ""
        pages = art.findtext("Pagination/MedlinePgn", "")

        doi = ""
        for aid in art.findall("ELocationID"):
            if aid.get("EIdType") == "doi":
                doi = aid.text or ""

        author_list = art.find("AuthorList")
        bib_authors, _, first_author_last, suspicions, has_collective_only = \
            _extract_authors(author_list)

        # Generate citation key
        key = f"{first_author_last}_{year}_{pmid}" if first_author_last else f"PMID_{pmid}"

        # Corporate / consortium guideline (e.g., KDIGO, AHA/ACC) — emit as
        # @misc so BibTeX styles render the body of the entry without trying
        # to format a personal author. Vancouver / AMA CSL handle both
        # @article and @misc with author = {{Organization Name}}.
        entry_type = "misc" if has_collective_only else "article"

        # Prepend suspicion comments so they survive .bib copy/paste audits.
        for note in suspicions:
            print(f"% [VERIFY] {note}")

        print(f"@{entry_type}{{{key},")
        print(f"  author    = {{{' and '.join(bib_authors)}}},")
        print(f"  title     = {{{title}}},")
        print(f"  journal   = {{{journal_full}}},")
        print(f"  year      = {{{year}}},")
        if volume:
            print(f"  volume    = {{{volume}}},")
        if issue:
            print(f"  number    = {{{issue}}},")
        if pages:
            print(f"  pages     = {{{pages}}},")
        if doi:
            print(f"  doi       = {{{doi}}},")
        print(f"  pmid      = {{{pmid}}},")

        # Anti-hallucination verification flag. Entries emitted by this script
        # originate from PubMed efetch XML, so a non-empty PMID is proof of
        # API provenance (verified=true). Missing PMID → verified=false and
        # downstream tooling (/verify-refs) will flag for manual check.
        verified = bool(pmid)
        verified_by = "pubmed+crossref" if (pmid and doi) else ("pubmed" if pmid else "")
        print(f"  verified  = {{{'true' if verified else 'false'}}},")
        if verified_by:
            print(f"  verified_by = {{{verified_by}}},")
            print(f"  verified_on = {{{date.today().isoformat()}}},")
        print("}")
        print()


if __name__ == "__main__":
    if len(sys.argv) < 2:
        print(__doc__)
        sys.exit(1)

    mode = sys.argv[1]
    data = sys.stdin.read()

    dispatch = {
        "esearch": parse_esearch,
        "esummary": parse_esummary,
        "efetch": parse_efetch,
        "bibtex": generate_bibtex,
    }

    func = dispatch.get(mode)
    if func is None:
        print(f"Unknown mode: {mode}. Use: {', '.join(dispatch.keys())}")
        sys.exit(1)

    func(data)

#!/bin/bash
# PubMed E-utilities CLI wrapper
# Fallback when PubMed MCP server is unavailable
# Usage: bash pubmed_eutils.sh <command> <args...>
#
# Commands:
#   search <query> [retmax]       -- Search PubMed, return PMIDs
#   fetch <pmid1,pmid2,...>       -- Fetch article metadata (XML)
#   fetch_json <pmid1,pmid2,...>  -- Fetch article summary (JSON, DocSum)
#   related <pmid> [retmax]       -- Find related articles
#   cite_lookup <title>           -- Search by exact title to verify citation
#
# Environment:
#   NCBI_API_KEY  -- Optional. Increases rate limit from 3/sec to 10/sec.
#                    Register at https://www.ncbi.nlm.nih.gov/account/settings/
#
# Rate limiting: 3 requests/second without API key, 10/sec with key.
# This script sleeps 350ms between calls for safety.

set -euo pipefail

BASE="https://eutils.ncbi.nlm.nih.gov/entrez/eutils"
TOOL="claude-code-search-lit"
EMAIL="noreply@example.com"
DB="pubmed"
SLEEP=0.35

# Build API key param if available
API_KEY_PARAM=""
if [ -n "${NCBI_API_KEY:-}" ]; then
  API_KEY_PARAM="&api_key=${NCBI_API_KEY}"
  SLEEP=0.1
fi

_sleep() { sleep "$SLEEP"; }

# Values reach Python as argv, never as source code. A query is free text ("Crohn's disease",
# "O'Brien[Author]"): pasted into a Python string literal, the first apostrophe was a SyntaxError
# that left the search term silently empty, and a crafted query ran its own Python.
_urlencode() {
  python3 -c 'import sys, urllib.parse; print(urllib.parse.quote(sys.argv[1]))' "$1"
}

_count() {  # _count <value> <name>: retmax must be a whole number or the command stops
  case "$1" in
    '' | *[!0-9]*) echo "{\"error\": \"$2 must be a whole number\"}" >&2; return 2 ;;
  esac
}

_curl() {
  local http_code body
  body=$(curl -sS -w '\n%{http_code}' -A "Mozilla/5.0 (${TOOL})" "$@")
  http_code=$(echo "$body" | tail -n1)
  body=$(echo "$body" | sed '$d')
  if [ "$http_code" -ge 400 ] 2>/dev/null; then
    echo "{\"error\": \"HTTP ${http_code}\", \"url\": \"$1\"}" >&2
    return 1
  fi
  echo "$body"
}

cmd_search() {
  local query="${1:?Usage: search <query> [retmax]}"
  local retmax="${2:-20}" term
  _count "$retmax" retmax
  # A separate assignment, so an encoding failure stops the script instead of searching term=.
  term="$(_urlencode "$query")"
  local url="${BASE}/esearch.fcgi?db=${DB}&term=${term}&retmax=${retmax}&retmode=json&tool=${TOOL}&email=${EMAIL}${API_KEY_PARAM}"
  _curl "$url"
}

cmd_fetch() {
  local ids="${1:?Usage: fetch <pmid1,pmid2,...>}"
  local url="${BASE}/efetch.fcgi?db=${DB}&id=${ids}&rettype=xml&retmode=xml&tool=${TOOL}&email=${EMAIL}${API_KEY_PARAM}"
  _curl "$url"
}

cmd_fetch_json() {
  local ids="${1:?Usage: fetch_json <pmid1,pmid2,...>}"
  local url="${BASE}/esummary.fcgi?db=${DB}&id=${ids}&retmode=json&tool=${TOOL}&email=${EMAIL}${API_KEY_PARAM}"
  _curl "$url"
}

cmd_related() {
  local pmid="${1:?Usage: related <pmid> [retmax]}"
  local retmax="${2:-10}"
  _count "$retmax" retmax
  local url="${BASE}/elink.fcgi?dbfrom=${DB}&db=${DB}&id=${pmid}&cmd=neighbor_score&retmode=json&tool=${TOOL}&email=${EMAIL}${API_KEY_PARAM}"
  local result
  result=$(_curl "$url")
  # Extract linked PMIDs and fetch their summaries
  local linked_ids
  linked_ids=$(echo "$result" | python3 -c "
import sys, json
retmax = int(sys.argv[1])
data = json.load(sys.stdin)
links = data.get('linksets', [{}])[0].get('linksetdbs', [{}])
for db in links:
    if db.get('linkname') == 'pubmed_pubmed':
        ids = [str(l['id']) for l in db.get('links', [])[:retmax]]
        print(','.join(ids))
        break
" "$retmax" 2>/dev/null || echo "")
  if [ -n "$linked_ids" ]; then
    _sleep
    cmd_fetch_json "$linked_ids"
  else
    echo '{"error": "No related articles found"}'
  fi
}

cmd_cite_lookup() {
  local title="${1:?Usage: cite_lookup <title>}"
  # Search by title field for exact verification
  cmd_search "${title}[Title]" 5
}

# Dispatch
case "${1:-help}" in
  search)      shift; cmd_search "$@" ;;
  fetch)       shift; cmd_fetch "$@" ;;
  fetch_json)  shift; cmd_fetch_json "$@" ;;
  related)     shift; cmd_related "$@" ;;
  cite_lookup) shift; cmd_cite_lookup "$@" ;;
  help|*)
    echo "Usage: bash pubmed_eutils.sh <command> <args...>"
    echo "Commands: search, fetch, fetch_json, related, cite_lookup"
    ;;
esac

#!/usr/bin/env python3
"""
Citation snowballing for search-lit (Phase 2.5 Citation Searching).

Expands a seed set of papers along the citation graph and emits API-verified
BibTeX candidates, deduplicated against an existing candidate pool.

Directions
----------
    backward : references the seed papers cite (cited-by-the-seed)
    forward  : papers that cite the seed (citing-the-seed)
    similar  : Semantic Scholar recommendations for the seed
    all      : backward + forward + similar (default)

Data source: Semantic Scholar Graph API (deterministic, no model memory).
    references     GET /graph/v1/paper/{id}/references
    citations      GET /graph/v1/paper/{id}/citations
    recommendations GET /recommendations/v1/papers/forpaper/{id}

Anti-hallucination contract
---------------------------
Snowball candidates carry `verified=false` + `verified_by=semantic_scholar`.
They are NOT cross-checked against PubMed/CrossRef here; downstream
`/verify-refs` confirms each entry and upgrades the flag. Nothing is ever
generated from memory.

Output contract (matches search-lit BibTeX section)
---------------------------------------------------
- BibTeX is APPENDED to the candidate pool (default references/library.bib).
- NEVER writes to manuscript/_src/refs.bib (that is /lit-sync's sole path).
- A PRISMA "records identified through citation searching" line is printed.
- Exit 0 only when every seed/direction answered in full. A failed fetch (network error, an
  error body) or a page the source says continues (`next`, i.e. more than --limit records) is
  printed as FAILED / TRUNCATED, the PRISMA line ends with INCOMPLETE, and the exit is 1.

Usage
-----
    # Live (network) — expand one DOI in all directions, dedup against pool
    python3 snowball.py --seed DOI:10.1000/synthetic.example \
        --pool references/library.bib --out references/library.bib

    # Multiple seeds from a file (one id per line), backward only
    python3 snowball.py --seed @seeds.txt --direction backward

    # Deterministic / offline (challenge-card verifier): read recorded JSON
    python3 snowball.py --seed DOI:10.0/seed1 --direction all \
        --offline-fixture fixture --as-of 2026-06-14 --stdout

Seed id formats accepted: `DOI:10.x/...`, `PMID:123456`, bare `10.x/...`
(treated as DOI), bare digits (treated as PMID), or a raw S2 paper id.
"""

from __future__ import annotations

import argparse
import json
import re
import sys
import urllib.parse
import urllib.request
from datetime import date
from pathlib import Path

S2_GRAPH = "https://api.semanticscholar.org/graph/v1/paper"
S2_REC = "https://api.semanticscholar.org/recommendations/v1/papers/forpaper"
S2_FIELDS = "title,year,venue,externalIds,authors.name"

DIRECTIONS = ("backward", "forward", "similar")


# --------------------------------------------------------------------------- #
# Seed id normalization
# --------------------------------------------------------------------------- #
def normalize_seed_id(raw: str) -> str:
    """Return an S2-acceptable paper id token (DOI:/PMID:/raw)."""
    s = raw.strip()
    if not s:
        return ""
    up = s.upper()
    if up.startswith("DOI:") or up.startswith("PMID:") or up.startswith("ARXIV:"):
        return s
    if s.startswith("10.") or "doi.org/" in s.lower():
        return "DOI:" + s.split("doi.org/")[-1]
    if s.isdigit():
        return "PMID:" + s
    return s  # assume raw S2 id


def fixture_slug(seed_id: str) -> str:
    """Filesystem-safe slug for an offline fixture filename."""
    return re.sub(r"[^A-Za-z0-9]+", "_", seed_id).strip("_")


# --------------------------------------------------------------------------- #
# Fetch (network or offline fixture)
# --------------------------------------------------------------------------- #
def _http_get_json(url: str) -> dict:
    req = urllib.request.Request(url, headers={"User-Agent": "medsci-skills/snowball"})
    with urllib.request.urlopen(req, timeout=30) as resp:  # noqa: S310 (trusted host)
        return json.loads(resp.read().decode("utf-8"))


def fetch_direction(seed_id: str, direction: str, limit: int,
                    fixture_dir: Path | None) -> tuple[list[dict], str | None]:
    """Return (papers, problem) for one seed+direction.

    Each paper dict has at least: title, year, venue, externalIds, authors.
    `problem` is None when the source answered in full, otherwise a one-line reason starting with
    `FAILED` (the fetch did not run, or its answer was not a result) or `TRUNCATED` (the source
    says more records exist beyond `--limit`). Either way the count for this seed+direction is a
    lower bound, not the number that exists, and the caller must say so.
    """
    if fixture_dir is not None:
        fpath = fixture_dir / f"{fixture_slug(seed_id)}.{direction}.json"
        if not fpath.exists():
            return [], None
        try:
            payload = json.loads(fpath.read_text())
        except (OSError, ValueError) as exc:
            return [], f"FAILED {direction} for {seed_id}: unreadable fixture {fpath.name} ({exc})"
    else:
        enc = urllib.parse.quote(seed_id, safe="")
        if direction == "backward":
            url = f"{S2_GRAPH}/{enc}/references?fields={S2_FIELDS}&limit={limit}"
        elif direction == "forward":
            url = f"{S2_GRAPH}/{enc}/citations?fields={S2_FIELDS}&limit={limit}"
        elif direction == "similar":
            url = f"{S2_REC}/{enc}?fields={S2_FIELDS}&limit={limit}"
        else:
            raise ValueError(f"unknown direction: {direction}")
        try:
            payload = _http_get_json(url)
        except Exception as exc:  # noqa: BLE001 — reported to the caller, never read as 0
            return [], f"FAILED {direction} for {seed_id}: {exc}"

    # Normalize payload shapes:
    #   references/citations: {"data": [{"citedPaper"|"citingPaper": {...}}], "next": N?}
    #   recommendations:      {"recommendedPapers": [{...}]} or {"data": [{...}]}
    # A body with neither list (an `error` payload, say) is not "no papers": the search did not run.
    if not isinstance(payload, dict) or not (
            isinstance(payload.get("data"), list)
            or isinstance(payload.get("recommendedPapers"), list)):
        detail = payload.get("error") or payload.get("message") if isinstance(payload, dict) else None
        return [], (f"FAILED {direction} for {seed_id}: response has no 'data' or "
                    f"'recommendedPapers' list" + (f" ({detail})" if detail else ""))
    rows = payload.get("data") or payload.get("recommendedPapers") or []
    papers = []
    for row in rows:
        paper = row.get("citedPaper") or row.get("citingPaper") or row
        if isinstance(paper, dict) and paper.get("title"):
            papers.append(paper)
    # The paginated endpoints carry `next` only when more records exist past this page.
    if payload.get("next") is not None:
        return papers, (f"TRUNCATED {direction} for {seed_id}: the source has more records "
                        f"after the first {len(rows)} (next offset {payload.get('next')}); "
                        "raise --limit")
    return papers, None


# --------------------------------------------------------------------------- #
# Dedup
# --------------------------------------------------------------------------- #
def norm_doi(doi: str | None) -> str:
    if not doi:
        return ""
    return doi.strip().lower().replace("https://doi.org/", "")


def norm_title(title: str | None) -> str:
    if not title:
        return ""
    return re.sub(r"[^a-z0-9]+", "", title.lower())


def parse_pool_keys(pool_path: Path | None) -> tuple[set[str], set[str]]:
    """Extract existing DOIs and normalized titles from a BibTeX pool."""
    dois: set[str] = set()
    titles: set[str] = set()
    if pool_path is None or not pool_path.exists():
        return dois, titles
    text = pool_path.read_text(errors="ignore")
    for m in re.finditer(r"doi\s*=\s*[{\"]([^}\"]+)[}\"]", text, re.I):
        dois.add(norm_doi(m.group(1)))
    for m in re.finditer(r"title\s*=\s*[{\"](.+?)[}\"]\s*,?\s*\n", text, re.I | re.S):
        titles.add(norm_title(m.group(1)))
    return dois, titles


# --------------------------------------------------------------------------- #
# BibTeX emit
# --------------------------------------------------------------------------- #
def _author_last(name: str) -> str:
    parts = name.strip().split()
    return parts[-1] if parts else "Anon"


def bibtex_key(paper: dict) -> str:
    authors = paper.get("authors") or []
    last = _author_last(authors[0]["name"]) if authors and authors[0].get("name") else "Anon"
    last = re.sub(r"[^A-Za-z]", "", last) or "Anon"
    year = str(paper.get("year") or "ND")
    title_word = ""
    for w in re.findall(r"[A-Za-z]{4,}", paper.get("title") or ""):
        if w.lower() not in {"with", "from", "using", "study", "analysis", "based"}:
            title_word = w.capitalize()
            break
    return f"{last}_{year}_{title_word or 'Snowball'}"


def to_bibtex(paper: dict, direction: str, as_of: str, key: str) -> str:
    ext = paper.get("externalIds") or {}
    doi = ext.get("DOI", "")
    pmid = ext.get("PubMed", "")
    authors = paper.get("authors") or []
    author_str = " and ".join(
        f"{_author_last(a['name'])}, {' '.join(a['name'].split()[:-1])}".strip().rstrip(",")
        for a in authors if a.get("name")
    ) or "Unknown"
    lines = [
        f"@article{{{key},",
        f"  author    = {{{author_str}}},",
        f"  title     = {{{paper.get('title', '').strip()}}},",
        f"  journal   = {{{paper.get('venue', '') or ''}}},",
        f"  year      = {{{paper.get('year') or ''}}},",
    ]
    if doi:
        lines.append(f"  doi       = {{{doi}}},")
    if pmid:
        lines.append(f"  pmid      = {{{pmid}}},")
    lines += [
        "  verified  = {false},",
        "  verified_by = {semantic_scholar},",
        f"  verified_on = {{{as_of}}},",
        "  source    = {citation_snowball},",
        f"  snowball_direction = {{{direction}}},",
        "}",
    ]
    return "\n".join(lines)


# --------------------------------------------------------------------------- #
# Main
# --------------------------------------------------------------------------- #
def load_seeds(raw: str) -> list[str]:
    if raw.startswith("@"):
        lines = Path(raw[1:]).read_text().splitlines()
        items = [ln for ln in lines if ln.strip() and not ln.strip().startswith("#")]
    else:
        items = re.split(r"[,\s]+", raw)
    return [normalize_seed_id(x) for x in items if x.strip()]


def main(argv: list[str] | None = None) -> int:
    ap = argparse.ArgumentParser(description="Citation snowballing for search-lit.")
    ap.add_argument("--seed", required=True,
                    help="comma/space-separated ids, or @file with one id per line")
    ap.add_argument("--direction", default="all",
                    choices=("all", *DIRECTIONS))
    ap.add_argument("--pool", default=None,
                    help="existing library.bib to dedup against")
    ap.add_argument("--out", default="references/library.bib",
                    help="BibTeX append target (NEVER manuscript/_src/refs.bib)")
    ap.add_argument("--limit", type=int, default=50, help="max per seed per direction")
    ap.add_argument("--offline-fixture", default=None,
                    help="dir of recorded JSON (<slug>.<direction>.json) for deterministic runs")
    ap.add_argument("--as-of", default=date.today().isoformat(),
                    help="verified_on date stamp (default today; set for reproducible output)")
    ap.add_argument("--stdout", action="store_true",
                    help="print BibTeX to stdout instead of appending to --out")
    args = ap.parse_args(argv)

    out_path = Path(args.out)
    if out_path.name == "refs.bib" and "_src" in str(out_path):
        ap.error("refusing to write manuscript/_src/refs.bib — that is /lit-sync's sole path")

    fixture_dir = Path(args.offline_fixture) if args.offline_fixture else None
    pool_dois, pool_titles = parse_pool_keys(Path(args.pool) if args.pool else None)

    seeds = load_seeds(args.seed)
    directions = list(DIRECTIONS) if args.direction == "all" else [args.direction]

    seen_doi = set(pool_dois)
    seen_title = set(pool_titles)
    seen_key: set[str] = set()
    counts = {d: 0 for d in directions}
    entries: list[str] = []
    raw_found = 0
    problems: list[str] = []

    for seed in seeds:
        for direction in directions:
            papers, problem = fetch_direction(seed, direction, args.limit, fixture_dir)
            if problem:
                problems.append(problem)
            for paper in papers:
                raw_found += 1
                ext = paper.get("externalIds") or {}
                d = norm_doi(ext.get("DOI"))
                t = norm_title(paper.get("title"))
                if (d and d in seen_doi) or (t and t in seen_title):
                    continue
                if d:
                    seen_doi.add(d)
                if t:
                    seen_title.add(t)
                key = bibtex_key(paper)
                base_key, n = key, 1
                while key in seen_key:
                    n += 1
                    key = f"{base_key}{chr(96 + n)}"  # _b, _c, ...
                seen_key.add(key)
                entries.append(to_bibtex(paper, direction, args.as_of, key))
                counts[direction] += 1

    new_total = len(entries)
    bibtex_blob = ("\n\n".join(entries) + "\n") if entries else ""

    if args.stdout or not entries:
        sys.stdout.write(bibtex_blob)
    else:
        out_path.parent.mkdir(parents=True, exist_ok=True)
        with out_path.open("a", encoding="utf-8") as fh:
            if out_path.exists() and out_path.stat().st_size > 0:
                fh.write("\n")
            fh.write(bibtex_blob)

    # PRISMA citation-searching line (stderr so --stdout BibTeX stays clean)
    breakdown = ", ".join(f"{d}={counts[d]}" for d in directions)
    pool_n = len(pool_dois) + len(pool_titles)
    for problem in problems:
        sys.stderr.write(f"[snowball] {problem}\n")
    incomplete = (
        f" INCOMPLETE: {len(problems)} seed/direction fetch(es) failed or were truncated; "
        "these counts are a lower bound and must not be recorded in the PRISMA flow."
        if problems else ""
    )
    sys.stderr.write(
        f"Records identified through citation searching (snowballing): "
        f"{raw_found} raw ({breakdown}); after dedup against existing pool: "
        f"{new_total} new candidates.{incomplete}\n"
    )
    sys.stderr.write(
        f"[snowball] seeds={len(seeds)} directions={'+'.join(directions)} "
        f"pool_keys={pool_n} -> {new_total} appended"
        f"{' (stdout)' if args.stdout else f' to {out_path}'}\n"
    )
    return 1 if problems else 0


if __name__ == "__main__":
    raise SystemExit(main())

#!/usr/bin/env python3
"""Does this row's DOI point at this row's paper?

A screening table's `doi` column is not always something a database handed over with the record. It
is often something a pipeline *worked out* — matched by title similarity against Crossref, at a
score somebody chose. When that match is wrong, the DOI is a valid, resolvable identifier for a
different paper, and nothing downstream can tell. Every check that follows treats it as ground
truth: deduplication, full-text retrieval, the reference list, the eligibility decision.

Two failures inside two days, from one such column:

  * a conference-abstract record carried the DOI of a **different study already in the cohort**,
    which produced a false limitation — a "missed eligible paper" that did not exist. It reached
    the supporting material of a submitted abstract.
  * another record's DOI resolved to an **entire supplement of abstracts** rather than to an
    article, sending a reader to look for a paper that was never there.

Neither is detectable by looking at the DOI. Both are obvious the moment you resolve it and read
the title back.

    DOI_NOT_THIS_RECORD  the DOI resolves, and to a different paper than the row describes
    DOI_IS_CONTAINER     the DOI resolves to an issue, a supplement, a book or proceedings — a
                         container, not an article
    DOI_IS_UPDATE_NOTICE the DOI resolves to a correction, erratum, retraction or similar notice
                         that updates another DOI (Crossref `update-to`), not to the article. Its
                         title is usually the article's own title behind a short prefix, so title
                         similarity alone clears it.
    DOI_UNRESOLVED       the DOI does not resolve at all (reported, never silently dropped)

WHAT THIS IS NOT
    It is not `/verify-refs`, which audits a manuscript's finished reference list against PubMed
    and Crossref. This runs earlier, on the screening table, where a wrong DOI is still cheap — and
    where the reference list does not exist yet.

    It also cannot tell you a DOI is *right*. A title that matches is consistent with the row; it
    is not proof the record was correctly extracted. It rules out one specific, recurring, and
    otherwise invisible way of being wrong.

DETERMINISM
    `--cache DIR` reads recorded Crossref responses (`<url-safe-doi>.json`) instead of the network,
    which is how the challenge card runs offline. With `--cache` and no recorded response, a row is
    UNRESOLVED — the cache is never quietly topped up from the network mid-run, because a check
    that is half-recorded and half-live is reproducible in neither direction.

Stdlib only.

Usage:
    check_doi_record_match.py --table screening.tsv [--title-col title] [--doi-col doi]
                             [--id-col record_id] [--email you@example.org]
                             [--threshold 0.90] [--cache DIR] [--json out.json] [--strict]

Exit: 0 clean (or findings without --strict), 1 findings with --strict, 2 unusable input.
"""

from __future__ import annotations

import argparse
import csv
import json
import re
import sys
import time
import unicodedata
import urllib.error
import urllib.parse
import urllib.request
from dataclasses import dataclass, field
from difflib import SequenceMatcher
from pathlib import Path
from typing import Dict, List, Optional

DETECTOR = "check_doi_record_match"

CROSSREF = "https://api.crossref.org/works/"

# Crossref `type` values that are an article-shaped thing a screening row can legitimately BE.
# Anything else is a container: resolving to it means the row points at the box, not the paper.
ARTICLE_TYPES = {
    "journal-article", "proceedings-article", "posted-content", "book-chapter",
    "report", "dissertation", "peer-review", "reference-entry", "other",
}

DOI_RE = re.compile(r"10\.\d{4,9}/[-._;()/:A-Za-z0-9]+")


@dataclass
class Finding:
    detector: str
    verdict: str
    record: Optional[str]
    summary: str
    evidence: List[str] = field(default_factory=list)


def normalise(s: str) -> str:
    """Compare what a title says, not how a database punctuated it."""
    s = unicodedata.normalize("NFKD", s or "").lower()
    s = re.sub(r"<[^>]+>", " ", s)          # Crossref titles carry JATS markup
    s = re.sub(r"[^a-z0-9\s]", " ", s)
    return " ".join(s.split())


def similarity(a: str, b: str) -> float:
    na, nb = normalise(a), normalise(b)
    if not na or not nb:
        return 0.0
    return SequenceMatcher(None, na, nb).ratio()


def clean_doi(raw: str) -> str:
    """Accept a DOI however a spreadsheet wrote it: bare, prefixed, or as a URL."""
    m = DOI_RE.search((raw or "").strip())
    return m.group(0).rstrip(".,;)") if m else ""


def cache_path(cache: Path, doi: str) -> Path:
    return cache / (urllib.parse.quote(doi, safe="") + ".json")


def resolve(doi: str, email: Optional[str], cache: Optional[Path],
            pause: float) -> Optional[dict]:
    """-> the Crossref `message` object, or None if it did not resolve."""
    if cache is not None:
        p = cache_path(cache, doi)
        if not p.is_file():
            return None
        try:
            return json.loads(p.read_text(encoding="utf-8")).get("message")
        except (json.JSONDecodeError, OSError):
            return None

    url = CROSSREF + urllib.parse.quote(doi, safe="/")
    ua = "check_doi_record_match/1.0"
    if email:
        ua += f" (mailto:{email})"
    req = urllib.request.Request(url, headers={"User-Agent": ua, "Accept": "application/json"})
    try:
        with urllib.request.urlopen(req, timeout=30) as r:  # noqa: S310 - fixed https host
            payload = json.loads(r.read().decode("utf-8"))
    except (urllib.error.URLError, json.JSONDecodeError, TimeoutError, OSError):
        return None
    finally:
        time.sleep(pause)
    return payload.get("message")


def resolved_title(msg: dict) -> str:
    t = msg.get("title") or []
    return t[0] if t else ""


def update_notice_targets(msg: dict) -> List[str]:
    """-> "type -> DOI" for each entry of Crossref's structured `update-to` list.

    A record that carries `update-to` is a notice about another DOI. A new-version update (a
    revised record of the same work) is still the work itself, so it is not counted here.
    """
    out: List[str] = []
    for u in msg.get("update-to") or []:
        if not isinstance(u, dict):
            continue
        kind = str(u.get("type") or "").lower()
        if "version" in kind:
            continue
        out.append(f"{kind or 'update'} -> {u.get('DOI') or '?'}")
    return out


def audit(rows: List[Dict[str, str]], id_col: str, title_col: str, doi_col: str,
          threshold: float, email: Optional[str], cache: Optional[Path],
          pause: float) -> List[Finding]:
    out: List[Finding] = []
    for i, row in enumerate(rows, start=2):  # 1 is the header; report the line a person can open
        rid = (row.get(id_col) or f"row {i}").strip() or f"row {i}"
        own_title = (row.get(title_col) or "").strip()
        doi = clean_doi(row.get(doi_col) or "")
        if not doi:
            continue
        if not own_title:
            out.append(Finding(
                DETECTOR, "DOI_UNRESOLVED", rid,
                f"{rid}: carries a DOI but no title, so the DOI cannot be checked against "
                "anything.",
                [f"doi: {doi}", "A DOI with nothing to compare it to is an assertion, not a fact."],
            ))
            continue

        msg = resolve(doi, email, cache, pause)
        if msg is None:
            out.append(Finding(
                DETECTOR, "DOI_UNRESOLVED", rid,
                f"{rid}: the DOI did not resolve.",
                [f"doi: {doi}", f"row title: {own_title[:80]!r}",
                 "Reported rather than dropped: an unresolvable DOI in a screening table is a "
                 "record nobody can retrieve, which is a finding of its own."],
            ))
            continue

        got = resolved_title(msg)
        ratio = similarity(own_title, got)
        kind = (msg.get("type") or "").lower()

        if kind and kind not in ARTICLE_TYPES:
            out.append(Finding(
                DETECTOR, "DOI_IS_CONTAINER", rid,
                f"{rid}: the DOI resolves to a {kind}, not to an article.",
                [f"doi: {doi}", f"resolved to: {got[:80]!r}", f"row title: {own_title[:80]!r}",
                 "Following this DOI leads to the volume the paper is in, or to a whole "
                 "supplement of abstracts. Anyone sent there looks for a paper that is not "
                 "separately registered."],
            ))
            continue

        updates = update_notice_targets(msg)
        if updates and normalise(own_title) != normalise(got):
            out.append(Finding(
                DETECTOR, "DOI_IS_UPDATE_NOTICE", rid,
                f"{rid}: the DOI resolves to a notice that updates another DOI "
                f"({'; '.join(updates)}), not to the article (title similarity {ratio:.2f}).",
                [f"doi: {doi}",
                 f"row title:      {own_title[:100]!r}",
                 f"resolved title: {got[:100]!r}",
                 "A correction or retraction notice repeats the article's title, so a "
                 "title-similarity match lands on it. The article's own DOI is the one the "
                 "notice points to."],
            ))
            continue

        if ratio < threshold:
            out.append(Finding(
                DETECTOR, "DOI_NOT_THIS_RECORD", rid,
                f"{rid}: the DOI resolves to a different paper (title similarity "
                f"{ratio:.2f} < {threshold:.2f}).",
                [f"doi: {doi}",
                 f"row title:      {own_title[:100]!r}",
                 f"resolved title: {got[:100]!r}",
                 "If this column was filled by matching titles against Crossref, this is what that "
                 "match got wrong — and downstream it is indistinguishable from a real record."],
            ))
    return out


def read_table(path: Path) -> List[Dict[str, str]]:
    text = path.read_text(encoding="utf-8-sig")
    delim = "\t" if path.suffix.lower() in {".tsv", ".tab"} or "\t" in text.split("\n")[0] else ","
    return list(csv.DictReader(text.splitlines(), delimiter=delim))


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__.split("\n")[0])
    ap.add_argument("--table", type=Path, required=True, help="screening table (.tsv or .csv)")
    ap.add_argument("--id-col", default="record_id")
    ap.add_argument("--title-col", default="title")
    ap.add_argument("--doi-col", default="doi")
    ap.add_argument("--threshold", type=float, default=0.90,
                    help="title similarity below which the DOI is another paper (default 0.90). "
                         "Pipelines that FILL a doi column commonly accept ~0.79, which is how "
                         "the wrong DOIs got in; do not set this to the value that produced them.")
    ap.add_argument("--email", help="contact address for the Crossref polite pool")
    ap.add_argument("--cache", type=Path,
                    help="directory of recorded Crossref responses; disables all network access")
    ap.add_argument("--pause", type=float, default=0.2, help="seconds between live requests")
    ap.add_argument("--json", type=Path)
    ap.add_argument("--strict", action="store_true")
    a = ap.parse_args()

    if not a.table.is_file():
        print(f"cannot read {a.table}", file=sys.stderr)
        return 2
    if a.cache is not None and not a.cache.is_dir():
        print(f"cache directory {a.cache} does not exist", file=sys.stderr)
        return 2

    rows = read_table(a.table)
    if not rows:
        print(f"{a.table} has no data rows", file=sys.stderr)
        return 2
    missing = [c for c in (a.title_col, a.doi_col) if c not in rows[0]]
    if missing:
        print(f"{a.table} has no column(s) named {', '.join(missing)} "
              f"(found: {', '.join(rows[0])})", file=sys.stderr)
        return 2

    findings = audit(rows, a.id_col, a.title_col, a.doi_col, a.threshold,
                     a.email, a.cache, a.pause)

    checked = sum(1 for r in rows if clean_doi(r.get(a.doi_col) or ""))
    if a.json:
        a.json.parent.mkdir(parents=True, exist_ok=True)
        a.json.write_text(json.dumps(
            {"detector": DETECTOR, "table": str(a.table), "rows": len(rows),
             "rows_with_doi": checked, "threshold": a.threshold,
             "source": "cache" if a.cache else "crossref",
             "findings": [f.__dict__ for f in findings]},
            indent=2, ensure_ascii=False) + "\n", encoding="utf-8")

    if not findings:
        print(f"OK: {checked} DOI(s) each resolve to the record that carries them.")
        return 0

    print(f"{len(findings)} finding(s) across {checked} DOI(s)\n")
    for f in findings:
        print(f"  [{f.verdict}] ({f.record})")
        print(f"      {f.summary}")
        for e in f.evidence:
            print(f"      - {e}")
        print()
    return 1 if a.strict else 0


if __name__ == "__main__":
    sys.exit(main())

#!/usr/bin/env bash
# Regression test: parse_pubmed.py keeps the whole title and abstract when PubMed marks them up.
#
# PubMed efetch XML carries inline markup inside <ArticleTitle> and <AbstractText>: italic species
# names, sub/superscripts. `findtext()` and `.text` return only the text before the first child
# element, so "Eradication of <i>Helicobacter pylori</i> infection." became
# `title = {Eradication of },` in an entry stamped `verified = {true}`, and an abstract reading
# "CO<sub>2</sub> levels rose." became "CO". The pre-fix parser fails checks 1-4; checks 5-6 are
# the negative control (a title with no markup is unchanged).
#
# No network: the XML is synthetic and inline.
set -u
HERE="$(cd "$(dirname "$0")" && pwd)"
PARSER="$HERE/../references/parse_pubmed.py"
TMP="$(mktemp -d)"
trap 'rm -rf "$TMP"' EXIT

cat >"$TMP/markup.xml" <<'XML'
<?xml version="1.0"?>
<PubmedArticleSet>
  <PubmedArticle>
    <MedlineCitation>
      <PMID>99999991</PMID>
      <Article>
        <Journal>
          <Title>Journal of Synthetic Medicine</Title>
          <ISOAbbreviation>J Synth Med</ISOAbbreviation>
          <JournalIssue><Volume>1</Volume><Issue>2</Issue><PubDate><Year>2024</Year></PubDate></JournalIssue>
        </Journal>
        <ArticleTitle>Eradication of <i>Helicobacter pylori</i> infection.</ArticleTitle>
        <Abstract>
          <AbstractText Label="BACKGROUND">CO<sub>2</sub> levels rose.</AbstractText>
        </Abstract>
        <AuthorList><Author><LastName>Researcher</LastName><ForeName>Ada</ForeName></Author></AuthorList>
        <ELocationID EIdType="doi">10.0/synthetic.markup</ELocationID>
      </Article>
    </MedlineCitation>
  </PubmedArticle>
</PubmedArticleSet>
XML

cat >"$TMP/plain.xml" <<'XML'
<?xml version="1.0"?>
<PubmedArticleSet>
  <PubmedArticle>
    <MedlineCitation>
      <PMID>99999992</PMID>
      <Article>
        <Journal>
          <Title>Journal of Synthetic Medicine</Title>
          <JournalIssue><PubDate><Year>2023</Year></PubDate></JournalIssue>
        </Journal>
        <ArticleTitle>Plain title without markup.</ArticleTitle>
        <Abstract><AbstractText>Plain abstract.</AbstractText></Abstract>
        <AuthorList><Author><LastName>Writer</LastName><ForeName>Dan</ForeName></Author></AuthorList>
      </Article>
    </MedlineCitation>
  </PubmedArticle>
</PubmedArticleSet>
XML

fail=0
ck() { if [ "$2" = "$3" ]; then printf '  PASS  %s\n' "$1"; else printf '  FAIL  %s (want %s got %s)\n' "$1" "$2" "$3"; fail=$((fail+1)); fi; }
has() { grep -qF -- "$2" "$1" && echo yes || echo no; }

python3 "$PARSER" bibtex <"$TMP/markup.xml" >"$TMP/markup.bib"
python3 "$PARSER" efetch <"$TMP/markup.xml" >"$TMP/markup.md"
python3 "$PARSER" bibtex <"$TMP/plain.xml" >"$TMP/plain.bib"
python3 "$PARSER" efetch <"$TMP/plain.xml" >"$TMP/plain.md"

# 1-2. a title with inline <i> keeps the text inside and after it
ck "bibtex title keeps the italic species name" yes \
  "$(has "$TMP/markup.bib" 'title     = {Eradication of Helicobacter pylori infection.},')"
ck "efetch title keeps the italic species name" yes \
  "$(has "$TMP/markup.md" '**Title**: Eradication of Helicobacter pylori infection.')"
# 3-4. an abstract with inline <sub> keeps the text inside and after it
ck "efetch abstract keeps the subscript and the rest of the sentence" yes \
  "$(has "$TMP/markup.md" '**Abstract**: BACKGROUND: CO2 levels rose.')"
ck "  ...and never stops at the first child element" no \
  "$(has "$TMP/markup.bib" 'title     = {Eradication of },')"

# 5-6. negative control: a title with no markup is unchanged
ck "plain bibtex title is unchanged" yes "$(has "$TMP/plain.bib" 'title     = {Plain title without markup.},')"
ck "plain efetch abstract is unchanged" yes "$(has "$TMP/plain.md" '**Abstract**: Plain abstract.')"

if [ "$fail" -eq 0 ]; then
  echo "PASS: parse_pubmed.py keeps titles and abstracts whole through inline markup."
else
  echo "FAIL: $fail check(s) failed." >&2
  exit 1
fi

#!/usr/bin/env bash
# Regression test: free-text queries in references/pubmed_eutils.sh are data, never code.
#
# `search` used to URL-encode the query by pasting it into Python source,
# `python3 -c "...quote('${query}')"`. Apostrophes are ordinary in queries (Crohn's disease,
# Alzheimer's, O'Brien[Author]). The first one closed the literal and caused a SyntaxError, the
# substitution came back empty, and the script sent `esearch?term=&retmax=...` and exited 0:
# a silent search for nothing, on the path `cite_lookup` shares. A crafted query could also
# close the literal itself and run its own Python, and `related` pasted `retmax` into source
# the same way.
#
# No network: `curl` is a stub on PATH that records each URL and answers with canned JSON.
# The pre-fix script fails 9 of the 14 assertions. It sends `term=`, runs both breakout payloads
# (their marker files appear) and accepts a non-numeric retmax. Check 5 passes before and after;
# it pins that retmax still truncates once it is passed as an argument.
set -u
HERE="$(cd "$(dirname "$0")" && pwd)"
EUTILS="$HERE/../references/pubmed_eutils.sh"
TMP="$(mktemp -d)"
trap 'rm -rf "$TMP"' EXIT

mkdir -p "$TMP/bin"
cat >"$TMP/bin/curl" <<'STUB'
#!/usr/bin/env bash
url="${*: -1}"
echo "$url" >>"$CALLS"
case "$url" in
  *elink.fcgi*) printf '%s' '{"linksets":[{"linksetdbs":[{"linkname":"pubmed_pubmed","links":[{"id":111},{"id":222},{"id":333}]}]}]}' ;;
  *) printf '%s' '{"esearchresult":{"idlist":[]}}' ;;
esac
printf '\n200\n'
STUB
chmod +x "$TMP/bin/curl"

fail=0
ck() { if [ "$2" = "$3" ]; then printf '  PASS  %s\n' "$1"; else printf '  FAIL  %s (want %s got %s)\n' "$1" "$2" "$3"; fail=$((fail+1)); fi; }
run() {  # run <args...> ; prints the exit code, URLs land in $TMP/calls
  : >"$TMP/calls"
  ( cd "$TMP" && PATH="$TMP/bin:$PATH" CALLS="$TMP/calls" NCBI_API_KEY= bash "$EUTILS" "$@" >"$TMP/out" 2>"$TMP/err" )
  echo $?
}
sent() { grep -qF -- "$1" "$TMP/calls" && echo yes || echo no; }

# 1. an apostrophe in a search query
ck "search with an apostrophe exits 0" 0 "$(run search "Crohn's disease[Title] AND MRI" 5)"
ck "  ...sends the whole encoded term" yes "$(sent 'term=Crohn%27s%20disease%5BTitle%5D%20AND%20MRI&retmax=5&')"
ck "  ...never sends an empty term" no "$(sent 'term=&')"

# 2. cite_lookup goes through the same path
ck "cite_lookup with an apostrophe exits 0" 0 "$(run cite_lookup "Crohn's disease: a review")"
ck "  ...sends the whole [Title] term" yes "$(sent 'term=Crohn%27s%20disease%3A%20a%20review%5BTitle%5D&retmax=5&')"

# 3. a query written to break out of the old Python string literal
MARK="$TMP/query-ran-code"
ck "breakout query exits 0" 0 "$(run search "x')); open('$MARK', 'w'); print(('" 5)"
ck "  ...runs no code" no "$([ -e "$MARK" ] && echo yes || echo no)"
ck "  ...is searched as literal text" yes "$(sent 'term=x%27%29%29%3B%20open%28')"

# 4. a related retmax written to break out of the old Python slice
MARK2="$TMP/retmax-ran-code"
rc="$(run related 12345 "1]]; open('$MARK2', 'w'); x=[[0")"
ck "breakout retmax is refused" yes "$([ "$rc" -ne 0 ] && echo yes || echo no)"
ck "  ...runs no code" no "$([ -e "$MARK2" ] && echo yes || echo no)"
ck "  ...before any request" 0 "$(wc -l <"$TMP/calls" | tr -d ' ')"

# 5. related still honours retmax, now passed as an argument
ck "related 12345 2 exits 0" 0 "$(run related 12345 2)"
ck "  ...fetches only the first 2 linked PMIDs" yes "$(sent 'esummary.fcgi?db=pubmed&id=111,222&')"

# 6. a non-numeric search retmax
rc="$(run search "MRI" "5&api_key=x")"
ck "non-numeric search retmax is refused" yes "$([ "$rc" -ne 0 ] && echo yes || echo no)"

if [ "$fail" -eq 0 ]; then
  echo "PASS: pubmed_eutils.sh sends free text as data, never as code."
else
  echo "FAIL: $fail check(s) failed." >&2
  exit 1
fi