{
  "skill_name": "agent-ready",
  "evals": [
    {
      "id": 1,
      "prompt": "Current Agent Score: 87/100. Warnings: llms.txt directive buried past 50% on some pages; 1/2 pages support .md URLs; Accept: text/markdown returns markdown with the wrong Content-Type; content starts at 17%; llms.txt covers 0/2 sitemap doc pages. Propose concrete edits grouped by check. Do not invent missing pages.",
      "expected_output": "Fixes grouped by AFDocs check with exact directive markup, .md/Content-Type headers, coverage regeneration, and no invented API or CLI.",
      "files": [],
      "assertions": [
        "Groups changes by the named AFDocs checks",
        "Places the llms.txt directive in the HTML body near the top",
        "Sets Content-Type to text/markdown for Accept negotiation",
        "Does not invent documentation pages that were not in the report or repo"
      ]
    },
    {
      "id": 2,
      "prompt": "Improve how ready https://example.com/docs is for agents. Is Agentic 93/100. Failures: JSON error responses, no typed error model in OpenAPI, no versioning policy, no rate-limit headers. Partial: docs not linked from homepage, operationIds present but schemas untyped. Inspect the existing codebase before changing files.",
      "expected_output": "Inspects routes and OpenAPI first, implements RFC 9457 errors, typed 4xx/5xx schemas, a declared /api/v1 or version header, RateLimit headers, and homepage/llms.txt links without adding a fake CLI.",
      "files": [],
      "assertions": [
        "Inspects existing API or OpenAPI before adding endpoints",
        "Uses a JSON problem or typed error schema for 4xx and 5xx",
        "Declares versioning and rate-limit headers rather than only describing them in prose",
        "Does not publish a CLI package unless one already exists or the user asked for one"
      ]
    },
    {
      "id": 3,
      "prompt": "Our CDN logs show Claude and Codex user agents hitting /docs with a 404 rate around 30%: they fetch a page, 404 on a guessed sibling path, then try another. We already serve .md twins for every page and /docs/llms.txt exists. Nothing shows up in GA. Fix it in this Next.js repo.",
      "expected_output": "Reads the twin route first, adds a server-rendered index link (absolute llms.txt URL) to the first lines of every markdown twin plus Link headers, does not rewrite prose, explains why GA shows nothing, adds a route test, and quotes curl output for a twin and llms.txt.",
      "files": [],
      "assertions": [
        "Puts a link to llms.txt in the first lines of every markdown twin, emitted by the route rather than hand-edited per page",
        "Advertises llms.txt and the markdown alternate in Link headers",
        "States that client-side analytics cannot observe agent traffic and points at server logs as the measure",
        "Does not rewrite page prose or invent new pages",
        "Quotes status and Content-Type from a fetched twin and from llms.txt after the change"
      ]
    }
  ],
  "routing": {
    "should_trigger": [
      "Current Agent Score: 87/100. Warnings: llms.txt directive buried past 50% on some pages; 1/2 pages support .md URLs.",
      "Improve how ready https://blode.co/iconsmith is for agents. Is Agentic 93/100. JSON error responses failed.",
      "Paste this isitagentready report and fix the failing checks.",
      "Fix llms.txt coverage so AFDocs stops failing.",
      "Make this documentation site agent-ready.",
      "Mintlify score is 87, implement the markdown negotiation warning.",
      "npx afdocs check failed content-negotiation and llms-txt-coverage.",
      "Add RFC 9457 error bodies and RateLimit headers to the public API from this is-agentic report.",
      "Agents keep 404ing on guessed URLs under /docs even though every page has a .md version.",
      "Run url-discovery-bench against our docs and fix whatever makes the failed-request count high.",
      "Mintlify's State of Knowledge report says markdown plus llms.txt cuts agent 404s 20x. Get our docs there.",
      "Our changelog and help center are HTML only while the docs have markdown twins. Make them agent-readable too."
    ],
    "near_miss": [
      {
        "prompt": "Audit this explanation page: it explains why transactions provide atomicity, has no quickstart, and links to /guides/transactions for usage.",
        "expected": "docs-writing"
      },
      {
        "prompt": "Audit my CLI help text and piped JSON output.",
        "expected": "dx-audit"
      },
      {
        "prompt": "Does this chat agent approval gate earn trust?",
        "expected": "ax-audit"
      },
      {
        "prompt": "Why did organic traffic drop after the migration?",
        "expected": "seo"
      },
      {
        "prompt": "Write a README from scratch for this CLI.",
        "expected": "readme-creator"
      },
      {
        "prompt": "Summarize the 2026 State of Knowledge report for the team in three paragraphs.",
        "expected": "none"
      },
      {
        "prompt": "Write a CLAUDE.md for this docs repo so agents follow our terminology.",
        "expected": "agents-md"
      }
    ]
  }
}

# APIs, OpenAPI, MCP, and CLIs (Is Agentic)

Read when the report is Is Agentic / Ora, or findings mention JSON errors, OpenAPI, versioning, rate limits, function calling, developer-portal search, or a CLI/MCP gap.

Recommended checks apply only when scan evidence already shows an API, OAuth, GraphQL, MCP, or developer portal. If the probe found no API, do not invent one. Improve docs discovery instead.

## Contents

- JSON errors
- OpenAPI error model
- Versioning and deprecation
- Rate limits
- Function calling
- Discoverability
- CLI
- MCP

## JSON errors

Agents cannot parse HTML error pages. Every API error body is JSON. Default to RFC 9457 `application/problem+json`:

```http
HTTP/1.1 400 Bad Request
Content-Type: application/problem+json

{
  "type": "https://example.com/errors/invalid-parameter",
  "title": "Invalid parameter",
  "status": 400,
  "detail": "limit must be between 1 and 100",
  "code": "invalid_parameter",
  "hint": "Retry with limit=100 or omit limit to use the default."
}
```

Keep `code` stable, `detail` specific, `hint` a next action. Map 400 invalid input, 401/403 auth, 404 missing, 405 method, 429 quota, 5xx unavailable. Do not change successful response shapes.

## OpenAPI error model

Give 4xx and 5xx a typed schema (the problem document or a shared `Error` object). Reference it from operations instead of untyped `string` bodies.

Every operation needs a unique `operationId`, a description, and typed request/response schemas. Partial function-calling scores are usually untyped request bodies or missing error schemas, not missing IDs.

Document rate-limit and deprecation headers in the spec (`RateLimit`, `RateLimit-Policy`, `Retry-After`, `Deprecation`, `Sunset`).

Publish the spec at a stable URL (`/openapi.json` or `/openapi.yaml`) and list that URL in `llms.txt`.

## Versioning and deprecation

Pick one advertised strategy and implement it:

- URL path: `/api/v1/...` for breaking changes. New major = new path.
- Or a documented version header, named in the spec.

Publish how sunset works: RFC 9745 `Deprecation` / `Sunset` headers plus a docs page with the timeline. A compatibility alias is allowed if it is marked deprecated and does not silently change behavior.

## Rate limits

On probed API responses, send IETF structured fields, plus `Retry-After` on 429:

```http
RateLimit-Policy: "default";q=120;w=60
RateLimit: "default";r=50;t=30
```

Document the quota next to the API (requests per window, what `q`/`r`/`t` mean). If there is no limiter yet, adding headers that claim a policy you do not enforce is a lie; implement the limiter or document "no limit" only when that is true. Prefer implementing a real window if the report failed this check on a public API.

## Function calling

| Required | Why |
|----------|-----|
| Unique `operationId` | Tool name |
| JSON Schema for params and bodies | No `additionalProperties: true` on unconstrained objects unless the API is a bag of extensions |
| Human description per operation | The model reads this as the tool doc |

Do not add unused properties to pass a linter. Type what the handler already validates.

## Discoverability

Predictable URLs: `/docs`, `/api`, `/developers`, `/openapi.json`. Link them from the homepage and from `llms.txt`. Put the product name in `<title>` and H1 of those pages so name search can hit them.

Do not add a fake developer portal. A single docs index that links auth, endpoints, and an example request is enough for the homepage-link check.

## CLI

A sentence in `llms.txt` is not a CLI. Pass means a package on npm, PyPI, or Homebrew that can run a real command.

If the product has no CLI and nobody asked to build one, remove the llms.txt mention (or move it under Optional as a planned item) and record it as a product decision. If they asked to ship one, scaffold with `scaffold-cli` and audit with `dx-audit`; this skill only requires the published entry and a link from `llms.txt`.

## MCP

Only if an MCP server already exists or the user asked to publish one.

Advertise the live transport URL in a server card (current well-known paths scanners still probe: `/.well-known/mcp/server-card.json`, compatibility copies at `/.well-known/mcp.json` / `/mcp/server-card`). Include `serverInfo` (name, version, description) and the transports you actually serve.

Auth belongs in RFC 9728 protected-resource metadata when the endpoint returns 401, not as a fictional card field. Anonymous read-only servers skip OAuth documents.

List the MCP URL in `llms.txt` and, when you have an RFC 9727 catalog, in `/.well-known/api-catalog`.

A docs MCP server needs both a search tool and a fetch-by-path tool. Across Mintlify-hosted servers in 2026 the calls split 53% search to 47% path retrieval, depending on whether the agent already knows the page; a server with only one strands the other half. Return markdown from both, and make the path tool accept the same slugs `llms.txt` lists.

# Docs: AFDocs and Mintlify Agent Score

Read when the report is Mintlify Agent Score, `npx afdocs`, or any `llms.txt` / markdown / truncation finding.

Follow the published formats exactly. Do not invent page copy; reuse titles and URLs already in the docs. Ask for source files when a listed page is not in the repo.

## Contents

- Baseline
- `llms.txt`
- Directives
- Markdown twins
- Size and structure
- Auth
- Other knowledge surfaces
- Framework routes

## Baseline

```bash
npx afdocs check <docs-url> --format scorecard --sampling deterministic
```

Iterate with `--checks <id>` plus its dependencies. `llms-txt-valid` needs `llms-txt-exists`. Markdown size, fence, and parity checks need `.md` URLs or `Accept: text/markdown`.

Candidate `llms.txt` locations: `{docsBase}/llms.txt`, `{origin}/llms.txt`, `{origin}/docs/llms.txt`. Canonical is the longest directory prefix of the URL you scored. Serve it on the docs host; a cross-host redirect is a warn.

## `llms.txt`

[llmstxt.org](https://llmstxt.org/) order: optional BOM, required H1, blockquote summary, optional notes (no extra headings), then H2 sections of lists. Each item is `[title](url)` plus optional `: one-line description`. An `## Optional` section is for skippable links.

Keep the file under 50,000 characters. Split with nested `llms.txt` files before 100,000. Link markdown URLs, not HTML, once twins exist.

Regenerate from the docs inventory at build time so `llms-txt-coverage` stays current. Default AFDocs thresholds: pass at 95% of sitemap doc pages, fail under 80%. Built-in exclusions include `/blog`, `/pricing`, `/about`, `/legal`, `/login`. If the index is curated, say so in the summary rather than stuffing unrelated URLs.

```text
# Acme API

> Acme processes payments through a REST API. Agents start at this index, then fetch markdown twins.

## Docs

- [Authentication](https://docs.acme.com/auth.md): keys, headers, sandbox vs live
- [Rate limits](https://docs.acme.com/rate-limits.md): quotas and 429 retry

## Optional

- [Changelog](https://docs.acme.com/changelog.md): dated notes
```

Use the site's real titles and URLs, never invented pages.

## Directives

Must appear in the HTML **body**, not `<head>`, `<nav>`, or `<script>`, and in the first 50% of the converted page. Server-render them.

The markdown twin's directive matters more than the HTML one. Converting a page to markdown removes the sidebar and breadcrumbs, so an agent that lands on a twin has nothing to navigate with and guesses sibling URLs. The index link removed most of those guesses in Mintlify's benchmark (numbers in SKILL.md, Priority). Ship the twin link before anything else in this file.

HTML (visually hidden, stays in the DOM):

```html
<p class="sr-only">
  For AI agents: the documentation index is at
  <a href="/llms.txt">/llms.txt</a>.
  Append <code>.md</code> to this URL or send
  <code>Accept: text/markdown</code> for markdown.
</p>
```

Markdown twin, first lines after any title (absolute URLs, since agents fetch twins from any base):

```markdown
> ## Documentation index
> [Documentation index](https://docs.acme.com/llms.txt)
> [HTML page](https://docs.acme.com/auth)
> Use the index to discover pages before guessing URLs.
```

Emit it from the same route that renders the twin, so no page can miss it. Inlining the whole `llms.txt` into every page is the benchmark's fourth arm; the report does not publish its result, so prefer the link and keep the twin small.

Use the actual `llms.txt` path (`/docs/llms.txt` when that is canonical). Also emit HTTP links when you can set headers without fighting the framework:

```http
Link: </llms.txt>; rel="describedby", </docs/example.md>; rel="alternate"; type="text/markdown"
```

## Markdown twins

Every docs HTML URL must answer at the same path with `.md` appended (or `index.md` on directory URLs) with `200` and a markdown body.

Honor `Accept: text/markdown` on the HTML URL:

| Header | Value |
|--------|--------|
| `Content-Type` | `text/markdown; charset=utf-8` |
| `Vary` | `Accept` |
| Unsupported combination | `406` when the client rejects every representation you offer |

`text/plain` or HTML with a markdown body is a warn. Serve the same source the HTML was built from so `markdown-content-parity` holds. Mark human-only chrome with `data-markdown-ignore` rather than drifting the two copies.

Cache `llms.txt` and markdown with `max-age` of 300–3600 plus `ETag` or `Last-Modified`. Multi-day `max-age` with no revalidation fails `cache-header-hygiene`.

## Size and structure

| Check | Pass | Fix |
|-------|------|-----|
| `rendering-strategy` | Substantive HTML without JS | Enable SSR/SSG; do not ship empty SPA shells |
| `content-start-position` | Main content in first 10% of converted text | Move sidebars/breadcrumbs after or beside main; do not serialize the full nav first |
| `page-size-*` | Under 50k characters | Split mega-pages; do not serialize every tab into one document |
| `tabbed-content-serialization` | Tabs under 50k serialized | Per-variant URLs or `?lang=` returning one variant |
| `section-header-quality` | Headers unique across tabs | `Step 1 (Python)` not `Step 1` |
| `markdown-code-fence-validity` | Fences closed with the same delimiter | Nested fences use a longer run of backticks |
| `http-status-codes` / redirects | Real 404s, same-host HTTP 3xx | No JS-only redirects, no soft 404s |

## Auth

Public reference and getting-started pages should fetch without a session. If a gate is required, publish an ungated `llms.txt` plus an alternate path (public mirror, shipped SDK docs). Do not put public docs behind bot challenges that HTML-only agents cannot pass.

## Other knowledge surfaces

Agents answer from whatever the origin serves, and most companies keep product knowledge on five or more surfaces while only the docs are agent-readable. Walk the sitemap and the homepage nav for changelog, release notes, help center, community, status, and API status pages. For each surface the product owns, decide once:

| Surface | Twins and index entry | Why |
|---------|----------------------|-----|
| Changelog, release notes, migration guides | Yes, under `## Optional` if the main list is long | Agents act on stale parameters; the changelog is where the fix lives |
| Help center, FAQ, troubleshooting | Yes | These answer the questions the docs did not |
| Community threads, forum | Only when the company hosts and stands behind the answers | Unvetted answers propagate as fact |
| Marketing, pricing, legal, careers | No | AFDocs excludes them; listing them dilutes the index |

A surface on a different host or CMS is a product decision to record, not something to fake with a copy in the docs repo.

## Framework routes

This file is the AFDocs contract (shape, headers, thresholds). In a Next.js App Router site, implement the handlers with `seo`: that skill's `nextjs-implementation.md` for the route and `proxy.ts` matcher, and `answer-engines.md` for Accept/`Vary`/canonicals. Inspect the app's existing `llms.txt` route before adding another.

# Site discovery (Is It Agent Ready)

Read when the report is from [isitagentready.com](https://isitagentready.com/) or findings name robots, sitemap, Link headers, DNS-AID, Content Signals, API catalog, Auth.md, skills index, or commerce protocols.

These checks advertise **existing** surfaces. A well-known JSON file that describes a server you do not run is worse than omitting the file.

## Contents

- Always worth doing
- Only when the product has the surface
- Leave unless asked

## Always worth doing

Valid `/robots.txt` as `text/plain` with `User-agent` groups and a `Sitemap:` line pointing at a working `/sitemap.xml` (or the sitemap the app already emits).

Named AI-bot groups in addition to `*`, matching the owner's training vs retrieval policy. Do not copy a full allow-list from training data; `seo` owns which bots mean search vs training (`answer-engines.md` on that skill). Carry forward existing private-path disallows.

Homepage (and docs HTML) `Link` headers for what you actually publish, for example:

```http
Link: </.well-known/api-catalog>; rel="api-catalog", </openapi.json>; rel="service-desc"; type="application/openapi+json", </llms.txt>; rel="describedby", </docs>; rel="alternate"; type="text/markdown"
```

Only include relations whose targets return 200 with the advertised type.

`Auth.md` (or `/docs` auth page listed from `llms.txt`) when the product has credentials. One page: how to get a key, the header name, and a failed-auth example that matches the JSON error model.

## Only when the product has the surface

| Check | Publish when | Where |
|-------|--------------|--------|
| RFC 9727 API catalog | A public HTTP API exists | `/.well-known/api-catalog` as `application/linkset+json` pointing at the OpenAPI URL |
| OAuth / OIDC discovery | You are an authorization server | `/.well-known/oauth-authorization-server` or `openid-configuration` |
| RFC 9728 protected resource | The API or MCP requires a bearer token | `/.well-known/oauth-protected-resource` |
| MCP server card | A live MCP endpoint exists | Well-known card for the transport you actually serve |
| Agent skills index | You publish installable skills | `/.well-known/agent-skills/index.json` with URL plus SHA-256 of each `SKILL.md` |
| A2A agent card | You speak A2A | `/.well-known/agent-card.json` |
| WebMCP | The page really calls `navigator.modelContext` | Do not stub the call |
| Content Signals | You have an explicit train/search policy | `Content-Signal:` in robots.txt (`ai-train`, `search`, `ai-input`) |
| Web Bot Auth | You verify HTTP message signatures | `/.well-known/http-message-signatures-directory` |
| DNS-AID | You can add DNS HTTPS/SVCB records | `_index._agents` (and `_mcp._agents` if MCP is live), DNSSEC if available |

DNS-AID and DNSSEC need registrar or DNS-provider access. Implement HTTP discovery first, then record DNS as a remaining recommendation.

## Leave unless asked

x402, UCP, ACP, MPP, and other commerce protocols. Adding them on a docs site that does not charge agents is advertising a payment flow that does not exist.

`llms-full.txt` is optional. Prefer a small `llms.txt` plus markdown twins over a dump that exceeds 50k characters.

Site-type hint for Is Agentic reports (does not change the score):

```html
<meta name="is-agentic-site-type" content="content">
```

Values: `content` (docs), `business`, `app`, `store`. One tag, one value.

# Verification

Read at Step 6. Quote command output (status, content-type, a header, or the first lines of a body). A green local build is not evidence that `llms.txt` is public.

## Probe the files you changed

Use the public origin when the user has one; otherwise the local preview origin. Same path in both cases.

Run the bundled script first, resolved against the installed skill directory. It prints one TSV line per URL (status, content-type, link, location, url) and exits 1 on any non-200 or a `Content-Type` that does not match `--expect-type`:

```bash
scripts/check-surfaces.sh --origin "$ORIGIN" /llms.txt /robots.txt /sitemap.xml /docs/example.md
scripts/check-surfaces.sh --accept text/markdown --expect-type text/markdown "$ORIGIN/docs/example"
scripts/check-surfaces.sh --origin "$ORIGIN" /openapi.json /.well-known/api-catalog --out surfaces.tsv
```

Quote its output lines as the evidence. Redirects count as misses because it does not follow them; a same-host 3xx you intend is fine, but say so. Use curl directly for what the script does not do: bodies, POST probes, and the HTML-Accept control.

```bash
curl -sSI "$ORIGIN/llms.txt"
curl -sS "$ORIGIN/llms.txt" | head
curl -sSI "$ORIGIN/docs/example.md"
curl -sSI "$ORIGIN/docs/example" -H "Accept: text/markdown"
curl -sSI "$ORIGIN/docs/example" -H "Accept: text/html"
curl -sSI "$ORIGIN/openapi.json"
curl -sSI "$ORIGIN/.well-known/api-catalog"
curl -sS -o /dev/null -D - -X POST "$ORIGIN/api/v1/does-not-exist"
```

Adjust paths to the repo. Expect:

- `llms.txt`: 200, `text/plain` or markdown-compatible type, H1 plus links
- `.md` and `Accept: text/markdown`: 200, `content-type: text/markdown`, body is markdown
- HTML Accept: existing HTML page, not the markdown body
- API errors: JSON problem document, not an HTML shell
- Rate-limit headers present on API responses you claim to throttle
- Catalog and OpenAPI: parseable JSON, targets 200

After docs edits, rerun:

```bash
npx afdocs check "$DOCS_URL" --sampling deterministic --format scorecard
```

After API edits, rerun `npx is-agentic <domain> --json` against the deployed origin if the user authorized a deploy; otherwise report local curl as preview evidence and name that the scanner still sees production.

## Server logs

Client-side analytics cannot see agents. Measure readership and navigation failures from the server or CDN log instead, before and after the change:

```bash
# requests for machine-readable routes
grep -E '(\.md|llms(-full)?\.txt) HTTP' access.log | wc -l
# 404s from recognized AI clients, by path
grep -iE 'claude|gptbot|chatgpt-user|oai-searchbot|perplexitybot|anthropic|codex' access.log \
  | awk '$9 == 404 {print $7}' | sort | uniq -c | sort -rn | head -20
```

Adjust field numbers to the log format. Bursts of 404s where the path is a plausible sibling of a real page (`/docs/auth/api-keys` when only `/docs/authentication` exists) are the navigation failure; a falling count after the twin link ships is the evidence to quote.

## Navigation benchmark

When the user wants the failed-request number itself, run Mintlify's open-source [url-discovery-bench](https://github.com/mintlify/url-discovery-bench) against the site. It needs Python 3.9+, plus Claude Code or the Codex CLI for the agent arms, and spends model tokens, so confirm the spend first.

```bash
git clone https://github.com/mintlify/url-discovery-bench && cd url-discovery-bench
pip install -r requirements.txt
python -m url_discovery_bench.run --dataset dataset/<site>.json --agents claude
python -m url_discovery_bench.report jobs/<jobname>
```

Write `dataset/<site>.json` from the site's real pages and questions in the shape of the bundled datasets. Compare the `md-link` arm against `html` and `md`; the report's published baseline is 2.23, 1.42, and 0.11 failed requests per task. Quote the site's own numbers, not the baseline.

## Tests

Add a test that hits the route or handler: markdown content-type, problem+json status and `code`, OpenAPI fragment for 4xx, `RateLimit` header. Match the project's runner. Do not add a browser screenshot as the only proof of a header.

## Remaining recommendations

Put in the summary, not in code:

- DNS-AID / DNSSEC
- Publishing an npm/Homebrew CLI that does not exist yet
- Ungating docs that legal or billing require a session for
- Commerce protocols
- Changing bot-training policy

#!/usr/bin/env bash
#
# Probes the machine-readable surfaces a site advertises and prints one TSV
# line per URL: status, content-type, link, location, url. Exits 1 when any
# probe misses: a non-200 status, a curl failure, or a Content-Type that does
# not start with --expect-type. Redirects are not followed; a 3xx is a miss
# with its Location shown, because agents that do not follow it see the miss.
#
#   check-surfaces.sh --origin https://docs.example.com
#       probes the standard set: /llms.txt /robots.txt /sitemap.xml
#   check-surfaces.sh --origin https://docs.example.com /guide.md /openapi.json
#       probes the given paths on that origin (absolute URLs also accepted)
#   check-surfaces.sh --accept text/markdown --expect-type text/markdown https://docs.example.com/guide
#       content negotiation: the HTML URL must answer with markdown
#   check-surfaces.sh --origin https://docs.example.com --out surfaces.tsv
#       also writes the TSV to a file for the report
#
# Portable to bash 3.2 (macOS default): no associative arrays, no mapfile.

set -uo pipefail

ORIGIN=""
ACCEPT=""
EXPECT_TYPE=""
OUT=""
# llms-full.txt on a large site runs several megabytes; a surface that takes
# longer than this to answer is itself a finding, so do not wait longer.
MAX_TIME=20

usage() { sed -n '2,20p' "$0" | sed 's/^# \{0,1\}//'; }

TARGETS=""
while [ $# -gt 0 ]; do
  case "$1" in
    --origin) ORIGIN="${2%/}"; shift 2 ;;
    --accept) ACCEPT="$2"; shift 2 ;;
    --expect-type) EXPECT_TYPE="$2"; shift 2 ;;
    --out) OUT="$2"; shift 2 ;;
    -h|--help) usage; exit 0 ;;
    -*) echo "unknown option: $1" >&2; usage >&2; exit 2 ;;
    *) TARGETS="$TARGETS
$1"; shift ;;
  esac
done

if [ -z "$TARGETS" ]; then
  if [ -z "$ORIGIN" ]; then
    echo "give --origin URL, one or more paths or URLs, or both" >&2
    exit 2
  fi
  TARGETS="
/llms.txt
/robots.txt
/sitemap.xml"
fi

# header_value <headers-file> <name>: first matching header's value, lowercase
# name match, trailing CR stripped. Empty when absent.
header_value() {
  awk -v name="$2" 'BEGIN { IGNORECASE = 1 }
    tolower($0) ~ "^" tolower(name) ":" { sub(/^[^:]*:[ \t]*/, ""); sub(/\r$/, ""); print; exit }' "$1"
}

HEADERS="$(mktemp)"
trap 'rm -f "$HEADERS"' EXIT

misses=0
lines="status	content-type	link	location	url"

probe() {
  url="$1"
  if [ -n "$ACCEPT" ]; then
    curl -sS -o /dev/null -D "$HEADERS" --max-time "$MAX_TIME" -H "Accept: $ACCEPT" "$url" 2>/dev/null
  else
    curl -sS -o /dev/null -D "$HEADERS" --max-time "$MAX_TIME" "$url" 2>/dev/null
  fi
  rc=$?
  if [ $rc -ne 0 ]; then
    status="000"; ctype=""; link=""; location="curl exit $rc"
  else
    # HTTP/1.1 and HTTP/2 status lines both carry the code in field 2.
    status="$(awk 'NR == 1 { print $2 }' "$HEADERS")"
    ctype="$(header_value "$HEADERS" content-type)"
    link="$(header_value "$HEADERS" link)"
    location="$(header_value "$HEADERS" location)"
  fi
  miss=0
  [ "$status" = "200" ] || miss=1
  if [ -n "$EXPECT_TYPE" ]; then
    case "$ctype" in
      "$EXPECT_TYPE"*) ;;
      *) miss=1 ;;
    esac
  fi
  [ $miss -eq 0 ] || misses=$((misses + 1))
  lines="$lines
$status	$ctype	$link	$location	$url"
}

echo "$TARGETS" | while IFS= read -r target; do
  [ -n "$target" ] || continue
  case "$target" in
    http://*|https://*) echo "$target" ;;
    *)
      if [ -z "$ORIGIN" ]; then
        echo "path $target needs --origin" >&2
        exit 2
      fi
      echo "$ORIGIN/${target#/}"
      ;;
  esac
done > "$HEADERS.urls" || exit 2

while IFS= read -r url; do
  [ -n "$url" ] || continue
  probe "$url"
done < "$HEADERS.urls"
rm -f "$HEADERS.urls"

printf '%s\n' "$lines"
if [ -n "$OUT" ]; then
  printf '%s\n' "$lines" > "$OUT"
  echo "wrote $OUT" >&2
fi

if [ $misses -gt 0 ]; then
  echo "$misses surface(s) missed" >&2
  exit 1
fi