{
  "skill_name": "broken-link-checker",
  "evals": [
    {
      "id": 1,
      "prompt": "check https://example.com for broken links and tell me which pages need fixes first",
      "expected_output": "A broken-link audit that summarizes scan coverage, separates internal vs external issues, and maps each broken URL back to its source page.",
      "files": [],
      "expectations": [
        "Requests or uses a target URL",
        "Reports how many pages were scanned and how many broken links were found",
        "Separates internal broken links from external broken links",
        "Maps broken links back to source pages",
        "Gives actionable fix guidance such as updating links or adding redirects"
      ]
    }
  ]
}

#!/usr/bin/env python3
"""
Broken Link Checker for NotFair.
Crawls a website starting from a given URL and identifies broken links (HTTP 4xx/5xx).

Usage:
  python3 checker.py --url https://example.com --max-pages 50
"""

import argparse
import json
import os
import sys
import urllib.parse
import urllib.request
import urllib.error
import urllib.robotparser
from concurrent.futures import ThreadPoolExecutor, as_completed
from collections import deque
from html.parser import HTMLParser


class LinkParser(HTMLParser):
    def __init__(self, base_url):
        super().__init__()
        self.base_url = base_url
        self.links = set()

    def handle_starttag(self, tag, attrs):
        if tag == 'a':
            for attr, value in attrs:
                if attr == 'href':
                    # Use urljoin to handle relative links and then urldefrag to remove fragments
                    url = urllib.parse.urljoin(self.base_url, value)
                    url = urllib.parse.urldefrag(url)[0]
                    if url.startswith('http'):
                        self.links.add(url)


def check_url(url, timeout=10):
    """Checks a URL and returns (status_code, error_msg)."""
    headers = {'User-Agent': 'NotFairBrokenLinkChecker/1.0'}

    # Try HEAD first for efficiency
    req = urllib.request.Request(url, method='HEAD', headers=headers)
    try:
        with urllib.request.urlopen(req, timeout=timeout) as response:
            return response.getcode(), None
    except urllib.error.HTTPError as e:
        # Many servers block HEAD, fall back to GET for accuracy
        if e.code in (403, 405, 501):
            req = urllib.request.Request(url, method='GET', headers=headers)
            try:
                # We only need the headers/status, so we don't read the body here
                with urllib.request.urlopen(req, timeout=timeout) as response:
                    return response.getcode(), None
            except urllib.error.HTTPError as e2:
                return e2.code, str(e2.reason)
            except Exception as e2:
                return None, str(e2)
        return e.code, str(e.reason)
    except urllib.error.URLError as e:
        return None, str(e.reason)
    except Exception as e:
        return None, str(e)


def crawl(start_url, max_pages=50):
    parsed_start = urllib.parse.urlparse(start_url)
    domain = parsed_start.netloc

    # Robots.txt check
    rp = urllib.robotparser.RobotFileParser()
    try:
        robots_url = urllib.parse.urljoin(start_url, '/robots.txt')
        robots_request = urllib.request.Request(
            robots_url,
            headers={'User-Agent': 'NotFairBrokenLinkChecker/1.0'},
        )
        with urllib.request.urlopen(robots_request, timeout=10) as response:
            robots_text = response.read().decode('utf-8', errors='ignore')
        rp.set_url(robots_url)
        rp.parse(robots_text.splitlines())
    except Exception as e:
        print(f"Warning: Could not read robots.txt: {e}", file=sys.stderr)
        # Default to allowing if robots.txt is missing or unreachable
        rp = None

    visited = set()
    queue = deque([start_url])
    broken_links = []
    pages_crawled = 0

    print(f"Starting crawl of {start_url} (limit: {max_pages} pages)...", file=sys.stderr)

    while queue and pages_crawled < max_pages:
        current_url = queue.popleft()
        if current_url in visited:
            continue

        # Check robots.txt Disallow
        if rp and not rp.can_fetch("NotFairBrokenLinkChecker/1.0", current_url):
            print(f"Skipping (blocked by robots.txt): {current_url}", file=sys.stderr)
            continue

        visited.add(current_url)
        pages_crawled += 1

        print(f"[{pages_crawled}/{max_pages}] Checking: {current_url}", file=sys.stderr)

        try:
            req = urllib.request.Request(current_url, headers={'User-Agent': 'NotFairBrokenLinkChecker/1.0'})
            with urllib.request.urlopen(req, timeout=10) as response:
                status = response.getcode()
                if status >= 400:
                    broken_links.append({"url": current_url, "status": status, "reason": "Page itself is broken"})
                    continue

                content_type = response.headers.get('Content-Type', '')
                if 'text/html' not in content_type:
                    continue

                html = response.read().decode('utf-8', errors='ignore')
                parser = LinkParser(current_url)
                parser.feed(html)

                found_links = list(parser.links)
                with ThreadPoolExecutor(max_workers=5) as executor:
                    future_to_url = {executor.submit(check_url, link): link for link in found_links}
                    for future in as_completed(future_to_url):
                        link = future_to_url[future]
                        try:
                            link_status, reason = future.result()
                        except Exception as e:
                            link_status, reason = None, str(e)

                        if link_status is None or link_status >= 400:
                            broken_links.append({
                                "source": current_url,
                                "target": link,
                                "status": link_status,
                                "reason": reason
                            })

                        # Add internal links to queue
                        if urllib.parse.urlparse(link).netloc == domain and link not in visited:
                            queue.append(link)

        except Exception as e:
            print(f"Error crawling {current_url}: {e}", file=sys.stderr)
            broken_links.append({"url": current_url, "status": None, "reason": str(e)})

    return broken_links


def main():
    parser = argparse.ArgumentParser(description="Broken Link Checker")
    parser.add_argument("--url", required=True, help="Starting URL")
    parser.add_argument("--max-pages", type=int, default=50, help="Maximum pages to crawl")
    parser.add_argument("--output", help="Output JSON file")

    args = parser.parse_args()

    broken = crawl(args.url, args.max_pages)

    result = {
        "start_url": args.url,
        "max_pages": args.max_pages,
        "broken_links_count": len(broken),
        "broken_links": broken
    }

    if args.output:
        with open(args.output, 'w') as f:
            json.dump(result, f, indent=2)
    else:
        print(json.dumps(result, indent=2))


if __name__ == "__main__":
    main()