{
  "skill_name": "seo-analysis",
  "evals": [
    {
      "id": 1,
      "prompt": "Can you run an SEO audit for me?",
      "expected_output": "The skill asks for the website URL upfront before doing anything else (before running preflight, before trying to list GSC properties). It should not assume a URL or proceed without one.",
      "files": [],
      "expectations": [
        "The response asks the user for their website URL before taking any action",
        "The skill does not proceed with preflight or GSC steps without first getting the URL",
        "The question is friendly and gives an example format (e.g. https://yoursite.com)"
      ]
    },
    {
      "id": 2,
      "prompt": "My site is https://acme-saas.com. I want a full SEO audit but I don't have gcloud installed or Google Search Console set up. What can you do?",
      "expected_output": "The skill gracefully handles missing gcloud/GSC by falling back to a technical-only audit. It should still perform metadata audit, schema markup audit, technical crawl of the homepage and key pages, and offer to run /meta-tags-optimizer and /schema-markup-generator based on what it finds. It should be honest about what it can and can't do without GSC data.",
      "files": [],
      "expectations": [
        "The skill acknowledges the URL https://acme-saas.com and confirms it will use this",
        "The skill does not fail or stop when gcloud is unavailable — it falls back to Phase 5 technical audit",
        "The response includes a crawl or fetch of the site's homepage",
        "The response includes a metadata audit section (title tags, meta descriptions)",
        "The response includes a schema markup audit section",
        "The response offers to run /meta-tags-optimizer or /schema-markup-generator as follow-up"
      ]
    },
    {
      "id": 3,
      "prompt": "My website is https://flowmetrics.io (a SaaS analytics tool). Here is my Google Search Console data from the past 90 days:\n\n```json\n{\"site\": \"sc-domain:flowmetrics.io\", \"period\": {\"start\": \"2025-12-22\", \"end\": \"2026-03-21\", \"days\": 90}, \"summary\": {\"clicks\": 8240, \"impressions\": 412000, \"ctr\": 2.0, \"position\": 14.3}, \"top_queries\": [{\"query\": \"analytics dashboard software\", \"clicks\": 890, \"impressions\": 42000, \"ctr\": 2.1, \"position\": 8.4}, {\"query\": \"flowmetrics\", \"clicks\": 720, \"impressions\": 3200, \"ctr\": 22.5, \"position\": 1.2}, {\"query\": \"saas metrics dashboard\", \"clicks\": 610, \"impressions\": 38000, \"ctr\": 1.6, \"position\": 9.1}, {\"query\": \"mrr tracking tool\", \"clicks\": 480, \"impressions\": 28000, \"ctr\": 1.7, \"position\": 11.2}, {\"query\": \"churn rate calculator\", \"clicks\": 320, \"impressions\": 19000, \"ctr\": 1.7, \"position\": 12.8}, {\"query\": \"best saas analytics tools\", \"clicks\": 280, \"impressions\": 31000, \"ctr\": 0.9, \"position\": 7.2}, {\"query\": \"ltv cac ratio tool\", \"clicks\": 180, \"impressions\": 14000, \"ctr\": 1.3, \"position\": 15.4}, {\"query\": \"subscription analytics software\", \"clicks\": 170, \"impressions\": 22000, \"ctr\": 0.8, \"position\": 6.8}, {\"query\": \"how to reduce saas churn\", \"clicks\": 150, \"impressions\": 18000, \"ctr\": 0.8, \"position\": 8.9}, {\"query\": \"arr tracking spreadsheet\", \"clicks\": 90, \"impressions\": 12000, \"ctr\": 0.75, \"position\": 13.1}], \"top_pages\": [{\"page\": \"https://flowmetrics.io/\", \"clicks\": 2100, \"impressions\": 85000, \"ctr\": 2.5, \"position\": 7.2}, {\"page\": \"https://flowmetrics.io/features\", \"clicks\": 1200, \"impressions\": 62000, \"ctr\": 1.9, \"position\": 11.4}, {\"page\": \"https://flowmetrics.io/pricing\", \"clicks\": 980, \"impressions\": 44000, \"ctr\": 2.2, \"position\": 9.8}, {\"page\": \"https://flowmetrics.io/blog/reduce-saas-churn\", \"clicks\": 620, \"impressions\": 38000, \"ctr\": 1.6, \"position\": 8.1}, {\"page\": \"https://flowmetrics.io/blog/mrr-vs-arr\", \"clicks\": 440, \"impressions\": 29000, \"ctr\": 1.5, \"position\": 12.3}], \"position_buckets\": {\"1-3\": [{\"query\": \"flowmetrics\", \"clicks\": 720, \"impressions\": 3200, \"ctr\": 22.5, \"position\": 1.2}], \"4-10\": [{\"query\": \"analytics dashboard software\", \"clicks\": 890, \"impressions\": 42000, \"ctr\": 2.1, \"position\": 8.4}, {\"query\": \"saas metrics dashboard\", \"clicks\": 610, \"impressions\": 38000, \"ctr\": 1.6, \"position\": 9.1}, {\"query\": \"best saas analytics tools\", \"clicks\": 280, \"impressions\": 31000, \"ctr\": 0.9, \"position\": 7.2}, {\"query\": \"subscription analytics software\", \"clicks\": 170, \"impressions\": 22000, \"ctr\": 0.8, \"position\": 6.8}, {\"query\": \"how to reduce saas churn\", \"clicks\": 150, \"impressions\": 18000, \"ctr\": 0.8, \"position\": 8.9}], \"11-20\": [{\"query\": \"mrr tracking tool\", \"clicks\": 480, \"impressions\": 28000, \"ctr\": 1.7, \"position\": 11.2}, {\"query\": \"churn rate calculator\", \"clicks\": 320, \"impressions\": 19000, \"ctr\": 1.7, \"position\": 12.8}, {\"query\": \"ltv cac ratio tool\", \"clicks\": 180, \"impressions\": 14000, \"ctr\": 1.3, \"position\": 15.4}, {\"query\": \"arr tracking spreadsheet\", \"clicks\": 90, \"impressions\": 12000, \"ctr\": 0.75, \"position\": 13.1}], \"21+\": []}, \"ctr_opportunities\": [{\"query\": \"best saas analytics tools\", \"clicks\": 280, \"impressions\": 31000, \"ctr\": 0.9, \"position\": 7.2}, {\"query\": \"subscription analytics software\", \"clicks\": 170, \"impressions\": 22000, \"ctr\": 0.8, \"position\": 6.8}, {\"query\": \"how to reduce saas churn\", \"clicks\": 150, \"impressions\": 18000, \"ctr\": 0.8, \"position\": 8.9}], \"ctr_gaps_by_page\": [{\"query\": \"best saas analytics tools\", \"page\": \"https://flowmetrics.io/\", \"clicks\": 280, \"impressions\": 31000, \"ctr\": 0.9, \"position\": 7.2}, {\"query\": \"subscription analytics software\", \"page\": \"https://flowmetrics.io/features\", \"clicks\": 170, \"impressions\": 22000, \"ctr\": 0.8, \"position\": 6.8}, {\"query\": \"how to reduce saas churn\", \"page\": \"https://flowmetrics.io/blog/reduce-saas-churn\", \"clicks\": 150, \"impressions\": 18000, \"ctr\": 0.8, \"position\": 8.9}], \"cannibalization\": [{\"query\": \"saas analytics\", \"competing_pages\": [{\"page\": \"https://flowmetrics.io/\", \"clicks\": 310, \"impressions\": 18000, \"ctr\": 1.7, \"position\": 8.2}, {\"page\": \"https://flowmetrics.io/features\", \"clicks\": 190, \"impressions\": 14000, \"ctr\": 1.4, \"position\": 11.8}], \"total_impressions\": 32000, \"total_clicks\": 500}], \"comparison\": {\"period\": \"2026-02-22 to 2026-03-21\", \"prior_period\": \"2026-01-24 to 2026-02-21\", \"declining_pages\": [{\"page\": \"https://flowmetrics.io/blog/mrr-vs-arr\", \"clicks_now\": 110, \"clicks_prev\": 220, \"change_pct\": -50.0}], \"declining_queries\": [{\"query\": \"mrr arr difference\", \"clicks_now\": 40, \"clicks_prev\": 95, \"change_pct\": -57.9}]}, \"device_split\": [{\"device\": \"DESKTOP\", \"clicks\": 5340, \"impressions\": 264000, \"ctr\": 2.0, \"position\": 13.8}, {\"device\": \"MOBILE\", \"clicks\": 2620, \"impressions\": 138000, \"ctr\": 1.9, \"position\": 15.1}, {\"device\": \"TABLET\", \"clicks\": 280, \"impressions\": 10000, \"ctr\": 2.8, \"position\": 12.4}], \"country_split\": [{\"country\": \"usa\", \"clicks\": 4120, \"impressions\": 198000, \"ctr\": 2.1, \"position\": 13.2}, {\"country\": \"gbr\", \"clicks\": 920, \"impressions\": 48000, \"ctr\": 1.9, \"position\": 14.8}, {\"country\": \"can\", \"clicks\": 680, \"impressions\": 34000, \"ctr\": 2.0, \"position\": 14.1}, {\"country\": \"ind\", \"clicks\": 310, \"impressions\": 52000, \"ctr\": 0.6, \"position\": 18.4}, {\"country\": \"aus\", \"clicks\": 290, \"impressions\": 14000, \"ctr\": 2.1, \"position\": 14.6}], \"search_type_split\": [{\"type\": \"web\", \"clicks\": 8100, \"impressions\": 408000, \"ctr\": 2.0, \"position\": 14.3}, {\"type\": \"image\", \"clicks\": 140, \"impressions\": 4000, \"ctr\": 3.5, \"position\": 8.2}]}\n```\n\nPlease give me a full SEO analysis. I want to understand my quick wins, keyword gaps, metadata issues, and schema problems.",
      "expected_output": "A comprehensive SEO analysis report that includes: Traffic Snapshot table, Quick Wins section with specific actionable recommendations tied to exact queries and pages, Keyword Gaps section (orphan keywords + topic clusters + business gaps), Metadata Issues table, Schema Gaps table, Content Opportunities, Traffic Drops section (specifically about the blog/mrr-vs-arr 50% drop), cannibalization analysis, and a prioritized 30-Day Action Plan. Also offers /meta-tags-optimizer, /schema-markup-generator, and /keyword-research as follow-up actions.",
      "files": [],
      "expectations": [
        "Report has a Traffic Snapshot table with the correct metrics (8240 clicks, 412000 impressions, 2.0% CTR, position 14.3)",
        "Quick Wins section specifically mentions 'best saas analytics tools' (position 7.2, 31000 impressions, 0.9% CTR) with a specific fix recommendation",
        "Quick Wins section specifically mentions 'subscription analytics software' (position 6.8, 22000 impressions, 0.8% CTR)",
        "Keyword Gaps section includes orphan keywords — specifically 'mrr tracking tool', 'churn rate calculator', or 'ltv cac ratio tool' as queries without dedicated pages",
        "Keyword Cannibalization section addresses 'saas analytics' competing between homepage and /features",
        "Traffic Drops section addresses the blog/mrr-vs-arr page dropping 50% (-110 clicks)",
        "Metadata Issues section is present (checks real page metadata or notes it needs crawling)",
        "Schema Gaps section is present with site type identified as SaaS and expected schema types listed",
        "The report offers to run /meta-tags-optimizer, /schema-markup-generator, or /keyword-research as follow-up",
        "30-Day Action Plan table is present with at least 3 prioritized actions"
      ]
    }
  ]
}

# Google Search Console API Setup Guide

## Step 0 — Which Google Account Has GSC Access?

This is the most common source of confusion. You need to authenticate with the
**exact Google account** that has access to Search Console for this site.

Check which account it is:
1. Go to https://search.google.com/search-console
2. Note which Google account you're logged in as (top-right corner)
3. Use that same account in all the steps below

If you have multiple Google accounts (work email, personal Gmail, different org),
make sure you pick the right one. A valid gcloud token from the wrong account will
appear to work but return no GSC properties.

---

## Step 1 — Install the gcloud CLI

Skip this step if you already have gcloud installed (`gcloud --version` to check).

### macOS (Homebrew)

```bash
brew install google-cloud-sdk
```

### Linux (Debian/Ubuntu)

```bash
# Install prerequisites
sudo apt-get install -y curl apt-transport-https ca-certificates gnupg

# Add the Google Cloud GPG key (modern keyring method, works on Debian 12+/Ubuntu 22.04+)
curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg \
  | sudo gpg --dearmor -o /usr/share/keyrings/cloud.google.gpg

# Add the Cloud SDK apt repository
echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] \
  https://packages.cloud.google.com/apt cloud-sdk main" \
  | sudo tee /etc/apt/sources.list.d/google-cloud-sdk.list

sudo apt-get update && sudo apt-get install -y google-cloud-cli
```

### Linux (RPM/Fedora/RHEL)

```bash
sudo tee /etc/yum.repos.d/google-cloud-sdk.repo << EOM
[google-cloud-cli]
name=Google Cloud CLI
baseurl=https://packages.cloud.google.com/yum/repos/cloud-sdk-el8-x86_64
enabled=1
gpgcheck=1
repo_gpgcheck=0
gpgkey=https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
EOM
sudo dnf install google-cloud-cli
```

### Windows

```powershell
winget install Google.CloudSDK
```

Or download the installer:
https://cloud.google.com/sdk/docs/install#windows

---

## Step 2 — Initialize gcloud (First-Time Users)

Skip this step if you've used gcloud before and already have a project configured
(`gcloud config get-value project` to check).

```bash
gcloud init
```

This interactive wizard will:
1. **Log you into Google** — a browser window opens. Sign in with the account from Step 0.
2. **Select or create a GCP project** — if you don't have one, choose "Create a new project" and give it any name (e.g., `my-seo-tools`). The project is free — it's just a container for API access.

After `gcloud init` completes, verify:
```bash
gcloud config get-value project
# Should print your project name, e.g. "my-seo-tools"
```

---

## Step 3 — Enable the Search Console API

The Search Console API must be enabled in your GCP project before you can pull data.

```bash
gcloud services enable searchconsole.googleapis.com
```

This is a one-time step per project. It's free — the Search Console API has no charges.

**If you get a billing error**: Some GCP projects require a billing account even for free APIs. Either:
- Link a billing account at https://console.cloud.google.com/billing (you won't be charged for Search Console API usage)
- Or create a new project at https://console.cloud.google.com/projectcreate and try again

---

## Step 4 — Authenticate for Search Console Access (OAuth)

This is the key step. You need Application Default Credentials (ADC) with the
Search Console scopes. The `--scopes` flag is required — omitting it defaults to
the broad `cloud-platform` scope, which asks for access to BigQuery, Compute Engine,
and other services you don't need.

```bash
gcloud auth application-default login \
  --scopes=https://www.googleapis.com/auth/webmasters,https://www.googleapis.com/auth/webmasters.readonly
```

A browser window opens. **Log in with the Google account from Step 0** — the one
that has access to Search Console for your site.

You'll see a consent screen asking to grant "Search Console API" access.
Click **Allow**.

The token is stored at `~/.config/gcloud/application_default_credentials.json`
and auto-refreshes — you won't need to do this again unless the token is revoked.

The scripts auto-detect the quota project from your gcloud config, so no extra
setup is needed.

---

## Step 5 — Verify Everything Works

```bash
SKILL_SCRIPTS=$(find ~/.claude/plugins ~/.claude/skills ~/.codex/skills .agents/skills -type d -name scripts -path "*seo-analysis*" 2>/dev/null | head -1)
python3 "$SKILL_SCRIPTS/list_gsc_sites.py"
```

This should list your Search Console properties. If it does, you're done.

---

## Property Types in Search Console

GSC has two types of properties:

- **Domain property**: `sc-domain:example.com` — covers all URLs, protocols, subdomains
- **URL-prefix property**: `https://example.com/` — covers only that exact prefix

Domain properties are better (more complete data). The analysis scripts handle both.

---

## Troubleshooting

**"No Search Console properties found"**: gcloud is working but the wrong Google
account is authenticated. Re-run Step 4 and log in with the account that has GSC
access (see Step 0).

**"Access Not Configured" / HTTP 403 with "API not enabled"**: The Search Console
API isn't enabled in your GCP project. Run Step 3:
```bash
gcloud services enable searchconsole.googleapis.com
```

**"The caller does not have permission"**: The authenticated account doesn't have
access to the specific GSC property. Verify at
https://search.google.com/search-console → Settings → Users and permissions.

**"insufficient_scope" or 403 on API calls despite valid token**: Your ADC token
was created without `--scopes` (which defaults to `cloud-platform`), or was set up
for a different Google service. The preflight script now detects this automatically
and re-authenticates. To fix manually, re-run Step 4:
```bash
gcloud auth application-default login \
  --scopes=https://www.googleapis.com/auth/webmasters,https://www.googleapis.com/auth/webmasters.readonly
```

**"quota project not set" / 403 with quota error**: The scripts auto-detect the
quota project from `gcloud config`. If this still fails, set it explicitly:
```bash
gcloud auth application-default set-quota-project "$(gcloud config get-value project)"
```

**"No project configured" / gcloud init never run**: Run Step 2:
```bash
gcloud init
```

**Token expired**: ADC tokens auto-refresh. If you get persistent auth errors,
re-run Step 4.

**Billing required error when enabling API**: Link a billing account at
https://console.cloud.google.com/billing — the Search Console API is free, but
some GCP projects require billing to be configured.

#!/usr/bin/env python3
"""
Pull and analyze Google Search Console data.
Outputs structured JSON for the seo-analysis skill to process.

Usage:
  python3 analyze_gsc.py --site "sc-domain:example.com" --days 90
  python3 analyze_gsc.py --site "https://example.com/" --days 28
"""

import argparse
import json
import os
import re
import subprocess
import sys
import tempfile
import urllib.parse
import urllib.request
import urllib.error
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import date, timedelta

from _gcloud import adc_access_token, adc_config_dir, gcloud_run
from _uid import portable_uid, secure_write_json


DEFAULT_PAGE_GROUPS = [
    ("blog",         r"/blog/"),
    ("products",     r"/product"),
    ("locations",    r"/location"),
    ("services",     r"/service"),
    ("pricing",      r"/pricing"),
    ("docs",         r"/docs?/"),
    ("about",        r"/about"),
    ("faq",          r"/faq"),
    ("landing",      r"/lp/"),
    ("case-studies", r"/case-studi"),
]


def get_quota_project():
    """Return the quota_project_id from the ADC JSON file, or None."""
    adc_dir = adc_config_dir()
    adc_path = os.path.join(adc_dir, "application_default_credentials.json")
    try:
        with open(adc_path) as f:
            data = json.load(f)
        if isinstance(data, dict):
            return data.get("quota_project_id") or None
    except (OSError, ValueError):
        pass
    return None


def get_access_token():
    try:
        result = adc_access_token()
    except FileNotFoundError:
        print("ERROR: gcloud not found. Install it and authenticate:", file=sys.stderr)
        print("  https://cloud.google.com/sdk/docs/install", file=sys.stderr)
        sys.exit(1)
    except subprocess.TimeoutExpired:
        print("ERROR: gcloud timed out after 15s. Check your network or gcloud installation.", file=sys.stderr)
        sys.exit(1)
    if result.returncode != 0:
        print("ERROR: Not authenticated. Run:", file=sys.stderr)
        print("  gcloud auth application-default login \\", file=sys.stderr)
        print("    --scopes=https://www.googleapis.com/auth/webmasters,"
              "https://www.googleapis.com/auth/webmasters.readonly", file=sys.stderr)
        sys.exit(1)
    token = result.stdout.strip()
    if not token:
        print("ERROR: gcloud returned an empty token. Re-authenticate:", file=sys.stderr)
        print("  gcloud auth application-default login \\", file=sys.stderr)
        print("    --scopes=https://www.googleapis.com/auth/webmasters,"
              "https://www.googleapis.com/auth/webmasters.readonly", file=sys.stderr)
        sys.exit(1)
    return token


def gsc_query(token, site_url, body):
    """Call the Search Analytics query endpoint."""
    encoded = urllib.parse.quote(site_url, safe="")
    url = f"https://searchconsole.googleapis.com/webmasters/v3/sites/{encoded}/searchAnalytics/query"
    data = json.dumps(body).encode()
    headers = {"Authorization": f"Bearer {token}", "Content-Type": "application/json"}
    quota_project = get_quota_project()
    if quota_project:
        headers["x-goog-user-project"] = quota_project
    req = urllib.request.Request(url, data=data, headers=headers)
    try:
        with urllib.request.urlopen(req, timeout=30) as resp:
            return json.loads(resp.read())
    except urllib.error.HTTPError as e:
        err_body = e.read().decode() if e.fp else "(no body)"
        print(f"GSC API error {e.code}: {err_body}", file=sys.stderr)
        return {"rows": []}
    except urllib.error.URLError as e:
        print(f"GSC API network error: {e.reason}", file=sys.stderr)
        return {"rows": []}


def date_range(days_ago_start, days_ago_end=3):
    """Return (start, end) date strings. GSC data typically lags ~3 days."""
    end = date.today() - timedelta(days=days_ago_end)
    start = end - timedelta(days=days_ago_start)
    return start.isoformat(), end.isoformat()


def pull_top_queries(token, site, start, end, row_limit=50):
    body = {
        "startDate": start, "endDate": end,
        "dimensions": ["query"],
        "rowLimit": row_limit,
        "orderBy": [{"fieldName": "impressions", "sortOrder": "DESCENDING"}]
    }
    data = gsc_query(token, site, body)
    rows = []
    for r in data.get("rows", []):
        rows.append({
            "query": r["keys"][0],
            "clicks": r["clicks"],
            "impressions": r["impressions"],
            "ctr": round(r["ctr"] * 100, 2),
            "position": round(r["position"], 1)
        })
    return rows


def pull_top_pages(token, site, start, end, row_limit=50):
    body = {
        "startDate": start, "endDate": end,
        "dimensions": ["page"],
        "rowLimit": row_limit,
        "orderBy": [{"fieldName": "clicks", "sortOrder": "DESCENDING"}]
    }
    data = gsc_query(token, site, body)
    rows = []
    for r in data.get("rows", []):
        rows.append({
            "page": r["keys"][0],
            "clicks": r["clicks"],
            "impressions": r["impressions"],
            "ctr": round(r["ctr"] * 100, 2),
            "position": round(r["position"], 1)
        })
    return rows


def pull_position_buckets(token, site, start, end):
    """Queries by position bucket: 1-3 (winners), 4-10 (low-hanging fruit), 11-20 (almost there), 21+."""
    body = {
        "startDate": start, "endDate": end,
        "dimensions": ["query"],
        "rowLimit": 1000,
        "orderBy": [{"fieldName": "impressions", "sortOrder": "DESCENDING"}]
    }
    data = gsc_query(token, site, body)
    buckets = {"1-3": [], "4-10": [], "11-20": [], "21+": []}
    for r in data.get("rows", []):
        pos = r["position"]
        entry = {
            "query": r["keys"][0],
            "clicks": r["clicks"],
            "impressions": r["impressions"],
            "ctr": round(r["ctr"] * 100, 2),
            "position": round(pos, 1)
        }
        if pos <= 3:
            buckets["1-3"].append(entry)
        elif pos <= 10:
            buckets["4-10"].append(entry)
        elif pos <= 20:
            buckets["11-20"].append(entry)
        else:
            buckets["21+"].append(entry)
    return buckets


def pull_period_comparison(token, site, days):
    """Compare current period vs prior period to find declines."""
    end_curr = date.today() - timedelta(days=3)
    start_curr = end_curr - timedelta(days=days)
    end_prev = start_curr - timedelta(days=1)
    start_prev = end_prev - timedelta(days=days)

    def fetch(start, end, dim):
        body = {
            "startDate": start.isoformat(), "endDate": end.isoformat(),
            "dimensions": [dim], "rowLimit": 200,
            "orderBy": [{"fieldName": "clicks", "sortOrder": "DESCENDING"}]
        }
        data = gsc_query(token, site, body)
        return {r["keys"][0]: r for r in data.get("rows", [])}

    # Pages comparison
    curr_pages = fetch(start_curr, end_curr, "page")
    prev_pages = fetch(start_prev, end_prev, "page")

    page_changes = []
    for page, curr in curr_pages.items():
        if page in prev_pages:
            prev = prev_pages[page]
            delta = curr["clicks"] - prev["clicks"]
            pct = round((delta / max(prev["clicks"], 1)) * 100, 1)
            if pct < -20 and prev["clicks"] > 10:  # Only flag meaningful drops
                page_changes.append({
                    "page": page,
                    "clicks_now": curr["clicks"],
                    "clicks_prev": prev["clicks"],
                    "click_delta": delta,
                    "absolute_click_loss": abs(delta),
                    "change_pct": pct,
                    "impressions_now": curr.get("impressions", 0),
                    "impressions_prev": prev.get("impressions", 0),
                    "impression_delta": curr.get("impressions", 0) - prev.get("impressions", 0),
                    "ctr_now": round(curr.get("ctr", 0) * 100, 2),
                    "ctr_prev": round(prev.get("ctr", 0) * 100, 2),
                    "position_now": round(curr.get("position", 0), 1),
                    "position_prev": round(prev.get("position", 0), 1),
                })
    page_changes.sort(key=lambda x: (-x.get("absolute_click_loss", 0), x["change_pct"]))

    # Queries comparison
    curr_q = fetch(start_curr, end_curr, "query")
    prev_q = fetch(start_prev, end_prev, "query")

    query_changes = []
    for q, curr in curr_q.items():
        if q in prev_q:
            prev = prev_q[q]
            delta = curr["clicks"] - prev["clicks"]
            pct = round((delta / max(prev["clicks"], 1)) * 100, 1)
            if pct < -25 and prev["clicks"] > 5:
                query_changes.append({
                    "query": q,
                    "clicks_now": curr["clicks"],
                    "clicks_prev": prev["clicks"],
                    "click_delta": delta,
                    "absolute_click_loss": abs(delta),
                    "change_pct": pct,
                    "impressions_now": curr.get("impressions", 0),
                    "impressions_prev": prev.get("impressions", 0),
                    "impression_delta": curr.get("impressions", 0) - prev.get("impressions", 0),
                    "ctr_now": round(curr.get("ctr", 0) * 100, 2),
                    "ctr_prev": round(prev.get("ctr", 0) * 100, 2),
                    "position_now": round(curr.get("position", 0), 1),
                    "position_prev": round(prev.get("position", 0), 1),
                })
    query_changes.sort(key=lambda x: (-x.get("absolute_click_loss", 0), x["change_pct"]))

    # --- Decompose each declining page into its top losing queries ---
    # For each page that lost meaningful traffic, fetch its top queries from
    # both periods so we can see which queries drove the decline.
    page_query_decompositions = []
    for pg in page_changes[:10]:
        page_url = pg["page"]
        # Fetch top queries for this page URL in both periods
        def _page_queries(start, end):
            body = {
                "startDate": start.isoformat(), "endDate": end.isoformat(),
                "dimensions": ["query"],
                "dimensionFilterGroups": [{
                    "filters": [{"dimension": "page", "operator": "equals", "expression": page_url}]
                }],
                "rowLimit": 25,
                "orderBy": [{"fieldName": "clicks", "sortOrder": "DESCENDING"}]
            }
            data = gsc_query(token, site, body)
            return {r["keys"][0]: r for r in data.get("rows", [])}

        curr_page_q = _page_queries(start_curr, end_curr)
        prev_page_q = _page_queries(start_prev, end_prev)

        # Compute query-level deltas — only for queries present in BOTH periods
        # to avoid inflating losses from truncated top-25 results.
        page_losing_queries = []
        shared_queries = set(curr_page_q.keys()) & set(prev_page_q.keys())
        for q in shared_queries:
            c = curr_page_q[q]
            p = prev_page_q[q]
            clicks_now = c.get("clicks", 0)
            clicks_prev = p.get("clicks", 0)
            click_delta = clicks_now - clicks_prev
            if click_delta >= 0:
                continue  # only losing queries matter
            pct = round((click_delta / max(clicks_prev, 1)) * 100, 1)
            page_losing_queries.append({
                "query": q,
                "clicks_now": clicks_now,
                "clicks_prev": clicks_prev,
                "click_delta": click_delta,
                "absolute_click_loss": abs(click_delta),
                "change_pct": pct,
                "impressions_now": c.get("impressions", 0),
                "impressions_prev": p.get("impressions", 0),
                "impression_delta": c.get("impressions", 0) - p.get("impressions", 0),
                "ctr_now": round(c.get("ctr", 0) * 100, 2) if c else 0,
                "ctr_prev": round(p.get("ctr", 0) * 100, 2) if p else 0,
                "position_now": round(c.get("position", 0), 1) if c else None,
                "position_prev": round(p.get("position", 0), 1) if p else None,
            })

        page_losing_queries.sort(key=lambda x: -x["absolute_click_loss"])
        if page_losing_queries:
            page_query_decompositions.append({
                "page": page_url,
                "top_losing_queries": page_losing_queries[:10],
            })

    return {
        "period": f"{start_curr.isoformat()} to {end_curr.isoformat()}",
        "prior_period": f"{start_prev.isoformat()} to {end_prev.isoformat()}",
        "declining_pages": page_changes[:20],
        "declining_queries": query_changes[:20],
        "page_query_decompositions": page_query_decompositions,
    }


def pull_summary(token, site, start, end):
    """Overall totals."""
    body = {"startDate": start, "endDate": end, "dimensions": []}
    data = gsc_query(token, site, body)
    rows = data.get("rows", [{}])
    r = rows[0] if rows else {}
    return {
        "clicks": r.get("clicks", 0),
        "impressions": r.get("impressions", 0),
        "ctr": round(r.get("ctr", 0) * 100, 2),
        "position": round(r.get("position", 0), 1)
    }


def pull_device_split(token, site, start, end):
    body = {
        "startDate": start, "endDate": end,
        "dimensions": ["device"],
        "rowLimit": 10
    }
    data = gsc_query(token, site, body)
    return [
        {"device": r["keys"][0], "clicks": r["clicks"], "impressions": r["impressions"],
         "ctr": round(r["ctr"] * 100, 2), "position": round(r["position"], 1)}
        for r in data.get("rows", [])
    ]


def pull_country_split(token, site, start, end, row_limit=20):
    """Top countries by clicks. Surfaces geo opportunities and region-specific problems."""
    body = {
        "startDate": start, "endDate": end,
        "dimensions": ["country"],
        "rowLimit": row_limit,
        "orderBy": [{"fieldName": "clicks", "sortOrder": "DESCENDING"}]
    }
    data = gsc_query(token, site, body)
    return [
        {"country": r["keys"][0], "clicks": r["clicks"], "impressions": r["impressions"],
         "ctr": round(r["ctr"] * 100, 2), "position": round(r["position"], 1)}
        for r in data.get("rows", [])
    ]


def pull_search_type_split(token, site, start, end):
    """Breakdown by search type: web, image, video, news, discover, googleNews.
    Many sites have Discover or image traffic they don't know about."""
    search_types = ["web", "image", "video", "news", "discover", "googleNews"]
    results = []
    for stype in search_types:
        body = {
            "startDate": start, "endDate": end,
            "dimensions": [],
            "type": stype
        }
        data = gsc_query(token, site, body)
        rows = data.get("rows", [{}])
        r = rows[0] if rows else {}
        clicks = r.get("clicks", 0)
        if clicks > 0:
            results.append({
                "type": stype,
                "clicks": clicks,
                "impressions": r.get("impressions", 0),
                "ctr": round(r.get("ctr", 0) * 100, 2),
                "position": round(r.get("position", 0), 1)
            })
    results.sort(key=lambda x: x["clicks"], reverse=True)
    return results


def pull_query_page_rows(token, site, start, end, row_limit=2000):
    """Pull [query, page] dimension data in one call. Expensive — reused for both
    cannibalization detection and page-level CTR gap analysis."""
    body = {
        "startDate": start, "endDate": end,
        "dimensions": ["query", "page"],
        "rowLimit": row_limit,
        "orderBy": [{"fieldName": "impressions", "sortOrder": "DESCENDING"}]
    }
    data = gsc_query(token, site, body)
    return data.get("rows", [])


def _cannibalization_winner(pages):
    """Pick the canonical winner page. Primary: best (lowest) position. Tiebreaker: most clicks."""
    return min(pages, key=lambda p: (p["position"], -p["clicks"]))


def derive_cannibalization(rows, min_impressions=100):
    """Find queries where multiple pages compete for the same keyword.
    Returns structured winner/loser scoring and recommended action.
    Input: raw rows from pull_query_page_rows."""
    query_pages = {}
    for r in rows:
        query, page = r["keys"]
        if query not in query_pages:
            query_pages[query] = []
        query_pages[query].append({
            "page": page,
            "clicks": r["clicks"],
            "impressions": r["impressions"],
            "ctr": round(r["ctr"] * 100, 2),
            "position": round(r["position"], 1)
        })

    cannibalized = []
    for query, pages in query_pages.items():
        if len(pages) > 1:
            total_impressions = sum(p["impressions"] for p in pages)
            if total_impressions >= min_impressions:
                winner = _cannibalization_winner(pages)
                winner_page = winner["page"]
                loser_pages = [p["page"] for p in pages if p["page"] != winner_page]

                # Possible SERP domination: all pages in top 5, positions within 2 of each other
                positions = [p["position"] for p in pages]
                is_domination = max(positions) <= 5 and (max(positions) - min(positions)) <= 2.0
                action = ("monitor: possible SERP domination"
                          if is_domination else
                          "consolidate: 301 redirect losers to winner or add canonical")

                # Determine the actual deciding factor for the winner reason
                all_same_position = all(p["position"] == winner["position"] for p in pages)
                winner_reason = (f"most clicks ({winner['clicks']})" if all_same_position
                                 else f"best position ({winner['position']})")

                cannibalized.append({
                    "query": query,
                    "winner_page": winner_page,
                    "winner_reason": winner_reason,
                    "loser_pages": loser_pages,
                    "recommended_action": action,
                    "competing_pages": sorted(pages, key=lambda x: x["position"]),
                    "total_impressions": total_impressions,
                    "total_clicks": sum(p["clicks"] for p in pages)
                })

    cannibalized.sort(key=lambda x: x["total_impressions"], reverse=True)
    return cannibalized[:30]


def derive_ctr_gaps_by_page(rows, min_impressions=200, max_ctr=3.0, max_position=20):
    """High-impression, low-CTR at query+page level — pinpoints exactly which page
    to rewrite the title/meta for. Input: raw rows from pull_query_page_rows."""
    gaps = []
    for r in rows:
        ctr_pct = r["ctr"] * 100
        if r["impressions"] >= min_impressions and ctr_pct < max_ctr and r["position"] <= max_position:
            gaps.append({
                "query": r["keys"][0],
                "page": r["keys"][1],
                "clicks": r["clicks"],
                "impressions": r["impressions"],
                "ctr": round(ctr_pct, 2),
                "position": round(r["position"], 1)
            })
    gaps.sort(key=lambda x: x["impressions"], reverse=True)
    return gaps[:25]


def classify_branded(query, brand_terms):
    """Return True if query contains any brand term (case-insensitive substring match)."""
    if not brand_terms:
        return False
    q = query.lower()
    return any(term.lower() in q for term in brand_terms)


def derive_branded_split(rows, brand_terms):
    """Split query+page traffic into branded vs non-branded segments.
    Input: raw rows from pull_query_page_rows. Returns None if no brand_terms provided."""
    if not brand_terms:
        return None

    # Aggregate per unique query (qp_rows can have multiple rows per query across pages)
    query_stats = {}
    for r in rows:
        query = r["keys"][0]
        imp = r["impressions"]
        if query not in query_stats:
            query_stats[query] = {
                "clicks": 0, "impressions": 0, "weighted_pos": 0.0,
                "branded": classify_branded(query, brand_terms)
            }
        query_stats[query]["clicks"] += r["clicks"]
        query_stats[query]["impressions"] += imp
        query_stats[query]["weighted_pos"] += r["position"] * imp

    branded, non_branded = [], []
    for query, s in query_stats.items():
        imp = s["impressions"]
        pos = round(s["weighted_pos"] / imp, 1) if imp > 0 else 0.0
        ctr = round(s["clicks"] / imp * 100, 2) if imp > 0 else 0.0
        entry = {"query": query, "clicks": s["clicks"], "impressions": imp,
                 "ctr": ctr, "position": pos}
        (branded if s["branded"] else non_branded).append(entry)

    def summarize(query_list):
        if not query_list:
            return {"clicks": 0, "impressions": 0, "ctr": 0.0, "position": 0.0,
                    "query_count": 0, "top_queries": []}
        total_clicks = sum(q["clicks"] for q in query_list)
        total_imp = sum(q["impressions"] for q in query_list)
        ctr = round(total_clicks / total_imp * 100, 2) if total_imp > 0 else 0.0
        weighted_pos = sum(q["position"] * q["impressions"] for q in query_list)
        pos = round(weighted_pos / total_imp, 1) if total_imp > 0 else 0.0
        top = sorted(query_list, key=lambda x: x["impressions"], reverse=True)[:20]
        return {"clicks": total_clicks, "impressions": total_imp, "ctr": ctr,
                "position": pos, "query_count": len(query_list), "top_queries": top}

    return {"branded": summarize(branded), "non_branded": summarize(non_branded)}


def _url_path(url):
    """Extract lowercase path from a full URL or path string."""
    try:
        path = urllib.parse.urlparse(url).path
    except Exception:
        path = url
    return path.rstrip("/").lower() or "/"


def cluster_page_groups(pages, groups=None):
    """Group pages by URL path pattern. Returns per-group aggregate stats sorted by clicks.
    pages: list of dicts with 'page', 'clicks', 'impressions', 'ctr', 'position' keys."""
    patterns = groups or DEFAULT_PAGE_GROUPS
    buckets = {name: {"clicks": 0, "impressions": 0, "pos_weighted": 0.0, "count": 0}
               for name, _ in patterns}
    buckets["other"] = {"clicks": 0, "impressions": 0, "pos_weighted": 0.0, "count": 0}

    for p in pages:
        path = _url_path(p["page"])
        group = "other"
        for name, pattern in patterns:
            if re.search(pattern, path):
                group = name
                break
        imp = p["impressions"]
        buckets[group]["clicks"] += p["clicks"]
        buckets[group]["impressions"] += imp
        buckets[group]["pos_weighted"] += p["position"] * imp
        buckets[group]["count"] += 1

    results = []
    for name, b in buckets.items():
        if b["count"] == 0:
            continue
        imp = b["impressions"]
        ctr = round(b["clicks"] / imp * 100, 2) if imp > 0 else 0.0
        pos = round(b["pos_weighted"] / imp, 1) if imp > 0 else 0.0
        results.append({"group": name, "page_count": b["count"],
                        "clicks": b["clicks"], "impressions": imp, "ctr": ctr, "position": pos})

    results.sort(key=lambda x: x["clicks"], reverse=True)
    return results


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--site", required=True, help="GSC property URL")
    parser.add_argument("--days", type=int, default=90, help="Days of data to pull")
    parser.add_argument("--brand-terms", default="",
                        help="Comma-separated brand names for branded vs non-branded split, e.g. 'Acme,AcmeCorp'")
    _default_out = os.path.join(tempfile.gettempdir(), f"gsc_analysis_{portable_uid()}.json")
    parser.add_argument("--output", default=_default_out, help="Output file")
    args = parser.parse_args()
    brand_terms = [t.strip() for t in args.brand_terms.split(",") if t.strip()]

    print(f"Pulling {args.days} days of GSC data for: {args.site}", file=sys.stderr)

    token = get_access_token()
    start, end = date_range(args.days)

    # All GSC calls are independent — run them concurrently to cut wall-clock
    # time from ~9 sequential round-trips down to the slowest single call.
    tasks = {
        "summary":    lambda: pull_summary(token, args.site, start, end),
        "queries":    lambda: pull_top_queries(token, args.site, start, end),
        "pages":      lambda: pull_top_pages(token, args.site, start, end),
        "buckets":    lambda: pull_position_buckets(token, args.site, start, end),
        "comparison": lambda: pull_period_comparison(token, args.site, 28),
        "devices":    lambda: pull_device_split(token, args.site, start, end),
        "countries":  lambda: pull_country_split(token, args.site, start, end),
        "search_types": lambda: pull_search_type_split(token, args.site, start, end),
        "qp_rows":    lambda: pull_query_page_rows(token, args.site, start, end),
    }

    results = {}
    print(f"Fetching {len(tasks)} data sets in parallel...", file=sys.stderr)
    with ThreadPoolExecutor(max_workers=len(tasks)) as pool:
        futures = {pool.submit(fn): name for name, fn in tasks.items()}
        for future in as_completed(futures):
            name = futures[future]
            try:
                results[name] = future.result()
                print(f"  ✓ {name}", file=sys.stderr)
            except Exception as exc:
                print(f"  ✗ {name}: {exc}", file=sys.stderr)
                results[name] = {}

    summary    = results["summary"]
    queries    = results["queries"]
    pages      = results["pages"]
    buckets    = results["buckets"]
    comparison = results["comparison"]
    devices    = results["devices"]
    countries  = results["countries"]
    search_types = results["search_types"]
    qp_rows    = results["qp_rows"]

    cannibalization   = derive_cannibalization(qp_rows)
    ctr_gaps_by_page  = derive_ctr_gaps_by_page(qp_rows)

    # High-impression, low-CTR queries (query-level, for quick title/snippet targeting)
    ctr_opportunities = [
        q for q in queries
        if q["impressions"] > 500 and q["ctr"] < 3.0 and q["position"] <= 20
    ]
    ctr_opportunities.sort(key=lambda x: x["impressions"], reverse=True)

    print("Deriving branded/non-branded split...", file=sys.stderr)
    branded_split = derive_branded_split(qp_rows, brand_terms)

    print("Clustering pages by section...", file=sys.stderr)
    page_groups = cluster_page_groups(pages)

    result = {
        "site": args.site,
        "period": {"start": start, "end": end, "days": args.days},
        "summary": summary,
        "top_queries": queries[:30],
        "top_pages": pages[:30],
        "position_buckets": {
            k: sorted(v, key=lambda x: x["impressions"], reverse=True)[:20]
            for k, v in buckets.items()
        },
        "ctr_opportunities": ctr_opportunities[:20],
        "ctr_gaps_by_page": ctr_gaps_by_page,
        "cannibalization": cannibalization,
        "comparison": comparison,
        "device_split": devices,
        "country_split": countries,
        "search_type_split": search_types,
        "branded_split": branded_split,
        "page_groups": page_groups
    }

    secure_write_json(args.output, result)

    print(f"\nDone. Results saved to {args.output}", file=sys.stderr)
    print(f"\nSummary: {summary['clicks']:,} clicks | {summary['impressions']:,} impressions | "
          f"CTR {summary['ctr']}% | Avg position {summary['position']}", file=sys.stderr)
    if cannibalization:
        print(f"Cannibalization: {len(cannibalization)} queries with competing pages found", file=sys.stderr)
    if search_types:
        type_summary = ", ".join(f"{t['type']}={t['clicks']:,}" for t in search_types)
        print(f"Search types: {type_summary}", file=sys.stderr)


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""Detect which CMS is configured via environment variables.

Checks for CMS-specific env vars and prints the detected CMS type to stdout.
Used by seo-analysis to determine which preflight/fetch scripts to run.

No external dependencies — uses only Python stdlib.

Exit codes:
  0 — CMS found (prints: wordpress | contentful | ghost | strapi)
  2 — No CMS configured
"""

import os
import sys


def load_env_file(path):
    env = {}
    try:
        with open(path) as f:
            for line in f:
                line = line.strip()
                if not line or line.startswith("#") or "=" not in line:
                    continue
                key, _, raw_value = line.partition("=")
                key = key.strip()
                value = raw_value.strip().strip('"').strip("'")
                if key:
                    env[key] = value
    except (OSError, PermissionError):
        pass
    return env


def find_and_load_env():
    env = {}
    search = os.path.abspath(os.getcwd())
    for _ in range(6):
        for name in (".env.local", ".env"):
            candidate = os.path.join(search, name)
            if os.path.isfile(candidate):
                env.update(load_env_file(candidate))
        parent = os.path.dirname(search)
        if parent == search:
            break
        search = parent
    return env


def main():
    file_env = find_and_load_env()

    def get(key):
        return os.environ.get(key) or file_env.get(key, "")

    # Check each CMS by its unique required env var.
    # Order matters when multiple are set: prefer the most recently added CMSes
    # (WordPress, Contentful, Ghost) over the original Strapi support, so that
    # users who migrate don't silently fall back to Strapi.
    if get("WP_URL"):
        print("wordpress")
        sys.exit(0)
    if get("CONTENTFUL_SPACE_ID"):
        print("contentful")
        sys.exit(0)
    if get("GHOST_URL"):
        print("ghost")
        sys.exit(0)
    if get("STRAPI_URL"):
        print("strapi")
        sys.exit(0)

    sys.exit(2)


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""Fetch published content from Contentful for SEO analysis.

Paginates through all entries of a content type, resolves linked SEO entries,
and outputs structured JSON in the normalized CMS content format.

Contentful's Delivery API only returns published content by default.
No external dependencies — uses only Python stdlib.

Usage:
  python3 fetch_contentful_content.py
  python3 fetch_contentful_content.py --content-type blogPost --output /tmp/cf.json

Environment variables (or .env / .env.local):
  CONTENTFUL_SPACE_ID        Required. Space ID from Settings → General Settings.
  CONTENTFUL_DELIVERY_TOKEN  Required. Content Delivery API access token.
  CONTENTFUL_CONTENT_TYPE    Required. API Identifier for your content type.
  CONTENTFUL_ENVIRONMENT     Optional. Environment ID (default: master).
"""

import argparse
import json
import os
import sys
import tempfile
import time
import urllib.error
import urllib.parse
import urllib.request

from _uid import portable_uid, secure_write_json


PAGE_SIZE = 1000  # Contentful max per request
_RETRY_CODES = {429, 502, 503, 504}
_CONTENTFUL_API = "https://cdn.contentful.com"


# ── Config loading ────────────────────────────────────────────────────────────

def load_env_file(path):
    env = {}
    try:
        with open(path) as f:
            for line in f:
                line = line.strip()
                if not line or line.startswith("#") or "=" not in line:
                    continue
                key, _, raw_value = line.partition("=")
                key = key.strip()
                value = raw_value.strip().strip('"').strip("'")
                if key:
                    env[key] = value
    except (OSError, PermissionError):
        pass
    return env


def find_and_load_env():
    env = {}
    search = os.path.abspath(os.getcwd())
    for _ in range(6):
        for name in (".env.local", ".env"):
            candidate = os.path.join(search, name)
            if os.path.isfile(candidate):
                env.update(load_env_file(candidate))
        parent = os.path.dirname(search)
        if parent == search:
            break
        search = parent
    return env


def get_config():
    file_env = find_and_load_env()

    def get(key):
        return os.environ.get(key) or file_env.get(key, "")

    return (
        get("CONTENTFUL_SPACE_ID"),
        get("CONTENTFUL_DELIVERY_TOKEN"),
        get("CONTENTFUL_CONTENT_TYPE"),
        get("CONTENTFUL_ENVIRONMENT") or "master",
    )


# ── HTTP helper with retry ────────────────────────────────────────────────────

def contentful_get(token, path, params=None, timeout=30, retries=3):
    full_url = f"{_CONTENTFUL_API}{path}"
    if params:
        full_url = f"{full_url}?{urllib.parse.urlencode(params)}"
    req = urllib.request.Request(
        full_url,
        headers={"Authorization": f"Bearer {token}"},
    )
    last_exc = None
    for attempt in range(retries):
        try:
            with urllib.request.urlopen(req, timeout=timeout) as resp:
                return json.loads(resp.read())
        except urllib.error.HTTPError as e:
            if e.code in _RETRY_CODES and attempt < retries - 1:
                wait = 2 ** attempt
                print(f"  HTTP {e.code} — retrying in {wait}s...", file=sys.stderr)
                time.sleep(wait)
                last_exc = e
                continue
            body = e.read().decode()[:200] if e.fp else "(no body)"
            print(f"ERROR: Contentful API {e.code}: {body}", file=sys.stderr)
            sys.exit(1)
        except urllib.error.URLError as e:
            if attempt < retries - 1:
                wait = 2 ** attempt
                print(f"  Network error ({e.reason}) — retrying in {wait}s...", file=sys.stderr)
                time.sleep(wait)
                last_exc = e
                continue
            print(f"ERROR: Network error reaching Contentful: {e.reason}", file=sys.stderr)
            sys.exit(1)
    raise last_exc


# ── SEO field extraction ──────────────────────────────────────────────────────

def extract_seo_fields(fields, includes_by_id):
    """Extract SEO meta title and description from a Contentful entry's fields.

    Tries three patterns in priority order:
    1. Linked SEO entry: fields.seo → linked entry with fields.title + fields.description
    2. Direct SEO fields: fields.seoTitle, fields.metaTitle, fields.seo_title, etc.
    3. Content fields fallback: fields.title, fields.description, fields.excerpt
    """
    meta_title = ""
    meta_description = ""
    has_meta_image = False
    has_meta_social = False

    # Pattern 1: linked SEO component entry
    seo_ref = fields.get("seo")
    if isinstance(seo_ref, dict) and seo_ref.get("sys", {}).get("type") == "Link":
        linked_id = seo_ref.get("sys", {}).get("id")
        linked = includes_by_id.get(linked_id, {})
        linked_fields = linked.get("fields", {})
        meta_title = linked_fields.get("title") or linked_fields.get("metaTitle") or ""
        meta_description = linked_fields.get("description") or linked_fields.get("metaDescription") or ""
        has_meta_image = bool(linked_fields.get("image") or linked_fields.get("ogImage"))
        has_meta_social = bool(linked_fields.get("openGraphTitle") or linked_fields.get("twitterTitle"))

    # Pattern 2: direct SEO fields
    if not meta_title:
        meta_title = (
            fields.get("seoTitle") or
            fields.get("metaTitle") or
            fields.get("seo_title") or
            fields.get("meta_title") or
            ""
        )
    if not meta_description:
        meta_description = (
            fields.get("seoDescription") or
            fields.get("metaDescription") or
            fields.get("seo_description") or
            fields.get("meta_description") or
            ""
        )

    # Do NOT fall back to fields["title"] — that's the content title, not an SEO
    # meta title override. Entries without explicit SEO titles should be flagged
    # as missing_meta_title=True so the audit surfaces them. The "title" field
    # is preserved in the normalised entry for display purposes.

    return {
        "meta_title": meta_title,
        "meta_description": meta_description,
        "has_meta_image": has_meta_image,
        "has_meta_social": has_meta_social,
    }


# ── Response normalisation ────────────────────────────────────────────────────

def normalise_entry(item, includes_by_id):
    """Normalise a Contentful entry to the shared CMS entry format."""
    sys_data = item.get("sys", {})
    fields = item.get("fields", {})

    document_id = sys_data.get("id", "")
    published_at = sys_data.get("createdAt") or ""  # Delivery API = publishedAt equivalent
    updated_at = sys_data.get("updatedAt") or ""
    locale = sys_data.get("locale") or ""

    # Slug: try slug, then title-derived, then id
    slug = fields.get("slug") or fields.get("url") or fields.get("path") or ""
    title = fields.get("title") or fields.get("name") or fields.get("heading") or ""

    seo = extract_seo_fields(fields, includes_by_id)

    meta_title_len = len(seo["meta_title"])
    meta_desc_len = len(seo["meta_description"])

    return {
        "document_id": document_id,
        "id": document_id,
        "title": title,
        "slug": slug,
        "published_at": published_at,
        "updated_at": updated_at,
        "created_at": published_at,
        "locale": locale,
        "seo": seo,
        "missing_meta_title": not seo["meta_title"],
        "missing_meta_description": not seo["meta_description"],
        "meta_title_too_long": meta_title_len > 60,
        "meta_description_too_long": meta_desc_len > 160,
        "meta_description_too_short": 0 < meta_desc_len < 70,
    }


# ── Pagination ────────────────────────────────────────────────────────────────

def fetch_all_entries(space_id, token, content_type, environment):
    """Paginate through all entries. Returns list of normalised entries."""
    path = f"/spaces/{space_id}/environments/{environment}/entries"
    all_entries = []
    skip = 0
    total = None

    while True:
        params = {
            "content_type": content_type,
            "limit": PAGE_SIZE,
            "skip": skip,
            "include": 1,  # Resolve one level of linked entries (SEO components)
            "order": "-sys.updatedAt",
        }

        page_num = skip // PAGE_SIZE + 1
        print(f"  Fetching page {page_num}...", file=sys.stderr)
        data = contentful_get(token, path, params)

        if total is None:
            total = data.get("total", 0)
            print(f"  {total} entries total", file=sys.stderr)

        items = data.get("items", [])
        if not items:
            break

        # Build a lookup of included entries (linked SEO components, etc.)
        includes = data.get("includes", {})
        includes_by_id = {}
        for entry in includes.get("Entry", []):
            entry_id = entry.get("sys", {}).get("id")
            if entry_id:
                includes_by_id[entry_id] = entry
        for asset in includes.get("Asset", []):
            asset_id = asset.get("sys", {}).get("id")
            if asset_id:
                includes_by_id[asset_id] = asset

        for item in items:
            all_entries.append(normalise_entry(item, includes_by_id))

        skip += len(items)
        print(f"  {len(all_entries)}/{total} entries fetched", file=sys.stderr)

        if skip >= total:
            break

    return all_entries


# ── SEO audit ─────────────────────────────────────────────────────────────────

def build_seo_audit(entries):
    missing_title = []
    missing_desc = []
    title_long = []
    desc_too_long = []
    desc_too_short = []
    broken_ids = set()

    for e in entries:
        broken = False
        if e["missing_meta_title"]:
            missing_title.append(e)
            broken = True
        if e["missing_meta_description"]:
            missing_desc.append(e)
            broken = True
        if e["meta_title_too_long"]:
            title_long.append(e)
            broken = True
        if e["meta_description_too_long"]:
            desc_too_long.append(e)
            broken = True
        if e["meta_description_too_short"]:
            desc_too_short.append(e)
            broken = True
        if broken:
            broken_ids.add(e["document_id"])

    return {
        "total": len(entries),
        "missing_meta_title": len(missing_title),
        "missing_meta_description": len(missing_desc),
        "meta_title_too_long": len(title_long),
        "meta_description_too_short": len(desc_too_short),
        "meta_description_too_long": len(desc_too_long),
        "complete_seo": len(entries) - len(broken_ids),
        "entries_missing_meta_title": [
            {"document_id": e["document_id"], "title": e["title"], "slug": e["slug"]}
            for e in missing_title[:20]
        ],
        "entries_missing_meta_description": [
            {"document_id": e["document_id"], "title": e["title"], "slug": e["slug"]}
            for e in missing_desc[:20]
        ],
        "entries_title_too_long": [
            {
                "document_id": e["document_id"],
                "title": e["title"],
                "meta_title": e["seo"]["meta_title"],
                "length": len(e["seo"]["meta_title"]),
            }
            for e in title_long[:20]
        ],
    }


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--content-type", help="Override CONTENTFUL_CONTENT_TYPE env var")
    parser.add_argument("--output", help="Output JSON file path (default: secure tempfile)")
    args = parser.parse_args()

    space_id, token, content_type, environment = get_config()
    if args.content_type:
        content_type = args.content_type

    if not space_id:
        print("CONTENTFUL_NOT_CONFIGURED: Set CONTENTFUL_SPACE_ID and CONTENTFUL_DELIVERY_TOKEN.", file=sys.stderr)
        sys.exit(2)
    if not token:
        print("ERROR: CONTENTFUL_DELIVERY_TOKEN is not set.", file=sys.stderr)
        sys.exit(1)
    if not content_type:
        print("ERROR: CONTENTFUL_CONTENT_TYPE is not set.", file=sys.stderr)
        sys.exit(1)

    print(f"Fetching {content_type} from Contentful space {space_id}...", file=sys.stderr)
    entries = fetch_all_entries(space_id, token, content_type, environment)

    seo_audit = build_seo_audit(entries)

    result = {
        "cms_type": "contentful",
        "cms_url": f"https://app.contentful.com/spaces/{space_id}",
        "content_type": content_type,
        "total_published": len(entries),
        "seo_audit": seo_audit,
        "entries": entries,
    }

    if args.output:
        out_path = args.output
    else:
        out_path = os.path.join(tempfile.gettempdir(), f"cms_content_{portable_uid()}.json")

    secure_write_json(out_path, result)

    print(f"\nDone. {len(entries)} entries saved to {out_path}", file=sys.stderr)
    print(
        f"SEO completeness: {seo_audit['complete_seo']}/{seo_audit['total']} entries fully complete",
        file=sys.stderr,
    )
    if seo_audit["missing_meta_title"]:
        print(f"  Missing meta title:        {seo_audit['missing_meta_title']}", file=sys.stderr)
    if seo_audit["missing_meta_description"]:
        print(f"  Missing meta description:  {seo_audit['missing_meta_description']}", file=sys.stderr)
    if seo_audit["meta_title_too_long"]:
        print(f"  Meta title too long (>60): {seo_audit['meta_title_too_long']}", file=sys.stderr)
    if seo_audit["meta_description_too_short"]:
        print(f"  Meta desc too short (<70): {seo_audit['meta_description_too_short']}", file=sys.stderr)


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""Fetch published content from Ghost for SEO analysis.

Paginates through all published posts or pages, extracts Ghost's native SEO
fields (meta_title, meta_description), and outputs structured JSON in the
normalized CMS content format consumed by seo-analysis.

Ghost has native meta_title and meta_description fields on every post/page —
no plugin needed.

Supports Ghost 4.x+ (/ghost/api/content/) and 3.x (/ghost/api/v3/content/).
No external dependencies — uses only Python stdlib.

Usage:
  python3 fetch_ghost_content.py
  python3 fetch_ghost_content.py --content-type pages --output /tmp/ghost.json

Environment variables (or .env / .env.local):
  GHOST_URL           Required. Ghost instance URL, e.g. https://myblog.ghost.io
  GHOST_CONTENT_KEY   Required. Content API key from Settings → Integrations.
  GHOST_CONTENT_TYPE  Optional. 'posts' or 'pages' (default: posts)
"""

import argparse
import ipaddress
import json
import os
import socket
import sys
import tempfile
import time
import urllib.error
import urllib.parse
import urllib.request

from _uid import portable_uid, secure_write_json


PAGE_SIZE = 100  # Ghost supports up to at least 100 per request
_RETRY_CODES = {429, 502, 503, 504}
_API_PATHS = ["/ghost/api/content", "/ghost/api/v3/content"]

_SEO_FIELDS = "id,title,slug,published_at,updated_at,meta_title,meta_description,og_image,og_title,og_description,twitter_title,twitter_description"


# ── Config loading ────────────────────────────────────────────────────────────

def load_env_file(path):
    env = {}
    try:
        with open(path) as f:
            for line in f:
                line = line.strip()
                if not line or line.startswith("#") or "=" not in line:
                    continue
                key, _, raw_value = line.partition("=")
                key = key.strip()
                value = raw_value.strip().strip('"').strip("'")
                if key:
                    env[key] = value
    except (OSError, PermissionError):
        pass
    return env


def find_and_load_env():
    env = {}
    search = os.path.abspath(os.getcwd())
    for _ in range(6):
        for name in (".env.local", ".env"):
            candidate = os.path.join(search, name)
            if os.path.isfile(candidate):
                env.update(load_env_file(candidate))
        parent = os.path.dirname(search)
        if parent == search:
            break
        search = parent
    return env


def get_config():
    file_env = find_and_load_env()

    def get(key):
        return os.environ.get(key) or file_env.get(key, "")

    return (
        get("GHOST_URL").rstrip("/"),
        get("GHOST_CONTENT_KEY"),
        get("GHOST_CONTENT_TYPE") or "posts",
    )


def _is_private_ip(ip_str):
    try:
        addr = ipaddress.ip_address(ip_str)
        return addr.is_loopback or addr.is_private or addr.is_link_local or addr.is_reserved
    except ValueError:
        return False


def validate_url(url):
    try:
        parsed = urllib.parse.urlparse(url)
    except Exception:
        print("ERROR: GHOST_URL is not a valid URL.", file=sys.stderr)
        sys.exit(1)
    if parsed.scheme not in ("http", "https"):
        print("ERROR: GHOST_URL must use http:// or https://", file=sys.stderr)
        sys.exit(1)
    hostname = parsed.hostname or ""
    if not hostname:
        print("ERROR: GHOST_URL has no hostname.", file=sys.stderr)
        sys.exit(1)
    if _is_private_ip(hostname):
        print(f"ERROR: GHOST_URL is a private/local address ('{hostname}').", file=sys.stderr)
        sys.exit(1)
    if hostname.lower() == "localhost":
        print("ERROR: GHOST_URL points to localhost.", file=sys.stderr)
        sys.exit(1)
    try:
        for info in socket.getaddrinfo(hostname, None, type=socket.SOCK_STREAM):
            if _is_private_ip(info[4][0]):
                print(f"ERROR: GHOST_URL resolves to an internal address ({info[4][0]}).", file=sys.stderr)
                sys.exit(1)
    except (socket.gaierror, OSError):
        pass


# ── HTTP helper with retry ────────────────────────────────────────────────────

def ghost_get(base_url, api_path, content_key, resource, params, timeout=30, retries=3):
    all_params = {"key": content_key, **params}
    full_url = f"{base_url}{api_path}/{resource}/?{urllib.parse.urlencode(all_params)}"
    req = urllib.request.Request(full_url, headers={"Accept-Version": "v5.0"})
    last_exc = None
    for attempt in range(retries):
        try:
            with urllib.request.urlopen(req, timeout=timeout) as resp:
                return json.loads(resp.read())
        except urllib.error.HTTPError as e:
            if e.code in _RETRY_CODES and attempt < retries - 1:
                wait = 2 ** attempt
                print(f"  HTTP {e.code} — retrying in {wait}s...", file=sys.stderr)
                time.sleep(wait)
                last_exc = e
                continue
            body = e.read().decode()[:200] if e.fp else "(no body)"
            print(f"ERROR: Ghost API {e.code}: {body}", file=sys.stderr)
            sys.exit(1)
        except urllib.error.URLError as e:
            if attempt < retries - 1:
                wait = 2 ** attempt
                print(f"  Network error ({e.reason}) — retrying in {wait}s...", file=sys.stderr)
                time.sleep(wait)
                last_exc = e
                continue
            print(f"ERROR: Network error reaching Ghost: {e.reason}", file=sys.stderr)
            sys.exit(1)
    raise last_exc


def detect_api_path(base_url, content_key, content_type):
    """Try API paths in order, return the first that works.

    Uses a direct urllib probe so that 404s on the first path don't terminate
    the process — ghost_get calls sys.exit on errors, which would prevent
    the fallback from running.
    """
    for api_path in _API_PATHS:
        try:
            url = (
                f"{base_url}{api_path}/{content_type}/?"
                f"{urllib.parse.urlencode({'key': content_key, 'limit': 1, 'fields': 'id'})}"
            )
            req = urllib.request.Request(url, headers={"Accept-Version": "v5.0"})
            with urllib.request.urlopen(req, timeout=10) as resp:
                resp.read()
                return api_path
        except urllib.error.HTTPError as e:
            if e.code == 403:
                print("ERROR: Ghost returned 403 Forbidden.", file=sys.stderr)
                print("Your GHOST_CONTENT_KEY is invalid or has been revoked.", file=sys.stderr)
                print("Regenerate it in: Settings → Integrations", file=sys.stderr)
                sys.exit(1)
            # 404 = wrong API path; try next
            continue
        except urllib.error.URLError as e:
            print(f"ERROR: Cannot reach Ghost at {base_url}", file=sys.stderr)
            print(f"  Network error: {e.reason}", file=sys.stderr)
            sys.exit(1)
    print(f"ERROR: Could not find Ghost Content API at {base_url}", file=sys.stderr)
    print("Check that GHOST_URL points to your Ghost instance.", file=sys.stderr)
    sys.exit(1)


# ── Response normalisation ────────────────────────────────────────────────────

def normalise_entry(item):
    """Normalise a Ghost post/page to the shared CMS entry format."""
    # Ghost returns null for meta_title/meta_description when not explicitly set.
    # Do NOT fall back to item["title"] here — that would hide the fact that no
    # SEO override exists, causing missing_meta_title to be False for posts that
    # have never been SEO-optimized. The entry's "title" field is preserved
    # separately for display purposes.
    meta_title = item.get("meta_title") or ""
    meta_description = item.get("meta_description") or ""
    has_meta_image = bool(item.get("og_image"))
    has_meta_social = bool(item.get("og_title") or item.get("twitter_title"))

    seo = {
        "meta_title": meta_title,
        "meta_description": meta_description,
        "has_meta_image": has_meta_image,
        "has_meta_social": has_meta_social,
    }

    meta_title_len = len(seo["meta_title"])
    meta_desc_len = len(seo["meta_description"])

    return {
        "document_id": item.get("id", ""),
        "id": item.get("id"),
        "title": item.get("title") or "",
        "slug": item.get("slug") or "",
        "published_at": item.get("published_at") or "",
        "updated_at": item.get("updated_at") or "",
        "created_at": item.get("published_at") or "",
        "locale": "",
        "seo": seo,
        "missing_meta_title": not meta_title,
        "missing_meta_description": not meta_description,
        "meta_title_too_long": meta_title_len > 60,
        "meta_description_too_long": meta_desc_len > 160,
        "meta_description_too_short": 0 < meta_desc_len < 70,
    }


# ── Pagination ────────────────────────────────────────────────────────────────

def fetch_all_entries(base_url, content_key, content_type):
    """Paginate through all published entries. Returns list of normalised entries."""
    api_path = detect_api_path(base_url, content_key, content_type)
    all_entries = []
    page = 1
    total_pages = None

    while True:
        params = {
            "limit": PAGE_SIZE,
            "page": page,
            "fields": _SEO_FIELDS,
            "order": "published_at desc",
        }

        print(f"  Fetching page {page}...", file=sys.stderr)
        data = ghost_get(base_url, api_path, content_key, content_type, params)

        items = data.get(content_type, [])
        if not items:
            break

        pagination = data.get("meta", {}).get("pagination", {})
        if total_pages is None:
            total_pages = pagination.get("pages", 1)
            total = pagination.get("total", 0)
            print(f"  {total} published {content_type} across {total_pages} page(s)", file=sys.stderr)

        for item in items:
            all_entries.append(normalise_entry(item))

        print(f"  Page {page}/{total_pages} — {len(all_entries)} fetched", file=sys.stderr)

        if page >= total_pages:
            break
        page += 1

    return all_entries


# ── SEO audit ─────────────────────────────────────────────────────────────────

def build_seo_audit(entries):
    missing_title = []
    missing_desc = []
    title_long = []
    desc_too_long = []
    desc_too_short = []
    broken_ids = set()

    for e in entries:
        broken = False
        if e["missing_meta_title"]:
            missing_title.append(e)
            broken = True
        if e["missing_meta_description"]:
            missing_desc.append(e)
            broken = True
        if e["meta_title_too_long"]:
            title_long.append(e)
            broken = True
        if e["meta_description_too_long"]:
            desc_too_long.append(e)
            broken = True
        if e["meta_description_too_short"]:
            desc_too_short.append(e)
            broken = True
        if broken:
            broken_ids.add(e["document_id"])

    return {
        "total": len(entries),
        "missing_meta_title": len(missing_title),
        "missing_meta_description": len(missing_desc),
        "meta_title_too_long": len(title_long),
        "meta_description_too_short": len(desc_too_short),
        "meta_description_too_long": len(desc_too_long),
        "complete_seo": len(entries) - len(broken_ids),
        "entries_missing_meta_title": [
            {"document_id": e["document_id"], "title": e["title"], "slug": e["slug"]}
            for e in missing_title[:20]
        ],
        "entries_missing_meta_description": [
            {"document_id": e["document_id"], "title": e["title"], "slug": e["slug"]}
            for e in missing_desc[:20]
        ],
        "entries_title_too_long": [
            {
                "document_id": e["document_id"],
                "title": e["title"],
                "meta_title": e["seo"]["meta_title"],
                "length": len(e["seo"]["meta_title"]),
            }
            for e in title_long[:20]
        ],
    }


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--content-type", help="Override GHOST_CONTENT_TYPE (posts or pages)")
    parser.add_argument("--output", help="Output JSON file path (default: secure tempfile)")
    args = parser.parse_args()

    base_url, content_key, content_type = get_config()
    if args.content_type:
        content_type = args.content_type

    if not base_url:
        print("GHOST_NOT_CONFIGURED: Set GHOST_URL and GHOST_CONTENT_KEY to enable Ghost integration.", file=sys.stderr)
        sys.exit(2)
    if not content_key:
        print("ERROR: GHOST_CONTENT_KEY is not set.", file=sys.stderr)
        sys.exit(1)

    validate_url(base_url)

    print(f"Fetching {content_type} from {base_url}...", file=sys.stderr)
    entries = fetch_all_entries(base_url, content_key, content_type)

    seo_audit = build_seo_audit(entries)

    result = {
        "cms_type": "ghost",
        "cms_url": base_url,
        "content_type": content_type,
        "total_published": len(entries),
        "seo_audit": seo_audit,
        "entries": entries,
    }

    if args.output:
        out_path = args.output
    else:
        out_path = os.path.join(tempfile.gettempdir(), f"cms_content_{portable_uid()}.json")

    secure_write_json(out_path, result)

    print(f"\nDone. {len(entries)} entries saved to {out_path}", file=sys.stderr)
    print(
        f"SEO completeness: {seo_audit['complete_seo']}/{seo_audit['total']} entries fully complete",
        file=sys.stderr,
    )
    if seo_audit["missing_meta_title"]:
        print(f"  Missing meta title:        {seo_audit['missing_meta_title']}", file=sys.stderr)
    if seo_audit["missing_meta_description"]:
        print(f"  Missing meta description:  {seo_audit['missing_meta_description']}", file=sys.stderr)
    if seo_audit["meta_title_too_long"]:
        print(f"  Meta title too long (>60): {seo_audit['meta_title_too_long']}", file=sys.stderr)
    if seo_audit["meta_description_too_short"]:
        print(f"  Meta desc too short (<70): {seo_audit['meta_description_too_short']}", file=sys.stderr)


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""Fetch published content from Strapi for SEO analysis.

Paginates through all published entries, extracts SEO fields (official
strapi-community/plugin-seo component + common custom field names),
and outputs structured JSON for the seo-analysis skill.

Supports Strapi v4 (nested attributes) and v5 (flat response).
No external dependencies — uses only Python stdlib.

Usage:
  python3 fetch_strapi_content.py
  python3 fetch_strapi_content.py --content-type blog-posts --output /tmp/strapi.json

Environment variables (or .env / .env.local):
  STRAPI_URL            Required. Base URL, e.g. https://cms.example.com
  STRAPI_API_KEY        Required. Full-access API token.
  STRAPI_CONTENT_TYPE   Optional. Plural API ID (default: articles)
  STRAPI_VERSION        Optional. Force '4' or '5' if auto-detection is wrong.
"""

import argparse
import ipaddress
import json
import os
import socket
import sys
import tempfile
import time
import urllib.error
import urllib.parse
import urllib.request

from _uid import portable_uid, secure_write_json


PAGE_SIZE = 100  # Strapi default max; configurable up to 250 in config/api.js

_RETRY_CODES = {429, 502, 503, 504}


# ── SSRF protection ───────────────────────────────────────────────────────────

def _is_private_ip(ip_str):
    try:
        addr = ipaddress.ip_address(ip_str)
        return addr.is_loopback or addr.is_private or addr.is_link_local or addr.is_reserved
    except ValueError:
        return False


def validate_url(url):
    """Block SSRF targets. Called before any HTTP requests are made."""
    try:
        parsed = urllib.parse.urlparse(url)
    except Exception:
        print("ERROR: STRAPI_URL is not a valid URL.", file=sys.stderr)
        sys.exit(1)
    if parsed.scheme not in ("http", "https"):
        print(f"ERROR: STRAPI_URL must use http:// or https://", file=sys.stderr)
        sys.exit(1)
    hostname = parsed.hostname or ""
    if not hostname:
        print("ERROR: STRAPI_URL has no hostname.", file=sys.stderr)
        sys.exit(1)
    if _is_private_ip(hostname):
        print(f"ERROR: STRAPI_URL is a private/local address ('{hostname}').", file=sys.stderr)
        sys.exit(1)
    if hostname.lower() == "localhost":
        print("ERROR: STRAPI_URL points to localhost.", file=sys.stderr)
        sys.exit(1)
    # DNS-based check (best-effort)
    try:
        for info in socket.getaddrinfo(hostname, None, type=socket.SOCK_STREAM):
            if _is_private_ip(info[4][0]):
                print(f"ERROR: STRAPI_URL resolves to an internal address ({info[4][0]}).", file=sys.stderr)
                sys.exit(1)
    except (socket.gaierror, OSError):
        pass  # non-fatal; let the request fail naturally


# ── Config loading ────────────────────────────────────────────────────────────

def load_env_file(path):
    env = {}
    try:
        with open(path) as f:
            for line in f:
                line = line.strip()
                if not line or line.startswith("#") or "=" not in line:
                    continue
                key, _, raw_value = line.partition("=")
                key = key.strip()
                value = raw_value.strip().strip('"').strip("'")
                if key:
                    env[key] = value
    except (OSError, PermissionError):
        pass
    return env


def find_and_load_env():
    env = {}
    search = os.path.abspath(os.getcwd())
    for _ in range(6):
        for name in (".env.local", ".env"):
            candidate = os.path.join(search, name)
            if os.path.isfile(candidate):
                env.update(load_env_file(candidate))
        parent = os.path.dirname(search)
        if parent == search:
            break
        search = parent
    return env


def get_config():
    file_env = find_and_load_env()

    def get(key):
        return os.environ.get(key) or file_env.get(key, "")

    return (
        get("STRAPI_URL").rstrip("/"),
        get("STRAPI_API_KEY"),
        get("STRAPI_CONTENT_TYPE") or "articles",
        get("STRAPI_VERSION"),  # "4" or "5" explicit override
    )


# ── HTTP helper with retry ────────────────────────────────────────────────────

def strapi_get(base_url, api_key, path, params, timeout=30, retries=3):
    full_url = f"{base_url}{path}?{urllib.parse.urlencode(params)}"
    req = urllib.request.Request(
        full_url,
        headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
    )
    last_exc = None
    for attempt in range(retries):
        try:
            with urllib.request.urlopen(req, timeout=timeout) as resp:
                return json.loads(resp.read())
        except urllib.error.HTTPError as e:
            if e.code in _RETRY_CODES and attempt < retries - 1:
                wait = 2 ** attempt
                print(f"  HTTP {e.code} on page — retrying in {wait}s...", file=sys.stderr)
                time.sleep(wait)
                last_exc = e
                continue
            body = e.read().decode()[:200] if e.fp else "(no body)"
            print(f"ERROR: Strapi API {e.code} on {path}: {body}", file=sys.stderr)
            sys.exit(1)
        except urllib.error.URLError as e:
            if attempt < retries - 1:
                wait = 2 ** attempt
                print(f"  Network error ({e.reason}) — retrying in {wait}s...", file=sys.stderr)
                time.sleep(wait)
                last_exc = e
                continue
            print(f"ERROR: Network error on {base_url}{path}: {e.reason}", file=sys.stderr)
            sys.exit(1)
    raise last_exc


# ── Version detection ─────────────────────────────────────────────────────────

def detect_version(data, version_hint):
    """Return 4 or 5. Explicit hint wins; otherwise infer from response structure."""
    if version_hint in ("4", "5"):
        return int(version_hint)
    items = data.get("data", [])
    if items:
        return 4 if "attributes" in items[0] else 5
    # Empty collection: fall back to v5. Set STRAPI_VERSION=4 to override.
    return 5


def publication_params(version):
    """Return the correct publication filter for the Strapi version."""
    if version == 4:
        return {"publicationState": "live"}
    return {"status": "published"}


# ── Response normalisation ────────────────────────────────────────────────────

def extract_seo_component(seo_val):
    """Normalise the SEO component from plugin-seo. Returns {} if absent."""
    if not seo_val or not isinstance(seo_val, dict):
        return {}
    # v4 may wrap under .data.attributes; v5 is already flat
    attrs = seo_val.get("attributes", seo_val)
    return {
        "meta_title": attrs.get("metaTitle") or "",
        "meta_description": attrs.get("metaDescription") or "",
        "has_meta_image": bool(attrs.get("metaImage")),
        "has_meta_social": bool(attrs.get("metaSocial")),
    }


def normalise_entry(raw, v4):
    """Return a flat dict regardless of v4/v5 response format."""
    if v4:
        document_id = str(raw.get("id", ""))
        attrs = raw.get("attributes", {})
    else:
        document_id = raw.get("documentId", "")
        attrs = raw

    item_id = raw.get("id")
    title = attrs.get("title") or attrs.get("name") or attrs.get("heading") or ""
    slug = attrs.get("slug") or attrs.get("url") or ""
    published_at = attrs.get("publishedAt") or ""
    updated_at = attrs.get("updatedAt") or ""
    created_at = attrs.get("createdAt") or ""
    locale = attrs.get("locale") or ""

    # Official plugin-seo component
    seo_component = extract_seo_component(attrs.get("seo"))

    # Fallback: common root-level custom fields when plugin is not installed
    if not seo_component.get("meta_title"):
        seo_component["meta_title"] = (
            attrs.get("metaTitle") or attrs.get("seoTitle") or
            attrs.get("meta_title") or attrs.get("seo_title") or ""
        )
    if not seo_component.get("meta_description"):
        seo_component["meta_description"] = (
            attrs.get("metaDescription") or attrs.get("seoDescription") or
            attrs.get("meta_description") or attrs.get("seo_description") or ""
        )

    # Detect which schema was used — push_strapi_seo.py needs this to write correctly
    seo_schema = "component" if attrs.get("seo") else "root_fields"

    meta_title_len = len(seo_component.get("meta_title", ""))
    meta_desc_len = len(seo_component.get("meta_description", ""))

    return {
        "document_id": document_id,
        "id": item_id,
        "title": title,
        "slug": slug,
        "published_at": published_at,
        "updated_at": updated_at,
        "created_at": created_at,
        "locale": locale,
        "seo": seo_component,
        "seo_schema": seo_schema,  # "component" | "root_fields"
        "missing_meta_title": not seo_component.get("meta_title"),
        "missing_meta_description": not seo_component.get("meta_description"),
        "meta_title_too_long": meta_title_len > 60,
        "meta_description_too_long": meta_desc_len > 160,
        "meta_description_too_short": 0 < meta_desc_len < 70,
    }


# ── Pagination ────────────────────────────────────────────────────────────────

def fetch_all_entries(base_url, api_key, content_type, version_hint):
    """Paginate through all published entries. Returns (entries, strapi_version)."""
    path = f"/api/{content_type}"
    all_entries = []

    # Version detection: skip probe when version_hint is already explicit
    if version_hint in ("4", "5"):
        strapi_version = int(version_hint)
        print(f"  Strapi v{strapi_version} (from STRAPI_VERSION)", file=sys.stderr)
    else:
        # Probe without publication filter so response structure reveals v4 vs v5
        probe_data = strapi_get(base_url, api_key, path, {"pagination[page]": 1, "pagination[pageSize]": 1})
        strapi_version = detect_version(probe_data, version_hint)
        print(f"  Strapi v{strapi_version} detected", file=sys.stderr)

    pub_filter = publication_params(strapi_version)
    page = 1

    while True:
        params = {
            "populate": "seo,seo.metaImage,seo.metaSocial",
            **pub_filter,
            "pagination[page]": page,
            "pagination[pageSize]": PAGE_SIZE,
            "sort[0]": "publishedAt:desc",
        }

        print(f"  Fetching page {page}...", file=sys.stderr)
        data = strapi_get(base_url, api_key, path, params)

        items = data.get("data", [])
        if not items:
            break

        for raw in items:
            all_entries.append(normalise_entry(raw, v4=(strapi_version == 4)))

        pagination = data.get("meta", {}).get("pagination", {})
        page_count = pagination.get("pageCount", 1)
        total = pagination.get("total", len(all_entries))

        print(
            f"  Page {page}/{page_count} — {len(all_entries)}/{total} entries fetched",
            file=sys.stderr,
        )

        if page >= page_count:
            break
        page += 1

    return all_entries, strapi_version


# ── SEO completeness audit ────────────────────────────────────────────────────

def build_seo_audit(entries):
    missing_title = []
    missing_desc = []
    title_long = []
    desc_too_long = []
    desc_too_short = []
    broken_ids = set()

    for e in entries:
        broken = False
        if e["missing_meta_title"]:
            missing_title.append(e)
            broken = True
        if e["missing_meta_description"]:
            missing_desc.append(e)
            broken = True
        if e["meta_title_too_long"]:
            title_long.append(e)
            broken = True
        if e["meta_description_too_long"]:
            desc_too_long.append(e)
            broken = True
        if e["meta_description_too_short"]:
            desc_too_short.append(e)
            broken = True
        if broken:
            broken_ids.add(e["document_id"])

    return {
        "total": len(entries),
        "missing_meta_title": len(missing_title),
        "missing_meta_description": len(missing_desc),
        "meta_title_too_long": len(title_long),
        "meta_description_too_short": len(desc_too_short),
        "meta_description_too_long": len(desc_too_long),
        "complete_seo": len(entries) - len(broken_ids),
        "entries_missing_meta_title": [
            {"document_id": e["document_id"], "title": e["title"], "slug": e["slug"]}
            for e in missing_title[:20]
        ],
        "entries_missing_meta_description": [
            {"document_id": e["document_id"], "title": e["title"], "slug": e["slug"]}
            for e in missing_desc[:20]
        ],
        "entries_title_too_long": [
            {
                "document_id": e["document_id"],
                "title": e["title"],
                "meta_title": e["seo"]["meta_title"],
                "length": len(e["seo"]["meta_title"]),
            }
            for e in title_long[:20]
        ],
    }


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--content-type", help="Override STRAPI_CONTENT_TYPE env var")
    parser.add_argument("--output", help="Output JSON file path (default: secure tempfile)")
    args = parser.parse_args()

    base_url, api_key, content_type, version_hint = get_config()
    if args.content_type:
        content_type = args.content_type

    if not base_url:
        print("STRAPI_NOT_CONFIGURED: Set STRAPI_URL and STRAPI_API_KEY to enable Strapi integration.", file=sys.stderr)
        sys.exit(2)
    if not api_key:
        print("ERROR: STRAPI_API_KEY is not set.", file=sys.stderr)
        sys.exit(1)

    validate_url(base_url)

    print(f"Fetching {content_type} from {base_url}...", file=sys.stderr)
    entries, version = fetch_all_entries(base_url, api_key, content_type, version_hint)

    seo_audit = build_seo_audit(entries)

    result = {
        "strapi_url": base_url,
        "content_type": content_type,
        "strapi_version": version,
        "total_published": len(entries),
        "seo_audit": seo_audit,
        "entries": entries,
    }

    if args.output:
        out_path = args.output
    else:
        out_path = os.path.join(tempfile.gettempdir(), f"strapi_content_{portable_uid()}.json")

    secure_write_json(out_path, result)

    print(f"\nDone. {len(entries)} entries saved to {out_path}", file=sys.stderr)
    print(
        f"SEO completeness: {seo_audit['complete_seo']}/{seo_audit['total']} entries fully complete",
        file=sys.stderr,
    )
    if seo_audit["missing_meta_title"]:
        print(f"  Missing meta title:        {seo_audit['missing_meta_title']}", file=sys.stderr)
    if seo_audit["missing_meta_description"]:
        print(f"  Missing meta description:  {seo_audit['missing_meta_description']}", file=sys.stderr)
    if seo_audit["meta_title_too_long"]:
        print(f"  Meta title too long (>60): {seo_audit['meta_title_too_long']}", file=sys.stderr)
    if seo_audit["meta_description_too_short"]:
        print(f"  Meta desc too short (<70): {seo_audit['meta_description_too_short']}", file=sys.stderr)


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""Fetch published content from WordPress REST API for SEO analysis.

Paginates through all published posts (or pages/custom types), extracts SEO
fields from Yoast SEO or RankMath if present, and outputs structured JSON
in the normalized CMS content format consumed by seo-analysis.

No external dependencies — uses only Python stdlib.

Usage:
  python3 fetch_wordpress_content.py
  python3 fetch_wordpress_content.py --content-type pages --output /tmp/wp.json

Environment variables (or .env / .env.local):
  WP_URL            Required. Base URL, e.g. https://myblog.com
  WP_USERNAME       Required. WordPress username.
  WP_APP_PASSWORD   Required. Application Password (spaces OK).
  WP_CONTENT_TYPE   Optional. REST API slug (default: posts)
"""

import argparse
import base64
import ipaddress
import json
import os
import socket
import sys
import tempfile
import time
import urllib.error
import urllib.parse
import urllib.request

from _uid import portable_uid, secure_write_json


PAGE_SIZE = 100
_RETRY_CODES = {429, 502, 503, 504}


# ── SSRF protection ───────────────────────────────────────────────────────────

def _is_private_ip(ip_str):
    try:
        addr = ipaddress.ip_address(ip_str)
        return addr.is_loopback or addr.is_private or addr.is_link_local or addr.is_reserved
    except ValueError:
        return False


def validate_url(url):
    try:
        parsed = urllib.parse.urlparse(url)
    except Exception:
        print("ERROR: WP_URL is not a valid URL.", file=sys.stderr)
        sys.exit(1)
    if parsed.scheme not in ("http", "https"):
        print("ERROR: WP_URL must use http:// or https://", file=sys.stderr)
        sys.exit(1)
    hostname = parsed.hostname or ""
    if not hostname:
        print("ERROR: WP_URL has no hostname.", file=sys.stderr)
        sys.exit(1)
    if _is_private_ip(hostname):
        print(f"ERROR: WP_URL is a private/local address ('{hostname}').", file=sys.stderr)
        sys.exit(1)
    if hostname.lower() == "localhost":
        print("ERROR: WP_URL points to localhost.", file=sys.stderr)
        sys.exit(1)
    try:
        for info in socket.getaddrinfo(hostname, None, type=socket.SOCK_STREAM):
            if _is_private_ip(info[4][0]):
                print(f"ERROR: WP_URL resolves to an internal address ({info[4][0]}).", file=sys.stderr)
                sys.exit(1)
    except (socket.gaierror, OSError):
        pass


# ── Config loading ────────────────────────────────────────────────────────────

def load_env_file(path):
    env = {}
    try:
        with open(path) as f:
            for line in f:
                line = line.strip()
                if not line or line.startswith("#") or "=" not in line:
                    continue
                key, _, raw_value = line.partition("=")
                key = key.strip()
                value = raw_value.strip().strip('"').strip("'")
                if key:
                    env[key] = value
    except (OSError, PermissionError):
        pass
    return env


def find_and_load_env():
    env = {}
    search = os.path.abspath(os.getcwd())
    for _ in range(6):
        for name in (".env.local", ".env"):
            candidate = os.path.join(search, name)
            if os.path.isfile(candidate):
                env.update(load_env_file(candidate))
        parent = os.path.dirname(search)
        if parent == search:
            break
        search = parent
    return env


def get_config():
    file_env = find_and_load_env()

    def get(key):
        return os.environ.get(key) or file_env.get(key, "")

    return (
        get("WP_URL").rstrip("/"),
        get("WP_USERNAME"),
        get("WP_APP_PASSWORD"),
        get("WP_CONTENT_TYPE") or "posts",
    )


def make_auth_header(username, app_password):
    password = app_password.replace(" ", "")
    credentials = base64.b64encode(f"{username}:{password}".encode()).decode()
    return f"Basic {credentials}"


# ── HTTP helper with retry ────────────────────────────────────────────────────

def wp_get(base_url, auth_header, path, params, timeout=30, retries=3):
    full_url = f"{base_url}{path}?{urllib.parse.urlencode(params)}"
    req = urllib.request.Request(
        full_url,
        headers={"Authorization": auth_header, "Accept": "application/json"},
    )
    last_exc = None
    for attempt in range(retries):
        try:
            with urllib.request.urlopen(req, timeout=timeout) as resp:
                body = json.loads(resp.read())
                total = int(resp.headers.get("X-WP-Total", 0))
                total_pages = int(resp.headers.get("X-WP-TotalPages", 1))
                return body, total, total_pages
        except urllib.error.HTTPError as e:
            if e.code in _RETRY_CODES and attempt < retries - 1:
                wait = 2 ** attempt
                print(f"  HTTP {e.code} — retrying in {wait}s...", file=sys.stderr)
                time.sleep(wait)
                last_exc = e
                continue
            body = e.read().decode()[:200] if e.fp else "(no body)"
            print(f"ERROR: WordPress API {e.code} on {path}: {body}", file=sys.stderr)
            sys.exit(1)
        except urllib.error.URLError as e:
            if attempt < retries - 1:
                wait = 2 ** attempt
                print(f"  Network error ({e.reason}) — retrying in {wait}s...", file=sys.stderr)
                time.sleep(wait)
                last_exc = e
                continue
            print(f"ERROR: Network error on {base_url}{path}: {e.reason}", file=sys.stderr)
            sys.exit(1)
    raise last_exc


# ── SEO field extraction ──────────────────────────────────────────────────────

def extract_seo_fields(item):
    """Extract meta title and description from WordPress post/page.

    Priority:
    1. Yoast SEO — yoast_head_json.title / yoast_head_json.description
    2. RankMath  — meta.rank_math_title / meta.rank_math_description
    3. Title     — title.rendered (raw page title, no SEO override)
    """
    meta_title = ""
    meta_description = ""
    has_meta_image = False
    has_meta_social = False

    yoast = item.get("yoast_head_json") or {}
    if yoast:
        meta_title = yoast.get("title") or ""
        meta_description = yoast.get("description") or ""
        has_meta_image = bool(yoast.get("og_image"))
        has_meta_social = bool(yoast.get("og_title") or yoast.get("twitter_title"))

    if not meta_title or not meta_description:
        meta_obj = item.get("meta") or {}
        if not meta_title:
            meta_title = (
                meta_obj.get("rank_math_title") or
                meta_obj.get("_yoast_wpseo_title") or
                ""
            )
        if not meta_description:
            meta_description = (
                meta_obj.get("rank_math_description") or
                meta_obj.get("_yoast_wpseo_metadesc") or
                ""
            )

    # Do NOT fall back to title.rendered — a post with no Yoast/RankMath title
    # should be flagged as missing_meta_title=True, not silently pass because it
    # has a page title. The entry's "title" field is preserved for display.

    return {
        "meta_title": meta_title,
        "meta_description": meta_description,
        "has_meta_image": has_meta_image,
        "has_meta_social": has_meta_social,
    }


# ── Response normalisation ────────────────────────────────────────────────────

def normalise_entry(item):
    """Normalise a WordPress REST API post/page to the shared CMS entry format."""
    seo = extract_seo_fields(item)

    title_obj = item.get("title") or {}
    slug = item.get("slug") or ""
    published_at = item.get("date_gmt") or item.get("date") or ""
    updated_at = item.get("modified_gmt") or item.get("modified") or ""

    meta_title_len = len(seo["meta_title"])
    meta_desc_len = len(seo["meta_description"])

    return {
        "document_id": str(item.get("id", "")),
        "id": item.get("id"),
        "title": title_obj.get("rendered") or "",
        "slug": slug,
        "published_at": published_at,
        "updated_at": updated_at,
        "created_at": published_at,  # WP doesn't separate creation from publish
        "locale": "",
        "seo": seo,
        "missing_meta_title": not seo["meta_title"],
        "missing_meta_description": not seo["meta_description"],
        "meta_title_too_long": meta_title_len > 60,
        "meta_description_too_long": meta_desc_len > 160,
        "meta_description_too_short": 0 < meta_desc_len < 70,
    }


# ── Pagination ────────────────────────────────────────────────────────────────

def fetch_all_entries(base_url, auth_header, content_type):
    """Paginate through all published entries. Returns list of normalised entries."""
    path = f"/wp-json/wp/v2/{content_type}"
    all_entries = []
    page = 1
    total_pages = None

    while True:
        params = {
            "status": "publish",
            "per_page": PAGE_SIZE,
            "page": page,
            # Fetch fields needed for SEO extraction plus pagination headers
            "_fields": "id,slug,title,date,date_gmt,modified,modified_gmt,yoast_head_json,meta",
        }

        print(f"  Fetching page {page}...", file=sys.stderr)
        items, total, tp = wp_get(base_url, auth_header, path, params)

        if total_pages is None:
            total_pages = tp
            print(f"  {total} published {content_type} across {total_pages} page(s)", file=sys.stderr)

        if not items:
            break

        for item in items:
            all_entries.append(normalise_entry(item))

        print(f"  Page {page}/{total_pages} — {len(all_entries)}/{total} fetched", file=sys.stderr)

        if page >= total_pages:
            break
        page += 1

    return all_entries


# ── SEO audit ─────────────────────────────────────────────────────────────────

def build_seo_audit(entries):
    missing_title = []
    missing_desc = []
    title_long = []
    desc_too_long = []
    desc_too_short = []
    broken_ids = set()

    for e in entries:
        broken = False
        if e["missing_meta_title"]:
            missing_title.append(e)
            broken = True
        if e["missing_meta_description"]:
            missing_desc.append(e)
            broken = True
        if e["meta_title_too_long"]:
            title_long.append(e)
            broken = True
        if e["meta_description_too_long"]:
            desc_too_long.append(e)
            broken = True
        if e["meta_description_too_short"]:
            desc_too_short.append(e)
            broken = True
        if broken:
            broken_ids.add(e["document_id"])

    return {
        "total": len(entries),
        "missing_meta_title": len(missing_title),
        "missing_meta_description": len(missing_desc),
        "meta_title_too_long": len(title_long),
        "meta_description_too_short": len(desc_too_short),
        "meta_description_too_long": len(desc_too_long),
        "complete_seo": len(entries) - len(broken_ids),
        "entries_missing_meta_title": [
            {"document_id": e["document_id"], "title": e["title"], "slug": e["slug"]}
            for e in missing_title[:20]
        ],
        "entries_missing_meta_description": [
            {"document_id": e["document_id"], "title": e["title"], "slug": e["slug"]}
            for e in missing_desc[:20]
        ],
        "entries_title_too_long": [
            {
                "document_id": e["document_id"],
                "title": e["title"],
                "meta_title": e["seo"]["meta_title"],
                "length": len(e["seo"]["meta_title"]),
            }
            for e in title_long[:20]
        ],
    }


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--content-type", help="Override WP_CONTENT_TYPE env var")
    parser.add_argument("--output", help="Output JSON file path (default: secure tempfile)")
    args = parser.parse_args()

    base_url, username, app_password, content_type = get_config()
    if args.content_type:
        content_type = args.content_type

    if not base_url:
        print("WP_NOT_CONFIGURED: Set WP_URL, WP_USERNAME, and WP_APP_PASSWORD to enable WordPress integration.", file=sys.stderr)
        sys.exit(2)
    if not username or not app_password:
        print("ERROR: WP_USERNAME and WP_APP_PASSWORD must both be set.", file=sys.stderr)
        sys.exit(1)

    validate_url(base_url)
    auth_header = make_auth_header(username, app_password)

    print(f"Fetching {content_type} from {base_url}...", file=sys.stderr)
    entries = fetch_all_entries(base_url, auth_header, content_type)

    seo_audit = build_seo_audit(entries)

    result = {
        "cms_type": "wordpress",
        "cms_url": base_url,
        "content_type": content_type,
        "total_published": len(entries),
        "seo_audit": seo_audit,
        "entries": entries,
    }

    if args.output:
        out_path = args.output
    else:
        out_path = os.path.join(tempfile.gettempdir(), f"cms_content_{portable_uid()}.json")

    secure_write_json(out_path, result)

    print(f"\nDone. {len(entries)} entries saved to {out_path}", file=sys.stderr)
    print(
        f"SEO completeness: {seo_audit['complete_seo']}/{seo_audit['total']} entries fully complete",
        file=sys.stderr,
    )
    if seo_audit["missing_meta_title"]:
        print(f"  Missing meta title:        {seo_audit['missing_meta_title']}", file=sys.stderr)
    if seo_audit["missing_meta_description"]:
        print(f"  Missing meta description:  {seo_audit['missing_meta_description']}", file=sys.stderr)
    if seo_audit["meta_title_too_long"]:
        print(f"  Meta title too long (>60): {seo_audit['meta_title_too_long']}", file=sys.stderr)
    if seo_audit["meta_description_too_short"]:
        print(f"  Meta desc too short (<70): {seo_audit['meta_description_too_short']}", file=sys.stderr)


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""List all Google Search Console properties for the authenticated account."""

import json
import os
import subprocess
import sys
import tempfile
import urllib.request
import urllib.error

from _gcloud import adc_access_token, adc_config_dir, gcloud_run
from _uid import portable_uid, secure_write_json


def get_quota_project():
    """Return the quota_project_id from the ADC JSON file, or None."""
    adc_dir = adc_config_dir()
    adc_path = os.path.join(adc_dir, "application_default_credentials.json")
    try:
        with open(adc_path) as f:
            data = json.load(f)
        if isinstance(data, dict):
            return data.get("quota_project_id") or None
    except (OSError, ValueError):
        pass
    return None


def get_access_token():
    try:
        result = adc_access_token()
    except FileNotFoundError:
        print("ERROR: gcloud not found. Install it and authenticate:", file=sys.stderr)
        print("  https://cloud.google.com/sdk/docs/install", file=sys.stderr)
        sys.exit(1)
    except subprocess.TimeoutExpired:
        print("ERROR: gcloud timed out after 15s. Check your network or gcloud installation.", file=sys.stderr)
        sys.exit(1)
    if result.returncode != 0:
        print("ERROR: Could not get access token. Run:", file=sys.stderr)
        print("  gcloud auth application-default login \\", file=sys.stderr)
        print("    --scopes=https://www.googleapis.com/auth/webmasters,"
              "https://www.googleapis.com/auth/webmasters.readonly", file=sys.stderr)
        sys.exit(1)
    token = result.stdout.strip()
    if not token:
        print("ERROR: gcloud returned an empty token. Re-authenticate:", file=sys.stderr)
        print("  gcloud auth application-default login \\", file=sys.stderr)
        print("    --scopes=https://www.googleapis.com/auth/webmasters,"
              "https://www.googleapis.com/auth/webmasters.readonly", file=sys.stderr)
        sys.exit(1)
    return token


def list_sites(token):
    url = "https://searchconsole.googleapis.com/webmasters/v3/sites"
    headers = {"Authorization": f"Bearer {token}"}
    quota_project = get_quota_project()
    if quota_project:
        headers["x-goog-user-project"] = quota_project
    req = urllib.request.Request(url, headers=headers)
    try:
        with urllib.request.urlopen(req, timeout=30) as resp:
            data = json.loads(resp.read())
            return data.get("siteEntry", [])
    except urllib.error.HTTPError as e:
        err_body = e.read().decode() if e.fp else "(no body)"
        print(f"ERROR {e.code}: {err_body}", file=sys.stderr)
        sys.exit(1)
    except urllib.error.URLError as e:
        print(f"ERROR: Network failure: {e.reason}", file=sys.stderr)
        sys.exit(1)


def main():
    token = get_access_token()
    sites = list_sites(token)

    if not sites:
        print("No Search Console properties found for this account.")
        print("Make sure you're logged in with the right Google account.")
        sys.exit(0)

    print(f"Found {len(sites)} Search Console properties:\n")
    for i, site in enumerate(sites, 1):
        ptype = "Domain" if site["siteUrl"].startswith("sc-domain:") else "URL-prefix"
        level = site.get("permissionLevel", "unknown")
        print(f"  {i}. {site['siteUrl']}")
        print(f"     Type: {ptype} | Permission: {level}")

    # Also output as JSON for machine parsing
    sites_path = os.path.join(tempfile.gettempdir(), f"gsc_sites_{portable_uid()}.json")
    secure_write_json(sites_path, sites)
    print(f"\n(Full list saved to {sites_path})")


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""
Pull PageSpeed Insights data for one or more URLs.
Outputs structured JSON with Core Web Vitals, performance scores,
and optimization opportunities.

Usage:
  python3 pagespeed.py --urls "https://example.com,https://example.com/about"
  python3 pagespeed.py --urls "https://example.com" --strategy mobile
  python3 pagespeed.py --urls "https://example.com" --api-key "YOUR_KEY"
"""

import argparse
import json
import os
import sys
import tempfile
import urllib.parse
import urllib.request
import urllib.error
from concurrent.futures import ThreadPoolExecutor, as_completed

from _uid import portable_uid, secure_write_json


PSI_API = "https://www.googleapis.com/pagespeedonline/v5/runPagespeed"


def _load_api_key_from_env_files():
    """Try to load PAGESPEED_API_KEY from .env files if not in environment."""
    for env_file in [".env", ".env.local", os.path.expanduser("~/.toprank/.env")]:
        if os.path.isfile(env_file):
            try:
                with open(env_file) as f:
                    for line in f:
                        line = line.strip()
                        if line.startswith("PAGESPEED_API_KEY=") and not line.startswith("#"):
                            val = line.split("=", 1)[1].strip().strip("'\"")
                            if val:
                                return val
            except OSError:
                pass
    return None


def run_pagespeed(url, strategy="mobile", api_key=None):
    """Call PageSpeed Insights API for a single URL and strategy."""
    params = {
        "url": url,
        "strategy": strategy,
        "category": "PERFORMANCE",
    }
    if api_key:
        params["key"] = api_key

    api_url = f"{PSI_API}?{urllib.parse.urlencode(params)}"
    req = urllib.request.Request(api_url)
    try:
        with urllib.request.urlopen(req, timeout=60) as resp:
            return json.loads(resp.read())
    except urllib.error.HTTPError as e:
        err_body = e.read().decode() if e.fp else "(no body)"
        print(f"PSI API error {e.code} for {url}: {err_body}", file=sys.stderr)
        if e.code == 429:
            print("", file=sys.stderr)
            print("  HINT: Quota exceeded. Fix options:", file=sys.stderr)
            print("  1. Create an API key: https://console.cloud.google.com/apis/credentials", file=sys.stderr)
            print("     Then: export PAGESPEED_API_KEY='your-key'", file=sys.stderr)
            print("  2. Enable the API: gcloud services enable pagespeedonline.googleapis.com", file=sys.stderr)
        return None
    except urllib.error.URLError as e:
        print(f"PSI API network error for {url}: {e.reason}", file=sys.stderr)
        return None
    except Exception as e:
        print(f"PSI API unexpected error for {url}: {e}", file=sys.stderr)
        return None


def extract_crux_metric(metric_data):
    """Extract Chrome UX Report (field data) metric value and rating."""
    if not metric_data:
        return None
    percentile = metric_data.get("percentile")
    category = metric_data.get("category")
    distributions = metric_data.get("distributions", [])
    return {
        "value": percentile,
        "rating": category,  # FAST, AVERAGE, SLOW
        "distributions": [
            {"min": d.get("min", 0), "max": d.get("max"), "proportion": round(d.get("proportion", 0), 4)}
            for d in distributions
        ] if distributions else []
    }


def extract_field_data(loading_experience):
    """Extract CrUX field data from the loading experience object."""
    if not loading_experience or not loading_experience.get("metrics"):
        return None

    metrics = loading_experience["metrics"]
    result = {
        "overall_category": loading_experience.get("overall_category"),
    }

    metric_map = {
        "LARGEST_CONTENTFUL_PAINT_MS": "lcp",
        "INTERACTION_TO_NEXT_PAINT": "inp",
        "CUMULATIVE_LAYOUT_SHIFT_SCORE": "cls",
        "FIRST_CONTENTFUL_PAINT_MS": "fcp",
        "EXPERIMENTAL_TIME_TO_FIRST_BYTE": "ttfb",
    }

    for api_key, short_key in metric_map.items():
        if api_key in metrics:
            result[short_key] = extract_crux_metric(metrics[api_key])

    return result


def extract_lab_data(lighthouse_result):
    """Extract Lighthouse lab data (synthetic test results)."""
    if not lighthouse_result:
        return None

    audits = lighthouse_result.get("audits", {})
    categories = lighthouse_result.get("categories", {})

    perf_score = None
    perf_category = categories.get("performance", {})
    if perf_category:
        perf_score = perf_category.get("score")
        if perf_score is not None:
            perf_score = round(perf_score * 100)

    def get_audit_value(audit_id, field="numericValue"):
        audit = audits.get(audit_id, {})
        val = audit.get(field)
        display = audit.get("displayValue", "")
        score = audit.get("score")
        return {"value": val, "display": display, "score": score}

    lab = {
        "performance_score": perf_score,
        "fcp": get_audit_value("first-contentful-paint"),
        "lcp": get_audit_value("largest-contentful-paint"),
        "cls": get_audit_value("cumulative-layout-shift"),
        "tbt": get_audit_value("total-blocking-time"),
        "si": get_audit_value("speed-index"),
        "tti": get_audit_value("interactive"),
    }

    return lab


def extract_opportunities(lighthouse_result, max_items=10):
    """Extract top optimization opportunities from Lighthouse."""
    if not lighthouse_result:
        return []

    audits = lighthouse_result.get("audits", {})
    categories = lighthouse_result.get("categories", {})
    perf = categories.get("performance", {})
    audit_refs = perf.get("auditRefs", [])

    # Collect opportunity-type audits that failed
    opportunities = []
    for ref in audit_refs:
        if ref.get("group") != "opportunity":
            continue
        audit_id = ref.get("id", "")
        audit = audits.get(audit_id, {})
        score = audit.get("score")
        if score is not None and score >= 0.9:
            continue  # already passing
        details = audit.get("details", {})
        overallSavingsMs = details.get("overallSavingsMs") or audit.get("numericValue", 0)
        if not overallSavingsMs or overallSavingsMs <= 0:
            continue
        overallSavingsBytes = details.get("overallSavingsBytes", 0)

        opportunities.append({
            "id": audit_id,
            "title": audit.get("title", audit_id),
            "description": audit.get("description", ""),
            "savings_ms": round(overallSavingsMs),
            "savings_bytes": overallSavingsBytes,
            "score": score,
            "display": audit.get("displayValue", ""),
        })

    opportunities.sort(key=lambda x: x["savings_ms"], reverse=True)
    return opportunities[:max_items]


def extract_diagnostics(lighthouse_result, max_items=10):
    """Extract diagnostic audit findings from Lighthouse."""
    if not lighthouse_result:
        return []

    audits = lighthouse_result.get("audits", {})
    categories = lighthouse_result.get("categories", {})
    perf = categories.get("performance", {})
    audit_refs = perf.get("auditRefs", [])

    diagnostics = []
    for ref in audit_refs:
        if ref.get("group") != "diagnostics":
            continue
        audit_id = ref.get("id", "")
        audit = audits.get(audit_id, {})
        score = audit.get("score")
        if score is not None and score >= 0.9:
            continue

        diagnostics.append({
            "id": audit_id,
            "title": audit.get("title", audit_id),
            "description": audit.get("description", ""),
            "display": audit.get("displayValue", ""),
            "score": score,
        })

    diagnostics.sort(key=lambda x: (x["score"] or 0))
    return diagnostics[:max_items]


def analyze_url(url, strategy, api_key):
    """Run PageSpeed analysis for a single URL and return structured results."""
    print(f"  Analyzing {url} ({strategy})...", file=sys.stderr)
    raw = run_pagespeed(url, strategy=strategy, api_key=api_key)
    if not raw:
        return {"url": url, "strategy": strategy, "error": "API call failed"}

    lighthouse = raw.get("lighthouseResult", {})

    result = {
        "url": url,
        "strategy": strategy,
        "field_data": extract_field_data(raw.get("loadingExperience")),
        "origin_field_data": extract_field_data(raw.get("originLoadingExperience")),
        "lab_data": extract_lab_data(lighthouse),
        "opportunities": extract_opportunities(lighthouse),
        "diagnostics": extract_diagnostics(lighthouse),
    }

    score = (result["lab_data"] or {}).get("performance_score")
    if score is not None:
        print(f"  \u2713 {url}: score {score}/100", file=sys.stderr)
    else:
        print(f"  \u2713 {url}: done (no score)", file=sys.stderr)

    return result


def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--urls", required=True,
                        help="Comma-separated URLs to analyze")
    parser.add_argument("--strategy", default="mobile",
                        choices=["mobile", "desktop"],
                        help="Test strategy (default: mobile)")
    parser.add_argument("--both-strategies", action="store_true",
                        help="Run both mobile and desktop")
    parser.add_argument("--api-key", default=os.environ.get("PAGESPEED_API_KEY", ""),
                        help="Google API key (optional, increases rate limits)")
    _default_out = os.path.join(tempfile.gettempdir(), f"pagespeed_{portable_uid()}.json")
    parser.add_argument("--output", default=_default_out, help="Output file")
    args = parser.parse_args()

    urls = [u.strip() for u in args.urls.split(",") if u.strip()]
    if not urls:
        print("ERROR: No URLs provided.", file=sys.stderr)
        sys.exit(1)

    strategies = ["mobile", "desktop"] if args.both_strategies else [args.strategy]
    api_key = args.api_key or _load_api_key_from_env_files()
    if api_key:
        print("Using PageSpeed API key.", file=sys.stderr)
    else:
        print("No API key found. Requests may hit quota limits.", file=sys.stderr)
        print("  Set PAGESPEED_API_KEY or add to ~/.toprank/.env", file=sys.stderr)

    tasks = []
    for url in urls:
        for strategy in strategies:
            tasks.append((url, strategy))

    print(f"Running PageSpeed analysis for {len(urls)} URL(s), "
          f"strategy: {', '.join(strategies)}...", file=sys.stderr)

    results = []
    # PSI API has rate limits; use modest concurrency
    max_workers = min(len(tasks), 4)
    with ThreadPoolExecutor(max_workers=max_workers) as pool:
        futures = {
            pool.submit(analyze_url, url, strategy, api_key): (url, strategy)
            for url, strategy in tasks
        }
        for future in as_completed(futures):
            url, strategy = futures[future]
            try:
                result = future.result()
                results.append(result)
            except Exception as exc:
                print(f"  \u2717 {url} ({strategy}): {exc}", file=sys.stderr)
                results.append({"url": url, "strategy": strategy, "error": str(exc)})

    # Sort results: by URL then strategy for consistent output
    results.sort(key=lambda x: (x["url"], x.get("strategy", "")))

    # Build summary
    scored = [r for r in results if (r.get("lab_data") or {}).get("performance_score") is not None]
    summary = {
        "urls_tested": len(urls),
        "strategies": strategies,
        "results_count": len(results),
        "avg_performance_score": (
            round(sum(r["lab_data"]["performance_score"] for r in scored) / len(scored))
            if scored else None
        ),
    }

    output = {
        "summary": summary,
        "results": results,
    }

    secure_write_json(args.output, output)

    print(f"\nDone. Results saved to {args.output}", file=sys.stderr)
    if summary["avg_performance_score"] is not None:
        print(f"Average performance score: {summary['avg_performance_score']}/100", file=sys.stderr)


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""Pre-flight check for seo-analysis skill.

Verifies gcloud is installed, a GCP project is configured, the Search Console
API is enabled, and Google ADC credentials are configured with the correct scope.
No external dependencies — uses only Python stdlib and the gcloud CLI.

Exit codes:
  0 — all dependencies ready
  1 — unrecoverable error (gcloud missing, auth failed, etc.)
"""

import json
import os
import platform
import shutil
import subprocess
import sys
import urllib.request

from _gcloud import adc_access_token, adc_config_dir, gcloud_run


def check_python_version():
    if sys.version_info < (3, 8):
        print(f"ERROR: Python 3.8+ required (you have {sys.version.split()[0]})", file=sys.stderr)
        print("  Upgrade: https://python.org/downloads", file=sys.stderr)
        sys.exit(1)


def check_gcloud():
    """Verify gcloud CLI is installed; print OS-specific install instructions if not."""
    if shutil.which("gcloud"):
        return

    system = platform.system()
    print("ERROR: gcloud CLI not found.", file=sys.stderr)
    print("", file=sys.stderr)

    if system == "Darwin":
        print("Install with Homebrew (recommended):", file=sys.stderr)
        print("  brew install google-cloud-sdk", file=sys.stderr)
        print("", file=sys.stderr)
        print("Or download the installer:", file=sys.stderr)
        print("  https://cloud.google.com/sdk/docs/install#mac", file=sys.stderr)
    elif system == "Linux":
        distro = ""
        try:
            with open("/etc/os-release") as f:
                distro = f.read().lower()
        except FileNotFoundError:
            pass

        if "ubuntu" in distro or "debian" in distro:
            print("Install with apt:", file=sys.stderr)
            print("  sudo apt-get install google-cloud-cli", file=sys.stderr)
        elif "fedora" in distro or "rhel" in distro or "centos" in distro:
            print("Install with dnf:", file=sys.stderr)
            print("  sudo dnf install google-cloud-cli", file=sys.stderr)
        else:
            print("Install via curl:", file=sys.stderr)
            print("  curl https://sdk.cloud.google.com | bash", file=sys.stderr)
        print("", file=sys.stderr)
        print("Full guide: https://cloud.google.com/sdk/docs/install#linux", file=sys.stderr)
    elif system == "Windows":
        print("Install with winget:", file=sys.stderr)
        print("  winget install Google.CloudSDK", file=sys.stderr)
        print("", file=sys.stderr)
        print("Or download the installer:", file=sys.stderr)
        print("  https://dl.google.com/dl/cloudsdk/channels/rapid/GoogleCloudSDKInstaller.exe", file=sys.stderr)
    else:
        print("See: https://cloud.google.com/sdk/docs/install", file=sys.stderr)

    sys.exit(1)


def check_gcloud_project():
    """Ensure gcloud has an active project. Run gcloud init if not."""
    try:
        result = gcloud_run(
            ["gcloud", "config", "get-value", "project"],
            capture_output=True, text=True, timeout=15,
        )
    except subprocess.TimeoutExpired:
        print("ERROR: gcloud timed out. Check your network.", file=sys.stderr)
        sys.exit(1)

    project = result.stdout.strip()
    # gcloud prints "(unset)" to stderr when no project is set
    if project and project != "(unset)":
        print(f"GCP project: {project}", file=sys.stderr)
        return

    # No project configured — first-time gcloud user
    print("No GCP project configured.", file=sys.stderr)

    if not sys.stdin.isatty():
        print("Run in an interactive terminal:", file=sys.stderr)
        print("  gcloud init", file=sys.stderr)
        print("This will create or select a Google Cloud project.", file=sys.stderr)
        sys.exit(1)

    print("Running 'gcloud init' to set up your project...", file=sys.stderr)
    print("", file=sys.stderr)
    init_result = gcloud_run(["gcloud", "init"])
    if init_result.returncode != 0:
        print("", file=sys.stderr)
        print("ERROR: gcloud init failed or was cancelled.", file=sys.stderr)
        print("Run 'gcloud init' manually and try again.", file=sys.stderr)
        sys.exit(1)

    # Verify project was set
    verify = gcloud_run(
        ["gcloud", "config", "get-value", "project"],
        capture_output=True, text=True, timeout=15,
    )
    project = verify.stdout.strip()
    if not project or project == "(unset)":
        print("ERROR: No project selected during gcloud init.", file=sys.stderr)
        print("Run 'gcloud init' again and select or create a project.", file=sys.stderr)
        sys.exit(1)

    print(f"GCP project: {project}", file=sys.stderr)


def check_search_console_api():
    """Ensure the Search Console API is enabled in the active project."""
    try:
        result = gcloud_run(
            ["gcloud", "services", "list", "--enabled",
             "--filter=config.name:searchconsole.googleapis.com",
             "--format=value(config.name)"],
            capture_output=True, text=True, timeout=30,
        )
    except subprocess.TimeoutExpired:
        print("WARNING: Timed out checking Search Console API status.", file=sys.stderr)
        print("If you get API errors later, run:", file=sys.stderr)
        print("  gcloud services enable searchconsole.googleapis.com", file=sys.stderr)
        return  # non-fatal — let it fail later with a clear error

    if "searchconsole.googleapis.com" in result.stdout:
        print("Search Console API: enabled", file=sys.stderr)
        return

    # API not enabled — try to enable it automatically
    print("Search Console API is not enabled. Enabling it now...", file=sys.stderr)
    enable_result = gcloud_run(
        ["gcloud", "services", "enable", "searchconsole.googleapis.com"],
        capture_output=True, text=True, timeout=60,
    )
    if enable_result.returncode == 0:
        print("Search Console API: enabled", file=sys.stderr)
        return

    # Enable failed — print manual instructions
    print("", file=sys.stderr)
    print("ERROR: Could not enable the Search Console API automatically.", file=sys.stderr)
    stderr_msg = enable_result.stderr.strip()
    if stderr_msg:
        print(f"  Reason: {stderr_msg}", file=sys.stderr)
    print("", file=sys.stderr)
    print("Enable it manually:", file=sys.stderr)
    print("  gcloud services enable searchconsole.googleapis.com", file=sys.stderr)
    print("", file=sys.stderr)
    print("Or via the Cloud Console:", file=sys.stderr)
    print("  https://console.cloud.google.com/apis/library/searchconsole.googleapis.com", file=sys.stderr)
    sys.exit(1)


_GSC_SCOPES = (
    "https://www.googleapis.com/auth/webmasters",
    "https://www.googleapis.com/auth/webmasters.readonly",
)
_GSC_SCOPES_ARG = ",".join(_GSC_SCOPES)


def _token_has_gsc_scope(token):
    """Return True if the token includes at least one Search Console scope.

    Calls the Google tokeninfo endpoint to inspect the granted scopes.
    Returns None if the check cannot be completed (network error, etc.).
    """
    try:
        url = f"https://oauth2.googleapis.com/tokeninfo?access_token={token}"
        with urllib.request.urlopen(url, timeout=10) as resp:
            data = json.loads(resp.read().decode())
        granted = set(data.get("scope", "").split())
        return bool(granted & set(_GSC_SCOPES))
    except Exception:
        return None  # can't verify — caller decides what to do


def check_adc_credentials():
    """Check ADC credentials exist with correct Search Console scope.

    If credentials are missing or have the wrong scope (e.g. cloud-platform from
    a prior bare `gcloud auth application-default login`), re-authenticates with
    the correct scopes.
    """
    try:
        result = adc_access_token()
    except subprocess.TimeoutExpired:
        print("ERROR: gcloud timed out checking credentials. Check your network.", file=sys.stderr)
        sys.exit(1)

    needs_auth = True
    if result.returncode == 0 and result.stdout.strip():
        token = result.stdout.strip()
        has_scope = _token_has_gsc_scope(token)
        if has_scope is True:
            print("Google credentials: OK (Search Console scope confirmed)", file=sys.stderr)
            return
        elif has_scope is None:
            # tokeninfo unreachable — assume credentials are fine, let the API call fail
            print("Google credentials: found (scope check skipped — no network)", file=sys.stderr)
            return
        else:
            # Credentials exist but have the wrong scope (e.g. cloud-platform)
            print("WARNING: Existing credentials are missing the Search Console scope.", file=sys.stderr)
            print("  This happens when `gcloud auth application-default login` was run", file=sys.stderr)
            print("  without --scopes, granting broad cloud-platform access instead.", file=sys.stderr)
            print("  Re-authenticating with the correct scope now...", file=sys.stderr)
            print("", file=sys.stderr)
            needs_auth = True

    if needs_auth and not sys.stdin.isatty():
        print("ERROR: No Application Default Credentials with Search Console scope.", file=sys.stderr)
        print("Run in an interactive terminal:", file=sys.stderr)
        print("  gcloud auth application-default login \\", file=sys.stderr)
        print(f"    --scopes={_GSC_SCOPES_ARG}", file=sys.stderr)
        sys.exit(1)

    print("Opening browser for Google authentication...", file=sys.stderr)
    print("(Log in with the Google account that has access to Search Console.)", file=sys.stderr)
    print("", file=sys.stderr)
    auth_result = gcloud_run(
        ["gcloud", "auth", "application-default", "login",
         f"--scopes={_GSC_SCOPES_ARG}"],
    )
    if auth_result.returncode != 0:
        print("", file=sys.stderr)
        print("ERROR: Authentication failed or was cancelled.", file=sys.stderr)
        print("Run this manually and try again:", file=sys.stderr)
        print("  gcloud auth application-default login \\", file=sys.stderr)
        print(f"    --scopes={_GSC_SCOPES_ARG}", file=sys.stderr)
        sys.exit(1)
    print("Authentication successful.", file=sys.stderr)


def _get_adc_quota_project():
    """Return the quota_project_id from the ADC JSON file, or None if absent."""
    adc_dir = adc_config_dir()
    adc_path = os.path.join(adc_dir, "application_default_credentials.json")
    if not os.path.isfile(adc_path):
        return None
    try:
        with open(adc_path) as f:
            data = json.load(f)
        if not isinstance(data, dict):
            return None
        return data.get("quota_project_id") or None
    except (OSError, ValueError):
        return None


def check_quota_project():
    """Ensure ADC has a quota project set; auto-configure it from the active gcloud project.

    Without a quota project, user-credential ADC calls to Search Console return 403:
      "The searchconsole.googleapis.com API requires a quota project."
    Fix: gcloud auth application-default set-quota-project PROJECT_ID

    Returns True if quota project is confirmed set, False if it could not be configured.
    """
    if _get_adc_quota_project():
        return True  # already configured

    # Look up the active gcloud project
    try:
        result = gcloud_run(
            ["gcloud", "config", "get-value", "project"],
            capture_output=True, text=True, timeout=15,
        )
    except subprocess.TimeoutExpired:
        print("WARNING: gcloud timed out getting project for quota setup.", file=sys.stderr)
        return False

    project = result.stdout.strip() if result.returncode == 0 else ""
    if not project or project == "(unset)":
        print("WARNING: Cannot set quota project — no active GCP project.", file=sys.stderr)
        print("  Run: gcloud auth application-default set-quota-project YOUR_PROJECT_ID", file=sys.stderr)
        return False

    print(f"Setting ADC quota project to '{project}'...", file=sys.stderr)
    try:
        set_result = gcloud_run(
            ["gcloud", "auth", "application-default", "set-quota-project", project],
            capture_output=True, text=True, timeout=30,
        )
    except subprocess.TimeoutExpired:
        print("WARNING: gcloud timed out setting quota project.", file=sys.stderr)
        print(f"  Run manually: gcloud auth application-default set-quota-project '{project}'", file=sys.stderr)
        return False
    if set_result.returncode == 0:
        print(f"ADC quota project: {project}", file=sys.stderr)
        return True
    else:
        print("WARNING: Could not set quota project automatically.", file=sys.stderr)
        print(f"  Run manually: gcloud auth application-default set-quota-project '{project}'", file=sys.stderr)
        stderr_msg = set_result.stderr.strip()
        if stderr_msg:
            print(f"  Reason: {stderr_msg}", file=sys.stderr)
        return False


def check_pagespeed_api():
    """Ensure the PageSpeed Insights API is enabled in the active project.
    Non-fatal — PageSpeed is optional but recommended."""
    try:
        result = gcloud_run(
            ["gcloud", "services", "list", "--enabled",
             "--filter=config.name:pagespeedonline.googleapis.com",
             "--format=value(config.name)"],
            capture_output=True, text=True, timeout=30,
        )
    except subprocess.TimeoutExpired:
        print("WARNING: Timed out checking PageSpeed API status.", file=sys.stderr)
        return

    if result.returncode != 0:
        print("WARNING: Could not check PageSpeed API status (gcloud error).", file=sys.stderr)
        print("  PageSpeed analysis will still work with an API key.", file=sys.stderr)
        return

    if "pagespeedonline.googleapis.com" in result.stdout:
        print("PageSpeed Insights API: enabled", file=sys.stderr)
        return

    print("PageSpeed Insights API is not enabled. Enabling it now...", file=sys.stderr)
    try:
        enable_result = gcloud_run(
            ["gcloud", "services", "enable", "pagespeedonline.googleapis.com"],
            capture_output=True, text=True, timeout=60,
        )
    except subprocess.TimeoutExpired:
        print("WARNING: Timed out enabling PageSpeed API.", file=sys.stderr)
        print("  To enable manually: gcloud services enable pagespeedonline.googleapis.com", file=sys.stderr)
        return
    if enable_result.returncode == 0:
        print("PageSpeed Insights API: enabled", file=sys.stderr)
    else:
        print("WARNING: Could not enable PageSpeed Insights API.", file=sys.stderr)
        print("  PageSpeed analysis will still work with an API key.", file=sys.stderr)
        print("  To enable manually: gcloud services enable pagespeedonline.googleapis.com", file=sys.stderr)


def check_pagespeed_api_key():
    """Check if a PageSpeed API key is available. Suggest creating one if not.
    The PSI API requires an API key for reliable access (without one, requests
    may be rejected with quota errors)."""
    if os.environ.get("PAGESPEED_API_KEY"):
        print("PageSpeed API key: found in environment", file=sys.stderr)
        return

    # Check .env and .env.local files in common locations
    for env_file in [".env", ".env.local", os.path.expanduser("~/.toprank/.env")]:
        if os.path.isfile(env_file):
            try:
                with open(env_file) as f:
                    for line in f:
                        stripped = line.strip()
                        if stripped.startswith("PAGESPEED_API_KEY=") and not stripped.startswith("#"):
                            val = stripped.split("=", 1)[1].strip().strip("'\"")
                            if val:
                                print(f"PageSpeed API key: found in {env_file}", file=sys.stderr)
                                return
            except OSError:
                pass

    print("", file=sys.stderr)
    print("NOTE: No PageSpeed API key found.", file=sys.stderr)
    print("  The PageSpeed Insights API works best with an API key.", file=sys.stderr)
    print("  Without one, requests may hit quota limits.", file=sys.stderr)
    print("", file=sys.stderr)
    print("  To create one:", file=sys.stderr)
    print("  1. Go to https://console.cloud.google.com/apis/credentials", file=sys.stderr)
    print("  2. Click 'Create Credentials' > 'API key'", file=sys.stderr)
    print("  3. Set: export PAGESPEED_API_KEY='your-key-here'", file=sys.stderr)
    print("     Or add to ~/.toprank/.env: PAGESPEED_API_KEY=your-key-here", file=sys.stderr)
    print("", file=sys.stderr)


def main():
    check_python_version()
    check_gcloud()
    check_gcloud_project()
    check_search_console_api()
    check_pagespeed_api()
    check_adc_credentials()
    quota_ok = check_quota_project()
    check_pagespeed_api_key()
    if quota_ok:
        print("OK: All dependencies ready.", file=sys.stderr)
    else:
        print("OK: All dependencies ready (quota project not confirmed — GSC may return 403).", file=sys.stderr)


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""Pre-flight check for Contentful Content Delivery API integration.

Verifies CONTENTFUL_SPACE_ID, CONTENTFUL_DELIVERY_TOKEN, and
CONTENTFUL_CONTENT_TYPE are set, tests the Delivery API, and reports the
total published entry count for the configured content type.

No external dependencies — uses only Python stdlib.

Exit codes:
  0 — Contentful connection ready
  1 — unrecoverable error (missing config, auth failed, wrong content type, etc.)
  2 — Contentful not configured (CONTENTFUL_SPACE_ID not set) — non-fatal, caller skips
"""

import json
import os
import sys
import time
import urllib.error
import urllib.parse
import urllib.request


_RETRY_CODES = {429, 502, 503, 504}
_CONTENTFUL_API = "https://cdn.contentful.com"


# ── Config loading ────────────────────────────────────────────────────────────

def load_env_file(path):
    env = {}
    try:
        with open(path) as f:
            for line in f:
                line = line.strip()
                if not line or line.startswith("#") or "=" not in line:
                    continue
                key, _, raw_value = line.partition("=")
                key = key.strip()
                value = raw_value.strip().strip('"').strip("'")
                if key:
                    env[key] = value
    except (OSError, PermissionError):
        pass
    return env


def find_and_load_env():
    env = {}
    search = os.path.abspath(os.getcwd())
    for _ in range(6):
        for name in (".env.local", ".env"):
            candidate = os.path.join(search, name)
            if os.path.isfile(candidate):
                env.update(load_env_file(candidate))
        parent = os.path.dirname(search)
        if parent == search:
            break
        search = parent
    return env


def get_config():
    file_env = find_and_load_env()

    def get(key):
        return os.environ.get(key) or file_env.get(key, "")

    return (
        get("CONTENTFUL_SPACE_ID"),
        get("CONTENTFUL_DELIVERY_TOKEN"),
        get("CONTENTFUL_CONTENT_TYPE"),
        get("CONTENTFUL_ENVIRONMENT") or "master",
    )


# ── HTTP helper with retry ────────────────────────────────────────────────────

def contentful_get(token, path, params=None, timeout=15, retries=3):
    """GET {_CONTENTFUL_API}{path}?{params} with Bearer auth."""
    full_url = f"{_CONTENTFUL_API}{path}"
    if params:
        full_url = f"{full_url}?{urllib.parse.urlencode(params)}"
    req = urllib.request.Request(
        full_url,
        headers={"Authorization": f"Bearer {token}"},
    )
    last_exc = None
    for attempt in range(retries):
        try:
            with urllib.request.urlopen(req, timeout=timeout) as resp:
                return json.loads(resp.read())
        except urllib.error.HTTPError as e:
            if e.code in _RETRY_CODES and attempt < retries - 1:
                wait = 2 ** attempt
                print(f"  HTTP {e.code} — retrying in {wait}s...", file=sys.stderr)
                time.sleep(wait)
                last_exc = e
                continue
            raise
        except urllib.error.URLError as e:
            if attempt < retries - 1:
                wait = 2 ** attempt
                print(f"  Network error ({e.reason}) — retrying in {wait}s...", file=sys.stderr)
                time.sleep(wait)
                last_exc = e
                continue
            raise
    raise last_exc


# ── Validation checks ─────────────────────────────────────────────────────────

def check_config(space_id, token, content_type):
    if not space_id:
        print("CONTENTFUL_NOT_CONFIGURED: Set CONTENTFUL_SPACE_ID and CONTENTFUL_DELIVERY_TOKEN to enable Contentful integration.", file=sys.stderr)
        sys.exit(2)

    if not token:
        print("ERROR: CONTENTFUL_DELIVERY_TOKEN is not set.", file=sys.stderr)
        print("", file=sys.stderr)
        print("Create a Content Delivery API key in Contentful:", file=sys.stderr)
        print("  Settings → API Keys → Add API Key → copy Content Delivery API - access token", file=sys.stderr)
        print("", file=sys.stderr)
        print("Then set it in .env.local:", file=sys.stderr)
        print("  CONTENTFUL_DELIVERY_TOKEN=your_token_here", file=sys.stderr)
        sys.exit(1)

    if not content_type:
        print("ERROR: CONTENTFUL_CONTENT_TYPE is not set.", file=sys.stderr)
        print("Set the content type API ID in .env.local:", file=sys.stderr)
        print("  CONTENTFUL_CONTENT_TYPE=blogPost", file=sys.stderr)
        print("Find it in: Content model → [your type] → API Identifier", file=sys.stderr)
        sys.exit(1)

    print(f"Contentful space: {space_id}", file=sys.stderr)


def check_connectivity(space_id, token, content_type, environment):
    """Probe the space and content type, return total entry count."""
    # First verify auth by fetching the space
    try:
        contentful_get(token, f"/spaces/{space_id}")
    except urllib.error.HTTPError as e:
        if e.code == 401:
            print("ERROR: Contentful returned 401 Unauthorized.", file=sys.stderr)
            print("Your CONTENTFUL_DELIVERY_TOKEN is invalid or expired.", file=sys.stderr)
            print("Regenerate it in: Settings → API Keys", file=sys.stderr)
            sys.exit(1)
        if e.code == 404:
            print(f"ERROR: Space '{space_id}' not found (404).", file=sys.stderr)
            print("Check CONTENTFUL_SPACE_ID in Settings → General Settings.", file=sys.stderr)
            sys.exit(1)
        # Non-fatal for space check — content type check below will surface the real error
    except urllib.error.URLError as e:
        print(f"ERROR: Cannot reach Contentful API.", file=sys.stderr)
        print(f"  Network error: {e.reason}", file=sys.stderr)
        sys.exit(1)

    # Verify content type and get count
    try:
        data = contentful_get(token, f"/spaces/{space_id}/environments/{environment}/entries", {
            "content_type": content_type,
            "limit": 1,
        })
    except urllib.error.HTTPError as e:
        body = e.read().decode()[:300] if e.fp else "(no body)"
        if e.code == 400:
            print(f"ERROR: Contentful returned 400 Bad Request.", file=sys.stderr)
            print(f"Content type '{content_type}' may not exist.", file=sys.stderr)
            print("Check CONTENTFUL_CONTENT_TYPE — find it in: Content model → [your type] → API Identifier", file=sys.stderr)
            sys.exit(1)
        if e.code == 404:
            print(f"ERROR: Environment '{environment}' not found.", file=sys.stderr)
            print(f"Check CONTENTFUL_ENVIRONMENT (default: master).", file=sys.stderr)
            sys.exit(1)
        print(f"ERROR: Contentful API error {e.code}. Response: {body}", file=sys.stderr)
        sys.exit(1)

    total = data.get("total", 0)
    print(f"Contentful ready | space: {space_id} | env: {environment} | content type: {content_type} | {total} entries", file=sys.stderr)
    return total


def main():
    space_id, token, content_type, environment = get_config()
    check_config(space_id, token, content_type)
    total = check_connectivity(space_id, token, content_type, environment)
    if total == 0:
        print(f"WARNING: No entries found in content type '{content_type}'.", file=sys.stderr)
    print(f"OK: Contentful ready ({content_type}, {total} entries)", file=sys.stderr)


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""Pre-flight check for Ghost Content API integration.

Verifies GHOST_URL and GHOST_CONTENT_KEY are set, tests connectivity
via the Ghost Content API, and reports the total published entry count.

Supports Ghost 4.x+ (/ghost/api/content/) and 3.x (/ghost/api/v3/content/).
No external dependencies — uses only Python stdlib.

Exit codes:
  0 — Ghost connection ready
  1 — unrecoverable error (missing config, auth failed, etc.)
  2 — Ghost not configured (GHOST_URL not set) — non-fatal, caller skips
"""

import ipaddress
import json
import os
import socket
import sys
import time
import urllib.error
import urllib.parse
import urllib.request


_RETRY_CODES = {429, 502, 503, 504}
# API path candidates: try newest first, fall back to v3
_API_PATHS = ["/ghost/api/content", "/ghost/api/v3/content"]


# ── Config loading ────────────────────────────────────────────────────────────

def load_env_file(path):
    env = {}
    try:
        with open(path) as f:
            for line in f:
                line = line.strip()
                if not line or line.startswith("#") or "=" not in line:
                    continue
                key, _, raw_value = line.partition("=")
                key = key.strip()
                value = raw_value.strip().strip('"').strip("'")
                if key:
                    env[key] = value
    except (OSError, PermissionError):
        pass
    return env


def find_and_load_env():
    env = {}
    search = os.path.abspath(os.getcwd())
    for _ in range(6):
        for name in (".env.local", ".env"):
            candidate = os.path.join(search, name)
            if os.path.isfile(candidate):
                env.update(load_env_file(candidate))
        parent = os.path.dirname(search)
        if parent == search:
            break
        search = parent
    return env


def get_config():
    file_env = find_and_load_env()

    def get(key):
        return os.environ.get(key) or file_env.get(key, "")

    url = get("GHOST_URL").rstrip("/")
    content_key = get("GHOST_CONTENT_KEY")
    content_type = get("GHOST_CONTENT_TYPE") or "posts"
    return url, content_key, content_type


# ── Security: SSRF protection ─────────────────────────────────────────────────

def _is_private_ip(ip_str):
    try:
        addr = ipaddress.ip_address(ip_str)
        return addr.is_loopback or addr.is_private or addr.is_link_local or addr.is_reserved
    except ValueError:
        return False


def validate_url(url):
    try:
        parsed = urllib.parse.urlparse(url)
    except Exception:
        print("ERROR: GHOST_URL is not a valid URL.", file=sys.stderr)
        sys.exit(1)

    if parsed.scheme not in ("http", "https"):
        print(f"ERROR: GHOST_URL must use http:// or https:// (got '{parsed.scheme}://')", file=sys.stderr)
        sys.exit(1)

    hostname = parsed.hostname or ""
    if not hostname:
        print("ERROR: GHOST_URL has no hostname.", file=sys.stderr)
        sys.exit(1)

    if _is_private_ip(hostname):
        print(f"ERROR: GHOST_URL is a private/local address ('{hostname}'). Use a public Ghost URL.", file=sys.stderr)
        sys.exit(1)

    if hostname.lower() == "localhost":
        print("ERROR: GHOST_URL points to localhost. Use a reachable Ghost URL.", file=sys.stderr)
        sys.exit(1)

    try:
        for info in socket.getaddrinfo(hostname, None, type=socket.SOCK_STREAM):
            if _is_private_ip(info[4][0]):
                print(f"ERROR: GHOST_URL resolves to an internal address ({info[4][0]}). Use a public Ghost URL.", file=sys.stderr)
                sys.exit(1)
    except (socket.gaierror, OSError):
        pass


# ── HTTP helper with retry ────────────────────────────────────────────────────

def ghost_get(url, api_path, content_key, resource, params=None, timeout=15, retries=3):
    """GET {url}{api_path}/{resource}/?key={key}&{params}."""
    all_params = {"key": content_key, **(params or {})}
    full_url = f"{url}{api_path}/{resource}/?{urllib.parse.urlencode(all_params)}"
    req = urllib.request.Request(full_url, headers={"Accept-Version": "v5.0"})
    last_exc = None
    for attempt in range(retries):
        try:
            with urllib.request.urlopen(req, timeout=timeout) as resp:
                return json.loads(resp.read())
        except urllib.error.HTTPError as e:
            if e.code in _RETRY_CODES and attempt < retries - 1:
                wait = 2 ** attempt
                print(f"  HTTP {e.code} — retrying in {wait}s...", file=sys.stderr)
                time.sleep(wait)
                last_exc = e
                continue
            raise
        except urllib.error.URLError as e:
            if attempt < retries - 1:
                wait = 2 ** attempt
                print(f"  Network error ({e.reason}) — retrying in {wait}s...", file=sys.stderr)
                time.sleep(wait)
                last_exc = e
                continue
            raise
    raise last_exc


# ── Validation checks ─────────────────────────────────────────────────────────

def check_config(url, content_key):
    if not url:
        print("GHOST_NOT_CONFIGURED: Set GHOST_URL and GHOST_CONTENT_KEY to enable Ghost integration.", file=sys.stderr)
        sys.exit(2)

    if not content_key:
        print("ERROR: GHOST_CONTENT_KEY is not set.", file=sys.stderr)
        print("", file=sys.stderr)
        print("Create a Content API key in Ghost admin:", file=sys.stderr)
        print("  Settings → Integrations → Add custom integration → copy Content API Key", file=sys.stderr)
        print("", file=sys.stderr)
        print("Then set it in .env.local:", file=sys.stderr)
        print("  GHOST_CONTENT_KEY=your_key_here", file=sys.stderr)
        sys.exit(1)

    validate_url(url)
    print(f"Ghost URL: {url}", file=sys.stderr)


def check_connectivity(url, content_key, content_type):
    """Try each API path version; return (api_path, total)."""
    last_error = None

    for api_path in _API_PATHS:
        try:
            data = ghost_get(url, api_path, content_key, content_type, {"limit": 1, "fields": "id"})
            total = data.get("meta", {}).get("pagination", {}).get("total", 0)
            version = "v4+" if "/v3/" not in api_path else "v3"
            print(f"Ghost {version} detected | content type: {content_type} | {total} published entries", file=sys.stderr)
            return api_path, total
        except urllib.error.HTTPError as e:
            if e.code == 403:
                print("ERROR: Ghost returned 403 Forbidden.", file=sys.stderr)
                print("Your GHOST_CONTENT_KEY is invalid or has been revoked.", file=sys.stderr)
                print("Regenerate it in: Settings → Integrations", file=sys.stderr)
                sys.exit(1)
            if e.code == 404 and api_path == _API_PATHS[-1]:
                print(f"ERROR: Could not find Ghost Content API at {url}", file=sys.stderr)
                print("Check that GHOST_URL points to your Ghost instance (not a CDN/proxy URL).", file=sys.stderr)
                sys.exit(1)
            last_error = e
            continue
        except urllib.error.URLError as e:
            print(f"ERROR: Cannot reach Ghost at {url}", file=sys.stderr)
            print(f"  Network error: {e.reason}", file=sys.stderr)
            sys.exit(1)

    if last_error:
        body = last_error.read().decode()[:200] if last_error.fp else "(no body)"
        print(f"ERROR: Ghost API error {last_error.code}. Response: {body}", file=sys.stderr)
    sys.exit(1)


def main():
    url, content_key, content_type = get_config()
    check_config(url, content_key)
    api_path, total = check_connectivity(url, content_key, content_type)
    if total == 0:
        print(f"WARNING: No published entries found in '{content_type}'.", file=sys.stderr)
    print(f"OK: Ghost ready ({content_type}, {total} published)", file=sys.stderr)


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""Pre-flight check for Strapi integration.

Verifies STRAPI_URL and STRAPI_API_KEY are set, tests connectivity,
detects API version (v4 or v5), and confirms the content type is accessible.
Credentials are loaded from environment variables or .env files.

No external dependencies — uses only Python stdlib.

Exit codes:
  0 — Strapi connection ready
  1 — unrecoverable error (missing config, auth failed, wrong content type, etc.)
  2 — Strapi not configured (STRAPI_URL not set) — non-fatal, caller skips
"""

import ipaddress
import json
import os
import socket
import sys
import time
import urllib.error
import urllib.parse
import urllib.request


# ── Config loading ────────────────────────────────────────────────────────────

def load_env_file(path):
    env = {}
    try:
        with open(path) as f:
            for line in f:
                line = line.strip()
                if not line or line.startswith("#") or "=" not in line:
                    continue
                key, _, raw_value = line.partition("=")
                key = key.strip()
                value = raw_value.strip().strip('"').strip("'")
                if key:
                    env[key] = value
    except (OSError, PermissionError):
        pass
    return env


def find_and_load_env():
    env = {}
    search = os.path.abspath(os.getcwd())
    for _ in range(6):
        for name in (".env.local", ".env"):
            candidate = os.path.join(search, name)
            if os.path.isfile(candidate):
                env.update(load_env_file(candidate))
        parent = os.path.dirname(search)
        if parent == search:
            break
        search = parent
    return env


def get_config():
    file_env = find_and_load_env()

    def get(key):
        return os.environ.get(key) or file_env.get(key, "")

    url = get("STRAPI_URL").rstrip("/")
    api_key = get("STRAPI_API_KEY")
    content_type = get("STRAPI_CONTENT_TYPE") or "articles"
    version_hint = get("STRAPI_VERSION")  # "4" or "5" — explicit override
    return url, api_key, content_type, version_hint


# ── Security: SSRF protection ─────────────────────────────────────────────────

def _is_private_ip(ip_str):
    """Return True if the IP is loopback, private, link-local, or reserved."""
    try:
        addr = ipaddress.ip_address(ip_str)
        return addr.is_loopback or addr.is_private or addr.is_link_local or addr.is_reserved
    except ValueError:
        return False


def _hostname_resolves_to_internal(hostname):
    """Resolve hostname and check if any address is internal. Non-fatal on DNS failure."""
    try:
        infos = socket.getaddrinfo(hostname, None, type=socket.SOCK_STREAM)
        for info in infos:
            ip = info[4][0]
            if _is_private_ip(ip):
                return True, ip
    except (socket.gaierror, OSError):
        pass
    return False, None


def validate_url(url):
    """Validate URL scheme and block SSRF targets (localhost, RFC1918, link-local)."""
    try:
        parsed = urllib.parse.urlparse(url)
    except Exception:
        print("ERROR: STRAPI_URL is not a valid URL.", file=sys.stderr)
        sys.exit(1)

    if parsed.scheme not in ("http", "https"):
        print(f"ERROR: STRAPI_URL must use http:// or https:// (got '{parsed.scheme}://')", file=sys.stderr)
        sys.exit(1)

    hostname = parsed.hostname or ""
    if not hostname:
        print("ERROR: STRAPI_URL has no hostname.", file=sys.stderr)
        sys.exit(1)

    # If the hostname is a literal IP, check it directly via ipaddress (no DNS needed).
    # If it's a hostname, fall through to the DNS-based check below.
    if _is_private_ip(hostname):
        print(f"ERROR: STRAPI_URL is a private/local address ('{hostname}'). Use a public CMS URL.", file=sys.stderr)
        sys.exit(1)

    if hostname.lower() in ("localhost",):
        print(f"ERROR: STRAPI_URL points to localhost. Use a reachable CMS URL.", file=sys.stderr)
        sys.exit(1)

    # DNS-based check for hostnames that resolve to internal IPs (best-effort)
    is_internal, resolved_ip = _hostname_resolves_to_internal(hostname)
    if is_internal:
        print(f"ERROR: STRAPI_URL resolves to an internal address ({resolved_ip}). Use a public CMS URL.", file=sys.stderr)
        sys.exit(1)


# ── HTTP helper with retry ────────────────────────────────────────────────────

_RETRY_CODES = {429, 502, 503, 504}


def strapi_get(url, api_key, path, params=None, timeout=15, retries=3):
    """GET {url}{path}?{params}. Retries on transient failures with backoff."""
    full_url = f"{url}{path}"
    if params:
        full_url = f"{full_url}?{urllib.parse.urlencode(params)}"
    req = urllib.request.Request(
        full_url,
        headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
    )
    last_exc = None
    for attempt in range(retries):
        try:
            with urllib.request.urlopen(req, timeout=timeout) as resp:
                return json.loads(resp.read())
        except urllib.error.HTTPError as e:
            if e.code in _RETRY_CODES and attempt < retries - 1:
                wait = 2 ** attempt
                print(f"  HTTP {e.code} — retrying in {wait}s...", file=sys.stderr)
                time.sleep(wait)
                last_exc = e
                continue
            raise
        except urllib.error.URLError as e:
            if attempt < retries - 1:
                wait = 2 ** attempt
                print(f"  Network error ({e.reason}) — retrying in {wait}s...", file=sys.stderr)
                time.sleep(wait)
                last_exc = e
                continue
            raise
    raise last_exc  # all retries exhausted


# ── Version detection ─────────────────────────────────────────────────────────

def detect_version(data, version_hint):
    """Return 4 or 5. Explicit hint wins; otherwise infer from response structure."""
    if version_hint in ("4", "5"):
        return int(version_hint)
    items = data.get("data", [])
    if items:
        return 4 if "attributes" in items[0] else 5
    # Empty collection: cannot infer from items. Check pagination meta shape.
    # v4 pagination has "total", v5 also has "total" — not distinguishable here.
    # Fall back to v5 (newer default). User can override with STRAPI_VERSION=4.
    return 5


def publication_param(version):
    """Return the correct publication filter param for the detected Strapi version."""
    if version == 4:
        return {"publicationState": "live"}
    return {"status": "published"}


# ── Validation checks ─────────────────────────────────────────────────────────

def check_config(url, api_key):
    if not url:
        print("STRAPI_NOT_CONFIGURED: Set STRAPI_URL and STRAPI_API_KEY to enable Strapi integration.", file=sys.stderr)
        sys.exit(2)

    if not api_key:
        print("ERROR: STRAPI_API_KEY is not set.", file=sys.stderr)
        print("", file=sys.stderr)
        print("Create a Full-access API token in Strapi admin:", file=sys.stderr)
        print("  Settings → Global settings → API Tokens → Create new API Token", file=sys.stderr)
        print("", file=sys.stderr)
        print("Then set it in .env.local:", file=sys.stderr)
        print("  STRAPI_API_KEY=your_token_here", file=sys.stderr)
        sys.exit(1)

    validate_url(url)
    print(f"Strapi URL: {url}", file=sys.stderr)


def check_connectivity(url, api_key, content_type, version_hint):
    """Probe the content type, detect version, return (version, total)."""
    # Probe without publication filter first — lets us detect version from response.
    probe_params = {"pagination[page]": 1, "pagination[pageSize]": 1}
    try:
        data = strapi_get(url, api_key, f"/api/{content_type}", probe_params)
    except urllib.error.HTTPError as e:
        body = e.read().decode()[:200] if e.fp else "(no body)"  # truncate — don't leak stack traces
        if e.code == 401:
            print("ERROR: Strapi returned 401 Unauthorized.", file=sys.stderr)
            print("Your STRAPI_API_KEY is invalid or expired.", file=sys.stderr)
            print("Regenerate it in: Settings → API Tokens", file=sys.stderr)
            sys.exit(1)
        if e.code == 403:
            print("ERROR: Strapi returned 403 Forbidden.", file=sys.stderr)
            print(f"The API token lacks permission to read '{content_type}'.", file=sys.stderr)
            print("Use a Full-access token or grant find/findOne permissions.", file=sys.stderr)
            sys.exit(1)
        if e.code == 404:
            print(f"ERROR: Content type '{content_type}' not found (404).", file=sys.stderr)
            print("Check the plural API ID in Strapi admin:", file=sys.stderr)
            print("  Content-Type Builder → [your type] → API ID (plural)", file=sys.stderr)
            print("Override with: STRAPI_CONTENT_TYPE=blog-posts  (in .env.local)", file=sys.stderr)
            sys.exit(1)
        print(f"ERROR: Strapi API error {e.code}. Response: {body}", file=sys.stderr)
        sys.exit(1)
    except urllib.error.URLError as e:
        print(f"ERROR: Cannot reach Strapi at {url}", file=sys.stderr)
        print(f"  Network error: {e.reason}", file=sys.stderr)
        sys.exit(1)

    version = detect_version(data, version_hint)

    # Now fetch published count with the correct version-specific param
    pub_params = {"pagination[page]": 1, "pagination[pageSize]": 1, **publication_param(version)}
    try:
        pub_data = strapi_get(url, api_key, f"/api/{content_type}", pub_params)
    except Exception:
        pub_data = data  # fall back to probe result

    total = pub_data.get("meta", {}).get("pagination", {}).get("total", 0)

    if version_hint and version_hint not in ("4", "5"):
        print(f"WARNING: STRAPI_VERSION='{version_hint}' is invalid. Use '4' or '5'. Detected v{version}.", file=sys.stderr)

    print(f"Strapi v{version} detected | content type: {content_type} | {total} published entries", file=sys.stderr)
    return version, total


def main():
    url, api_key, content_type, version_hint = get_config()
    check_config(url, api_key)
    version, total = check_connectivity(url, api_key, content_type, version_hint)
    if total == 0:
        print(f"WARNING: No published entries found in '{content_type}'.", file=sys.stderr)
        if not version_hint:
            print("  If this is a v4 instance with live content, set STRAPI_VERSION=4 in .env.local", file=sys.stderr)
    print(f"OK: Strapi ready (v{version}, {content_type}, {total} published)", file=sys.stderr)


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""Pre-flight check for WordPress REST API integration.

Verifies WP_URL, WP_USERNAME, and WP_APP_PASSWORD are set, tests connectivity
via the WordPress REST API, detects installed SEO plugins (Yoast, RankMath),
and reports the total published entry count.

Uses WordPress Application Passwords (WP 5.6+) for authentication.
No external dependencies — uses only Python stdlib.

Exit codes:
  0 — WordPress connection ready
  1 — unrecoverable error (missing config, auth failed, wrong content type, etc.)
  2 — WordPress not configured (WP_URL not set) — non-fatal, caller skips
"""

import base64
import ipaddress
import json
import os
import socket
import sys
import time
import urllib.error
import urllib.parse
import urllib.request


_RETRY_CODES = {429, 502, 503, 504}


# ── Config loading ────────────────────────────────────────────────────────────

def load_env_file(path):
    env = {}
    try:
        with open(path) as f:
            for line in f:
                line = line.strip()
                if not line or line.startswith("#") or "=" not in line:
                    continue
                key, _, raw_value = line.partition("=")
                key = key.strip()
                value = raw_value.strip().strip('"').strip("'")
                if key:
                    env[key] = value
    except (OSError, PermissionError):
        pass
    return env


def find_and_load_env():
    env = {}
    search = os.path.abspath(os.getcwd())
    for _ in range(6):
        for name in (".env.local", ".env"):
            candidate = os.path.join(search, name)
            if os.path.isfile(candidate):
                env.update(load_env_file(candidate))
        parent = os.path.dirname(search)
        if parent == search:
            break
        search = parent
    return env


def get_config():
    file_env = find_and_load_env()

    def get(key):
        return os.environ.get(key) or file_env.get(key, "")

    url = get("WP_URL").rstrip("/")
    username = get("WP_USERNAME")
    app_password = get("WP_APP_PASSWORD")
    content_type = get("WP_CONTENT_TYPE") or "posts"
    return url, username, app_password, content_type


# ── Security: SSRF protection ─────────────────────────────────────────────────

def _is_private_ip(ip_str):
    try:
        addr = ipaddress.ip_address(ip_str)
        return addr.is_loopback or addr.is_private or addr.is_link_local or addr.is_reserved
    except ValueError:
        return False


def validate_url(url):
    """Validate URL scheme and block SSRF targets."""
    try:
        parsed = urllib.parse.urlparse(url)
    except Exception:
        print("ERROR: WP_URL is not a valid URL.", file=sys.stderr)
        sys.exit(1)

    if parsed.scheme not in ("http", "https"):
        print(f"ERROR: WP_URL must use http:// or https:// (got '{parsed.scheme}://')", file=sys.stderr)
        sys.exit(1)

    hostname = parsed.hostname or ""
    if not hostname:
        print("ERROR: WP_URL has no hostname.", file=sys.stderr)
        sys.exit(1)

    if _is_private_ip(hostname):
        print(f"ERROR: WP_URL is a private/local address ('{hostname}'). Use a public WordPress URL.", file=sys.stderr)
        sys.exit(1)

    if hostname.lower() == "localhost":
        print("ERROR: WP_URL points to localhost. Use a reachable WordPress URL.", file=sys.stderr)
        sys.exit(1)

    try:
        for info in socket.getaddrinfo(hostname, None, type=socket.SOCK_STREAM):
            if _is_private_ip(info[4][0]):
                print(f"ERROR: WP_URL resolves to an internal address ({info[4][0]}). Use a public WordPress URL.", file=sys.stderr)
                sys.exit(1)
    except (socket.gaierror, OSError):
        pass  # non-fatal; let the request fail naturally


# ── HTTP helper ───────────────────────────────────────────────────────────────

def wp_get(url, auth_header, path, params=None, timeout=15, retries=3):
    """GET {url}{path}?{params} with Basic auth. Retries on transient failures."""
    full_url = f"{url}{path}"
    if params:
        full_url = f"{full_url}?{urllib.parse.urlencode(params)}"
    req = urllib.request.Request(
        full_url,
        headers={"Authorization": auth_header, "Accept": "application/json"},
    )
    last_exc = None
    for attempt in range(retries):
        try:
            with urllib.request.urlopen(req, timeout=timeout) as resp:
                body = json.loads(resp.read())
                total = int(resp.headers.get("X-WP-Total", 0))
                total_pages = int(resp.headers.get("X-WP-TotalPages", 1))
                return body, total, total_pages
        except urllib.error.HTTPError as e:
            if e.code in _RETRY_CODES and attempt < retries - 1:
                wait = 2 ** attempt
                print(f"  HTTP {e.code} — retrying in {wait}s...", file=sys.stderr)
                time.sleep(wait)
                last_exc = e
                continue
            raise
        except urllib.error.URLError as e:
            if attempt < retries - 1:
                wait = 2 ** attempt
                print(f"  Network error ({e.reason}) — retrying in {wait}s...", file=sys.stderr)
                time.sleep(wait)
                last_exc = e
                continue
            raise
    raise last_exc


def make_auth_header(username, app_password):
    """Build Basic auth header from WP username + Application Password."""
    # Application Passwords are displayed with spaces for readability — strip them.
    password = app_password.replace(" ", "")
    credentials = base64.b64encode(f"{username}:{password}".encode()).decode()
    return f"Basic {credentials}"


# ── Validation checks ─────────────────────────────────────────────────────────

def check_config(url, username, app_password):
    if not url:
        print("WP_NOT_CONFIGURED: Set WP_URL, WP_USERNAME, and WP_APP_PASSWORD to enable WordPress integration.", file=sys.stderr)
        sys.exit(2)

    if not username:
        print("ERROR: WP_USERNAME is not set.", file=sys.stderr)
        print("", file=sys.stderr)
        print("Set your WordPress username in .env.local:", file=sys.stderr)
        print("  WP_USERNAME=your_username", file=sys.stderr)
        sys.exit(1)

    if not app_password:
        print("ERROR: WP_APP_PASSWORD is not set.", file=sys.stderr)
        print("", file=sys.stderr)
        print("Create an Application Password in WordPress admin:", file=sys.stderr)
        print("  Users → Profile → Application Passwords → Add New Application Password", file=sys.stderr)
        print("", file=sys.stderr)
        print("Then set it in .env.local:", file=sys.stderr)
        print("  WP_APP_PASSWORD=xxxx xxxx xxxx xxxx xxxx xxxx", file=sys.stderr)
        sys.exit(1)

    validate_url(url)
    print(f"WordPress URL: {url}", file=sys.stderr)


def check_connectivity(url, auth_header, content_type):
    """Probe the REST API, detect SEO plugins, return total published count."""
    try:
        items, total, _ = wp_get(url, auth_header, f"/wp-json/wp/v2/{content_type}", {
            "status": "publish",
            "per_page": 1,
            "_fields": "id,yoast_head_json",
        })
    except urllib.error.HTTPError as e:
        body = e.read().decode()[:300] if e.fp else "(no body)"
        if e.code == 401:
            print("ERROR: WordPress returned 401 Unauthorized.", file=sys.stderr)
            print("Your WP_USERNAME or WP_APP_PASSWORD is invalid.", file=sys.stderr)
            print("Regenerate the Application Password in: Users → Profile → Application Passwords", file=sys.stderr)
            sys.exit(1)
        if e.code == 403:
            print("ERROR: WordPress returned 403 Forbidden.", file=sys.stderr)
            print(f"The user '{content_type}' may not have REST API access.", file=sys.stderr)
            sys.exit(1)
        if e.code == 404:
            print(f"ERROR: Content type '{content_type}' not found (404).", file=sys.stderr)
            print("Check the content type slug. Common values: posts, pages.", file=sys.stderr)
            print("Override with: WP_CONTENT_TYPE=your-type  (in .env.local)", file=sys.stderr)
            sys.exit(1)
        print(f"ERROR: WordPress API error {e.code}. Response: {body}", file=sys.stderr)
        sys.exit(1)
    except urllib.error.URLError as e:
        print(f"ERROR: Cannot reach WordPress at {url}", file=sys.stderr)
        print(f"  Network error: {e.reason}", file=sys.stderr)
        sys.exit(1)

    # Detect SEO plugins from first item's fields
    seo_plugin = "none detected"
    if items:
        item = items[0] if isinstance(items, list) else items
        if "yoast_head_json" in item:
            seo_plugin = "Yoast SEO"

    if seo_plugin == "none detected":
        # Check REST API namespaces for plugin detection
        try:
            ns_data, _, _ = wp_get(url, auth_header, "/wp-json/", timeout=10, retries=1)
            namespaces = ns_data.get("namespaces", [])
            if any(ns.startswith("yoast") for ns in namespaces):
                seo_plugin = "Yoast SEO"
            elif any(ns.startswith("rankmath") for ns in namespaces):
                seo_plugin = "RankMath"
        except Exception:
            pass

    print(f"WordPress REST API ready | content type: {content_type} | {total} published | SEO plugin: {seo_plugin}", file=sys.stderr)
    return total, seo_plugin


def main():
    url, username, app_password, content_type = get_config()
    check_config(url, username, app_password)
    auth_header = make_auth_header(username, app_password)
    total, seo_plugin = check_connectivity(url, auth_header, content_type)
    if total == 0:
        print(f"WARNING: No published entries found in '{content_type}'.", file=sys.stderr)
    print(f"OK: WordPress ready ({content_type}, {total} published, {seo_plugin})", file=sys.stderr)


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""Push SEO metadata updates back to Strapi.

Reads a batch of recommended SEO updates, shows a diff of proposed changes,
and writes them to Strapi after confirmation.

Supports Strapi v4 (PUT /api/{type}/{id}) and v5 (PUT /api/{type}/{documentId}).
Locale-aware: for localized v5 content, pass --locale or include 'locale' in batch.
No external dependencies — uses only Python stdlib.

Usage — single entry:
  python3 push_strapi_seo.py \\
    --document-id clkgylmcc000008lcdd868feh \\
    --meta-title "New Title | Brand" \\
    --meta-description "Compelling 120-char description."

  # Localized v5:
  python3 push_strapi_seo.py \\
    --document-id clkgylmcc000008lcdd868feh \\
    --locale fr \\
    --meta-title "Nouveau titre | Marque"

Usage — batch from file:
  python3 push_strapi_seo.py --batch-file /tmp/seo_updates.json [--yes]

Batch file format (JSON array):
  [
    {
      "document_id": "clkgylmcc000008lcdd868feh",  // v5
      "id": 42,                                      // v4 fallback
      "locale": "en",                                // optional, v5 localized
      "seo_schema": "component",                     // "component" | "root_fields"
      "meta_title": "New Title | Brand",
      "meta_description": "New description.",
      "updated_at": "2024-01-20T14:30:00.000Z"      // optional: refuse if stale
    }
  ]

Environment variables (or .env / .env.local):
  STRAPI_URL            Required.
  STRAPI_API_KEY        Required. Must be Full-access token (not read-only).
  STRAPI_CONTENT_TYPE   Optional (default: articles).
  STRAPI_VERSION        Optional. Force '4' or '5'.
"""

import argparse
import ipaddress
import json
import os
import socket
import sys
import time
import urllib.error
import urllib.parse
import urllib.request


_RETRY_CODES = {429, 502, 503, 504}


# ── SSRF protection ───────────────────────────────────────────────────────────

def _is_private_ip(ip_str):
    try:
        addr = ipaddress.ip_address(ip_str)
        return addr.is_loopback or addr.is_private or addr.is_link_local or addr.is_reserved
    except ValueError:
        return False


def validate_url(url):
    """Block SSRF targets. Called before any HTTP requests are made."""
    try:
        parsed = urllib.parse.urlparse(url)
    except Exception:
        print("ERROR: STRAPI_URL is not a valid URL.", file=sys.stderr)
        sys.exit(1)
    if parsed.scheme not in ("http", "https"):
        print(f"ERROR: STRAPI_URL must use http:// or https://", file=sys.stderr)
        sys.exit(1)
    hostname = parsed.hostname or ""
    if not hostname:
        print("ERROR: STRAPI_URL has no hostname.", file=sys.stderr)
        sys.exit(1)
    if _is_private_ip(hostname):
        print(f"ERROR: STRAPI_URL is a private/local address ('{hostname}').", file=sys.stderr)
        sys.exit(1)
    if hostname.lower() == "localhost":
        print("ERROR: STRAPI_URL points to localhost.", file=sys.stderr)
        sys.exit(1)
    # DNS-based check (best-effort)
    try:
        for info in socket.getaddrinfo(hostname, None, type=socket.SOCK_STREAM):
            if _is_private_ip(info[4][0]):
                print(f"ERROR: STRAPI_URL resolves to an internal address ({info[4][0]}).", file=sys.stderr)
                sys.exit(1)
    except (socket.gaierror, OSError):
        pass  # non-fatal; let the request fail naturally


# ── Config loading ────────────────────────────────────────────────────────────

def load_env_file(path):
    env = {}
    try:
        with open(path) as f:
            for line in f:
                line = line.strip()
                if not line or line.startswith("#") or "=" not in line:
                    continue
                key, _, raw_value = line.partition("=")
                key = key.strip()
                value = raw_value.strip().strip('"').strip("'")
                if key:
                    env[key] = value
    except (OSError, PermissionError):
        pass
    return env


def find_and_load_env():
    env = {}
    search = os.path.abspath(os.getcwd())
    for _ in range(6):
        for name in (".env.local", ".env"):
            candidate = os.path.join(search, name)
            if os.path.isfile(candidate):
                env.update(load_env_file(candidate))
        parent = os.path.dirname(search)
        if parent == search:
            break
        search = parent
    return env


def get_config():
    file_env = find_and_load_env()

    def get(key):
        return os.environ.get(key) or file_env.get(key, "")

    return (
        get("STRAPI_URL").rstrip("/"),
        get("STRAPI_API_KEY"),
        get("STRAPI_CONTENT_TYPE") or "articles",
        get("STRAPI_VERSION"),
    )


# ── HTTP helpers with retry ───────────────────────────────────────────────────

def _headers(api_key):
    return {"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"}


def strapi_get(base_url, api_key, path, params=None, timeout=15, retries=3):
    full_url = f"{base_url}{path}"
    if params:
        full_url = f"{full_url}?{urllib.parse.urlencode(params)}"
    req = urllib.request.Request(full_url, headers=_headers(api_key))
    last_exc = None
    for attempt in range(retries):
        try:
            with urllib.request.urlopen(req, timeout=timeout) as resp:
                return json.loads(resp.read())
        except urllib.error.HTTPError as e:
            if e.code in _RETRY_CODES and attempt < retries - 1:
                time.sleep(2 ** attempt)
                last_exc = e
                continue
            raise
        except urllib.error.URLError as e:
            if attempt < retries - 1:
                time.sleep(2 ** attempt)
                last_exc = e
                continue
            raise
    raise last_exc


def strapi_put(base_url, api_key, path, payload, timeout=30, retries=3):
    full_url = f"{base_url}{path}"
    data = json.dumps(payload).encode()
    req = urllib.request.Request(
        full_url, data=data, method="PUT", headers=_headers(api_key)
    )
    last_exc = None
    for attempt in range(retries):
        try:
            with urllib.request.urlopen(req, timeout=timeout) as resp:
                return json.loads(resp.read())
        except urllib.error.HTTPError as e:
            if e.code in _RETRY_CODES and attempt < retries - 1:
                time.sleep(2 ** attempt)
                last_exc = e
                continue
            raise
        except urllib.error.URLError as e:
            if attempt < retries - 1:
                time.sleep(2 ** attempt)
                last_exc = e
                continue
            raise
    raise last_exc


# ── Version detection ─────────────────────────────────────────────────────────

def detect_version(raw, version_hint):
    """Detect v4 vs v5 from a single-entry GET response."""
    if version_hint in ("4", "5"):
        return int(version_hint)
    data = raw.get("data", {})
    return 4 if "attributes" in data else 5


# ── Current SEO value fetching ────────────────────────────────────────────────

def fetch_current_seo(base_url, api_key, content_type, entry_id, locale, version):
    params = {"populate": "seo"}
    if version == 5 and locale:
        params["locale"] = locale
    try:
        data = strapi_get(base_url, api_key, f"/api/{content_type}/{entry_id}", params)
    except Exception as e:
        print(f"  WARNING: Could not fetch current values for {entry_id}: {e}", file=sys.stderr)
        return {}, {}

    raw = data.get("data", {})
    attrs = raw.get("attributes", raw)  # works for both v4 and v5

    seo = attrs.get("seo") or {}
    if isinstance(seo, dict) and "data" in seo:
        seo = seo["data"].get("attributes", seo)

    current_seo = {
        "meta_title": seo.get("metaTitle") or attrs.get("metaTitle") or "",
        "meta_description": seo.get("metaDescription") or attrs.get("metaDescription") or "",
        "updated_at": attrs.get("updatedAt") or "",
    }
    return current_seo, attrs


# ── Payload builder ───────────────────────────────────────────────────────────

def build_payload(update, current_attrs, seo_schema):
    """Build the PUT payload. Respects the schema in use (component vs root fields)."""
    seo_patch = {}
    if "meta_title" in update:
        seo_patch["metaTitle"] = update["meta_title"]
    if "meta_description" in update:
        seo_patch["metaDescription"] = update["meta_description"]

    if not seo_patch:
        return None

    # Determine schema: prefer explicit flag from fetch output, else detect from attrs
    use_component = seo_schema == "component"
    if seo_schema not in ("component", "root_fields"):
        # Auto-detect: if current attrs have a 'seo' key, use component schema
        use_component = "seo" in (current_attrs or {})

    if use_component:
        # Merge with existing SEO component to avoid clobbering unrelated fields
        existing_seo = (current_attrs or {}).get("seo") or {}
        if isinstance(existing_seo, dict) and "data" in existing_seo:
            existing_seo = existing_seo["data"].get("attributes", {})
        merged_seo = {**{k: v for k, v in existing_seo.items()
                         if k not in ("metaImage", "metaSocial")},
                      **seo_patch}
        return {"data": {"seo": merged_seo}}
    else:
        # Root-level SEO fields
        return {"data": seo_patch}


# ── Diff display ──────────────────────────────────────────────────────────────

def print_diff(entry_id, update, current, locale=None):
    loc_label = f" [{locale}]" if locale else ""
    print(f"\n  Entry: {entry_id}{loc_label}", file=sys.stderr)
    for field in ("meta_title", "meta_description"):
        if field not in update:
            continue
        old_val = current.get(field) or "(empty)"
        new_val = update[field]
        label = "Meta Title" if field == "meta_title" else "Meta Description"
        char_limit = 60 if field == "meta_title" else 160
        print(f"  {label}:", file=sys.stderr)
        print(f"    - {old_val}", file=sys.stderr)
        print(f"    + {new_val}", file=sys.stderr)
        if len(new_val) > char_limit:
            print(f"    WARNING: exceeds {char_limit} chars ({len(new_val)})", file=sys.stderr)
        if len(new_val) == 0:
            print(f"    WARNING: new value is empty — will blank this field", file=sys.stderr)


# ── Confirmation ──────────────────────────────────────────────────────────────

def confirm_batch(count, auto_yes=False):
    if auto_yes:
        return True
    if not sys.stdin.isatty():
        print("ERROR: --yes flag required in non-interactive mode.", file=sys.stderr)
        sys.exit(1)
    print(f"\nApply {count} SEO update(s) to Strapi? [y/N] ", end="", file=sys.stderr, flush=True)
    return input().strip().lower() in ("y", "yes")


# ── Core update logic ─────────────────────────────────────────────────────────

def process_updates(base_url, api_key, content_type, updates, version_hint, auto_yes=False):
    if not updates:
        print("No updates to apply.", file=sys.stderr)
        return

    # Short-circuit: if version_hint is explicit, skip the probe entirely
    if version_hint in ("4", "5"):
        version = int(version_hint)
    else:
        version = 5  # default; probe to confirm
        for upd in updates:
            entry_id = upd.get("document_id") or str(upd.get("id", ""))
            if not entry_id:
                continue
            try:
                probe = strapi_get(base_url, api_key, f"/api/{content_type}/{entry_id}")
                version = detect_version(probe, version_hint)
                break
            except Exception:
                continue

    print(f"Strapi v{version} | {content_type}", file=sys.stderr)

    # Build diffs
    print(f"\nProposed changes ({len(updates)} entries):", file=sys.stderr)
    print("-" * 60, file=sys.stderr)

    enriched = []
    skipped = 0
    for upd in updates:
        entry_id = upd.get("document_id") or str(upd.get("id", ""))
        locale = upd.get("locale") or ""
        seo_schema = upd.get("seo_schema") or "auto"

        if not entry_id:
            print(f"  SKIP: entry missing document_id/id: {upd}", file=sys.stderr)
            skipped += 1
            continue

        current_seo, current_attrs = fetch_current_seo(
            base_url, api_key, content_type, entry_id, locale, version
        )

        # Stale-write guard: refuse if entry was modified since the batch was generated
        expected_updated_at = upd.get("updated_at") or ""
        live_updated_at = current_seo.get("updated_at") or ""
        if expected_updated_at and live_updated_at and expected_updated_at != live_updated_at:
            print(
                f"  SKIP {entry_id}: entry was modified since analysis "
                f"(expected {expected_updated_at}, got {live_updated_at}). "
                f"Re-run analysis before pushing.",
                file=sys.stderr,
            )
            skipped += 1
            continue

        print_diff(entry_id, upd, current_seo, locale)
        enriched.append((entry_id, upd, current_seo, current_attrs, locale, seo_schema))

    print("-" * 60, file=sys.stderr)

    if not enriched:
        print("No valid entries to update after review.", file=sys.stderr)
        sys.exit(0 if skipped == 0 else 1)

    if not confirm_batch(len(enriched), auto_yes):
        print("Aborted. No changes written.", file=sys.stderr)
        sys.exit(0)

    # Apply updates
    success = 0
    failed = 0
    for entry_id, upd, _current, current_attrs, locale, seo_schema in enriched:
        payload = build_payload(upd, current_attrs, seo_schema)
        if not payload:
            print(f"  SKIP {entry_id}: nothing to update", file=sys.stderr)
            continue

        path = f"/api/{content_type}/{entry_id}"
        params = {}
        if version == 5 and locale:
            params["locale"] = locale
        if params:
            path = f"{path}?{urllib.parse.urlencode(params)}"

        try:
            strapi_put(base_url, api_key, path, payload)
            loc_label = f" [{locale}]" if locale else ""
            print(f"  OK   {entry_id}{loc_label}", file=sys.stderr)
            success += 1
        except urllib.error.HTTPError as e:
            body = e.read().decode()[:200] if e.fp else "(no body)"
            print(f"  FAIL {entry_id}: HTTP {e.code}: {body}", file=sys.stderr)
            failed += 1
        except urllib.error.URLError as e:
            print(f"  FAIL {entry_id}: network error: {e.reason}", file=sys.stderr)
            failed += 1

    print(f"\nDone. {success} updated, {failed} failed, {skipped} skipped.", file=sys.stderr)
    if failed:
        sys.exit(1)


# ── CLI ───────────────────────────────────────────────────────────────────────

def main():
    parser = argparse.ArgumentParser(description="Push SEO updates to Strapi")
    parser.add_argument("--content-type", help="Override STRAPI_CONTENT_TYPE")
    parser.add_argument("--document-id", help="Strapi v5 documentId (single entry)")
    parser.add_argument("--id", type=int, help="Strapi v4 numeric id (single entry)")
    parser.add_argument("--locale", help="Locale for v5 localized content (e.g. 'fr', 'en')")
    parser.add_argument("--meta-title", help="New meta title (max 60 chars)")
    parser.add_argument("--meta-description", help="New meta description (70-160 chars)")
    parser.add_argument("--batch-file", help="JSON file with array of update objects")
    parser.add_argument("--yes", "-y", action="store_true", help="Skip confirmation prompt")
    args = parser.parse_args()

    base_url, api_key, content_type, version_hint = get_config()
    if args.content_type:
        content_type = args.content_type

    if not base_url:
        print("ERROR: STRAPI_URL is not set.", file=sys.stderr)
        sys.exit(1)
    if not api_key:
        print("ERROR: STRAPI_API_KEY is not set.", file=sys.stderr)
        sys.exit(1)

    validate_url(base_url)

    if args.batch_file:
        try:
            with open(args.batch_file) as f:
                updates = json.load(f)
            if not isinstance(updates, list):
                print("ERROR: batch file must contain a JSON array.", file=sys.stderr)
                sys.exit(1)
        except (OSError, json.JSONDecodeError) as e:
            print(f"ERROR: Could not read batch file: {e}", file=sys.stderr)
            sys.exit(1)
    elif args.document_id or args.id:
        if not args.meta_title and not args.meta_description:
            print("ERROR: Provide --meta-title and/or --meta-description.", file=sys.stderr)
            sys.exit(1)
        update = {}
        if args.document_id:
            update["document_id"] = args.document_id
        if args.id:
            update["id"] = args.id
        if args.locale:
            import re
            if not re.match(r"^[a-z]{2}(-[A-Z]{2})?$", args.locale):
                print(f"ERROR: Invalid locale '{args.locale}'. Expected format: 'en' or 'en-US'.", file=sys.stderr)
                sys.exit(1)
            update["locale"] = args.locale
        if args.meta_title:
            update["meta_title"] = args.meta_title
        if args.meta_description:
            update["meta_description"] = args.meta_description
        updates = [update]
    else:
        parser.print_help()
        sys.exit(1)

    process_updates(base_url, api_key, content_type, updates, version_hint, auto_yes=args.yes)


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""
Display a human-readable summary of the GSC analysis JSON output.

Usage:
  python3 show_gsc.py [path/to/gsc_analysis.json]

If no path is given, reads from the default temp file location.
"""

import json
import os
import sys
import tempfile

from _uid import portable_uid

DEFAULT_PATH = os.path.join(tempfile.gettempdir(), f"gsc_analysis_{portable_uid()}.json")


def fmt_ctr(ctr):
    """CTR is stored as a percentage (e.g. 4.79 means 4.79%)."""
    return f"{ctr:.2f}%"


def show(path):
    with open(path) as f:
        data = json.load(f)

    summary = data.get("summary", {})
    print(f"\nSite: {data.get('site', '?')}")
    period = data.get("period", {})
    print(f"Period: {period.get('start', '?')} to {period.get('end', '?')} ({period.get('days', '?')} days)")
    print(f"\nSummary: {summary.get('clicks', 0):,} clicks | {summary.get('impressions', 0):,} impressions "
          f"| CTR {fmt_ctr(summary.get('ctr', 0))} | Avg position {summary.get('position', 0)}")

    # Top pages
    top_pages = data.get("top_pages", [])
    if top_pages:
        print(f"\n=== TOP {len(top_pages)} PAGES ===")
        for i, p in enumerate(top_pages, 1):
            print(f"  {i:2}. {p['clicks']:5,} clk | {p['impressions']:7,} imp "
                  f"| CTR {fmt_ctr(p['ctr'])} | pos {p['position']} | {p['page']}")

    # Top queries
    top_queries = data.get("top_queries", [])
    if top_queries:
        print(f"\n=== TOP {len(top_queries)} QUERIES ===")
        for i, q in enumerate(top_queries, 1):
            print(f"  {i:2}. {q['clicks']:5,} clk | {q['impressions']:7,} imp "
                  f"| CTR {fmt_ctr(q['ctr'])} | pos {q['position']} | {q['query']}")

    # Position buckets
    buckets = data.get("position_buckets", {})
    if buckets:
        print("\n=== POSITION BUCKETS ===")
        for bucket_name in ["1-3", "4-10", "11-20", "21+"]:
            rows = buckets.get(bucket_name, [])
            print(f"  [{bucket_name}]: {len(rows)} queries")

    # CTR opportunities
    ctr_opps = data.get("ctr_opportunities", [])
    if ctr_opps:
        print(f"\n=== CTR OPPORTUNITIES (high impressions, low CTR) ===")
        for q in ctr_opps[:10]:
            print(f"  {q['impressions']:6,} imp | CTR {fmt_ctr(q['ctr'])} | pos {q['position']} | {q['query']}")

    # Cannibalization
    cannib = data.get("cannibalization", [])
    if cannib:
        print(f"\n=== CANNIBALIZATION ({len(cannib)} queries) ===")
        for c in cannib[:5]:
            print(f"  '{c['query']}' → winner: {c['winner_page']}")
            print(f"    losers: {', '.join(c['loser_pages'])}")
            print(f"    action: {c['recommended_action']}")

    # Declining pages/queries
    comparison = data.get("comparison", {})
    declining_pages = comparison.get("declining_pages", [])
    declining_queries = comparison.get("declining_queries", [])
    comp_period = comparison.get("period", "?")
    comp_prior = comparison.get("prior_period", "?")
    if declining_pages or declining_queries:
        print(f"\n=== TRAFFIC CHANGES ({comp_period} vs {comp_prior}) ===")
        if declining_pages:
            print(f"  Declining pages ({len(declining_pages)}):")
            for p in declining_pages[:5]:
                print(f"    {p['change_pct']:+.1f}% | {p['clicks_now']:,} → {p['clicks_prev']:,} | {p['page']}")
        if declining_queries:
            print(f"  Declining queries ({len(declining_queries)}):")
            for q in declining_queries[:5]:
                print(f"    {q['change_pct']:+.1f}% | {q['clicks_now']:,} → {q['clicks_prev']:,} | {q['query']}")

    # Device split
    devices = data.get("device_split", [])
    if devices:
        print("\n=== DEVICE SPLIT ===")
        for d in devices:
            print(f"  {d['device']:10} {d['clicks']:6,} clk | CTR {fmt_ctr(d['ctr'])} | pos {d['position']}")

    # Search type split
    search_types = data.get("search_type_split", [])
    if search_types:
        print("\n=== SEARCH TYPE SPLIT ===")
        for t in search_types:
            print(f"  {t['type']:12} {t['clicks']:6,} clk | CTR {fmt_ctr(t['ctr'])} | pos {t['position']}")

    # Branded split
    branded = data.get("branded_split")
    if branded:
        b = branded.get("branded", {})
        nb = branded.get("non_branded", {})
        print("\n=== BRANDED vs NON-BRANDED ===")
        print(f"  Branded:     {b.get('clicks', 0):6,} clk | {b.get('impressions', 0):,} imp "
              f"| CTR {fmt_ctr(b.get('ctr', 0))} | {b.get('query_count', 0)} queries")
        print(f"  Non-branded: {nb.get('clicks', 0):6,} clk | {nb.get('impressions', 0):,} imp "
              f"| CTR {fmt_ctr(nb.get('ctr', 0))} | {nb.get('query_count', 0)} queries")

    # Page groups
    page_groups = data.get("page_groups", [])
    if page_groups:
        print("\n=== PAGE GROUPS ===")
        for g in page_groups:
            print(f"  {g['group']:15} {g['page_count']:3} pages | {g['clicks']:6,} clk "
                  f"| CTR {fmt_ctr(g['ctr'])} | pos {g['position']}")

    print()


if __name__ == "__main__":
    path = sys.argv[1] if len(sys.argv) > 1 else DEFAULT_PATH
    if not os.path.exists(path):
        print(f"ERROR: GSC data file not found: {path}", file=sys.stderr)
        print("Run analyze_gsc.py first to generate it.", file=sys.stderr)
        sys.exit(1)
    show(path)

#!/usr/bin/env python3
"""
Display PageSpeed Insights results in a terminal-friendly format.
Reads the JSON output from pagespeed.py.

Usage:
  python3 show_pagespeed.py
  python3 show_pagespeed.py --input /path/to/pagespeed.json
"""

import argparse
import json
import os
import sys
import tempfile

from _uid import portable_uid


def rating_indicator(rating):
    """Return a visual indicator for CrUX rating."""
    if not rating:
        return "?"
    return {"FAST": "GOOD", "AVERAGE": "NEEDS WORK", "SLOW": "POOR"}.get(rating, rating)


def score_indicator(score):
    """Return a visual indicator for Lighthouse score."""
    if score is None:
        return "N/A"
    if score >= 90:
        return f"{score} (Good)"
    elif score >= 50:
        return f"{score} (Needs Work)"
    else:
        return f"{score} (Poor)"


def format_ms(val):
    """Format milliseconds for display."""
    if val is None:
        return "N/A"
    if val >= 1000:
        return f"{val / 1000:.1f} s"
    return f"{val:.0f} ms"


def format_bytes(val):
    """Format bytes for display."""
    if not val:
        return ""
    if val >= 1_048_576:
        return f"{val / 1_048_576:.1f} MB"
    if val >= 1024:
        return f"{val / 1024:.0f} KB"
    return f"{val} B"


def format_cls(val, is_crux=False):
    """Format CLS value.
    CrUX API returns CLS as an integer (CLS * 100), e.g. 10 means 0.10.
    Lighthouse returns CLS as a float, e.g. 0.10."""
    if val is None:
        return "N/A"
    if isinstance(val, (int, float)):
        if is_crux:
            return f"{val / 100:.3f}"
        return f"{val:.3f}"
    return str(val)


def print_section(title):
    print(f"\n{'=' * 60}")
    print(f"  {title}")
    print(f"{'=' * 60}")


def print_result(result):
    """Print a single URL's PageSpeed result."""
    url = result.get("url", "?")
    strategy = result.get("strategy", "?")

    if result.get("error"):
        print(f"\n  {url} ({strategy}): ERROR - {result['error']}")
        return

    print(f"\n  URL: {url}")
    print(f"  Strategy: {strategy}")

    # Lab data (Lighthouse)
    lab = result.get("lab_data")
    if lab:
        score = lab.get("performance_score")
        print(f"\n  Performance Score: {score_indicator(score)}")
        print(f"  {'─' * 40}")

        metrics = [
            ("First Contentful Paint", lab.get("fcp", {})),
            ("Largest Contentful Paint", lab.get("lcp", {})),
            ("Total Blocking Time", lab.get("tbt", {})),
            ("Cumulative Layout Shift", lab.get("cls", {})),
            ("Speed Index", lab.get("si", {})),
            ("Time to Interactive", lab.get("tti", {})),
        ]

        print(f"  {'Metric':<30} {'Value':<15} {'Score'}")
        print(f"  {'─' * 55}")
        for name, data in metrics:
            if not data:
                continue
            display = data.get("display", "")
            metric_score = data.get("score")
            if metric_score is not None:
                metric_score = f"{round(metric_score * 100)}/100"
            else:
                metric_score = "N/A"
            print(f"  {name:<30} {display:<15} {metric_score}")

    # Field data (CrUX - real user data)
    field = result.get("field_data")
    if field:
        print(f"\n  Real-User Data (Chrome UX Report)")
        print(f"  {'─' * 40}")
        overall = field.get("overall_category")
        if overall:
            print(f"  Overall: {rating_indicator(overall)}")

        crux_metrics = [
            ("LCP", field.get("lcp"), "ms", False),
            ("INP", field.get("inp"), "ms", False),
            ("CLS", field.get("cls"), "", True),
            ("FCP", field.get("fcp"), "ms", False),
            ("TTFB", field.get("ttfb"), "ms", False),
        ]

        print(f"  {'Metric':<8} {'Value':<12} {'Rating'}")
        print(f"  {'─' * 35}")
        for name, data, unit, is_cls in crux_metrics:
            if not data:
                continue
            val = data.get("value")
            rating = data.get("rating")
            if is_cls:
                display = format_cls(val, is_crux=True)
            elif unit == "ms":
                display = format_ms(val)
            else:
                display = str(val) if val is not None else "N/A"
            print(f"  {name:<8} {display:<12} {rating_indicator(rating)}")

    # Origin field data (site-wide CrUX)
    origin = result.get("origin_field_data")
    if origin and origin.get("overall_category"):
        print(f"\n  Origin (Site-Wide) Real-User Data")
        print(f"  {'─' * 40}")
        print(f"  Overall: {rating_indicator(origin.get('overall_category'))}")

        crux_metrics = [
            ("LCP", origin.get("lcp"), "ms", False),
            ("INP", origin.get("inp"), "ms", False),
            ("CLS", origin.get("cls"), "", True),
        ]
        for name, data, unit, is_cls in crux_metrics:
            if not data:
                continue
            val = data.get("value")
            rating = data.get("rating")
            if is_cls:
                display = format_cls(val, is_crux=True)
            elif unit == "ms":
                display = format_ms(val)
            else:
                display = str(val) if val is not None else "N/A"
            print(f"  {name:<8} {display:<12} {rating_indicator(rating)}")

    # Opportunities
    opportunities = result.get("opportunities", [])
    if opportunities:
        print(f"\n  Top Optimization Opportunities")
        print(f"  {'─' * 55}")
        print(f"  {'Opportunity':<40} {'Savings'}")
        print(f"  {'─' * 55}")
        for opp in opportunities:
            title = opp.get("title", "?")
            if len(title) > 38:
                title = title[:35] + "..."
            savings = format_ms(opp.get("savings_ms"))
            bytes_saved = opp.get("savings_bytes", 0)
            extra = f" ({format_bytes(bytes_saved)})" if bytes_saved else ""
            print(f"  {title:<40} {savings}{extra}")

    # Diagnostics
    diagnostics = result.get("diagnostics", [])
    if diagnostics:
        print(f"\n  Diagnostics")
        print(f"  {'─' * 55}")
        for diag in diagnostics:
            title = diag.get("title", "?")
            display = diag.get("display", "")
            score = diag.get("score")
            indicator = ""
            if score is not None:
                indicator = f" (score: {round(score * 100)}/100)"
            suffix = f" — {display}" if display else ""
            print(f"  - {title}{suffix}{indicator}")


def main():
    parser = argparse.ArgumentParser()
    _default_in = os.path.join(tempfile.gettempdir(), f"pagespeed_{portable_uid()}.json")
    parser.add_argument("--input", default=_default_in, help="Input JSON file")
    args = parser.parse_args()

    if not os.path.isfile(args.input):
        print(f"ERROR: File not found: {args.input}", file=sys.stderr)
        print("Run pagespeed.py first to generate the data.", file=sys.stderr)
        sys.exit(1)

    with open(args.input) as f:
        data = json.load(f)

    summary = data.get("summary", {})
    results = data.get("results", [])

    print_section("PageSpeed Insights Report")
    print(f"  URLs tested: {summary.get('urls_tested', '?')}")
    print(f"  Strategies: {', '.join(summary.get('strategies', []))}")
    avg = summary.get("avg_performance_score")
    if avg is not None:
        print(f"  Average score: {score_indicator(avg)}")

    for result in results:
        print(f"\n{'─' * 60}")
        print_result(result)

    print(f"\n{'=' * 60}")


if __name__ == "__main__":
    main()

#!/usr/bin/env python3
"""
Run the Google Search Console URL Inspection API on a list of URLs.
Outputs structured JSON for the seo-analysis skill to process.

The URL Inspection API returns per-page: indexing status, mobile usability verdict,
rich result status, last crawl time, referring sitemaps, and coverage state.

Required OAuth scope: https://www.googleapis.com/auth/webmasters
(Not just webmasters.readonly — URL Inspection requires the broader scope.)

Usage:
  python3 url_inspection.py --site "sc-domain:example.com" \
    --urls "https://example.com/,https://example.com/pricing"

  python3 url_inspection.py --site "https://example.com/" \
    --urls-file /tmp/urls.txt

  python3 url_inspection.py --site "sc-domain:example.com" \
    --urls "https://example.com/,https://example.com/blog" \
    --output /tmp/inspection_results.json
"""

import argparse
import json
import os
import subprocess
import sys
import tempfile
import time
import urllib.parse
import urllib.request
import urllib.error
from concurrent.futures import ThreadPoolExecutor, as_completed

from _gcloud import adc_access_token, adc_config_dir, gcloud_run
from _uid import portable_uid, secure_write_json


def get_quota_project():
    """Return the quota_project_id from the ADC JSON file, or None."""
    adc_dir = adc_config_dir()
    adc_path = os.path.join(adc_dir, "application_default_credentials.json")
    try:
        with open(adc_path) as f:
            data = json.load(f)
        if isinstance(data, dict):
            return data.get("quota_project_id") or None
    except (OSError, ValueError):
        pass
    return None


def get_access_token():
    try:
        result = adc_access_token()
    except FileNotFoundError:
        print("ERROR: gcloud not found. Install it from https://cloud.google.com/sdk/docs/install",
              file=sys.stderr)
        sys.exit(1)
    except subprocess.TimeoutExpired:
        print("ERROR: gcloud timed out after 15s.", file=sys.stderr)
        sys.exit(1)

    if result.returncode != 0:
        print("ERROR: Not authenticated. Run:", file=sys.stderr)
        print("  gcloud auth application-default login \\", file=sys.stderr)
        print("    --scopes=https://www.googleapis.com/auth/webmasters,"
              "https://www.googleapis.com/auth/webmasters.readonly", file=sys.stderr)
        sys.exit(1)

    token = result.stdout.strip()
    if not token:
        print("ERROR: gcloud returned an empty token.", file=sys.stderr)
        sys.exit(1)
    return token


def inspect_url(token, site_url, inspection_url):
    """Call the URL Inspection API for one URL. Returns the raw API response dict."""
    endpoint = "https://searchconsole.googleapis.com/v1/urlInspection/index:inspect"
    body = json.dumps({
        "inspectionUrl": inspection_url,
        "siteUrl": site_url
    }).encode()

    headers = {
        "Authorization": f"Bearer {token}",
        "Content-Type": "application/json",
    }
    quota_project = get_quota_project()
    if quota_project:
        headers["x-goog-user-project"] = quota_project
    req = urllib.request.Request(endpoint, data=body, headers=headers)

    try:
        with urllib.request.urlopen(req, timeout=30) as resp:
            return json.loads(resp.read()), None
    except urllib.error.HTTPError as e:
        err_body = e.read().decode() if e.fp else ""
        if e.code == 403:
            return None, (
                f"403 Forbidden for {inspection_url}. "
                "The URL Inspection API requires the broader 'webmasters' OAuth scope "
                "(not just 'webmasters.readonly'). Re-authenticate with:\n"
                "  gcloud auth application-default login \\\n"
                "    --scopes=https://www.googleapis.com/auth/webmasters,"
                "https://www.googleapis.com/auth/webmasters.readonly"
            )
        return None, f"HTTP {e.code} for {inspection_url}: {err_body[:200]}"
    except urllib.error.URLError as e:
        return None, f"Network error for {inspection_url}: {e.reason}"


def normalize_site_url_for_inspection(site_url, url):
    """
    For domain properties (sc-domain:example.com), the inspectionUrl must be
    an absolute URL. For URL-prefix properties, it must start with the prefix.
    If the caller passes a path like '/pricing', resolve it against the site URL.
    """
    if url.startswith("http://") or url.startswith("https://"):
        return url

    # Strip the sc-domain: prefix to get the bare domain
    if site_url.startswith("sc-domain:"):
        domain = site_url[len("sc-domain:"):]
        return f"https://{domain.rstrip('/')}{url}"
    else:
        base = site_url.rstrip("/")
        return f"{base}{url}"


def _as_dict(value):
    return value if isinstance(value, dict) else {}


def _as_list(value):
    return value if isinstance(value, list) else []


def parse_inspection_result(raw, url):
    """Extract the fields we care about from the API response."""
    ir = _as_dict(_as_dict(raw).get("inspectionResult"))

    # Index status
    index_result = _as_dict(ir.get("indexStatusResult"))
    indexing_state = index_result.get("coverageState", "UNKNOWN")
    verdict = index_result.get("verdict", "UNKNOWN")
    last_crawl = index_result.get("lastCrawlTime", None)
    referring_sitemaps = _as_list(index_result.get("referringSitemaps"))
    crawled_as = index_result.get("crawledAs", None)
    google_canonical = index_result.get("googleCanonical", None)
    user_canonical = index_result.get("userDeclaredCanonical", None)
    page_fetch_state = index_result.get("pageFetchState", None)
    robots_txt_state = index_result.get("robotsTxtState", None)
    indexing_state_value = index_result.get("indexingState", None)

    # Mobile usability
    mobile_result = _as_dict(ir.get("mobileUsabilityResult"))
    mobile_verdict = mobile_result.get("verdict", "VERDICT_UNSPECIFIED")
    mobile_issues = [
        issue.get("issueType", "UNKNOWN")
        for issue in _as_list(mobile_result.get("issues"))
        if isinstance(issue, dict)
    ]

    # Rich results
    rich_result = _as_dict(ir.get("richResultsResult"))
    rich_verdict = rich_result.get("verdict", "VERDICT_UNSPECIFIED")
    rich_items = []
    for item in _as_list(rich_result.get("detectedItems")):
        if not isinstance(item, dict):
            continue
        for ri in _as_list(item.get("items")):
            if not isinstance(ri, dict):
                continue
            item_entry = {
                "name": ri.get("name", ""),
                "issues": [
                    issue.get("issueMessage", "")
                    for issue in _as_list(ri.get("issues"))
                    if isinstance(issue, dict)
                ]
            }
            rich_items.append(item_entry)

    return {
        "url": url,
        "index_status": {
            "verdict": verdict,
            "coverage_state": indexing_state,
            "last_crawl_time": last_crawl,
            "crawled_as": crawled_as,
            "indexing_state": indexing_state_value,
            "page_fetch_state": page_fetch_state,
            "robots_txt_state": robots_txt_state,
            "referring_sitemaps": referring_sitemaps,
            "google_canonical": google_canonical,
            "user_declared_canonical": user_canonical,
        },
        "mobile_usability": {
            "verdict": mobile_verdict,
            "issues": mobile_issues
        },
        "rich_results": {
            "verdict": rich_verdict,
            "detected_items": rich_items
        }
    }


def summarize_findings(results):
    """Produce a high-level summary flags for easy parsing by the skill."""
    not_indexed = [r for r in results if r.get("index_status", {}).get("verdict") not in ("PASS", "NEUTRAL", "VERDICT_UNSPECIFIED")]
    mobile_issues = [r for r in results if r.get("mobile_usability", {}).get("verdict") not in ("MOBILE_FRIENDLY", "VERDICT_UNSPECIFIED")]
    rich_errors = [r for r in results if r.get("rich_results", {}).get("verdict") == "FAIL"]
    no_sitemaps = [r for r in results if not r.get("index_status", {}).get("referring_sitemaps")]

    import datetime
    stale_crawl = []
    for r in results:
        lc = r.get("index_status", {}).get("last_crawl_time")
        if lc:
            try:
                crawl_dt = datetime.datetime.fromisoformat(lc.replace("Z", "+00:00"))
                now = datetime.datetime.now(datetime.timezone.utc)
                days_since = (now - crawl_dt).days
                if days_since > 60:
                    stale_crawl.append({
                        "url": r["url"],
                        "last_crawl_time": lc,
                        "days_since_crawl": days_since
                    })
            except (ValueError, TypeError):
                pass

    return {
        "total_urls_inspected": len(results),
        "not_indexed_count": len(not_indexed),
        "mobile_issues_count": len(mobile_issues),
        "rich_result_errors_count": len(rich_errors),
        "no_sitemap_count": len(no_sitemaps),
        "stale_crawl_count": len(stale_crawl),
        "not_indexed_urls": [r["url"] for r in not_indexed],
        "mobile_issue_urls": [r["url"] for r in mobile_issues],
        "rich_error_urls": [r["url"] for r in rich_errors],
        "stale_crawl_urls": stale_crawl
    }


def main():
    parser = argparse.ArgumentParser(
        description="Run GSC URL Inspection API on a list of URLs."
    )
    parser.add_argument("--site", required=True,
                        help="GSC property (sc-domain:example.com or https://example.com/)")
    parser.add_argument("--urls",
                        help="Comma-separated list of URLs to inspect")
    parser.add_argument("--urls-file",
                        help="File with one URL per line")
    parser.add_argument("--max-urls", type=int, default=5,
                        help="Maximum number of URLs to inspect (API limit: 2000/day). Default: 5")
    _default_out = os.path.join(tempfile.gettempdir(),
                                f"url_inspection_{portable_uid()}.json")
    parser.add_argument("--output", default=_default_out,
                        help="Output JSON file path")
    parser.add_argument("--delay", type=float, default=0.1,
                        help="Seconds between concurrent API calls to avoid rate limiting. Default: 0.1")
    parser.add_argument("--concurrency", type=int, default=3,
                        help="Number of concurrent URL inspections. Default: 3")
    args = parser.parse_args()

    # Collect URLs
    urls = []
    if args.urls:
        urls.extend([u.strip() for u in args.urls.split(",") if u.strip()])
    if args.urls_file:
        with open(args.urls_file) as f:
            urls.extend([line.strip() for line in f if line.strip()])

    if not urls:
        print("ERROR: Provide --urls or --urls-file", file=sys.stderr)
        sys.exit(1)

    # Deduplicate and cap
    seen = set()
    deduped = []
    for u in urls:
        if u not in seen:
            seen.add(u)
            deduped.append(u)
    urls = deduped[:args.max_urls]

    if len(deduped) > args.max_urls:
        print(f"Note: Capped at {args.max_urls} URLs (had {len(deduped)}). "
              f"Pass --max-urls N to inspect more.", file=sys.stderr)

    print(f"Inspecting {len(urls)} URLs for site: {args.site} "
          f"(concurrency={args.concurrency})", file=sys.stderr)

    token = get_access_token()

    # Normalize all URLs upfront
    absolute_urls = [normalize_site_url_for_inspection(args.site, u) for u in urls]

    results = []
    errors = []
    abort_403 = False

    def _inspect_one(absolute_url):
        time.sleep(args.delay)  # small stagger to avoid thundering herd
        return absolute_url, inspect_url(token, args.site, absolute_url)

    with ThreadPoolExecutor(max_workers=args.concurrency) as pool:
        futures = {pool.submit(_inspect_one, url): url for url in absolute_urls}
        for future in as_completed(futures):
            try:
                absolute_url, (raw, error) = future.result()
            except Exception as exc:
                absolute_url = futures[future]
                print(f"  ERROR [{absolute_url}]: unexpected error: {exc}", file=sys.stderr)
                errors.append({"url": absolute_url, "error": str(exc)})
                continue
            if error:
                print(f"  ERROR [{absolute_url}]: {error[:80]}", file=sys.stderr)
                errors.append({"url": absolute_url, "error": error})
                if "403" in str(error):
                    abort_403 = True
            else:
                parsed = parse_inspection_result(raw, absolute_url)
                results.append(parsed)
                verdict = parsed.get("index_status", {}).get("verdict", "UNKNOWN")
                mobile = parsed.get("mobile_usability", {}).get("verdict", "VERDICT_UNSPECIFIED")
                print(f"  ✓ {absolute_url} — Index: {verdict} | Mobile: {mobile}",
                      file=sys.stderr)

    if abort_403:
        print("\nOne or more 403 errors — URL Inspection requires 'webmasters' scope.",
              file=sys.stderr)
        print("Re-authenticate with the broader scope and retry:", file=sys.stderr)
        print("  gcloud auth application-default login \\", file=sys.stderr)
        print("    --scopes=https://www.googleapis.com/auth/webmasters,"
              "https://www.googleapis.com/auth/webmasters.readonly", file=sys.stderr)

    summary = summarize_findings(results)

    output = {
        "site": args.site,
        "inspected_at": __import__("datetime").datetime.utcnow().isoformat() + "Z",
        "summary": summary,
        "results": results,
        "errors": errors
    }

    secure_write_json(args.output, output)

    print(f"\nDone. Results saved to {args.output}", file=sys.stderr)
    print(f"Summary: {summary['total_urls_inspected']} inspected | "
          f"{summary['not_indexed_count']} not indexed | "
          f"{summary['mobile_issues_count']} mobile issues | "
          f"{summary['rich_result_errors_count']} rich result errors | "
          f"{summary['stale_crawl_count']} stale crawl",
          file=sys.stderr)

    if errors:
        print(f"\n{len(errors)} URL(s) failed inspection:", file=sys.stderr)
        for e in errors:
            print(f"  {e['url']}: {e['error'][:100]}", file=sys.stderr)


if __name__ == "__main__":
    main()

"""Helpers for invoking the gcloud CLI portably."""

import os
import shutil
import subprocess
import sys


def gcloud_command(args):
    """Return a subprocess argument list for a gcloud command.

    On Windows, the Google Cloud SDK installs gcloud as gcloud.cmd. Python's
    shell=False path resolution does not expand PATHEXT for CreateProcess, so
    invoke the resolved batch file through cmd.exe.
    """
    if not args or args[0] != "gcloud":
        raise ValueError("gcloud_command expects args starting with 'gcloud'")

    gcloud = shutil.which("gcloud")
    if sys.platform == "win32" and gcloud and gcloud.lower().endswith((".cmd", ".bat")):
        return ["cmd", "/c", gcloud, *args[1:]]

    return [gcloud or "gcloud", *args[1:]]


def gcloud_run(args, *run_args, **run_kwargs):
    """Run gcloud with subprocess.run using the portable command wrapper."""
    return subprocess.run(gcloud_command(args), *run_args, **run_kwargs)


# Search Console needs an explicitly-requested scope. Service-account ADC
# (GOOGLE_APPLICATION_CREDENTIALS) mints a cloud-platform-only token by default,
# which searchconsole.googleapis.com rejects with 403
# ACCESS_TOKEN_SCOPE_INSUFFICIENT.
GSC_SCOPES = (
    "https://www.googleapis.com/auth/webmasters.readonly,"
    "https://www.googleapis.com/auth/cloud-platform"
)


def adc_access_token(scopes=GSC_SCOPES, timeout=15):
    """Mint an Application Default Credentials access token.

    Requests `scopes` explicitly so that service-account credentials can reach
    Search Console. User credentials that were not granted those scopes at
    login cannot mint them, so fall back to an unscoped request rather than
    failing outright.
    """
    attempts = []
    if scopes:
        attempts.append(
            ["gcloud", "auth", "application-default", "print-access-token",
             f"--scopes={scopes}"]
        )
    attempts.append(["gcloud", "auth", "application-default", "print-access-token"])

    result = None
    for args in attempts:
        result = gcloud_run(args, capture_output=True, text=True, timeout=timeout)
        if result.returncode == 0 and result.stdout.strip():
            return result
    return result


def adc_config_dir():
    """Return the gcloud config directory that holds the ADC file.

    CLOUDSDK_CONFIG wins when set. Otherwise gcloud writes its config to
    %APPDATA%\\gcloud on Windows and ~/.config/gcloud everywhere else.
    """
    if os.environ.get("CLOUDSDK_CONFIG"):
        return os.environ["CLOUDSDK_CONFIG"]
    if sys.platform == "win32":
        appdata = os.environ.get("APPDATA")
        if appdata:
            return os.path.join(appdata, "gcloud")
    return os.path.join(os.path.expanduser("~"), ".config", "gcloud")

"""Portable per-user identifier and secure tmp-file helpers.

Why this exists
---------------
The seo-analysis scripts cache intermediate JSON in the system tempdir between
invocations (e.g. ``analyze_gsc.py`` writes, ``show_gsc.py`` reads). Filenames
are keyed by a per-user suffix so that on shared POSIX hosts two users running
the scripts in the same ``/tmp`` don't collide.

Two primitives:

- :func:`portable_uid` — stable, path-safe identifier for the current user.
  Uses ``os.getuid()`` on POSIX (preserving the historical filename suffix on
  Linux/macOS), sanitized ``getpass.getuser()`` on Windows where ``getuid``
  doesn't exist, and a hashed env-based fallback if neither is available.

- :func:`secure_write_json` — atomic, mode-0600, symlink-safe JSON write,
  matching the pattern already used in the CMS fetchers
  (``fetch_*_content.py``). Writes to a fresh ``mkstemp()`` file in the
  destination directory, then ``os.replace()``s it into place. Defends
  against symlink attacks on shared tmpdirs because POSIX ``rename(2)``
  removes a pre-existing symlink at the destination instead of following it.

Usage as a module: ``python3 -m _uid`` prints the portable uid (used by the
SKILL.md heredoc in place of the previous ``os.getuid()`` shell call).
"""

from __future__ import annotations

import hashlib
import json
import os
import re
import sys
import tempfile
from typing import Any

_SAFE_CHARS = re.compile(r"[^A-Za-z0-9_-]")


def portable_uid() -> str:
    """Return a stable, path-safe identifier for the current user."""
    getuid = getattr(os, "getuid", None)
    if getuid is not None:
        return str(getuid())

    import getpass
    try:
        username = getpass.getuser()
    except Exception:
        username = ""

    # Sanitize: env-var-derived usernames could contain path separators or
    # null bytes; we interpolate the result into a filesystem path.
    safe = _SAFE_CHARS.sub("_", username)[:32].strip("_")
    if safe:
        return safe

    seed = os.environ.get("APPDATA") or os.environ.get("USERPROFILE") or ""
    if seed:
        return hashlib.sha1(seed.encode("utf-8", "replace")).hexdigest()[:8]

    return f"pid{os.getpid()}"


def secure_write_json(path: str, data: Any) -> None:
    """Atomically write ``data`` as JSON to ``path`` with mode 0600.

    Defends against symlink-based clobber attacks on shared tempdirs.
    """
    out_dir = os.path.dirname(path) or "."
    fd, tmp_path = tempfile.mkstemp(dir=out_dir, suffix=".json.tmp")
    try:
        try:
            os.chmod(tmp_path, 0o600)
        except (OSError, NotImplementedError):
            # Windows / unusual filesystems may not honor POSIX modes.
            pass
        with os.fdopen(fd, "w") as f:
            json.dump(data, f, indent=2)
        os.replace(tmp_path, path)
    except Exception:
        try:
            os.unlink(tmp_path)
        except OSError:
            pass
        raise


if __name__ == "__main__":
    sys.stdout.write(portable_uid())