import argparse
import asyncio

from .config import config
from .http import close_http_client


def build_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(
        prog="groksearch_cli",
        description="GrokSearch CLI - Standalone web search via Grok API",
    )
    parser.add_argument("--api-url", help="Override GROK_API_URL")
    parser.add_argument("--api-key", help="Override GROK_API_KEY")
    parser.add_argument("--debug", action="store_true", help="Enable debug output")

    subparsers = parser.add_subparsers(dest="command", required=True)

    p_search = subparsers.add_parser("web_search", help="Perform web search")
    p_search.add_argument("--query", "-q", required=True, help="Search query")
    p_search.add_argument("--platform", "-p", default="", help="Focus platforms (e.g., 'GitHub,Reddit')")
    p_search.add_argument("--min-results", type=int, default=3, help="Minimum results")
    p_search.add_argument("--max-results", type=int, default=10, help="Maximum results")
    p_search.add_argument(
        "--extra-sources",
        type=int,
        default=0,
        help="Number of additional Tavily results merged into output (requires TAVILY_API_KEY)",
    )
    p_search.add_argument("--raw", action="store_true", help="Output raw response without JSON parsing")

    p_fetch = subparsers.add_parser("web_fetch", help="Fetch webpage content")
    p_fetch.add_argument("--url", "-u", required=True, help="URL to fetch")
    p_fetch.add_argument("--out", "-o", help="Output file path")
    p_fetch.add_argument("--via", choices=["grok", "tavily"], default="grok", help="Fetch backend (default: grok)")

    p_map = subparsers.add_parser("web_map", help="Map a website's structure (Tavily)")
    p_map.add_argument("--url", "-u", required=True, help="Root URL to map")
    p_map.add_argument("--instructions", default="", help="Natural language filter for crawler")
    p_map.add_argument("--max-depth", type=int, default=1, help="Max traversal depth (1-5)")
    p_map.add_argument("--max-breadth", type=int, default=20, help="Max links per page")
    p_map.add_argument("--limit", type=int, default=50, help="Total link limit")
    p_map.add_argument("--timeout", type=int, default=150, help="Operation timeout (seconds)")

    p_crawl = subparsers.add_parser("web_crawl", help="Crawl a website's pages (Tavily)")
    p_crawl.add_argument("--url", "-u", required=True, help="Root URL to crawl")
    p_crawl.add_argument("--instructions", default="", help="Natural language filter for crawler")
    p_crawl.add_argument(
        "--max-depth", type=int, default=None, help="Max crawl depth (default: TAVILY_CRAWL_MAX_DEPTH)"
    )
    p_crawl.add_argument(
        "--max-breadth", type=int, default=None, help="Max links per page (default: TAVILY_CRAWL_MAX_BREADTH)"
    )
    p_crawl.add_argument("--limit", type=int, default=None, help="Total page limit (default: TAVILY_CRAWL_LIMIT)")
    p_crawl.add_argument("--timeout", type=int, default=None, help="Operation timeout (default: TAVILY_CRAWL_TIMEOUT)")
    p_crawl.add_argument("--select-paths", default=None, help="Comma-separated path regexes to include")
    p_crawl.add_argument("--exclude-paths", default=None, help="Comma-separated path regexes to exclude")

    p_research = subparsers.add_parser("web_research", help="Run a cited research task (Tavily, async)")
    p_research.add_argument("--input", "-i", required=True, help="Research question or task")
    p_research.add_argument(
        "--model", default=None, help="Research model: mini|pro|auto (default: TAVILY_RESEARCH_MODEL)"
    )
    p_research.add_argument(
        "--output-length", default=None, help="short|standard|long (default: TAVILY_RESEARCH_OUTPUT_LENGTH)"
    )
    p_research.add_argument("--citation-format", default=None, help="numbered|mla|apa|chicago")
    p_research.add_argument(
        "--output-schema",
        default=None,
        help="Path to a JSON Schema file for structured output (default: TAVILY_RESEARCH_OUTPUT_SCHEMA)",
    )

    p_config = subparsers.add_parser("get_config_info", help="Show configuration and test connection")
    p_config.add_argument("--no-test", action="store_true", help="Skip connection test")

    p_model = subparsers.add_parser("switch_model", help="Switch Grok model")
    p_model.add_argument("--model", "-m", required=True, help="Model ID to switch to")

    p_toggle = subparsers.add_parser("toggle_builtin_tools", help="Toggle built-in WebSearch/WebFetch")
    p_toggle.add_argument("--action", "-a", default="status", help="Action: on/off/status")
    p_toggle.add_argument("--root", "-r", help="Project root path (default: auto-detect via .git)")

    return parser


async def run_command(args, commands: dict):
    try:
        await commands[args.command](args)
    finally:
        await close_http_client()


def parse_and_run(commands: dict) -> None:
    parser = build_parser()
    args = parser.parse_args()

    if args.api_url or args.api_key:
        config.set_overrides(args.api_url, args.api_key)

    asyncio.run(run_command(args, commands))

import asyncio
import json
import sys
import time
from pathlib import Path

import httpx

from .config import config
from .formatting import merge_search_results
from .provider import GrokSearchProvider
from .tavily import (
    _call_tavily_crawl,
    _call_tavily_extract,
    _call_tavily_map,
    _call_tavily_research,
    _call_tavily_search,
    _tavily_unavailable_reason,
)


async def cmd_web_search(args):
    try:
        provider = GrokSearchProvider(config.grok_api_url, config.grok_api_key, config.grok_model)
        extra_sources = getattr(args, "extra_sources", 0) or 0

        if args.raw:
            grok_result = await provider.search(args.query, args.platform, args.min_results, args.max_results)
            print(grok_result)
            return

        if extra_sources > 0 and _tavily_unavailable_reason() is None:
            grok_task = provider.search(args.query, args.platform, args.min_results, args.max_results)
            tavily_task = _call_tavily_search(args.query, extra_sources)
            grok_result, tavily_results = await asyncio.gather(grok_task, tavily_task)
        else:
            grok_result = await provider.search(args.query, args.platform, args.min_results, args.max_results)
            tavily_results = None

        merged = merge_search_results(grok_result, tavily_results, tavily_requested=extra_sources > 0)
        print(json.dumps(merged, ensure_ascii=False, indent=2))
    except ValueError as e:
        print(json.dumps({"error": str(e)}, ensure_ascii=False), file=sys.stderr)
        sys.exit(1)
    except httpx.HTTPStatusError as e:
        print(json.dumps({"error": f"API error: {e.response.status_code}"}, ensure_ascii=False), file=sys.stderr)
        sys.exit(1)


async def cmd_web_fetch(args):
    via = getattr(args, "via", "grok")
    try:
        if via == "tavily":
            reason = _tavily_unavailable_reason()
            if reason:
                print(json.dumps({"error": reason}, ensure_ascii=False), file=sys.stderr)
                sys.exit(1)
            result = await _call_tavily_extract(args.url)
            if result is None:
                print(
                    json.dumps({"error": "Tavily extract failed or returned empty content"}, ensure_ascii=False),
                    file=sys.stderr,
                )
                sys.exit(1)
        else:
            provider = GrokSearchProvider(config.grok_api_url, config.grok_api_key, config.grok_model)
            result = await provider.fetch(args.url)

        if args.out:
            Path(args.out).write_text(result, encoding="utf-8")
            print(f"Content saved to {args.out}")
        else:
            print(result)
    except ValueError as e:
        print(f"Error: {e}", file=sys.stderr)
        sys.exit(1)
    except httpx.HTTPStatusError as e:
        print(f"API error: {e.response.status_code}", file=sys.stderr)
        sys.exit(1)


async def _print_or_fail(coro):
    """Print a Tavily command result, or report a config error as JSON on stderr.

    Tuning values are validated lazily on property access, so an invalid TAVILY_* env
    surfaces as a ValueError from inside the call rather than at startup.
    """
    try:
        print(await coro)
    except ValueError as e:
        print(json.dumps({"error": str(e)}, ensure_ascii=False), file=sys.stderr)
        sys.exit(1)


async def cmd_web_map(args):
    await _print_or_fail(
        _call_tavily_map(
            args.url,
            args.instructions,
            args.max_depth,
            args.max_breadth,
            args.limit,
            args.timeout,
        )
    )


async def cmd_web_crawl(args):
    await _print_or_fail(
        _call_tavily_crawl(
            args.url,
            args.instructions,
            args.max_depth,
            args.max_breadth,
            args.limit,
            args.timeout,
            args.select_paths,
            args.exclude_paths,
        )
    )


async def cmd_web_research(args):
    await _print_or_fail(
        _call_tavily_research(
            args.input,
            args.model,
            args.output_length,
            args.citation_format,
            getattr(args, "output_schema", None),
        )
    )


async def cmd_get_config_info(args):
    config_info = config.get_config_info()

    if not args.no_test:
        test_result = {"status": "Not tested", "message": "", "response_time_ms": 0}
        try:
            api_url = config.grok_api_url
            api_key = config.grok_api_key
            models_url = f"{api_url}/models"

            start_time = time.time()
            async with httpx.AsyncClient(timeout=10.0) as client:
                response = await client.get(
                    models_url,
                    headers={"Authorization": f"Bearer {api_key}", "Content-Type": "application/json"},
                )
                response_time = (time.time() - start_time) * 1000

                if response.status_code == 200:
                    test_result["status"] = "✅ Connection Successful"
                    test_result["response_time_ms"] = round(response_time, 2)
                    try:
                        models_data = response.json()
                    except ValueError as e:
                        # A 200 with a non-JSON body still means the endpoint is reachable.
                        if config.debug_enabled:
                            print(f"[DEBUG] Could not parse models response: {e}", file=sys.stderr)
                        models_data = {}
                    if "data" in models_data:
                        model_count = len(models_data["data"])
                        test_result["message"] = f"Retrieved {model_count} models"
                        test_result["available_models"] = [
                            m.get("id") for m in models_data["data"] if isinstance(m, dict)
                        ]
                else:
                    test_result["status"] = "⚠️ Connection Issue"
                    test_result["message"] = f"HTTP {response.status_code}"

        except httpx.TimeoutException:
            test_result["status"] = "❌ Connection Timeout"
            test_result["message"] = "Request timed out (10s)"
        except Exception as e:
            test_result["status"] = "❌ Connection Failed"
            test_result["message"] = str(e)

        config_info["connection_test"] = test_result

    print(json.dumps(config_info, ensure_ascii=False, indent=2))


async def cmd_switch_model(args):
    try:
        previous = config.set_model(args.model)
        result = {
            "status": "✅ Success",
            "previous_model": previous,
            "current_model": args.model,
            "config_file": str(config.config_file),
        }
        print(json.dumps(result, ensure_ascii=False, indent=2))
    except Exception as e:
        print(json.dumps({"status": "❌ Failed", "error": str(e)}, ensure_ascii=False), file=sys.stderr)
        sys.exit(1)


async def cmd_toggle_builtin_tools(args):
    if args.root:
        root = Path(args.root)
        if not root.exists():
            print(
                json.dumps({"error": f"Specified root does not exist: {args.root}"}, ensure_ascii=False),
                file=sys.stderr,
            )
            sys.exit(1)
    else:
        root = Path.cwd()
        while root != root.parent and not (root / ".git").exists():
            root = root.parent
        if not (root / ".git").exists():
            print(
                json.dumps(
                    {
                        "error": "No .git directory found. Use --root to specify project root.",
                        "hint": "Run this command from within a git repository or specify --root PATH",
                    },
                    ensure_ascii=False,
                ),
                file=sys.stderr,
            )
            sys.exit(1)

    settings_path = root / ".claude" / "settings.json"
    tools = ["WebFetch", "WebSearch"]

    if settings_path.exists():
        with open(settings_path, encoding="utf-8") as f:
            settings = json.load(f)
    else:
        settings = {"permissions": {"deny": []}}

    deny = settings.setdefault("permissions", {}).setdefault("deny", [])
    blocked = all(t in deny for t in tools)

    action = args.action.lower()
    if action in ["on", "enable"]:
        for t in tools:
            if t not in deny:
                deny.append(t)
        settings_path.parent.mkdir(parents=True, exist_ok=True)
        with open(settings_path, "w", encoding="utf-8") as f:
            json.dump(settings, f, ensure_ascii=False, indent=2)
        msg = "Built-in tools disabled"
        blocked = True
    elif action in ["off", "disable"]:
        deny[:] = [t for t in deny if t not in tools]
        settings_path.parent.mkdir(parents=True, exist_ok=True)
        with open(settings_path, "w", encoding="utf-8") as f:
            json.dump(settings, f, ensure_ascii=False, indent=2)
        msg = "Built-in tools enabled"
        blocked = False
    else:
        msg = f"Built-in tools currently {'disabled' if blocked else 'enabled'}"

    print(
        json.dumps(
            {"blocked": blocked, "deny_list": deny, "file": str(settings_path), "message": msg},
            ensure_ascii=False,
            indent=2,
        )
    )

import json
import math
import os
from pathlib import Path

from .env import load_dotenv

load_dotenv()


class Config:
    _instance = None
    _FALLBACK_MODEL = "grok-4-1-fast"

    def __new__(cls):
        if cls._instance is None:
            cls._instance = super().__new__(cls)
            cls._instance._config_file = None
            cls._instance._cached_model = None
            cls._instance._override_url = None
            cls._instance._override_key = None
        return cls._instance

    @property
    def config_file(self) -> Path:
        if self._config_file is None:
            config_dir = Path.home() / ".config" / "grok-search"
            config_dir.mkdir(parents=True, exist_ok=True)
            self._config_file = config_dir / "config.json"
        return self._config_file

    def _load_config_file(self) -> dict:
        if not self.config_file.exists():
            return {}
        try:
            with open(self.config_file, encoding="utf-8") as f:
                return json.load(f)
        except (OSError, json.JSONDecodeError):
            return {}

    def _save_config_file(self, config_data: dict) -> None:
        with open(self.config_file, "w", encoding="utf-8") as f:
            json.dump(config_data, f, ensure_ascii=False, indent=2)

    def set_overrides(self, api_url: str | None, api_key: str | None):
        self._override_url = api_url
        self._override_key = api_key

    @property
    def debug_enabled(self) -> bool:
        return os.getenv("GROK_DEBUG", "false").lower() in ("true", "1", "yes")

    @property
    def retry_max_attempts(self) -> int:
        return int(os.getenv("GROK_RETRY_MAX_ATTEMPTS", "3"))

    @property
    def retry_multiplier(self) -> float:
        return float(os.getenv("GROK_RETRY_MULTIPLIER", "1"))

    @property
    def retry_max_wait(self) -> int:
        return int(os.getenv("GROK_RETRY_MAX_WAIT", "10"))

    @property
    def tavily_enabled(self) -> bool:
        return os.getenv("TAVILY_ENABLED", "true").lower() in ("true", "1", "yes")

    @property
    def tavily_api_url(self) -> str:
        return os.getenv("TAVILY_API_URL", "https://api.tavily.com")

    @property
    def tavily_api_key(self) -> str | None:
        return os.getenv("TAVILY_API_KEY")

    @property
    def tavily_search_depth(self) -> str:
        depth = os.getenv("TAVILY_SEARCH_DEPTH", "advanced").lower()
        if depth not in ("advanced", "basic", "fast", "ultra-fast"):
            raise ValueError(f"Invalid TAVILY_SEARCH_DEPTH: {depth}. Use advanced/basic/fast/ultra-fast")
        return depth

    @property
    def tavily_chunks_per_source(self) -> int:
        # The API accepts 1-5 ("or 'auto'"), though the docs page still renders 1-3.
        chunks = int(os.getenv("TAVILY_CHUNKS_PER_SOURCE", "3"))
        if not 1 <= chunks <= 5:
            raise ValueError(f"Invalid TAVILY_CHUNKS_PER_SOURCE: {chunks}. API accepts 1-5")
        return chunks

    @property
    def tavily_topic(self) -> str:
        topic = os.getenv("TAVILY_TOPIC", "general").lower()
        if topic not in ("general", "news", "finance"):
            raise ValueError(f"Invalid TAVILY_TOPIC: {topic}. Use general/news/finance")
        return topic

    @property
    def tavily_time_range(self) -> str:
        value = os.getenv("TAVILY_TIME_RANGE", "").lower()
        if not value:
            return ""
        if value not in ("day", "week", "month", "year", "d", "w", "m", "y"):
            raise ValueError(f"Invalid TAVILY_TIME_RANGE: {value}. Use day/week/month/year")
        return value

    @staticmethod
    def _split_csv(value: str) -> list[str]:
        return [item.strip() for item in value.split(",") if item.strip()]

    @staticmethod
    def _bounded_float(name: str, default: float, low: float | None = None, high: float | None = None) -> float:
        raw = os.getenv(name, str(default))
        try:
            value = float(raw)
        except ValueError as e:
            raise ValueError(f"Invalid {name}: {raw!r}. Must be a number") from e
        if not math.isfinite(value) or (low is not None and value < low) or (high is not None and value > high):
            bounds = f"{low}-{high}" if high is not None else f">={low}"
            raise ValueError(f"Invalid {name}: {raw!r}. Must be {bounds}")
        return value

    @staticmethod
    def _positive_int(name: str, default: int) -> int:
        raw = os.getenv(name, str(default))
        try:
            value = int(raw)
        except ValueError as e:
            raise ValueError(f"Invalid {name}: {raw!r}. Must be an integer") from e
        if value < 1:
            raise ValueError(f"Invalid {name}: {raw!r}. Must be >= 1")
        return value

    @property
    def tavily_include_domains(self) -> list[str]:
        return self._split_csv(os.getenv("TAVILY_INCLUDE_DOMAINS", ""))

    @property
    def tavily_exclude_domains(self) -> list[str]:
        return self._split_csv(os.getenv("TAVILY_EXCLUDE_DOMAINS", ""))

    @property
    def tavily_include_domains_mode(self) -> str:
        value = os.getenv("TAVILY_INCLUDE_DOMAINS_MODE", "filter").lower()
        if value not in ("filter", "boost"):
            raise ValueError(f"Invalid TAVILY_INCLUDE_DOMAINS_MODE: {value}. Use filter/boost")
        return value

    @property
    def tavily_include_usage(self) -> bool:
        return os.getenv("TAVILY_INCLUDE_USAGE", "false").lower() in ("true", "1", "yes")

    @property
    def tavily_extract_depth(self) -> str:
        depth = os.getenv("TAVILY_EXTRACT_DEPTH", "basic").lower()
        if depth not in ("basic", "advanced"):
            raise ValueError(f"Invalid TAVILY_EXTRACT_DEPTH: {depth}. Use basic/advanced")
        return depth

    @property
    def tavily_include_answer(self) -> bool | str:
        """Return the value for Tavily's `include_answer`.

        The API accepts only True, False, 'basic' or 'advanced' — the string "false" is
        rejected with a 400, so the disabled state must be the boolean.
        """
        value = os.getenv("TAVILY_INCLUDE_ANSWER", "false").lower()
        if value in ("false", "0", "no", "none", ""):
            return False
        if value in ("true", "1", "yes", "basic"):
            return "basic"
        if value == "advanced":
            return "advanced"
        raise ValueError(f"Invalid TAVILY_INCLUDE_ANSWER: {value}. Use false/basic/advanced")

    @property
    def tavily_extract_timeout(self) -> float:
        return self._bounded_float("TAVILY_EXTRACT_TIMEOUT", 30.0, low=1.0, high=60.0)

    @property
    def tavily_crawl_timeout(self) -> float:
        return self._bounded_float("TAVILY_CRAWL_TIMEOUT", 150.0, low=10.0, high=150.0)

    @property
    def tavily_crawl_max_depth(self) -> int:
        return self._positive_int("TAVILY_CRAWL_MAX_DEPTH", 1)

    @property
    def tavily_crawl_max_breadth(self) -> int:
        return self._positive_int("TAVILY_CRAWL_MAX_BREADTH", 20)

    @property
    def tavily_crawl_limit(self) -> int:
        return self._positive_int("TAVILY_CRAWL_LIMIT", 50)

    @property
    def tavily_crawl_allow_external(self) -> bool:
        return os.getenv("TAVILY_CRAWL_ALLOW_EXTERNAL", "true").lower() in ("true", "1", "yes")

    @property
    def tavily_crawl_select_paths(self) -> list[str]:
        return self._split_csv(os.getenv("TAVILY_CRAWL_SELECT_PATHS", ""))

    @property
    def tavily_crawl_exclude_paths(self) -> list[str]:
        return self._split_csv(os.getenv("TAVILY_CRAWL_EXCLUDE_PATHS", ""))

    @property
    def tavily_research_model(self) -> str:
        model = os.getenv("TAVILY_RESEARCH_MODEL", "auto").lower()
        if model not in ("mini", "pro", "auto"):
            raise ValueError(f"Invalid TAVILY_RESEARCH_MODEL: {model}. Use mini/pro/auto")
        return model

    @property
    def tavily_research_citation_format(self) -> str:
        value = os.getenv("TAVILY_RESEARCH_CITATION_FORMAT", "numbered").lower()
        if value not in ("numbered", "mla", "apa", "chicago"):
            raise ValueError(f"Invalid TAVILY_RESEARCH_CITATION_FORMAT: {value}. Use numbered/mla/apa/chicago")
        return value

    @property
    def tavily_research_output_length(self) -> str:
        value = os.getenv("TAVILY_RESEARCH_OUTPUT_LENGTH", "standard").lower()
        if value not in ("short", "standard", "long"):
            raise ValueError(f"Invalid TAVILY_RESEARCH_OUTPUT_LENGTH: {value}. Use short/standard/long")
        return value

    @property
    def tavily_research_output_schema(self) -> str:
        """Path to a JSON Schema file; blank means unstructured text output."""
        return os.getenv("TAVILY_RESEARCH_OUTPUT_SCHEMA", "").strip()

    @property
    def tavily_research_timeout(self) -> float:
        return self._bounded_float("TAVILY_RESEARCH_TIMEOUT", 300.0, low=0.0)

    @property
    def tavily_research_poll_interval(self) -> float:
        return self._bounded_float("TAVILY_RESEARCH_POLL_INTERVAL", 5.0, low=0.0)

    @property
    def grok_api_url(self) -> str:
        if self._override_url:
            return self._override_url
        url = os.getenv("GROK_API_URL")
        if not url:
            raise ValueError("GROK_API_URL not configured. Set environment variable or use --api-url")
        return url.rstrip("/")

    @property
    def grok_api_key(self) -> str:
        if self._override_key:
            return self._override_key
        key = os.getenv("GROK_API_KEY")
        if not key:
            raise ValueError("GROK_API_KEY not configured. Set environment variable or use --api-key")
        return key

    def _apply_model_suffix(self, model: str) -> str:
        try:
            url = self.grok_api_url
        except ValueError:
            return model
        if "openrouter" in url and ":online" not in model:
            return f"{model}:online"
        return model

    @property
    def grok_model(self) -> str:
        if self._cached_model is not None:
            return self._cached_model
        config_data = self._load_config_file()
        file_model = config_data.get("model")
        if file_model:
            self._cached_model = self._apply_model_suffix(file_model)
            return self._cached_model
        env_model = os.getenv("GROK_MODEL")
        if env_model:
            self._cached_model = self._apply_model_suffix(env_model)
            return self._cached_model
        self._cached_model = self._apply_model_suffix(self._FALLBACK_MODEL)
        return self._cached_model

    def set_model(self, model: str) -> str:
        previous = self.grok_model
        config_data = self._load_config_file()
        config_data["model"] = model
        self._save_config_file(config_data)
        self._cached_model = self._apply_model_suffix(model)
        return previous

    @staticmethod
    def _mask_api_key(key: str) -> str:
        if not key or len(key) <= 8:
            return "***"
        return f"{key[:4]}{'*' * (len(key) - 8)}{key[-4:]}"

    def get_config_info(self) -> dict:
        try:
            api_url = self.grok_api_url
            api_key_raw = self.grok_api_key
            api_key_masked = self._mask_api_key(api_key_raw)
            config_status = "✅ Configuration Complete"
        except ValueError as e:
            api_url = "Not configured"
            api_key_masked = "Not configured"
            config_status = f"❌ Error: {str(e)}"

        # A malformed tuning value raises ValueError on access; report it per-key instead
        # of breaking the whole config dump. Values are property names, resolved lazily.
        unset_when_empty = {
            "TAVILY_TIME_RANGE",
            "TAVILY_INCLUDE_DOMAINS",
            "TAVILY_EXCLUDE_DOMAINS",
            "TAVILY_CRAWL_SELECT_PATHS",
            "TAVILY_CRAWL_EXCLUDE_PATHS",
            "TAVILY_RESEARCH_OUTPUT_SCHEMA",
        }
        tuning_props = {
            "TAVILY_SEARCH_DEPTH": "tavily_search_depth",
            "TAVILY_CHUNKS_PER_SOURCE": "tavily_chunks_per_source",
            "TAVILY_INCLUDE_ANSWER": "tavily_include_answer",
            "TAVILY_EXTRACT_TIMEOUT": "tavily_extract_timeout",
            "TAVILY_EXTRACT_DEPTH": "tavily_extract_depth",
            "TAVILY_TOPIC": "tavily_topic",
            "TAVILY_TIME_RANGE": "tavily_time_range",
            "TAVILY_INCLUDE_DOMAINS": "tavily_include_domains",
            "TAVILY_EXCLUDE_DOMAINS": "tavily_exclude_domains",
            "TAVILY_INCLUDE_DOMAINS_MODE": "tavily_include_domains_mode",
            "TAVILY_INCLUDE_USAGE": "tavily_include_usage",
            "TAVILY_CRAWL_TIMEOUT": "tavily_crawl_timeout",
            "TAVILY_CRAWL_MAX_DEPTH": "tavily_crawl_max_depth",
            "TAVILY_CRAWL_MAX_BREADTH": "tavily_crawl_max_breadth",
            "TAVILY_CRAWL_LIMIT": "tavily_crawl_limit",
            "TAVILY_CRAWL_ALLOW_EXTERNAL": "tavily_crawl_allow_external",
            "TAVILY_CRAWL_SELECT_PATHS": "tavily_crawl_select_paths",
            "TAVILY_CRAWL_EXCLUDE_PATHS": "tavily_crawl_exclude_paths",
            "TAVILY_RESEARCH_MODEL": "tavily_research_model",
            "TAVILY_RESEARCH_CITATION_FORMAT": "tavily_research_citation_format",
            "TAVILY_RESEARCH_OUTPUT_LENGTH": "tavily_research_output_length",
            "TAVILY_RESEARCH_OUTPUT_SCHEMA": "tavily_research_output_schema",
            "TAVILY_RESEARCH_TIMEOUT": "tavily_research_timeout",
            "TAVILY_RESEARCH_POLL_INTERVAL": "tavily_research_poll_interval",
        }

        def _get(key: str, prop: str):
            try:
                value = getattr(self, prop)
            except ValueError as e:
                return f"❌ {e}"
            return "(unset)" if key in unset_when_empty and not value else value

        tuning = {k: _get(k, p) for k, p in tuning_props.items()}

        return {
            "GROK_API_URL": api_url,
            "GROK_API_KEY": api_key_masked,
            "GROK_MODEL": self.grok_model,
            "GROK_DEBUG": self.debug_enabled,
            "TAVILY_API_URL": self.tavily_api_url,
            "TAVILY_ENABLED": self.tavily_enabled,
            "TAVILY_API_KEY": self._mask_api_key(self.tavily_api_key) if self.tavily_api_key else "Not configured",
            **tuning,
            "config_status": config_status,
        }


config = Config()

import os
from pathlib import Path


def load_dotenv(env_path: Path | None = None) -> bool:
    search_paths = []
    if env_path:
        search_paths.append(env_path)
    else:
        # Get grok-search root directory (scripts/groksearch/../..)
        skill_root = Path(__file__).resolve().parent.parent.parent
        search_paths.append(skill_root / ".env")
        search_paths.append(skill_root / "scripts" / ".env")

    for path in search_paths:
        if path.exists():
            try:
                with open(path, encoding="utf-8") as f:
                    for line in f:
                        line = line.strip()
                        if not line or line.startswith("#"):
                            continue
                        if "=" not in line:
                            continue
                        key, _, value = line.partition("=")
                        key = key.strip()
                        value = value.strip()
                        if (value.startswith('"') and value.endswith('"')) or (
                            value.startswith("'") and value.endswith("'")
                        ):
                            value = value[1:-1]
                        if key and key not in os.environ:
                            os.environ[key] = value
                return True
            except OSError:
                continue
    return False

import json
import re
import sys


def extract_json(text: str) -> str:
    match = re.search(r"```(?:json)?\s*\n?([\s\S]*?)\n?```", text)
    if match:
        text = match.group(1).strip()

    try:
        data = json.loads(text)
        if isinstance(data, list):
            standardized = []
            for item in data:
                if isinstance(item, dict):
                    standardized.append(
                        {
                            "title": item.get("title", ""),
                            "url": item.get("url", item.get("link", "")),
                            "description": item.get(
                                "description",
                                item.get("content", item.get("snippet", item.get("summary", ""))),
                            ),
                        }
                    )
            return json.dumps(standardized, ensure_ascii=False, indent=2)
        return json.dumps(data, ensure_ascii=False, indent=2)
    except json.JSONDecodeError:
        return json.dumps({"error": "Failed to parse JSON", "raw": text[:500]}, ensure_ascii=False, indent=2)


def merge_search_results(
    grok_raw: str,
    tavily_results: list[dict] | None,
    tavily_requested: bool = False,
) -> list[dict]:
    extracted = extract_json(grok_raw)
    try:
        grok_items = json.loads(extracted)
        if not isinstance(grok_items, list):
            grok_items = []
    except json.JSONDecodeError:
        grok_items = []

    # Tavily reports credit usage as a non-result entry carrying only a "usage" key.
    usage = None
    if tavily_results:
        for r in tavily_results:
            if set(r) == {"usage"}:
                usage = r["usage"]
        tavily_results = [r for r in tavily_results if set(r) != {"usage"}]

    seen_urls = {item.get("url", "").strip() for item in grok_items if item.get("url", "").strip()}
    merged = list(grok_items)

    if tavily_results:
        for r in tavily_results:
            url = r.get("url", "").strip()
            if not url or url in seen_urls:
                continue
            seen_urls.add(url)
            merged.append(
                {
                    "title": r.get("title", ""),
                    "url": url,
                    "description": r.get("content", ""),
                    "provider": "tavily",
                }
            )
    elif tavily_requested:
        # Keep the top-level type a JSON array so list consumers are unaffected; disclose the
        # degradation on the first item instead of injecting a synthetic pseudo-result.
        print(
            "WARNING: Tavily extra sources were requested but returned no results; output contains Grok sources only",
            file=sys.stderr,
        )
        if merged:
            merged[0]["degraded"] = "tavily_unavailable"

    # Attach usage to the first item rather than adding an element, so the array stays
    # a list of results for downstream consumers.
    if usage and merged:
        merged[0]["tavily_usage"] = usage

    return merged

from datetime import UTC, datetime
from email.utils import parsedate_to_datetime

import httpx
from tenacity import AsyncRetrying as _AsyncRetrying
from tenacity import retry_if_exception, stop_after_attempt, wait_random_exponential
from tenacity.wait import wait_base

RETRYABLE_STATUS_CODES = {408, 429, 500, 502, 503, 504}
_DEFAULT_TIMEOUT = httpx.Timeout(connect=10.0, read=120.0, write=15.0, pool=None)
_http_client: httpx.AsyncClient | None = None


def _is_retryable_exception(exc) -> bool:
    if isinstance(exc, (httpx.TimeoutException, httpx.NetworkError, httpx.ConnectError)):
        return True
    if isinstance(exc, httpx.HTTPStatusError):
        return exc.response.status_code in RETRYABLE_STATUS_CODES
    return False


class _WaitWithRetryAfter(wait_base):
    def __init__(self, multiplier: float, max_wait: int):
        self._base_wait = wait_random_exponential(multiplier=multiplier, max=max_wait)

    def __call__(self, retry_state):
        if retry_state.outcome and retry_state.outcome.failed:
            exc = retry_state.outcome.exception()
            if isinstance(exc, httpx.HTTPStatusError) and exc.response.status_code == 429:
                retry_after = self._parse_retry_after(exc.response)
                if retry_after is not None:
                    return retry_after
        return self._base_wait(retry_state)

    def _parse_retry_after(self, response: httpx.Response) -> float | None:
        header = response.headers.get("Retry-After")
        if not header:
            return None
        header = header.strip()
        if header.isdigit():
            return float(header)
        try:
            retry_dt = parsedate_to_datetime(header)
            if retry_dt.tzinfo is None:
                retry_dt = retry_dt.replace(tzinfo=UTC)
            delay = (retry_dt - datetime.now(UTC)).total_seconds()
            return max(0.0, delay)
        except (TypeError, ValueError):
            return None


async def get_http_client() -> httpx.AsyncClient:
    global _http_client
    if _http_client is None or _http_client.is_closed:
        _http_client = httpx.AsyncClient(
            timeout=_DEFAULT_TIMEOUT,
            follow_redirects=True,
            limits=httpx.Limits(max_connections=10, max_keepalive_connections=5),
        )
    return _http_client


async def close_http_client():
    global _http_client
    if _http_client is not None and not _http_client.is_closed:
        await _http_client.aclose()
        _http_client = None


def retry_attempts(config, before_sleep=None):
    return _AsyncRetrying(
        stop=stop_after_attempt(config.retry_max_attempts),
        wait=_WaitWithRetryAfter(config.retry_multiplier, config.retry_max_wait),
        retry=retry_if_exception(_is_retryable_exception),
        before_sleep=before_sleep,
        reraise=True,
    )

SEARCH_PROMPT = """# Role: Search Assistant

Return search results as a JSON array. Each result must have exactly these fields:
- "title": string, result title
- "url": string, valid URL
- "description": string, 20-50 word summary

Output ONLY valid JSON array, no markdown, no explanation.

Example:
[
  {"title": "Example", "url": "https://example.com", "description": "Brief description"}
]
"""

FETCH_PROMPT = """# Role: Web Content Fetcher

Fetch the webpage content and convert to structured Markdown:
- Preserve all headings, paragraphs, lists, tables, code blocks
- Include metadata header: source URL, title, fetch timestamp
- Do NOT summarize - return complete content
- Use UTF-8 encoding
"""

import json
import sys
from datetime import UTC, datetime

from .config import config
from .http import get_http_client, retry_attempts
from .prompts import FETCH_PROMPT, SEARCH_PROMPT


def _get_local_time_info() -> str:
    try:
        local_tz = datetime.now().astimezone().tzinfo
        local_now = datetime.now(local_tz)
    except Exception:
        local_now = datetime.now(UTC)

    weekdays = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"]
    return (
        f"[Current Time Context]\n"
        f"- Date: {local_now.strftime('%Y-%m-%d')} ({weekdays[local_now.weekday()]})\n"
        f"- Time: {local_now.strftime('%H:%M:%S')}\n"
    )


def _needs_time_context(query: str) -> bool:
    keywords = [
        "current",
        "now",
        "today",
        "tomorrow",
        "yesterday",
        "this week",
        "last week",
        "next week",
        "latest",
        "recent",
        "recently",
        "up-to-date",
        "当前",
        "现在",
        "今天",
        "最新",
        "最近",
    ]
    query_lower = query.lower()
    return any(kw in query_lower or kw in query for kw in keywords)


class GrokSearchProvider:
    def __init__(self, api_url: str, api_key: str, model: str):
        self.api_url = api_url.rstrip("/")
        self.api_key = api_key
        self.model = model
        self._headers = {
            "Authorization": f"Bearer {api_key}",
            "Content-Type": "application/json",
        }

    async def search(self, query: str, platform: str = "", min_results: int = 3, max_results: int = 10) -> str:
        platform_prompt = f"\n\nFocus on platforms: {platform}" if platform else ""
        return_prompt = f"\n\nReturn {min_results}-{max_results} results as JSON array."
        time_context = _get_local_time_info() + "\n" if _needs_time_context(query) else ""

        payload = {
            "model": self.model,
            "messages": [
                {"role": "system", "content": SEARCH_PROMPT},
                {"role": "user", "content": time_context + query + platform_prompt + return_prompt},
            ],
        }

        # Enable web search for supported endpoints
        if "openrouter" not in config.grok_api_url:
            # xAI native and compatible endpoints: explicit tools array
            payload["tools"] = [{"type": "web_search"}]

        return await self._execute(payload)

    async def fetch(self, url: str) -> str:
        payload = {
            "model": self.model,
            "messages": [
                {"role": "system", "content": FETCH_PROMPT},
                {"role": "user", "content": f"{url}\n\nFetch and return structured Markdown."},
            ],
        }
        return await self._execute(payload)

    async def _execute(self, payload: dict) -> str:
        try:
            content = await self._execute_stream(payload)
            if not content:
                raise ValueError("Streaming response returned empty content")
            return content
        except Exception as e:
            if config.debug_enabled:
                print(f"[DEBUG] Streaming failed: {e}, falling back to non-streaming", file=sys.stderr)
            return await self._execute_non_stream(payload)

    async def _execute_non_stream(self, payload: dict) -> str:
        payload_copy = {**payload, "stream": False}
        client = await get_http_client()

        async for attempt in retry_attempts(config):
            with attempt:
                response = await client.post(
                    f"{self.api_url}/chat/completions",
                    headers=self._headers,
                    json=payload_copy,
                )
                response.raise_for_status()
                data = response.json()
                choices = data.get("choices", [])
                if choices:
                    return choices[0].get("message", {}).get("content", "")
                return ""

    async def _execute_stream(self, payload: dict) -> str:
        payload_copy = {**payload, "stream": True}
        client = await get_http_client()

        async for attempt in retry_attempts(config):
            with attempt:
                async with client.stream(
                    "POST",
                    f"{self.api_url}/chat/completions",
                    headers=self._headers,
                    json=payload_copy,
                ) as response:
                    response.raise_for_status()
                    return await self._parse_streaming_response(response)

    async def _parse_streaming_response(self, response) -> str:
        content = ""
        full_body_buffer = []

        async for line in response.aiter_lines():
            line = line.strip()
            if not line:
                continue
            full_body_buffer.append(line)

            if line.startswith("data:"):
                if line in ("data: [DONE]", "data:[DONE]"):
                    continue
                try:
                    json_str = line[5:].lstrip()
                    data = json.loads(json_str)
                    choices = data.get("choices", [])
                    if choices:
                        delta = choices[0].get("delta", {})
                        if "content" in delta:
                            content += delta["content"]
                except (json.JSONDecodeError, IndexError):
                    continue

        if not content and full_body_buffer:
            try:
                full_text = "".join(full_body_buffer)
                data = json.loads(full_text)
                if "choices" in data and data["choices"]:
                    message = data["choices"][0].get("message", {})
                    content = message.get("content", "")
            except json.JSONDecodeError:
                pass

        return content

import asyncio
import json
import sys
import time
from pathlib import Path

import httpx

from .config import config
from .http import RETRYABLE_STATUS_CODES, get_http_client, retry_attempts


def _split_csv(value) -> list[str]:
    if not value:
        return []
    if isinstance(value, (list, tuple)):
        return [str(v).strip() for v in value if str(v).strip()]
    return [item.strip() for item in str(value).split(",") if item.strip()]


def _tavily_unavailable_reason() -> str | None:
    if not config.tavily_enabled:
        return "Tavily integration disabled"
    if not config.tavily_api_key:
        return "TAVILY_API_KEY not configured"
    return None


def _tavily_headers() -> dict:
    return {
        "Authorization": f"Bearer {config.tavily_api_key}",
        "Content-Type": "application/json",
    }


def _status_message(action: str, status: int, body: str) -> str:
    if status == 401:
        return "ERROR: TAVILY_API_KEY invalid or missing"
    if status == 429:
        return "ERROR: Tavily quota exceeded (rate limit or usage cap)"
    if status == 432:
        return "ERROR: Tavily account suspended or payment required"
    if status == 400:
        return f"ERROR: Tavily rejected {action} parameters: {body[:200]}"
    return f"ERROR: Tavily {action} HTTP {status}: {body[:200]}"


def _report_terminal_failure(action: str, exc: Exception) -> None:
    """Report a failure that terminated Tavily execution.

    Only annotate retry exhaustion when the failure was actually retryable — 401/403/400/432
    are attempted exactly once, so claiming "after N attempts" there would be a false report.
    """
    if isinstance(exc, httpx.HTTPStatusError):
        status = exc.response.status_code
        msg = _status_message(action, status, exc.response.text)
        if status in RETRYABLE_STATUS_CODES:
            msg = f"{msg} (after {config.retry_max_attempts} attempts)"
        print(msg, file=sys.stderr)
    else:
        print(
            f"ERROR: Tavily {action} failed after {config.retry_max_attempts} attempts: {exc}",
            file=sys.stderr,
        )


def _report_retry(retry_state) -> None:
    exc = retry_state.outcome.exception() if retry_state.outcome else None
    delay = retry_state.next_action.sleep if retry_state.next_action else 0
    print(
        f"WARNING: Tavily attempt {retry_state.attempt_number} failed ({exc}); retrying in {delay:.1f}s",
        file=sys.stderr,
    )


async def _post_tavily_json(endpoint: str, body: dict, request_timeout: httpx.Timeout | None = None) -> dict:
    client = await get_http_client()
    request_kwargs = {"headers": _tavily_headers(), "json": body}
    if request_timeout is not None:
        request_kwargs["timeout"] = request_timeout

    # retry_attempts uses reraise=True, so the loop either returns or raises.
    async for attempt in retry_attempts(config, before_sleep=_report_retry):
        with attempt:
            response = await client.post(endpoint, **request_kwargs)
            response.raise_for_status()
            return response.json()


async def _call_tavily_search(query: str, max_results: int = 6) -> list[dict] | None:
    if _tavily_unavailable_reason():
        return None
    endpoint = f"{config.tavily_api_url.rstrip('/')}/search"
    body = {
        "query": query,
        "max_results": max_results,
        "search_depth": config.tavily_search_depth,
        "chunks_per_source": config.tavily_chunks_per_source,
        "include_raw_content": False,
        "include_answer": config.tavily_include_answer,
    }
    # Optional filters: omitted entirely when unset so the API defaults apply.
    for key, value in (
        ("topic", config.tavily_topic),
        ("time_range", config.tavily_time_range),
        ("include_domains", config.tavily_include_domains),
        ("exclude_domains", config.tavily_exclude_domains),
        ("include_usage", config.tavily_include_usage),
    ):
        if value:
            body[key] = value
    # include_domains_mode requires include_domains; sending it alone is a 400.
    if config.tavily_include_domains:
        body["include_domains_mode"] = config.tavily_include_domains_mode
    try:
        data = await _post_tavily_json(endpoint, body)
        results = data.get("results", [])
        # Lift usage out even when there are no results, so credits spent on an empty
        # search are not silently dropped.
        out = [
            {
                "title": r.get("title", ""),
                "url": r.get("url", ""),
                "content": r.get("content", ""),
                "score": r.get("score", 0),
            }
            for r in results
        ]
        # Surfaced as a sibling key so merge_search_results (which treats this as a result
        # list) can lift it out without mistaking it for a result.
        if data.get("usage"):
            out.append({"usage": data["usage"]})
        return out or None
    except httpx.HTTPStatusError as e:
        _report_terminal_failure("search", e)
        return None
    except (httpx.TimeoutException, httpx.NetworkError) as e:
        _report_terminal_failure("search", e)
        return None


async def _call_tavily_extract(url: str) -> str | None:
    if _tavily_unavailable_reason():
        return None
    endpoint = f"{config.tavily_api_url.rstrip('/')}/extract"
    body = {
        "urls": [url],
        "format": "markdown",
        "extract_depth": config.tavily_extract_depth,
        "timeout": config.tavily_extract_timeout,
    }
    try:
        # Tavily's own `timeout` (max 60s) reports a clean per-URL failure; give the client
        # headroom beyond it so Tavily's diagnosable error wins over an opaque read timeout.
        request_timeout = httpx.Timeout(connect=10.0, read=config.tavily_extract_timeout + 10.0, write=15.0, pool=None)
        data = await _post_tavily_json(endpoint, body, request_timeout)
        results = data.get("results", [])
        failed = data.get("failed_results", [])
        if failed:
            print(f"WARNING: Tavily extract failed for {len(failed)} URL(s): {failed}", file=sys.stderr)
        if results:
            content = results[0].get("raw_content", "")
            return content if content and content.strip() else None
        return None
    except httpx.HTTPStatusError as e:
        _report_terminal_failure("extract", e)
        return None
    except (httpx.TimeoutException, httpx.NetworkError) as e:
        _report_terminal_failure("extract", e)
        return None


async def _call_tavily_crawl(
    url: str,
    instructions: str = "",
    max_depth: int | None = None,
    max_breadth: int | None = None,
    limit: int | None = None,
    timeout: int | None = None,
    select_paths: list[str] | None = None,
    exclude_paths: list[str] | None = None,
) -> str:
    reason = _tavily_unavailable_reason()
    if reason:
        return f"Configuration error: {reason}"
    endpoint = f"{config.tavily_api_url.rstrip('/')}/crawl"
    timeout = config.tavily_crawl_timeout if timeout is None else timeout
    body = {
        "url": url,
        "max_depth": config.tavily_crawl_max_depth if max_depth is None else max_depth,
        "max_breadth": config.tavily_crawl_max_breadth if max_breadth is None else max_breadth,
        "limit": config.tavily_crawl_limit if limit is None else limit,
        "allow_external": config.tavily_crawl_allow_external,
        "extract_depth": config.tavily_extract_depth,
        "format": "markdown",
        "timeout": timeout,
    }
    if config.tavily_include_usage:
        body["include_usage"] = True
    if instructions:
        body["instructions"] = instructions
    for key, value in (
        ("select_paths", _split_csv(config.tavily_crawl_select_paths if select_paths is None else select_paths)),
        ("exclude_paths", _split_csv(config.tavily_crawl_exclude_paths if exclude_paths is None else exclude_paths)),
    ):
        if value:
            body[key] = value
    try:
        # `timeout` here bounds the crawl server-side, so the client must wait longer.
        request_timeout = httpx.Timeout(connect=10.0, read=float(timeout) + 10.0, write=15.0, pool=None)
        data = await _post_tavily_json(endpoint, body, request_timeout)
        payload = {
            "base_url": data.get("base_url", ""),
            "results": data.get("results", []),
            "response_time": data.get("response_time", 0),
        }
        if data.get("usage"):
            payload["usage"] = data["usage"]
        return json.dumps(payload, ensure_ascii=False, indent=2)
    except httpx.TimeoutException:
        return f"Crawl timeout: request exceeded {timeout}s after {config.retry_max_attempts} attempts"
    except httpx.HTTPStatusError as e:
        status = e.response.status_code
        retry_note = f" (after {config.retry_max_attempts} attempts)" if status in RETRYABLE_STATUS_CODES else ""
        return f"HTTP error: {status} - {e.response.text[:200]}{retry_note}"
    except Exception as e:
        return f"Crawl error: {str(e)}"


_SCHEMA_TYPES = ("object", "string", "integer", "number", "array")


def _validate_output_schema(schema: dict) -> str | None:
    """Return an error message if `schema` would be rejected, else None.

    The API requires a 'properties' object; each property needs a type (one of
    object/string/integer/number/array) and a description. 'items' is required when
    type is array, 'properties' when type is object.
    """
    if not isinstance(schema, dict):
        return "output_schema must be a JSON object"
    props = schema.get("properties")
    if not isinstance(props, dict) or not props:
        return "output_schema must include a non-empty 'properties' object"

    def check(name: str, spec: dict, depth: int = 0) -> str | None:
        if not isinstance(spec, dict):
            return f"property '{name}' must be an object"
        ptype = spec.get("type")
        if ptype not in _SCHEMA_TYPES:
            return f"property '{name}' needs a type in {_SCHEMA_TYPES} (got {ptype!r})"
        if not spec.get("description"):
            return f"property '{name}' needs a description"
        if ptype == "array":
            items = spec.get("items")
            if not isinstance(items, dict) or "type" not in items:
                return f"property '{name}' of type array needs 'items' with a type"
        if ptype == "object":
            nested = spec.get("properties")
            if not isinstance(nested, dict) or not nested:
                return f"property '{name}' of type object needs non-empty 'properties'"
            if depth < 8:
                for child, child_spec in nested.items():
                    err = check(child, child_spec, depth + 1)
                    if err:
                        return err
        return None

    for name, spec in props.items():
        err = check(name, spec)
        if err:
            return err
    required = schema.get("required")
    if required is not None:
        if not isinstance(required, list) or not required:
            return "'required' must be a non-empty array when present"
        unknown = [r for r in required if r not in props]
        if unknown:
            return f"'required' names not present in properties: {unknown}"
    return None


def _load_output_schema() -> dict | None:
    """Read TAVILY_RESEARCH_OUTPUT_SCHEMA (a JSON file path). Raises ValueError if invalid."""
    path = config.tavily_research_output_schema
    if not path:
        return None
    try:
        with open(path, encoding="utf-8") as f:
            schema = json.load(f)
    except OSError as e:
        raise ValueError(f"Cannot read TAVILY_RESEARCH_OUTPUT_SCHEMA file {path}: {e}") from e
    except json.JSONDecodeError as e:
        raise ValueError(f"TAVILY_RESEARCH_OUTPUT_SCHEMA file {path} is not valid JSON: {e}") from e
    error = _validate_output_schema(schema)
    if error:
        raise ValueError(f"Invalid TAVILY_RESEARCH_OUTPUT_SCHEMA ({path}): {error}")
    return schema


async def _call_tavily_research(
    input_text: str,
    model: str | None = None,
    output_length: str | None = None,
    citation_format: str | None = None,
    output_schema: dict | None = None,
    include_domains: list[str] | None = None,
    exclude_domains: list[str] | None = None,
) -> str:
    reason = _tavily_unavailable_reason()
    if reason:
        return f"Configuration error: {reason}"
    if output_schema is None:
        try:
            output_schema = _load_output_schema()
        except ValueError as e:
            return f"Configuration error: {e}"
    elif isinstance(output_schema, str):
        # CLI passes a file path; resolve and validate it the same way.
        try:
            schema = json.loads(Path(output_schema).read_text(encoding="utf-8"))
        except (OSError, json.JSONDecodeError) as e:
            return f"Configuration error: cannot read output_schema file {output_schema}: {e}"
        if error := _validate_output_schema(schema):
            return f"Configuration error: invalid output_schema ({output_schema}): {error}"
        output_schema = schema
    elif error := _validate_output_schema(output_schema):
        return f"Configuration error: invalid output_schema: {error}"
    base = config.tavily_api_url.rstrip("/")
    body = {
        "input": input_text,
        "model": config.tavily_research_model if model is None else model,
        "citation_format": config.tavily_research_citation_format if citation_format is None else citation_format,
        "output_length": config.tavily_research_output_length if output_length is None else output_length,
    }
    if output_schema is not None:
        body["output_schema"] = output_schema
    for key, value in (
        ("include_domains", _split_csv(config.tavily_include_domains if include_domains is None else include_domains)),
        ("exclude_domains", _split_csv(config.tavily_exclude_domains if exclude_domains is None else exclude_domains)),
    ):
        if value:
            body[key] = value

    try:
        created = await _post_tavily_json(f"{base}/research", body)
    except httpx.HTTPStatusError as e:
        return f"HTTP error: {e.response.status_code} - {e.response.text[:200]}"
    except (httpx.TimeoutException, httpx.NetworkError) as e:
        return f"Research submit failed: {e}"

    request_id = created.get("request_id")
    if not request_id:
        return json.dumps({"error": "Research task rejected", "response": created}, ensure_ascii=False, indent=2)

    # Research is asynchronous: poll until completed/failed or the budget runs out.
    deadline = time.monotonic() + config.tavily_research_timeout
    client = await get_http_client()
    while True:
        await asyncio.sleep(min(config.tavily_research_poll_interval, max(0.0, deadline - time.monotonic())))
        if time.monotonic() >= deadline:
            return json.dumps(
                {
                    "request_id": request_id,
                    "status": "timeout",
                    "message": f"Not finished within {config.tavily_research_timeout}s",
                },
                ensure_ascii=False,
                indent=2,
            )
        params = {"include_usage": "true"} if config.tavily_include_usage else None
        try:
            response = await client.get(
                f"{base}/research/{request_id}",
                headers=_tavily_headers(),
                params=params,
            )
            response.raise_for_status()
        except httpx.HTTPStatusError as e:
            return json.dumps(
                {
                    "request_id": request_id,
                    "status": "poll_error",
                    "error": f"HTTP {e.response.status_code} - {e.response.text[:200]}",
                },
                ensure_ascii=False,
                indent=2,
            )
        except (httpx.TimeoutException, httpx.NetworkError) as e:
            # The task was created and may still be consuming credits; the caller needs
            # request_id to re-fetch it, so poll failures keep the JSON contract too.
            return json.dumps(
                {
                    "request_id": request_id,
                    "status": "poll_error",
                    "error": f"{e}",
                },
                ensure_ascii=False,
                indent=2,
            )

        data = response.json()
        status = data.get("status")
        if status in ("completed", "failed"):
            if config.tavily_include_usage and not data.get("usage"):
                print("WARNING: Tavily research usage unavailable for this request", file=sys.stderr)
            return json.dumps(data, ensure_ascii=False, indent=2)


async def _call_tavily_map(
    url: str,
    instructions: str = "",
    max_depth: int = 1,
    max_breadth: int = 20,
    limit: int = 50,
    timeout: int = 150,
) -> str:
    reason = _tavily_unavailable_reason()
    if reason:
        return f"Configuration error: {reason}"
    endpoint = f"{config.tavily_api_url.rstrip('/')}/map"
    body = {
        "url": url,
        "max_depth": max_depth,
        "max_breadth": max_breadth,
        "limit": limit,
        "timeout": timeout,
    }
    if config.tavily_include_usage:
        body["include_usage"] = True
    if instructions:
        body["instructions"] = instructions
    try:
        request_timeout = httpx.Timeout(connect=10.0, read=float(timeout) + 5.0, write=15.0, pool=None)
        data = await _post_tavily_json(endpoint, body, request_timeout)
        payload = {
            "base_url": data.get("base_url", ""),
            "results": data.get("results", []),
            "response_time": data.get("response_time", 0),
        }
        if data.get("usage"):
            payload["usage"] = data["usage"]
        return json.dumps(payload, ensure_ascii=False, indent=2)
    except httpx.TimeoutException:
        return f"Map timeout: request exceeded {timeout}s after {config.retry_max_attempts} attempts"
    except httpx.HTTPStatusError as e:
        status = e.response.status_code
        retry_note = f" (after {config.retry_max_attempts} attempts)" if status in RETRYABLE_STATUS_CODES else ""
        return f"HTTP error: {status} - {e.response.text[:200]}{retry_note}"
    except Exception as e:
        return f"Map error: {str(e)}"

"""Internal modules for the GrokSearch CLI."""

"""Tests for grok-search CLI: Config + Tavily + commands."""

import json
import sys
from pathlib import Path
from unittest.mock import AsyncMock, MagicMock

import httpx
import pytest

SCRIPTS_DIR = Path(__file__).parent.parent
sys.path.insert(0, str(SCRIPTS_DIR))


@pytest.fixture(autouse=True)
def reset_config_singleton(monkeypatch):
    """Reset Config singleton state and clear env vars between tests."""
    for k in [
        "GROK_API_URL",
        "GROK_API_KEY",
        "GROK_MODEL",
        "GROK_DEBUG",
        "TAVILY_API_URL",
        "TAVILY_API_KEY",
        "TAVILY_ENABLED",
        "TAVILY_SEARCH_DEPTH",
        "TAVILY_CHUNKS_PER_SOURCE",
        "TAVILY_INCLUDE_ANSWER",
        "TAVILY_EXTRACT_TIMEOUT",
        "TAVILY_EXTRACT_DEPTH",
        "TAVILY_TOPIC",
        "TAVILY_TIME_RANGE",
        "TAVILY_INCLUDE_DOMAINS",
        "TAVILY_EXCLUDE_DOMAINS",
        "TAVILY_INCLUDE_DOMAINS_MODE",
        "TAVILY_INCLUDE_USAGE",
        "TAVILY_CRAWL_TIMEOUT",
        "TAVILY_CRAWL_MAX_DEPTH",
        "TAVILY_CRAWL_MAX_BREADTH",
        "TAVILY_CRAWL_LIMIT",
        "TAVILY_CRAWL_ALLOW_EXTERNAL",
        "TAVILY_CRAWL_SELECT_PATHS",
        "TAVILY_CRAWL_EXCLUDE_PATHS",
        "TAVILY_RESEARCH_MODEL",
        "TAVILY_RESEARCH_CITATION_FORMAT",
        "TAVILY_RESEARCH_OUTPUT_LENGTH",
        "TAVILY_RESEARCH_TIMEOUT",
        "TAVILY_RESEARCH_POLL_INTERVAL",
        "GROK_RETRY_MAX_ATTEMPTS",
        "GROK_RETRY_MULTIPLIER",
        "GROK_RETRY_MAX_WAIT",
    ]:
        monkeypatch.delenv(k, raising=False)
    import groksearch_cli

    groksearch_cli.Config._instance = None
    yield
    groksearch_cli.Config._instance = None


# ============================================================================
# Task 1.1: retry_* properties
# ============================================================================


class TestRetryConfig:
    def test_default_retry_values(self):
        from groksearch_cli import Config

        cfg = Config()
        assert cfg.retry_max_attempts == 3
        assert cfg.retry_multiplier == 1.0
        assert cfg.retry_max_wait == 10

    def test_env_overrides_retry(self, monkeypatch):
        monkeypatch.setenv("GROK_RETRY_MAX_ATTEMPTS", "5")
        monkeypatch.setenv("GROK_RETRY_MULTIPLIER", "2.5")
        monkeypatch.setenv("GROK_RETRY_MAX_WAIT", "20")
        from groksearch_cli import Config

        cfg = Config()
        assert cfg.retry_max_attempts == 5
        assert cfg.retry_multiplier == 2.5
        assert cfg.retry_max_wait == 20


# ============================================================================
# Task 1.2: tavily_* config properties
# ============================================================================


class TestTavilyConfig:
    def test_tavily_api_url_default(self):
        from groksearch_cli import Config

        assert Config().tavily_api_url == "https://api.tavily.com"

    def test_search_tuning_defaults(self):
        from groksearch_cli import Config

        cfg = Config()
        assert cfg.tavily_search_depth == "advanced"
        assert cfg.tavily_chunks_per_source == 3
        assert cfg.tavily_include_answer is False
        assert cfg.tavily_extract_timeout == 30.0

    def test_search_tuning_env_overrides(self, monkeypatch):
        monkeypatch.setenv("TAVILY_SEARCH_DEPTH", "ultra-fast")
        monkeypatch.setenv("TAVILY_CHUNKS_PER_SOURCE", "5")
        monkeypatch.setenv("TAVILY_INCLUDE_ANSWER", "advanced")
        monkeypatch.setenv("TAVILY_EXTRACT_TIMEOUT", "45")
        from groksearch_cli import Config

        cfg = Config()
        assert cfg.tavily_search_depth == "ultra-fast"
        assert cfg.tavily_chunks_per_source == 5
        assert cfg.tavily_include_answer == "advanced"
        assert cfg.tavily_extract_timeout == 45.0

    @pytest.mark.parametrize(
        "raw,expected",
        [
            ("false", False),
            ("0", False),
            ("no", False),
            ("", False),
            ("true", "basic"),
            ("1", "basic"),
            ("basic", "basic"),
            ("advanced", "advanced"),
        ],
    )
    def test_include_answer_aliases(self, monkeypatch, raw, expected):
        monkeypatch.setenv("TAVILY_INCLUDE_ANSWER", raw)
        from groksearch_cli import Config

        value = Config().tavily_include_answer
        assert value == expected
        # Tavily rejects the string "false" with a 400 — the disabled state must be boolean.
        if expected is False:
            assert value is False

    def test_invalid_search_depth_rejected(self, monkeypatch):
        monkeypatch.setenv("TAVILY_SEARCH_DEPTH", "deep")
        from groksearch_cli import Config

        with pytest.raises(ValueError, match="TAVILY_SEARCH_DEPTH"):
            _ = Config().tavily_search_depth

    def test_invalid_include_answer_rejected(self, monkeypatch):
        monkeypatch.setenv("TAVILY_INCLUDE_ANSWER", "maybe")
        from groksearch_cli import Config

        with pytest.raises(ValueError, match="TAVILY_INCLUDE_ANSWER"):
            _ = Config().tavily_include_answer

    def test_filter_defaults(self):
        from groksearch_cli import Config

        cfg = Config()
        assert cfg.tavily_topic == "general"
        assert cfg.tavily_time_range == ""
        assert cfg.tavily_include_domains == []
        assert cfg.tavily_exclude_domains == []
        assert cfg.tavily_include_domains_mode == "filter"
        assert cfg.tavily_include_usage is False
        assert cfg.tavily_extract_depth == "basic"

    def test_domain_lists_parsed_from_csv(self, monkeypatch):
        monkeypatch.setenv("TAVILY_INCLUDE_DOMAINS", "reuters.com, bloomberg.com ,")
        monkeypatch.setenv("TAVILY_EXCLUDE_DOMAINS", "espn.com")
        from groksearch_cli import Config

        cfg = Config()
        assert cfg.tavily_include_domains == ["reuters.com", "bloomberg.com"]
        assert cfg.tavily_exclude_domains == ["espn.com"]

    def test_chunks_per_source_out_of_range_rejected(self, monkeypatch):
        # Live API: "chunks_per_source must be an integer between 1 and 5, or 'auto'"
        monkeypatch.setenv("TAVILY_CHUNKS_PER_SOURCE", "6")
        from groksearch_cli import Config

        with pytest.raises(ValueError, match="TAVILY_CHUNKS_PER_SOURCE"):
            _ = Config().tavily_chunks_per_source

    @pytest.mark.parametrize(
        "var,value",
        [
            ("TAVILY_TOPIC", "bogus"),
            ("TAVILY_TIME_RANGE", "decade"),
            ("TAVILY_INCLUDE_DOMAINS_MODE", "strict"),
            ("TAVILY_EXTRACT_DEPTH", "deep"),
        ],
    )
    def test_invalid_filter_values_surface_in_config_info(self, monkeypatch, var, value):
        """A bad tuning value must be reported in the dump, not crash it."""
        monkeypatch.setenv(var, value)
        from groksearch_cli import Config

        info = Config().get_config_info()
        assert var in info
        assert "❌" in info[var], f"{var} should be flagged as invalid"

    def test_domains_mode_omitted_without_include_domains(self, monkeypatch):
        """include_domains_mode alone is a 400 — it must not be sent unless domains are set."""
        import asyncio

        monkeypatch.setenv("TAVILY_API_KEY", "tvly-test")
        monkeypatch.setenv("TAVILY_INCLUDE_DOMAINS_MODE", "boost")
        import groksearch_cli

        groksearch_cli.Config._instance = None

        mock_response = MagicMock()
        mock_response.raise_for_status = MagicMock()
        mock_response.json = MagicMock(return_value={"results": []})
        mock_client = AsyncMock()
        mock_client.post = AsyncMock(return_value=mock_response)

        async def _get_client():
            return mock_client

        monkeypatch.setattr(groksearch_cli, "get_http_client", _get_client)

        asyncio.run(groksearch_cli._call_tavily_search("q"))
        body = mock_client.post.call_args.kwargs["json"]
        assert "include_domains_mode" not in body
        assert "include_domains" not in body
        # Unset optional filters must be omitted entirely, not sent as empty values.
        assert "topic" in body and body["topic"] == "general"
        assert "time_range" not in body
        assert "include_usage" not in body

    def test_filters_included_when_configured(self, monkeypatch):
        import asyncio

        monkeypatch.setenv("TAVILY_API_KEY", "tvly-test")
        monkeypatch.setenv("TAVILY_INCLUDE_DOMAINS", "reuters.com,bloomberg.com")
        monkeypatch.setenv("TAVILY_INCLUDE_DOMAINS_MODE", "boost")
        monkeypatch.setenv("TAVILY_EXCLUDE_DOMAINS", "espn.com")
        monkeypatch.setenv("TAVILY_TIME_RANGE", "month")
        monkeypatch.setenv("TAVILY_INCLUDE_USAGE", "true")
        import groksearch_cli

        groksearch_cli.Config._instance = None

        mock_response = MagicMock()
        mock_response.raise_for_status = MagicMock()
        mock_response.json = MagicMock(return_value={"results": []})
        mock_client = AsyncMock()
        mock_client.post = AsyncMock(return_value=mock_response)

        async def _get_client():
            return mock_client

        monkeypatch.setattr(groksearch_cli, "get_http_client", _get_client)

        asyncio.run(groksearch_cli._call_tavily_search("q"))
        body = mock_client.post.call_args.kwargs["json"]
        assert body["include_domains"] == ["reuters.com", "bloomberg.com"]
        assert body["include_domains_mode"] == "boost"
        assert body["exclude_domains"] == ["espn.com"]
        assert body["time_range"] == "month"
        assert body["include_usage"] is True

    def test_tavily_api_url_override(self, monkeypatch):
        monkeypatch.setenv("TAVILY_API_URL", "https://custom.tavily/v2")
        from groksearch_cli import Config

        assert Config().tavily_api_url == "https://custom.tavily/v2"

    def test_tavily_api_key_unset_returns_none(self):
        from groksearch_cli import Config

        assert Config().tavily_api_key is None

    def test_tavily_api_key_set(self, monkeypatch):
        monkeypatch.setenv("TAVILY_API_KEY", "tvly-abc123")
        from groksearch_cli import Config

        assert Config().tavily_api_key == "tvly-abc123"

    def test_tavily_enabled_default_true(self):
        from groksearch_cli import Config

        assert Config().tavily_enabled is True

    def test_tavily_enabled_false(self, monkeypatch):
        monkeypatch.setenv("TAVILY_ENABLED", "false")
        from groksearch_cli import Config

        assert Config().tavily_enabled is False

    @pytest.mark.parametrize(
        "var,raw",
        [
            ("TAVILY_CRAWL_TIMEOUT", "abc"),
            ("TAVILY_CRAWL_TIMEOUT", "500"),
            ("TAVILY_CRAWL_TIMEOUT", "nan"),
            ("TAVILY_CRAWL_LIMIT", "0"),
            ("TAVILY_EXTRACT_TIMEOUT", "61"),
            ("TAVILY_RESEARCH_POLL_INTERVAL", "-1"),
        ],
    )
    def test_numeric_tuning_out_of_range_rejected(self, monkeypatch, var, raw):
        """Numeric tuning must reject non-finite / out-of-range values with a named error."""
        monkeypatch.setenv(var, raw)
        from groksearch_cli import Config

        prop = {
            "TAVILY_CRAWL_TIMEOUT": "tavily_crawl_timeout",
            "TAVILY_CRAWL_LIMIT": "tavily_crawl_limit",
            "TAVILY_EXTRACT_TIMEOUT": "tavily_extract_timeout",
            "TAVILY_RESEARCH_POLL_INTERVAL": "tavily_research_poll_interval",
        }[var]
        with pytest.raises(ValueError, match=var):
            _ = getattr(Config(), prop)


# ============================================================================
# Task 1.3-1.4: _apply_model_suffix and grok_model integration
# ============================================================================


class TestModelSuffix:
    def test_openrouter_appends_online(self, monkeypatch, tmp_path):
        monkeypatch.setenv("GROK_API_URL", "https://openrouter.ai/api/v1")
        monkeypatch.setenv("GROK_API_KEY", "sk-test")
        monkeypatch.setenv("GROK_MODEL", "grok-4-fast")
        monkeypatch.setenv("HOME", str(tmp_path))
        monkeypatch.setenv("USERPROFILE", str(tmp_path))
        from groksearch_cli import Config

        assert Config().grok_model == "grok-4-fast:online"

    def test_openrouter_with_existing_online_no_double(self, monkeypatch, tmp_path):
        monkeypatch.setenv("GROK_API_URL", "https://openrouter.ai/api/v1")
        monkeypatch.setenv("GROK_API_KEY", "sk-test")
        monkeypatch.setenv("GROK_MODEL", "grok-4-fast:online")
        monkeypatch.setenv("HOME", str(tmp_path))
        monkeypatch.setenv("USERPROFILE", str(tmp_path))
        from groksearch_cli import Config

        assert Config().grok_model == "grok-4-fast:online"

    def test_non_openrouter_no_suffix(self, monkeypatch, tmp_path):
        monkeypatch.setenv("GROK_API_URL", "https://api.x.ai/v1")
        monkeypatch.setenv("GROK_API_KEY", "sk-test")
        monkeypatch.setenv("GROK_MODEL", "grok-4-fast")
        monkeypatch.setenv("HOME", str(tmp_path))
        monkeypatch.setenv("USERPROFILE", str(tmp_path))
        from groksearch_cli import Config

        assert Config().grok_model == "grok-4-fast"


# ============================================================================
# Task 1.5: get_config_info exposes Tavily fields
# ============================================================================


class TestConfigInfoOutput:
    def test_get_config_info_includes_tavily_fields(self, monkeypatch):
        monkeypatch.setenv("GROK_API_URL", "https://api.x.ai/v1")
        monkeypatch.setenv("GROK_API_KEY", "sk-grok-secret")
        monkeypatch.setenv("TAVILY_API_KEY", "tvly-secretkey1234")
        from groksearch_cli import Config

        info = Config().get_config_info()
        assert "TAVILY_API_URL" in info
        assert "TAVILY_ENABLED" in info
        assert "TAVILY_API_KEY" in info
        # Masked
        assert info["TAVILY_API_KEY"] != "tvly-secretkey1234"
        assert "tvly" in info["TAVILY_API_KEY"]
        assert "1234" in info["TAVILY_API_KEY"]

    def test_get_config_info_tavily_unset_label(self, monkeypatch):
        monkeypatch.setenv("GROK_API_URL", "https://api.x.ai/v1")
        monkeypatch.setenv("GROK_API_KEY", "sk-test")
        from groksearch_cli import Config

        info = Config().get_config_info()
        assert info["TAVILY_API_KEY"] in ("Not configured", "未配置")

    def test_get_config_info_invalid_value_is_per_key(self, monkeypatch):
        """One bad tuning value must flag only its own key, not the whole tuning block."""
        monkeypatch.setenv("GROK_API_URL", "https://api.x.ai/v1")
        monkeypatch.setenv("GROK_API_KEY", "sk-test")
        monkeypatch.setenv("TAVILY_TOPIC", "bogus")
        from groksearch_cli import Config

        info = Config().get_config_info()
        assert "❌" in info["TAVILY_TOPIC"]
        assert info["TAVILY_SEARCH_DEPTH"] == "advanced"
        assert info["TAVILY_CRAWL_LIMIT"] == 50


# ============================================================================
# Task 2: Tavily call functions
# ============================================================================


class TestTavilyCallFunctions:
    @pytest.mark.asyncio
    async def test_call_tavily_search_returns_none_when_no_key(self):
        from groksearch_cli import _call_tavily_search

        result = await _call_tavily_search("query", max_results=3)
        assert result is None

    @pytest.mark.asyncio
    async def test_call_tavily_search_returns_results(self, monkeypatch):
        monkeypatch.setenv("TAVILY_API_KEY", "tvly-test")
        import groksearch_cli

        groksearch_cli.Config._instance = None

        mock_response = MagicMock()
        mock_response.raise_for_status = MagicMock()
        mock_response.json = MagicMock(
            return_value={"results": [{"title": "Test", "url": "https://example.com", "content": "Body", "score": 0.9}]}
        )

        mock_client = AsyncMock()
        mock_client.post = AsyncMock(return_value=mock_response)

        async def _get_client():
            return mock_client

        monkeypatch.setattr(groksearch_cli, "get_http_client", _get_client)

        result = await groksearch_cli._call_tavily_search("test query", max_results=3)
        assert result == [{"title": "Test", "url": "https://example.com", "content": "Body", "score": 0.9}]
        # Verify body
        call_args = mock_client.post.call_args
        body = call_args.kwargs["json"]
        assert body["query"] == "test query"
        assert body["max_results"] == 3
        assert body["search_depth"] == "advanced"
        assert body["chunks_per_source"] == 3
        assert body["include_raw_content"] is False
        assert body["include_answer"] is False

    @pytest.mark.asyncio
    async def test_search_body_uses_tuning_config(self, monkeypatch):
        monkeypatch.setenv("TAVILY_API_KEY", "tvly-test")
        monkeypatch.setenv("TAVILY_SEARCH_DEPTH", "ultra-fast")
        monkeypatch.setenv("TAVILY_CHUNKS_PER_SOURCE", "5")
        monkeypatch.setenv("TAVILY_INCLUDE_ANSWER", "advanced")
        import groksearch_cli

        groksearch_cli.Config._instance = None

        mock_response = MagicMock()
        mock_response.raise_for_status = MagicMock()
        mock_response.json = MagicMock(return_value={"results": []})
        mock_client = AsyncMock()
        mock_client.post = AsyncMock(return_value=mock_response)

        async def _get_client():
            return mock_client

        monkeypatch.setattr(groksearch_cli, "get_http_client", _get_client)

        await groksearch_cli._call_tavily_search("q")
        body = mock_client.post.call_args.kwargs["json"]
        assert body["search_depth"] == "ultra-fast"
        assert body["chunks_per_source"] == 5
        assert body["include_answer"] == "advanced"

    @pytest.mark.asyncio
    async def test_call_tavily_search_exception_returns_none(self, monkeypatch, capsys):
        monkeypatch.setenv("TAVILY_API_KEY", "tvly-test")
        import groksearch_cli

        groksearch_cli.Config._instance = None

        mock_client = AsyncMock()
        mock_client.post = AsyncMock(side_effect=httpx.NetworkError("network"))

        async def _get_client():
            return mock_client

        monkeypatch.setattr(groksearch_cli, "get_http_client", _get_client)

        result = await groksearch_cli._call_tavily_search("q")
        assert result is None
        captured = capsys.readouterr()
        # Contract: the message must disclose retry exhaustion, not read as a first-attempt failure.
        assert "after 3 attempts" in captured.err
        assert "network" in captured.err

    @pytest.mark.asyncio
    async def test_call_tavily_search_retries_persistent_network_error(self, monkeypatch, capsys):
        """A network error that survives every retry reports exhaustion, not a transient failure."""
        monkeypatch.setenv("TAVILY_API_KEY", "tvly-test")
        import groksearch_cli

        groksearch_cli.Config._instance = None

        attempts = {"n": 0}
        mock_client = AsyncMock()

        async def _post(*a, **kw):
            attempts["n"] += 1
            raise httpx.ConnectError("persistent")

        mock_client.post = _post

        async def _get_client():
            return mock_client

        monkeypatch.setattr(groksearch_cli, "get_http_client", _get_client)

        result = await groksearch_cli._call_tavily_search("q")
        assert result is None
        assert attempts["n"] == 3, "all retry attempts must be made before reporting"
        err = capsys.readouterr().err
        assert "3 attempts" in err

    @pytest.mark.asyncio
    async def test_call_tavily_search_429_reports_after_retries(self, monkeypatch, capsys):
        """429 is in RETRYABLE_STATUS_CODES, so it is retried before being reported."""
        monkeypatch.setenv("TAVILY_API_KEY", "tvly-test")
        import groksearch_cli

        groksearch_cli.Config._instance = None

        attempts = {"n": 0}
        mock_client = AsyncMock()

        async def _post(*a, **kw):
            attempts["n"] += 1
            req = httpx.Request("POST", "https://api.tavily.com/search")
            raise httpx.HTTPStatusError("429", request=req, response=httpx.Response(429, request=req))

        mock_client.post = _post

        async def _get_client():
            return mock_client

        monkeypatch.setattr(groksearch_cli, "get_http_client", _get_client)

        result = await groksearch_cli._call_tavily_search("q")
        assert result is None
        assert attempts["n"] == 3
        err = capsys.readouterr().err
        assert "quota exceeded" in err
        assert "3 attempts" in err

    @pytest.mark.asyncio
    async def test_call_tavily_search_non_retryable_401_never_claims_retries(self, monkeypatch, capsys):
        """401/403/400/432 are attempted exactly once — claiming '(after N attempts)' would be false."""
        monkeypatch.setenv("TAVILY_API_KEY", "tvly-test")
        import groksearch_cli

        groksearch_cli.Config._instance = None

        attempts = {"n": 0}
        mock_client = AsyncMock()

        async def _post(*a, **kw):
            attempts["n"] += 1
            req = httpx.Request("POST", "https://api.tavily.com/search")
            raise httpx.HTTPStatusError("401", request=req, response=httpx.Response(401, request=req))

        mock_client.post = _post

        async def _get_client():
            return mock_client

        monkeypatch.setattr(groksearch_cli, "get_http_client", _get_client)

        result = await groksearch_cli._call_tavily_search("q")
        assert result is None
        assert attempts["n"] == 1, "401 is not retryable and must not enter the retry loop"
        err = capsys.readouterr().err
        assert "TAVILY_API_KEY invalid or missing" in err
        assert "attempts" not in err, "a single-attempt fatal error must not be labelled as retry-exhausted"

    @pytest.mark.asyncio
    async def test_call_tavily_search_reports_intermediate_retries(self, monkeypatch, capsys):
        """Intermediate attempts must be visible: a 3-attempt failure must not look like a 1-attempt one."""
        monkeypatch.setenv("TAVILY_API_KEY", "tvly-test")
        import groksearch_cli

        groksearch_cli.Config._instance = None

        mock_client = AsyncMock()

        async def _post(*a, **kw):
            raise httpx.ConnectError("persistent")

        mock_client.post = _post

        async def _get_client():
            return mock_client

        monkeypatch.setattr(groksearch_cli, "get_http_client", _get_client)

        await groksearch_cli._call_tavily_search("q")
        err = capsys.readouterr().err
        assert "attempt 1 failed" in err
        assert "attempt 2 failed" in err
        # The final attempt fails without scheduling another retry, so no "attempt 3 failed" progress line.
        assert "attempt 3 failed" not in err

    @pytest.mark.asyncio
    async def test_call_tavily_extract_returns_content(self, monkeypatch):
        monkeypatch.setenv("TAVILY_API_KEY", "tvly-test")
        import groksearch_cli

        groksearch_cli.Config._instance = None

        mock_response = MagicMock()
        mock_response.raise_for_status = MagicMock()
        mock_response.json = MagicMock(return_value={"results": [{"raw_content": "# Page\nContent here"}]})
        mock_client = AsyncMock()
        mock_client.post = AsyncMock(return_value=mock_response)

        async def _get_client():
            return mock_client

        monkeypatch.setattr(groksearch_cli, "get_http_client", _get_client)

        result = await groksearch_cli._call_tavily_extract("https://example.com")
        assert result == "# Page\nContent here"
        body = mock_client.post.call_args.kwargs["json"]
        assert body["urls"] == ["https://example.com"]
        assert body["format"] == "markdown"
        # Tavily's own timeout (API range 1.0-60.0) must be sent, not left to the default.
        assert body["timeout"] == 30.0
        # The HTTP client must outlast Tavily's timeout so Tavily's error wins over ours.
        client_timeout = mock_client.post.call_args.kwargs["timeout"]
        assert client_timeout.read > body["timeout"]

    @pytest.mark.asyncio
    async def test_call_tavily_extract_timeout_configurable(self, monkeypatch):
        monkeypatch.setenv("TAVILY_API_KEY", "tvly-test")
        monkeypatch.setenv("TAVILY_EXTRACT_TIMEOUT", "55")
        import groksearch_cli

        groksearch_cli.Config._instance = None

        mock_response = MagicMock()
        mock_response.raise_for_status = MagicMock()
        mock_response.json = MagicMock(return_value={"results": [{"raw_content": "x"}]})
        mock_client = AsyncMock()
        mock_client.post = AsyncMock(return_value=mock_response)

        async def _get_client():
            return mock_client

        monkeypatch.setattr(groksearch_cli, "get_http_client", _get_client)

        await groksearch_cli._call_tavily_extract("https://example.com")
        body = mock_client.post.call_args.kwargs["json"]
        assert body["timeout"] == 55.0

    @pytest.mark.asyncio
    async def test_call_tavily_extract_no_key_returns_none(self):
        from groksearch_cli import _call_tavily_extract

        result = await _call_tavily_extract("https://example.com")
        assert result is None

    @pytest.mark.asyncio
    async def test_call_tavily_map_returns_json_string(self, monkeypatch):
        monkeypatch.setenv("TAVILY_API_KEY", "tvly-test")
        import groksearch_cli

        groksearch_cli.Config._instance = None

        mock_response = MagicMock()
        mock_response.raise_for_status = MagicMock()
        mock_response.json = MagicMock(
            return_value={
                "base_url": "https://docs.python.org",
                "results": ["https://docs.python.org/3"],
                "response_time": 1.2,
            }
        )
        mock_client = AsyncMock()
        mock_client.post = AsyncMock(return_value=mock_response)

        async def _get_client():
            return mock_client

        monkeypatch.setattr(groksearch_cli, "get_http_client", _get_client)

        result = await groksearch_cli._call_tavily_map(
            "https://docs.python.org", instructions="api docs", max_depth=2, max_breadth=10, limit=20, timeout=60
        )
        data = json.loads(result)
        assert data["base_url"] == "https://docs.python.org"
        body = mock_client.post.call_args.kwargs["json"]
        assert body["url"] == "https://docs.python.org"
        assert body["max_depth"] == 2
        assert body["max_breadth"] == 10
        assert body["limit"] == 20
        assert body["timeout"] == 60
        assert body["instructions"] == "api docs"

    @pytest.mark.asyncio
    async def test_call_tavily_map_no_key_returns_error_string(self):
        from groksearch_cli import _call_tavily_map

        result = await _call_tavily_map(
            "https://x.com", instructions="", max_depth=1, max_breadth=20, limit=50, timeout=150
        )
        assert "TAVILY_API_KEY" in result


# ============================================================================
# Task 3: web_search --extra-sources merging
# ============================================================================


class TestWebSearchExtraSources:
    @pytest.mark.asyncio
    async def test_extra_sources_merges_results(self, monkeypatch, capsys):
        monkeypatch.setenv("GROK_API_URL", "https://api.x.ai/v1")
        monkeypatch.setenv("GROK_API_KEY", "sk-test")
        monkeypatch.setenv("TAVILY_API_KEY", "tvly-test")
        import groksearch_cli

        groksearch_cli.Config._instance = None

        async def fake_grok_search(*args, **kwargs):
            return json.dumps(
                [
                    {"title": "Grok1", "url": "https://a.com", "description": "A"},
                ]
            )

        async def fake_tavily(query, max_results=6):
            return [
                {"title": "Tav1", "url": "https://a.com", "content": "dup"},
                {"title": "Tav2", "url": "https://b.com", "content": "B"},
            ]

        monkeypatch.setattr(groksearch_cli.GrokSearchProvider, "search", fake_grok_search)
        monkeypatch.setattr(groksearch_cli, "_call_tavily_search", fake_tavily)

        args = MagicMock(query="test", platform="", min_results=3, max_results=10, extra_sources=2, raw=False)
        await groksearch_cli.cmd_web_search(args)
        out = capsys.readouterr().out
        data = json.loads(out)
        urls = [d["url"] for d in data]
        assert "https://a.com" in urls
        assert "https://b.com" in urls
        # Tavily-marked entries
        providers = [d.get("provider") for d in data]
        assert "tavily" in providers
        # Grok url should appear only once (dedup)
        assert urls.count("https://a.com") == 1

    @pytest.mark.asyncio
    async def test_extra_sources_zero_no_tavily_call(self, monkeypatch, capsys):
        monkeypatch.setenv("GROK_API_URL", "https://api.x.ai/v1")
        monkeypatch.setenv("GROK_API_KEY", "sk-test")
        monkeypatch.setenv("TAVILY_API_KEY", "tvly-test")
        import groksearch_cli

        groksearch_cli.Config._instance = None

        called = {"tavily": 0}

        async def fake_grok_search(*a, **kw):
            return json.dumps([{"title": "G", "url": "https://g.com", "description": "G"}])

        async def fake_tavily(*a, **kw):
            called["tavily"] += 1
            return None

        monkeypatch.setattr(groksearch_cli.GrokSearchProvider, "search", fake_grok_search)
        monkeypatch.setattr(groksearch_cli, "_call_tavily_search", fake_tavily)

        args = MagicMock(query="test", platform="", min_results=3, max_results=10, extra_sources=0, raw=False)
        await groksearch_cli.cmd_web_search(args)
        assert called["tavily"] == 0

    @pytest.mark.asyncio
    async def test_extra_sources_tavily_failure_does_not_block(self, monkeypatch, capsys):
        monkeypatch.setenv("GROK_API_URL", "https://api.x.ai/v1")
        monkeypatch.setenv("GROK_API_KEY", "sk-test")
        monkeypatch.setenv("TAVILY_API_KEY", "tvly-test")
        import groksearch_cli

        groksearch_cli.Config._instance = None

        async def fake_grok_search(*a, **kw):
            return json.dumps([{"title": "G", "url": "https://g.com", "description": "G"}])

        async def fake_tavily(*a, **kw):
            return None  # simulate failure

        monkeypatch.setattr(groksearch_cli.GrokSearchProvider, "search", fake_grok_search)
        monkeypatch.setattr(groksearch_cli, "_call_tavily_search", fake_tavily)

        args = MagicMock(query="t", platform="", min_results=3, max_results=10, extra_sources=3, raw=False)
        await groksearch_cli.cmd_web_search(args)
        captured = capsys.readouterr()
        data = json.loads(captured.out)
        assert any(d["url"] == "https://g.com" for d in data)
        # Requested-but-unavailable extra sources must be disclosed, not silently dropped.
        assert "Tavily extra sources were requested" in captured.err
        assert data[0].get("degraded") == "tavily_unavailable"
        # The disclosure must not inject a synthetic element into the results array.
        assert all(d.get("url") for d in data)

    @pytest.mark.asyncio
    async def test_extra_sources_zero_not_marked_degraded(self, monkeypatch, capsys):
        """No degradation marker when Tavily was never requested."""
        monkeypatch.setenv("GROK_API_URL", "https://api.x.ai/v1")
        monkeypatch.setenv("GROK_API_KEY", "sk-test")
        monkeypatch.setenv("TAVILY_API_KEY", "tvly-test")
        import groksearch_cli

        groksearch_cli.Config._instance = None

        async def fake_grok_search(*a, **kw):
            return json.dumps([{"title": "G", "url": "https://g.com", "description": "G"}])

        monkeypatch.setattr(groksearch_cli.GrokSearchProvider, "search", fake_grok_search)

        args = MagicMock(query="t", platform="", min_results=3, max_results=10, extra_sources=0, raw=False)
        await groksearch_cli.cmd_web_search(args)
        captured = capsys.readouterr()
        data = json.loads(captured.out)
        assert "degraded" not in data[0]
        assert "Tavily extra sources" not in captured.err


# ============================================================================
# Task 4: web_fetch --via {grok|tavily}
# ============================================================================


class TestGrokStreamingPreference:
    @pytest.mark.asyncio
    async def test_search_uses_streaming_request(self, monkeypatch):
        monkeypatch.setenv("GROK_API_URL", "https://api.example/v1")
        from groksearch.provider import GrokSearchProvider

        provider = GrokSearchProvider("https://api.example/v1", "sk-test", "grok-test")
        execute_stream = AsyncMock(return_value="# Example")
        execute_non_stream = AsyncMock(side_effect=AssertionError("search must not start with a non-streaming request"))
        monkeypatch.setattr(provider, "_execute_stream", execute_stream)
        monkeypatch.setattr(provider, "_execute_non_stream", execute_non_stream)

        result = await provider.search("example")

        assert result == "# Example"
        execute_stream.assert_awaited_once()
        execute_non_stream.assert_not_awaited()

    @pytest.mark.asyncio
    async def test_fetch_uses_streaming_request(self, monkeypatch):
        monkeypatch.setenv("GROK_API_URL", "https://api.example/v1")
        from groksearch.provider import GrokSearchProvider

        provider = GrokSearchProvider("https://api.example/v1", "sk-test", "grok-test")
        execute_stream = AsyncMock(return_value="# Example")
        execute_non_stream = AsyncMock(side_effect=AssertionError("fetch must not start with a non-streaming request"))
        monkeypatch.setattr(provider, "_execute_stream", execute_stream)
        monkeypatch.setattr(provider, "_execute_non_stream", execute_non_stream)

        result = await provider.fetch("https://example.com")

        assert result == "# Example"
        execute_stream.assert_awaited_once()
        execute_non_stream.assert_not_awaited()

    @pytest.mark.asyncio
    async def test_stream_failure_falls_back_to_non_stream(self, monkeypatch):
        monkeypatch.setenv("GROK_API_URL", "https://api.example/v1")
        from groksearch.provider import GrokSearchProvider

        provider = GrokSearchProvider("https://api.example/v1", "sk-test", "grok-test")
        calls = []

        async def execute_stream(payload):
            calls.append("stream")
            raise httpx.ReadTimeout("stream failed")

        async def execute_non_stream(payload):
            calls.append("non-stream")
            return "fallback"

        monkeypatch.setattr(provider, "_execute_stream", execute_stream)
        monkeypatch.setattr(provider, "_execute_non_stream", execute_non_stream)

        result = await provider.fetch("https://example.com")

        assert result == "fallback"
        assert calls == ["stream", "non-stream"]

    @pytest.mark.asyncio
    async def test_empty_stream_falls_back_to_non_stream(self, monkeypatch):
        monkeypatch.setenv("GROK_API_URL", "https://api.example/v1")
        from groksearch.provider import GrokSearchProvider

        provider = GrokSearchProvider("https://api.example/v1", "sk-test", "grok-test")
        execute_stream = AsyncMock(return_value="")
        execute_non_stream = AsyncMock(return_value="fallback")
        monkeypatch.setattr(provider, "_execute_stream", execute_stream)
        monkeypatch.setattr(provider, "_execute_non_stream", execute_non_stream)

        result = await provider.search("example")

        assert result == "fallback"
        execute_stream.assert_awaited_once()
        execute_non_stream.assert_awaited_once()


class TestWebFetchViaTavily:
    @pytest.mark.asyncio
    async def test_via_tavily_calls_extract(self, monkeypatch, capsys):
        monkeypatch.setenv("GROK_API_URL", "https://api.x.ai/v1")
        monkeypatch.setenv("GROK_API_KEY", "sk-test")
        monkeypatch.setenv("TAVILY_API_KEY", "tvly-test")
        import groksearch_cli

        groksearch_cli.Config._instance = None

        async def fake_extract(url):
            return f"# Extracted: {url}"

        monkeypatch.setattr(groksearch_cli, "_call_tavily_extract", fake_extract)

        args = MagicMock(url="https://example.com", out=None, via="tavily")
        await groksearch_cli.cmd_web_fetch(args)
        out = capsys.readouterr().out
        assert "Extracted: https://example.com" in out

    @pytest.mark.asyncio
    async def test_via_tavily_no_key_errors(self, monkeypatch, capsys):
        monkeypatch.setenv("GROK_API_URL", "https://api.x.ai/v1")
        monkeypatch.setenv("GROK_API_KEY", "sk-test")
        import groksearch_cli

        groksearch_cli.Config._instance = None

        args = MagicMock(url="https://example.com", out=None, via="tavily")
        with pytest.raises(SystemExit) as exc:
            await groksearch_cli.cmd_web_fetch(args)
        assert exc.value.code != 0

    @pytest.mark.asyncio
    async def test_via_grok_default_path(self, monkeypatch, capsys):
        monkeypatch.setenv("GROK_API_URL", "https://api.x.ai/v1")
        monkeypatch.setenv("GROK_API_KEY", "sk-test")
        import groksearch_cli

        groksearch_cli.Config._instance = None

        async def fake_fetch(self, url):
            return f"GROK FETCH: {url}"

        monkeypatch.setattr(groksearch_cli.GrokSearchProvider, "fetch", fake_fetch)

        args = MagicMock(url="https://example.com", out=None, via="grok")
        await groksearch_cli.cmd_web_fetch(args)
        out = capsys.readouterr().out
        assert "GROK FETCH" in out


# ============================================================================
# Task 5: web_map subcommand
# ============================================================================


class TestWebMap:
    @pytest.mark.asyncio
    async def test_web_map_command_calls_tavily_map(self, monkeypatch, capsys):
        monkeypatch.setenv("TAVILY_API_KEY", "tvly-test")
        import groksearch_cli

        groksearch_cli.Config._instance = None

        captured = {}

        async def fake_map(url, instructions, max_depth, max_breadth, limit, timeout):
            captured.update(locals())
            return json.dumps({"base_url": url, "results": [], "response_time": 0.5})

        monkeypatch.setattr(groksearch_cli, "_call_tavily_map", fake_map)

        args = MagicMock(
            url="https://docs.python.org", instructions="api", max_depth=2, max_breadth=15, limit=30, timeout=120
        )
        await groksearch_cli.cmd_web_map(args)
        out = capsys.readouterr().out
        data = json.loads(out)
        assert data["base_url"] == "https://docs.python.org"
        assert captured["max_depth"] == 2
        assert captured["max_breadth"] == 15
        assert captured["limit"] == 30
        assert captured["timeout"] == 120

    def test_web_map_argparse_registered(self):
        # Build parser to verify subcommand registration
        from io import StringIO

        import groksearch_cli

        old_stderr = sys.stderr
        sys.stderr = StringIO()
        try:
            with pytest.raises(SystemExit):
                # Trigger parser to print help on a known subcommand
                old_argv = sys.argv
                sys.argv = ["groksearch_cli", "web_map", "--help"]
                try:
                    groksearch_cli.main()
                except SystemExit as e:
                    # argparse prints help and exits 0
                    if e.code != 0:
                        raise
                    raise
                finally:
                    sys.argv = old_argv
        finally:
            sys.stderr = old_stderr


# ============================================================================
# Task 6: web_crawl + web_research
# ============================================================================


class TestWebCrawl:
    @pytest.mark.asyncio
    async def test_crawl_command_passes_options(self, monkeypatch, capsys):
        monkeypatch.setenv("TAVILY_API_KEY", "tvly-test")
        import groksearch_cli

        groksearch_cli.Config._instance = None

        captured = {}

        async def fake_crawl(url, instructions, max_depth, max_breadth, limit, timeout, select_paths, exclude_paths):
            captured.update(locals())
            return json.dumps({"base_url": url, "results": [], "response_time": 1.0})

        monkeypatch.setattr(groksearch_cli, "_call_tavily_crawl", fake_crawl)

        args = MagicMock(
            url="https://docs.python.org",
            instructions="api",
            max_depth=2,
            max_breadth=10,
            limit=20,
            timeout=60,
            select_paths="/docs/.*",
            exclude_paths="/blog/.*",
        )
        await groksearch_cli.cmd_web_crawl(args)
        data = json.loads(capsys.readouterr().out)
        assert data["base_url"] == "https://docs.python.org"
        assert captured["max_depth"] == 2
        assert captured["limit"] == 20
        assert captured["select_paths"] == "/docs/.*"

    @pytest.mark.asyncio
    async def test_crawl_body_uses_config_defaults(self, monkeypatch):
        """Options default to the configured values when not passed on the CLI."""
        monkeypatch.setenv("TAVILY_API_KEY", "tvly-test")
        monkeypatch.setenv("TAVILY_CRAWL_MAX_DEPTH", "3")
        monkeypatch.setenv("TAVILY_CRAWL_LIMIT", "25")
        monkeypatch.setenv("TAVILY_CRAWL_ALLOW_EXTERNAL", "false")
        monkeypatch.setenv("TAVILY_CRAWL_SELECT_PATHS", "/docs/.*,/api/.*")
        import groksearch_cli

        groksearch_cli.Config._instance = None

        mock_response = MagicMock()
        mock_response.raise_for_status = MagicMock()
        mock_response.json = MagicMock(return_value={"base_url": "u", "results": []})
        mock_client = AsyncMock()
        mock_client.post = AsyncMock(return_value=mock_response)

        async def _get_client():
            return mock_client

        monkeypatch.setattr(groksearch_cli, "get_http_client", _get_client)

        await groksearch_cli._call_tavily_crawl("https://x.com")
        body = mock_client.post.call_args.kwargs["json"]
        assert body["max_depth"] == 3
        assert body["limit"] == 25
        assert body["allow_external"] is False
        # CSV config must reach the API as a real list, not a raw string.
        assert body["select_paths"] == ["/docs/.*", "/api/.*"]
        assert "include_usage" not in body  # False is omitted

    @pytest.mark.asyncio
    async def test_crawl_no_key_returns_config_error(self):
        from groksearch_cli import _call_tavily_crawl

        result = await _call_tavily_crawl("https://x.com")
        assert "TAVILY_API_KEY" in result

    def test_crawl_argparse_registered(self):
        import groksearch_cli

        parser = groksearch_cli.build_parser()
        args = parser.parse_args(["web_crawl", "--url", "https://x.com"])
        assert args.url == "https://x.com"
        assert args.max_depth is None  # falls back to config


class TestWebResearch:
    @pytest.mark.asyncio
    async def test_research_polls_until_completed(self, monkeypatch, capsys):
        monkeypatch.setenv("TAVILY_API_KEY", "tvly-test")
        monkeypatch.setenv("TAVILY_RESEARCH_POLL_INTERVAL", "0")
        import groksearch_cli

        groksearch_cli.Config._instance = None

        posted = {}

        class FakeResponse:
            def __init__(self, payload, status=200):
                self._payload = payload
                self.status_code = status

            def raise_for_status(self):
                return None

            def json(self):
                return self._payload

        states = [{"status": "in_progress"}, {"status": "completed", "content": "Answer", "sources": []}]

        class FakeClient:
            async def post(self, url, **kw):
                posted.update(kw.get("json") or {})
                return FakeResponse({"request_id": "req-1", "status": "pending"})

            async def get(self, url, **kw):
                return FakeResponse(states.pop(0))

        async def _get_client():
            return FakeClient()

        monkeypatch.setattr(groksearch_cli, "get_http_client", _get_client)

        result = await groksearch_cli._call_tavily_research("What is Tavily?")
        data = json.loads(result)
        assert data["status"] == "completed"
        assert data["content"] == "Answer"
        assert posted["input"] == "What is Tavily?"

    @pytest.mark.asyncio
    async def test_research_body_uses_config(self, monkeypatch):
        monkeypatch.setenv("TAVILY_API_KEY", "tvly-test")
        monkeypatch.setenv("TAVILY_RESEARCH_MODEL", "pro")
        monkeypatch.setenv("TAVILY_RESEARCH_CITATION_FORMAT", "apa")
        monkeypatch.setenv("TAVILY_RESEARCH_OUTPUT_LENGTH", "long")
        monkeypatch.setenv("TAVILY_RESEARCH_POLL_INTERVAL", "0")
        monkeypatch.setenv("TAVILY_INCLUDE_DOMAINS", "reuters.com")
        import groksearch_cli

        groksearch_cli.Config._instance = None

        posted = {}

        class FakeResponse:
            def __init__(self, payload):
                self._payload = payload

            def raise_for_status(self):
                return None

            def json(self):
                return self._payload

        class FakeClient:
            async def post(self, url, **kw):
                posted.update(kw.get("json") or {})
                return FakeResponse({"request_id": "r"})

            async def get(self, url, **kw):
                return FakeResponse({"status": "completed", "content": ""})

        async def _get_client():
            return FakeClient()

        monkeypatch.setattr(groksearch_cli, "get_http_client", _get_client)

        await groksearch_cli._call_tavily_research("q")
        assert posted["model"] == "pro"
        assert posted["citation_format"] == "apa"
        assert posted["output_length"] == "long"
        assert posted["include_domains"] == ["reuters.com"]

    @pytest.mark.asyncio
    async def test_research_timeout_returns_request_id(self, monkeypatch):
        """A task that never finishes must return its request_id, not hang."""
        monkeypatch.setenv("TAVILY_API_KEY", "tvly-test")
        monkeypatch.setenv("TAVILY_RESEARCH_TIMEOUT", "0")
        monkeypatch.setenv("TAVILY_RESEARCH_POLL_INTERVAL", "0")
        import groksearch_cli

        groksearch_cli.Config._instance = None

        class FakeResponse:
            def __init__(self, payload):
                self._payload = payload

            def raise_for_status(self):
                return None

            def json(self):
                return self._payload

        class FakeClient:
            async def post(self, url, **kw):
                return FakeResponse({"request_id": "req-slow"})

            async def get(self, url, **kw):
                return FakeResponse({"status": "in_progress"})

        async def _get_client():
            return FakeClient()

        monkeypatch.setattr(groksearch_cli, "get_http_client", _get_client)

        data = json.loads(await groksearch_cli._call_tavily_research("q"))
        assert data["status"] == "timeout"
        assert data["request_id"] == "req-slow"

    @pytest.mark.asyncio
    async def test_research_rejected_task_reports_response(self, monkeypatch):
        monkeypatch.setenv("TAVILY_API_KEY", "tvly-test")
        import groksearch_cli

        groksearch_cli.Config._instance = None

        class FakeResponse:
            def raise_for_status(self):
                return None

            def json(self):
                return {"detail": {"error": "bad input"}}

        class FakeClient:
            async def post(self, url, **kw):
                return FakeResponse()

        async def _get_client():
            return FakeClient()

        monkeypatch.setattr(groksearch_cli, "get_http_client", _get_client)

        data = json.loads(await groksearch_cli._call_tavily_research("q"))
        assert "error" in data

    def test_research_argparse_registered(self):
        import groksearch_cli

        parser = groksearch_cli.build_parser()
        args = parser.parse_args(["web_research", "--input", "What is Tavily?"])
        assert args.input == "What is Tavily?"
        assert args.model is None  # falls back to config
        assert args.output_schema is None

    @pytest.mark.asyncio
    async def test_research_poll_network_error_keeps_request_id(self, monkeypatch):
        """A poll failure after task creation must return JSON with request_id, not a bare string."""
        monkeypatch.setenv("TAVILY_API_KEY", "tvly-test")
        monkeypatch.setenv("TAVILY_RESEARCH_POLL_INTERVAL", "0")
        import groksearch_cli

        groksearch_cli.Config._instance = None

        class FakeResponse:
            def raise_for_status(self):
                return None

            def json(self):
                return {"request_id": "r"}

        class FakeClient:
            async def post(self, url, **kw):
                return FakeResponse()

            async def get(self, url, **kw):
                raise httpx.ConnectError("poll dropped")

        async def _get_client():
            return FakeClient()

        monkeypatch.setattr(groksearch_cli, "get_http_client", _get_client)

        data = json.loads(await groksearch_cli._call_tavily_research("q"))
        assert data["request_id"] == "r"
        assert data["status"] == "poll_error"
        assert "poll dropped" in data["error"]

    @pytest.mark.asyncio
    async def test_research_poll_budget_bounds_sleep(self, monkeypatch):
        """The poll interval must be clamped to the remaining budget so a large interval cannot overshoot."""
        monkeypatch.setenv("TAVILY_API_KEY", "tvly-test")
        monkeypatch.setenv("TAVILY_RESEARCH_TIMEOUT", "0")
        monkeypatch.setenv("TAVILY_RESEARCH_POLL_INTERVAL", "30")
        import groksearch_cli

        groksearch_cli.Config._instance = None

        class FakeResponse:
            def raise_for_status(self):
                return None

            def json(self):
                return {"request_id": "r"}

        class FakeClient:
            async def post(self, url, **kw):
                return FakeResponse()

            async def get(self, url, **kw):
                raise AssertionError("must not poll after the budget is spent")

        async def _get_client():
            return FakeClient()

        async def _sleep(seconds):
            # With budget 0 the clamped sleep must be 0, not the configured 30s interval.
            assert seconds == 0

        import asyncio

        monkeypatch.setattr(groksearch_cli, "get_http_client", _get_client)
        monkeypatch.setattr(asyncio, "sleep", _sleep)

        data = json.loads(await groksearch_cli._call_tavily_research("q"))
        assert data["status"] == "timeout"
        assert data["request_id"] == "r"

    @pytest.mark.asyncio
    async def test_research_invalid_model_returns_json_error(self, monkeypatch, capsys):
        """An invalid research tuning env surfaces as a JSON error, not a bare traceback."""
        monkeypatch.setenv("TAVILY_API_KEY", "tvly-test")
        monkeypatch.setenv("TAVILY_RESEARCH_MODEL", "huge")
        import groksearch_cli

        groksearch_cli.Config._instance = None

        with pytest.raises(SystemExit) as excinfo:
            await groksearch_cli.cmd_web_research(
                MagicMock(input="q", model=None, output_length=None, citation_format=None, output_schema=None)
            )
        assert excinfo.value.code == 1
        err = capsys.readouterr().err
        data = json.loads(err)
        assert "TAVILY_RESEARCH_MODEL" in data["error"]


VALID_SCHEMA = {
    "properties": {
        "company": {"type": "string", "description": "The company name"},
        "metrics": {
            "type": "array",
            "description": "Key metrics",
            "items": {"type": "string"},
        },
    },
    "required": ["company"],
}


class TestResearchOutputSchema:
    def test_valid_schema_passes_validation(self):
        from groksearch_cli import _validate_output_schema

        assert _validate_output_schema(VALID_SCHEMA) is None

    @pytest.mark.parametrize(
        "schema,needle",
        [
            ({"properties": {}}, "non-empty 'properties'"),
            ({"type": "object"}, "non-empty 'properties'"),
            ({"properties": {"a": {"description": "d"}}}, "needs a type"),
            ({"properties": {"a": {"type": "blob", "description": "d"}}}, "needs a type"),
            ({"properties": {"a": {"type": "string"}}}, "needs a description"),
            ({"properties": {"a": {"type": "array", "description": "d"}}}, "needs 'items'"),
            ({"properties": {"a": {"type": "object", "description": "d"}}}, "needs non-empty 'properties'"),
            (
                {"properties": {"a": {"type": "string", "description": "d"}}, "required": ["b"]},
                "not present in properties",
            ),
            ({"properties": {"a": {"type": "string", "description": "d"}}, "required": []}, "non-empty array"),
        ],
    )
    def test_invalid_schemas_rejected_with_reason(self, schema, needle):
        """Rejecting locally is cheaper than a 400 round-trip; the message must name the fault."""
        from groksearch_cli import _validate_output_schema

        error = _validate_output_schema(schema)
        assert error and needle in error

    def test_nested_object_validated(self):
        from groksearch_cli import _validate_output_schema

        schema = {
            "properties": {
                "financials": {
                    "type": "object",
                    "description": "breakdown",
                    "properties": {"income": {"type": "string"}},  # missing description
                }
            }
        }
        assert "income" in _validate_output_schema(schema)

    @pytest.mark.asyncio
    async def test_output_schema_sent_in_body(self, monkeypatch):
        monkeypatch.setenv("TAVILY_API_KEY", "tvly-test")
        monkeypatch.setenv("TAVILY_RESEARCH_POLL_INTERVAL", "0")
        import groksearch_cli

        groksearch_cli.Config._instance = None

        posted = {}

        class FakeResponse:
            def __init__(self, payload):
                self._payload = payload

            def raise_for_status(self):
                return None

            def json(self):
                return self._payload

        class FakeClient:
            async def post(self, url, **kw):
                posted.update(kw.get("json") or {})
                return FakeResponse({"request_id": "r"})

            async def get(self, url, **kw):
                return FakeResponse({"status": "completed", "content": {"company": "Acme"}})

        async def _get_client():
            return FakeClient()

        monkeypatch.setattr(groksearch_cli, "get_http_client", _get_client)

        result = await groksearch_cli._call_tavily_research("q", output_schema=VALID_SCHEMA)
        assert posted["output_schema"] == VALID_SCHEMA
        # Structured output arrives as an object in `content`, not a string.
        assert json.loads(result)["content"] == {"company": "Acme"}

    @pytest.mark.asyncio
    async def test_invalid_output_schema_stops_before_any_request(self, monkeypatch):
        monkeypatch.setenv("TAVILY_API_KEY", "tvly-test")
        import groksearch_cli

        groksearch_cli.Config._instance = None

        called = {"post": 0}

        class FakeClient:
            async def post(self, url, **kw):
                called["post"] += 1
                raise AssertionError("must not call the API with an invalid schema")

        async def _get_client():
            return FakeClient()

        monkeypatch.setattr(groksearch_cli, "get_http_client", _get_client)

        result = await groksearch_cli._call_tavily_research(
            "q", output_schema={"properties": {"a": {"type": "string"}}}
        )
        assert result.startswith("Configuration error:")
        assert called["post"] == 0

    @pytest.mark.asyncio
    async def test_schema_loaded_from_file(self, monkeypatch, tmp_path):
        schema_file = tmp_path / "schema.json"
        schema_file.write_text(json.dumps(VALID_SCHEMA), encoding="utf-8")
        monkeypatch.setenv("TAVILY_API_KEY", "tvly-test")
        monkeypatch.setenv("TAVILY_RESEARCH_POLL_INTERVAL", "0")
        monkeypatch.setenv("TAVILY_RESEARCH_OUTPUT_SCHEMA", str(schema_file))
        import groksearch_cli

        groksearch_cli.Config._instance = None

        posted = {}

        class FakeResponse:
            def __init__(self, payload):
                self._payload = payload

            def raise_for_status(self):
                return None

            def json(self):
                return self._payload

        class FakeClient:
            async def post(self, url, **kw):
                posted.update(kw.get("json") or {})
                return FakeResponse({"request_id": "r"})

            async def get(self, url, **kw):
                return FakeResponse({"status": "completed", "content": {}})

        async def _get_client():
            return FakeClient()

        monkeypatch.setattr(groksearch_cli, "get_http_client", _get_client)

        await groksearch_cli._call_tavily_research("q")
        assert posted["output_schema"] == VALID_SCHEMA

    @pytest.mark.asyncio
    async def test_missing_schema_file_reports_config_error(self, monkeypatch, tmp_path):
        monkeypatch.setenv("TAVILY_API_KEY", "tvly-test")
        monkeypatch.setenv("TAVILY_RESEARCH_OUTPUT_SCHEMA", str(tmp_path / "nope.json"))
        import groksearch_cli

        groksearch_cli.Config._instance = None

        result = await groksearch_cli._call_tavily_research("q")
        assert result.startswith("Configuration error:")
        assert "nope.json" in result

    @pytest.mark.asyncio
    async def test_malformed_schema_file_reports_config_error(self, monkeypatch, tmp_path):
        bad = tmp_path / "bad.json"
        bad.write_text("{not json", encoding="utf-8")
        monkeypatch.setenv("TAVILY_API_KEY", "tvly-test")
        monkeypatch.setenv("TAVILY_RESEARCH_OUTPUT_SCHEMA", str(bad))
        import groksearch_cli

        groksearch_cli.Config._instance = None

        result = await groksearch_cli._call_tavily_research("q")
        assert result.startswith("Configuration error:")
        assert "not valid JSON" in result

    @pytest.mark.asyncio
    async def test_no_schema_means_omitted_from_body(self, monkeypatch):
        monkeypatch.setenv("TAVILY_API_KEY", "tvly-test")
        monkeypatch.setenv("TAVILY_RESEARCH_POLL_INTERVAL", "0")
        import groksearch_cli

        groksearch_cli.Config._instance = None

        posted = {}

        class FakeResponse:
            def __init__(self, payload):
                self._payload = payload

            def raise_for_status(self):
                return None

            def json(self):
                return self._payload

        class FakeClient:
            async def post(self, url, **kw):
                posted.update(kw.get("json") or {})
                return FakeResponse({"request_id": "r"})

            async def get(self, url, **kw):
                return FakeResponse({"status": "completed", "content": "text"})

        async def _get_client():
            return FakeClient()

        monkeypatch.setattr(groksearch_cli, "get_http_client", _get_client)

        await groksearch_cli._call_tavily_research("q")
        assert "output_schema" not in posted


class TestTavilyUsageInResults:
    @pytest.m