#!/usr/bin/env python3
from __future__ import annotations

import argparse
from pathlib import Path

from rag_memory_lib import build_corpus, load_corpus, load_embedding_vectors, render_context_pack, retrieve, slugify


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description="Create an LLM-ready Agentic SWMM RAG context pack.")
    parser.add_argument("--query", required=True)
    parser.add_argument("--memory-dir", type=Path, default=Path("memory/modeling-memory"))
    parser.add_argument("--runs-dir", type=Path, default=Path("runs"))
    parser.add_argument("--index-dir", type=Path, default=None)
    parser.add_argument("--repo-root", type=Path, default=Path.cwd())
    parser.add_argument("--top-k", type=int, default=6)
    parser.add_argument("--project", default=None)
    parser.add_argument("--retriever", choices=("keyword", "hybrid"), default="hybrid")
    parser.add_argument("--format", choices=("markdown",), default="markdown")
    parser.add_argument("--obsidian-dir", type=Path, default=None)
    return parser.parse_args()


def main() -> int:
    args = parse_args()
    entries = []
    embedding_vectors = []
    if args.index_dir:
        entries = load_corpus(args.index_dir / "corpus.jsonl")
        embedding_vectors = load_embedding_vectors(args.index_dir / "embedding_index.json")
    if not entries:
        entries = build_corpus(args.memory_dir, args.runs_dir, args.repo_root)
    matches = retrieve(
        entries,
        args.query,
        args.top_k,
        project=args.project,
        retriever=args.retriever,
        embedding_vectors=embedding_vectors,
    )
    context = render_context_pack(args.query, matches)
    if args.obsidian_dir:
        args.obsidian_dir.mkdir(parents=True, exist_ok=True)
        out_path = args.obsidian_dir / f"RAG Query - {slugify(args.query)}.md"
        out_path.write_text(context, encoding="utf-8")
        print(str(out_path))
    else:
        print(context)
    return 0


if __name__ == "__main__":
    raise SystemExit(main())

#!/usr/bin/env python3
"""Build the Agentic SWMM RAG memory corpus.

PRD M6 hygiene contract (post-processing layer; the underlying 761-LOC
``rag_memory_lib.build_corpus`` is left untouched):

- Every emitted entry must have a non-empty ``case_name``. Resolution
  order: existing entry value, ``experiment_provenance.json:case_id``,
  ``experiment_provenance.json:case_name``, ``modeling_memory_index``
  ``case_name`` for matching ``run_id``, audit-note frontmatter
  ``case:`` field, parent run-dir name. If all sources fail for an
  audit-derived entry, the script exits non-zero with a clear message.
- Every emitted entry carries a ``schema_version`` field, copied from
  the source ``experiment_provenance.json:schema_version`` when
  available, falling back to the lessons file marker, falling back to
  the package default ``DEFAULT_SCHEMA_VERSION``.
- A one-line summary is written to stderr after a successful build.

ME-2 (issue #62) bounded-forgetting contract:

- Pattern sections in ``lessons_learned.md`` whose metadata says
  ``status: retired`` are stripped from the emitted corpus entry text
  entirely.
- Surviving (active / dormant) pattern sections expose
  ``pattern_status`` + ``pattern_confidence`` dicts on the entry so
  the retrieval layer can downweight dormant patterns.
"""

from __future__ import annotations

import argparse
import json
import re
import sys
from pathlib import Path
from typing import Any, Iterable

from rag_memory_lib import build_corpus, write_corpus


DEFAULT_SCHEMA_VERSION = "1.1"
_SCHEMA_VERSION_RE = re.compile(r"schema_version\s*[:=]\s*([0-9]+\.[0-9]+)")
_AUDIT_FRONTMATTER_CASE_RE = re.compile(r"^case\s*:\s*(.+?)\s*$", flags=re.MULTILINE)


# ME-2 (issue #62) lifecycle filter. We reach into
# ``agentic_swmm.memory.lessons_metadata`` for the YAML parser so the
# rules stay in one place; falling back to ``None`` lets the corpus
# build still succeed in environments that ship the rag scripts
# standalone (e.g. test fixtures).
try:
    sys.path.insert(0, str(Path(__file__).resolve().parents[3]))
    from agentic_swmm.memory.lessons_metadata import (  # type: ignore[import-not-found]
        _iter_pattern_spans,
        read_metadata,
    )
except Exception:  # noqa: BLE001 — degrade gracefully
    _iter_pattern_spans = None  # type: ignore[assignment]
    read_metadata = None  # type: ignore[assignment]


def _filter_lessons_text(text: str) -> tuple[str, dict[str, str], dict[str, float]]:
    """Strip retired pattern sections + collect lifecycle weights.

    Returns ``(filtered_text, pattern_status, pattern_confidence)``
    where ``pattern_status`` maps surviving pattern names to their
    ``status`` and ``pattern_confidence`` maps them to their
    ``confidence_score`` (capped at 1.0 so the retrieval layer can use
    it as a 0..1 weight without further normalisation).

    When the metadata helpers are unavailable (e.g. standalone test
    install) the function returns the input unchanged.
    """
    if _iter_pattern_spans is None or read_metadata is None:
        return text, {}, {}

    spans = list(_iter_pattern_spans(text))
    if not spans:
        return text, {}, {}

    pattern_status: dict[str, str] = {}
    pattern_confidence: dict[str, float] = {}
    # Walk back-to-front so the running offsets stay valid as we
    # excise retired sections.
    out = text
    for name, start, end in reversed(spans):
        block = text[start:end]
        meta = read_metadata(block)
        if not meta:
            continue
        status = str(meta.get("status") or "active").lower()
        score = float(meta.get("confidence_score") or 0.0)
        if status == "retired":
            out = out[:start] + out[end:]
            continue
        pattern_status[name] = status
        pattern_confidence[name] = min(1.0, max(0.0, score))
    return out, pattern_status, pattern_confidence


def _apply_lessons_filter_to_entries(entries: list[dict[str, Any]]) -> None:
    """In-place: strip retired patterns + annotate lifecycle weights.

    Targets entries whose ``source_path`` ends in ``lessons_learned.md``
    or ``lessons_archived.md``. The archive file is normally not in
    the source set, but we treat it defensively the same way.
    """
    for entry in entries:
        source = str(entry.get("source_path") or "")
        if not source.endswith("lessons_learned.md"):
            continue
        text = entry.get("text")
        if not isinstance(text, str):
            continue
        filtered, status_map, confidence_map = _filter_lessons_text(text)
        entry["text"] = filtered
        if status_map:
            entry["pattern_status"] = status_map
        if confidence_map:
            entry["pattern_confidence"] = confidence_map


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description="Build an Agentic SWMM RAG memory corpus.")
    parser.add_argument("--memory-dir", type=Path, default=Path("memory/modeling-memory"))
    parser.add_argument("--runs-dir", type=Path, default=Path("runs"))
    parser.add_argument("--out-dir", type=Path, default=Path("memory/rag-memory"))
    parser.add_argument("--repo-root", type=Path, default=Path.cwd())
    parser.add_argument(
        "--allow-missing-case-name",
        action="store_true",
        help="Do not exit non-zero when an audit-derived entry has no resolvable case_name.",
    )
    parser.add_argument(
        "--include-embeddings",
        action="store_true",
        help=(
            "Also emit embedding_index.json (hashed-cosine vectors). "
            "Off by default — no caller on main passes retriever='hybrid' "
            "to rag_memory_lib.retrieve(). See P1-2 in #79."
        ),
    )
    return parser.parse_args()


def _lessons_schema_version(memory_dir: Path) -> str | None:
    path = memory_dir / "lessons_learned.md"
    if not path.is_file():
        return None
    try:
        head = path.read_text(encoding="utf-8")[:4000]
    except OSError:
        return None
    match = _SCHEMA_VERSION_RE.search(head)
    return match.group(1) if match else None


def _modeling_index_case_names(memory_dir: Path) -> dict[str, str]:
    out: dict[str, str] = {}
    path = memory_dir / "modeling_memory_index.json"
    if not path.is_file():
        return out
    try:
        parsed = json.loads(path.read_text(encoding="utf-8"))
    except (OSError, json.JSONDecodeError):
        return out
    if not isinstance(parsed, dict):
        return out
    for record in parsed.get("records", []) or []:
        if not isinstance(record, dict):
            continue
        run_id = record.get("run_id")
        case_name = record.get("case_name")
        if run_id and case_name:
            out[str(run_id)] = str(case_name)
    return out


def _audit_dir_for(run_dir: Path) -> Path:
    new = run_dir / "09_audit"
    if new.is_dir():
        return new
    return run_dir


def _read_provenance(audit_dir: Path) -> dict[str, Any]:
    path = audit_dir / "experiment_provenance.json"
    if not path.is_file():
        return {}
    try:
        parsed = json.loads(path.read_text(encoding="utf-8"))
    except (OSError, json.JSONDecodeError):
        return {}
    return parsed if isinstance(parsed, dict) else {}


def _read_audit_note_case(audit_dir: Path) -> str | None:
    path = audit_dir / "experiment_note.md"
    if not path.is_file():
        return None
    try:
        head = path.read_text(encoding="utf-8")[:4000]
    except OSError:
        return None
    match = _AUDIT_FRONTMATTER_CASE_RE.search(head)
    return match.group(1).strip() if match else None


def _candidate_run_dirs(runs_dir: Path) -> Iterable[Path]:
    if not runs_dir.is_dir():
        return []
    return [path for path in runs_dir.rglob("*") if path.is_dir() and (path / "09_audit").is_dir() or _looks_like_run_dir(path)]


def _looks_like_run_dir(path: Path) -> bool:
    if not path.is_dir():
        return False
    for marker in ("experiment_provenance.json", "09_audit/experiment_provenance.json"):
        if (path / marker).is_file():
            return True
    return False


def _resolve_case_name_for_entry(
    entry: dict[str, Any],
    *,
    repo_root: Path,
    runs_dir: Path,
    index_case_names: dict[str, str],
) -> tuple[str | None, str | None]:
    """Return ``(case_name, schema_version)`` for ``entry``."""
    existing = entry.get("case_name")
    if existing:
        return str(existing), None

    run_id = entry.get("run_id")
    source_path = entry.get("source_path")

    # 1. modeling_memory_index by run_id.
    if run_id and run_id in index_case_names:
        return index_case_names[str(run_id)], None

    # 2. provenance / audit-note via the source path.
    if source_path:
        candidate = (repo_root / source_path).resolve()
        # walk upward looking for an audit dir.
        for parent in [candidate.parent, *candidate.parents]:
            try:
                parent.relative_to(repo_root.resolve())
            except ValueError:
                break
            audit_dir = _audit_dir_for(parent)
            provenance = _read_provenance(audit_dir)
            if provenance:
                case_id = provenance.get("case_id") or provenance.get("case_name")
                schema_version = provenance.get("schema_version")
                if case_id:
                    return str(case_id), (str(schema_version) if schema_version else None)
                note_case = _read_audit_note_case(audit_dir)
                if note_case:
                    return note_case, (str(schema_version) if schema_version else None)
            if audit_dir != parent and parent != repo_root.resolve():
                # Use the run-dir name as last-resort fallback.
                return parent.name, None

    # 3. By run_id within runs/.
    if run_id and runs_dir.is_dir():
        match = next(
            (path for path in runs_dir.rglob(str(run_id)) if path.is_dir()),
            None,
        )
        if match is not None:
            return match.name, None

    # 4. Curated memory documents (lessons / index / proposals): synthesise a
    #    stable, greppable case_name from the source path stem. These entries
    #    have no run, so the PRD's case_name field is informational only.
    if source_path:
        source = Path(str(source_path))
        curated_stems = {
            "lessons_learned",
            "modeling_memory_index",
            "project_memory_index",
            "skill_update_proposals",
            "benchmark_verification_plan",
        }
        if source.stem in curated_stems or "modeling-memory" in source.parts:
            return f"_curated_memory_{source.stem}", None

    return (str(run_id) if run_id else None), None


def _post_process(
    entries: list[dict[str, Any]],
    *,
    repo_root: Path,
    runs_dir: Path,
    memory_dir: Path,
    allow_missing_case_name: bool,
) -> tuple[list[dict[str, Any]], list[str], str]:
    """Apply case_name + schema_version hygiene.

    Returns the cleaned entries, a list of source_paths with missing
    case_name (empty unless ``allow_missing_case_name`` was set), and
    the schema version stamped on entries that lacked one.
    """
    index_case_names = _modeling_index_case_names(memory_dir)
    lessons_schema = _lessons_schema_version(memory_dir)
    fallback_schema = lessons_schema or DEFAULT_SCHEMA_VERSION

    missing: list[str] = []
    cleaned: list[dict[str, Any]] = []
    for entry in entries:
        case_name, derived_schema = _resolve_case_name_for_entry(
            entry,
            repo_root=repo_root,
            runs_dir=runs_dir,
            index_case_names=index_case_names,
        )
        if not case_name:
            missing.append(str(entry.get("source_path") or entry.get("run_id") or "<unknown>"))
            if not allow_missing_case_name:
                continue
        if case_name:
            entry["case_name"] = case_name
        # Override the lib-default "1.0" placeholder with the
        # derived-per-source or PRD-default schema_version. The PRD
        # contract is: every entry carries the latest known schema
        # version, not the lib's hardcoded default.
        if derived_schema:
            entry["schema_version"] = derived_schema
        elif not entry.get("schema_version") or str(entry.get("schema_version")) == "1.0":
            entry["schema_version"] = fallback_schema
        cleaned.append(entry)

    return cleaned, missing, fallback_schema


def main() -> int:
    args = parse_args()
    entries = build_corpus(args.memory_dir, args.runs_dir, args.repo_root)
    # ME-2 (issue #62): drop retired pattern sections + tag lifecycle
    # weights on the lessons entry BEFORE case-name hygiene so the
    # downstream hygiene checks see the same body the retrieval layer
    # will see.
    _apply_lessons_filter_to_entries(entries)
    cleaned, missing, schema = _post_process(
        entries,
        repo_root=args.repo_root,
        runs_dir=args.runs_dir,
        memory_dir=args.memory_dir,
        allow_missing_case_name=args.allow_missing_case_name,
    )

    if missing and not args.allow_missing_case_name:
        print(
            json.dumps(
                {
                    "error": "build_memory_corpus refused to emit entries with empty case_name",
                    "missing_sources": missing,
                    "hint": "Add case_id / case_name to experiment_provenance.json or pass --allow-missing-case-name to bypass.",
                },
                indent=2,
            ),
            file=sys.stderr,
        )
        return 1

    write_corpus(cleaned, args.out_dir, include_embeddings=args.include_embeddings)
    distinct_cases = {entry.get("case_name") for entry in cleaned if entry.get("case_name")}
    print(
        json.dumps(
            {
                "entry_count": len(cleaned),
                "out_dir": str(args.out_dir),
                "embedding_backend": "local-hashed-token-char-ngram",
                "schema_version": schema,
                "distinct_cases": len(distinct_cases),
            },
            sort_keys=True,
        )
    )
    print(
        f"built corpus: {len(cleaned)} entries, {len(distinct_cases)} distinct cases, schema={schema}",
        file=sys.stderr,
    )
    return 0


if __name__ == "__main__":
    raise SystemExit(main())

#!/usr/bin/env python3
from __future__ import annotations

import argparse
import json
from pathlib import Path

from rag_memory_lib import (
    build_corpus,
    build_failure_advice_query,
    extract_run_problem,
    load_corpus,
    load_embedding_vectors,
    now_utc,
    read_run_evidence,
    render_failure_advice,
    retrieve,
    should_generate_failure_advice,
    suggested_checks,
    write_json,
)


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description="Generate retrieval-grounded failure advice for an audited Agentic SWMM run.")
    parser.add_argument("--run-dir", type=Path, required=True)
    parser.add_argument("--memory-dir", type=Path, default=Path("memory/modeling-memory"))
    parser.add_argument("--runs-dir", type=Path, default=Path("runs"))
    parser.add_argument("--index-dir", type=Path, default=Path("memory/rag-memory"))
    parser.add_argument("--repo-root", type=Path, default=Path.cwd())
    parser.add_argument("--top-k", type=int, default=5)
    parser.add_argument("--retriever", choices=("keyword", "hybrid"), default="hybrid")
    parser.add_argument("--out-json", type=Path, default=None)
    parser.add_argument("--out-md", type=Path, default=None)
    parser.add_argument("--force", action="store_true", help="Write advice even if trigger conditions are not met.")
    parser.add_argument("--no-write", action="store_true", help="Print JSON result without writing failure_advice files.")
    return parser.parse_args()


def main() -> int:
    args = parse_args()
    repo_root = args.repo_root.resolve()
    run_dir = args.run_dir.resolve()
    evidence = read_run_evidence(run_dir)
    problem = extract_run_problem(evidence, run_dir, repo_root)
    should_trigger, trigger_reasons = should_generate_failure_advice(problem)

    if not should_trigger and not args.force:
        result = {
            "schema_version": "1.0",
            "generated_by": "swmm-rag-memory",
            "generated_at_utc": now_utc(),
            "run_id": problem.get("run_id"),
            "advice_written": False,
            "triggered": False,
            "trigger_reasons": [],
            "boundary": "No failure advice was generated because trigger conditions were not met.",
        }
        print(json.dumps(result, indent=2, sort_keys=True, ensure_ascii=False))
        return 0

    entries = load_corpus(args.index_dir / "corpus.jsonl")
    embedding_vectors = load_embedding_vectors(args.index_dir / "embedding_index.json") if entries else []
    if not entries:
        entries = build_corpus(args.memory_dir, args.runs_dir, repo_root)
    query = build_failure_advice_query(problem)
    matches = retrieve(
        entries,
        query,
        args.top_k,
        project=problem.get("project_key"),
        retriever=args.retriever,
        embedding_vectors=embedding_vectors,
    )

    advice = {
        "schema_version": "1.0",
        "generated_by": "swmm-rag-memory",
        "generated_at_utc": now_utc(),
        "retrieval_grounded": True,
        "human_reviewed": False,
        "benchmark_verified": False,
        "advice_written": not args.no_write,
        "triggered": should_trigger or args.force,
        "trigger_reasons": trigger_reasons if should_trigger else ["forced"],
        "retriever": args.retriever,
        "query": query,
        "current_run_problem": problem,
        "retrieved_memory": matches,
        "suggested_next_checks": suggested_checks(problem, matches),
        "boundary": "Retrieval-grounded advice only. No model files, workflow code, or skill definitions were modified.",
    }

    if not args.no_write:
        out_json = args.out_json or (run_dir / "failure_advice.json")
        out_md = args.out_md or (run_dir / "failure_advice.md")
        write_json(out_json, advice)
        out_md.parent.mkdir(parents=True, exist_ok=True)
        out_md.write_text(render_failure_advice(advice), encoding="utf-8")
        advice["failure_advice_json"] = str(out_json)
        advice["failure_advice_md"] = str(out_md)

    print(json.dumps(advice, indent=2, sort_keys=True, ensure_ascii=False))
    return 0


if __name__ == "__main__":
    raise SystemExit(main())

"""rag_memory_lib — corpus + retrieval helpers for swmm-rag-memory.

Two retrieval paths are supported, gated by the ``retriever`` argument on
``retrieve()``:

* ``retriever="keyword"`` (default) — pure inverted-index / token-overlap
  matching. This is what every caller in ``main`` currently passes through
  to (search ``retrieve(`` in repo). Always cheap; no embedding side-files
  are needed at query time.

* ``retriever="hybrid"`` — adds a 384-dim hashed-embedding cosine score
  blended in at ``HYBRID_KEYWORD_WEIGHT / HYBRID_SEMANTIC_WEIGHT /
  HYBRID_METADATA_WEIGHT``. The hashed embedding is intentionally
  cheap-to-derive but very dense on small corpora (~98% non-zero on the
  current 26-entry memory). The path is kept for the LID-paper IP work
  and any caller that opts in explicitly. It is feature-flagged off by
  default to avoid writing ~3 MB of unused index data on every audit
  refresh (P1-2 in #79).

To opt in to hybrid retrieval:

  >>> entries = load_corpus(corpus_path)
  >>> vectors = load_embedding_vectors(embedding_index_path)
  >>> retrieve(entries, query, 5, retriever="hybrid", embedding_vectors=vectors)

When building the corpus from a writer, pass ``include_embeddings=True`` to
``write_corpus`` to also emit ``embedding_index.json``. Default is False.
"""

from __future__ import annotations

import json
import math
import re
from collections import Counter
from datetime import datetime, timezone
from pathlib import Path
from typing import Any


TOKEN_RE = re.compile(r"[A-Za-z0-9_./:-]+|[\u4e00-\u9fff]+")
EMBEDDING_DIMENSIONS = 384
HYBRID_KEYWORD_WEIGHT = 0.62
HYBRID_SEMANTIC_WEIGHT = 0.28
HYBRID_METADATA_WEIGHT = 0.10
QUERY_EXPANSIONS = {
    "洪峰": ["peak", "flow", "peak_flow", "peak_flow_parse_missing"],
    "峰值": ["peak", "flow", "peak_flow"],
    "流量": ["flow", "inflow", "outflow"],
    "没读到": ["parse", "missing", "parse_missing"],
    "没有读到": ["parse", "missing", "parse_missing"],
    "解析不到": ["parse", "missing", "parse_missing"],
    "水量平衡": ["continuity", "continuity_error", "flow_routing", "runoff_quantity"],
    "连续性": ["continuity", "continuity_error"],
    "管道坡度": ["conduit_slope", "conduit_slope_suspicious"],
    "坡度": ["slope", "conduit_slope_suspicious"],
    "缺证据": ["missing", "missing_evidence", "partial_run"],
    "证据不足": ["missing", "missing_evidence", "partial_run"],
    "失败": ["failure", "failed", "failure_patterns"],
}
STOPWORDS = {
    "the",
    "a",
    "an",
    "and",
    "or",
    "to",
    "of",
    "in",
    "for",
    "with",
    "is",
    "are",
    "this",
    "that",
    "what",
    "why",
    "how",
    "怎么办",
    "为什么",
    "这个",
    "那个",
}


def now_utc() -> str:
    return datetime.now(timezone.utc).isoformat(timespec="seconds")


def read_json(path: Path) -> Any:
    if not path.exists():
        return None
    try:
        return json.loads(path.read_text(encoding="utf-8"))
    except (OSError, json.JSONDecodeError):
        return None


def read_text(path: Path) -> str:
    if not path.exists():
        return ""
    try:
        return path.read_text(encoding="utf-8")
    except OSError:
        return ""


def write_json(path: Path, value: Any) -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_text(json.dumps(value, indent=2, sort_keys=True, ensure_ascii=False), encoding="utf-8")


def relpath(path: Path, root: Path) -> str:
    try:
        return str(path.resolve().relative_to(root.resolve()))
    except ValueError:
        return str(path)


def tokenize(text: str) -> list[str]:
    tokens = [match.group(0).lower() for match in TOKEN_RE.finditer(text)]
    out: list[str] = []
    for token in tokens:
        token = token.strip("._-:/")
        if not token or token in STOPWORDS:
            continue
        out.append(token)
        if "_" in token:
            out.extend(part for part in token.split("_") if part and part not in STOPWORDS)
        if "-" in token:
            out.extend(part for part in token.split("-") if part and part not in STOPWORDS)
    return out


def expand_query_tokens(query: str, tokens: list[str]) -> list[str]:
    expanded = list(tokens)
    lowered = query.lower()
    for phrase, additions in QUERY_EXPANSIONS.items():
        if phrase.lower() in lowered:
            expanded.extend(additions)
    token_set = set(tokens)
    if {"peak", "flow"} <= token_set:
        expanded.append("peak_flow_parse_missing")
    if {"parse", "missing"} <= token_set:
        expanded.extend(["parse_missing", "peak_flow_parse_missing", "continuity_parse_missing"])
    if "continuity" in token_set and ("high" in token_set or "error" in token_set):
        expanded.append("continuity_error_high")
    return expanded


def stable_hash(value: str) -> int:
    h = 2166136261
    for byte in value.encode("utf-8"):
        h ^= byte
        h = (h * 16777619) & 0xFFFFFFFF
    return h


def char_ngrams(text: str) -> list[str]:
    clean = re.sub(r"\s+", " ", text.lower()).strip()
    grams: list[str] = []
    for n in (3, 4, 5):
        if len(clean) < n:
            continue
        grams.extend(clean[i : i + n] for i in range(len(clean) - n + 1))
    return grams


def embedding_features(text: str) -> list[str]:
    tokens = tokenize(text)
    features: list[str] = []
    features.extend(f"tok:{token}" for token in tokens)
    for left, right in zip(tokens, tokens[1:]):
        features.append(f"bi:{left}_{right}")
    features.extend(f"char:{gram}" for gram in char_ngrams(text))
    return features


def hashed_embedding(text: str, dimensions: int = EMBEDDING_DIMENSIONS) -> dict[str, float]:
    counts: Counter[int] = Counter()
    for feature in embedding_features(text):
        idx = stable_hash(feature) % dimensions
        sign = -1.0 if stable_hash("sign:" + feature) % 2 else 1.0
        counts[idx] += sign
    norm = math.sqrt(sum(float(value) * float(value) for value in counts.values()))
    if not norm:
        return {}
    return {str(idx): round(float(value) / norm, 6) for idx, value in sorted(counts.items()) if value}


def cosine_sparse(left: dict[str, float], right: dict[str, float]) -> float:
    if not left or not right:
        return 0.0
    if len(left) > len(right):
        left, right = right, left
    return sum(value * right.get(key, 0.0) for key, value in left.items())


def compact_json(value: Any, max_chars: int = 1800) -> str:
    text = json.dumps(value, sort_keys=True, ensure_ascii=False)
    if len(text) <= max_chars:
        return text
    return text[: max_chars - 3] + "..."


def excerpt(text: str, terms: set[str], max_chars: int = 600) -> str:
    clean = re.sub(r"\s+", " ", text).strip()
    if len(clean) <= max_chars:
        return clean
    lower = clean.lower()
    positions = [lower.find(term.lower()) for term in terms if term and lower.find(term.lower()) >= 0]
    start = max(0, min(positions) - 160) if positions else 0
    end = min(len(clean), start + max_chars)
    prefix = "..." if start > 0 else ""
    suffix = "..." if end < len(clean) else ""
    return prefix + clean[start:end] + suffix


def listify(value: Any) -> list[str]:
    if value is None:
        return []
    if isinstance(value, list):
        return [str(item) for item in value if item not in (None, "")]
    return [str(value)]


def infer_source_type(path: Path) -> str:
    name = path.name
    if name == "resolution_memory.json":
        return "resolution_memory"
    if name == "failure_advice.json":
        return "failure_advice"
    if name == "failure_advice.md":
        return "failure_advice_note"
    if name == "memory_summary.json":
        return "run_memory"
    if name == "modeling_memory_index.json":
        return "global_memory_index"
    if name == "run_memory_summaries.json":
        return "run_memory_index"
    if name == "project_memory.json":
        return "project_memory"
    if name == "experiment_note.md":
        return "experiment_note"
    if name == "model_diagnostics.json":
        return "model_diagnostics"
    if name.endswith(".md"):
        return "memory_note"
    return "memory_artifact"


def entry_from_record(record: dict[str, Any], source_path: Path, repo_root: Path, source_type: str) -> dict[str, Any]:
    fields = [
        record.get("run_id"),
        record.get("case_name"),
        record.get("project_key"),
        record.get("workflow_mode"),
        record.get("objective"),
        " ".join(listify(record.get("failure_patterns"))),
        " ".join(listify(record.get("model_diagnostic_ids"))),
        " ".join(listify(record.get("warnings"))),
        " ".join(listify(record.get("evidence_boundary_notes"))),
        " ".join(listify(record.get("next_run_cautions"))),
        compact_json(record.get("metrics", {}), max_chars=900),
    ]
    text = "\n".join(str(item) for item in fields if item)
    return {
        "schema_version": "1.0",
        "source_type": source_type,
        "source_path": relpath(source_path, repo_root),
        "run_id": record.get("run_id"),
        "project_key": record.get("project_key"),
        "case_name": record.get("case_name"),
        "workflow_mode": record.get("workflow_mode"),
        "qa_status": record.get("qa_status"),
        "failure_patterns": listify(record.get("failure_patterns")),
        "model_diagnostic_ids": listify(record.get("model_diagnostic_ids")),
        "next_run_cautions": listify(record.get("next_run_cautions")),
        "text": text,
    }


def entry_from_file(path: Path, repo_root: Path, source_type: str, metadata: dict[str, Any] | None = None) -> dict[str, Any]:
    metadata = normalize_entry_metadata(metadata or {})
    if path.suffix == ".json":
        parsed = read_json(path)
        text = compact_json(parsed if parsed is not None else {}, max_chars=3000)
    else:
        text = read_text(path)
    return {
        "schema_version": "1.0",
        "source_type": source_type,
        "source_path": relpath(path, repo_root),
        "run_id": metadata.get("run_id"),
        "project_key": metadata.get("project_key"),
        "case_name": metadata.get("case_name"),
        "workflow_mode": metadata.get("workflow_mode"),
        "qa_status": metadata.get("qa_status"),
        "failure_patterns": listify(metadata.get("failure_patterns")),
        "model_diagnostic_ids": listify(metadata.get("model_diagnostic_ids")),
        "next_run_cautions": listify(metadata.get("next_run_cautions")),
        "retrieval_grounded": metadata.get("retrieval_grounded"),
        "human_reviewed": metadata.get("human_reviewed"),
        "benchmark_verified": metadata.get("benchmark_verified"),
        "text": text,
    }


def normalize_entry_metadata(metadata: dict[str, Any]) -> dict[str, Any]:
    if "current_run_problem" in metadata and isinstance(metadata["current_run_problem"], dict):
        problem = metadata["current_run_problem"]
        out = dict(metadata)
        for key in ("run_id", "project_key", "case_name", "workflow_mode", "qa_status", "failure_patterns", "model_diagnostic_ids", "next_run_cautions"):
            if out.get(key) in (None, [], "") and problem.get(key) not in (None, [], ""):
                out[key] = problem.get(key)
        return out
    if "problem" in metadata and isinstance(metadata["problem"], dict):
        problem = metadata["problem"]
        out = dict(metadata)
        if out.get("failure_patterns") in (None, [], ""):
            out["failure_patterns"] = problem.get("failure_patterns")
        if out.get("model_diagnostic_ids") in (None, [], ""):
            out["model_diagnostic_ids"] = problem.get("diagnostics")
        return out
    return metadata


def load_modeling_index(memory_dir: Path) -> list[dict[str, Any]]:
    parsed = read_json(memory_dir / "modeling_memory_index.json")
    if isinstance(parsed, dict) and isinstance(parsed.get("records"), list):
        return [item for item in parsed["records"] if isinstance(item, dict)]
    return []


def records_by_run(records: list[dict[str, Any]]) -> dict[str, dict[str, Any]]:
    out: dict[str, dict[str, Any]] = {}
    for record in records:
        run_id = record.get("run_id")
        if run_id:
            out[str(run_id)] = record
    return out


def build_corpus(memory_dir: Path, runs_dir: Path, repo_root: Path) -> list[dict[str, Any]]:
    entries: list[dict[str, Any]] = []
    records = load_modeling_index(memory_dir)
    run_records = records_by_run(records)
    for record in records:
        entries.append(entry_from_record(record, memory_dir / "modeling_memory_index.json", repo_root, "run_record"))

    for path in sorted(memory_dir.glob("*.md")):
        entries.append(entry_from_file(path, repo_root, infer_source_type(path)))

    for path in sorted(memory_dir.glob("projects/*/project_memory.json")):
        parsed = read_json(path)
        metadata = parsed if isinstance(parsed, dict) else {}
        entries.append(entry_from_file(path, repo_root, "project_memory", metadata))

    for path in sorted(memory_dir.glob("projects/*/project_memory.md")):
        project_key = path.parent.name
        entries.append(entry_from_file(path, repo_root, "project_memory_note", {"project_key": project_key}))

    if runs_dir.exists():
        wanted = {"memory_summary.json", "experiment_note.md", "model_diagnostics.json", "failure_advice.json", "failure_advice.md", "resolution_memory.json"}
        for path in sorted(p for p in runs_dir.rglob("*") if p.is_file() and p.name in wanted):
            parsed = read_json(path) if path.suffix == ".json" else None
            metadata = parsed if isinstance(parsed, dict) else {}
            run_id = metadata.get("run_id")
            if run_id and str(run_id) in run_records:
                merged = {**run_records[str(run_id)], **metadata}
            else:
                merged = metadata
            entries.append(entry_from_file(path, repo_root, infer_source_type(path), merged))

    deduped: list[dict[str, Any]] = []
    seen: set[tuple[str, str | None, str]] = set()
    for entry in entries:
        key = (str(entry.get("source_path")), entry.get("run_id"), str(entry.get("source_type")))
        if key in seen:
            continue
        seen.add(key)
        entry["tokens"] = sorted(set(tokenize(" ".join([entry.get("text", ""), compact_json(entry, max_chars=1200)]))))
        deduped.append(entry)
    return deduped


def write_corpus(
    entries: list[dict[str, Any]],
    out_dir: Path,
    *,
    include_embeddings: bool = False,
) -> None:
    """Write ``corpus.jsonl`` + ``keyword_index.json`` (+ optional embeddings).

    ``include_embeddings=True`` additionally emits ``embedding_index.json``
    for the hashed-cosine hybrid retriever path. Default is ``False`` per
    P1-2 in #79 — no caller on ``main`` currently passes
    ``retriever="hybrid"`` to ``retrieve()``, and the index file is ~3 MB
    on every audit refresh.
    """
    out_dir.mkdir(parents=True, exist_ok=True)
    with (out_dir / "corpus.jsonl").open("w", encoding="utf-8") as handle:
        for entry in entries:
            handle.write(json.dumps(entry, sort_keys=True, ensure_ascii=False) + "\n")
    index: dict[str, list[int]] = {}
    for idx, entry in enumerate(entries):
        for token in entry.get("tokens", []):
            index.setdefault(str(token), []).append(idx)
    write_json(
        out_dir / "keyword_index.json",
        {
            "schema_version": "1.0",
            "generated_by": "swmm-rag-memory",
            "generated_at_utc": now_utc(),
            "entry_count": len(entries),
            "token_count": len(index),
            "index": index,
        },
    )
    if include_embeddings:
        write_json(
            out_dir / "embedding_index.json",
            {
                "schema_version": "1.0",
                "generated_by": "swmm-rag-memory",
                "backend": "local-hashed-token-char-ngram",
                "dimensions": EMBEDDING_DIMENSIONS,
                "generated_at_utc": now_utc(),
                "entry_count": len(entries),
                "vectors": [hashed_embedding(str(entry.get("text") or "")) for entry in entries],
            },
        )


def load_corpus(path: Path) -> list[dict[str, Any]]:
    entries: list[dict[str, Any]] = []
    if not path.exists():
        return entries
    for line in path.read_text(encoding="utf-8").splitlines():
        if not line.strip():
            continue
        try:
            parsed = json.loads(line)
        except json.JSONDecodeError:
            continue
        if isinstance(parsed, dict):
            entries.append(parsed)
    return entries


def load_embedding_vectors(path: Path) -> list[dict[str, float]]:
    parsed = read_json(path)
    if not isinstance(parsed, dict) or not isinstance(parsed.get("vectors"), list):
        return []
    vectors: list[dict[str, float]] = []
    for vector in parsed["vectors"]:
        if isinstance(vector, dict):
            vectors.append({str(key): float(value) for key, value in vector.items()})
    return vectors


def score_entry(entry: dict[str, Any], query_tokens: list[str], project: str | None = None) -> tuple[float, list[str]]:
    entry_tokens = set(str(token) for token in entry.get("tokens", []))
    query_set = set(query_tokens)
    matched = sorted(query_set & entry_tokens)
    if not matched:
        return 0.0, []

    token_score = sum(1.0 + math.log1p(len(token)) / 4.0 for token in matched)
    tag_text = " ".join(
        listify(entry.get("failure_patterns"))
        + listify(entry.get("model_diagnostic_ids"))
        + listify(entry.get("project_key"))
        + listify(entry.get("run_id"))
        + listify(entry.get("case_name"))
    ).lower()
    tag_matches = [token for token in query_set if token in tag_text]
    score = token_score + 2.5 * len(tag_matches)
    if entry.get("source_type") in {"run_record", "run_memory", "project_memory"}:
        score += 1.0
    if project and str(entry.get("project_key") or "").lower() == project.lower():
        score += 4.0
    elif project:
        score *= 0.35
    return score, sorted(set(matched + tag_matches))


def metadata_score(entry: dict[str, Any], query_tokens: list[str], project: str | None = None) -> float:
    query_set = set(query_tokens)
    tags = set(
        tokenize(
            " ".join(
                listify(entry.get("failure_patterns"))
                + listify(entry.get("model_diagnostic_ids"))
                + listify(entry.get("project_key"))
                + listify(entry.get("run_id"))
                + listify(entry.get("case_name"))
                + listify(entry.get("workflow_mode"))
            )
        )
    )
    score = min(1.0, len(query_set & tags) / 4.0)
    if project and str(entry.get("project_key") or "").lower() == project.lower():
        score = min(1.0, score + 0.5)
    return score


def entry_confidence(entry: dict[str, Any]) -> float:
    """Strongest pattern confidence for an entry, in [0, 1].

    ``pattern_confidence`` (set by build_memory_corpus) is the decay-aware
    lessons-lifecycle score, so reusing it makes retrieval recency-aware
    without a second decay model. Returns the neutral 1.0 when an entry
    carries no confidence annotation, so un-annotated corpora rank exactly as
    before.
    """
    conf = entry.get("pattern_confidence")
    if not isinstance(conf, dict) or not conf:
        return 1.0
    values = [
        float(v)
        for v in conf.values()
        if isinstance(v, (int, float)) and not isinstance(v, bool)
    ]
    if not values:
        return 1.0
    return max(0.0, min(1.0, max(values)))


def confidence_factor(confidence: float) -> float:
    """Map confidence [0,1] to a gentle score multiplier [0.5, 1.0].

    A low-confidence (stale / dormant) precedent is dampened up to 50% so it
    ranks below a fresh, high-confidence one for the same textual match —
    without erasing it (a weak precedent can still be the only match).
    """
    return 0.5 + 0.5 * max(0.0, min(1.0, confidence))


def retrieve(
    entries: list[dict[str, Any]],
    query: str,
    top_k: int,
    project: str | None = None,
    retriever: str = "keyword",
    embedding_vectors: list[dict[str, float]] | None = None,
) -> list[dict[str, Any]]:
    query_tokens = expand_query_tokens(query, tokenize(query))
    query_vector = hashed_embedding(" ".join([query, " ".join(query_tokens)])) if retriever == "hybrid" else {}
    scored: list[dict[str, Any]] = []
    max_keyword = 1.0
    raw_scores: list[tuple[dict[str, Any], float, list[str], float, float]] = []
    for idx, entry in enumerate(entries):
        score, matched_terms = score_entry(entry, query_tokens, project=project)
        semantic = 0.0
        if retriever == "hybrid":
            if embedding_vectors and idx < len(embedding_vectors):
                entry_vector = embedding_vectors[idx]
            else:
                entry_vector = hashed_embedding(str(entry.get("text") or ""))
            semantic = max(0.0, cosine_sparse(query_vector, entry_vector))
        if score <= 0 and semantic <= 0:
            continue
        meta = metadata_score(entry, query_tokens, project=project)
        max_keyword = max(max_keyword, score)
        raw_scores.append((entry, score, matched_terms, semantic, meta))

    for entry, keyword_score, matched_terms, semantic, meta in raw_scores:
        if retriever == "hybrid":
            normalized_keyword = keyword_score / max_keyword if max_keyword else 0.0
            final_score = (
                HYBRID_KEYWORD_WEIGHT * normalized_keyword
                + HYBRID_SEMANTIC_WEIGHT * semantic
                + HYBRID_METADATA_WEIGHT * meta
            ) * 100.0
        else:
            final_score = keyword_score
        # Recency-aware ranking: weight by the entry's pattern confidence so a
        # stale/dormant precedent ranks below a fresh one for the same match.
        # Neutral (1.0) when the entry carries no confidence annotation.
        confidence = entry_confidence(entry)
        final_score = final_score * confidence_factor(confidence)
        result = {key: value for key, value in entry.items() if key not in {"tokens", "text"}}
        result["score"] = round(final_score, 3)
        result["confidence"] = round(confidence, 3)
        result["keyword_score"] = round(keyword_score, 3)
        result["semantic_score"] = round(semantic, 3)
        result["metadata_score"] = round(meta, 3)
        result["matched_terms"] = matched_terms
        result["excerpt"] = excerpt(str(entry.get("text") or ""), set(matched_terms))
        scored.append(result)
    scored.sort(key=lambda item: (-float(item["score"]), str(item.get("source_path"))))
    return scored[:top_k]


def render_context_pack(query: str, matches: list[dict[str, Any]]) -> str:
    lines = [
        "# Retrieved Agentic SWMM Memory Context",
        "",
        "## User Question",
        query,
        "",
        "## Relevant Historical Evidence",
    ]
    if not matches:
        lines.append("- No relevant Agentic SWMM memory was retrieved.")
    for idx, match in enumerate(matches, start=1):
        lines.extend(
            [
                f"{idx}. Source: `{match.get('source_path')}`",
                f"   - Type: `{match.get('source_type')}`",
                f"   - Run: `{match.get('run_id') or 'n/a'}`",
                f"   - Project: `{match.get('project_key') or 'n/a'}`",
                f"   - Failure patterns: `{', '.join(match.get('failure_patterns') or []) or 'none'}`",
                f"   - Diagnostics: `{', '.join(match.get('model_diagnostic_ids') or []) or 'none'}`",
                f"   - Matched terms: `{', '.join(match.get('matched_terms') or [])}`",
                f"   - Review state: retrieval_grounded=`{match.get('retrieval_grounded')}`, human_reviewed=`{match.get('human_reviewed')}`, benchmark_verified=`{match.get('benchmark_verified')}`",
                f"   - Evidence excerpt: {match.get('excerpt') or 'n/a'}",
            ]
        )
    lines.extend(
        [
            "",
            "## Answer Constraints",
            "- Use retrieved Agentic SWMM memory as historical context, not as proof of a new model result.",
            "- Distinguish confirmed audit evidence from inference.",
            "- Cite source paths and run ids when making a memory-grounded claim.",
            "- Keep missing evidence and QA limitations visible.",
            "",
        ]
    )
    return "\n".join(lines)


def slugify(value: str) -> str:
    slug = re.sub(r"[^A-Za-z0-9._-]+", "-", value.lower()).strip("-._")
    return slug[:80] or "rag-query"


def read_run_evidence(run_dir: Path) -> dict[str, Any]:
    memory_summary = read_json(run_dir / "memory_summary.json")
    provenance = read_json(run_dir / "experiment_provenance.json")
    comparison = read_json(run_dir / "comparison.json")
    diagnostics = read_json(run_dir / "model_diagnostics.json")
    note_text = read_text(run_dir / "experiment_note.md")
    return {
        "memory_summary": memory_summary if isinstance(memory_summary, dict) else {},
        "provenance": provenance if isinstance(provenance, dict) else {},
        "comparison": comparison if isinstance(comparison, dict) else {},
        "diagnostics": diagnostics if isinstance(diagnostics, dict) else {},
        "experiment_note": note_text,
    }


def extract_run_problem(evidence: dict[str, Any], run_dir: Path, repo_root: Path) -> dict[str, Any]:
    memory_summary = evidence.get("memory_summary") or {}
    provenance = evidence.get("provenance") or {}
    comparison = evidence.get("comparison") or {}
    diagnostics = evidence.get("diagnostics") or {}
    artifacts = provenance.get("artifacts") if isinstance(provenance.get("artifacts"), dict) else {}
    missing_evidence = listify(memory_summary.get("missing_evidence"))
    if not missing_evidence and artifacts:
        for artifact_id, record in artifacts.items():
            if isinstance(record, dict) and record.get("exists") is False:
                missing_evidence.append(str(artifact_id))

    stderr_excerpt = ""
    stderr_record = artifacts.get("runner_stderr") if isinstance(artifacts, dict) else None
    stderr_path = None
    if isinstance(stderr_record, dict):
        stderr_value = stderr_record.get("absolute_path") or stderr_record.get("relative_path")
        if stderr_value:
            candidate = Path(str(stderr_value))
            stderr_path = candidate if candidate.is_absolute() else repo_root / candidate
    if stderr_path and stderr_path.exists():
        stderr_excerpt = excerpt(read_text(stderr_path), set(), max_chars=800)

    diagnostic_ids = listify(memory_summary.get("model_diagnostic_ids"))
    if not diagnostic_ids:
        diagnostic_ids = [str(item.get("id")) for item in diagnostics.get("diagnostics", []) if isinstance(item, dict) and item.get("id")]

    failure_patterns = listify(memory_summary.get("failure_patterns"))
    if not failure_patterns:
        metrics = provenance.get("metrics") if isinstance(provenance.get("metrics"), dict) else {}
        if metrics.get("peak_flow") is None:
            failure_patterns.append("peak_flow_parse_missing")
        if metrics.get("continuity_error") is None:
            failure_patterns.append("continuity_parse_missing")
        if missing_evidence:
            failure_patterns.append("partial_run")
        if not failure_patterns:
            failure_patterns.append("no_detected_failure")

    qa_status = memory_summary.get("qa_status") or ((provenance.get("qa") or {}).get("status") if isinstance(provenance.get("qa"), dict) else None) or "unknown"
    diagnostic_status = memory_summary.get("model_diagnostics_status") or diagnostics.get("status") or "unknown"
    comparison_status = memory_summary.get("comparison_status")
    if not comparison_status:
        comparison_status = "mismatch" if any(isinstance(c, dict) and c.get("same") is False for c in comparison.get("checks", []) or []) else "unknown"

    return {
        "run_id": memory_summary.get("run_id") or provenance.get("run_id") or run_dir.name,
        "run_dir": relpath(run_dir, repo_root),
        "project_key": memory_summary.get("project_key"),
        "case_name": memory_summary.get("case_name") or provenance.get("case_name") or run_dir.name,
        "workflow_mode": memory_summary.get("workflow_mode") or provenance.get("workflow_mode"),
        "audit_status": memory_summary.get("audit_status") or provenance.get("status"),
        "qa_status": qa_status,
        "comparison_status": comparison_status,
        "failure_patterns": sorted(set(failure_patterns)),
        "model_diagnostic_ids": sorted(set(diagnostic_ids)),
        "missing_evidence": sorted(set(missing_evidence)),
        "warnings": sorted(set(listify(memory_summary.get("warnings")) + listify(provenance.get("warnings")))),
        "next_run_cautions": listify(memory_summary.get("next_run_cautions")),
        "stderr_excerpt": stderr_excerpt,
        "model_diagnostics_status": diagnostic_status,
    }


def should_generate_failure_advice(problem: dict[str, Any]) -> tuple[bool, list[str]]:
    reasons: list[str] = []
    if str(problem.get("audit_status") or "").lower() not in {"", "none", "pass"}:
        reasons.append("audit_status_not_pass")
    if str(problem.get("qa_status") or "").lower() not in {"", "none", "pass"}:
        reasons.append("qa_status_not_pass")
    if str(problem.get("comparison_status") or "").lower() == "mismatch":
        reasons.append("comparison_mismatch")
    if str(problem.get("model_diagnostics_status") or "").lower() in {"warning", "fail", "error"}:
        reasons.append("model_diagnostics_not_pass")
    patterns = set(problem.get("failure_patterns") or [])
    if patterns and patterns != {"no_detected_failure"}:
        reasons.append("failure_patterns_detected")
    if problem.get("missing_evidence"):
        reasons.append("missing_evidence_detected")
    if problem.get("warnings"):
        reasons.append("warnings_detected")
    if problem.get("stderr_excerpt"):
        reasons.append("stderr_present")
    return bool(reasons), sorted(set(reasons))


def build_failure_advice_query(problem: dict[str, Any]) -> str:
    parts = [
        str(problem.get("case_name") or ""),
        str(problem.get("workflow_mode") or ""),
        " ".join(problem.get("failure_patterns") or []),
        " ".join(problem.get("model_diagnostic_ids") or []),
        " ".join(problem.get("missing_evidence") or []),
        " ".join(problem.get("warnings") or []),
        str(problem.get("stderr_excerpt") or ""),
    ]
    return " ".join(part for part in parts if part).strip() or str(problem.get("run_id") or "failed run")


def render_failure_advice(advice: dict[str, Any]) -> str:
    problem = advice["current_run_problem"]
    lines = [
        "# Failure Advice",
        "",
        "This file is retrieval-grounded advice. It is not proof of model correctness and it did not modify model files.",
        "",
        "## Current Run Evidence",
        f"- Run ID: `{problem.get('run_id')}`",
        f"- Run directory: `{problem.get('run_dir')}`",
        f"- Project: `{problem.get('project_key') or 'n/a'}`",
        f"- Audit status: `{problem.get('audit_status')}`",
        f"- QA status: `{problem.get('qa_status')}`",
        f"- Diagnostic status: `{problem.get('model_diagnostics_status')}`",
        f"- Failure patterns: `{', '.join(problem.get('failure_patterns') or []) or 'none'}`",
        f"- Model diagnostics: `{', '.join(problem.get('model_diagnostic_ids') or []) or 'none'}`",
        f"- Missing evidence: `{', '.join(problem.get('missing_evidence') or []) or 'none'}`",
        f"- Trigger reasons: `{', '.join(advice.get('trigger_reasons') or [])}`",
        "",
    ]
    if problem.get("warnings"):
        lines.extend(["## Warnings", *[f"- {item}" for item in problem["warnings"]], ""])
    if problem.get("stderr_excerpt"):
        lines.extend(["## Stderr Excerpt", "```text", str(problem["stderr_excerpt"]), "```", ""])

    lines.append("## Retrieved Similar Memory")
    matches = advice.get("retrieved_memory") or []
    if not matches:
        lines.append("- No similar historical memory was retrieved.")
    for idx, match in enumerate(matches, start=1):
        lines.extend(
            [
                f"{idx}. `{match.get('source_path')}`",
                f"   - Run: `{match.get('run_id') or 'n/a'}`",
                f"   - Project: `{match.get('project_key') or 'n/a'}`",
                f"   - Source type: `{match.get('source_type')}`",
                f"   - Failure patterns: `{', '.join(match.get('failure_patterns') or []) or 'none'}`",
                f"   - Diagnostics: `{', '.join(match.get('model_diagnostic_ids') or []) or 'none'}`",
                f"   - Review state: retrieval_grounded=`{match.get('retrieval_grounded')}`, human_reviewed=`{match.get('human_reviewed')}`, benchmark_verified=`{match.get('benchmark_verified')}`",
                f"   - Matched terms: `{', '.join(match.get('matched_terms') or [])}`",
                f"   - Excerpt: {match.get('excerpt') or 'n/a'}",
            ]
        )

    lines.extend(["", "## Suggested Next Checks"])
    suggestions = advice.get("suggested_next_checks") or []
    if suggestions:
        lines.extend(f"- {item}" for item in suggestions)
    else:
        lines.append("- Inspect current run evidence manually before changing model inputs or workflow code.")

    lines.extend(
        [
            "",
            "## Boundary",
            "- This advice is generated from current audit evidence plus retrieved historical memory.",
            "- It does not edit SWMM model files, workflow code, or skill definitions.",
            "- Scientific modeling changes require human review and benchmark verification.",
            "- A repair should only become `resolution_memory.json` after verification evidence exists.",
            "",
        ]
    )
    return "\n".join(lines)


def suggested_checks(problem: dict[str, Any], matches: list[dict[str, Any]]) -> list[str]:
    checks: list[str] = []
    patterns = set(problem.get("failure_patterns") or [])
    diagnostics = set(problem.get("model_diagnostic_ids") or [])
    missing = set(problem.get("missing_evidence") or [])
    if "peak_flow_parse_missing" in patterns or "peak_qa" in missing:
        checks.append("Check whether the SWMM report contains `Node Inflow Summary`; if not, document the fallback section before accepting peak-flow evidence.")
    if "continuity_parse_missing" in patterns or "continuity_qa" in missing:
        checks.append("Check whether continuity tables are present in the runner report and referenced by the run manifest.")
    if "missing_inp" in patterns or "model_inp" in missing:
        checks.append("Record the runnable SWMM INP handoff before treating the run as reproducible.")
    if "continuity_error_high" in diagnostics:
        checks.append("Inspect routing step, storage, and external inflow/outflow accounting before treating the result as hydrologic evidence.")
    if "conduit_slope_suspicious" in diagnostics:
        checks.append("Check node invert elevations, conduit direction, and conduit length before changing hydrologic parameters.")
    if "partial_run" in patterns:
        checks.append("Keep this as partial-run evidence until the missing artifacts are produced or explicitly waived.")
    for match in matches[:3]:
        for caution in match.get("next_run_cautions") or []:
            if caution not in checks:
                checks.append(str(caution))
    if not checks:
        checks.append("Use the retrieved source paths to inspect the closest historical runs before deciding on a repair.")
    return checks

#!/usr/bin/env python3
from __future__ import annotations

import argparse
import json
import subprocess
from pathlib import Path
from typing import Any

from rag_memory_lib import extract_run_problem, now_utc, read_json, read_run_evidence, relpath, write_json


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description="Record a verified Agentic SWMM failure resolution memory.")
    parser.add_argument("--run-dir", type=Path, required=True)
    parser.add_argument("--repo-root", type=Path, default=Path.cwd())
    parser.add_argument("--action-taken", required=True, help="Human-readable repair action.")
    parser.add_argument("--verification", action="append", default=[], help="Verification command or evidence. Repeatable.")
    parser.add_argument("--file-changed", action="append", default=[], help="Changed file path. Repeatable.")
    parser.add_argument("--retrieved-source", action="append", default=[], help="Retrieved memory source path used for the repair. Repeatable.")
    parser.add_argument("--human-reviewed", action="store_true")
    parser.add_argument("--benchmark-verified", action="store_true")
    parser.add_argument("--out", type=Path, default=None)
    return parser.parse_args()


def git_head(repo_root: Path) -> str | None:
    proc = subprocess.run(["git", "rev-parse", "HEAD"], cwd=repo_root, capture_output=True, text=True)
    return proc.stdout.strip() if proc.returncode == 0 else None


def load_retrieved_memory(run_dir: Path, explicit_sources: list[str]) -> list[dict[str, Any]]:
    advice = read_json(run_dir / "failure_advice.json")
    memories: list[dict[str, Any]] = []
    if isinstance(advice, dict):
        for item in advice.get("retrieved_memory") or []:
            if isinstance(item, dict):
                memories.append(
                    {
                        "source_path": item.get("source_path"),
                        "run_id": item.get("run_id"),
                        "matched_terms": item.get("matched_terms", []),
                    }
                )
    for source in explicit_sources:
        if source and source not in {str(item.get("source_path")) for item in memories}:
            memories.append({"source_path": source, "run_id": None, "matched_terms": []})
    return memories


def main() -> int:
    args = parse_args()
    repo_root = args.repo_root.resolve()
    run_dir = args.run_dir.resolve()
    evidence = read_run_evidence(run_dir)
    problem = extract_run_problem(evidence, run_dir, repo_root)
    status = "verified" if args.human_reviewed and args.benchmark_verified else "draft"
    resolution = {
        "schema_version": "1.0",
        "generated_by": "swmm-rag-memory",
        "generated_at_utc": now_utc(),
        "run_id": problem.get("run_id"),
        "run_dir": relpath(run_dir, repo_root),
        "problem": {
            "failure_patterns": problem.get("failure_patterns", []),
            "diagnostics": problem.get("model_diagnostic_ids", []),
            "missing_evidence": problem.get("missing_evidence", []),
            "stderr_excerpt": problem.get("stderr_excerpt", ""),
        },
        "retrieved_memory_used": load_retrieved_memory(run_dir, args.retrieved_source),
        "resolution": {
            "action_taken": args.action_taken,
            "files_changed": args.file_changed,
            "verification": args.verification,
            "status": status,
            "source_commit": git_head(repo_root),
        },
        "retrieval_grounded": bool(load_retrieved_memory(run_dir, args.retrieved_source)),
        "human_reviewed": bool(args.human_reviewed),
        "benchmark_verified": bool(args.benchmark_verified),
        "boundary": "This records a repair for this workflow. It is not a universal SWMM modeling rule.",
    }
    out_path = args.out or (run_dir / "resolution_memory.json")
    write_json(out_path, resolution)
    print(json.dumps({"ok": True, "resolution_memory": str(out_path), "status": status}, indent=2, sort_keys=True))
    return 0


if __name__ == "__main__":
    raise SystemExit(main())

#!/usr/bin/env python3
from __future__ import annotations

import argparse
import json
import subprocess
import sys
from pathlib import Path


REPO_ROOT = Path(__file__).resolve().parents[3]
SUMMARIZE_MEMORY = REPO_ROOT / "skills" / "swmm-modeling-memory" / "scripts" / "summarize_memory.py"
BUILD_CORPUS = REPO_ROOT / "skills" / "swmm-rag-memory" / "scripts" / "build_memory_corpus.py"
GENERATE_ADVICE = REPO_ROOT / "skills" / "swmm-rag-memory" / "scripts" / "generate_failure_advice.py"


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description="Refresh Agentic SWMM modeling memory and RAG advice after an audited run.")
    parser.add_argument("--run-dir", type=Path, required=True, help="Audited run directory to inspect for failure advice.")
    parser.add_argument("--runs-dir", type=Path, default=Path("runs"))
    parser.add_argument("--memory-dir", type=Path, default=Path("memory/modeling-memory"))
    parser.add_argument("--rag-dir", type=Path, default=Path("memory/rag-memory"))
    parser.add_argument("--repo-root", type=Path, default=REPO_ROOT)
    parser.add_argument("--retriever", choices=("keyword", "hybrid"), default="hybrid")
    parser.add_argument("--top-k", type=int, default=5)
    parser.add_argument("--refresh-modeling-memory", action="store_true", help="Regenerate memory/modeling-memory before building the RAG corpus. Off by default to avoid overwriting curated memory records.")
    parser.add_argument("--no-advice", action="store_true", help="Only refresh RAG indexes.")
    return parser.parse_args()


def run_command(command: list[str], cwd: Path) -> subprocess.CompletedProcess[str]:
    return subprocess.run(command, cwd=cwd, text=True, capture_output=True, check=True)


def main() -> int:
    args = parse_args()
    repo_root = args.repo_root.resolve()
    steps: list[dict[str, object]] = []

    if args.refresh_modeling_memory:
        summarize = run_command(
            [
                sys.executable,
                str(SUMMARIZE_MEMORY),
                "--runs-dir",
                str(args.runs_dir),
                "--out-dir",
                str(args.memory_dir),
            ],
            repo_root,
        )
        steps.append({"step": "summarize_memory", "ok": True, "stdout": summarize.stdout.strip()})
    else:
        steps.append(
            {
                "step": "summarize_memory",
                "ok": True,
                "skipped": True,
                "reason": "Use --refresh-modeling-memory to regenerate curated modeling-memory outputs.",
            }
        )

    build = run_command(
        [
            sys.executable,
            str(BUILD_CORPUS),
            "--memory-dir",
            str(args.memory_dir),
            "--runs-dir",
            str(args.runs_dir),
            "--out-dir",
            str(args.rag_dir),
            "--repo-root",
            str(repo_root),
        ],
        repo_root,
    )
    steps.append({"step": "build_rag_corpus", "ok": True, "stdout": build.stdout.strip()})

    advice_written = False
    if not args.no_advice:
        advice = run_command(
            [
                sys.executable,
                str(GENERATE_ADVICE),
                "--run-dir",
                str(args.run_dir),
                "--memory-dir",
                str(args.memory_dir),
                "--runs-dir",
                str(args.runs_dir),
                "--index-dir",
                str(args.rag_dir),
                "--repo-root",
                str(repo_root),
                "--retriever",
                args.retriever,
                "--top-k",
                str(args.top_k),
            ],
            repo_root,
        )
        advice_payload = json.loads(advice.stdout)
        advice_written = bool(advice_payload.get("advice_written"))
        steps.append(
            {
                "step": "generate_failure_advice",
                "ok": True,
                "advice_written": advice_written,
                "triggered": advice_payload.get("triggered"),
                "trigger_reasons": advice_payload.get("trigger_reasons", []),
            }
        )

    if advice_written:
        rebuild = run_command(
            [
                sys.executable,
                str(BUILD_CORPUS),
                "--memory-dir",
                str(args.memory_dir),
                "--runs-dir",
                str(args.runs_dir),
                "--out-dir",
                str(args.rag_dir),
                "--repo-root",
                str(repo_root),
            ],
            repo_root,
        )
        steps.append({"step": "rebuild_rag_corpus_after_advice", "ok": True, "stdout": rebuild.stdout.strip()})

    print(
        json.dumps(
            {
                "ok": True,
                "run_dir": str(args.run_dir),
                "memory_dir": str(args.memory_dir),
                "rag_dir": str(args.rag_dir),
                "steps": steps,
                "boundary": "Post-run memory refresh only. No SWMM model files, workflow code, or skill definitions were modified.",
            },
            indent=2,
            sort_keys=True,
        )
    )
    return 0


if __name__ == "__main__":
    raise SystemExit(main())

#!/usr/bin/env python3
from __future__ import annotations

import argparse
import json
from pathlib import Path

from rag_memory_lib import build_corpus, load_corpus, load_embedding_vectors, render_context_pack, retrieve


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description="Retrieve relevant Agentic SWMM memory for a query.")
    parser.add_argument("--query", required=True)
    parser.add_argument("--memory-dir", type=Path, default=Path("memory/modeling-memory"))
    parser.add_argument("--runs-dir", type=Path, default=Path("runs"))
    parser.add_argument("--index-dir", type=Path, default=None)
    parser.add_argument("--repo-root", type=Path, default=Path.cwd())
    parser.add_argument("--top-k", type=int, default=5)
    parser.add_argument("--project", default=None)
    parser.add_argument("--retriever", choices=("keyword", "hybrid"), default="keyword")
    parser.add_argument("--format", choices=("json", "markdown"), default="json")
    return parser.parse_args()


def main() -> int:
    args = parse_args()
    entries = []
    embedding_vectors = []
    if args.index_dir:
        entries = load_corpus(args.index_dir / "corpus.jsonl")
        embedding_vectors = load_embedding_vectors(args.index_dir / "embedding_index.json")
    if not entries:
        entries = build_corpus(args.memory_dir, args.runs_dir, args.repo_root)
    matches = retrieve(
        entries,
        args.query,
        args.top_k,
        project=args.project,
        retriever=args.retriever,
        embedding_vectors=embedding_vectors,
    )
    if args.format == "markdown":
        print(render_context_pack(args.query, matches))
    else:
        print(
            json.dumps(
                {
                    "schema_version": "1.0",
                    "generated_by": "swmm-rag-memory",
                    "query": args.query,
                    "retriever": args.retriever,
                    "project_filter": args.project,
                    "match_count": len(matches),
                    "matches": matches,
                },
                indent=2,
                sort_keys=True,
                ensure_ascii=False,
            )
        )
    return 0


if __name__ == "__main__":
    raise SystemExit(main())