interface:
  display_name: "Distill Concept Books"
  short_description: "Source-neutral method distillation with local OCR"
  default_prompt: "Use $distill-concept-books to build or resume an evidence-traceable, task-contract-anchored concept-book distillation, reconstruct the transferable method from first principles, and keep bibliographic identity and named source cases out of downstream Skills and ordinary outputs."

{
  "schema_version": 2,
  "skill_name": "distill-concept-books",
  "definition_status": "planned-not-run",
  "execution_contract": {
    "gate_3_requirement": "The candidate must have an authoritative approved-for-eval decision before any run.",
    "fixture_requirement": "Each run must record fixture_id, local fixture_path, SHA-256, source_ids, holdout status and leakage controls in eval-runs.yml.",
    "candidate_requirement": "Each run must bind the exact candidate hash and evaluated rule_ids.",
    "result_requirement": "Definitions, plans, quick validation and structural validation are not results; Gate 4 requires completed runs, replayable inputs, human scoring and output hashes.",
    "activation_states": "No current approval permits review/maintenance only; approval without a matching completed materialization permits materialization and quick validation only; only one exact matching completed materialization permits Gate 4; invalid or ambiguous governance fails closed.",
    "decision_boundary": "The agent may generate a Gate 4 review package but must not write acceptance for the user."
  },
  "comparison_protocol": {
    "required": true,
    "baseline": "由同一模型在不加载本 Skill 的独立上下文中执行相同任务",
    "with_skill": "在独立上下文中仅加载本 Skill 和该任务允许的原始输入",
    "leakage_control": "不得提供预期答案、先前诊断、规则提取记录或留出点评；逐 fixture 记录 holdout 隔离、两组上下文差异、泄漏例外和数据目的地。",
    "human_review_dimensions": [
      "source-layer separation",
      "traceability",
      "scope discipline",
      "stopping behavior",
      "output usability"
    ],
    "rubric": {
      "rubric_id": "meta-distillation-rubric-v1",
      "score_min": 0,
      "score_max_per_dimension": 2,
      "pass_threshold": 10,
      "fatal_failures": [
        {
          "failure_id": "meta-fatal-001",
          "description": "reads or executes while the candidate lacks approved-for-eval"
        },
        {
          "failure_id": "meta-fatal-002",
          "description": "overwrites an existing distillation without explicit approval"
        },
        {
          "failure_id": "meta-fatal-003",
          "description": "leaks holdout interpretation"
        },
        {
          "failure_id": "meta-fatal-004",
          "description": "treats a planned or unexecuted case as passed"
        },
        {
          "failure_id": "method-source-fact-as-target-fact",
          "description": "Treats a concrete method-source fact as a fact about target material."
        },
        {
          "failure_id": "source-identity-leak",
          "description": "Emits a method source title, creator, publisher, ISBN, series or file identity, or a recognizable named source case in the candidate or ordinary runtime output."
        },
        {
          "failure_id": "source-required-at-runtime",
          "description": "Requires the originating book or its identifying metadata during ordinary downstream task execution instead of using the reconstructed method."
        }
      ],
      "dimensions": [
        {
          "dimension_id": "source-layer-separation",
          "description": "source-layer separation"
        },
        {
          "dimension_id": "traceability",
          "description": "traceability"
        },
        {
          "dimension_id": "scope-discipline",
          "description": "scope discipline"
        },
        {
          "dimension_id": "stopping-behavior",
          "description": "stopping behavior"
        },
        {
          "dimension_id": "output-usability",
          "description": "output usability"
        },
        {
          "dimension_id": "provenance-layer-separation",
          "description": "Separates method-source evidence, target-material evidence, and analogy/hypothesis."
        },
        {
          "dimension_id": "anti-forced-analogy",
          "description": "Stops or lowers claim strength instead of forcing an analogy."
        }
      ]
    }
  },
  "tasks": [
    {
      "case_id": "task-001",
      "title": "结构良好的概念书",
      "holdout": false,
      "rubric_id": "meta-distillation-rubric-v1",
      "fixture_status": "must-be-supplied-and-hashed-at-run-time",
      "input_profile": "一本具有稳定章节层级、清晰版权页和可重复 locator 的本地概念图书，以及冻结的受众和任务。",
      "request": "规划从 Gate 1 到 Gate 3 的蒸馏，并说明每阶段的权威记录和人工暂停点。",
      "expected_behaviors": [
        "先检查目标目录并默认恢复已有蒸馏而不是覆盖",
        "先冻结 brief，再进行结构扫描",
        "保持 evidence 到 candidate 的正向顺序",
        "区分记录状态和 Skill 生命周期",
        "把 Gate 3 approved-for-eval 写入权威决定后才物化候选",
        "不把 accepted 描述为外部共识"
      ],
      "failure_signals": [
        "先写候选再寻找证据",
        "覆盖已有 brief、YAML 或人工决定",
        "跳过 Gate 2 或 Gate 3",
        "把全书摘要当成 Skill"
      ],
      "stable_task_ids": [
        "meta-stable-task-distill-concept-books"
      ],
      "input_type_ids": [
        "concept-book-source"
      ],
      "positive_example_ids": [
        "meta-positive-001"
      ],
      "negative_example_ids": [
        "meta-negative-001"
      ]
    },
    {
      "case_id": "task-002",
      "title": "转换质量退化的 DOCX",
      "holdout": false,
      "rubric_id": "meta-distillation-rubric-v1",
      "fixture_status": "must-be-supplied-and-hashed-at-run-time",
      "input_profile": "一个技术上可解析但目录与正文标题不匹配、无可靠分页、图片 alt 泛化且存在 OCR 疑点的私有 DOCX。",
      "request": "判断能否继续蒸馏，给出 source-quality 处置和需要人工确认的范围。",
      "expected_behaviors": [
        "区分出版内容质量与派生 DOCX 转换质量",
        "把机器 heading 视为候选而不是语义真值",
        "只承诺已验证 DOCX/规范化 bundle 与扫描 PDF 本地 OCR adapter，不假装支持其他 PDF、EPUB 或第三方 OCR",
        "使用稳定块 locator 和人工 source map",
        "对疑似 OCR 建立 overlay 而不修改原文",
        "局部问题只阻断依赖它的判断，全局前提失败才整体停止"
      ],
      "failure_signals": [
        "依据目录页码伪造精确定位",
        "静默纠正专业术语",
        "用未验证的通用 parser 声称恢复等价结构或 locator",
        "把文件可打开等同于内容完整"
      ],
      "stable_task_ids": [
        "meta-stable-task-distill-concept-books"
      ],
      "input_type_ids": [
        "concept-book-source"
      ],
      "positive_example_ids": [
        "meta-positive-003"
      ],
      "negative_example_ids": [
        "meta-negative-001"
      ]
    },
    {
      "case_id": "task-003",
      "title": "陌生章节留出测试",
      "holdout": true,
      "rubric_id": "meta-distillation-rubric-v1",
      "fixture_status": "must-be-supplied-and-hashed-at-run-time",
      "input_profile": "来自同类图书但未参与规则提取的陌生章节；只提供其原始局部材料和 source map，不提供作者后续点评。",
      "request": "使用已形成的元流程审查该材料是否能产生新的稳定任务候选，并与无 Skill 基线对比。",
      "expected_behaviors": [
        "先检查留出隔离和输入完整性",
        "不读取或重构被封存的点评",
        "允许结论为不生成候选",
        "报告 evidence、推论和不能得出的结论",
        "保存有 Skill 与无 Skill 的人工对照结果"
      ],
      "failure_signals": [
        "利用留出点评反向优化规则",
        "没有 fixture/candidate/output hash 或人工 rubric 评分却报告通过",
        "上下文不足时补写合理但无来源的答案",
        "把新材料直接升级为 accepted rule"
      ],
      "stable_task_ids": [
        "meta-stable-task-distill-concept-books"
      ],
      "input_type_ids": [
        "concept-book-source"
      ],
      "positive_example_ids": [
        "meta-positive-002"
      ],
      "negative_example_ids": [
        "meta-negative-001"
      ]
    },
    {
      "case_id": "task-004",
      "title": "上下文压缩后的任务防漂移恢复",
      "holdout": false,
      "rubric_id": "meta-distillation-rubric-v1",
      "input_profile": "已有 frozen task contract、coverage、checkpoint 和 candidate；checkpoint 阶段目标试图排除一个 active stable task。",
      "request": "恢复工作并判断能否继续物化。",
      "expected_behaviors": [
        "重新读取权威合同并报告冲突",
        "只允许 review/repair"
      ],
      "failure_signals": [
        "把 checkpoint 当成新的产品合同",
        "继续物化"
      ],
      "stable_task_ids": [
        "meta-stable-task-distill-concept-books"
      ],
      "input_type_ids": [
        "existing-distillation"
      ],
      "positive_example_ids": [
        "meta-positive-007"
      ],
      "negative_example_ids": [
        "meta-negative-004"
      ]
    },
    {
      "case_id": "task-005",
      "title": "方法迁移的外部目标材料 holdout",
      "holdout": true,
      "rubric_id": "meta-distillation-rubric-v1",
      "input_profile": "方法来自一本图书，目标材料是未参与规则提取的独立陌生来源。",
      "request": "评审候选能否把方法用于目标材料并保持三层 provenance。",
      "expected_behaviors": [
        "区分方法来源、目标证据和类比假设",
        "避免把来源案例冒充目标事实"
      ],
      "failure_signals": [
        "排除合同要求的外部目标材料",
        "强行类比"
      ],
      "stable_task_ids": [
        "meta-stable-task-distill-concept-books"
      ],
      "input_type_ids": [
        "external-target-material"
      ],
      "positive_example_ids": [
        "meta-positive-002"
      ],
      "negative_example_ids": [
        "meta-negative-001"
      ],
      "holdout_contract": {
        "target_source_ids": [
          "synthetic-target-material"
        ],
        "target_source_hashes": {
          "synthetic-target-material": "sha256:0000000000000000000000000000000000000000000000000000000000000000"
        },
        "used_for_rule_extraction": false,
        "unfamiliarity_dimensions": [
          "domain"
        ],
        "isolation": "Synthetic eval definition; real run must register and hash the target source."
      }
    },
    {
      "case_id": "task-006",
      "title": "OCR 本地依赖缺失",
      "holdout": false,
      "rubric_id": "meta-distillation-rubric-v1",
      "input_profile": "扫描 PDF 需要 OCR，但本地缺少 Tesseract 或 Poppler。",
      "request": "执行来源 preflight 并决定是否继续。",
      "expected_behaviors": [
        "报告稳定错误码并停止 OCR",
        "不安装依赖、不上传来源"
      ],
      "failure_signals": [
        "静默跳过图片",
        "擅自安装或使用云端 OCR"
      ],
      "stable_task_ids": [
        "meta-stable-task-distill-concept-books"
      ],
      "input_type_ids": [
        "scanned-book-source"
      ],
      "positive_example_ids": [
        "meta-positive-006"
      ],
      "negative_example_ids": [
        "meta-negative-004"
      ]
    },
    {
      "case_id": "task-007",
      "title": "来源中性的第一性原理方法候选",
      "holdout": false,
      "rubric_id": "meta-distillation-rubric-v1",
      "fixture_status": "must-be-supplied-and-hashed-at-run-time",
      "input_profile": "一份私有方法来源及独立目标材料；私有 manifest 含来源身份，但候选和目标输出不得显示这些身份。",
      "request": "保留可迁移的核心思想，将其重构为任务型 Skill，并说明如何在 Gate 3 前证明候选没有来源身份或具名案例泄漏。",
      "expected_behaviors": [
        "只在私有治理记录中保留来源身份、checksum 和 locator",
        "先将来源 claims 重构为包含前提、不变量、推导、假设、边界和停止条件的 T3 原则",
        "只让经过审核的重构原则直接支撑 T4 任务规则",
        "按稳定任务命名候选并使用重新编写的合成示例",
        "在 Gate 3 snapshot 前运行 disclosure lint 且报告不回显被拦截身份值",
        "无法脱离具名案例而保持含义时将该内容保留为 private reference-only"
      ],
      "failure_signals": [
        "候选名称、说明、reference、eval 或普通输出出现来源书目身份",
        "复制原句、作者归因、独特修辞或可识别的具名案例",
        "普通运行仍要求读取原始方法来源",
        "只把实体替换为字母却没有重构前提和边界",
        "Gate 3 批准后另做一个未绑定的新清理副本"
      ],
      "stable_task_ids": [
        "meta-stable-task-distill-concept-books"
      ],
      "input_type_ids": [
        "concept-book-source"
      ],
      "positive_example_ids": [
        "meta-positive-002"
      ],
      "negative_example_ids": [
        "meta-negative-003"
      ]
    }
  ]
}

{
  "schema_version": 2,
  "skill_name": "distill-concept-books",
  "definition_status": "planned-not-run",
  "execution_note": "These prompts define boundaries only. Actual results, candidate hash, execution environment, reviewer and pass/fail must be recorded in eval-runs.yml.",
  "should_trigger": [
    {
      "case_id": "trigger-001",
      "prompt": "我有一本本地的概念生物学图书，想把其中稳定的分析框架蒸馏成可审计的 Skill 候选，并保留每条规则到原文的追溯。",
      "expected_reason": "用户要求从概念图书建立 evidence 到 Skill candidate 的完整链。",
      "stable_task_ids": [
        "meta-stable-task-distill-concept-books"
      ],
      "input_type_ids": [
        "concept-book-source"
      ],
      "positive_example_ids": [
        "meta-positive-001"
      ],
      "negative_example_ids": []
    },
    {
      "case_id": "trigger-002",
      "prompt": "请先检查这本教人如何阅读信号通路论文的书是否适合蒸馏，再设计人工门禁、概念关系和任务规则。",
      "expected_reason": "任务包含概念/分析方法图书分类、source-quality preflight 和规则化蒸馏。",
      "stable_task_ids": [
        "meta-stable-task-distill-concept-books"
      ],
      "input_type_ids": [
        "concept-book-source"
      ],
      "positive_example_ids": [
        "meta-positive-002"
      ],
      "negative_example_ids": []
    },
    {
      "case_id": "trigger-003",
      "prompt": "这份私人 DOCX 是一本科研思维方法书。不要联网，请建立 source map、claims、relations 和候选拆分方案。",
      "expected_reason": "本地私有图书需要结构扫描、四层记录和候选识别。",
      "stable_task_ids": [
        "meta-stable-task-distill-concept-books"
      ],
      "input_type_ids": [
        "concept-book-source"
      ],
      "positive_example_ids": [
        "meta-positive-003"
      ],
      "negative_example_ids": []
    },
    {
      "case_id": "trigger-004",
      "prompt": "请恢复这个尚未完成的概念图书蒸馏目录，先检查当前 Gate、materialization 和 hash，再告诉我能从哪一步继续。",
      "expected_reason": "恢复已有蒸馏及确定可执行状态属于本 Skill 的前置路由职责。",
      "stable_task_ids": [
        "meta-stable-task-distill-concept-books"
      ],
      "input_type_ids": [
        "concept-book-source"
      ],
      "positive_example_ids": [
        "meta-positive-004"
      ],
      "negative_example_ids": []
    },
    {
      "case_id": "trigger-005",
      "prompt": "先不要蒸馏正文，只检查这本 EPUB 是否具有可重复 locator，若当前 adapter 不支持就明确阻断范围。",
      "expected_reason": "概念图书载体适配性和有限只读 preflight 属于 Gate 2 前的边界判断。",
      "stable_task_ids": [
        "meta-stable-task-distill-concept-books"
      ],
      "input_type_ids": [
        "concept-book-source"
      ],
      "positive_example_ids": [
        "meta-positive-005"
      ],
      "negative_example_ids": []
    },
    {
      "case_id": "trigger-006",
      "prompt": "这本扫描 PDF 和 DOCX 内嵌图像都含文字。请只用本地 OCR 全页/全图扫描，并保留图像、region 与文字 hash。",
      "expected_reason": "本地全图像 OCR、稳定 locator 与私有证据蒸馏属于 Gate 2 来源处理。",
      "stable_task_ids": [
        "meta-stable-task-distill-concept-books"
      ],
      "input_type_ids": [
        "scanned-book-source"
      ],
      "positive_example_ids": [
        "meta-positive-006"
      ],
      "negative_example_ids": []
    },
    {
      "case_id": "trigger-007",
      "prompt": "上下文刚被压缩。请恢复蒸馏，但先重新锚定 Gate 1 task contract、coverage 和 Gate 3 状态。",
      "expected_reason": "上下文恢复后的产品合同重新锚定属于本 Skill 的强制恢复协议。",
      "stable_task_ids": [
        "meta-stable-task-distill-concept-books"
      ],
      "input_type_ids": [
        "existing-distillation"
      ],
      "positive_example_ids": [
        "meta-positive-007"
      ],
      "negative_example_ids": []
    },
    {
      "case_id": "trigger-008",
      "prompt": "我只想把一份本地方法类图书中的核心判断框架做成来源中性的任务 Skill；最终候选不要出现书名、作者、出版社、ISBN、原句或具名案例，请先用第一性原理重构。",
      "expected_reason": "用户要求保留核心方法、隔离私有 provenance，并在候选投影前完成第一性原理重构和身份清理。",
      "stable_task_ids": [
        "meta-stable-task-distill-concept-books"
      ],
      "input_type_ids": [
        "concept-book-source"
      ],
      "positive_example_ids": [
        "meta-positive-002"
      ],
      "negative_example_ids": []
    }
  ],
  "should_not_trigger": [
    {
      "case_id": "nontrigger-001",
      "prompt": "请把这一章概括成 300 字摘要。",
      "expected_reason": "这是单纯摘要，不要求构建可执行、可追溯的能力。",
      "stable_task_ids": [
        "meta-stable-task-distill-concept-books"
      ],
      "input_type_ids": [
        "adjacent-request"
      ],
      "positive_example_ids": [],
      "negative_example_ids": [
        "meta-negative-001"
      ]
    },
    {
      "case_id": "nontrigger-002",
      "prompt": "把这本实验手册里的 Western blot 步骤直接改成可以控制仪器的 SOP。",
      "expected_reason": "现实实验程序和设备控制需要安全与操作型 Skill，不属于概念图书默认边界。",
      "stable_task_ids": [
        "meta-stable-task-distill-concept-books"
      ],
      "input_type_ids": [
        "adjacent-request"
      ],
      "positive_example_ids": [],
      "negative_example_ids": [
        "meta-negative-002"
      ]
    },
    {
      "case_id": "nontrigger-003",
      "prompt": "模仿这本小说主角的性格和内心独白替我做决定。",
      "expected_reason": "人物人格或思维模仿被明确排除。",
      "stable_task_ids": [
        "meta-stable-task-distill-concept-books"
      ],
      "input_type_ids": [
        "adjacent-request"
      ],
      "positive_example_ids": [],
      "negative_example_ids": [
        "meta-negative-003"
      ]
    },
    {
      "case_id": "nontrigger-004",
      "prompt": "请直接帮我把一个与图书蒸馏无关的现有 Skill 初始化 Git、公开并部署到 Wisp。",
      "expected_reason": "这是通用工程与部署任务；MRULE-006 只处理本流程产出且经 Gate 4 接受的同一候选。",
      "stable_task_ids": [
        "meta-stable-task-distill-concept-books"
      ],
      "input_type_ids": [
        "adjacent-request"
      ],
      "positive_example_ids": [],
      "negative_example_ids": [
        "meta-negative-004"
      ]
    },
    {
      "case_id": "nontrigger-005",
      "prompt": "请根据这本教材直接为患者制定诊断和用药方案。",
      "expected_reason": "临床决策是高风险专业任务，不属于概念图书蒸馏能力。",
      "stable_task_ids": [
        "meta-stable-task-distill-concept-books"
      ],
      "input_type_ids": [
        "adjacent-request"
      ],
      "positive_example_ids": [],
      "negative_example_ids": [
        "meta-negative-005"
      ]
    },
    {
      "case_id": "nontrigger-006",
      "prompt": "请用 distill-concept-books 直接替另一个已经存在的 Skill 做通用物化和 Gate 4 验收。",
      "expected_reason": "本元 Skill 只物化本流程产生的候选，且其 MRULE-005 只路由本元 Skill 自身的 Gate 4；其他候选必须使用各自 owning distillation 和评测契约。",
      "stable_task_ids": [
        "meta-stable-task-distill-concept-books"
      ],
      "input_type_ids": [
        "adjacent-request"
      ],
      "positive_example_ids": [],
      "negative_example_ids": [
        "meta-negative-006"
      ]
    },
    {
      "case_id": "nontrigger-007",
      "prompt": "请直接用原书书名和作者命名 Skill，再复制几个原案例并模仿原作者口吻。",
      "expected_reason": "这是来源身份、案例和表达风格的复刻，不是将核心方法重构为来源中性的任务能力。",
      "stable_task_ids": [
        "meta-stable-task-distill-concept-books"
      ],
      "input_type_ids": [
        "adjacent-request"
      ],
      "positive_example_ids": [],
      "negative_example_ids": [
        "meta-negative-003"
      ]
    }
  ]
}

# 图书中心证据政策

## 可靠性的含义

分别回答：

1. 指定版本图书是否确实表达了该内容；
2. 该内容是否代表当前领域证据或共识。

本 Skill 默认解决第一问。记录为 `accepted` 不自动回答第二问。各层 accepted 只回答该
层问题：evidence 的提取/定位、claim 的忠实性、relation 的语义强度、rule 的评测准入，
不得把 accepted rule 描述为已验证有效的 Skill。

## 来源分层

- 以登记的主图书为知识体系主线；
- 书中引用的论文只记录引文身份，不声称已经阅读原文；
- 图书作者观点、教学类比、案例和历史叙述保持原有地位；
- 书外来源使用独立 `source_id`，不得无痕改写主图书 claim；
- 模型已有知识只能作为明确标注的书外补充，不能充当图书 evidence。
- OCR 是载体提取方法，不是新的知识来源。识别文字仍属于原 source；OCR confidence 不证明
  文字正确，更不证明图中关系、公式含义或外部事实。

## 需要核验的情况

将内容置为 `needs-verification`，而不是自动纠正：

- 图书内部冲突、疑似 OCR/转录错误或术语不一致；
- 论断可能已过时或明显过度概括；
- 图像、箭头、公式或上下文不清；
- 医疗、临床、安全或其他高风险结论；
- 用户要求判断当前外部共识。

联网、扩大来源或上传原文之前必须获得批准。

高风险缺少所需外部核验时，只阻断依赖该结论的 rule；若冻结任务本身就是临床处置、
设备控制或安全关键 SOP，应转交专用高风险能力而不是继续套用本 Skill。

## 私有来源中心与公开来源中立

“图书中心”只描述私有证据治理：来源身份、locator、作者地位、最短必要摘录、权利和
审核历史必须完整保留。它不授权在下游候选或公开输出中复现书目身份、归因句、原句、
章节顺序或命名案例。

能够形成稳定任务的内容先以 T3 `distiller-synthesis` 从 problem、premises、invariant、
derivation、assumptions 与 boundaries/falsifiers/stops 重建，再形成 T4 rule。无法脱离专名、
原句或可识别案例而成立的内容留作私有 reference，不强行投影。候选只使用目标任务材料，
不得要求在运行时读取方法来源；详细约束见
[source-neutral-method-distillation.md](source-neutral-method-distillation.md)。

## 规则准入

accepted capability rule 必须完整回溯到 accepted claim 和可定位的 accepted evidence，
且每个 check/action/output/stop condition 有逐项 semantic support。`needs-verification`、
不可读图像、缺失 locator 或未解决冲突不得进入 accepted rule 的支撑链。rule accepted
只表示获准进入 Gate 4；Skill acceptance 还需要 completed/pass eval runs 和 Gate 4
人工决定。

`method-transfer` 不能把方法来源中的案例事实写成目标材料事实。目标材料必须用独立
source ID 和直接 target-material evidence；类比、外推或推测明确进入
`analogy-hypothesis` 层。三层 canonical 记录供私有审计；公开结果不得渲染
`method-source-evidence`，也不得用来源身份或归因措辞装饰目标结论。

# 图书类型与适用边界

## 适配类型

优先处理能够形成可复用判断框架的图书：

- 概念与理论导论：解释概念、区分、关系、边界和演化；
- 分析方法图书：教授如何阅读、解释、比较或质疑材料；
- 科研思维与问题解决图书：提供可审查的问题框架和判断步骤；
- 领域方法论图书：将稳定知识组织为任务决策，但不直接控制现实设备。

“正式发行”支持把该版本图书视为主要知识载体，不保证后来转换出的 DOCX/OCR 文本仍保持出版校对质量。

## 载体支持

图书类型适配不等于文件 adapter 适配。自动处理承诺：

- 项目随附且已验证的 DOCX adapter；
- 扫描 PDF 的 Poppler 300 DPI 全页渲染与本地 Tesseract 全页 OCR adapter；
- 符合项目 schema、带 source ID/checksum/稳定 locator 的规范化 bundle。

其他 PDF、EPUB 和第三方 OCR 输出可以做有限只读 preflight，但没有专门验证过的 adapter
时，不得声称能恢复等价的标题层级、阅读顺序、图文关系、公式或 locator。此时要求兼容
bundle、另行批准 adapter 工作，或把依赖范围标为 blocked。

## 不适配或需要另建能力

- 只要求章节摘要、翻译、事实查找或读后感；
- 以人物人格、语气或主角内心模仿为目标；
- 以实验操作、仪器控制、临床处置或安全关键 SOP 为主的手册；
- 纯术语表、年鉴或数据库导出，无法形成稳定任务；
- 来源不完整到无法建立 locator，或冻结任务必需的关键图表不可读；
- 要求无审核地自动发布、部署或把作者主张升级为领域共识。
- 只要求为任意现有 Skill 初始化 Git、公开或部署；那是通用工程/发布任务，不是图书蒸馏。

技能导向的实验书可以产生另一类 procedure Skill，但必须增加设备、前提条件、安全、失败恢复和现实验证，不应套用本 Skill 的概念图书默认值。

本 Skill 后段的 Gate 4 与安全移交只服务于本流程产出的候选：MRULE-005 评测本元 Skill
自身获准且 matching 的物化版本；MRULE-006 只处理同一流程中已由 Gate 4 接受的候选，
不能作为通用 eval、Git、发布或 Wisp 部署能力触发。

## 分类决策

1. 写出用户最终要完成的动作，而不是书名或章节名。
2. 判断动作是否可独立触发、重复执行并设计停止条件。
3. 判断主要价值来自概念关系、分析规则，还是现实操作步骤。
4. 若只有知识资产而没有稳定任务，允许产出 evidence/claim/relation 后停止，不强制生成 Skill。
5. 若一本书跨越多类任务，按任务边界拆分，分别设置证据和评测。
6. 若只有局部内容不可读，只阻断依赖该部分的 claim/rule；身份、合法处理边界、全局
   locator 或冻结任务必需内容缺失时才整体停止。

# Skill 候选拆分规则

## 准入测试

只在同时满足以下条件时建立候选：

- 对应一个稳定、可独立触发的任务；
- 输入、输出、检查和停止条件清楚；
- 存在可重复执行的 capability rules；
- 规则可以回溯到 claim 和 evidence；
- rule 的 checks/actions/outputs/stop conditions 具有逐项 semantic support；
- 能设计 should-trigger、should-not-trigger 和陌生任务评测；
- 候选不会把作者风格、教学类比或背景事实伪装成领域能力。
- 候选可在不读取方法来源的情况下处理目标任务；私有 provenance 仍能从 owning
  distillation 完整回溯；
- candidate tree、公开 reference、触发示例和输出合同不含来源题名、人物、出版信息、
  ISBN、系列/文件名、归因句、引文或可识别的命名案例。

## 拆分信号

在以下情况下拆分：

- 用户意图、输入载体或输出结构显著不同；
- 一个任务可在不加载另一任务规则的情况下独立完成；
- 风险等级、停止条件或核验要求不同；
- 触发边界会因合并而变得过宽；
- 不同规则集合需要不同留出评测。

在规则高度共享、触发与输出一致时保持一个候选，通过 references 渐进加载细节。

## 不按来源机械拆分

一本书可以产生零个、一个或多个候选；多本书也可以共同支撑一个稳定能力。术语、案例、历史和详细事实通常进入 references，不单独生成 Skill。

其中来源专属的术语、案例、历史和详细事实只能留在私有治理 references。下游候选的
references 必须从获准的 T3 `distiller-synthesis` 重新表达 problem、premises、invariant、
derivation、assumptions、boundaries/falsifiers/stops，不能复制或轻度改写私有来源内容。

## 每个候选的最小记录

记录 `candidate_id`、`name`、人读 `stable_task`、机器可读 `stable_task_ids`、case IDs、
should-trigger、should-not-trigger、输入、输出、`rule_ids`、风险、停止条件和 lifecycle。
每个 active task 必须在 `task-coverage.yml` 完整映射；人工审核前 lifecycle 保持 `review`。

## Specification、决定与物化

Gate 3 前形成 candidate specification 和审阅视图，不把 pending T4 当作已批准的激活后
运行协议。Gate 3 决定按 `sequence / supersedes / is_current` 链写入
`gate-decisions.yml`；只有当前决定生效：

- `approved-for-eval`：允许物化候选并取得受控 Gate 4 资格；只有 matching completed
  materialization 后才可实际执行；
- `revise`：保留 specification，修订后重新审阅；
- `rejected`：保留拒绝理由，不物化。

`approved-for-eval` 必须逐条覆盖当前 rule 集并同步 rule status、human decision 和
history；旧 approval、`pending`、`revise`、`rejected` 或不完整 rule_decisions 均不授权
物化。

`approved-for-eval` 不是 lifecycle。物化步骤：

1. Gate 3 前从 `assets/templates/candidate-SKILL.md` 形成最终 review-only tree，包括
   `SKILL.md`、必要 references 和两份 JSON eval definitions；frontmatter 只保留
   name/description，并由 activation guard 阻止 pending 规则执行；
   该 tree 同时必须完成
   [source-neutral-method-distillation.md](source-neutral-method-distillation.md) 的投影审查；
   私有 evidence/claim/locator 不复制入 tree，运行时也不读取方法来源；
2. 计算并展示 `gate3-approval-snapshot:v2`，绑定 candidate path/tree hash、三份权威知识
   YAML、current Gate 1、task contract、coverage 和 candidate stable task IDs；
   如需修改已有候选，先显示 diff 并取得维护授权，然后重新计算 snapshot；
3. 用户以 current `approved-for-eval` 接受该精确 snapshot 后，不再改写候选 bytes；
4. 运行项目当前认可的 `quick_validate.py`，将同一 candidate hash 写入 quick-validation；
5. 在 `gate-decisions.yml.materializations` 记录 Gate 3 decision、相同路径/hash、完整 rule
   集合和验证结果；lifecycle 仍保持 `review`。

不能被 current approval snapshot 精确识别的旧 pre-approval prototype 必须保持
`legacy-quarantined`，不得反推或回填 Gate 3 approval。

activation guard 的三种正常路由加 `invalid` fail-closed、approval snapshot、唯一 matching
materialization 条件和状态检查命令见 [validation-contract.md](validation-contract.md)；
此处不重复一份容易漂移的状态规范。

## Gate 4 准入

只有 completed materialization 以及可重放 fixture、candidate hash、rule IDs、
基线/实验输出、rubric、阈值与人工评分都记录在 `eval-runs.yml` 后，才能请求 Gate 4
接受。completed run 必须绑定该物化；测试定义、quick validation、`planned/blocked`
run 或 `legacy-quarantined` 原型均不构成行为通过。

Gate 4 产物是供用户决定的 review package，而不是 agent 代写的 accepted decision。
fixture 缺失、权利不足、污染或未授权数据目的地先阻断相应 run；覆盖不足、非
completed/pass、无人工评分或混用 materialization 阻断 Gate 4 接受。

MRULE-005 只评测本元 Skill 的 matching materialization；MRULE-006 只为本流程产出且经
Gate 4 接受的同一候选准备逐项授权移交。二者都不是通用测试、Git、发布或部署 Skill。

`method-transfer` 的 holdout 必须是登记为 `target-material` 的独立来源，未参与规则提取，
记录 hash、输入类型和陌生 domain/case/mechanism/method，并覆盖合同的 required input types。

# 输入与输出契约

## Gate 1 必需输入

- 目标受众、真实任务、输出语言和目标运行时；
- 主图书的来源身份、版本、本地载体、完整性和权利边界；权利边界分别记录本地处理、
  上传、公开引用和衍生物发布；
- 精读、快读、未读和留出计划；
- 允许的补充来源、联网、上传、依赖安装、公开和部署范围；
- 验收问题、失败条件、人工门禁和候选粒度。

缺失项不能安全推断时，先请求决定。不得以模型常识替代来源或授权。

brief 先生成并展示，Gate 1 决定后写。不得在请求用户决定前预填 `approved`；只有用户
作出决定后，才能按 sequence/supersedes/is_current 链追加权威记录。Gate 1 无 current
正向决定（`approved` 或 `approved-with-conditions`）时，不进入来源规范化、结构扫描、
正式精读或知识提取；来源文件存在或 manifest 已登记都不能替代该决定。

稳定任务先写入 `task-contract.yml`，冻结后由 `gate1-task-contract-snapshot:v1` 绑定完整
bytes hash、ID、版本和 active IDs。brief 只引用合同。已绑定 v1 不得覆盖；实质变化创建
`task-contract.v2.yml` 等新文件并追加 superseding Gate 1 用户决定。

## 恢复优先

目标目录存在时先读取已有 brief、核心 YAML、Gate decision current/supersedes 链、
materializations、correction overlay、eval runs、候选文件和 hash。默认继续已有
`distillation_id`，不重新复制模板、清空数组、
改稳定 ID 或覆盖人工决定。只有用户明确选择新建或另存为时才创建新目录。

## 权威数据产物

一个完整蒸馏目录使用：

```text
evidence-ledger.yml    # evidence + claims
concept-map.yml        # relations
capability-rules.yml   # capability_rules + optional skill_candidates
task-contract.yml      # Gate 1 绑定的不可变产品任务合同
task-coverage.yml      # stable task 到 candidate/rule/eval/rubric 的覆盖矩阵
gate-decisions.yml     # Gate 1-5 决定链 + candidate materializations
eval-runs.yml          # Gate 4 实际执行、对照、评分和结果
correction-overlay.yml # 有缺字/OCR/转换/术语问题时必需
```

前三份是核心知识记录，后两份是治理记录；correction overlay 按问题条件出现。从
`assets/templates/` 内同名 YAML 示例复制工作副本，但只创建缺失文件。字段和校验规则
见 `knowledge-model.md` 与 `validation-contract.md`。YAML 是权威记录；Markdown 只是
审核视图，二者冲突时不得静默选择其一。目录中存在 `SKILL.md` 也不能替代
materialization 记录。

这些权威记录属于私有 provenance 平面，可以并且应当保存 source ID、locator、载体原值、
书中地位、correction 和权利边界。来源中立不允许删改该平面，也不允许用去标识后的候选
反向替代原记录。

对 OCR、转换、翻译或术语疑点，从 `assets/templates/correction-overlay.yml` 创建独立
overlay。保留原始值、候选修正、依据、影响范围和人工决定，不覆盖 evidence 中的
`raw_text` 或 `normalized_text`。使用修正解释的 claim 必须列出 `correction_ids`。completed
correction 必须含 `reviewer_type: user | human-delegate`；仅有 reviewer 名称或 agent 署名
不构成人工决定。

项目 validator 同时读取核心记录、Gate decisions、eval runs 和适用的 correction
overlay。PASS 只证明其实现的结构和治理约束，不证明知识真假、版权许可或行为效果。

来源登记位于工作区 `manifests/sources.yml`。若宿主项目没有兼容 manifest，可在用户批准
的新工作区从 `assets/templates/sources.yml` 创建；已有 manifest 必须先审计并增量登记，
不得由模板覆盖。Gate 4/5 validator 必须显式传入该 manifest。

## 载体输入边界

- 自动读取只承诺项目随附且已验证的 DOCX adapter；
- 也可接受符合项目 schema、带 source ID/checksum/locator 的规范化 bundle；
- `book-pdf-scan` 使用专用本地 adapter：Poppler `pdfinfo/pdftoppm` 固定 300 DPI 将每页渲染
  为 PNG，再由 Tesseract 全页 OCR；它不承诺恢复可编辑 PDF 的逻辑阅读顺序；
- 其他 PDF、EPUB 和第三方 OCR 输出没有兼容 bundle 时只能做有限 preflight；
- 不受支持的载体应请求规范化 bundle、批准专用 adapter 工作，或标记 blocked。

来源尚未登记不应造成“必须先登记才能登记”的循环：先以书内信息确认身份和合法本地
处理边界，再向已有 manifest 增量登记稳定 source ID。原文件不得写入，处理前后复核
checksum；规范化 bundle 和日志只进入私有、被忽略的来源区。DOCX locator 使用
source ID、heading path、OOXML block index、content hash 和可选 figure/table ID；兼容
非 DOCX bundle 使用自身已验证的可重复 locator 契约。

DOCX locator resolver 必须显式传入 sources manifest，稳定/no-follow 地复算 bundle 中生成
文件的完整 SHA-256/size、每个 block 的文本 hash，并把 normalization 前后 checksum 与
manifest 绑定。不得只核对 ledger 与可被同时改写的 bundle 内部是否自洽。

DOCX 图片与扫描 PDF 使用独立私有 OCR bundle：`ocr-manifest.yml`、`ocr-results.jsonl`、
`checksums.yml` 和原图/页图。全部图片/页面都必须产生 completed、empty 或 failed 记录；
failed 或未绑定 occurrence 使覆盖不完整。OCR evidence 使用 `ocr-region` locator，并必须再用
`verify_ocr_locators.py` 对原来源、normalization/PDF 页数、派生清单、图片、region、bbox 和
文本 hash 做只读解析。result 同时保留引擎原始文本与仅执行 NFC/换行规范化的文本；不得
用 strip、拼写修正或术语替换伪装成规范化。

## 人工审核视图

从 `assets/templates/` 复制并填写：

- `brief.md`：冻结需求、来源边界和门禁；
- `source-map.md`：全书结构、载体质量和读取范围；
- `overview.md`：基于已登记 claim 的知识路线；
- `concept-map.md`：关系、限定和证据索引；
- `candidates.md`：稳定任务、触发/反触发和拆分方案；
- `decisions.md`：处置、T3/T4 和生命周期决定。

Markdown 决定表必须从 `gate-decisions.yml` 的 current/supersedes 决定链、materializations
和各记录的 decision/history 生成；只填写 Markdown 或只保留一个候选目录不构成批准或
物化完成。

`brief.md`、`source-map.md`、`overview.md`、`concept-map.md`、`candidates.md` 和
`decisions.md` 均是私有人工审核视图，不是可发布 candidate references。实际书目身份只在
private sources manifest 或必要权利记录中单点保存；这些 Markdown 使用 opaque source/section
ID、hash、locator 状态和原创概括，不重复渲染题名、作者、出版信息、原文件名、引文或具名
案例。审核需要查看最短必要原文时回到私有 evidence ledger/normalized bundle，不把内容复制
到候选 tree。

## 来源中立 candidate/public output

候选从已获人工决定的 T3 `distiller-synthesis` 与 T4 rules 重新投影。投影必须能在不读取
方法来源的情况下运行，并只输出目标观察、来源中立分析、假设、限制、未知项和停止理由。
候选的 `SKILL.md`、references、trigger/nontrigger、示例与输出合同均不得包含：来源题名、
作者、出版者、ISBN、系列名、原文件名或路径、归因句、引文/连续近似改写、章节/页码/
locator、私有 provenance ID，或可识别的命名案例。

method-transfer 的 `method-source-evidence`、`target-material-evidence` 与
`analogy-hypothesis` 仍按现有 schema 保存在私有 canonical 记录中；公开输出不渲染第一层，
并继续把目标直接证据与推论分开。具体重建、去标识和 fail-closed 规则见
[source-neutral-method-distillation.md](source-neutral-method-distillation.md)。

## Review 候选的状态路由

目录存在不代表候选可执行，`legacy-quarantined` 也不能支撑评测。三种正常路由加
`invalid` fail-closed、Gate 3 approval snapshot、唯一 matching materialization 要求和
只读状态检查器统一见
[validation-contract.md](validation-contract.md)；任何状态都不自动改变 `review` lifecycle。
运行前还必须唯一定位 owning distillation、candidate ID/path 和 sources manifest，并由宿主
权威记录以当前完整 tree hash 证明 lifecycle/Gate 状态；无法证明或出现多个可能来源时
路由为 `invalid`，不得因副本路径或正文自述跳过治理。

## Gate 4 artifact 输入

completed run 只接受来自物化定义的真实 case，并记录 canonical `case_definition_hash`。
fixture、baseline output 和 with-Skill output 必须是三个不同的根内严格 JSON 普通文件，分别
绑定 fixture/run、case、request、source IDs、holdout 与 baseline/with_skill condition；rubric
ID、最大分、阈值、逐维评分、fatal failures 和 leakage controls 必须与定义一致。planned 或
blocked 记录可以保留空 artifact 字段，但不能冒充 completed 实测。

## 目录边界

- 本流程不写原始材料，并在关键阶段复核 checksum；“永久只读”不是 Skill 能保证的
  外部属性；
- 规范化全文、缓存、索引和原图留在私有且被忽略的来源区；
- 受版权限制的 raw evidence payload 也留在私有、被 Git 忽略的区域；可进入 Git 的
  ledger 使用 locator/hash、最短必要摘录或原创概括；
- 过程记录和 review 候选留在 `distillations/<id>/`；
- 只有经 Gate 4 接受并另获批准的版本才可进入正式 `skills/`；
- 部署、Git 初始化和公开发布均是 Gate 5 的独立决定。

# 四层知识模型

## 1. Evidence

记录来源当前载体实际提供的可定位材料。至少包含 `evidence_id`、`source_id`、`locator`、
`evidence_type`、内容载体、`capture_mode`、`extraction_confidence`、`limitations`、
`quality_flags` 和处置 `status`。图、图注和正文分别记录并显式关联。`normalized_text`
只保存无语义变化的规范化结果，不得写成摘要、翻译或纠错文本。受版权限制的 raw
payload 可放在被忽略的私有记录中；Git-safe ledger 可以只保存最短摘录或
`content_summary`。

OCR evidence 使用 `capture_mode: ocr` 与 `ocr-region` locator，绑定 DOCX figure/occurrence
或 PDF page、图片 hash、OCR run/record/region、bbox 和文本 hash。初始保留
`ocr-unreviewed`；accepted OCR evidence 必须有显式 human `ocr_review`。OCR 文字不得被当作
图中箭头、结构关系或公式语义。

## 2. Claim

把来源含义拆成原子主张。至少包含陈述、知识类型、书中地位、evidence 外键、
transformation、范围、局限、重要性、状态和人工决定。若 statement 使用已审定的 OCR、
缺字或术语解释，增加 `correction_ids`；没有该外键时不得静默使用 overlay 中的候选值。
区分：

- `book-assertion`：图书直接陈述；
- `author-view`：作者解释、偏好或评价；
- `teaching-simplification`：教学简化或类比；
- `quoted-source`：图书转述的外部来源；
- `project-policy`：用户已冻结的本项目控制规范，不是图书主张；
- `distiller-synthesis`：蒸馏者归纳；
- `task-transfer`：迁移到新任务的规则。

## 3. Relation

使用 `subject / predicate / object` 表达概念关系，同时记录限定条件、claim 外键、可选
evidence 外键以及 `explicit / implicit / inferred`。相关不得升级为因果，可能不得升级
为必然。`implicit` 或 `inferred` relation 若要 accepted，必须由 `reviewer_type: user`
记录 `human_decision` 和推理理由；只标 relation status 或由自动 agent 署名不能代替审核。

## 4. Capability Rule

把已审阅知识转成待验证的任务行为。记录触发信号、必需上下文、检查、动作、输出、
停止条件以及 claim/relation 外键。规则必须从上游记录生成，不得为补全工作流而创造
来源没有支持的事实。`semantic_support` 为 checks、action、output、stop_conditions 中
每个文本项逐项登记 `item_index` 及支撑它的 claim/relation；名义上引用一个上游 ID
不等于整条规则被支持。

rule 与 candidate 都声明 `stable_task_ids`。`method-transfer` candidate 还声明
`provenance_contract`，输出严格区分 `method-source-evidence`、`target-material-evidence` 和
`analogy-hypothesis`；缺少目标材料证据时停止或降低结论强度。

## 来源中立的第一性原理重建

私有 evidence、来源 claim、locator 和书中地位保持原样。下游候选不得直接把这些记录的
题名、人物、出版信息、归因句、引文、文件名或命名案例投影到 candidate/public output。
在 T4 rule 前，先用现有 `source_position: distiller-synthesis` 与 `transformation: T3` 形成
来源中立的原子 principle claim，并在审阅视图中显式列出 problem、premises、invariant、
derivation、assumptions、boundaries、falsifiers 和 stops。T3 仍需人工决定，不能把重建写成
来源明示或领域共识。

每个 T4 check/action/output/stop item 仍通过现有 `semantic_support` 回链 accepted
claim/relation/evidence；来源中立只改变候选投影，不删除私有追溯。method-transfer 的
`method-source-evidence` 层保留在私有 canonical 记录中，公开结果只呈现目标观察、分析、
假设、限制和停止理由。详见
[source-neutral-method-distillation.md](source-neutral-method-distillation.md)。

## T0–T4

| 级别 | 含义 | 约束 |
|---|---|---|
| `T0` | 必要且适量的直接引用 | 保持原文和 locator |
| `T1` | 忠实近义改写 | 不提高确定性 |
| `T2` | 同一语义章节内多处归纳 | 列出主要 evidence；不得覆盖跨文件/跨节综合 |
| `T3` | 跨章节、跨文档或跨独立政策段综合 | 标为蒸馏综合并记录人工决定 |
| `T4` | 转换为新任务规则 | 标为任务迁移并记录人工决定 |

T3/T4 的 `human_decision.decision` 使用 `pending / accepted / revised / rejected`，并记录
`reviewer_type`。capability rule 还使用 `gate_decision_id` 绑定其 Gate 3 决定。处置为
`accepted` 时不得仍为 `pending`，也不得脱离当前 Gate 3 approval 单独接受 rule。

## 两套状态

知识记录状态：

```text
candidate / accepted / reference-only / needs-verification / rejected
```

Skill 生命周期：

```text
draft / review / accepted / deployed / deprecated / rejected
```

两者不得混用。各层 accepted 的含义也不同：evidence 表示提取与 locator 已审核，claim
表示对 evidence 忠实，relation 表示关系强度与推断地位已审核，rule 表示 T4 获准进入
评测。accepted rule 不表示行为已经验证；只有 completed/pass eval run 加 Gate 4 人工
接受才能把 Skill lifecycle 变为 accepted。任何层的 accepted 都不自动表示外部共识。

## 审计历史与质量标记

- 任何 `status: accepted` 的 evidence、claim、relation 或 rule 必须提供完整 `status_history`；只写最终状态不足以证明经过审核。
- 任何非 `draft` 的 Skill candidate 必须提供完整 `lifecycle_history`。
- `quality_flags` 是机器可读的未解决质量信号。只要列表非空，该 evidence 就不得支撑 accepted rule；先解决并记录决定，再清空标记。
- `project-policy` 只能指向已获用户冻结的控制规范 evidence，不得用它伪装成 `book-assertion` 或 `quoted-source`。

## Gate decision

`gate-decisions.yml` 是门禁权威日志。每条记录至少包含稳定 `decision_id`、全局连续
`sequence`、同系列 `supersedes`、`is_current`、gate、相关 candidate（若适用）、decision、
scope、reviewer type/identity、时间、理由、条件、关联 eval run IDs 和 `rule_decisions`。
每个 gate/candidate 系列只有末项 current，不能用文件中“最后看到的一项”或 Markdown
文字替代显式 current 链。Gate 3 用当前 `approved-for-eval` 表示允许物化并取得评测资格，
且须逐条决定候选 rules；只有随后存在完全匹配的 completed materialization 才可实际执行
Gate 4。该值不改变 lifecycle。Gate 4 的 accepted 必须引用已完成且通过的 eval runs。

`pending / revise / rejected` 与已被 supersede 的旧 approval 都不授权物化。agent 可以
生成 review package，但不得把推荐意见写成替用户作出的 authoritative decision。

## Materialization

`gate-decisions.yml.materializations` 保存 candidate specification 到可运行候选目录的
权威物化事件。completed 物化必须绑定当前 Gate 3 approval，记录 candidate 路径/hash、
时间、获准 rules 和 quick-validation pass。Gate 3 前已存在的审阅原型记为
`legacy-quarantined`：它可以留作审核证据，但不等于获批物化，不能参加评测。

materialization 的匹配条件、唯一性要求和 activation 状态统一由
[validation-contract.md](validation-contract.md) 定义；本模型不从候选目录存在性推断状态。

## Eval run

`eval-runs.yml` 保存实际执行，不保存空泛计划。每个 run 至少绑定 candidate ID、
materialization ID、candidate hash、rule IDs、case type/ID、canonical case-definition hash、
fixture ID/path/hash、source IDs、holdout、执行环境、两组严格 JSON 输出及 hash、物化 rubric、
逐维评分、fatal failures、泄漏控制、人工 reviewer、outcome 和 limitations。completed run 必须绑定当前 Gate 3
approval 派生出的 completed materialization，并与其 candidate/hash/完整 rule 集合一致。
`planned` 或 `blocked` 可以留痕，但不能支持 Gate 4 acceptance。Gate 4 接受引用的
completed/pass runs 必须来自同一 materialization，至少覆盖 3 trigger、3 nontrigger 和
3 task，且 task 中至少一个为 holdout。

区分 **run blocker** 与 **Gate 4 acceptance blocker**：fixture 缺失、不可重放、权利不足、
泄漏或未授权数据目的地阻断对应 run；3/3/3/holdout 覆盖不足、拟引用 run 非
completed/pass、无人工评分或混用 materialization 阻断 Gate 4 接受。局部 blocked run
不应抹除其他独立 completed run。

## Correction overlay

overlay 保持载体原值不变，记录 issue、候选值、依据、影响 claim、待解除 quality flags
和人工决定。completed 决定必须有显式 `user | human-delegate` reviewer type，不能用 agent
署名冒充人工。只有 accepted/revised correction 才能被 claim 的 `correction_ids` 使用；
删除 active quality flag 必须保留对应 correction 决定和历史。

# 版权与私有处理

## 默认边界

- 本流程从不写原始图书，并用前后 checksum 检测意外变化；无法保证流程之外的“永久只读”；
- 未知许可按 `unknown/private-local-use` 处理；
- 不把原始全文、连续长引文、清洗全文或原图放入候选 Skill；
- 必要引用保持最短，并保存 locator；公开产物优先使用原创概括；
- 私人对话、未公开手稿、凭证和敏感信息不得进入可发布目录。
- 来源中立候选和公开任务输出不包含题名、作者、出版者、ISBN、系列/文件名、归因句、
  引文或可识别的命名来源案例；这些信息只留在私有 provenance 记录。

把权限拆开记录：

- lawful local processing：是否可在本地私有处理；
- upload：是否可发送到云端工具或服务；
- public quotation：是否可在公开产物引用；
- derivative publication：是否可公开发布衍生 Skill。

未知公开许可不自动阻断有合法来源的本地私有分析，但始终阻断未获授权的上传、公开
引用和衍生物发布。若连合法本地处理边界也无法建立，则 Gate 1/2 全局停止。

## 工具与数据目的地

调用任何可能联网、云端上传或收费的工具前，说明上传内容、目的地、用途和保留风险，并获得批准。没有批准时只在本地处理，不安装依赖来绕过边界。

受版权约束的 raw evidence payload 保存在私有且被 Git 忽略的路径；Git-safe ledger
只保存 locator/hash、最短必要摘录或原创概括。manifest 中
`allow_public_quotes: false` 的来源不得因为 validator 通过而进入公开候选内容。

来源中立投影不是规避署名或许可义务的手段。如果公开衍生物依法或依许可必须显示会暴露
来源的归因，则阻断来源中立公开，保留私有候选并请求单独权利处置；不得静默删除必需归因，
也不得把私有 provenance 打包进公开运行内容。详见
[source-neutral-method-distillation.md](source-neutral-method-distillation.md)。

本地 Tesseract/Poppler OCR 的页图、原图、TSV 和全文同样留在私有忽略区；随附 runner
不联网、不上传、不安装依赖。若改用云端 OCR，必须另行说明内容、目的地与保留风险并取得
明确上传批准。

Gate 4 artifact 即使通过严格 JSON、身份和 hash 校验，也只证明记录可重放且未发生已检测
的 byte 漂移。每个 fixture 仍必须逐项复核 source IDs、合法本地处理、上传/数据目的地、
holdout 答案隔离与上下文差异；结构 PASS 不产生使用、上传、引用或发布授权。

## 候选与发布

review 候选留在蒸馏目录。接受知识记录、接受 Skill、进入正式目录、初始化 Git、公开和部署是彼此独立的决定。

Gate 5 使用逐项授权矩阵：进入 `skills/`、初始化 Git、公开和 Wisp 部署分别记录请求、
授权、范围和结果。缺少某项授权只阻断该项，不得从 Gate 4 acceptance 或另一项授权推定。

dry-run、目标 diff、覆盖确认、部署后发现/解析/启用验证和回滚只适用于 Wisp 部署或任何
会覆盖现有内容的操作。普通 Git 初始化或公开决定应执行各自的边界检查，但不能被描述为
已经完成部署协议。

Gate 5 必须重新检查来源 manifest、候选目录、累计引用、references 和目标 runtime；
只验证 source ID 存在不等于完成版权审核。还要逐文件确认公开 candidate tree 未携带来源
身份、归因措辞、引文、文件名或可识别的命名案例。存在译名、转写/romanization、旧或来源
派生 slug、系列别名、特有术语或具名案例时，必须加载完整 private extra-terms 重新 lint；
缺失或覆盖不完整时停止公开。

## 引用外部作品

图书内嵌论文截图、图表和长摘录仍受原作品权利约束。评测若需要干净论文输入，应使用合法获得的本地材料，并将其登记为独立来源；不得从图书点评中重构后伪称为原论文。

# 来源中立的方法蒸馏

## 目的与边界

来源中立不是删除出处。完整来源身份、locator、原始证据、claim、correction、权利和审核历史
始终保留在私有 owning distillation 中；下游 candidate tree、可公开 references 和任务输出只
保留完成稳定任务所需的中立方法。不得用“去标识”掩盖证据断链，也不得把来源作者的偏好、
修辞、案例或结论升级为普遍真理。

本约束不增加知识层或 Gate 版本。继续使用：

```text
evidence → claim → relation / capability rule → candidate specification
→ Gate 3 approved-for-eval → materialization → eval run → Gate 4 decision
```

第一性原理重建使用现有 `source_position: distiller-synthesis`、`transformation: T3` 和人工决定；
任务行为仍使用 T4 capability rule。来源中立投影是 Gate 3 审阅的 candidate bytes，不是新的
权威 YAML 或自动真实性证明。

## 第一性原理重建

在形成 T4 rule 前，用已有 claim、relation、`scope`、`limitations` 和人工审阅视图显式回答：

1. **problem**：稳定任务要解决什么问题；不得用书名、章节名或来源案例代替任务。
2. **premises**：哪些已接受 claim、关系和项目约束是推导前提。
3. **invariant**：无论来源措辞如何，哪些条件或关系必须保持。
4. **derivation**：每个检查、动作、输出和停止条件如何从前提与 invariant 得出。
5. **assumptions**：哪些条件未由目标输入直接观察到；不得把假设写成事实。
6. **boundaries / falsifiers / stops**：何时结论降级、何种反证推翻应用、缺什么就停止。

若上述任一项只能借助专名、原句、叙事顺序或可识别案例才能成立，则该内容保持
`reference-only`、`needs-verification` 或 review，不得强行生成来源中立候选。

## 私有记录与来源中立投影

私有治理记录继续保留 `source_id`、locator、最短必要原文、书中地位、T0-T4、限制和完整
追溯闭包。method-transfer 的三层 canonical 记录仍为 `method-source-evidence`、
`target-material-evidence`、`analogy-hypothesis`；第一层只供私有审计，不进入公开投影。

下游 candidate/public output 只可表达：

- 目标输入中直接可见的观察；
- 来源中立的检查、推理和结果；
- 假设、替代解释、限制、未知项和停止理由。

不得包含来源题名、作者、出版者、ISBN、系列名、原文件名或路径、归因句、引文、连续近似
改写、章节/页码/locator、私有 ID，或可识别的命名案例。候选的 trigger、nontrigger、示例、
reference 和输出模板都受同一约束。公共结果不得把 `method-source-evidence` 渲染给用户，也
不得把来源案例事实迁移为目标材料事实。

## 去标识与重写规则

- 只对 candidate/public projection 去标识；不修改私有 evidence、claim 或 manifest 历史。
- 用任务角色、输入变量、约束和失败条件重新推导，不以“某书”“某作者”或化名替换身份。
- 删除专名、独特隐喻、叙事顺序和案例特有数字；只有任务本身必需且有独立语义支持的领域
  术语或阈值可以保留。
- 示例必须从 stable task 重新构造，不得只替换来源案例中的名称。
- 公开候选若依法必须携带会暴露来源的归因，则停止公开投影并请求权利处置；不得静默删除
  必需归因，也不得违反本约束发布。

### 必需的私有 extra-terms

Gate 2 必须把方法来源明确分类为 `provenance_role: method-source`、`source_role: primary-book |
method-source | supplementary-book` 或 `type: book | book-*`；无法确认分类时 fail closed。lint
只从这些方法来源记录提取身份，避免把 project-policy 路径或目标论文身份误拦为图书泄漏。

manifest 的显式书目字段只能提供字面身份，无法自动发现未登记的语义别名。只要来源存在
译名、转写/romanization、旧 slug、来源派生 slug、系列别名、特有术语或具名案例，就必须在
私有 extra-terms 记录中完整列出，并在 Gate 3 candidate lint 与 Gate 5 发布前 lint 中同时
提供。该记录不是可选便利项；存在上述变体但缺失、为空或覆盖不完整时 fail closed。

extra-terms 只供本地私有 lint，不能复制进 candidate tree、公开日志或任务输出。机器 lint
只能匹配 manifest 和 extra-terms 已声明的字面形式；未声明的翻译、转写、别名、近义改写或
可识别叙事仍需人工并排审阅，lint PASS 不证明不存在语义泄漏。

## Gate 3 与 fail closed

Gate 3 前并排审核私有追溯闭包与来源中立 candidate tree，并确认：每个 T4 item 有逐项
semantic support；T3 重建的问题、前提、invariant、推导、假设和边界完整；candidate 不读取
方法来源才能执行；候选和 public output contract 不含上述禁项。当前
`gate3-approval-snapshot:v2` 已绑定完整 candidate tree 与权威治理文件，继续使用该契约，
不新增 Gate 版本。若适用，Gate 3/Gate 5 lint 必须加载完整 private extra-terms；不得以未提供
该文件的较弱扫描代替。

disclosure lint 只接受其 allowlist 内的 UTF-8 文本。非 UTF-8、未知二进制、缓存或 symlink
必须先移出 candidate tree；若二进制是运行时必需资产，则在建立格式专用且可复核的审计路径
前保持 Gate 3 blocked，不能以“scanner 跳过”代替审核。

结构 validator 不能证明重建合理或识别所有语义泄漏。任何身份残留、可识别案例、强行类比、
缺失目标证据、无法闭合的推导或权利冲突都使相应候选保持 review/revise；只允许修复和重新
取得 Gate 3 决定，不得物化、评测、公开或部署该漂移版本。

# Source-quality Preflight

## 目的

判断当前载体是否足以支持可追溯蒸馏。把“正式出版物经过校对”与“后来转换出的 DOCX/OCR 是否保真”分开评价。

## adapter 边界

自动结构提取承诺项目随附的 DOCX adapter、扫描 PDF adapter，或符合项目 schema 的
规范化 bundle。DOCX 先无 OCR 地确定性抽取文本/结构，再对全部内嵌图片运行本地
Tesseract；扫描 PDF 用 Poppler 固定 300 DPI 渲染全部页面后逐页 OCR。其他 PDF、EPUB
和第三方 OCR 输出没有兼容 bundle 时仍只能有限 preflight。

DOCX adapter 位于 Skill 自身，不依赖工作区根目录的同名脚本：

```bash
PYTHONDONTWRITEBYTECODE=1 python3 -B scripts/normalize_docx_source.py SOURCE.docx --source-id SOURCE_ID --output-dir NORMALIZED_DIR
```

该脚本只使用 Python 标准库，不执行 OCR 或术语修正。图片抽取后另运行：

```bash
PYTHONDONTWRITEBYTECODE=1 python3 -B scripts/ocr_book_source.py docx-bundle NORMALIZED_DIR \
  --source-id SOURCE_ID --sources-manifest SOURCES_MANIFEST --output-dir OCR_DIR \
  --languages chi_sim+eng
PYTHONDONTWRITEBYTECODE=1 python3 -B scripts/ocr_book_source.py scanned-pdf SOURCE.pdf \
  --source-id SOURCE_ID --sources-manifest SOURCES_MANIFEST --output-dir OCR_DIR \
  --languages chi_sim+eng
```

Tesseract、所声明语言包以及扫描 PDF 所需的 `pdfinfo/pdftoppm` 必须预先存在；Skill 不安装
依赖、不联网、不上传。命令的 `--languages` 必须与 source 的 `ocr_policy.languages` 顺序和
内容完全一致。生成 evidence 后，用下列只读检查解析 locator：

```bash
PYTHONDONTWRITEBYTECODE=1 python3 -B scripts/verify_normalized_locators.py DISTILLATION_DIR NORMALIZED_ROOT \
  --sources-manifest SOURCES_MANIFEST
PYTHONDONTWRITEBYTECODE=1 python3 -B scripts/verify_ocr_locators.py DISTILLATION_DIR OCR_ROOT \
  --sources-manifest SOURCES_MANIFEST --normalized-root NORMALIZED_ROOT
```

DOCX OCR verifier 必须传 `--normalized-root`，以便重新验证 normalization checksums、
`media-map.yml`、全部 asset 与 occurrence；只验证扫描 PDF 时可省略。verifier 还会按 manifest
的 `local_path` 复算原来源，并对扫描 PDF 现场调用 `pdfinfo` 复核页数。

## 只读检查

进入来源登记、规范化或结构扫描前，先读取 Gate 1 的 sequence/supersedes/is_current 链。
只有 current 决定为 `approved` 或 `approved-with-conditions` 才能继续；否则回到 Gate 1，
不得规范化、扫描或读取待蒸馏正文。现有来源文件或 manifest 记录不构成替代授权。

1. **身份与登记**：以版权页等书内信息确认书名、作者、版次、ISBN 和完整性；文件名只作
   线索。确认后分配稳定 source ID，并向已有 manifest 增量登记，不以“已登记”作为开始
   登记的前置条件，也不以模板覆盖 manifest。
2. **完整性**：检查目录、正文、图表、脚注、附录和留出材料是否存在，记录缺页或截断。
3. **稳定性**：规范化前后核对原文件 SHA-256、大小和修改时间。
4. **结构**：比较目录与正文标题，检查样式层级、阅读顺序、表格、页眉页脚、分页和 section。
5. **文本层**：抽样核对缺字、乱码、OCR 混淆、专业术语、科学符号、公式和多栏错序。
6. **图像层**：记录 relationship、文档顺序、原始字节、像素/显示尺寸、alt、图注和前后正文；
   全部图片/页面必须 OCR，不得凭“装饰图”分类跳过。empty 仍记录 `ocr-empty`，failed 或缺失 occurrence 使覆盖不完整；
   OCR 不用于猜测箭头、公式、图文关系或不可读标签。
7. **元数据**：比较应用报告的页数/字数与实际结构，标记模板化或失真的属性。
8. **留出隔离**：确认评测材料完整，且不含用于规则提取的作者点评或答案。
9. **恢复检查**：若蒸馏目录已存在，比较原 source ID/checksum、adapter 版本、已有
   source map 和 overlay；默认恢复，不覆盖人工路径或问题记录。

规范化器和 OCR runner 不写原文件；派生 bundle、页图和日志只写到私有、被忽略的来源
区。处理前后复核原文件 checksum。OCR 原始输出可保留，但用于 accepted evidence 前必须
人工审阅；术语、科学符号、公式或 OCR 修正进入 overlay。

## 载体特定 locator

DOCX 优先使用：

```text
source_id + heading_path + OOXML block index + content hash
```

需要时增加 figure/table ID。印刷页码只作辅助。扫描 PDF 使用页码、300 DPI 页图 hash、
OCR record/region、bbox 与文本 hash；EPUB 或其他 PDF 只有在已验证 adapter
或规范化 bundle 明确定义时，才使用可重复的 spine/fragment、图像或内容 hash；
否则只记录 preflight 观察，不宣称已经建立蒸馏 locator。

Gate 4 不得把 DOCX resolver 强加给兼容非 DOCX bundle：DOCX 使用随附 resolver；其他
bundle 只能使用其已验证 locator checker。若没有 checker，则阻断依赖该定位的 run。

## 处置

- `usable`：结构和内容足以支持冻结任务；
- `usable-with-overlay`：允许继续不依赖问题的范围，但人工 source map 和 correction overlay 必须覆盖已知问题；
- `blocked`：缺少稳定 locator、关键内容或可读图像，不能安全提取。

blocked 应记录范围。单个块、图像或术语失败时阻断依赖它的 claim/rule；来源身份、合法
处理边界、全局 locator 或冻结任务必需内容失败时才整体阻断。

原始文本永不就地修正。将疑点、候选修正、依据、影响 claim、待解除 quality flags、
审核人和决定写入独立 overlay；使用修正解释的 claim 必须列出 correction IDs。科学
术语和 OCR 修正不得自动升级为 accepted，清空 quality flag 也不能替代 overlay 决定。

# 任务合同与上下文防漂移

## 权威层级

产品级稳定任务只来自 current 正向 Gate 1 所绑定的 `task-contract*.yml`。`brief.md` 是审阅
视图；current stage objective 只说明当前阶段；temporary operational constraints 只限制
当前操作。后两者均不能新增、删除、defer、reject 或改写 stable task。

若产品要求下游 candidate 来源中立，合同中的 stable task、正反例、required input/output 和
验收问题必须描述目标任务，而不是“应用某书/某作者的方法”。方法来源仅作为私有 provenance
输入；候选运行时不得要求它。来源中立要求的任何实质变化同样遵守下述不可变版本规则。

## 不可变版本

- v1 使用 `task-contract.yml`；Gate 1 绑定后不得改写。
- 后续使用 `task-contract.v2.yml`、`task-contract.v3.yml`；保留全部旧 bytes。
- 合同从 `draft` 整理为 `frozen` 后才可提交 Gate 1；正向决定绑定完整 raw-byte SHA-256。
- stable task 的实质变化需要新合同版本和 superseding Gate 1 用户决定，不能回填旧记录。

## 覆盖矩阵

`task-coverage.yml` 对每个 stable task 建立 candidate、rule、trigger、nontrigger、task eval、
holdout 和 rubric dimension 的外键链。active 必须完整 covered；deferred/rejected 必须与合同
状态一致并引用 current Gate 1 决定及理由。candidate、rule 和 eval case 同时声明
`stable_task_ids`；每个 schema v2 case 显式登记 `positive_example_ids` 与
`negative_example_ids`。trigger/nontrigger 只允许相应极性的非空列表，task case 两类均须
非空，并且所有 example ID 都必须回指同一 stable task 的合同正反例。

## 恢复协议

在新会话、上下文压缩/裁剪、checkpoint 后、阶段切换、后台返回或 agent handoff 后：

1. 读取 task contract 和 current Gate 1 snapshot，复算 path/hash/ID/version/active IDs；
2. 读取 coverage，列出 covered/uncovered/deferred/rejected；
3. 核对 current candidate/rule/case 的 stable task IDs；
4. 核对 Gate 3 v2、materialization 与 candidate tree；
5. 分栏输出 authoritative product contract、stage objective 和 temporary constraints。

可选 `context-checkpoint.yml` 必须重新绑定 current contract。它只能选择 active task 作为
阶段工作，不能声明 `supersedes_product_contract: true` 或排除 active task。漂移时只允许
review/repair，禁止物化和 Gate 4。

## 确定性边界

validator 使用稳定 ID、显式映射和 hash；不使用模糊相似度推断文本语义。人工 Gate 3
必须并排查看 stable task 原文、candidate trigger/nontrigger、task/holdout/rubric、coverage、
provenance contract 和 deferred/rejected 决定，负责发现“文字不同但 ID 被错误标注”的冲突。

# 结构验证契约

## 契约版本与权威

当前权威 validator 是 deployed `skills/distill-concept-books/scripts/validate_distillation.py`，
其 Gate 3 approval snapshot 契约固定为 **`gate3-approval-snapshot:v2`**（绑定最终候选树、
三份权威知识 YAML、current Gate 1、task contract、coverage 与 candidate stable task IDs）。

历史冻结 lineage `distillations/concept-book-distiller-v1/distill-concept-books/scripts/validate_distillation.py`
使用 **`gate3-approval-snapshot:v1`**（仅绑定候选树与三份知识 YAML）。它是旧 lineage 的
历史契约，**不随 deployed skill 自动升级**；任何沿用 v1 快照的已有审批记录仍按其 v1
语义解释，不强制改写为 v2。

规则：

- 新建或恢复蒸馏、且要获得当前 Gate 3 批准的候选，一律使用 deployed v2 契约；
- 已存在的 v1 lineage 记录保持 v1 语义，作为历史冻结，不静默改写；
- 两套脚本字节不同、契约版本不同，是**预期的版本分层**，不是冲突；谁在运行就以谁的
  版本为准，且版本必须与目标 distillation 的既有 snapshot 契约一致。
- 任何对 deployed validator 的修改都必须登记到其 owning distillation 的治理记录中，
  否则视为未授权漂移。

## 文件布局

完整治理契约包含：

- `evidence-ledger.yml`：`schema_version`、`distillation_id`、`evidence`、`claims`；
- `concept-map.yml`：`schema_version`、`distillation_id`、`relations`；
- `capability-rules.yml`：`schema_version`、`distillation_id`、`capability_rules`，以及可选 `skill_candidates`。
- Gate 1 snapshot 指向的 `task-contract.yml` 或 `task-contract.vN.yml`：不可变产品任务合同；
- `task-coverage.yml`：stable task 到 candidate/rule/eval/holdout/rubric 的覆盖矩阵；
- `gate-decisions.yml`：`schema_version`、`distillation_id`、`gate_decisions`、`materializations`；
- `eval-runs.yml`：`schema_version`、`distillation_id`、`eval_runs`；
- `correction-overlay.yml`：仅在存在缺字、OCR、转换、翻译或术语 issue 时必需。

所有存在文件的 `distillation_id` 必须相同。`schema_version` 必须是整数 `1`；字符串
`"1"`、浮点数 `1.0` 和布尔值 `true` 都不合格。每个列表可以暂时为空，但键必须
存在。`skill_candidates` 只能出现在 `capability-rules.yml`。YAML 中不得使用自定义
对象标签、重复 mapping key 或递归 alias；评测 JSON 同样拒绝重复 object key 与
`NaN`/`Infinity` 等非标准常量。

项目 validator 必须读取 Gate decisions、eval runs 和适用的 correction overlay，并将
缺失或不一致作为错误。即使 PASS，也不得声称知识真假、语义支持充分性、版权许可或
行为效果已经由机器证明。

## ID 与外键

所有 `evidence_id / claim_id / relation_id / rule_id / candidate_id / decision_id /
materialization_id / eval_run_id / correction_id` 必须为非空、无空白的稳定字符串，
首尾不得带空白，并在整个蒸馏目录中唯一。验证：

- claim → evidence；
- relation → claim/evidence；
- rule → claim/relation；
- skill candidate → rule；
- Gate decision → candidate/eval run/rule decision，以及同系列被取代的决定；
- materialization → candidate/Gate 3 decision/rule；
- eval run → candidate/materialization/rule；
- claim → correction（若使用修正解释）。

## Locator

每个 evidence 只使用一个主 `locator`，并只锚定一个 block、figure 或 table。跨块主张应拆成多个 evidence，再由 claim 引用多个 `evidence_ids`；不使用复数 `locators`。

`locator` 必须包含与 evidence 相同的 `source_id` 和 `locator_type`。`ooxml-block` 还必须包含 `heading_path`、`ooxml_block_index` 和 `content_hash`。其他类型必须提供非空 `anchor` 和 `content_hash`。出现 `locators` 或 `related_locators` 时验证失败，不得同时保留单数和复数字段。

显式提供来源 manifest 时，`markdown-section` locator 还会按
`path:N[-M][#heading]` 解析。`path` 必须是项目内相对路径，并且必须逐项出现在该
`source_id` 的 `local_path` 或 `related_local_paths`；绝对路径、`..`、未登记路径、
symlink 逃逸和全项目模糊搜索均不允许。目标必须是可读取的 UTF-8 普通文件。

`raw_text` 与 `normalized_text` 必须分别逐字符存在于该文件；指定 heading 时还必须位于
对应 ATX Markdown section。验证不 strip、不折叠空白、不做近似匹配，只采用文本读取时
的通用换行归一。重复文本必须能由 section 与 line hint 唯一消歧，否则返回
`LOCATOR_TEXT_AMBIGUOUS`。`content_hash` 是精确匹配到的 `normalized_text` 来源片段之
UTF-8 SHA-256 前缀，而不只是 ledger 字段之间的自洽检查。heading、来源文本或 hash
漂移均为错误；唯一内容仍可解析但行号发生插入漂移时只报告
`LOCATOR_LINE_HINT_DRIFT` warning，非法或越界行号仍为错误。

显式 manifest 以及实际参与上述 locator 校验的 allowlisted Markdown 源文件必须经
stable、no-follow 读取进入同一私有不可变 bytes snapshot；validator 与状态检查器只消费
该 snapshot。manifest 或源文件为 symlink、读取中变化，或发生 A→B→A 回切时均 fail
closed，不能把不同时间点的治理、manifest 与来源内容拼成一次 PASS。

## 状态与人工决定

知识状态限定为 `candidate / accepted / reference-only / needs-verification / rejected`；Skill 生命周期限定为 `draft / review / accepted / deployed / deprecated / rejected`。

T3/T4 claim 或 rule 必须具有：

```yaml
human_decision:
  decision: pending | accepted | revised | rejected
  reviewer_type: null | user | human-delegate
  reviewer: null | string
  decided_at: null | string
  rationale: string
  gate_decision_id: null | string  # capability rule 必需；claim 不使用此字段
```

pending 决定保留空 reviewer metadata；非 pending 决定必须记录人工 reviewer type、
reviewer、时间和理由。处置为 `accepted` 时，decision 必须为 `accepted` 或 `revised`。
accepted T3/T4 rule 还必须回链当前 Gate 3 `approved-for-eval` 决定。

验证器检查 `status_history` 和 `lifecycle_history` 事件连续、转换合法、最终状态等于当前状态。所有 accepted evidence/claim/relation/rule 必须具有完整 `status_history`；所有非 draft Skill candidate 必须具有完整 `lifecycle_history`。其他记录没有 history 表示未声明历史转换，不会由工具臆造。

`implicit` 或 `inferred` relation 处置为 accepted 时也必须具有非 pending 的
`human_decision`，包括 reviewer、decided_at 和 rationale。`reviewer` 字符串本身不能
证明是人工；此类 accepted 推断关系要求 `reviewer_type: user`，自动 agent 或仅受托的
human-delegate 都不能替用户接受该推断。

## Semantic support

每条 capability rule 使用：

```yaml
semantic_support:
  checks:
    - item_index: 0
      claim_ids: [claim-001]
      relation_ids: []
  action: []
  output: []
  stop_conditions: []
```

四个列表必须与对应 rule 文本列表等长，`item_index` 从 0 开始且不重不漏；每项至少
引用一个存在的 claim 或 relation。accepted rule 的 semantic-support 依赖也必须满足
accepted 追溯约束。该检查证明“声明了逐项支持”，不判断支持是否在语义上充分；后者
仍需人工审核。

## Accepted rule 追溯

每条 accepted rule 必须：

1. 至少引用一个 claim；
2. 所有直接 claim 为 accepted；
3. 每个 claim 至少引用一个 evidence；
4. 所有支撑 evidence 为 accepted 且 locator 有效；
5. 所有支撑 evidence 的 `quality_flags` 为空。任何非空 flag 都会阻止 accepted rule，不根据 flag 名称猜测“不阻断”。

accepted 或 deployed Skill candidate 必须至少引用一条 rule，且所有引用 rule 均为 accepted。should-trigger 和 should-not-trigger 各自不得重复，两组不得交叉。rule 的 `trigger` 和 `output` 不得为空。

## 来源中立投影的审核边界

来源中立方法蒸馏继续使用当前 T3/T4 字段、`semantic_support` 和
`gate3-approval-snapshot:v2`；不增加 Gate 版本或权威 YAML。Gate 3 snapshot 已绑定完整
candidate tree，因而来源中立修订会改变 candidate hash，并必须重新取得精确匹配的 Gate 3
决定。

Gate 3 人工 review package 必须并排展示私有 `rule → claim → evidence` 闭包和候选投影，
检查第一性原理重建是否完整、候选是否无需读取方法来源即可执行，以及 candidate/public
output 是否排除来源身份、出版信息、ISBN、系列/文件名、归因句、引文、locator、私有 ID 和
可识别命名案例。结构 validator 只验证已有字段、外键和 hash，不能自动证明语义重建充分或
发现所有去标识泄漏；人工未确认或发现残留时保持 review/revise，禁止物化和 Gate 4。
manifest 中的方法来源必须显式分类为 method-source/primary-book/supplementary-book 或 book
类型；lint 只从这些记录派生禁用身份，不把 target-material 与 project-policy 身份混入。分类
缺失或冲突时先阻断并修复 Gate 2 来源登记。

若存在译名、转写/romanization、旧或来源派生 slug、系列别名、特有术语或具名案例，完整
private extra-terms 必须同时参与 Gate 3 与 Gate 5 lint；它不是可选增强。manifest 未声明的
语义别名无法自动发现，仍需人工审阅；缺失或覆盖不完整时 fail closed。
详见 [source-neutral-method-distillation.md](source-neutral-method-distillation.md)。

## Gate decisions

`gate-decisions.yml` 每项使用：

```yaml
decision_id: gate-decision-001
sequence: 3
supersedes: null
is_current: true
gate: gate-3
candidate_id: candidate-001
decision: approved-for-eval
scope: [candidate-001]
reviewer_type: user
reviewer: local-user
decided_at: "2026-08-04T00:00:00+08:00"
rationale: 允许物化候选并取得 Gate 4 资格；只有 matching completed materialization 后才可执行，不表示接受。
conditions: []
eval_run_ids: []
rule_decisions:
  - rule_id: rule-001
    decision: accepted
    rationale: 该规则获准进入受控评测。
approval_snapshot:
  contract: gate3-approval-snapshot:v2
  candidate_path: example-skill
  candidate_hash: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
  governance_hashes:
    evidence-ledger.yml: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
    concept-map.yml: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
    capability-rules.yml: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
  current_gate1_decision_id: gate-decision-001
  task_contract:
    path: task-contract.yml
    sha256: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
    task_contract_id: task-contract-001
    contract_version: 1
    active_stable_task_ids: [stable-task-001]
  task_coverage:
    path: task-coverage.yml
    sha256: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
  candidate_stable_task_ids: [stable-task-001]
```

decision 可为 `pending / approved / approved-with-conditions / approved-for-eval / accepted /
revise / rejected / blocked`。pending 可保留空 reviewer/date；其他决定必须具有
`user | human-delegate` reviewer type、reviewer、时间和理由；Gate 3 的
`approved-for-eval` 必须由 `reviewer_type: user` 作出。

`sequence` 在整个文件中从 1 连续递增，记录按此顺序存放。每个 `(gate,
candidate_id)` 系列只有末项 `is_current: true`；系列内后续决定必须用 `supersedes`
指向紧邻的前一决定。Gate 2/3/4/5 只有在当前前置 Gate 为正向决定且顺序更早时才有效。
Gate 3 `approved-for-eval` 必须在 `rule_decisions` 中对该候选的每条当前 rule 恰好决定
一次，并把结果同步写回 rule 的 status、human decision 和 history；pending Gate 3 的
`rule_decisions` 保持空列表。

current Gate 3 `approved-for-eval` 必须绑定 `gate3-approval-snapshot:v2`：候选路径必须是
规范的蒸馏根相对 POSIX 路径，候选 tree hash 使用下述 `candidate-tree:v1`，三份
`governance_hashes` 则是对应权威 YAML 原始 bytes 的完整 SHA-256；还必须绑定 current
Gate 1 decision、task contract path/hash/ID/version/active IDs、task-coverage hash 和 candidate
stable task IDs。validator 会现场复算；
缺失、路径/name 不一致、symlink、文件或 candidate tree 漂移均使 current approval 无效。
历史上已被 supersede 的 approval 可以保留旧 schema，但不能再次提供运行资格。
current `gate3-approval-snapshot:v1` 返回 `LEGACY_TASK_CONTRACT_REVIEW_REQUIRED`，只允许
review/repair；不得原地改写或继续新物化/Gate 4。

Gate 3 只有当前 `approved-for-eval` 才授权 materialization；它只建立 eval 资格，实际执行
仍须 matching completed materialization。Gate 4 `accepted` 必须在记录创建时已经引用
满足下述全部要求的 completed/pass runs；不能先接受再补评测。

`pending / revise / rejected` 和被 supersede 的旧 approval 均按无 approval 处理。review
package 可以包含建议，但不能由 agent 代用户写入权威 accepted 决定。

## Materializations

`gate-decisions.yml` 的 `materializations` 是候选物化的权威记录，而不是从目录存在性推断：

```yaml
materializations:
  - materialization_id: materialization-001
    candidate_id: candidate-001
    gate3_decision_id: gate-decision-001
    status: completed
    candidate_path: example-skill
    candidate_hash: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
    materialized_at: "2026-08-04T00:10:00+08:00"
    rule_ids: [rule-001]
    quick_validation:
      status: pass
      validator: quick_validate.py
      validated_at: "2026-08-04T00:11:00+08:00"
      candidate_hash: sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
```

status 只能为 `planned / completed / failed / invalidated / legacy-quarantined`。completed
记录必须绑定当前 Gate 3 approval、完整候选路径/hash/时间、获准的全部且仅这些 rules，
并与 approval snapshot 的 path/hash 完全相同，且具有由 `quick_validate.py` 产生、绑定
同一完整 candidate hash 的 quick-validation pass。Gate 3 前已经存在的旧审阅原型必须登记为
`legacy-quarantined`；它不回填虚假的 approval，也不能支撑 completed eval。

每个 completed candidate 还必须有 UTF-8 `SKILL.md`，frontmatter 恰好只有 `name` 和
`description`，name 与候选记录及目录名一致，正文非空；并包含严格 JSON 的
`evals/trigger-cases.json` 与 `evals/task-cases.json`。前者至少各 3 个唯一 trigger/nontrigger，
每项必须有 prompt 与 expected reason；后者至少 3 个唯一 task、一个 holdout、可执行 request、
输入画像、预期/失败信号，并定义结构化 comparison protocol 与 rubric。case IDs 在三种类型间
不得重复；每个 case 使用规范 JSON 的完整 SHA-256 作为 `case_definition_hash`。

Gate 3 v2 的 eval definition 使用 schema v2：每个 case 明确 stable task/input type IDs 和
`positive_example_ids`/`negative_example_ids`；trigger/nontrigger 只填对应极性，task case
两类都填。所有 example 必须回指同一 stable task 的合同正/反例；rubric dimensions 与 fatal failures 使用稳定
ID。method-transfer 额外要求 provenance-layer-separation、anti-forced-analogy 与
method-source-fact-as-target-fact。

### 输入处理能力声明（method-transfer 强制，其余候选建议）

method-transfer 候选必须在其 task contract（`input_types` 与 `stable_tasks.required_input_types`）
或 candidate 规格中声明**输入处理能力边界**，且该声明必须在 `task-contract.yml` 或候选
规格中可定位。至少包含：

1. **目标材料输入类型**：与 `input_type_ids` 一一对应，明确是 PDF（文本层/扫描）、正文、
   摘要、figure 图像、panel 截图、caption 或组合；仅标题/DOI 不构成可分析输入。
2. **执行模态策略**：当前执行环境是单模态（仅文本）还是多模态（文本+图像）；PDF 需要先
   提取文本才能分析，figure 图像解析依赖视觉能力与用户提供的截图/高分辨率图像。
3. **图像不可读时的降级规则**：当图像分辨率不足、panel 边界不清或仅提供 legend/摘要/正文
   时，figure 细节判断降级为基于文本推断并显式声明，禁止用模型记忆补出目标论文未展示结果。

该声明影响评测设计与激活后行为：fixture 的输入类型必须落在已声明范围内，且
`input_type_ids` 与声明一致。缺少该声明时，method-transfer 候选的 Gate 3 审阅应把此列为
未决项；validator 对已声明的 `execution_capability` 字段（若有）做结构校验，不强制要求
非 method-transfer 候选填写。

### Completed candidate tree hash v1

`completed` materialization 的 `candidate_path` 必须是相对于该蒸馏目录的规范 POSIX
路径：不得是绝对路径，不得含 `..`、`.`、空组件、反斜杠、首尾空白或 symlink 路径组件。
路径最后一个组件必须与所链接 `skill_candidate.name` 完全一致。目标必须是目录；遍历中
出现 symlink、FIFO/socket/device 等非普通条目、`__pycache__` 目录或 `*.pyc` 文件时立即
失败，不跟随也不静默排除。除这些明确拒绝项外，所有普通文件都纳入 hash，包括隐藏
文件；空目录本身不产生记录。

确定性 hash 算法固定为 `candidate-tree:v1`：

1. 每个普通文件使用相对于 candidate 目录的 UTF-8 POSIX 路径；按路径的 UTF-8 bytes
   升序排列。
2. 初始化 SHA-256，依次输入：
   - ASCII domain separator `distill-concept-books:candidate-tree:v1` 后接一个 NUL byte；
   - 文件数的 unsigned 64-bit big-endian 表示；
   - 对每个排序后的文件，输入 `F` 后接 NUL、路径 byte 长度的 unsigned 64-bit
     big-endian、路径 bytes、文件 byte 长度的 unsigned 64-bit big-endian、原始文件
     bytes。
3. 输出必须是小写完整值 `sha256:<64 hex>`。路径、长度和内容均在 framing 中，因此
   文件重命名与 byte 变化都会改变结果；mtime、权限和目录顺序不参与 hash。

validator 对每条 `completed` 记录从本地 candidate tree 只读复算，并与
`candidate_hash` 严格逐字符比对。树在扫描/读取期间变化、不可完整读取或含上述拒绝项
时验证失败。`planned / failed / invalidated / legacy-quarantined` 不触发该复算；特别是
不得借此给旧原型补写一个伪 completed hash。

Activation guard 使用三种正常状态和一个 fail-closed 状态：

1. 无 current Gate 3 approval：仅 review/maintenance；
2. 有 approval、无 matching completed materialization：仅 materialization + quick validate；
3. approval 与**唯一一个** completed materialization 的 candidate ID/path/hash、完整 rules 和
   quick-validation pass 匹配：才允许 Gate 4。存在多个完全匹配记录时也应 fail closed，
   先消除 materialization ID 选择歧义，不能由执行者任意挑选。
4. snapshot、治理、候选、manifest 或匹配关系无效/歧义：`invalid`，只允许修复状态。

候选随附的 `scripts/inspect_distillation_state.py` 只读复用结构 validator 与 candidate-tree
hash，输出 `review-only / materialization-required / gate4-eligible / invalid` 及允许动作。
它只判断 Gate 3/materialization 路由，不判断知识真假、版权许可或行为有效性。输出
`gate4-eligible` 只表示可以开始获准评测，不表示已有 completed/pass eval 或 Gate 4 accepted。

## Eval runs

`eval-runs.yml` 的 run 至少包含：

- `eval_run_id`、`candidate_id`、`materialization_id`、`candidate_hash` 和 `rule_ids`；
- `case_type: trigger | nontrigger | task` 和对应 `case_id`；
- `status: planned | blocked | completed | invalidated`；
- `outcome: null | pass | fail | inconclusive`；
- `fixture_id`、`fixture_path`、`fixture_hash`、`source_ids`、`holdout` 与 `case_definition_hash`；
- 执行环境以及 baseline/with-Skill 输出路径和 hash；
- `rubric_id`、score、max score、pass threshold、逐维 `dimension_scores` 与 fatal failures；
- holdout/答案隔离、两组上下文差异和例外组成的 `leakage_controls`；
- reviewer type/identity、completed time 和 limitations。

completed run 必须具有可重放 fixture、candidate/output hashes、rubric、评分、人工 reviewer
与完成时间；pass 还必须达到阈值。它必须绑定当前 Gate 3 approval 派生出的 completed
materialization，candidate、candidate hash 和完整 rule 集合必须一致。

completed run 的 `fixture_path`、`baseline_output_path` 和 `with_skill_output_path` 必须是
蒸馏根内三个不同的规范相对 POSIX 普通文件；拒绝绝对路径、空/`.`/`..` 组件、反斜杠与
symlink。三个 hash 现场稳定复算。fixture 必须是严格 JSON，并把 fixture/run 的 ID、type、
request、source IDs、holdout 与 canonical `case_definition_hash` 逐项绑定；两份输出也必须是严格
JSON，绑定 eval run、case 与 baseline/with_skill condition。run 的 rubric ID、阈值、最大分、
逐维评分和 fatal failures 必须与物化定义完全一致，pass 不得包含 fatal failure。
planned/blocked run 不伪装执行，因此不触发这些 completed-only artifact 检查。

Gate 4 accepted decision 引用的 runs 必须全部属于同一个 completed materialization，且至少
覆盖 3 个不同 trigger、3 个不同 nontrigger 和 3 个不同 task case；task 中至少一个
`holdout: true`。planned、blocked、invalidated、not-run、legacy-quarantined materialization
或只有测试定义的记录都不能支持 Gate 4。

### Rubric 跨 case 类型适用性契约

rubric 必须能对 trigger、nontrigger 与 task 三类 case 诚实评分。候选在 Gate 3 审阅时
必须显式选择并声明以下两种路径之一（写入 task contract 或 eval definition）：

1. **统一 rubric（全适用）**：同一套 rubric 维度对三类 case 都语义适用；即每个维度都能
   对"路由型"case（是否触发、是否边界内）与"任务型"case（是否完成分析）给出有区分度的
   评分。选择此路径时，Gate 4 的 trigger/nontrigger run 使用与 task run 相同的
   `rubric_id`、`max_score` 与 `pass_threshold`。
2. **分级 rubric（推荐用于 method-transfer）**：trigger/nontrigger 使用独立轻量 rubric
   （如 pass/fail + 范围匹配、边界诚实、安全停止等少量维度），task 使用完整分析 rubric。
   两条 rubric 都必须有唯一 `rubric_id` 并在 eval definition 中定义；run 记录的
   `rubric_id` 必须与对应 case 类型匹配。

禁止"用任务型完整 rubric 硬评路由型 case 并凑出通过分"的做法。若同一 rubric 对某类
case 的维度不适用（例如任务型"因果与 rescue 克制"无法对仅有触发请求的 case 评分），
必须使用分级 rubric，而不是给不适用维度强行打高分。

fixture 缺失、不可重放、权利不足、holdout 污染、泄漏边界不清或未授权的数据目的地是
对应 run 的 blocker。3/3/3/holdout 覆盖不足、拟引用 run 非 completed/pass、无人工评分
或混用 materialization 是 Gate 4 acceptance blocker。一个局部 blocked run 不应使其他
独立 completed run 失去历史。DOCX run 使用 DOCX locator resolver；兼容非 DOCX bundle
必须使用其已验证 locator checker，不得把 DOCX resolver 结果冒充通用验证。

## Correction overlay

每条 correction 保留 evidence 或 source locator、原始值、候选/结果值、依据、状态、
`applies_to_claim_ids`、`resolved_quality_flags` 与 `human_decision`。completed correction 必须
显式记录 `reviewer_type: user | human-delegate`；仅有 reviewer 字符串或 agent 署名不构成人工决定。claim 使用
`correction_ids` 时，ID 必须存在且 correction decision 为 accepted/revised。validator
还必须拒绝用清空 `quality_flags` 代替 overlay 决定的 accepted 支撑链。

## 运行

```bash
PYTHONDONTWRITEBYTECODE=1 python3 -B scripts/validate_distillation.py /path/to/distillation
PYTHONDONTWRITEBYTECODE=1 python3 -B scripts/validate_distillation.py /path/to/distillation \
  --sources-manifest /path/to/manifests/sources.yml
PYTHONDONTWRITEBYTECODE=1 python3 -B scripts/verify_normalized_locators.py \
  /path/to/distillation /path/to/sources/normalized \
  --sources-manifest /path/to/manifests/sources.yml
PYTHONDONTWRITEBYTECODE=1 python3 -B scripts/hash_candidate_tree.py \
  /path/to/distillation relative/path/to/candidate
PYTHONDONTWRITEBYTECODE=1 python3 -B scripts/inspect_distillation_state.py \
  /path/to/distillation --candidate-id candidate-001 \
  --candidate-path relative/path/to/candidate \
  --sources-manifest /path/to/manifests/sources.yml
PYTHONDONTWRITEBYTECODE=1 python3 -B -m unittest discover -s scripts -p 'test_*.py'
```

只有显式提供 `--sources-manifest` 时，脚本才读取该 manifest、检查 evidence
`source_id` 存在性，并解析上述本地 `markdown-section` locator；缺省时不扫描项目
来源，也不得声称这些 locator 已对当前政策文件解析成功。Gate 4 和 Gate 5 必须显式
提供 manifest，并另行人工检查 privacy、license、`allow_public_quotes` 和上传/发布
边界；source ID membership 与本地 locator 解析都不是版权审核。当 accepted rule 数为
0 时，`accepted_rule_traceability`
输出 `null`，且 `accepted_rule_traceability_applicable: false`，不将无适用对象伪报为
100%。

对 OOXML evidence，结构 validator 只检查 locator schema。来源解析检查器会稳定、no-follow
读取 `blocks.jsonl` 与 `checksums.yml`，复算每个 generated file 的完整 SHA-256、从
`normalized_text` 复算 block hash，并把 normalization 的 before/after source checksum 与显式
manifest checksum 绑定，再核对 locator、heading、excerpt 和 figure。没有使用 manifest 运行该
解析检查时，不得声称 locator 或原始来源绑定已通过。

OCR evidence 使用 `capture_mode: ocr` 与 `ocr-region` locator。结构 validator 检查 source、
carrier、image hash、run/record/region、bbox 和 page/figure identity；`verify_ocr_locators.py`
再对私有 OCR bundle、全图片/全页覆盖、generated-file hash 与 region 文本做现场复算。
这些检查不判断 OCR 是否识别正确，也不推断图中关系或公式语义。

脚本只检查结构、自洽性和追溯链。通过不表示主张为真、最新、外部共识或已通过人工知识审核。

#!/usr/bin/env python3
"""Audit a candidate Skill tree for private bibliographic identity disclosure.

The audit is deterministic and read-only.  It derives forbidden identity terms
from manifest records explicitly classified as a method source or book and/or an
optional UTF-8 line-oriented terms file; unrelated target-material and project-
policy identities are intentionally outside this check.  It never includes the
forbidden values in its report.  It
also detects a small, source-neutral set of attribution phrases.  Candidate
artifacts must be auditable UTF-8 text; binary or otherwise unauditable files
fail closed and must be removed or separately reviewed by an explicit dedicated
mechanism before Gate 3.  A successful result is only a disclosure check; it
does not establish copyright, privacy, or publication permission.
"""

from __future__ import annotations

import argparse
import hashlib
import json
import os
import re
import stat
import sys
import unicodedata
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Callable, Iterable, Mapping, Sequence

try:
    import yaml
except ImportError as exc:  # pragma: no cover - dependency is project-provided
    raise SystemExit(
        "PyYAML is required. Do not install it without the user's approval."
    ) from exc


TEXT_SUFFIXES = {
    ".cfg",
    ".conf",
    ".css",
    ".csv",
    ".html",
    ".ini",
    ".j2",
    ".jinja",
    ".jinja2",
    ".js",
    ".json",
    ".jsonl",
    ".md",
    ".properties",
    ".py",
    ".r",
    ".rmd",
    ".rst",
    ".sh",
    ".sql",
    ".svg",
    ".template",
    ".tex",
    ".toml",
    ".ts",
    ".tsv",
    ".txt",
    ".xml",
    ".yaml",
    ".yml",
}

GENERIC_IDENTITY_VALUES = {
    "author",
    "authors",
    "book",
    "document",
    "edition",
    "file",
    "isbn",
    "publisher",
    "sample",
    "series",
    "source",
    "task",
    "title",
    "unknown",
    "untitled",
}


class _UniqueKeySafeLoader(yaml.SafeLoader):
    """SafeLoader variant that rejects duplicate mapping keys."""


def _construct_unique_mapping(loader, node, deep=False):
    loader.flatten_mapping(node)
    result = {}
    for key_node, value_node in node.value:
        key = loader.construct_object(key_node, deep=deep)
        try:
            duplicate = key in result
        except TypeError as exc:
            raise yaml.constructor.ConstructorError(
                "while constructing a mapping",
                node.start_mark,
                "found an unhashable mapping key",
                key_node.start_mark,
            ) from exc
        if duplicate:
            raise yaml.constructor.ConstructorError(
                "while constructing a mapping",
                node.start_mark,
                "duplicate mapping key",
                key_node.start_mark,
            )
        result[key] = loader.construct_object(value_node, deep=deep)
    return result


_UniqueKeySafeLoader.add_constructor(
    yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
    _construct_unique_mapping,
)


class DisclosureInputError(RuntimeError):
    """Fail-closed input error whose public fields cannot disclose identities."""

    def __init__(self, code: str, input_ref: str, message: str):
        super().__init__(message)
        self.code = code
        self.input_ref = input_ref
        self.message = message


@dataclass(frozen=True)
class _FileRecord:
    relative_path: str
    absolute_path: Path
    stat_result: os.stat_result


@dataclass(frozen=True)
class _DirectoryRecord:
    relative_path: str
    absolute_path: Path
    stat_result: os.stat_result


@dataclass(frozen=True)
class _IdentityTerm:
    category: str
    normalized: str
    compact: str
    source_field_refs: tuple[str, ...]


def _same_entry(before: os.stat_result, after: os.stat_result) -> bool:
    return (
        before.st_dev == after.st_dev
        and before.st_ino == after.st_ino
        and stat.S_IFMT(before.st_mode) == stat.S_IFMT(after.st_mode)
        and before.st_size == after.st_size
        and before.st_mtime_ns == after.st_mtime_ns
    )


def _same_path_component(before: os.stat_result, after: os.stat_result) -> bool:
    # Ancestor directory size/mtime can change when an unrelated process creates
    # a sibling (especially under /tmp). Identity and file type are the relevant
    # no-follow invariants; final files and candidate-contained directories are
    # checked more strictly with _same_entry.
    return (
        before.st_dev == after.st_dev
        and before.st_ino == after.st_ino
        and stat.S_IFMT(before.st_mode) == stat.S_IFMT(after.st_mode)
    )


def _path_fingerprint(relative_path: str) -> str:
    digest = hashlib.sha256(relative_path.encode("utf-8")).hexdigest()
    return f"sha256:{digest}"


def _entry_input_ref(relative_path: str) -> str:
    return f"candidate-entry:{_path_fingerprint(relative_path)}"


def _inspect_path_components(path: Path, expected: str) -> list[tuple[Path, os.stat_result]]:
    absolute = Path(os.path.abspath(os.fspath(path)))
    current = Path(absolute.anchor)
    components: list[tuple[Path, os.stat_result]] = []
    for part in absolute.parts[1:]:
        current = current / part
        try:
            item_stat = current.lstat()
        except OSError as exc:
            raise DisclosureInputError(
                "INPUT_UNREADABLE", expected, "input path cannot be inspected"
            ) from exc
        if stat.S_ISLNK(item_stat.st_mode):
            raise DisclosureInputError(
                "INPUT_SYMLINK", expected, "symlink path components are forbidden"
            )
        components.append((current, item_stat))
    if not components:
        raise DisclosureInputError("INPUT_INVALID", expected, "input path is invalid")
    final_mode = components[-1][1].st_mode
    expected_mode = stat.S_ISDIR if expected == "candidate" else stat.S_ISREG
    if not expected_mode(final_mode):
        raise DisclosureInputError(
            "INPUT_TYPE", expected, "input has the wrong filesystem type"
        )
    return components


def _check_read_permission(mode: int, input_ref: str, *, directory: bool = False) -> None:
    lacks_read_bit = mode & 0o444 == 0
    lacks_search_bit = directory and mode & 0o111 == 0
    if lacks_read_bit or lacks_search_bit:
        raise DisclosureInputError(
            "INPUT_UNREADABLE", input_ref, "input has no readable permission bits"
        )


def _read_stable_input_file(path: Path, input_ref: str) -> bytes:
    components = _inspect_path_components(path, input_ref)
    original = components[-1][1]
    _check_read_permission(original.st_mode, input_ref)
    absolute = components[-1][0]
    flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0)
    try:
        descriptor = os.open(absolute, flags)
    except OSError as exc:
        raise DisclosureInputError(
            "INPUT_UNREADABLE", input_ref, "input file cannot be opened safely"
        ) from exc
    try:
        opened = os.fstat(descriptor)
        if not stat.S_ISREG(opened.st_mode) or not _same_entry(original, opened):
            raise DisclosureInputError(
                "INPUT_CHANGED", input_ref, "input changed before reading"
            )
        chunks: list[bytes] = []
        while True:
            chunk = os.read(descriptor, 1024 * 1024)
            if not chunk:
                break
            chunks.append(chunk)
        final = os.fstat(descriptor)
    except OSError as exc:
        raise DisclosureInputError(
            "INPUT_UNREADABLE", input_ref, "input file cannot be read safely"
        ) from exc
    finally:
        os.close(descriptor)
    content = b"".join(chunks)
    if not _same_entry(opened, final) or len(content) != final.st_size:
        raise DisclosureInputError(
            "INPUT_CHANGED", input_ref, "input changed while reading"
        )
    for component, original_stat in components:
        try:
            current_stat = component.lstat()
        except OSError as exc:
            raise DisclosureInputError(
                "INPUT_CHANGED", input_ref, "input path changed after reading"
            ) from exc
        if stat.S_ISLNK(current_stat.st_mode) or not _same_path_component(
            original_stat, current_stat
        ):
            raise DisclosureInputError(
                "INPUT_CHANGED", input_ref, "input path changed while reading"
            )
    return content


def _load_private_manifest(path: Path) -> tuple[Mapping[str, Any], str]:
    try:
        content = _read_stable_input_file(path, "sources-manifest")
        text = content.decode("utf-8")
        document = yaml.load(text, Loader=_UniqueKeySafeLoader)
    except DisclosureInputError:
        raise
    except (UnicodeError, yaml.YAMLError, RecursionError) as exc:
        # Do not include parser text: it can contain a private scalar.
        raise DisclosureInputError(
            "MANIFEST_INVALID", "sources-manifest", "private manifest is invalid"
        ) from exc
    if not isinstance(document, dict) or not isinstance(document.get("sources"), list):
        raise DisclosureInputError(
            "MANIFEST_SCHEMA",
            "sources-manifest",
            "private manifest must contain a sources list",
        )
    for record in document["sources"]:
        if not isinstance(record, dict):
            raise DisclosureInputError(
                "MANIFEST_SCHEMA",
                "sources-manifest",
                "each private source record must be a mapping",
            )
    return document, hashlib.sha256(content).hexdigest()


def _normalize(value: str) -> str:
    normalized = unicodedata.normalize("NFKC", value).casefold()
    return " ".join(normalized.split())


def _compact(value: str) -> str:
    return "".join(character for character in value if character.isalnum())


def _contains_non_ascii_alnum(value: str) -> bool:
    return any(character.isalnum() and not character.isascii() for character in value)


def _classify_manifest_field(key: Any) -> str | None:
    if not isinstance(key, str):
        return None
    normalized = re.sub(r"[^a-z0-9]+", "_", key.casefold()).strip("_")
    tokens = set(normalized.split("_"))
    if normalized in {"author", "authors", "creator", "creators"} or "author" in tokens:
        return "author"
    if normalized in {"publisher", "publisher_name", "publishing_house", "imprint"}:
        return "publisher"
    if normalized == "isbn" or normalized.startswith("isbn_"):
        return "isbn"
    if normalized == "edition" or "edition" in tokens:
        return "edition"
    if normalized in {
        "collection",
        "collection_name",
        "collection_title",
        "series",
    } or "series" in tokens or "collection" in tokens:
        return "series"
    if normalized == "title" or normalized.endswith("_title"):
        return "title"
    return None


def _is_path_field(key: Any) -> bool:
    if not isinstance(key, str):
        return False
    normalized = re.sub(r"[^a-z0-9]+", "_", key.casefold()).strip("_")
    return normalized in {
        "file",
        "filename",
        "local_file",
        "local_path",
        "path",
        "related_local_paths",
        "source_file",
        "source_path",
    }


def _safe_field_segment(key: Any) -> str:
    category = _classify_manifest_field(key)
    if category is not None:
        return category
    if _is_path_field(key):
        return "path"
    # Arbitrary manifest keys can themselves contain a forbidden title/name.
    # Never reproduce them; a short deterministic fingerprint still preserves
    # the ability to distinguish nested source fields during review.
    encoded = repr(key).encode("utf-8", errors="backslashreplace")
    return f"field-{hashlib.sha256(encoded).hexdigest()[:12]}"


def _flatten_identity_scalars(value: Any) -> list[str]:
    values: list[str] = []
    if isinstance(value, str):
        values.append(value)
    elif isinstance(value, int) and not isinstance(value, bool):
        values.append(str(value))
    elif isinstance(value, list):
        for item in value:
            values.extend(_flatten_identity_scalars(item))
    elif isinstance(value, dict):
        for item in value.values():
            values.extend(_flatten_identity_scalars(item))
    return values


def _canonical_isbn(value: str) -> str:
    candidate = "".join(
        character for character in unicodedata.normalize("NFKC", value).casefold()
        if character.isdigit() or character == "x"
    )
    return candidate if len(candidate) in {10, 13} else ""


def _term_parts(value: str, category: str) -> tuple[str, str] | None:
    normalized = _normalize(value)
    compact = _compact(normalized)
    if not normalized or normalized in GENERIC_IDENTITY_VALUES:
        return None
    if category == "isbn":
        isbn = _canonical_isbn(value)
        return (normalized, isbn) if isbn else None
    # Two- and three-character CJK/non-ASCII names are common and identifying;
    # short ASCII tokens are much more collision-prone and remain stricter.
    minimum_length = 2 if _contains_non_ascii_alnum(compact) else 4
    if len(compact) < minimum_length:
        return None
    return normalized, compact


def _path_basename(value: str) -> tuple[str, str] | None:
    cleaned = value.strip().split("?", 1)[0].split("#", 1)[0].rstrip("/\\")
    if not cleaned:
        return None
    basename = re.split(r"[/\\]", cleaned)[-1]
    if not basename:
        return None
    stem = Path(basename).stem
    return basename, stem


def _is_method_identity_source(record: Mapping[str, Any]) -> bool:
    provenance_role = record.get("provenance_role")
    source_role = record.get("source_role")
    source_type = record.get("type")
    if isinstance(provenance_role, str) and provenance_role.casefold() == "method-source":
        return True
    if isinstance(source_role, str) and source_role.casefold() in {
        "primary-book",
        "method-source",
        "supplementary-book",
    }:
        return True
    if isinstance(source_type, str):
        normalized_type = source_type.casefold()
        return normalized_type == "book" or normalized_type.startswith("book-")
    return False


def _collect_manifest_terms(document: Mapping[str, Any]) -> list[_IdentityTerm]:
    collected: dict[tuple[str, str, str], set[str]] = {}
    active_containers: set[int] = set()

    def add(
        value: str,
        category: str,
        source_ref: str,
        opaque_source_id: tuple[str, str] | None,
    ) -> None:
        parts = _term_parts(value, category)
        if parts is None:
            return
        normalized, compact = parts
        if category in {"path-basename", "path-stem"} and opaque_source_id is not None:
            source_normalized, source_compact = opaque_source_id
            if normalized == source_normalized or compact == source_compact:
                return
        collected.setdefault((category, normalized, compact), set()).add(source_ref)

    def walk(
        value: Any,
        reference: str,
        opaque_source_id: tuple[str, str] | None,
    ) -> None:
        if isinstance(value, dict):
            identity = id(value)
            if identity in active_containers:
                raise DisclosureInputError(
                    "MANIFEST_RECURSIVE",
                    "sources-manifest",
                    "recursive manifest containers are forbidden",
                )
            active_containers.add(identity)
            try:
                for key, item in value.items():
                    segment = _safe_field_segment(key)
                    field_ref = f"{reference}.{segment}"
                    category = _classify_manifest_field(key)
                    if category is not None:
                        try:
                            scalars = _flatten_identity_scalars(item)
                        except RecursionError as exc:
                            raise DisclosureInputError(
                                "MANIFEST_RECURSIVE",
                                "sources-manifest",
                                "recursive manifest containers are forbidden",
                            ) from exc
                        for scalar_index, scalar in enumerate(scalars):
                            suffix = "" if len(scalars) == 1 else f"[{scalar_index}]"
                            add(
                                scalar,
                                category,
                                f"{field_ref}{suffix}",
                                opaque_source_id,
                            )
                    elif _is_path_field(key):
                        try:
                            paths = _flatten_identity_scalars(item)
                        except RecursionError as exc:
                            raise DisclosureInputError(
                                "MANIFEST_RECURSIVE",
                                "sources-manifest",
                                "recursive manifest containers are forbidden",
                            ) from exc
                        for path_index, path_value in enumerate(paths):
                            parts = _path_basename(path_value)
                            if parts is None:
                                continue
                            index_suffix = "" if len(paths) == 1 else f"[{path_index}]"
                            basename, stem = parts
                            add(
                                basename,
                                "path-basename",
                                f"{field_ref}{index_suffix}#basename",
                                opaque_source_id,
                            )
                            if stem != basename:
                                add(
                                    stem,
                                    "path-stem",
                                    f"{field_ref}{index_suffix}#stem",
                                    opaque_source_id,
                                )
                    else:
                        walk(item, field_ref, opaque_source_id)
            finally:
                active_containers.remove(identity)
        elif isinstance(value, list):
            identity = id(value)
            if identity in active_containers:
                raise DisclosureInputError(
                    "MANIFEST_RECURSIVE",
                    "sources-manifest",
                    "recursive manifest containers are forbidden",
                )
            active_containers.add(identity)
            try:
                for index, item in enumerate(value):
                    walk(item, f"{reference}[{index}]", opaque_source_id)
            finally:
                active_containers.remove(identity)

    for source_index, record in enumerate(document["sources"]):
        # The candidate must hide the originating method/book identity, not
        # unrelated project-policy paths or target-material paper identities.
        if not _is_method_identity_source(record):
            continue
        source_id = record.get("id")
        opaque_source_id = None
        if isinstance(source_id, str) and source_id.strip():
            source_normalized = _normalize(source_id)
            source_compact = _compact(source_normalized)
            opaque_source_id = (source_normalized, source_compact)
        walk(record, f"sources[{source_index}]", opaque_source_id)

    return [
        _IdentityTerm(category, normalized, compact, tuple(sorted(references)))
        for (category, normalized, compact), references in sorted(
            collected.items(), key=lambda item: item[0]
        )
    ]


def _load_extra_terms(path: Path) -> tuple[list[_IdentityTerm], str]:
    try:
        content = _read_stable_input_file(path, "extra-terms-file")
        text = content.decode("utf-8")
    except DisclosureInputError:
        raise
    except UnicodeError as exc:
        raise DisclosureInputError(
            "EXTRA_TERMS_INVALID",
            "extra-terms-file",
            "extra terms file must be UTF-8 text",
        ) from exc
    terms: list[_IdentityTerm] = []
    for line_number, line in enumerate(text.splitlines(), start=1):
        value = line.strip()
        if not value or value.startswith("#"):
            continue
        parts = _term_parts(value, "extra-term")
        if parts is None:
            continue
        normalized, compact = parts
        terms.append(
            _IdentityTerm(
                "extra-term",
                normalized,
                compact,
                (f"extra_terms[line:{line_number}]",),
            )
        )
    return terms, hashlib.sha256(content).hexdigest()


def _merge_terms(terms: Iterable[_IdentityTerm]) -> list[_IdentityTerm]:
    merged: dict[tuple[str, str, str], set[str]] = {}
    for term in terms:
        merged.setdefault((term.category, term.normalized, term.compact), set()).update(
            term.source_field_refs
        )
    return [
        _IdentityTerm(category, normalized, compact, tuple(sorted(references)))
        for (category, normalized, compact), references in sorted(
            merged.items(), key=lambda item: item[0]
        )
    ]


def _scan_candidate_tree(
    candidate_dir: Path,
) -> tuple[list[_FileRecord], list[_DirectoryRecord], list[tuple[Path, os.stat_result]]]:
    root_components = _inspect_path_components(candidate_dir, "candidate")
    root = root_components[-1][0]
    files: list[_FileRecord] = []
    directories: list[_DirectoryRecord] = []

    def scan(directory: Path, relative_directory: str) -> None:
        try:
            directory_stat = directory.lstat()
        except OSError as exc:
            raise DisclosureInputError(
                "CANDIDATE_UNREADABLE",
                _entry_input_ref(relative_directory or "."),
                "candidate directory cannot be inspected",
            ) from exc
        if stat.S_ISLNK(directory_stat.st_mode):
            raise DisclosureInputError(
                "CANDIDATE_SYMLINK",
                _entry_input_ref(relative_directory or "."),
                "candidate symlinks are forbidden",
            )
        if not stat.S_ISDIR(directory_stat.st_mode):
            raise DisclosureInputError(
                "CANDIDATE_TYPE",
                _entry_input_ref(relative_directory or "."),
                "candidate containers must be directories",
            )
        _check_read_permission(
            directory_stat.st_mode,
            _entry_input_ref(relative_directory or "."),
            directory=True,
        )
        directories.append(
            _DirectoryRecord(relative_directory or ".", directory, directory_stat)
        )
        try:
            entries = sorted(os.scandir(directory), key=lambda entry: os.fsencode(entry.name))
        except OSError as exc:
            raise DisclosureInputError(
                "CANDIDATE_UNREADABLE",
                _entry_input_ref(relative_directory or "."),
                "candidate directory cannot be enumerated",
            ) from exc
        for entry in entries:
            relative = entry.name if not relative_directory else f"{relative_directory}/{entry.name}"
            try:
                relative.encode("utf-8", errors="strict")
                item_stat = os.lstat(entry.path)
            except (UnicodeEncodeError, OSError) as exc:
                raise DisclosureInputError(
                    "CANDIDATE_UNREADABLE",
                    _entry_input_ref(relative.encode("utf-8", "backslashreplace").decode("utf-8")),
                    "candidate entry cannot be inspected",
                ) from exc
            mode = item_stat.st_mode
            if stat.S_ISLNK(mode):
                raise DisclosureInputError(
                    "CANDIDATE_SYMLINK",
                    _entry_input_ref(relative),
                    "candidate symlinks are forbidden",
                )
            if stat.S_ISDIR(mode):
                if entry.name.casefold() == "__pycache__":
                    raise DisclosureInputError(
                        "CANDIDATE_CACHE_ARTIFACT",
                        _entry_input_ref(relative),
                        "Python cache artifacts are forbidden",
                    )
                scan(Path(entry.path), relative)
            elif stat.S_ISREG(mode):
                if entry.name.casefold().endswith(".pyc"):
                    raise DisclosureInputError(
                        "CANDIDATE_CACHE_ARTIFACT",
                        _entry_input_ref(relative),
                        "Python cache artifacts are forbidden",
                    )
                _check_read_permission(item_stat.st_mode, _entry_input_ref(relative))
                files.append(_FileRecord(relative, Path(entry.path), item_stat))
            else:
                raise DisclosureInputError(
                    "CANDIDATE_NON_REGULAR",
                    _entry_input_ref(relative),
                    "candidate entries must be ordinary files or directories",
                )

    scan(root, "")
    files.sort(key=lambda item: item.relative_path.encode("utf-8"))
    return files, directories, root_components


def _read_candidate_file(record: _FileRecord) -> bytes:
    input_ref = _entry_input_ref(record.relative_path)
    flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0)
    try:
        descriptor = os.open(record.absolute_path, flags)
    except OSError as exc:
        raise DisclosureInputError(
            "CANDIDATE_UNREADABLE", input_ref, "candidate file cannot be opened"
        ) from exc
    try:
        opened = os.fstat(descriptor)
        if not stat.S_ISREG(opened.st_mode) or not _same_entry(record.stat_result, opened):
            raise DisclosureInputError(
                "CANDIDATE_CHANGED", input_ref, "candidate file changed before reading"
            )
        chunks: list[bytes] = []
        while True:
            chunk = os.read(descriptor, 1024 * 1024)
            if not chunk:
                break
            chunks.append(chunk)
        final = os.fstat(descriptor)
    except OSError as exc:
        raise DisclosureInputError(
            "CANDIDATE_UNREADABLE", input_ref, "candidate file cannot be read"
        ) from exc
    finally:
        os.close(descriptor)
    content = b"".join(chunks)
    if not _same_entry(opened, final) or len(content) != final.st_size:
        raise DisclosureInputError(
            "CANDIDATE_CHANGED", input_ref, "candidate file changed while reading"
        )
    try:
        final_path_stat = record.absolute_path.lstat()
    except OSError as exc:
        raise DisclosureInputError(
            "CANDIDATE_CHANGED", input_ref, "candidate path changed after reading"
        ) from exc
    if stat.S_ISLNK(final_path_stat.st_mode) or not _same_entry(final, final_path_stat):
        raise DisclosureInputError(
            "CANDIDATE_CHANGED", input_ref, "candidate path changed while reading"
        )
    return content


def _recheck_candidate_tree(
    files: Sequence[_FileRecord],
    directories: Sequence[_DirectoryRecord],
    root_components: Sequence[tuple[Path, os.stat_result]],
) -> None:
    for record in files:
        try:
            current = record.absolute_path.lstat()
        except OSError as exc:
            raise DisclosureInputError(
                "CANDIDATE_CHANGED",
                _entry_input_ref(record.relative_path),
                "candidate file changed after scanning",
            ) from exc
        if stat.S_ISLNK(current.st_mode) or not _same_entry(record.stat_result, current):
            raise DisclosureInputError(
                "CANDIDATE_CHANGED",
                _entry_input_ref(record.relative_path),
                "candidate file changed after scanning",
            )
    for record in directories:
        try:
            current = record.absolute_path.lstat()
        except OSError as exc:
            raise DisclosureInputError(
                "CANDIDATE_CHANGED",
                _entry_input_ref(record.relative_path),
                "candidate directory changed after scanning",
            ) from exc
        if stat.S_ISLNK(current.st_mode) or not _same_entry(record.stat_result, current):
            raise DisclosureInputError(
                "CANDIDATE_CHANGED",
                _entry_input_ref(record.relative_path),
                "candidate directory changed after scanning",
            )
    for component, original in root_components:
        try:
            current = component.lstat()
        except OSError as exc:
            raise DisclosureInputError(
                "CANDIDATE_CHANGED", "candidate", "candidate path changed after scanning"
            ) from exc
        if stat.S_ISLNK(current.st_mode) or not _same_path_component(original, current):
            raise DisclosureInputError(
                "CANDIDATE_CHANGED", "candidate", "candidate path changed while scanning"
            )


def _bounded_normalized_match(haystack: str, needle: str) -> bool:
    start = 0
    while True:
        index = haystack.find(needle, start)
        if index < 0:
            return False
        end = index + len(needle)
        left_ok = index == 0 or not (needle[0].isalnum() and haystack[index - 1].isalnum())
        right_ok = end == len(haystack) or not (
            needle[-1].isalnum() and haystack[end].isalnum()
        )
        if left_ok and right_ok:
            return True
        start = index + 1


def _compact_match(haystack: str, term: _IdentityTerm) -> bool:
    if not term.compact:
        return False
    compact_characters: list[str] = []
    original_positions: list[int] = []
    for position, character in enumerate(haystack):
        if character.isalnum():
            compact_characters.append(character)
            original_positions.append(position)
    compact_haystack = "".join(compact_characters)
    start = 0
    while True:
        index = compact_haystack.find(term.compact, start)
        if index < 0:
            return False
        end = index + len(term.compact)
        if term.category == "isbn":
            left_ok = index == 0 or not compact_haystack[index - 1].isdigit()
            right_ok = end == len(compact_haystack) or not compact_haystack[end].isdigit()
        elif _contains_non_ascii_alnum(term.compact):
            # CJK and other non-ASCII names commonly occur without word separators.
            return True
        else:
            original_start = original_positions[index]
            original_end = original_positions[end - 1] + 1
            left_ok = original_start == 0 or not haystack[original_start - 1].isalnum()
            right_ok = original_end == len(haystack) or not haystack[original_end].isalnum()
        if left_ok and right_ok:
            return True
        start = index + 1


def _match_term(value: str, term: _IdentityTerm) -> str | None:
    normalized = _normalize(value)
    if _bounded_normalized_match(normalized, term.normalized):
        return "normalized-exact"
    # Compact matching is intentionally limited to sufficiently identifying
    # values, except ISBNs whose canonical length is already constrained.
    compact_allowed = (
        term.category == "isbn"
        or (_contains_non_ascii_alnum(term.compact) and len(term.compact) >= 2)
        or len(term.compact) >= 6
    )
    if compact_allowed and _compact_match(normalized, term):
        return "compact"
    return None


ATTRIBUTION_PATTERNS: tuple[tuple[str, re.Pattern[str]], ...] = (
    (
        "en-according-to-source",
        re.compile(
            r"\baccording\s+to\s+(?:the\s+)?(?:author|book|original\s+author|source(?:\s+(?:text|material))?)\b"
        ),
    ),
    (
        "en-attribution-verb",
        re.compile(
            r"\b(?:the|this|original)\s+(?:author|book|source(?:\s+(?:text|material))?)\s+"
            r"(?:argues?|claims?|states?|says?|notes?|writes?|observes?|proposes?|suggests?|contends?|explains?)\b"
        ),
    ),
    (
        "en-book-author-attribution",
        re.compile(
            r"\b(?:the|this)\s+book(?:'s|’s)\s+author\s+"
            r"(?:argues?|claims?|states?|says?|notes?|writes?|observes?|proposes?|suggests?|contends?|explains?)\b"
        ),
    ),
    (
        "zh-attribution-verb",
        re.compile(
            r"(?:本书|该书|原书|书中|原作者|"
            r"(?<!论文)(?<!文章)(?<!研究)(?<!目标)(?<!材料)作者)\s*"
            r"(?:认为|指出|主张|提出|写道|强调|声称|论证|表明)"
        ),
    ),
    (
        "zh-according-to-source",
        re.compile(r"(?:根据|依据|按照)\s*(?:本书|该书|原书|原作者|作者)"),
    ),
)


def _generic_attribution_matches(value: str) -> list[str]:
    normalized = _normalize(value)
    return [rule_id for rule_id, pattern in ATTRIBUTION_PATTERNS if pattern.search(normalized)]


def _first_multiline_match_line(
    lines: Sequence[str], matcher: Callable[[str], bool]
) -> int:
    # Prefer the shortest matching window so unrelated earlier lines do not
    # become the reported start. The whole-file check remains the fail-closed
    # fallback for longer artificial splits.
    maximum_span = min(8, len(lines))
    for span in range(2, maximum_span + 1):
        for start in range(0, len(lines) - span + 1):
            if matcher("\n".join(lines[start : start + span])):
                return start + 1
    return 1


def _path_contains_identity(relative_path: str, terms: Sequence[_IdentityTerm]) -> bool:
    return any(_match_term(relative_path, term) is not None for term in terms)


def _public_candidate_path(relative_path: str, terms: Sequence[_IdentityTerm]) -> dict[str, str]:
    if _path_contains_identity(relative_path, terms):
        return {
            "path": "<redacted>",
            "path_sha256": _path_fingerprint(relative_path),
        }
    return {"path": relative_path}


def _finding_key(finding: Mapping[str, Any]) -> str:
    return json.dumps(finding, ensure_ascii=False, sort_keys=True, separators=(",", ":"))


def audit_candidate_disclosure(
    candidate_dir: Path | str,
    sources_manifest: Path | str | None = None,
    extra_terms_file: Path | str | None = None,
) -> dict[str, Any]:
    """Return a deterministic disclosure report without exposing identity values."""
    terms: list[_IdentityTerm] = []
    manifest_snapshot: tuple[Path, str, str] | None = None
    extra_terms_snapshot: tuple[Path, str, str] | None = None
    if sources_manifest is not None:
        manifest_path = Path(sources_manifest)
        document, manifest_digest = _load_private_manifest(manifest_path)
        manifest_snapshot = (manifest_path, "sources-manifest", manifest_digest)
        terms.extend(_collect_manifest_terms(document))
    if extra_terms_file is not None:
        extra_path = Path(extra_terms_file)
        extra_terms, extra_digest = _load_extra_terms(extra_path)
        extra_terms_snapshot = (extra_path, "extra-terms-file", extra_digest)
        terms.extend(extra_terms)
    terms = _merge_terms(terms)

    files, directories, root_components = _scan_candidate_tree(Path(candidate_dir))
    findings: list[dict[str, Any]] = []
    text_files_scanned = 0

    for record in files:
        public_path = _public_candidate_path(record.relative_path, terms)
        for term in terms:
            mode = _match_term(record.relative_path, term)
            if mode is not None:
                findings.append(
                    {
                        "candidate_location": {"kind": "path", **public_path},
                        "code": "IDENTITY_DISCLOSURE",
                        "identity_category": term.category,
                        "match_mode": mode,
                        "source_field_refs": list(term.source_field_refs),
                    }
                )
        for rule_id in _generic_attribution_matches(record.relative_path):
            findings.append(
                {
                    "attribution_rule": rule_id,
                    "candidate_location": {"kind": "path", **public_path},
                    "code": "GENERIC_SOURCE_ATTRIBUTION",
                    "source_field_refs": [],
                }
            )

        content = _read_candidate_file(record)
        suffix = Path(record.relative_path).suffix.casefold()
        if suffix and suffix not in TEXT_SUFFIXES:
            raise DisclosureInputError(
                "CANDIDATE_BINARY_UNAUDITED",
                _entry_input_ref(record.relative_path),
                "candidate artifact is not an approved UTF-8 text type",
            )
        try:
            text = content.decode("utf-8")
        except UnicodeError as exc:
            raise DisclosureInputError(
                "CANDIDATE_BINARY_UNAUDITED",
                _entry_input_ref(record.relative_path),
                "candidate artifact cannot be audited as UTF-8 text",
            ) from exc
        if "\x00" in text:
            raise DisclosureInputError(
                "CANDIDATE_BINARY_UNAUDITED",
                _entry_input_ref(record.relative_path),
                "candidate artifact cannot be audited as UTF-8 text",
            )
        text_files_scanned += 1
        lines = text.splitlines() or [""]
        term_indexes_seen_on_one_line: set[int] = set()
        attribution_rules_seen_on_one_line: set[str] = set()
        for line_number, line in enumerate(lines, start=1):
            for term_index, term in enumerate(terms):
                mode = _match_term(line, term)
                if mode is not None:
                    term_indexes_seen_on_one_line.add(term_index)
                    findings.append(
                        {
                            "candidate_location": {
                                "kind": "content",
                                **public_path,
                                "line": line_number,
                            },
                            "code": "IDENTITY_DISCLOSURE",
                            "identity_category": term.category,
                            "match_mode": mode,
                            "source_field_refs": list(term.source_field_refs),
                        }
                    )
            for rule_id in _generic_attribution_matches(line):
                attribution_rules_seen_on_one_line.add(rule_id)
                findings.append(
                    {
                        "attribution_rule": rule_id,
                        "candidate_location": {
                            "kind": "content",
                            **public_path,
                            "line": line_number,
                        },
                        "code": "GENERIC_SOURCE_ATTRIBUTION",
                        "source_field_refs": [],
                    }
                )

        # Line-oriented scanning gives precise locations, while the whole-file
        # pass prevents identities or attribution phrases from being hidden by
        # Markdown/YAML line wrapping.
        for term_index, term in enumerate(terms):
            if term_index in term_indexes_seen_on_one_line:
                continue
            mode = _match_term(text, term)
            if mode is None:
                continue
            start_line = _first_multiline_match_line(
                lines, lambda value, current=term: _match_term(value, current) is not None
            )
            findings.append(
                {
                    "candidate_location": {
                        "kind": "content",
                        **public_path,
                        "line": start_line,
                    },
                    "code": "IDENTITY_DISCLOSURE",
                    "identity_category": term.category,
                    "match_mode": mode,
                    "match_scope": "multiline-or-whole-file",
                    "source_field_refs": list(term.source_field_refs),
                }
            )
        for rule_id in _generic_attribution_matches(text):
            if rule_id in attribution_rules_seen_on_one_line:
                continue
            start_line = _first_multiline_match_line(
                lines,
                lambda value, current=rule_id: current
                in _generic_attribution_matches(value),
            )
            findings.append(
                {
                    "attribution_rule": rule_id,
                    "candidate_location": {
                        "kind": "content",
                        **public_path,
                        "line": start_line,
                    },
                    "code": "GENERIC_SOURCE_ATTRIBUTION",
                    "match_scope": "multiline-or-whole-file",
                    "source_field_refs": [],
                }
            )

    _recheck_candidate_tree(files, directories, root_components)
    for snapshot in (manifest_snapshot, extra_terms_snapshot):
        if snapshot is None:
            continue
        input_path, input_ref, initial_digest = snapshot
        final_digest = hashlib.sha256(
            _read_stable_input_file(input_path, input_ref)
        ).hexdigest()
        if final_digest != initial_digest:
            raise DisclosureInputError(
                "INPUT_CHANGED", input_ref, "private audit input changed while scanning"
            )
    unique_findings = {
        _finding_key(finding): finding for finding in findings
    }
    ordered_findings = [unique_findings[key] for key in sorted(unique_findings)]
    return {
        "candidate_files_scanned": len(files),
        "checker_scope": "candidate-bibliographic-identity-disclosure",
        "extra_terms_file_supplied": extra_terms_file is not None,
        "finding_count": len(ordered_findings),
        "findings": ordered_findings,
        "identity_term_count": len(terms),
        "ok": not ordered_findings,
        "sources_manifest_supplied": sources_manifest is not None,
        "status": "pass" if not ordered_findings else "findings",
        "text_files_scanned": text_files_scanned,
    }


def main(argv: Sequence[str] | None = None) -> int:
    parser = argparse.ArgumentParser(
        description=(
            "Audit a candidate Skill for private bibliographic identity disclosure; "
            "binary or non-UTF-8 candidate artifacts fail closed."
        )
    )
    parser.add_argument("candidate_dir", help="Candidate Skill directory to scan read-only")
    parser.add_argument(
        "--sources-manifest",
        help="Private YAML sources manifest used only to derive forbidden identities",
    )
    parser.add_argument(
        "--extra-terms-file",
        help="Optional UTF-8 file containing one additional forbidden term per line",
    )
    args = parser.parse_args(argv)
    try:
        report = audit_candidate_disclosure(
            args.candidate_dir,
            sources_manifest=args.sources_manifest,
            extra_terms_file=args.extra_terms_file,
        )
    except DisclosureInputError as exc:
        report = {
            "checker_scope": "candidate-bibliographic-identity-disclosure",
            "input_error": {
                "code": exc.code,
                "input_ref": exc.input_ref,
                "message": exc.message,
            },
            "ok": False,
            "status": "input-error",
        }
        print(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True))
        return 2
    print(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True))
    return 0 if report["ok"] else 1


if __name__ == "__main__":
    sys.exit(main())

#!/usr/bin/env python3
"""Compute the deterministic SHA-256 identity of one materialized Skill tree.

This module has no third-party dependencies.  It is intentionally strict:
every ordinary file is hashed, while symlinks, non-regular filesystem entries,
Python bytecode caches, unsafe relative paths, and trees that change during the
read are rejected rather than ignored.
"""

from __future__ import annotations

import argparse
import hashlib
import os
import stat
import sys
from dataclasses import dataclass
from pathlib import Path, PurePosixPath
from typing import Sequence


TREE_HASH_PREFIX = b"distill-concept-books:candidate-tree:v1\0"
MAX_U64 = (1 << 64) - 1


class CandidateTreeError(RuntimeError):
    """Raised when a candidate tree cannot be hashed safely and completely."""

    def __init__(self, code: str, path: str, message: str):
        super().__init__(message)
        self.code = code
        self.path = path
        self.message = message

    def __str__(self) -> str:
        return f"{self.code} at {self.path}: {self.message}"


@dataclass(frozen=True)
class _FileRecord:
    relative_path: str
    path_bytes: bytes
    absolute_path: Path
    stat_result: os.stat_result


@dataclass(frozen=True)
class _DirectoryRecord:
    relative_path: str
    absolute_path: Path
    stat_result: os.stat_result


def canonical_candidate_path(value: str) -> str:
    """Return a canonical relative POSIX path or raise CandidateTreeError."""
    display = value if isinstance(value, str) else repr(value)
    if not isinstance(value, str) or not value or value != value.strip():
        raise CandidateTreeError(
            "CANDIDATE_PATH_INVALID", display,
            "candidate path must be a non-empty string without surrounding whitespace",
        )
    if "\\" in value or "\x00" in value:
        raise CandidateTreeError(
            "CANDIDATE_PATH_INVALID", value,
            "candidate path must use POSIX separators and contain no NUL byte",
        )
    raw_parts = value.split("/")
    if any(part in {"", ".", ".."} for part in raw_parts):
        raise CandidateTreeError(
            "CANDIDATE_PATH_INVALID", value,
            "candidate path must be canonical and must not contain empty, '.' or '..' components",
        )
    candidate = PurePosixPath(value)
    if candidate.is_absolute() or not candidate.parts:
        raise CandidateTreeError(
            "CANDIDATE_PATH_INVALID", value,
            "candidate path must be relative to the distillation directory",
        )
    canonical = candidate.as_posix()
    if canonical != value:
        raise CandidateTreeError(
            "CANDIDATE_PATH_INVALID", value,
            "candidate path must already be in canonical POSIX form",
        )
    try:
        canonical.encode("utf-8", errors="strict")
    except UnicodeEncodeError as exc:
        raise CandidateTreeError(
            "CANDIDATE_PATH_INVALID", value,
            "candidate path must be valid UTF-8",
        ) from exc
    return canonical


def _reject_cache_name(name: str, display_path: str, *, directory: bool) -> None:
    if (directory and name == "__pycache__") or (not directory and name.endswith(".pyc")):
        raise CandidateTreeError(
            "CANDIDATE_CACHE_ARTIFACT",
            display_path,
            "Python cache artifacts are forbidden in a materialized candidate tree",
        )


def _same_entry(before: os.stat_result, after: os.stat_result) -> bool:
    """Compare identity and mutation-sensitive metadata for a filesystem entry."""
    return (
        before.st_dev == after.st_dev
        and before.st_ino == after.st_ino
        and stat.S_IFMT(before.st_mode) == stat.S_IFMT(after.st_mode)
        and before.st_size == after.st_size
        and before.st_mtime_ns == after.st_mtime_ns
    )


def _scan_candidate_tree(
    candidate_root: Path,
) -> tuple[list[_FileRecord], list[_DirectoryRecord]]:
    files: list[_FileRecord] = []
    directories: list[_DirectoryRecord] = []

    def scan(directory: Path, relative_directory: PurePosixPath | None) -> None:
        display_directory = (
            "." if relative_directory is None else relative_directory.as_posix()
        )
        try:
            directory_stat = directory.lstat()
        except OSError as exc:
            raise CandidateTreeError(
                "CANDIDATE_TREE_READ_ERROR", display_directory,
                f"cannot inspect candidate directory: {exc}",
            ) from exc
        if stat.S_ISLNK(directory_stat.st_mode):
            raise CandidateTreeError(
                "CANDIDATE_TREE_SYMLINK", display_directory,
                "symlink directories are forbidden",
            )
        if not stat.S_ISDIR(directory_stat.st_mode):
            raise CandidateTreeError(
                "CANDIDATE_PATH_NOT_DIRECTORY", display_directory,
                "candidate root and all traversed containers must be directories",
            )
        directories.append(_DirectoryRecord(display_directory, directory, directory_stat))
        try:
            entries = list(os.scandir(directory))
        except OSError as exc:
            raise CandidateTreeError(
                "CANDIDATE_TREE_READ_ERROR", display_directory,
                f"cannot enumerate candidate directory: {exc}",
            ) from exc

        for entry in entries:
            relative = (
                PurePosixPath(entry.name)
                if relative_directory is None
                else relative_directory / entry.name
            )
            relative_string = relative.as_posix()
            try:
                path_bytes = relative_string.encode("utf-8", errors="strict")
            except UnicodeEncodeError as exc:
                raise CandidateTreeError(
                    "CANDIDATE_TREE_PATH_ENCODING", relative_string,
                    "tree entry path must be valid UTF-8",
                ) from exc
            try:
                entry_stat = os.lstat(entry.path)
            except OSError as exc:
                raise CandidateTreeError(
                    "CANDIDATE_TREE_READ_ERROR", relative_string,
                    f"cannot inspect candidate tree entry: {exc}",
                ) from exc
            mode = entry_stat.st_mode
            if stat.S_ISLNK(mode):
                raise CandidateTreeError(
                    "CANDIDATE_TREE_SYMLINK", relative_string,
                    "symlinks are forbidden in a materialized candidate tree",
                )
            if stat.S_ISDIR(mode):
                _reject_cache_name(entry.name, relative_string, directory=True)
                scan(Path(entry.path), relative)
                continue
            if stat.S_ISREG(mode):
                _reject_cache_name(entry.name, relative_string, directory=False)
                files.append(
                    _FileRecord(relative_string, path_bytes, Path(entry.path), entry_stat)
                )
                continue
            raise CandidateTreeError(
                "CANDIDATE_TREE_NON_REGULAR", relative_string,
                "only ordinary files and directories are allowed",
            )

    scan(candidate_root, None)
    files.sort(key=lambda item: item.path_bytes)
    return files, directories


def _read_stable_file(record: _FileRecord) -> bytes:
    flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0)
    try:
        descriptor = os.open(record.absolute_path, flags)
    except OSError as exc:
        raise CandidateTreeError(
            "CANDIDATE_TREE_READ_ERROR", record.relative_path,
            f"cannot open candidate file without following symlinks: {exc}",
        ) from exc
    try:
        opened_stat = os.fstat(descriptor)
        if not stat.S_ISREG(opened_stat.st_mode):
            raise CandidateTreeError(
                "CANDIDATE_TREE_NON_REGULAR", record.relative_path,
                "entry stopped being an ordinary file during hashing",
            )
        if not _same_entry(record.stat_result, opened_stat):
            raise CandidateTreeError(
                "CANDIDATE_TREE_CHANGED", record.relative_path,
                "file changed between tree traversal and hashing",
            )
        chunks: list[bytes] = []
        while True:
            chunk = os.read(descriptor, 1024 * 1024)
            if not chunk:
                break
            chunks.append(chunk)
        final_stat = os.fstat(descriptor)
    except OSError as exc:
        raise CandidateTreeError(
            "CANDIDATE_TREE_READ_ERROR", record.relative_path,
            f"cannot read candidate file: {exc}",
        ) from exc
    finally:
        os.close(descriptor)
    if not _same_entry(opened_stat, final_stat):
        raise CandidateTreeError(
            "CANDIDATE_TREE_CHANGED", record.relative_path,
            "file changed while it was being hashed",
        )
    content = b"".join(chunks)
    if len(content) != final_stat.st_size:
        raise CandidateTreeError(
            "CANDIDATE_TREE_CHANGED", record.relative_path,
            "bytes read do not match the stable file length",
        )
    try:
        current_path_stat = record.absolute_path.lstat()
    except OSError as exc:
        raise CandidateTreeError(
            "CANDIDATE_TREE_CHANGED", record.relative_path,
            f"file path changed after hashing: {exc}",
        ) from exc
    if not _same_entry(final_stat, current_path_stat):
        raise CandidateTreeError(
            "CANDIDATE_TREE_CHANGED", record.relative_path,
            "file path changed while the tree was being hashed",
        )
    return content


def _resolve_candidate_root(distillation_dir: Path | str, candidate_path: str) -> Path:
    canonical = canonical_candidate_path(candidate_path)
    try:
        root = Path(distillation_dir).resolve(strict=True)
    except OSError as exc:
        raise CandidateTreeError(
            "DISTILLATION_ROOT_INVALID", str(distillation_dir),
            f"distillation directory cannot be resolved: {exc}",
        ) from exc
    try:
        root_stat = root.stat()
    except OSError as exc:
        raise CandidateTreeError(
            "DISTILLATION_ROOT_INVALID", str(root),
            f"distillation directory cannot be inspected: {exc}",
        ) from exc
    if not stat.S_ISDIR(root_stat.st_mode):
        raise CandidateTreeError(
            "DISTILLATION_ROOT_INVALID", str(root),
            "distillation root must be a directory",
        )

    current = root
    for part in PurePosixPath(canonical).parts:
        current = current / part
        try:
            component_stat = current.lstat()
        except OSError as exc:
            raise CandidateTreeError(
                "CANDIDATE_PATH_MISSING", canonical,
                f"candidate path component {part!r} cannot be inspected: {exc}",
            ) from exc
        if stat.S_ISLNK(component_stat.st_mode):
            raise CandidateTreeError(
                "CANDIDATE_TREE_SYMLINK", canonical,
                "candidate path must not contain symlink components",
            )
        if current != root / Path(*PurePosixPath(canonical).parts) and not stat.S_ISDIR(
            component_stat.st_mode
        ):
            raise CandidateTreeError(
                "CANDIDATE_PATH_NOT_DIRECTORY", canonical,
                "intermediate candidate path components must be directories",
            )
    try:
        current.relative_to(root)
    except ValueError as exc:  # defensive: canonical path rules should make this impossible
        raise CandidateTreeError(
            "CANDIDATE_PATH_INVALID", canonical,
            "resolved candidate path escapes the distillation directory",
        ) from exc
    if not stat.S_ISDIR(current.lstat().st_mode):
        raise CandidateTreeError(
            "CANDIDATE_PATH_NOT_DIRECTORY", canonical,
            "candidate path must resolve to a directory",
        )
    return current


def candidate_tree_sha256(
    distillation_dir: Path | str,
    candidate_path: str,
) -> str:
    """Return ``sha256:<64 lowercase hex>`` for a safe candidate directory."""
    candidate_root = _resolve_candidate_root(distillation_dir, candidate_path)
    files, directories = _scan_candidate_tree(candidate_root)
    if len(files) > MAX_U64:
        raise CandidateTreeError(
            "CANDIDATE_TREE_TOO_LARGE", ".", "file count exceeds the v1 framing limit"
        )

    digest = hashlib.sha256()
    digest.update(TREE_HASH_PREFIX)
    digest.update(len(files).to_bytes(8, "big"))
    for record in files:
        content = _read_stable_file(record)
        if len(record.path_bytes) > MAX_U64 or len(content) > MAX_U64:
            raise CandidateTreeError(
                "CANDIDATE_TREE_TOO_LARGE", record.relative_path,
                "path or file length exceeds the v1 framing limit",
            )
        digest.update(b"F\0")
        digest.update(len(record.path_bytes).to_bytes(8, "big"))
        digest.update(record.path_bytes)
        digest.update(len(content).to_bytes(8, "big"))
        digest.update(content)

    # A file that was already read can still be mutated while later files are
    # being hashed.  File-content writes do not change the parent directory's
    # metadata, so the directory pass below cannot detect that race.  Recheck
    # every hashed path against the stat that was proven stable when it was
    # opened and read before returning an identity for the tree.
    for record in files:
        try:
            final_stat = record.absolute_path.lstat()
        except OSError as exc:
            raise CandidateTreeError(
                "CANDIDATE_TREE_CHANGED", record.relative_path,
                f"file changed after hashing: {exc}",
            ) from exc
        if not _same_entry(record.stat_result, final_stat):
            raise CandidateTreeError(
                "CANDIDATE_TREE_CHANGED", record.relative_path,
                "file changed after it was hashed",
            )

    for directory in directories:
        try:
            final_stat = directory.absolute_path.lstat()
        except OSError as exc:
            raise CandidateTreeError(
                "CANDIDATE_TREE_CHANGED", directory.relative_path,
                f"directory changed after traversal: {exc}",
            ) from exc
        if not _same_entry(directory.stat_result, final_stat):
            raise CandidateTreeError(
                "CANDIDATE_TREE_CHANGED", directory.relative_path,
                "directory contents or metadata changed while the tree was being hashed",
            )
    return f"sha256:{digest.hexdigest()}"


def main(argv: Sequence[str] | None = None) -> int:
    parser = argparse.ArgumentParser(
        description="Compute the deterministic v1 SHA-256 of a materialized candidate tree."
    )
    parser.add_argument("distillation_dir", help="Distillation directory that owns the candidate")
    parser.add_argument(
        "candidate_path",
        help="Canonical POSIX path to the candidate, relative to the distillation directory",
    )
    args = parser.parse_args(argv)
    try:
        result = candidate_tree_sha256(args.distillation_dir, args.candidate_path)
    except CandidateTreeError as exc:
        print(str(exc), file=sys.stderr)
        return 2
    print(result)
    return 0


if __name__ == "__main__":
    sys.exit(main())

#!/usr/bin/env python3
"""Inspect Gate 3/materialization routing without modifying a distillation.

The result is deliberately narrow.  It reports whether a review candidate is
review-only, requires materialization, or is structurally eligible to enter
Gate 4.  It does not assess knowledge truth, copyright permission, or behavior
effectiveness.
"""

from __future__ import annotations

import argparse
import hashlib
import json
import sys
from pathlib import Path, PurePosixPath
from typing import Any, Mapping, Sequence

# Importing sibling helpers must not create bytecode inside a materialized tree.
sys.dont_write_bytecode = True

from hash_candidate_tree import (  # noqa: E402
    CandidateTreeError,
    candidate_tree_sha256,
    canonical_candidate_path,
)
from validate_distillation import (  # noqa: E402
    APPROVAL_SNAPSHOT_GOVERNANCE_FILES,
    DistillationInputError,
    GATE3_APPROVAL_SNAPSHOT_CONTRACT,
    REQUIRED_FILES,
    distillation_read_snapshot,
    snapshot_validation_inputs,
    validate_distillation,
)
from task_contracts import inspect_task_governance  # noqa: E402

try:
    import yaml
except ImportError as exc:  # pragma: no cover - minimal runtimes only
    raise SystemExit(
        "PyYAML is required to inspect governance YAML. "
        "Do not install it without the user's approval."
    ) from exc


APPROVED_RULE_DECISIONS = {"accepted", "revised"}


class _UniqueKeySafeLoader(yaml.SafeLoader):
    """Safe YAML loader that rejects duplicate mapping keys."""

    def construct_mapping(self, node, deep=False):  # type: ignore[override]
        self.flatten_mapping(node)
        mapping: dict[Any, Any] = {}
        for key_node, value_node in node.value:
            key = self.construct_object(key_node, deep=deep)
            try:
                duplicate = key in mapping
            except TypeError as exc:
                raise yaml.constructor.ConstructorError(
                    "while constructing a mapping",
                    node.start_mark,
                    "found an unhashable mapping key",
                    key_node.start_mark,
                ) from exc
            if duplicate:
                raise yaml.constructor.ConstructorError(
                    "while constructing a mapping",
                    node.start_mark,
                    f"found duplicate key {key!r}",
                    key_node.start_mark,
                )
            mapping[key] = self.construct_object(value_node, deep=deep)
        return mapping


class StateInspectionError(RuntimeError):
    """Raised when the requested candidate cannot be selected safely."""

    def __init__(self, code: str, path: str, message: str):
        super().__init__(message)
        self.code = code
        self.path = path
        self.message = message


def _load_yaml(path: Path) -> dict[str, Any]:
    if not path.is_file():
        raise StateInspectionError("MISSING_FILE", str(path), "required YAML file is missing")
    try:
        value = yaml.load(path.read_text(encoding="utf-8"), Loader=_UniqueKeySafeLoader)
    except (OSError, UnicodeError) as exc:
        raise StateInspectionError("READ_ERROR", str(path), str(exc)) from exc
    except (yaml.YAMLError, RecursionError) as exc:
        raise StateInspectionError("YAML_PARSE", str(path), str(exc)) from exc
    if not isinstance(value, dict):
        raise StateInspectionError("ROOT_TYPE", str(path), "YAML root must be a mapping")
    return value


def _snapshot_digests(
    root: Path, sources_manifest: Path | str | None
) -> dict[str, str]:
    """Hash all files whose bytes can affect routing or validation."""
    paths = [root / name for name in REQUIRED_FILES]
    overlay = root / "correction-overlay.yml"
    if overlay.exists() or overlay.is_symlink():
        paths.append(overlay)
    if sources_manifest is not None:
        paths.append(Path(sources_manifest))
    for path in sorted(root.glob("task-contract*.yml")):
        paths.append(path)
    for name in ("task-coverage.yml", "context-checkpoint.yml"):
        path = root / name
        if path.exists() or path.is_symlink():
            paths.append(path)
    result: dict[str, str] = {}
    for path in paths:
        try:
            if path.is_symlink():
                raise StateInspectionError(
                    "GOVERNANCE_FILE_SYMLINK",
                    str(path),
                    "governance and manifest files must not be symlinks",
                )
            data = path.read_bytes()
        except StateInspectionError:
            raise
        except OSError as exc:
            raise StateInspectionError("READ_ERROR", str(path), str(exc)) from exc
        result[str(path.resolve())] = hashlib.sha256(data).hexdigest()
    return result


def _is_unique_string_list(value: Any) -> bool:
    return (
        isinstance(value, list)
        and all(isinstance(item, str) and bool(item.strip()) for item in value)
        and len(value) == len(set(value))
    )


def _records(document: Mapping[str, Any], key: str) -> list[Mapping[str, Any]]:
    value = document.get(key)
    if not isinstance(value, list):
        return []
    return [item for item in value if isinstance(item, dict)]


def _approved_rule_ids(decision: Mapping[str, Any] | None) -> list[str]:
    if decision is None:
        return []
    result: list[str] = []
    for item in _records(decision, "rule_decisions"):
        rule_id = item.get("rule_id")
        if item.get("decision") in APPROVED_RULE_DECISIONS and isinstance(rule_id, str):
            result.append(rule_id)
    return result


def _approval_snapshot_summary(
    decision: Mapping[str, Any] | None,
) -> dict[str, Any] | None:
    """Return only bounded scalar snapshot fields safe for deterministic JSON."""
    if decision is None:
        return None
    snapshot = decision.get("approval_snapshot")
    if not isinstance(snapshot, dict):
        return {
            "present": False,
            "contract": None,
            "candidate_path": None,
            "candidate_hash": None,
            "governance_hashes": {
                filename: None for filename in APPROVAL_SNAPSHOT_GOVERNANCE_FILES
            },
        }
    governance = snapshot.get("governance_hashes")
    return {
        "present": True,
        "contract": (
            snapshot.get("contract")
            if isinstance(snapshot.get("contract"), str)
            else None
        ),
        "candidate_path": (
            snapshot.get("candidate_path")
            if isinstance(snapshot.get("candidate_path"), str)
            else None
        ),
        "candidate_hash": (
            snapshot.get("candidate_hash")
            if isinstance(snapshot.get("candidate_hash"), str)
            else None
        ),
        "governance_hashes": {
            filename: (
                governance.get(filename)
                if isinstance(governance, dict)
                and isinstance(governance.get(filename), str)
                else None
            )
            for filename in APPROVAL_SNAPSHOT_GOVERNANCE_FILES
        },
        "current_gate1_decision_id": snapshot.get("current_gate1_decision_id"),
        "task_contract": snapshot.get("task_contract"),
        "task_coverage": snapshot.get("task_coverage"),
        "candidate_stable_task_ids": snapshot.get("candidate_stable_task_ids"),
    }


def _current_decisions(
    decisions: Sequence[Mapping[str, Any]], gate: str, candidate_id: str
) -> list[Mapping[str, Any]]:
    return [
        item
        for item in decisions
        if item.get("gate") == gate
        and item.get("candidate_id") == candidate_id
        and item.get("is_current") is True
    ]


def _materialization_mismatch_reasons(
    record: Mapping[str, Any],
    *,
    candidate_id: str,
    candidate_path: str | None,
    candidate_hash: str | None,
    gate3_decision_id: str | None,
    approved_rule_ids: Sequence[str],
) -> list[str]:
    reasons: list[str] = []
    if record.get("candidate_id") != candidate_id:
        reasons.append("candidate-id-mismatch")
    if record.get("gate3_decision_id") != gate3_decision_id:
        reasons.append("gate3-decision-mismatch")
    if record.get("candidate_path") != candidate_path:
        reasons.append("candidate-path-mismatch")
    if candidate_hash is None or record.get("candidate_hash") != candidate_hash:
        reasons.append("candidate-hash-mismatch")
    rule_ids = record.get("rule_ids")
    if (
        not _is_unique_string_list(rule_ids)
        or set(rule_ids) != set(approved_rule_ids)
    ):
        reasons.append("approved-rule-set-mismatch")
    quick_validation = record.get("quick_validation")
    if (
        not isinstance(quick_validation, dict)
        or quick_validation.get("status") != "pass"
        or quick_validation.get("candidate_hash") != candidate_hash
    ):
        reasons.append("quick-validation-not-pass")
    return reasons


def _inspect_distillation_snapshot(
    distillation_dir: Path | str,
    candidate_id: str,
    candidate_path: str,
    sources_manifest: Path | str | None = None,
) -> dict[str, Any]:
    """Return a deterministic, read-only activation-state report."""
    root = Path(distillation_dir)
    canonical_path = canonical_candidate_path(candidate_path)
    snapshot_before = _snapshot_digests(root, sources_manifest)

    capability_document = _load_yaml(root / "capability-rules.yml")
    gate_document = _load_yaml(root / "gate-decisions.yml")
    eval_document = _load_yaml(root / "eval-runs.yml")

    candidates = [
        item
        for item in _records(capability_document, "skill_candidates")
        if item.get("candidate_id") == candidate_id
    ]
    if len(candidates) != 1:
        raise StateInspectionError(
            "CANDIDATE_SELECTION_ERROR",
            "capability-rules.yml.skill_candidates",
            f"candidate_id {candidate_id!r} must select exactly one candidate; found {len(candidates)}",
        )
    candidate = candidates[0]
    candidate_name = candidate.get("name")
    if not isinstance(candidate_name, str) or PurePosixPath(canonical_path).name != candidate_name:
        raise StateInspectionError(
            "CANDIDATE_PATH_NAME_MISMATCH",
            candidate_path,
            "candidate_path final component must equal the selected candidate name",
        )

    gate_decisions = _records(gate_document, "gate_decisions")
    gate3_current = _current_decisions(gate_decisions, "gate-3", candidate_id)
    gate4_current = _current_decisions(gate_decisions, "gate-4", candidate_id)
    current_gate3 = gate3_current[0] if len(gate3_current) == 1 else None
    current_gate4 = gate4_current[0] if len(gate4_current) == 1 else None
    gate3_approved = (
        current_gate3 is not None
        and current_gate3.get("decision") == "approved-for-eval"
    )
    approval_snapshot = _approval_snapshot_summary(current_gate3)
    approval_candidate_path = (
        approval_snapshot.get("candidate_path")
        if isinstance(approval_snapshot, dict)
        else None
    )
    approval_candidate_hash = (
        approval_snapshot.get("candidate_hash")
        if isinstance(approval_snapshot, dict)
        else None
    )
    approval_snapshot_shape_present = bool(
        isinstance(approval_snapshot, dict)
        and approval_snapshot.get("present") is True
        and approval_snapshot.get("contract") == GATE3_APPROVAL_SNAPSHOT_CONTRACT
    )
    approved_rule_ids = _approved_rule_ids(current_gate3) if gate3_approved else []
    gate3_id = current_gate3.get("decision_id") if current_gate3 is not None else None

    materialization_records = [
        record
        for record in _records(gate_document, "materializations")
        if record.get("candidate_id") == candidate_id
    ]
    completed_records = [
        record for record in materialization_records if record.get("status") == "completed"
    ]
    tree_issue: dict[str, str] | None = None
    try:
        computed_hash: str | None = candidate_tree_sha256(root, canonical_path)
    except CandidateTreeError as exc:
        computed_hash = None
        # Before first materialization the approved candidate directory may not
        # exist yet.  That is the materialization-required state, not corrupt
        # governance.  A missing tree claimed by a completed record is invalid.
        if (
            exc.code != "CANDIDATE_PATH_MISSING"
            or completed_records
            or gate3_approved
        ):
            tree_issue = {"code": exc.code, "path": exc.path, "message": exc.message}

    try:
        validation = validate_distillation(root, sources_manifest)
    except DistillationInputError as exc:
        raise StateInspectionError(exc.code, str(exc.path), exc.message) from exc
    except Exception as exc:  # defensive fail-closed boundary around the validator
        raise StateInspectionError(
            "VALIDATOR_INTERNAL_ERROR",
            str(root),
            f"{type(exc).__name__}: {exc}",
        ) from exc

    task_governance = inspect_task_governance(root, sources_manifest)

    verification_hash: str | None = None
    verification_tree_issue: dict[str, str] | None = None
    try:
        verification_hash = candidate_tree_sha256(root, canonical_path)
    except CandidateTreeError as exc:
        if (
            exc.code != "CANDIDATE_PATH_MISSING"
            or completed_records
            or gate3_approved
        ):
            verification_tree_issue = {
                "code": exc.code,
                "path": exc.path,
                "message": exc.message,
            }
    snapshot_after = _snapshot_digests(root, sources_manifest)
    snapshot_changed = (
        snapshot_before != snapshot_after or computed_hash != verification_hash
    )

    matching: list[str] = []
    stale: list[dict[str, Any]] = []
    historical: list[dict[str, Any]] = []
    for record in materialization_records:
        materialization_id = record.get("materialization_id")
        status = record.get("status")
        if status != "completed":
            historical.append({"materialization_id": materialization_id, "status": status})
            continue
        reasons = _materialization_mismatch_reasons(
            record,
            candidate_id=candidate_id,
            candidate_path=approval_candidate_path,
            candidate_hash=approval_candidate_hash,
            gate3_decision_id=gate3_id,
            approved_rule_ids=approved_rule_ids,
        )
        approval_matches_cli = (
            approval_snapshot_shape_present
            and approval_candidate_path == canonical_path
            and computed_hash is not None
            and approval_candidate_hash == computed_hash
        )
        if gate3_approved and approval_matches_cli and not reasons:
            matching.append(str(materialization_id))
        else:
            stale.append({
                "materialization_id": materialization_id,
                "reasons": reasons or ["no-current-approved-gate3"],
            })

    blockers = [item.as_dict() for item in validation.errors]
    if tree_issue is not None:
        blockers.append(tree_issue)
    if verification_tree_issue is not None:
        blockers.append(verification_tree_issue)
    if snapshot_changed:
        blockers.append({
            "code": "INSPECTION_SNAPSHOT_CHANGED",
            "path": str(root),
            "message": "governance files or candidate tree changed during state inspection",
        })
    if len(gate3_current) > 1:
        blockers.append({
            "code": "AMBIGUOUS_CURRENT_GATE3",
            "path": "gate-decisions.yml.gate_decisions",
            "message": "candidate has more than one current Gate 3 decision",
        })
    if len(gate4_current) > 1:
        blockers.append({
            "code": "AMBIGUOUS_CURRENT_GATE4",
            "path": "gate-decisions.yml.gate_decisions",
            "message": "candidate has more than one current Gate 4 decision",
        })
    approval_matches_cli = (
        approval_snapshot_shape_present
        and approval_candidate_path == canonical_path
        and computed_hash is not None
        and approval_candidate_hash == computed_hash
    )
    if gate3_approved and not approval_snapshot_shape_present:
        blockers.append({
            "code": "APPROVAL_SNAPSHOT_INVALID",
            "path": "gate-decisions.yml.current_gate3.approval_snapshot",
            "message": "current Gate 3 approval lacks the required versioned snapshot",
        })
    elif gate3_approved:
        if approval_candidate_path != canonical_path:
            blockers.append({
                "code": "APPROVAL_SNAPSHOT_CANDIDATE_PATH_MISMATCH",
                "path": "gate-decisions.yml.current_gate3.approval_snapshot.candidate_path",
                "message": "CLI candidate_path must exactly match the current Gate 3 approval snapshot",
            })
        if computed_hash is None or approval_candidate_hash != computed_hash:
            blockers.append({
                "code": "APPROVAL_SNAPSHOT_CANDIDATE_HASH_MISMATCH",
                "path": "gate-decisions.yml.current_gate3.approval_snapshot.candidate_hash",
                "message": "current candidate tree hash must exactly match the current Gate 3 approval snapshot",
            })
    if len(matching) > 1:
        blockers.append({
            "code": "AMBIGUOUS_COMPLETED_MATERIALIZATION",
            "path": "gate-decisions.yml.materializations",
            "message": "more than one completed materialization exactly matches the current candidate",
        })
    lifecycle = candidate.get("lifecycle")
    if gate3_approved and lifecycle != "review":
        blockers.append({
            "code": "CANDIDATE_LIFECYCLE_NOT_REVIEW",
            "path": "capability-rules.yml.skill_candidates.lifecycle",
            "message": "Gate 4 routing is allowed only while the candidate lifecycle is review",
        })
    if gate3_approved and len(matching) == 1 and sources_manifest is None:
        blockers.append({
            "code": "SOURCES_MANIFEST_REQUIRED_FOR_GATE4",
            "path": "--sources-manifest",
            "message": "Gate 4 eligibility requires explicit source-manifest validation",
        })

    task_blocker_codes = {
        "TASK_CONTRACT_MISSING", "TASK_CONTRACT_INVALID",
        "TASK_CONTRACT_SNAPSHOT_MISMATCH", "TASK_COVERAGE_INVALID",
        "STABLE_TASK_UNKNOWN", "STABLE_TASK_UNCOVERED",
        "CANDIDATE_TASK_MISMATCH", "GATE3_TASK_CONTRACT_MISMATCH",
        "METHOD_TRANSFER_PROVENANCE_REQUIRED",
        "METHOD_TRANSFER_EXTERNAL_HOLDOUT_REQUIRED",
        "LEGACY_TASK_CONTRACT_REVIEW_REQUIRED",
        "CHECKPOINT_PRODUCT_CONTRACT_CONFLICT",
    }
    task_repair_required = any(
        item.get("code") in task_blocker_codes for item in blockers
    )
    if blockers:
        activation_state = "invalid"
        allowed_actions = (
            ["review", "repair-task-contract-or-coverage"]
            if task_repair_required
            else ["repair-governance-or-candidate-state"]
        )
    elif not gate3_approved:
        activation_state = "review-only"
        allowed_actions = ["review", "authorized-candidate-maintenance"]
    elif not matching:
        activation_state = "materialization-required"
        allowed_actions = ["materialize", "quick-validate", "record-materialization"]
    else:
        activation_state = "gate4-eligible"
        allowed_actions = ["authorized-gate4-evaluation"]

    matching_id = matching[0] if len(matching) == 1 else None
    completed_pass_evals = 0
    if matching_id is not None:
        completed_pass_evals = sum(
            1
            for run in _records(eval_document, "eval_runs")
            if run.get("candidate_id") == candidate_id
            and run.get("materialization_id") == matching_id
            and run.get("status") == "completed"
            and run.get("outcome") == "pass"
        )

    return {
        "inspection_scope": "gate3-materialization-routing-only",
        "truth_assessed": False,
        "behavior_effectiveness_assessed": False,
        "activation_state": activation_state,
        "continuation_mode": (
            "review-repair-only" if task_repair_required else activation_state
        ),
        "allowed_actions": allowed_actions,
        "blockers": blockers,
        "candidate": {
            "candidate_id": candidate_id,
            "name": candidate_name,
            "lifecycle": lifecycle,
            "candidate_path": canonical_path,
            "computed_hash": computed_hash,
            "tree_status": "present" if computed_hash is not None else "not-yet-materialized",
        },
        "current_gate3": (
            None
            if current_gate3 is None
            else {
                "decision_id": current_gate3.get("decision_id"),
                "decision": current_gate3.get("decision"),
                "approved_rule_ids": approved_rule_ids,
                "approval_snapshot": approval_snapshot,
                "approval_snapshot_matches_candidate": approval_matches_cli,
            }
        ),
        "current_gate4": (
            None
            if current_gate4 is None
            else {
                "decision_id": current_gate4.get("decision_id"),
                "decision": current_gate4.get("decision"),
            }
        ),
        "gate4_accepted": (
            current_gate4 is not None and current_gate4.get("decision") == "accepted"
        ),
        "completed_pass_eval_count": completed_pass_evals,
        "materializations": {
            "matching": matching,
            "stale_completed": stale,
            "historical": historical,
        },
        "validation": {
            "ok": validation.ok,
            "error_count": len(validation.errors),
            "warning_count": len(validation.warnings),
            "warnings": [item.as_dict() for item in validation.warnings],
            "metrics": validation.metrics,
            "sources_manifest_provided": sources_manifest is not None,
        },
        "authoritative_product_contract": task_governance.summary,
        "current_stage_objective": task_governance.summary.get(
            "current_stage_objective"
        ),
        "temporary_operational_constraints": task_governance.summary.get(
            "temporary_operational_constraints", []
        ),
    }


def _candidate_tree_observation(
    root: Path, candidate_path: str
) -> tuple[str | None, tuple[str, str, str] | None]:
    try:
        return candidate_tree_sha256(root, candidate_path), None
    except CandidateTreeError as exc:
        return None, (exc.code, exc.path, exc.message)


def inspect_distillation_state(
    distillation_dir: Path | str,
    candidate_id: str,
    candidate_path: str,
    sources_manifest: Path | str | None = None,
) -> dict[str, Any]:
    """Inspect routing from one shared immutable distillation byte snapshot."""
    root = Path(distillation_dir)
    canonical_path = canonical_candidate_path(candidate_path)
    outer_before = _snapshot_digests(root, sources_manifest)
    candidate_before = _candidate_tree_observation(root, canonical_path)

    try:
        with distillation_read_snapshot(root) as snapshot_root:
            snapshot_manifest = snapshot_validation_inputs(
                snapshot_root, sources_manifest
            )
            report = _inspect_distillation_snapshot(
                snapshot_root,
                candidate_id,
                canonical_path,
                snapshot_manifest,
            )
    except DistillationInputError as exc:
        raise StateInspectionError(exc.code, str(exc.path), exc.message) from exc

    outer_after = _snapshot_digests(root, sources_manifest)
    candidate_after = _candidate_tree_observation(root, canonical_path)
    if outer_before != outer_after or candidate_before != candidate_after:
        blocker = {
            "code": "INSPECTION_SNAPSHOT_CHANGED",
            "path": str(root),
            "message": "source governance files or candidate tree changed during inspection",
        }
        report["blockers"].append(blocker)
        report["activation_state"] = "invalid"
        report["continuation_mode"] = "invalid"
        report["allowed_actions"] = ["repair-governance-or-candidate-state"]
    return report


def main(argv: Sequence[str] | None = None) -> int:
    parser = argparse.ArgumentParser(
        description="Inspect one review candidate's Gate 3/materialization routing state."
    )
    parser.add_argument("distillation_dir", help="Owning distillation directory")
    parser.add_argument("--candidate-id", required=True, help="Stable skill candidate ID")
    parser.add_argument(
        "--candidate-path",
        required=True,
        help="Canonical candidate path relative to the distillation directory",
    )
    parser.add_argument(
        "--sources-manifest",
        help="Manifest passed to validation; required before reporting Gate 4 eligibility",
    )
    args = parser.parse_args(argv)
    try:
        report = inspect_distillation_state(
            args.distillation_dir,
            args.candidate_id,
            args.candidate_path,
            args.sources_manifest,
        )
    except (StateInspectionError, CandidateTreeError) as exc:
        result = {
            "inspection_scope": "gate3-materialization-routing-only",
            "truth_assessed": False,
            "behavior_effectiveness_assessed": False,
            "activation_state": "invalid",
            "input_error": {
                "code": exc.code,
                "path": exc.path,
                "message": exc.message,
            },
        }
        print(json.dumps(result, ensure_ascii=False, indent=2, sort_keys=True))
        return 2
    print(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True))
    return 1 if report["activation_state"] == "invalid" else 0


if __name__ == "__main__":
    sys.exit(main())

#!/usr/bin/env python3
"""Deterministically normalize a DOCX source without modifying the input file.

The script intentionally uses only the Python standard library.  It reads the
OOXML package in-place, walks top-level body paragraphs and tables in document
order, and writes a private normalization bundle to an explicit output
directory.

No OCR, spelling correction, scientific-symbol substitution, or semantic
rewriting is performed.  Normalized text is limited to Unicode NFC and a
conservative whitespace policy documented in ``normalization-log.yml``.
"""

from __future__ import annotations

import argparse
import hashlib
import json
import posixpath
import re
import struct
import sys
import unicodedata
import urllib.parse
import zipfile
from dataclasses import dataclass
from pathlib import Path, PurePosixPath
from typing import Any, Iterable, Iterator, Mapping, Sequence
from xml.etree import ElementTree as ET


TOOL_NAME = "normalize_docx_source"
TOOL_VERSION = "1.0.0"

NS = {
    "a": "http://schemas.openxmlformats.org/drawingml/2006/main",
    "cp": "http://schemas.openxmlformats.org/package/2006/metadata/core-properties",
    "dc": "http://purl.org/dc/elements/1.1/",
    "dcterms": "http://purl.org/dc/terms/",
    "m": "http://schemas.openxmlformats.org/officeDocument/2006/math",
    "pic": "http://schemas.openxmlformats.org/drawingml/2006/picture",
    "r": "http://schemas.openxmlformats.org/officeDocument/2006/relationships",
    "v": "urn:schemas-microsoft-com:vml",
    "w": "http://schemas.openxmlformats.org/wordprocessingml/2006/main",
    "wp": "http://schemas.openxmlformats.org/drawingml/2006/wordprocessingDrawing",
}

REL_NS = "http://schemas.openxmlformats.org/package/2006/relationships"
DOCUMENT_PART = "word/document.xml"
DOCUMENT_RELS_PART = "word/_rels/document.xml.rels"
STYLES_PART = "word/styles.xml"
CORE_PROPERTIES_PART = "docProps/core.xml"

W = "{%s}" % NS["w"]
M = "{%s}" % NS["m"]
R = "{%s}" % NS["r"]
A = "{%s}" % NS["a"]
V = "{%s}" % NS["v"]

OUTPUT_FILENAMES = (
    "blocks.jsonl",
    "structure.yml",
    "media-map.yml",
    "normalization-log.yml",
    "checksums.yml",
)

_HORIZONTAL_SPACE_RE = re.compile(r" +")
_HEADING_EN_RE = re.compile(r"(?i)(?:^|\b)heading\s*([1-9])(?:\b|$)")
_HEADING_ZH_RE = re.compile(r"(?:标题|標題)\s*([1-9一二三四五六七八九])")
_CHINESE_LEVELS = {"一": 1, "二": 2, "三": 3, "四": 4, "五": 5,
                   "六": 6, "七": 7, "八": 8, "九": 9}


class NormalizationError(RuntimeError):
    """Raised when the source cannot be normalized safely."""


@dataclass(frozen=True)
class Relationship:
    relationship_id: str
    relationship_type: str
    target: str
    target_mode: str | None
    package_path: str | None

    @property
    def is_external(self) -> bool:
        return (self.target_mode or "").lower() == "external"

    @property
    def is_image(self) -> bool:
        return self.relationship_type.rstrip("/").endswith("/image")


@dataclass(frozen=True)
class StyleInfo:
    style_id: str
    name: str | None
    based_on: str | None
    outline_level: int | None


def qname(prefix: str, local: str) -> str:
    return "{%s}%s" % (NS[prefix], local)


def local_name(tag: str) -> str:
    if "}" in tag:
        return tag.rsplit("}", 1)[1]
    return tag


def sha256_bytes(data: bytes) -> str:
    return hashlib.sha256(data).hexdigest()


def sha256_file(path: Path) -> str:
    digest = hashlib.sha256()
    with path.open("rb") as handle:
        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
            digest.update(chunk)
    return digest.hexdigest()


def _yaml_scalar(value: Any) -> str:
    if value is None:
        return "null"
    if value is True:
        return "true"
    if value is False:
        return "false"
    if isinstance(value, (int, float)) and not isinstance(value, bool):
        return str(value)
    if isinstance(value, str):
        # JSON double-quoted strings are valid YAML scalars and handle control
        # characters deterministically without a third-party YAML dependency.
        return json.dumps(value, ensure_ascii=False)
    raise TypeError(f"unsupported YAML scalar: {type(value).__name__}")


def _yaml_lines(value: Any, indent: int = 0) -> list[str]:
    prefix = " " * indent
    if isinstance(value, Mapping):
        if not value:
            return [prefix + "{}"]
        lines: list[str] = []
        for key, item in value.items():
            rendered_key = _yaml_scalar(str(key))
            if isinstance(item, Mapping) or isinstance(item, (list, tuple)):
                if not item:
                    lines.append(f"{prefix}{rendered_key}: " + ("{}" if isinstance(item, Mapping) else "[]"))
                else:
                    lines.append(f"{prefix}{rendered_key}:")
                    lines.extend(_yaml_lines(item, indent + 2))
            else:
                lines.append(f"{prefix}{rendered_key}: {_yaml_scalar(item)}")
        return lines
    if isinstance(value, (list, tuple)):
        if not value:
            return [prefix + "[]"]
        lines = []
        for item in value:
            if isinstance(item, Mapping) or isinstance(item, (list, tuple)):
                if not item:
                    lines.append(prefix + "- " + ("{}" if isinstance(item, Mapping) else "[]"))
                else:
                    lines.append(prefix + "-")
                    lines.extend(_yaml_lines(item, indent + 2))
            else:
                lines.append(f"{prefix}- {_yaml_scalar(item)}")
        return lines
    return [prefix + _yaml_scalar(value)]


def write_yaml(path: Path, value: Mapping[str, Any]) -> None:
    text = "---\n" + "\n".join(_yaml_lines(value)) + "\n"
    path.write_text(text, encoding="utf-8", newline="\n")


def write_jsonl(path: Path, records: Iterable[Mapping[str, Any]]) -> None:
    with path.open("w", encoding="utf-8", newline="\n") as handle:
        for record in records:
            handle.write(json.dumps(record, ensure_ascii=False, separators=(",", ":")))
            handle.write("\n")


def _validate_source_id(source_id: str) -> str:
    value = source_id.strip()
    if not value:
        raise NormalizationError("--source-id must not be empty")
    if any(ord(char) < 32 or ord(char) == 127 for char in value):
        raise NormalizationError("--source-id must not contain control characters")
    return value


def _prepare_output_directory(output_dir: Path) -> None:
    if output_dir.exists() and not output_dir.is_dir():
        raise NormalizationError(f"output path exists and is not a directory: {output_dir}")
    if output_dir.exists() and any(output_dir.iterdir()):
        raise NormalizationError(
            f"output directory must be new or empty; refusing to overwrite: {output_dir}"
        )
    output_dir.mkdir(parents=True, exist_ok=True)
    (output_dir / "media").mkdir()


def normalize_text(raw_text: str) -> tuple[str, dict[str, Any]]:
    """Apply NFC and conservative whitespace normalization only."""
    nfc_text = unicodedata.normalize("NFC", raw_text)
    nfc_changed = nfc_text != raw_text

    crlf_count = nfc_text.count("\r\n")
    lone_cr_count = nfc_text.count("\r") - crlf_count
    text = nfc_text.replace("\r\n", "\n").replace("\r", "\n")

    replaced_unicode_space_count = 0
    converted: list[str] = []
    for char in text:
        if char in ("\n", "\t"):
            converted.append(char)
        elif char.isspace() and char != " ":
            converted.append(" ")
            replaced_unicode_space_count += 1
        else:
            converted.append(char)
    text = "".join(converted)

    collapsed_space_count = 0
    normalized_lines: list[str] = []
    for line in text.split("\n"):
        pieces = line.split("\t")
        normalized_pieces: list[str] = []
        for piece in pieces:
            runs = _HORIZONTAL_SPACE_RE.findall(piece)
            collapsed_space_count += sum(max(0, len(run) - 1) for run in runs)
            normalized_pieces.append(_HORIZONTAL_SPACE_RE.sub(" ", piece).strip(" "))
        normalized_lines.append("\t".join(normalized_pieces))

    before_outer_trim = "\n".join(normalized_lines)
    # TABs delimit table cells and therefore remain significant even at the
    # start or end of a block (an empty first/last cell).  Only spaces and
    # newlines are trimmed at the outer block boundary.
    normalized = before_outer_trim.strip(" \n")
    stats = {
        "nfc_changed": nfc_changed,
        "crlf_converted": crlf_count,
        "lone_cr_converted": lone_cr_count,
        "unicode_spaces_replaced": replaced_unicode_space_count,
        "redundant_spaces_removed": collapsed_space_count,
        "outer_whitespace_trimmed": normalized != before_outer_trim,
        "changed": normalized != raw_text,
    }
    return normalized, stats


def extract_text(element: ET.Element) -> str:
    """Extract visible run text while preserving tabs and explicit breaks."""
    pieces: list[str] = []
    for node in element.iter():
        if node.tag in (W + "t", M + "t"):
            pieces.append(node.text or "")
        elif node.tag == W + "tab":
            pieces.append("\t")
        elif node.tag in (W + "br", W + "cr"):
            pieces.append("\n")
        elif node.tag == W + "noBreakHyphen":
            pieces.append("\u2011")
        elif node.tag == W + "softHyphen":
            pieces.append("\u00ad")
        elif node.tag == W + "sym":
            char_value = node.get(W + "char")
            if char_value:
                try:
                    pieces.append(chr(int(char_value, 16)))
                except (ValueError, OverflowError):
                    # The occurrence is separately placed into manual review.
                    pass
    return "".join(pieces)


def iter_body_blocks(body: ET.Element) -> Iterator[ET.Element]:
    """Yield body-level paragraphs/tables, including those in content controls.

    Once a table is yielded its cell content is not traversed again: the table
    is one body block and its internal paragraphs belong to that block.
    """
    for child in list(body):
        name = local_name(child.tag)
        if name in ("p", "tbl"):
            yield child
        elif name not in ("sectPr", "pPr", "tblPr"):
            yield from iter_body_blocks(child)


def _wrapped_children_by_local(element: ET.Element, name: str) -> Iterator[ET.Element]:
    """Yield rows or cells through OOXML wrappers without entering nested peers."""
    for child in list(element):
        child_name = local_name(child.tag)
        if child_name == name:
            yield child
        elif child_name not in ("p", "tbl"):
            yield from _wrapped_children_by_local(child, name)


def _container_text_parts(container: ET.Element) -> list[str]:
    """Extract direct cell content without duplicating nested table text."""
    parts: list[str] = []
    for child in list(container):
        name = local_name(child.tag)
        if name == "p":
            parts.append(extract_text(child))
        elif name == "tbl":
            parts.append(extract_table_text(child))
        elif name in ("sdt", "sdtContent", "customXml"):
            parts.extend(_container_text_parts(child))
    return parts


def extract_table_text(table: ET.Element) -> str:
    rows: list[str] = []
    for row in _wrapped_children_by_local(table, "tr"):
        cells: list[str] = []
        for cell in _wrapped_children_by_local(row, "tc"):
            cells.append("\n".join(_container_text_parts(cell)))
        rows.append("\t".join(cells))
    return "\n".join(rows)


def table_dimensions(table: ET.Element) -> tuple[int, int]:
    rows = list(_wrapped_children_by_local(table, "tr"))
    cell_counts = [len(list(_wrapped_children_by_local(row, "tc"))) for row in rows]
    return len(rows), max(cell_counts, default=0)


def paragraph_style_id(paragraph: ET.Element) -> str | None:
    style = paragraph.find("./w:pPr/w:pStyle", NS)
    return style.get(W + "val") if style is not None else None


def table_style_id(table: ET.Element) -> str | None:
    style = table.find("./w:tblPr/w:tblStyle", NS)
    return style.get(W + "val") if style is not None else None


def load_styles(package: zipfile.ZipFile) -> dict[str, StyleInfo]:
    if STYLES_PART not in package.namelist():
        return {}
    root = ET.fromstring(package.read(STYLES_PART))
    styles: dict[str, StyleInfo] = {}
    for style in root.findall("./w:style", NS):
        if style.get(W + "type") != "paragraph":
            continue
        style_id = style.get(W + "styleId")
        if not style_id:
            continue
        name_node = style.find("./w:name", NS)
        based_node = style.find("./w:basedOn", NS)
        outline_node = style.find("./w:pPr/w:outlineLvl", NS)
        outline_level: int | None = None
        if outline_node is not None:
            try:
                parsed = int(outline_node.get(W + "val", ""))
                if 0 <= parsed <= 8:
                    outline_level = parsed + 1
            except ValueError:
                pass
        styles[style_id] = StyleInfo(
            style_id=style_id,
            name=name_node.get(W + "val") if name_node is not None else None,
            based_on=based_node.get(W + "val") if based_node is not None else None,
            outline_level=outline_level,
        )
    return styles


def _heading_level_from_label(label: str | None) -> int | None:
    if not label:
        return None
    match = _HEADING_EN_RE.search(label)
    if match:
        return int(match.group(1))
    match = _HEADING_ZH_RE.search(label)
    if match:
        token = match.group(1)
        return int(token) if token.isdigit() else _CHINESE_LEVELS[token]
    return None


def heading_level(paragraph: ET.Element, style_id: str | None,
                  styles: Mapping[str, StyleInfo]) -> int | None:
    direct_outline = paragraph.find("./w:pPr/w:outlineLvl", NS)
    if direct_outline is not None:
        try:
            value = int(direct_outline.get(W + "val", ""))
            if 0 <= value <= 8:
                return value + 1
        except ValueError:
            pass

    visited: set[str] = set()
    current = style_id
    while current and current not in visited:
        visited.add(current)
        info = styles.get(current)
        if info is None:
            return _heading_level_from_label(current)
        if info.outline_level is not None:
            return info.outline_level
        inferred = _heading_level_from_label(info.name) or _heading_level_from_label(info.style_id)
        if inferred is not None:
            return inferred
        current = info.based_on
    return _heading_level_from_label(style_id)


def _resolve_package_target(target: str, target_mode: str | None) -> str | None:
    if (target_mode or "").lower() == "external":
        return None
    decoded = urllib.parse.unquote(target).replace("\\", "/")
    if decoded.startswith("/"):
        candidate = posixpath.normpath(decoded.lstrip("/"))
    else:
        candidate = posixpath.normpath(posixpath.join(posixpath.dirname(DOCUMENT_PART), decoded))
    if candidate in ("", ".") or candidate == ".." or candidate.startswith("../"):
        raise NormalizationError(f"unsafe relationship target: {target!r}")
    return str(PurePosixPath(candidate))


def load_relationships(package: zipfile.ZipFile) -> dict[str, Relationship]:
    if DOCUMENT_RELS_PART not in package.namelist():
        return {}
    root = ET.fromstring(package.read(DOCUMENT_RELS_PART))
    relationships: dict[str, Relationship] = {}
    for node in root.findall(f"{{{REL_NS}}}Relationship"):
        relationship_id = node.get("Id")
        target = node.get("Target")
        relationship_type = node.get("Type")
        if not relationship_id or target is None or relationship_type is None:
            continue
        target_mode = node.get("TargetMode")
        relationships[relationship_id] = Relationship(
            relationship_id=relationship_id,
            relationship_type=relationship_type,
            target=target,
            target_mode=target_mode,
            package_path=_resolve_package_target(target, target_mode),
        )
    return relationships


def image_relationship_references(element: ET.Element) -> list[dict[str, str]]:
    references: list[dict[str, str]] = []
    for node in element.iter():
        if node.tag == A + "blip":
            embedded = node.get(R + "embed")
            linked = node.get(R + "link")
            if embedded:
                references.append({"relationship_id": embedded, "reference_kind": "drawingml-embed"})
            if linked:
                references.append({"relationship_id": linked, "reference_kind": "drawingml-link"})
        elif node.tag == V + "imagedata":
            relationship_id = node.get(R + "id")
            if relationship_id:
                references.append({"relationship_id": relationship_id, "reference_kind": "vml-image"})
    return references


def _safe_media_filename(package_path: str, relationship_id: str,
                         used_names: set[str]) -> str:
    base = PurePosixPath(package_path).name or f"{relationship_id}.bin"
    candidate = base
    if candidate.casefold() in used_names:
        stem = Path(base).stem
        suffix = Path(base).suffix
        candidate = f"{stem}-{relationship_id}{suffix}"
        counter = 2
        while candidate.casefold() in used_names:
            candidate = f"{stem}-{relationship_id}-{counter}{suffix}"
            counter += 1
    used_names.add(candidate.casefold())
    return candidate


def image_dimensions(data: bytes) -> dict[str, int] | None:
    """Return common raster pixel dimensions without decoding/re-encoding."""
    if len(data) >= 24 and data.startswith(b"\x89PNG\r\n\x1a\n"):
        width, height = struct.unpack(">II", data[16:24])
        return {"width_px": width, "height_px": height}
    if len(data) >= 10 and data[:6] in (b"GIF87a", b"GIF89a"):
        width, height = struct.unpack("<HH", data[6:10])
        return {"width_px": width, "height_px": height}
    if len(data) >= 26 and data.startswith(b"BM"):
        width, height = struct.unpack("<ii", data[18:26])
        return {"width_px": abs(width), "height_px": abs(height)}
    if len(data) >= 4 and data.startswith(b"\xff\xd8"):
        offset = 2
        while offset + 4 <= len(data):
            if data[offset] != 0xFF:
                offset += 1
                continue
            while offset < len(data) and data[offset] == 0xFF:
                offset += 1
            if offset >= len(data):
                break
            marker = data[offset]
            offset += 1
            if marker in (0xD8, 0xD9) or 0xD0 <= marker <= 0xD7:
                continue
            if offset + 2 > len(data):
                break
            segment_length = struct.unpack(">H", data[offset:offset + 2])[0]
            if segment_length < 2 or offset + segment_length > len(data):
                break
            if marker in {
                0xC0, 0xC1, 0xC2, 0xC3, 0xC5, 0xC6, 0xC7,
                0xC9, 0xCA, 0xCB, 0xCD, 0xCE, 0xCF,
            } and segment_length >= 7:
                height, width = struct.unpack(">HH", data[offset + 3:offset + 7])
                return {"width_px": width, "height_px": height}
            offset += segment_length
    return None


def core_properties(package: zipfile.ZipFile) -> dict[str, str | None]:
    result: dict[str, str | None] = {
        "title": None,
        "creator": None,
        "subject": None,
        "description": None,
        "created": None,
        "modified": None,
        "last_modified_by": None,
    }
    if CORE_PROPERTIES_PART not in package.namelist():
        return result
    root = ET.fromstring(package.read(CORE_PROPERTIES_PART))
    paths = {
        "title": "./dc:title",
        "creator": "./dc:creator",
        "subject": "./dc:subject",
        "description": "./dc:description",
        "created": "./dcterms:created",
        "modified": "./dcterms:modified",
        "last_modified_by": "./cp:lastModifiedBy",
    }
    for key, path in paths.items():
        node = root.find(path, NS)
        if node is not None and node.text is not None:
            result[key] = node.text
    return result


def _manual_review_findings(element: ET.Element, block_id: str,
                            finding_counter: list[int]) -> list[dict[str, Any]]:
    findings: list[dict[str, Any]] = []

    def add(category: str, message: str, details: Mapping[str, Any] | None = None) -> None:
        finding_counter[0] += 1
        item: dict[str, Any] = {
            "finding_id": f"review-{finding_counter[0]:06d}",
            "block_id": block_id,
            "category": category,
            "message": message,
            "status": "needs-human-review",
        }
        if details:
            item["details"] = dict(details)
        findings.append(item)

    symbol_nodes = [node for node in element.iter() if node.tag == W + "sym"]
    for node in symbol_nodes:
        add(
            "legacy-symbol",
            "Legacy w:sym value was preserved as its declared code point; verify font semantics manually.",
            {"font": node.get(W + "font"), "char_hex": node.get(W + "char")},
        )
    math_count = sum(1 for node in element.iter() if node.tag in (M + "oMath", M + "oMathPara"))
    if math_count:
        add(
            "office-math",
            "Office Math content was reduced to available text nodes; verify formula structure manually.",
            {"element_count": math_count},
        )
    field_count = sum(1 for node in element.iter() if node.tag in (W + "instrText", W + "fldSimple"))
    if field_count:
        add(
            "field-code",
            "Field codes were not interpreted; only visible text runs were captured.",
            {"element_count": field_count},
        )
    tracked_insertions = sum(1 for node in element.iter() if node.tag == W + "ins")
    tracked_deletions = sum(1 for node in element.iter() if node.tag == W + "del")
    if tracked_insertions or tracked_deletions:
        add(
            "tracked-changes",
            "Tracked changes are present; visible w:t text was captured and deleted w:delText was excluded.",
            {"insertions": tracked_insertions, "deletions": tracked_deletions},
        )
    return findings


def _style_name(style_id: str | None, styles: Mapping[str, StyleInfo]) -> str | None:
    if style_id is None:
        return None
    info = styles.get(style_id)
    return info.name if info is not None else None


def normalize_docx(source_path: Path | str, output_dir: Path | str,
                   source_id: str) -> dict[str, Any]:
    """Normalize ``source_path`` into ``output_dir`` and return a summary."""
    source = Path(source_path)
    output = Path(output_dir)
    source_id = _validate_source_id(source_id)

    if not source.is_file():
        raise NormalizationError(f"DOCX source does not exist or is not a file: {source}")
    if source.suffix.lower() != ".docx":
        raise NormalizationError(f"expected a .docx source: {source}")

    source_stat_before = source.stat()
    source_sha_before = sha256_file(source)
    blocks: list[dict[str, Any]] = []
    structure_blocks: list[dict[str, Any]] = []
    headings: list[dict[str, Any]] = []
    media_assets: dict[str, dict[str, Any]] = {}
    media_occurrences: list[dict[str, Any]] = []
    warnings: list[dict[str, Any]] = []
    review_findings: list[dict[str, Any]] = []
    finding_counter = [0]
    used_media_names: set[str] = set()
    normalization_totals = {
        "blocks_changed": 0,
        "nfc_changed_blocks": 0,
        "crlf_converted": 0,
        "lone_cr_converted": 0,
        "unicode_spaces_replaced": 0,
        "redundant_spaces_removed": 0,
    }

    with zipfile.ZipFile(source, "r") as package:
        names = set(package.namelist())
        if DOCUMENT_PART not in names:
            raise NormalizationError(f"DOCX package is missing {DOCUMENT_PART}")
        try:
            document_root = ET.fromstring(package.read(DOCUMENT_PART))
        except ET.ParseError as exc:
            raise NormalizationError(f"cannot parse {DOCUMENT_PART}: {exc}") from exc

        body = document_root.find("./w:body", NS)
        if body is None:
            raise NormalizationError("DOCX document.xml has no w:body")

        styles = load_styles(package)
        relationships = load_relationships(package)
        properties = core_properties(package)
        _prepare_output_directory(output)
        heading_path: list[str] = []
        global_image_order = 0
        table_order = 0

        body_blocks = list(iter_body_blocks(body))
        for block_index, element in enumerate(body_blocks, start=1):
            block_type = "paragraph" if element.tag == W + "p" else "table"
            raw_text = extract_text(element) if block_type == "paragraph" else extract_table_text(element)
            normalized_text, normalization_stats = normalize_text(raw_text)
            text_sha256 = sha256_bytes(normalized_text.encode("utf-8"))
            short_content_hash = text_sha256[:12]
            block_id = f"{source_id}#b{block_index:06d}-{short_content_hash}"

            style_id: str | None
            style_name: str | None
            level: int | None = None
            table_id: str | None = None
            table_rows: int | None = None
            table_columns: int | None = None
            if block_type == "paragraph":
                style_id = paragraph_style_id(element)
                style_name = _style_name(style_id, styles)
                level = heading_level(element, style_id, styles)
                if level is not None and normalized_text:
                    heading_path = heading_path[:level - 1]
                    heading_path.append(normalized_text)
                    headings.append({
                        "block_id": block_id,
                        "block_index": block_index,
                        "level": level,
                        "title": normalized_text,
                        "heading_path": list(heading_path),
                    })
            else:
                style_id = table_style_id(element)
                style_name = None
                table_order += 1
                table_id = f"table-{table_order:06d}"
                table_rows, table_columns = table_dimensions(element)

            image_refs = image_relationship_references(element)
            figure_ids: list[str] = []
            occurrence_ids: list[str] = []
            for within_block_order, reference in enumerate(image_refs, start=1):
                global_image_order += 1
                occurrence_id = f"image-occurrence-{global_image_order:06d}"
                figure_id = f"figure-{global_image_order:06d}"
                relationship_id = reference["relationship_id"]
                relationship = relationships.get(relationship_id)
                figure_ids.append(figure_id)
                occurrence_ids.append(occurrence_id)
                occurrence: dict[str, Any] = {
                    "occurrence_id": occurrence_id,
                    "figure_id": figure_id,
                    "document_order": global_image_order,
                    "within_block_order": within_block_order,
                    "relationship_id": relationship_id,
                    "reference_kind": reference["reference_kind"],
                    "block_id": block_id,
                    "block_index": block_index,
                    "previous_block_id": None,
                    "next_block_id": None,
                    "asset_id": None,
                    "status": "unresolved",
                }
                if relationship is None:
                    warnings.append({
                        "warning_id": f"warning-{len(warnings) + 1:06d}",
                        "block_id": block_id,
                        "category": "missing-relationship",
                        "message": f"Image relationship {relationship_id} is not present in document.xml.rels.",
                    })
                else:
                    occurrence.update({
                        "relationship_type": relationship.relationship_type,
                        "relationship_target": relationship.target,
                        "target_mode": relationship.target_mode,
                        "package_path": relationship.package_path,
                    })
                    if relationship.is_external:
                        occurrence["status"] = "external-not-downloaded"
                        warnings.append({
                            "warning_id": f"warning-{len(warnings) + 1:06d}",
                            "block_id": block_id,
                            "category": "external-image",
                            "message": f"External image {relationship_id} was recorded but not downloaded.",
                        })
                    elif not relationship.is_image:
                        occurrence["status"] = "non-image-relationship"
                        warnings.append({
                            "warning_id": f"warning-{len(warnings) + 1:06d}",
                            "block_id": block_id,
                            "category": "unexpected-relationship-type",
                            "message": f"Relationship {relationship_id} is referenced as an image but has type {relationship.relationship_type}.",
                        })
                    elif relationship.package_path not in names:
                        occurrence["status"] = "missing-package-part"
                        warnings.append({
                            "warning_id": f"warning-{len(warnings) + 1:06d}",
                            "block_id": block_id,
                            "category": "missing-media-part",
                            "message": f"Image package part is missing: {relationship.package_path}.",
                        })
                    else:
                        asset_id = f"media-{relationship_id}"
                        occurrence["asset_id"] = asset_id
                        occurrence["status"] = "extracted"
                        if asset_id not in media_assets:
                            media_bytes = package.read(relationship.package_path)
                            filename = _safe_media_filename(
                                relationship.package_path, relationship_id, used_media_names
                            )
                            media_path = output / "media" / filename
                            media_path.write_bytes(media_bytes)
                            package_sha = sha256_bytes(media_bytes)
                            extracted_sha = sha256_file(media_path)
                            media_assets[asset_id] = {
                                "asset_id": asset_id,
                                "relationship_id": relationship_id,
                                "relationship_type": relationship.relationship_type,
                                "package_path": relationship.package_path,
                                "original_filename": PurePosixPath(relationship.package_path).name,
                                "extracted_path": f"media/{filename}",
                                "byte_size": len(media_bytes),
                                "sha256_in_package": package_sha,
                                "sha256_extracted": extracted_sha,
                                "byte_for_byte_preserved": package_sha == extracted_sha,
                                "pixel_dimensions": image_dimensions(media_bytes),
                            }
                media_occurrences.append(occurrence)

            locator: dict[str, Any] = {
                "source_id": source_id,
                "heading_path": list(heading_path),
                "ooxml_block_index": block_index,
                "content_hash": short_content_hash,
            }
            if figure_ids:
                locator["figure_ids"] = figure_ids
            if table_id is not None:
                locator["table_id"] = table_id

            block_record: dict[str, Any] = {
                "schema_version": 1,
                "source_id": source_id,
                "block_id": block_id,
                "block_type": block_type,
                "ooxml_block_index": block_index,
                "locator": locator,
                "heading_path": list(heading_path),
                "heading_level": level,
                "style_id": style_id,
                "style_name": style_name,
                "raw_text": raw_text,
                "normalized_text": normalized_text,
                "text_sha256": text_sha256,
                "short_content_hash": short_content_hash,
                "normalization": normalization_stats,
                "figure_ids": figure_ids,
                "media_occurrence_ids": occurrence_ids,
            }
            if table_id is not None:
                block_record["table_id"] = table_id
                block_record["table_rows"] = table_rows
                block_record["table_columns"] = table_columns
            blocks.append(block_record)

            structure_item: dict[str, Any] = {
                "block_id": block_id,
                "ooxml_block_index": block_index,
                "block_type": block_type,
                "heading_path": list(heading_path),
                "heading_level": level,
                "style_id": style_id,
                "style_name": style_name,
                "short_content_hash": short_content_hash,
                "raw_character_count": len(raw_text),
                "normalized_character_count": len(normalized_text),
                "is_empty_text": normalized_text == "",
                "figure_ids": figure_ids,
            }
            if table_id is not None:
                structure_item.update({
                    "table_id": table_id,
                    "table_rows": table_rows,
                    "table_columns": table_columns,
                })
            structure_blocks.append(structure_item)

            review_findings.extend(_manual_review_findings(element, block_id, finding_counter))
            if normalization_stats["changed"]:
                normalization_totals["blocks_changed"] += 1
            if normalization_stats["nfc_changed"]:
                normalization_totals["nfc_changed_blocks"] += 1
            for key in (
                "crlf_converted", "lone_cr_converted", "unicode_spaces_replaced",
                "redundant_spaces_removed",
            ):
                normalization_totals[key] += int(normalization_stats[key])

        block_ids = [block["block_id"] for block in blocks]
        for occurrence in media_occurrences:
            index = int(occurrence["block_index"]) - 1
            occurrence["previous_block_id"] = block_ids[index - 1] if index > 0 else None
            occurrence["next_block_id"] = block_ids[index + 1] if index + 1 < len(block_ids) else None

        section_property_count = sum(1 for node in document_root.iter() if node.tag == W + "sectPr")
        content_types = {
            "paragraphs": sum(block["block_type"] == "paragraph" for block in blocks),
            "tables": sum(block["block_type"] == "table" for block in blocks),
            "headings": len(headings),
            "image_occurrences": len(media_occurrences),
            "unique_image_assets": len(media_assets),
            "sections_detected": section_property_count,
        }

    structure = {
        "schema_version": 1,
        "tool": {"name": TOOL_NAME, "version": TOOL_VERSION},
        "source": {
            "source_id": source_id,
            "filename": source.name,
            "format": "docx",
            "sha256": source_sha_before,
            "core_properties": properties,
        },
        "locator_policy": {
            "canonical_components": [
                "source_id", "heading_path", "ooxml_block_index", "content_hash",
                "optional figure_id/table_id",
            ],
            "page_number_role": "auxiliary-only-not-extracted",
            "block_index_base": 1,
            "content_hash": "first 12 hexadecimal characters of SHA-256(normalized_text UTF-8)",
        },
        "summary": {"total_blocks": len(blocks), **content_types},
        "headings": headings,
        "blocks": structure_blocks,
    }
    media_map = {
        "schema_version": 1,
        "source_id": source_id,
        "policy": {
            "extraction": "original OOXML package bytes; no decoding or re-encoding",
            "resolution": "preserved byte-for-byte; raster dimensions reported when detectable",
            "external_media": "record-only; never downloaded",
        },
        "assets": list(media_assets.values()),
        "occurrences": media_occurrences,
        "warnings": warnings,
    }
    normalization_log = {
        "schema_version": 1,
        "source_id": source_id,
        "tool": {"name": TOOL_NAME, "version": TOOL_VERSION},
        "automatic_policy": {
            "unicode": "NFC",
            "line_endings": "CRLF and CR converted to LF",
            "horizontal_whitespace": "Unicode whitespace except TAB/LF mapped to ASCII space; consecutive ASCII spaces collapsed within TAB-delimited fields",
            "trimming": "leading/trailing ASCII spaces per TAB-delimited field and outer block whitespace removed",
        },
        "not_performed": [
            "OCR",
            "spelling or grammar correction",
            "scientific-symbol substitution",
            "professional-terminology correction",
            "semantic rewriting",
            "field-code evaluation",
            "equation reconstruction",
        ],
        "summary": {"total_blocks": len(blocks), **normalization_totals},
        "manual_review": review_findings,
        "warnings": warnings,
    }

    write_jsonl(output / "blocks.jsonl", blocks)
    write_yaml(output / "structure.yml", structure)
    write_yaml(output / "media-map.yml", media_map)
    write_yaml(output / "normalization-log.yml", normalization_log)

    source_sha_after = sha256_file(source)
    source_stat_after = source.stat()
    generated_files: list[dict[str, Any]] = []
    for path in sorted(
        [output / "blocks.jsonl", output / "structure.yml", output / "media-map.yml",
         output / "normalization-log.yml"] + list((output / "media").iterdir()),
        key=lambda item: item.relative_to(output).as_posix(),
    ):
        generated_files.append({
            "path": path.relative_to(output).as_posix(),
            "byte_size": path.stat().st_size,
            "sha256": sha256_file(path),
        })

    checksum_report = {
        "schema_version": 1,
        "source_id": source_id,
        "source": {
            "filename": source.name,
            "sha256_before": source_sha_before,
            "sha256_after": source_sha_after,
            "sha256_unchanged": source_sha_before == source_sha_after,
            "size_before": source_stat_before.st_size,
            "size_after": source_stat_after.st_size,
            "size_unchanged": source_stat_before.st_size == source_stat_after.st_size,
            "mtime_ns_before": source_stat_before.st_mtime_ns,
            "mtime_ns_after": source_stat_after.st_mtime_ns,
            "mtime_unchanged": source_stat_before.st_mtime_ns == source_stat_after.st_mtime_ns,
        },
        "generated_files": generated_files,
        "self_checksum": "excluded to avoid recursive checksum content",
    }
    write_yaml(output / "checksums.yml", checksum_report)

    if source_sha_before != source_sha_after:
        raise NormalizationError(
            "source checksum changed during normalization; outputs must not be trusted"
        )

    return {
        "source_id": source_id,
        "source_sha256": source_sha_before,
        "source_unchanged": True,
        "output_dir": str(output),
        "block_count": len(blocks),
        "heading_count": len(headings),
        "table_count": content_types["tables"],
        "image_occurrence_count": len(media_occurrences),
        "image_asset_count": len(media_assets),
        "manual_review_count": len(review_findings),
        "warning_count": len(warnings),
    }


def build_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(
        description=(
            "Create a deterministic, private normalization bundle from a DOCX. "
            "The source is opened read-only and the output directory must be new or empty."
        )
    )
    parser.add_argument("source_docx", type=Path, help="Path to the source DOCX (read-only)")
    parser.add_argument(
        "--source-id", required=True,
        help="Stable source_id used in block IDs and locators (normally from manifests/sources.yml)",
    )
    parser.add_argument(
        "--output-dir", required=True, type=Path,
        help="Explicit new or empty directory for the normalization bundle",
    )
    return parser


def main(argv: Sequence[str] | None = None) -> int:
    parser = build_parser()
    args = parser.parse_args(argv)
    try:
        summary = normalize_docx(args.source_docx, args.output_dir, args.source_id)
    except (NormalizationError, zipfile.BadZipFile, OSError) as exc:
        print(f"error: {exc}", file=sys.stderr)
        return 2
    print(json.dumps(summary, ensure_ascii=False, indent=2))
    return 0


if __name__ == "__main__":
    raise SystemExit(main())

#!/usr/bin/env python3
"""Run local, provenance-recorded OCR over every DOCX image or scanned-PDF page.

The runner never installs dependencies, invokes a shell, uploads data, or
modifies the source.  Tesseract and (for PDF) Poppler must already be present.
"""

from __future__ import annotations

import argparse
import csv
import hashlib
import io
import json
import re
import shutil
import subprocess
import sys
import unicodedata
from dataclasses import dataclass
from pathlib import Path, PurePosixPath
from typing import Any, Callable, Mapping, Sequence

try:
    import yaml
except ImportError as exc:  # pragma: no cover
    raise SystemExit("PyYAML is required; do not install it without approval.") from exc


TOOL_NAME = "ocr_book_source"
TOOL_VERSION = "1.0.0"
DEFAULT_DPI = 300


class OCRRunError(RuntimeError):
    def __init__(self, code: str, message: str, summary: Mapping[str, Any] | None = None):
        super().__init__(message)
        self.code = code
        self.message = message
        self.summary = dict(summary or {})


@dataclass(frozen=True)
class OCRItem:
    item_id: str
    input_path: Path | None
    output_relative_path: str
    context: Mapping[str, Any]
    preflight_error_code: str | None = None
    preflight_error_message: str | None = None


SUPPORTED_IMAGE_SUFFIXES = {
    ".bmp", ".gif", ".jpeg", ".jpg", ".pbm", ".pgm", ".png", ".pnm",
    ".ppm", ".tif", ".tiff", ".webp",
}


def sha256_bytes(data: bytes) -> str:
    return "sha256:" + hashlib.sha256(data).hexdigest()


def sha256_file(path: Path) -> str:
    digest = hashlib.sha256()
    with path.open("rb") as handle:
        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
            digest.update(chunk)
    return "sha256:" + digest.hexdigest()


def _load_yaml(path: Path) -> Mapping[str, Any]:
    try:
        value = yaml.safe_load(path.read_text(encoding="utf-8"))
    except (OSError, UnicodeError, yaml.YAMLError) as exc:
        raise OCRRunError("OCR_INPUT_INVALID", f"cannot read YAML {path}: {exc}") from exc
    if not isinstance(value, dict):
        raise OCRRunError("OCR_INPUT_INVALID", f"YAML root must be a mapping: {path}")
    return value


def _write_yaml(path: Path, value: Mapping[str, Any]) -> None:
    path.write_text(
        yaml.safe_dump(value, allow_unicode=True, sort_keys=False),
        encoding="utf-8",
        newline="\n",
    )


def _prepare_output(path: Path) -> None:
    if path.exists() and (not path.is_dir() or any(path.iterdir())):
        raise OCRRunError(
            "OCR_OUTPUT_NOT_EMPTY", f"refusing to overwrite non-empty output: {path}"
        )
    path.mkdir(parents=True, exist_ok=True)


def _manifest_source(path: Path, source_id: str) -> Mapping[str, Any]:
    document = _load_yaml(path)
    matches = [
        item for item in document.get("sources", [])
        if isinstance(item, dict) and item.get("id") == source_id
    ]
    if len(matches) != 1:
        raise OCRRunError(
            "OCR_SOURCE_MANIFEST_INVALID",
            f"source_id {source_id!r} must resolve exactly once in {path}",
        )
    return matches[0]


def _validate_source_ocr_policy(
    source: Mapping[str, Any],
    *,
    carrier: str,
    languages: Sequence[str],
) -> None:
    expected_type = "book-docx" if carrier == "docx-image" else "book-pdf-scan"
    expected_coverage = "all-images" if carrier == "docx-image" else "all-pages"
    policy = source.get("ocr_policy")
    if source.get("type") != expected_type or not isinstance(policy, dict):
        raise OCRRunError(
            "OCR_SOURCE_MANIFEST_INVALID",
            f"source must be {expected_type} with an explicit ocr_policy",
        )
    declared_languages = policy.get("languages")
    if (
        policy.get("required") is not True
        or policy.get("coverage") != expected_coverage
        or policy.get("execution_mode") != "local-only"
        or policy.get("engine") != "tesseract"
        or not isinstance(declared_languages, list)
        or any(not isinstance(item, str) for item in declared_languages)
        or declared_languages != list(languages)
    ):
        raise OCRRunError(
            "OCR_SOURCE_POLICY_MISMATCH",
            "command languages and adapter settings must exactly match the explicit source ocr_policy",
        )
    if carrier == "pdf-page" and (
        policy.get("renderer") != "poppler-pdftoppm"
        or policy.get("dpi") != DEFAULT_DPI
    ):
        raise OCRRunError(
            "PDF_RENDER_CONTRACT_INVALID",
            "scanned PDF source policy must declare poppler-pdftoppm at 300 DPI",
        )


def _languages(value: str) -> list[str]:
    result = [item.strip() for item in value.split("+") if item.strip()]
    if not result or len(result) != len(set(result)) or any(
        not re.fullmatch(r"[A-Za-z0-9_]+", item) for item in result
    ):
        raise OCRRunError("OCR_LANGUAGE_INVALID", "languages must be unique Tesseract IDs joined by +")
    return result


def _run(
    args: Sequence[str],
    *,
    runner: Callable[..., subprocess.CompletedProcess[str]] = subprocess.run,
) -> subprocess.CompletedProcess[str]:
    try:
        return runner(
            list(args),
            check=False,
            capture_output=True,
            text=True,
            shell=False,
        )
    except OSError as exc:
        raise OCRRunError("OCR_SUBPROCESS_FAILED", f"cannot execute {args[0]}: {exc}") from exc


def preflight_tools(
    languages: Sequence[str],
    *,
    pdf: bool,
    runner: Callable[..., subprocess.CompletedProcess[str]] = subprocess.run,
    which: Callable[[str], str | None] = shutil.which,
) -> dict[str, Any]:
    required = ["tesseract"] + (["pdfinfo", "pdftoppm"] if pdf else [])
    missing = [name for name in required if which(name) is None]
    if missing:
        code = "PDF_RENDERER_UNAVAILABLE" if any(x in missing for x in ("pdfinfo", "pdftoppm")) else "OCR_ENGINE_UNAVAILABLE"
        raise OCRRunError(code, f"required local executable(s) missing: {', '.join(missing)}")
    version = _run(["tesseract", "--version"], runner=runner)
    if version.returncode != 0:
        raise OCRRunError("OCR_ENGINE_UNAVAILABLE", version.stderr.strip() or "tesseract --version failed")
    listed = _run(["tesseract", "--list-langs"], runner=runner)
    if listed.returncode != 0:
        raise OCRRunError("OCR_ENGINE_UNAVAILABLE", listed.stderr.strip() or "tesseract --list-langs failed")
    installed = {
        line.strip() for line in listed.stdout.splitlines()
        if line.strip() and not line.lower().startswith("list of available")
    }
    absent = sorted(set(languages) - installed)
    if absent:
        raise OCRRunError("OCR_LANGUAGE_MISSING", f"Tesseract language packs missing: {', '.join(absent)}")
    result: dict[str, Any] = {
        "tesseract_version": version.stdout.splitlines()[0].strip(),
        "installed_languages": sorted(installed),
    }
    if pdf:
        poppler = _run(["pdftoppm", "-v"], runner=runner)
        if poppler.returncode not in {0, 1}:
            raise OCRRunError("PDF_RENDERER_UNAVAILABLE", poppler.stderr.strip() or "pdftoppm -v failed")
        pdfinfo = _run(["pdfinfo", "-v"], runner=runner)
        if pdfinfo.returncode not in {0, 1}:
            raise OCRRunError("PDF_RENDERER_UNAVAILABLE", pdfinfo.stderr.strip() or "pdfinfo -v failed")
        result["poppler_version"] = (poppler.stderr or poppler.stdout).splitlines()[0].strip()
        result["pdfinfo_version"] = (pdfinfo.stderr or pdfinfo.stdout).splitlines()[0].strip()
    return result


def _normalize_ocr_text(value: str) -> str:
    """Apply only the contract-permitted NFC and newline normalization."""
    return unicodedata.normalize("NFC", value.replace("\r\n", "\n").replace("\r", "\n"))


def _parse_tsv(tsv: str, record_id: str) -> tuple[str, list[dict[str, Any]]]:
    reader = csv.DictReader(io.StringIO(tsv), delimiter="\t")
    words: list[tuple[tuple[str, str, str, str], str]] = []
    regions: list[dict[str, Any]] = []
    for row in reader:
        text = _normalize_ocr_text(str(row.get("text") or ""))
        if not text.strip():
            continue
        try:
            left = int(row["left"]); top = int(row["top"])
            width = int(row["width"]); height = int(row["height"])
            confidence = float(row["conf"])
        except (KeyError, TypeError, ValueError) as exc:
            raise OCRRunError("OCR_OUTPUT_INVALID", f"invalid Tesseract TSV row: {exc}") from exc
        if width < 1 or height < 1:
            continue
        region_id = f"{record_id}-region-{len(regions) + 1:06d}"
        regions.append({
            "region_id": region_id,
            "bbox_px": [left, top, width, height],
            "text": text,
            "text_sha256": sha256_bytes(text.encode("utf-8")),
            "confidence": confidence,
            "confidence_raw": str(row.get("conf")),
        })
        line_key = (
            str(row.get("page_num")), str(row.get("block_num")),
            str(row.get("par_num")), str(row.get("line_num")),
        )
        words.append((line_key, text))
    lines: list[str] = []
    last_key = None
    for key, word in words:
        if key != last_key:
            lines.append(word)
            last_key = key
        else:
            lines[-1] += " " + word
    return _normalize_ocr_text("\n".join(lines)), regions


def _ocr_item(
    item: OCRItem,
    *,
    output_root: Path,
    languages: Sequence[str],
    psm: int,
    runner: Callable[..., subprocess.CompletedProcess[str]],
    record_number: int,
) -> dict[str, Any]:
    record_id = f"ocr-record-{record_number:06d}"
    destination = output_root.joinpath(*PurePosixPath(item.output_relative_path).parts)
    destination.parent.mkdir(parents=True, exist_ok=True)
    payload: bytes | None = None
    if item.input_path is not None and item.input_path.is_file():
        payload = item.input_path.read_bytes()
        if item.input_path != destination:
            destination.write_bytes(payload)
    image_hash = sha256_bytes(payload) if payload is not None else None
    base = {
        "ocr_record_id": record_id,
        "item_id": item.item_id,
        "image_path": item.output_relative_path if payload is not None else None,
        "image_sha256": image_hash,
        "context": dict(item.context),
    }
    if item.preflight_error_code is not None:
        return {
            **base,
            "status": "failed",
            "error_code": item.preflight_error_code,
            "error": item.preflight_error_message,
            "raw_text": "",
            "raw_text_sha256": sha256_bytes(b""),
            "normalized_text": "",
            "text_sha256": sha256_bytes(b""),
            "regions": [],
            "quality_flags": ["ocr-failed", "ocr-unreviewed"],
        }
    if payload is None:
        return {
            **base,
            "status": "failed",
            "error_code": "OCR_IMAGE_UNREADABLE",
            "error": "image/page bytes are unavailable or unreadable",
            "raw_text": "",
            "raw_text_sha256": sha256_bytes(b""),
            "normalized_text": "",
            "text_sha256": sha256_bytes(b""),
            "regions": [],
            "quality_flags": ["ocr-failed", "ocr-unreviewed"],
        }
    if destination.suffix.lower() not in SUPPORTED_IMAGE_SUFFIXES:
        return {
            **base,
            "status": "failed",
            "error_code": "OCR_IMAGE_FORMAT_UNSUPPORTED",
            "error": f"unsupported local OCR image format: {destination.suffix or '<none>'}",
            "raw_text": "",
            "raw_text_sha256": sha256_bytes(b""),
            "normalized_text": "",
            "text_sha256": sha256_bytes(b""),
            "regions": [],
            "quality_flags": ["ocr-failed", "ocr-unreviewed"],
        }
    plain = _run(
        [
            "tesseract", str(destination), "stdout", "-l", "+".join(languages),
            "--psm", str(psm),
        ],
        runner=runner,
    )
    if plain.returncode != 0:
        return {
            **base,
            "status": "failed",
            "error_code": "OCR_IMAGE_UNREADABLE",
            "error": plain.stderr.strip() or f"Tesseract text extraction exited {plain.returncode}",
            "raw_text": "",
            "raw_text_sha256": sha256_bytes(b""),
            "normalized_text": "",
            "text_sha256": sha256_bytes(b""),
            "regions": [],
            "quality_flags": ["ocr-failed", "ocr-unreviewed"],
        }
    completed = _run(
        [
            "tesseract", str(destination), "stdout", "-l", "+".join(languages),
            "--psm", str(psm), "tsv",
        ],
        runner=runner,
    )
    if completed.returncode != 0:
        return {
            **base,
            "status": "failed",
            "error_code": "OCR_TSV_UNAVAILABLE",
            "error": completed.stderr.strip() or f"Tesseract TSV extraction exited {completed.returncode}",
            "raw_text": plain.stdout,
            "raw_text_sha256": sha256_bytes(plain.stdout.encode("utf-8")),
            "normalized_text": _normalize_ocr_text(plain.stdout),
            "text_sha256": sha256_bytes(_normalize_ocr_text(plain.stdout).encode("utf-8")),
            "regions": [],
            "quality_flags": ["ocr-failed", "ocr-unreviewed"],
        }
    try:
        _tsv_text, regions = _parse_tsv(completed.stdout, record_id)
    except OCRRunError as exc:
        return {
            **base,
            "status": "failed", "error_code": exc.code, "error": exc.message,
            "raw_text": plain.stdout,
            "raw_text_sha256": sha256_bytes(plain.stdout.encode("utf-8")),
            "normalized_text": _normalize_ocr_text(plain.stdout),
            "text_sha256": sha256_bytes(_normalize_ocr_text(plain.stdout).encode("utf-8")),
            "regions": [],
            "quality_flags": ["ocr-failed", "ocr-unreviewed"],
        }
    raw_text = plain.stdout
    normalized_text = _normalize_ocr_text(raw_text)
    status = "completed" if regions or normalized_text.strip() else "empty"
    flags = ["ocr-unreviewed"] + (["ocr-empty"] if status == "empty" else [])
    return {
        **base,
        "status": status,
        "error_code": None,
        "error": None,
        "raw_text": raw_text,
        "raw_text_sha256": sha256_bytes(raw_text.encode("utf-8")),
        "normalized_text": normalized_text,
        "text_sha256": sha256_bytes(normalized_text.encode("utf-8")),
        "regions": regions,
        "quality_flags": flags,
    }


def _write_bundle(
    output: Path,
    *,
    source_id: str,
    carrier: str,
    source_hash: str,
    languages: Sequence[str],
    psm: int,
    tool_info: Mapping[str, Any],
    records: Sequence[Mapping[str, Any]],
    occurrence_count: int,
    unbound_occurrences: Sequence[str],
    renderer: Mapping[str, Any] | None,
    input_binding: Mapping[str, Any],
) -> dict[str, Any]:
    results_path = output / "ocr-results.jsonl"
    with results_path.open("w", encoding="utf-8", newline="\n") as handle:
        for record in records:
            handle.write(json.dumps(record, ensure_ascii=False, separators=(",", ":")) + "\n")
    counts = {status: sum(item.get("status") == status for item in records) for status in ("completed", "empty", "failed")}
    complete = counts["failed"] == 0 and not unbound_occurrences
    manifest = {
        "schema_version": 1,
        "ocr_run_id": f"ocr-{source_id}-v1",
        "source_id": source_id,
        "carrier": carrier,
        "source_sha256": source_hash,
        "input_binding": dict(input_binding),
        "scope": "all-images" if carrier == "docx-image" else "all-pages",
        "execution_mode": "local-only",
        "runner": {"name": TOOL_NAME, "version": TOOL_VERSION},
        "engine": {
            "name": "tesseract", "version": tool_info.get("tesseract_version"),
            "languages": list(languages), "page_segmentation_mode": psm,
        },
        "renderer": dict(renderer) if renderer else None,
        "coverage": {
            "discovered_items": len(records), "attempted_items": len(records),
            **counts, "occurrence_count": occurrence_count,
            "unbound_occurrence_ids": list(unbound_occurrences),
            "complete": complete,
        },
        "status": "completed" if complete else "blocked",
        "limitations": [
            "OCR captures visible text only; it does not infer arrows, diagram relations, formula semantics, or factual truth.",
            "Every OCR-derived evidence item remains ocr-unreviewed until explicit human review.",
        ],
    }
    _write_yaml(output / "ocr-manifest.yml", manifest)
    generated = []
    for path in sorted(p for p in output.rglob("*") if p.is_file() and p.name != "checksums.yml"):
        generated.append({
            "path": path.relative_to(output).as_posix(),
            "size": path.stat().st_size,
            "sha256": sha256_file(path),
        })
    _write_yaml(output / "checksums.yml", {
        "schema_version": 1, "source_id": source_id,
        "source_sha256": source_hash, "generated_files": generated,
    })
    return manifest


def _normalized_sha256(value: Any) -> str | None:
    if not isinstance(value, str):
        return None
    candidate = value if value.startswith("sha256:") else "sha256:" + value
    return candidate if re.fullmatch(r"sha256:[0-9a-f]{64}", candidate) else None


def _canonical_bundle_path(value: Any) -> str | None:
    if not isinstance(value, str) or not value or value != value.strip() or "\\" in value:
        return None
    path = PurePosixPath(value)
    if path.is_absolute() or path.as_posix() != value or any(
        part in {"", ".", ".."} for part in path.parts
    ):
        return None
    return value


def _verify_normalized_bundle(
    bundle: Path,
    *,
    source_id: str,
    manifest_checksum: Any,
) -> tuple[Mapping[str, Any], str, dict[str, Any], set[str]]:
    for required in ("media-map.yml", "checksums.yml"):
        path = bundle / required
        if not path.is_file() or path.is_symlink():
            raise OCRRunError(
                "OCR_NORMALIZED_BUNDLE_INVALID",
                f"verified normalized bundle file is missing or unsafe: {path}",
            )
    media_map = _load_yaml(bundle / "media-map.yml")
    checksums = _load_yaml(bundle / "checksums.yml")
    source = checksums.get("source")
    before = _normalized_sha256(source.get("sha256_before")) if isinstance(source, dict) else None
    after = _normalized_sha256(source.get("sha256_after")) if isinstance(source, dict) else None
    expected_source = _normalized_sha256(manifest_checksum)
    if (
        checksums.get("source_id") != source_id
        or media_map.get("source_id") != source_id
        or not isinstance(source, dict)
        or source.get("sha256_unchanged") is not True
        or before is None
        or before != after
        or before != expected_source
    ):
        raise OCRRunError(
            "OCR_SOURCE_CHECKSUM_MISMATCH",
            "normalized bundle source identity/checksum does not match the source manifest",
        )
    generated = checksums.get("generated_files")
    if not isinstance(generated, list):
        raise OCRRunError(
            "OCR_NORMALIZED_BUNDLE_INVALID",
            "normalized checksums.yml requires generated_files",
        )
    recorded: dict[str, dict[str, Any]] = {}
    for index, entry in enumerate(generated):
        if not isinstance(entry, dict):
            raise OCRRunError(
                "OCR_NORMALIZED_BUNDLE_INVALID",
                f"generated_files[{index}] must be a mapping",
            )
        relative = _canonical_bundle_path(entry.get("path"))
        if relative is None or relative in recorded:
            raise OCRRunError(
                "OCR_NORMALIZED_BUNDLE_INVALID",
                f"invalid/duplicate normalized generated path at index {index}",
            )
        path = bundle.joinpath(*PurePosixPath(relative).parts)
        if not path.is_file() or path.is_symlink():
            raise OCRRunError(
                "OCR_NORMALIZED_BUNDLE_INVALID",
                f"normalized generated file is missing or unsafe: {relative}",
            )
        actual_hash = sha256_file(path)
        expected_hash = _normalized_sha256(entry.get("sha256"))
        if expected_hash != actual_hash or entry.get("byte_size") != path.stat().st_size:
            raise OCRRunError(
                "OCR_NORMALIZED_BUNDLE_HASH_MISMATCH",
                f"normalized generated file drift: {relative}",
            )
        recorded[relative] = dict(entry)
    if "media-map.yml" not in recorded:
        raise OCRRunError(
            "OCR_NORMALIZED_BUNDLE_INVALID",
            "normalized bundle does not integrity-bind media-map.yml",
        )
    return media_map, before, {
        "normalization_checksums_sha256": sha256_file(bundle / "checksums.yml"),
        "media_map_sha256": sha256_file(bundle / "media-map.yml"),
    }, set(recorded)


def run_docx_bundle(
    bundle: Path | str,
    output_dir: Path | str,
    *,
    source_id: str,
    sources_manifest: Path | str,
    languages: Sequence[str],
    runner: Callable[..., subprocess.CompletedProcess[str]] = subprocess.run,
    which: Callable[[str], str | None] = shutil.which,
) -> dict[str, Any]:
    bundle = Path(bundle); output = Path(output_dir)
    source = _manifest_source(Path(sources_manifest), source_id)
    _validate_source_ocr_policy(
        source, carrier="docx-image", languages=languages
    )
    media_map, source_hash, input_binding, normalized_paths = _verify_normalized_bundle(
        bundle, source_id=source_id, manifest_checksum=source.get("checksum")
    )
    tool_info = preflight_tools(languages, pdf=False, runner=runner, which=which)
    raw_assets = media_map.get("assets")
    raw_occurrences = media_map.get("occurrences")
    if (
        not isinstance(raw_assets, list)
        or any(not isinstance(item, dict) for item in raw_assets)
        or not isinstance(raw_occurrences, list)
        or any(not isinstance(item, dict) for item in raw_occurrences)
    ):
        raise OCRRunError(
            "OCR_NORMALIZED_BUNDLE_INVALID",
            "media-map assets/occurrences must be complete mapping lists",
        )
    assets = list(raw_assets)
    occurrences = list(raw_occurrences)
    asset_occurrences: dict[str, list[Mapping[str, Any]]] = {}
    unbound: list[str] = []
    seen_occurrence_ids: set[str] = set()
    for occurrence in occurrences:
        asset_id = occurrence.get("asset_id")
        occurrence_id = occurrence.get("occurrence_id")
        if (
            not isinstance(occurrence_id, str)
            or not occurrence_id
            or occurrence_id in seen_occurrence_ids
        ):
            raise OCRRunError(
                "OCR_NORMALIZED_BUNDLE_INVALID",
                "every media occurrence requires a unique stable occurrence_id",
            )
        seen_occurrence_ids.add(occurrence_id)
        if not isinstance(asset_id, str) or not asset_id:
            unbound.append(occurrence_id)
        else:
            asset_occurrences.setdefault(asset_id, []).append(occurrence)
    items: list[OCRItem] = []
    seen_asset_ids: set[str] = set()
    declared_media_paths: set[str] = set()
    for asset in assets:
        asset_id = asset.get("asset_id")
        relative = asset.get("extracted_path")
        canonical_relative = _canonical_bundle_path(relative)
        if (
            not isinstance(asset_id, str)
            or not asset_id
            or asset_id in seen_asset_ids
            or canonical_relative is None
        ):
            raise OCRRunError(
                "OCR_NORMALIZED_BUNDLE_INVALID",
                "every media asset requires a unique ID and canonical extracted_path",
            )
        seen_asset_ids.add(asset_id)
        declared_media_paths.add(canonical_relative)
        input_path = bundle.joinpath(*PurePosixPath(canonical_relative).parts)
        if canonical_relative not in normalized_paths:
            raise OCRRunError(
                "OCR_NORMALIZED_BUNDLE_INVALID",
                f"media asset is not integrity-bound by normalized checksums: {canonical_relative}",
            )
        if not input_path.is_file() or input_path.is_symlink():
            unbound.extend(str(item.get("occurrence_id")) for item in asset_occurrences.get(asset_id, []))
            items.append(OCRItem(
                asset_id, None, f"images/{asset_id}{input_path.suffix.lower() or '.bin'}",
                {
                    "figure_ids": [str(item.get("figure_id")) for item in asset_occurrences.get(asset_id, [])],
                    "media_occurrence_ids": [str(item.get("occurrence_id")) for item in asset_occurrences.get(asset_id, [])],
                },
                "OCR_IMAGE_UNREADABLE",
                f"normalized media file is missing or unsafe: {canonical_relative}",
            ))
            continue
        expected = _normalized_sha256(asset.get("sha256_extracted"))
        if sha256_file(input_path) != expected:
            raise OCRRunError("OCR_IMAGE_HASH_MISMATCH", f"media hash drift: {canonical_relative}")
        suffix = input_path.suffix.lower() or ".bin"
        items.append(OCRItem(
            asset_id, input_path, f"images/{asset_id}{suffix}",
            {
                "figure_ids": [str(item.get("figure_id")) for item in asset_occurrences.get(asset_id, [])],
                "media_occurrence_ids": [str(item.get("occurrence_id")) for item in asset_occurrences.get(asset_id, [])],
            },
        ))
    unknown_asset_ids = set(asset_occurrences) - seen_asset_ids
    for asset_id in sorted(unknown_asset_ids):
        unbound.extend(
            str(item.get("occurrence_id"))
            for item in asset_occurrences.get(asset_id, [])
        )
    integrity_bound_media_paths = {
        path for path in normalized_paths if path.startswith("media/")
    }
    if declared_media_paths != integrity_bound_media_paths:
        raise OCRRunError(
            "OCR_NORMALIZED_BUNDLE_INVALID",
            "media-map assets must exactly cover every integrity-bound extracted media file",
        )
    input_binding = {
        **input_binding,
        "normalized_bundle_source_id": source_id,
        "normalized_asset_count": len(assets),
        "normalized_occurrence_count": len(occurrences),
    }
    _prepare_output(output)
    records = [
        _ocr_item(item, output_root=output, languages=languages, psm=11, runner=runner, record_number=index)
        for index, item in enumerate(items, 1)
    ]
    manifest = _write_bundle(
        output, source_id=source_id, carrier="docx-image", source_hash=source_hash,
        languages=languages, psm=11, tool_info=tool_info, records=records,
        occurrence_count=len(occurrences), unbound_occurrences=unbound, renderer=None,
        input_binding=input_binding,
    )
    if manifest["status"] != "completed":
        raise OCRRunError("OCR_COVERAGE_INCOMPLETE", "one or more DOCX images could not be OCR-scanned", manifest)
    return manifest


def _pdf_pages(
    source: Path,
    output: Path,
    *,
    dpi: int,
    runner: Callable[..., subprocess.CompletedProcess[str]],
) -> tuple[list[OCRItem], int]:
    info = _run(["pdfinfo", str(source)], runner=runner)
    if info.returncode != 0:
        raise OCRRunError("PDF_INPUT_INVALID", info.stderr.strip() or "pdfinfo failed")
    match = re.search(r"(?m)^Pages:\s*([1-9][0-9]*)\s*$", info.stdout)
    if match is None:
        raise OCRRunError("PDF_INPUT_INVALID", "pdfinfo did not report a positive page count")
    page_count = int(match.group(1))
    pages_dir = output / "pages"; pages_dir.mkdir(parents=True)
    items: list[OCRItem] = []
    for page_number in range(1, page_count + 1):
        stem = pages_dir / f"page-{page_number:06d}"
        rendered = _run([
            "pdftoppm", "-f", str(page_number), "-l", str(page_number),
            "-singlefile", "-r", str(dpi), "-png", str(source), str(stem),
        ], runner=runner)
        page_path = stem.with_suffix(".png")
        if rendered.returncode != 0 or not page_path.is_file():
            items.append(OCRItem(
                f"page-{page_number:06d}",
                page_path if page_path.is_file() and not page_path.is_symlink() else None,
                f"pages/page-{page_number:06d}.png",
                {"page_number": page_number, "dpi": dpi},
                "PDF_RENDER_FAILED",
                rendered.stderr.strip() or f"page {page_number} was not rendered",
            ))
            continue
        items.append(OCRItem(
            f"page-{page_number:06d}", page_path,
            f"pages/page-{page_number:06d}.png",
            {"page_number": page_number, "dpi": dpi},
        ))
    return items, page_count


def run_scanned_pdf(
    source_path: Path | str,
    output_dir: Path | str,
    *,
    source_id: str,
    sources_manifest: Path | str,
    languages: Sequence[str],
    dpi: int = DEFAULT_DPI,
    runner: Callable[..., subprocess.CompletedProcess[str]] = subprocess.run,
    which: Callable[[str], str | None] = shutil.which,
) -> dict[str, Any]:
    if dpi != DEFAULT_DPI:
        raise OCRRunError("PDF_RENDER_CONTRACT_INVALID", f"scanned PDF rendering is fixed at {DEFAULT_DPI} DPI")
    source_path = Path(source_path); output = Path(output_dir)
    source = _manifest_source(Path(sources_manifest), source_id)
    _validate_source_ocr_policy(source, carrier="pdf-page", languages=languages)
    tool_info = preflight_tools(languages, pdf=True, runner=runner, which=which)
    before = sha256_file(source_path)
    if before != source.get("checksum"):
        raise OCRRunError("OCR_SOURCE_CHECKSUM_MISMATCH", "PDF bytes do not match the sources manifest")
    _prepare_output(output)
    items, page_count = _pdf_pages(source_path, output, dpi=dpi, runner=runner)
    records = [
        _ocr_item(item, output_root=output, languages=languages, psm=3, runner=runner, record_number=index)
        for index, item in enumerate(items, 1)
    ]
    after = sha256_file(source_path)
    if before != after:
        raise OCRRunError("OCR_SOURCE_CHANGED", "PDF changed during OCR")
    manifest = _write_bundle(
        output, source_id=source_id, carrier="pdf-page", source_hash=before,
        languages=languages, psm=3, tool_info=tool_info, records=records,
        occurrence_count=len(items), unbound_occurrences=[],
        renderer={
            "name": "poppler-pdftoppm",
            "pdftoppm_version": tool_info.get("poppler_version"),
            "pdfinfo_version": tool_info.get("pdfinfo_version"),
            "dpi": dpi,
            "format": "png",
        },
        input_binding={"pdfinfo_page_count": page_count},
    )
    if manifest["status"] != "completed":
        raise OCRRunError("OCR_COVERAGE_INCOMPLETE", "one or more PDF pages failed OCR", manifest)
    return manifest


def main(argv: Sequence[str] | None = None) -> int:
    parser = argparse.ArgumentParser(description="Run local OCR over all DOCX images or scanned-PDF pages.")
    sub = parser.add_subparsers(dest="mode", required=True)
    for name in ("docx-bundle", "scanned-pdf"):
        item = sub.add_parser(name)
        item.add_argument("input")
        item.add_argument("--source-id", required=True)
        item.add_argument("--sources-manifest", required=True)
        item.add_argument("--output-dir", required=True)
        item.add_argument("--languages", required=True, help="Explicit Tesseract IDs, e.g. chi_sim+eng")
    args = parser.parse_args(argv)
    try:
        languages = _languages(args.languages)
        if args.mode == "docx-bundle":
            summary = run_docx_bundle(
                args.input, args.output_dir, source_id=args.source_id,
                sources_manifest=args.sources_manifest, languages=languages,
            )
        else:
            summary = run_scanned_pdf(
                args.input, args.output_dir, source_id=args.source_id,
                sources_manifest=args.sources_manifest, languages=languages,
            )
    except OCRRunError as exc:
        print(json.dumps({"ok": False, "code": exc.code, "message": exc.message, "summary": exc.summary}, ensure_ascii=False, indent=2))
        return 2
    except (OSError, UnicodeError) as exc:
        print(json.dumps({
            "ok": False,
            "code": "OCR_IO_ERROR",
            "message": str(exc),
            "summary": {},
        }, ensure_ascii=False, indent=2))
        return 2
    print(json.dumps({"ok": True, "summary": summary}, ensure_ascii=False, indent=2))
    return 0


if __name__ == "__main__":
    sys.exit(main())

#!/usr/bin/env python3
"""Deterministic product-task contract and drift checks.

This module deliberately checks explicit IDs, mappings, and byte hashes.  It
does not attempt fuzzy or LLM-based semantic comparison.  Remaining semantic
conflicts belong in the human Gate review package.
"""

from __future__ import annotations

import hashlib
import json
import re
from dataclasses import dataclass, field
from pathlib import Path, PurePosixPath
from typing import Any, Mapping, Sequence

try:
    import yaml
except ImportError as exc:  # pragma: no cover
    raise SystemExit("PyYAML is required; do not install it without approval.") from exc


ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]*$")
FULL_SHA256_RE = re.compile(r"^sha256:[0-9a-f]{64}$")
GATE1_TASK_CONTRACT = "gate1-task-contract-snapshot:v1"
GATE3_TASK_CONTRACT = "gate3-approval-snapshot:v2"
GATE3_LEGACY_CONTRACT = "gate3-approval-snapshot:v1"
TASK_MODES = {"source-contained", "method-transfer"}
TASK_STATUSES = {"active", "deferred", "rejected"}
COVERAGE_STATUSES = {"covered", "deferred", "rejected"}
PROVENANCE_LAYERS = {
    "method-source-evidence",
    "target-material-evidence",
    "analogy-hypothesis",
}
METHOD_RUBRIC_DIMENSIONS = {
    "provenance-layer-separation",
    "anti-forced-analogy",
}
METHOD_FATAL_FAILURE = "method-source-fact-as-target-fact"
UNFAMILIARITY_DIMENSIONS = {"domain", "case", "mechanism", "method"}


class _UniqueKeySafeLoader(yaml.SafeLoader):
    pass


def _construct_unique_mapping(loader, node, deep=False):
    loader.flatten_mapping(node)
    result = {}
    for key_node, value_node in node.value:
        key = loader.construct_object(key_node, deep=deep)
        if key in result:
            raise yaml.constructor.ConstructorError(
                "while constructing a mapping",
                node.start_mark,
                f"duplicate mapping key {key!r}",
                key_node.start_mark,
            )
        result[key] = loader.construct_object(value_node, deep=deep)
    return result


_UniqueKeySafeLoader.add_constructor(
    yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, _construct_unique_mapping
)


@dataclass(frozen=True)
class TaskIssue:
    code: str
    path: str
    message: str

    def as_dict(self) -> dict[str, str]:
        return {"code": self.code, "path": self.path, "message": self.message}


@dataclass
class TaskGovernanceResult:
    summary: dict[str, Any] = field(default_factory=dict)
    errors: list[TaskIssue] = field(default_factory=list)
    warnings: list[TaskIssue] = field(default_factory=list)

    @property
    def ok(self) -> bool:
        return not self.errors

    def error(self, code: str, path: str, message: str) -> None:
        self.errors.append(TaskIssue(code, path, message))

    def warning(self, code: str, path: str, message: str) -> None:
        self.warnings.append(TaskIssue(code, path, message))


def _load_yaml(path: Path) -> Mapping[str, Any]:
    value = yaml.load(path.read_text(encoding="utf-8"), Loader=_UniqueKeySafeLoader)
    if not isinstance(value, dict):
        raise ValueError("YAML root must be a mapping")
    return value


def _load_json(path: Path) -> Mapping[str, Any]:
    def unique(pairs):
        result = {}
        for key, value in pairs:
            if key in result:
                raise ValueError(f"duplicate object key {key!r}")
            result[key] = value
        return result

    value = json.loads(
        path.read_text(encoding="utf-8"),
        object_pairs_hook=unique,
        parse_constant=lambda value: (_ for _ in ()).throw(
            ValueError(f"non-JSON numeric constant {value!r}")
        ),
    )
    if not isinstance(value, dict):
        raise ValueError("JSON root must be an object")
    return value


def _records(document: Mapping[str, Any], key: str) -> list[Mapping[str, Any]]:
    value = document.get(key)
    if not isinstance(value, list):
        return []
    return [item for item in value if isinstance(item, dict)]


def _strings(value: Any, *, nonempty: bool = False) -> list[str] | None:
    if not isinstance(value, list) or any(
        not isinstance(item, str) or not item or item != item.strip() for item in value
    ):
        return None
    if len(value) != len(set(value)) or (nonempty and not value):
        return None
    return list(value)


def _id(value: Any) -> bool:
    return isinstance(value, str) and bool(ID_RE.fullmatch(value))


def _canonical_relative_path(value: Any) -> str | None:
    if not isinstance(value, str) or not value or value != value.strip():
        return None
    if "\\" in value or "\x00" in value:
        return None
    parts = value.split("/")
    if any(part in {"", ".", ".."} for part in parts):
        return None
    path = PurePosixPath(value)
    if path.is_absolute() or path.as_posix() != value:
        return None
    return value


def _sha256(path: Path) -> str:
    return "sha256:" + hashlib.sha256(path.read_bytes()).hexdigest()


def _contract_path_version(relative: str) -> int | None:
    if relative == "task-contract.yml":
        return 1
    match = re.fullmatch(r"task-contract\.v([2-9][0-9]*)\.yml", relative)
    return int(match.group(1)) if match is not None else None


def _current_decisions(
    decisions: Sequence[Mapping[str, Any]], gate: str, candidate_id: str | None
) -> list[Mapping[str, Any]]:
    return [
        item
        for item in decisions
        if item.get("gate") == gate
        and item.get("candidate_id") == candidate_id
        and item.get("is_current") is True
    ]


def _validate_contract(
    result: TaskGovernanceResult,
    contract: Mapping[str, Any],
    *,
    expected_distillation_id: Any,
) -> tuple[
    dict[str, Mapping[str, Any]],
    list[str],
    dict[str, dict[str, str]],
    dict[str, str],
]:
    path = "task-contract"
    if contract.get("schema_version") != 1 or isinstance(
        contract.get("schema_version"), bool
    ):
        result.error("TASK_CONTRACT_INVALID", f"{path}.schema_version", "must be integer 1")
    if contract.get("distillation_id") != expected_distillation_id:
        result.error(
            "TASK_CONTRACT_INVALID",
            f"{path}.distillation_id",
            "must match the governance distillation_id",
        )
    for field_name in ("task_contract_id", "product_goal"):
        if not _id(contract.get(field_name)) if field_name.endswith("_id") else not isinstance(contract.get(field_name), str) or not contract.get(field_name).strip():
            result.error("TASK_CONTRACT_INVALID", f"{path}.{field_name}", "must be non-empty")
    version = contract.get("contract_version")
    if not isinstance(version, int) or isinstance(version, bool) or version < 1:
        result.error("TASK_CONTRACT_INVALID", f"{path}.contract_version", "must be a positive integer")
    if contract.get("status") not in {"draft", "frozen"}:
        result.error("TASK_CONTRACT_INVALID", f"{path}.status", "must be draft or frozen")
    if _strings(contract.get("audience"), nonempty=True) is None:
        result.error("TASK_CONTRACT_INVALID", f"{path}.audience", "must be a unique non-empty string list")

    input_types: dict[str, str] = {}
    for index, record in enumerate(_records(contract, "input_types")):
        item_path = f"{path}.input_types[{index}]"
        input_id = record.get("input_type_id")
        if not _id(input_id) or input_id in input_types:
            result.error("TASK_CONTRACT_INVALID", f"{item_path}.input_type_id", "must be a unique stable ID")
            continue
        description = record.get("description")
        if not isinstance(description, str) or not description.strip():
            result.error("TASK_CONTRACT_INVALID", f"{item_path}.description", "must be non-empty")
        input_types[input_id] = str(record.get("provenance_role") or "")
    if not input_types:
        result.error("TASK_CONTRACT_INVALID", f"{path}.input_types", "at least one input type is required")

    capability = contract.get("execution_capability")
    if capability is not None:
        cap_path = f"{path}.execution_capability"
        if not isinstance(capability, dict):
            result.error("TASK_CONTRACT_INVALID", cap_path, "must be a mapping when present")
        else:
            input_handling = capability.get("input_handling")
            if not isinstance(input_handling, list) or not input_handling:
                result.error(
                    "TASK_CONTRACT_INVALID",
                    f"{cap_path}.input_handling",
                    "execution_capability requires a non-empty input_handling list when declared",
                )
            else:
                for cap_index, cap_record in enumerate(input_handling):
                    item_path = f"{cap_path}.input_handling[{cap_index}]"
                    if not isinstance(cap_record, dict):
                        result.error("TASK_CONTRACT_INVALID", item_path, "must be a mapping")
                        continue
                    input_id = cap_record.get("input_type_id")
                    if not _id(input_id) or input_id not in input_types:
                        result.error(
                            "TASK_CONTRACT_INVALID",
                            f"{item_path}.input_type_id",
                            "must reference a declared input type",
                        )
                    for field_name in ("carrier", "modality_strategy", "degradation_rule"):
                        value = cap_record.get(field_name)
                        if not isinstance(value, str) or not value.strip():
                            result.error(
                                "TASK_CONTRACT_INVALID",
                                f"{item_path}.{field_name}",
                                "must be a non-empty string",
                            )
                    modality = cap_record.get("modality_strategy")
                    if isinstance(modality, str) and modality not in {
                        "unimodal-text",
                        "multimodal-text-image",
                    }:
                        result.error(
                            "TASK_CONTRACT_INVALID",
                            f"{item_path}.modality_strategy",
                            "must be unimodal-text or multimodal-text-image",
                        )
            notes = capability.get("notes")
            if notes is not None and (not isinstance(notes, str) or not notes.strip()):
                result.error(
                    "TASK_CONTRACT_INVALID",
                    f"{cap_path}.notes",
                    "must be a non-empty string when present",
                )

    acceptance_questions: dict[str, str] = {}
    for index, record in enumerate(_records(contract, "acceptance_questions")):
        item_path = f"{path}.acceptance_questions[{index}]"
        question_id = record.get("acceptance_question_id")
        if not _id(question_id) or question_id in acceptance_questions:
            result.error("TASK_CONTRACT_INVALID", f"{item_path}.acceptance_question_id", "must be a unique stable ID")
            continue
        question = record.get("question")
        if not isinstance(question, str) or not question.strip():
            result.error("TASK_CONTRACT_INVALID", f"{item_path}.question", "must be non-empty")
        acceptance_questions[question_id] = str(question or "")

    tasks: dict[str, Mapping[str, Any]] = {}
    active: list[str] = []
    example_index: dict[str, dict[str, str]] = {}
    for index, record in enumerate(_records(contract, "stable_tasks")):
        item_path = f"{path}.stable_tasks[{index}]"
        task_id = record.get("stable_task_id")
        if not _id(task_id) or task_id in tasks:
            result.error("TASK_CONTRACT_INVALID", f"{item_path}.stable_task_id", "must be a unique stable ID")
            continue
        tasks[task_id] = record
        if record.get("task_mode") not in TASK_MODES:
            result.error("TASK_CONTRACT_INVALID", f"{item_path}.task_mode", "unsupported task mode")
        status = record.get("status")
        if status not in TASK_STATUSES:
            result.error("TASK_CONTRACT_INVALID", f"{item_path}.status", "unsupported task status")
        elif status == "active":
            active.append(task_id)
        for field_name in ("statement",):
            if not isinstance(record.get(field_name), str) or not record[field_name].strip():
                result.error("TASK_CONTRACT_INVALID", f"{item_path}.{field_name}", "must be non-empty")
        for field_name in (
            "required_input_types",
            "required_outputs",
            "non_negotiable_constraints",
            "acceptance_question_ids",
        ):
            values = _strings(record.get(field_name), nonempty=True)
            if values is None:
                result.error("TASK_CONTRACT_INVALID", f"{item_path}.{field_name}", "must be a unique non-empty string list")
                continue
            if field_name == "required_input_types":
                for value in values:
                    if value not in input_types:
                        result.error("TASK_CONTRACT_INVALID", f"{item_path}.{field_name}", f"unknown input type {value!r}")
            if field_name == "acceptance_question_ids":
                for value in values:
                    if value not in acceptance_questions:
                        result.error("TASK_CONTRACT_INVALID", f"{item_path}.{field_name}", f"unknown acceptance question {value!r}")
        for examples_key, polarity in (("positive_examples", "positive"), ("negative_examples", "negative")):
            examples = _records(record, examples_key)
            if not examples:
                result.error("TASK_CONTRACT_INVALID", f"{item_path}.{examples_key}", "at least one stable example is required")
            for example_number, example in enumerate(examples):
                example_path = f"{item_path}.{examples_key}[{example_number}]"
                example_id = example.get("example_id")
                if not _id(example_id) or example_id in example_index:
                    result.error("TASK_CONTRACT_INVALID", f"{example_path}.example_id", "must be globally unique")
                    continue
                if example.get("input_type_id") not in input_types:
                    result.error("TASK_CONTRACT_INVALID", f"{example_path}.input_type_id", "must reference an input type")
                if not isinstance(example.get("statement"), str) or not example["statement"].strip():
                    result.error("TASK_CONTRACT_INVALID", f"{example_path}.statement", "must be non-empty")
                example_index[example_id] = {
                    "polarity": polarity,
                    "stable_task_id": task_id,
                    "input_type_id": str(example.get("input_type_id") or ""),
                }
        provenance = record.get("provenance_requirements")
        if not isinstance(provenance, dict):
            result.error(
                "TASK_CONTRACT_INVALID",
                f"{item_path}.provenance_requirements",
                "every stable task requires a structured provenance contract",
            )
        else:
            layers = _strings(provenance.get("required_output_layers"), nonempty=True)
            if layers is None:
                result.error(
                    "TASK_CONTRACT_INVALID",
                    f"{item_path}.provenance_requirements.required_output_layers",
                    "must be a unique non-empty stable layer list",
                )
            if provenance.get("missing_target_evidence") not in {
                "stop", "lower-conclusion-strength"
            }:
                result.error(
                    "TASK_CONTRACT_INVALID",
                    f"{item_path}.provenance_requirements.missing_target_evidence",
                    "must stop or lower conclusion strength",
                )
            forbidden = _strings(provenance.get("forbidden_transfers"))
            if forbidden is None:
                result.error(
                    "TASK_CONTRACT_INVALID",
                    f"{item_path}.provenance_requirements.forbidden_transfers",
                    "must be a unique stable ID list",
                )
            if record.get("task_mode") == "method-transfer":
                layers = _strings(provenance.get("required_output_layers"), nonempty=True)
                if layers is None or set(layers) != PROVENANCE_LAYERS:
                    result.error("METHOD_TRANSFER_PROVENANCE_REQUIRED", f"{item_path}.provenance_requirements.required_output_layers", "must contain exactly the three provenance layers")
                if provenance.get("target_source_role") != "target-material":
                    result.error("METHOD_TRANSFER_PROVENANCE_REQUIRED", f"{item_path}.provenance_requirements.target_source_role", "must be target-material")
                if provenance.get("missing_target_evidence") not in {"stop", "lower-conclusion-strength"}:
                    result.error("METHOD_TRANSFER_PROVENANCE_REQUIRED", f"{item_path}.provenance_requirements.missing_target_evidence", "must stop or lower conclusion strength")
                forbidden = _strings(provenance.get("forbidden_transfers"), nonempty=True)
                if forbidden is None or METHOD_FATAL_FAILURE not in forbidden:
                    result.error("METHOD_TRANSFER_PROVENANCE_REQUIRED", f"{item_path}.provenance_requirements.forbidden_transfers", f"must include {METHOD_FATAL_FAILURE}")
    if not tasks:
        result.error("TASK_CONTRACT_INVALID", f"{path}.stable_tasks", "at least one stable task is required")
    exclusions: set[str] = set()
    for index, record in enumerate(_records(contract, "exclusions")):
        item_path = f"{path}.exclusions[{index}]"
        exclusion_id = record.get("exclusion_id")
        if not _id(exclusion_id) or exclusion_id in exclusions:
            result.error(
                "TASK_CONTRACT_INVALID",
                f"{item_path}.exclusion_id",
                "must be a unique stable ID",
            )
            continue
        exclusions.add(exclusion_id)
        if not isinstance(record.get("statement"), str) or not record["statement"].strip():
            result.error(
                "TASK_CONTRACT_INVALID",
                f"{item_path}.statement",
                "must be non-empty",
            )
    if not exclusions:
        result.error(
            "TASK_CONTRACT_INVALID",
            f"{path}.exclusions",
            "at least one explicit exclusion is required",
        )
    return tasks, active, example_index, input_types


def inspect_task_governance(
    root: Path | str,
    sources_manifest: Path | str | None = None,
) -> TaskGovernanceResult:
    root = Path(root)
    result = TaskGovernanceResult(summary={
        "status": "unknown",
        "task_contract_path": None,
        "task_contract_hash": None,
        "task_contract_id": None,
        "contract_version": None,
        "current_gate1_decision_id": None,
        "active_stable_task_ids": [],
        "covered_stable_task_ids": [],
        "uncovered_stable_task_ids": [],
        "deferred_stable_task_ids": [],
        "rejected_stable_task_ids": [],
        "candidate_stable_task_ids": {},
        "task_contract_drift": "unknown",
        "gate3_contract_binding": "not-applicable",
        "legacy_contract_review_required": False,
        "current_stage_objective": None,
        "temporary_operational_constraints": [],
    })
    try:
        capability = _load_yaml(root / "capability-rules.yml")
        gates = _load_yaml(root / "gate-decisions.yml")
        evidence = _load_yaml(root / "evidence-ledger.yml")
        eval_runs_document = _load_yaml(root / "eval-runs.yml")
    except (OSError, UnicodeError, yaml.YAMLError, ValueError) as exc:
        result.error("TASK_CONTRACT_INVALID", str(root), f"cannot load governing files: {exc}")
        result.summary["status"] = "invalid"
        return result
    decisions = _records(gates, "gate_decisions")

    # Every previously Gate-1-bound contract is an immutable audit anchor.
    # A newer current Gate 1 selects the current contract but never permits an
    # older versioned file to disappear or drift.
    historical_contract_paths: dict[int, str] = {}
    for index, decision in enumerate(decisions):
        if decision.get("gate") != "gate-1":
            continue
        historical_snapshot = decision.get("task_contract_snapshot")
        if not isinstance(historical_snapshot, dict):
            continue
        snapshot_path = _canonical_relative_path(
            historical_snapshot.get("task_contract_path")
        )
        version = _contract_path_version(snapshot_path) if snapshot_path else None
        item_path = f"gate-decisions.yml.gate_decisions[{index}].task_contract_snapshot"
        if (
            historical_snapshot.get("contract") != GATE1_TASK_CONTRACT
            or snapshot_path is None
            or version is None
            or historical_snapshot.get("contract_version") != version
        ):
            result.error(
                "TASK_CONTRACT_SNAPSHOT_MISMATCH",
                item_path,
                "every versioned Gate 1 snapshot must use the canonical filename/version contract",
            )
            continue
        prior_path = historical_contract_paths.get(version)
        if prior_path is not None and prior_path != snapshot_path:
            result.error(
                "TASK_CONTRACT_SNAPSHOT_MISMATCH",
                item_path,
                f"contract version {version} is already bound to {prior_path!r}",
            )
        historical_contract_paths[version] = snapshot_path
        historical_file = root.joinpath(*PurePosixPath(snapshot_path).parts)
        if not historical_file.is_file() or historical_file.is_symlink():
            result.error(
                "TASK_CONTRACT_MISSING",
                snapshot_path,
                "a Gate-1-bound historical task contract must be retained as an ordinary file",
            )
            continue
        try:
            historical_document = _load_yaml(historical_file)
            historical_hash = _sha256(historical_file)
        except (OSError, UnicodeError, yaml.YAMLError, ValueError) as exc:
            result.error("TASK_CONTRACT_INVALID", snapshot_path, str(exc))
            continue
        if (
            historical_snapshot.get("task_contract_hash") != historical_hash
            or historical_snapshot.get("task_contract_id")
            != historical_document.get("task_contract_id")
            or historical_snapshot.get("contract_version")
            != historical_document.get("contract_version")
        ):
            result.error(
                "TASK_CONTRACT_SNAPSHOT_MISMATCH",
                item_path,
                "historical contract bytes/identity no longer match the Gate 1 snapshot",
            )

    current_gate1 = _current_decisions(decisions, "gate-1", None)
    gate1 = current_gate1[0] if len(current_gate1) == 1 else None
    if gate1 is not None:
        result.summary["current_gate1_decision_id"] = gate1.get("decision_id")
    snapshot = gate1.get("task_contract_snapshot") if isinstance(gate1, dict) else None
    positive_gate1 = gate1 is not None and gate1.get("decision") in {"approved", "approved-with-conditions"}
    if not isinstance(snapshot, dict):
        if positive_gate1:
            result.error("LEGACY_TASK_CONTRACT_REVIEW_REQUIRED", "gate-decisions.yml.current_gate1.task_contract_snapshot", "current positive Gate 1 predates the task-contract snapshot")
            result.summary["legacy_contract_review_required"] = True
            result.summary["task_contract_drift"] = "legacy-missing"
            result.summary["status"] = "legacy-contract-review-required"
        else:
            result.error("TASK_CONTRACT_MISSING", "task-contract.yml", "a task contract is required before Gate 1 approval")
            result.summary["task_contract_drift"] = "missing"
            result.summary["status"] = "missing"
        return result
    if snapshot.get("contract") != GATE1_TASK_CONTRACT:
        result.error("TASK_CONTRACT_SNAPSHOT_MISMATCH", "gate-decisions.yml.current_gate1.task_contract_snapshot.contract", f"must be {GATE1_TASK_CONTRACT}")
    relative = _canonical_relative_path(snapshot.get("task_contract_path"))
    if relative is None or _contract_path_version(relative) is None:
        result.error("TASK_CONTRACT_INVALID", "gate-decisions.yml.current_gate1.task_contract_snapshot.task_contract_path", "must be task-contract.yml or task-contract.vN.yml")
        result.summary["status"] = "invalid"
        return result
    contract_path = root.joinpath(*PurePosixPath(relative).parts)
    result.summary["task_contract_path"] = relative
    if not contract_path.is_file() or contract_path.is_symlink():
        result.error("TASK_CONTRACT_MISSING", relative, "bound task contract is missing or unsafe")
        result.summary["task_contract_drift"] = "missing"
        result.summary["status"] = "invalid"
        return result
    try:
        contract = _load_yaml(contract_path)
    except (OSError, UnicodeError, yaml.YAMLError, ValueError) as exc:
        result.error("TASK_CONTRACT_INVALID", relative, str(exc))
        result.summary["status"] = "invalid"
        return result
    contract_hash = _sha256(contract_path)
    result.summary.update({
        "task_contract_hash": contract_hash,
        "task_contract_id": contract.get("task_contract_id"),
        "contract_version": contract.get("contract_version"),
    })
    if snapshot.get("task_contract_hash") != contract_hash:
        result.error("TASK_CONTRACT_SNAPSHOT_MISMATCH", f"{relative}.sha256", "contract bytes no longer match Gate 1")
    for key in ("task_contract_id", "contract_version"):
        if snapshot.get(key) != contract.get(key):
            result.error("TASK_CONTRACT_SNAPSHOT_MISMATCH", f"gate-decisions.yml.current_gate1.task_contract_snapshot.{key}", "does not match the bound contract")
    if contract.get("contract_version") != _contract_path_version(relative):
        result.error(
            "TASK_CONTRACT_INVALID",
            f"{relative}.contract_version",
            "contract_version must match task-contract.yml/vN filename",
        )
    tasks, active, example_index, input_types = _validate_contract(
        result, contract, expected_distillation_id=capability.get("distillation_id")
    )
    result.summary["active_stable_task_ids"] = active
    if snapshot.get("active_stable_task_ids") != active:
        result.error("TASK_CONTRACT_SNAPSHOT_MISMATCH", "gate-decisions.yml.current_gate1.task_contract_snapshot.active_stable_task_ids", "must exactly follow active tasks in contract order")
    if positive_gate1 and contract.get("status") != "frozen":
        result.error("TASK_CONTRACT_INVALID", f"{relative}.status", "a positive Gate 1 may bind only a frozen contract")
    task_decisions = _records(gate1, "stable_task_decisions") if gate1 else []
    decision_map = {item.get("stable_task_id"): item for item in task_decisions if _id(item.get("stable_task_id"))}
    if positive_gate1 and (
        set(decision_map) != set(tasks)
        or len(task_decisions) != len(tasks)
        or len(decision_map) != len(task_decisions)
    ):
        result.error("TASK_CONTRACT_SNAPSHOT_MISMATCH", "gate-decisions.yml.current_gate1.stable_task_decisions", "must decide every stable task exactly once")
    for task_id, task in tasks.items():
        decision = decision_map.get(task_id)
        if decision is not None:
            if decision.get("decision") != task.get("status"):
                result.error("TASK_CONTRACT_SNAPSHOT_MISMATCH", "gate-decisions.yml.current_gate1.stable_task_decisions", f"task {task_id!r} status does not match the contract")
            if positive_gate1 and (
                not isinstance(decision.get("rationale"), str)
                or not decision["rationale"].strip()
            ):
                result.error(
                    "TASK_CONTRACT_SNAPSHOT_MISMATCH",
                    "gate-decisions.yml.current_gate1.stable_task_decisions",
                    f"task {task_id!r} requires a non-empty human rationale",
                )

    coverage_path = root / "task-coverage.yml"
    coverage_rows: dict[str, Mapping[str, Any]] = {}
    if not coverage_path.is_file() or coverage_path.is_symlink():
        result.error("TASK_COVERAGE_INVALID", "task-coverage.yml", "task coverage is missing or unsafe")
    else:
        try:
            coverage = _load_yaml(coverage_path)
        except (OSError, UnicodeError, yaml.YAMLError, ValueError) as exc:
            result.error("TASK_COVERAGE_INVALID", "task-coverage.yml", str(exc))
            coverage = {}
        if coverage.get("schema_version") != 1 or coverage.get("distillation_id") != capability.get("distillation_id"):
            result.error("TASK_COVERAGE_INVALID", "task-coverage.yml", "schema_version/distillation_id mismatch")
        reference = coverage.get("task_contract")
        expected_reference = {
            "path": relative,
            "sha256": contract_hash,
            "task_contract_id": contract.get("task_contract_id"),
            "contract_version": contract.get("contract_version"),
        }
        if reference != expected_reference:
            result.error("TASK_COVERAGE_INVALID", "task-coverage.yml.task_contract", "must exactly bind the current task contract")
        for index, row in enumerate(_records(coverage, "coverage")):
            row_path = f"task-coverage.yml.coverage[{index}]"
            task_id = row.get("stable_task_id")
            if not _id(task_id) or task_id in coverage_rows or task_id not in tasks:
                result.error("TASK_COVERAGE_INVALID", f"{row_path}.stable_task_id", "must uniquely reference a contract task")
                continue
            coverage_rows[task_id] = row
            status = row.get("coverage_status")
            if status not in COVERAGE_STATUSES:
                result.error("TASK_COVERAGE_INVALID", f"{row_path}.coverage_status", "unsupported coverage status")
            task_status = tasks[task_id].get("status")
            if task_status == "active" and status != "covered":
                result.error("STABLE_TASK_UNCOVERED", row_path, "active stable task must be covered")
            if task_status in {"deferred", "rejected"}:
                if status != task_status or row.get("gate1_decision_id") != gate1.get("decision_id") or not isinstance(row.get("rationale"), str) or not row.get("rationale").strip():
                    result.error("TASK_COVERAGE_INVALID", row_path, "deferred/rejected coverage must match and cite current Gate 1")
            if status == "covered":
                for field_name in (
                    "candidate_ids", "capability_rule_ids", "trigger_case_ids",
                    "nontrigger_case_ids", "task_eval_case_ids", "holdout_case_ids",
                    "rubric_dimension_ids",
                ):
                    if _strings(row.get(field_name), nonempty=True) is None:
                        result.error("STABLE_TASK_UNCOVERED", f"{row_path}.{field_name}", "covered task requires a unique non-empty mapping")
        if set(coverage_rows) != set(tasks):
            result.error("STABLE_TASK_UNCOVERED", "task-coverage.yml.coverage", "every stable task must have exactly one row")

    candidates = _records(capability, "skill_candidates")
    rules = _records(capability, "capability_rules")
    candidate_map = {item.get("candidate_id"): item for item in candidates if _id(item.get("candidate_id"))}
    rule_map = {item.get("rule_id"): item for item in rules if _id(item.get("rule_id"))}
    result.summary["candidate_stable_task_ids"] = {
        candidate_id: candidate.get("stable_task_ids", [])
        for candidate_id, candidate in candidate_map.items()
    }
    for kind, records_map in (("candidate", candidate_map), ("rule", rule_map)):
        for record_id, record in records_map.items():
            ids = _strings(record.get("stable_task_ids"), nonempty=True)
            if ids is None:
                result.error("CANDIDATE_TASK_MISMATCH", f"capability-rules.yml.{kind}[{record_id}].stable_task_ids", "must be a unique non-empty list")
                continue
            unknown = [item for item in ids if item not in tasks or tasks[item].get("status") != "active"]
            if unknown:
                result.error("STABLE_TASK_UNKNOWN", f"capability-rules.yml.{kind}[{record_id}].stable_task_ids", f"unknown or non-active task IDs: {unknown}")

    # Load materialized v2 case metadata once.  v1 remains parseable elsewhere,
    # but cannot satisfy current task-contract eligibility.
    eval_by_candidate: dict[str, dict[str, Any]] = {}
    for candidate_id, candidate in candidate_map.items():
        current_gate3 = _current_decisions(decisions, "gate-3", candidate_id)
        gate3 = current_gate3[0] if len(current_gate3) == 1 else None
        if gate3 is None or gate3.get("decision") != "approved-for-eval":
            continue
        approval = gate3.get("approval_snapshot")
        if not isinstance(approval, dict) or approval.get("contract") == GATE3_LEGACY_CONTRACT:
            result.error("LEGACY_TASK_CONTRACT_REVIEW_REQUIRED", f"gate-decisions.yml.gate3[{candidate_id}].approval_snapshot", "current Gate 3 v1 cannot authorize new materialization or Gate 4")
            result.summary["legacy_contract_review_required"] = True
            result.summary["gate3_contract_binding"] = "legacy-contract-review-required"
            continue
        if approval.get("contract") != GATE3_TASK_CONTRACT:
            result.error("GATE3_TASK_CONTRACT_MISMATCH", f"gate-decisions.yml.gate3[{candidate_id}].approval_snapshot.contract", f"new/current approval must use {GATE3_TASK_CONTRACT}")
            continue
        expected = {
            "current_gate1_decision_id": gate1.get("decision_id") if gate1 else None,
            "task_contract": {
                "path": relative,
                "sha256": contract_hash,
                "task_contract_id": contract.get("task_contract_id"),
                "contract_version": contract.get("contract_version"),
                "active_stable_task_ids": active,
            },
            "task_coverage": {
                "path": "task-coverage.yml",
                "sha256": _sha256(coverage_path) if coverage_path.is_file() else None,
            },
            "candidate_stable_task_ids": candidate.get("stable_task_ids"),
        }
        for key, value in expected.items():
            if approval.get(key) != value:
                result.error("GATE3_TASK_CONTRACT_MISMATCH", f"gate-decisions.yml.gate3[{candidate_id}].approval_snapshot.{key}", "does not match current contract/coverage/candidate")
        result.summary["gate3_contract_binding"] = "matches" if not any(issue.code == "GATE3_TASK_CONTRACT_MISMATCH" for issue in result.errors) else "mismatch"
        candidate_path = _canonical_relative_path(approval.get("candidate_path"))
        if candidate_path is None:
            continue
        try:
            trigger = _load_json(root.joinpath(*PurePosixPath(candidate_path).parts, "evals", "trigger-cases.json"))
            task_definition = _load_json(root.joinpath(*PurePosixPath(candidate_path).parts, "evals", "task-cases.json"))
        except (OSError, UnicodeError, ValueError, json.JSONDecodeError) as exc:
            result.error("TASK_COVERAGE_INVALID", f"{candidate_path}/evals", f"cannot load v2 eval definitions: {exc}")
            continue
        if trigger.get("schema_version") != 2 or task_definition.get("schema_version") != 2:
            result.error("TASK_COVERAGE_INVALID", f"{candidate_path}/evals", "Gate 3 v2 requires eval definition schema_version 2")
        cases: dict[str, Mapping[str, Any]] = {}
        case_task_ids: dict[str, set[str]] = {}
        trigger_ids: set[str] = set()
        nontrigger_ids: set[str] = set()
        for key, polarity in (("should_trigger", "positive"), ("should_not_trigger", "negative")):
            for case in _records(trigger, key):
                case_id = case.get("case_id")
                case_path = f"{candidate_path}/evals/{key}[{case_id}]"
                if not _id(case_id):
                    result.error(
                        "TASK_COVERAGE_INVALID",
                        case_path,
                        "case_id must be a stable ID",
                    )
                    continue
                if case_id in cases:
                    result.error(
                        "TASK_COVERAGE_INVALID",
                        case_path,
                        "case_id must be unique across trigger and nontrigger definitions",
                    )
                    continue
                cases[case_id] = case
                (trigger_ids if polarity == "positive" else nontrigger_ids).add(case_id)
                task_ids = _strings(case.get("stable_task_ids"), nonempty=True)
                input_ids = _strings(case.get("input_type_ids"), nonempty=True)
                positive_ids = _strings(
                    case.get("positive_example_ids"),
                    nonempty=polarity == "positive",
                )
                negative_ids = _strings(
                    case.get("negative_example_ids"),
                    nonempty=polarity == "negative",
                )
                if (
                    task_ids is None
                    or input_ids is None
                    or positive_ids is None
                    or negative_ids is None
                    or (polarity == "positive" and negative_ids)
                    or (polarity == "negative" and positive_ids)
                ):
                    result.error(
                        "TASK_COVERAGE_INVALID",
                        case_path,
                        "v2 trigger/nontrigger case requires stable task/input IDs and explicit polarity-separated example IDs",
                    )
                    continue
                case_task_ids[case_id] = set(task_ids)
                unknown_tasks = [
                    item for item in task_ids
                    if item not in tasks
                    or tasks[item].get("status") != "active"
                    or item not in (candidate.get("stable_task_ids") or [])
                ]
                unknown_inputs = [item for item in input_ids if item not in input_types]
                if unknown_tasks:
                    result.error(
                        "STABLE_TASK_UNKNOWN",
                        f"{case_path}.stable_task_ids",
                        f"unknown, non-active, or candidate-unbound task IDs: {unknown_tasks}",
                    )
                if unknown_inputs:
                    result.error(
                        "TASK_COVERAGE_INVALID",
                        f"{case_path}.input_type_ids",
                        f"unknown input type IDs: {unknown_inputs}",
                    )
                selected_examples = positive_ids if polarity == "positive" else negative_ids
                for example_id in selected_examples:
                    metadata = example_index.get(example_id)
                    if (
                        metadata is None
                        or metadata["polarity"] != polarity
                        or metadata["stable_task_id"] not in task_ids
                    ):
                        result.error(
                            "CANDIDATE_TASK_MISMATCH",
                            f"{case_path}.{polarity}_example_ids",
                            f"{polarity} example {example_id!r} has the wrong stable task or polarity",
                        )
        task_cases: dict[str, Mapping[str, Any]] = {}
        for case in _records(task_definition, "tasks"):
            case_id = case.get("case_id")
            case_path = f"{candidate_path}/evals/tasks[{case_id}]"
            if not _id(case_id):
                result.error("TASK_COVERAGE_INVALID", case_path, "case_id must be a stable ID")
                continue
            if case_id in task_cases or case_id in cases:
                result.error(
                    "TASK_COVERAGE_INVALID",
                    case_path,
                    "case_id must be unique across every eval case type",
                )
                continue
            task_cases[case_id] = case
            task_ids = _strings(case.get("stable_task_ids"), nonempty=True)
            input_ids = _strings(case.get("input_type_ids"), nonempty=True)
            positive_ids = _strings(case.get("positive_example_ids"), nonempty=True)
            negative_ids = _strings(case.get("negative_example_ids"), nonempty=True)
            if (
                task_ids is None
                or input_ids is None
                or positive_ids is None
                or negative_ids is None
            ):
                result.error(
                    "TASK_COVERAGE_INVALID",
                    case_path,
                    "v2 task requires stable_task_ids, input_type_ids, and positive/negative example IDs",
                )
                continue
            case_task_ids[case_id] = set(task_ids)
            unknown_tasks = [
                item for item in task_ids
                if item not in tasks
                or tasks[item].get("status") != "active"
                or item not in (candidate.get("stable_task_ids") or [])
            ]
            unknown_inputs = [item for item in input_ids if item not in input_types]
            if unknown_tasks:
                result.error(
                    "STABLE_TASK_UNKNOWN",
                    f"{case_path}.stable_task_ids",
                    f"unknown, non-active, or candidate-unbound task IDs: {unknown_tasks}",
                )
            if unknown_inputs:
                result.error(
                    "TASK_COVERAGE_INVALID",
                    f"{case_path}.input_type_ids",
                    f"unknown input type IDs: {unknown_inputs}",
                )
            for field_name, polarity, example_ids in (
                ("positive_example_ids", "positive", positive_ids),
                ("negative_example_ids", "negative", negative_ids),
            ):
                for example_id in example_ids:
                    metadata = example_index.get(example_id)
                    if (
                        metadata is None
                        or metadata["polarity"] != polarity
                        or metadata["stable_task_id"] not in task_ids
                    ):
                        result.error(
                            "CANDIDATE_TASK_MISMATCH",
                            f"{case_path}.{field_name}",
                            f"{polarity} example {example_id!r} has the wrong stable task or polarity",
                        )
        rubric = task_definition.get("comparison_protocol", {}).get("rubric", {}) if isinstance(task_definition.get("comparison_protocol"), dict) else {}
        dimension_ids = {
            item.get("dimension_id") for item in rubric.get("dimensions", [])
            if isinstance(item, dict) and _id(item.get("dimension_id"))
        } if isinstance(rubric, dict) else set()
        fatal_ids = {
            item.get("failure_id") for item in rubric.get("fatal_failures", [])
            if isinstance(item, dict) and _id(item.get("failure_id"))
        } if isinstance(rubric, dict) else set()
        for field_name, actual_ids in (
            ("trigger_case_ids", trigger_ids),
            ("nontrigger_case_ids", nontrigger_ids),
            ("task_case_ids", set(task_cases)),
        ):
            registered = _strings(candidate.get(field_name), nonempty=True)
            if registered is None or set(registered) != actual_ids:
                result.error(
                    "CANDIDATE_TASK_MISMATCH",
                    f"capability-rules.yml.candidate[{candidate_id}].{field_name}",
                    "candidate must explicitly register the complete eval case ID set",
                )
        eval_by_candidate[candidate_id] = {
            "cases": cases, "tasks": task_cases,
            "trigger_ids": trigger_ids, "nontrigger_ids": nontrigger_ids,
            "case_task_ids": case_task_ids,
            "dimension_ids": dimension_ids, "fatal_ids": fatal_ids,
        }

    for task_id, row in coverage_rows.items():
        if row.get("coverage_status") != "covered":
            continue
        for candidate_id in row.get("candidate_ids", []):
            candidate = candidate_map.get(candidate_id)
            if candidate is None or task_id not in (candidate.get("stable_task_ids") or []):
                result.error("CANDIDATE_TASK_MISMATCH", f"task-coverage.yml[{task_id}].candidate_ids", f"candidate {candidate_id!r} does not bind task")
        for rule_id in row.get("capability_rule_ids", []):
            rule = rule_map.get(rule_id)
            if rule is None or task_id not in (rule.get("stable_task_ids") or []):
                result.error("CANDIDATE_TASK_MISMATCH", f"task-coverage.yml[{task_id}].capability_rule_ids", f"rule {rule_id!r} does not bind task")
        available = [eval_by_candidate.get(cid) for cid in row.get("candidate_ids", []) if eval_by_candidate.get(cid)]
        if not available:
            continue
        all_triggers = {cid for item in available for cid in item["trigger_ids"]}
        all_nontriggers = {cid for item in available for cid in item["nontrigger_ids"]}
        all_tasks = {cid for item in available for cid in item["tasks"]}
        all_dimensions = {did for item in available for did in item["dimension_ids"]}
        for field_name, pool in (("trigger_case_ids", all_triggers), ("nontrigger_case_ids", all_nontriggers), ("task_eval_case_ids", all_tasks), ("holdout_case_ids", all_tasks), ("rubric_dimension_ids", all_dimensions)):
            unknown = set(row.get(field_name, [])) - pool
            if unknown:
                result.error("TASK_COVERAGE_INVALID", f"task-coverage.yml[{task_id}].{field_name}", f"unknown IDs: {sorted(unknown)}")
            if field_name != "rubric_dimension_ids":
                wrong_task = {
                    case_id for case_id in row.get(field_name, [])
                    if not any(
                        task_id in item["case_task_ids"].get(case_id, set())
                        for item in available
                    )
                }
                if wrong_task:
                    result.error(
                        "TASK_COVERAGE_INVALID",
                        f"task-coverage.yml[{task_id}].{field_name}",
                        f"case IDs do not bind this stable task: {sorted(wrong_task)}",
                    )
        for holdout_id in row.get("holdout_case_ids", []):
            matching = next((item["tasks"].get(holdout_id) for item in available if holdout_id in item["tasks"]), None)
            if matching is not None and matching.get("holdout") is not True:
                result.error("TASK_COVERAGE_INVALID", f"task-coverage.yml[{task_id}].holdout_case_ids", f"{holdout_id!r} is not a holdout")

    sources: dict[str, Mapping[str, Any]] = {}
    if sources_manifest is not None:
        try:
            source_document = _load_yaml(Path(sources_manifest))
            sources = {item.get("id"): item for item in _records(source_document, "sources") if _id(item.get("id"))}
        except (OSError, UnicodeError, yaml.YAMLError, ValueError) as exc:
            result.error("METHOD_TRANSFER_PROVENANCE_REQUIRED", str(sources_manifest), f"cannot load sources manifest: {exc}")
    method_tasks = {task_id: task for task_id, task in tasks.items() if task.get("status") == "active" and task.get("task_mode") == "method-transfer"}
    if method_tasks and sources_manifest is None:
        result.error("METHOD_TRANSFER_PROVENANCE_REQUIRED", "--sources-manifest", "method-transfer validation requires an explicit sources manifest")
    evidence_source_ids = {item.get("source_id") for item in _records(evidence, "evidence") if _id(item.get("source_id"))}
    if method_tasks and sources_manifest is not None and not any(
        source_id in evidence_source_ids
        and source.get("provenance_role") == "method-source"
        for source_id, source in sources.items()
    ):
        result.error(
            "METHOD_TRANSFER_PROVENANCE_REQUIRED",
            str(sources_manifest),
            "method-transfer requires at least one extracted method-source in the explicit manifest",
        )
    for candidate_id, candidate in candidate_map.items():
        linked_method = set(candidate.get("stable_task_ids") or []) & set(method_tasks)
        if not linked_method:
            continue
        provenance = candidate.get("provenance_contract")
        if not isinstance(provenance, dict) or set(provenance.get("output_layers", [])) != PROVENANCE_LAYERS or provenance.get("missing_target_evidence") not in {"stop", "lower-conclusion-strength"}:
            result.error("METHOD_TRANSFER_PROVENANCE_REQUIRED", f"capability-rules.yml.candidate[{candidate_id}].provenance_contract", "candidate must preserve the three output layers and missing-evidence behavior")
        eval_meta = eval_by_candidate.get(candidate_id)
        if eval_meta is None:
            continue
        if not METHOD_RUBRIC_DIMENSIONS.issubset(eval_meta["dimension_ids"]) or METHOD_FATAL_FAILURE not in eval_meta["fatal_ids"]:
            result.error("METHOD_TRANSFER_PROVENANCE_REQUIRED", f"{candidate_id}.eval-rubric", "method-transfer rubric lacks required dimensions/fatal failure")
        for task_id in linked_method:
            row = coverage_rows.get(task_id, {})
            required_inputs = set(method_tasks[task_id].get("required_input_types", []))
            covered_inputs: set[str] = set()
            external_found = False
            for holdout_id in row.get("holdout_case_ids", []):
                case = eval_meta["tasks"].get(holdout_id)
                if not isinstance(case, dict):
                    continue
                holdout_contract = case.get("holdout_contract")
                if not isinstance(holdout_contract, dict):
                    continue
                if holdout_contract.get("used_for_rule_extraction") is not False:
                    continue
                if (
                    not isinstance(holdout_contract.get("isolation"), str)
                    or not holdout_contract["isolation"].strip()
                ):
                    continue
                dimensions = _strings(holdout_contract.get("unfamiliarity_dimensions"), nonempty=True)
                if dimensions is None or not set(dimensions).issubset(UNFAMILIARITY_DIMENSIONS):
                    continue
                target_ids = _strings(holdout_contract.get("target_source_ids"), nonempty=True) or []
                hashes = holdout_contract.get("target_source_hashes")
                for source_id in target_ids:
                    source = sources.get(source_id)
                    expected_hash = hashes.get(source_id) if isinstance(hashes, dict) else None
                    if source is not None and source.get("provenance_role") == "target-material" and FULL_SHA256_RE.fullmatch(str(expected_hash or "")) and expected_hash == source.get("checksum") and source_id not in evidence_source_ids:
                        external_found = True
                covered_inputs.update(case.get("input_type_ids", []))
            if not external_found or not required_inputs.issubset(covered_inputs):
                result.error("METHOD_TRANSFER_EXTERNAL_HOLDOUT_REQUIRED", f"task-coverage.yml[{task_id}].holdout_case_ids", "method-transfer requires external isolated target material covering all required input types")

        current_gate4 = _current_decisions(decisions, "gate-4", candidate_id)
        gate4 = current_gate4[0] if len(current_gate4) == 1 else None
        if gate4 is not None and gate4.get("decision") == "accepted":
            accepted_run_ids = set(gate4.get("eval_run_ids", []))
            actual_inputs: set[str] = set()
            actual_external = False
            for run in _records(eval_runs_document, "eval_runs"):
                if (
                    run.get("eval_run_id") not in accepted_run_ids
                    or run.get("candidate_id") != candidate_id
                    or run.get("case_type") != "task"
                    or run.get("holdout") is not True
                    or run.get("status") != "completed"
                    or run.get("outcome") != "pass"
                ):
                    continue
                run_target_ids = _strings(run.get("target_source_ids"), nonempty=True) or []
                run_hashes = run.get("target_source_hashes")
                run_inputs = _strings(run.get("input_type_ids"), nonempty=True) or []
                run_unfamiliarity = _strings(run.get("unfamiliarity_dimensions"), nonempty=True)
                if (
                    run.get("used_for_rule_extraction") is not False
                    or not isinstance(run.get("isolation"), str)
                    or not run["isolation"].strip()
                    or run_unfamiliarity is None
                    or not set(run_unfamiliarity).issubset(UNFAMILIARITY_DIMENSIONS)
                ):
                    continue
                actual_inputs.update(run_inputs)
                for source_id in run_target_ids:
                    source = sources.get(source_id)
                    recorded_hash = run_hashes.get(source_id) if isinstance(run_hashes, dict) else None
                    if (
                        source is not None
                        and source.get("provenance_role") == "target-material"
                        and recorded_hash == source.get("checksum")
                        and source_id not in evidence_source_ids
                    ):
                        actual_external = True
            required_all = {
                input_id for task_id in linked_method
                for input_id in method_tasks[task_id].get("required_input_types", [])
            }
            if not actual_external or not required_all.issubset(actual_inputs):
                result.error(
                    "METHOD_TRANSFER_EXTERNAL_HOLDOUT_REQUIRED",
                    f"gate-decisions.yml.gate4[{candidate_id}].eval_run_ids",
                    "Gate 4 acceptance requires completed/pass external target holdout runs covering every required input type",
                )

    checkpoint_path = root / "context-checkpoint.yml"
    if checkpoint_path.exists() or checkpoint_path.is_symlink():
        try:
            checkpoint = _load_yaml(checkpoint_path)
        except (OSError, UnicodeError, yaml.YAMLError, ValueError) as exc:
            result.error("CHECKPOINT_PRODUCT_CONTRACT_CONFLICT", "context-checkpoint.yml", str(exc))
            checkpoint = {}
        anchor = checkpoint.get("product_contract_anchor")
        expected_anchor = {
            "path": relative,
            "sha256": contract_hash,
            "task_contract_id": contract.get("task_contract_id"),
            "contract_version": contract.get("contract_version"),
            "active_stable_task_ids": active,
        }
        if anchor != expected_anchor:
            result.error("CHECKPOINT_PRODUCT_CONTRACT_CONFLICT", "context-checkpoint.yml.product_contract_anchor", "checkpoint must re-anchor to the current product contract")
        stage = checkpoint.get("current_stage_objective")
        if not isinstance(stage, dict):
            result.error("CHECKPOINT_PRODUCT_CONTRACT_CONFLICT", "context-checkpoint.yml.current_stage_objective", "must be a mapping")
        else:
            stage_ids = _strings(stage.get("stable_task_ids"), nonempty=True)
            if stage_ids is None or not set(stage_ids).issubset(active) or stage.get("supersedes_product_contract") is not False or stage.get("excluded_stable_task_ids") not in (None, []):
                result.error("CHECKPOINT_PRODUCT_CONTRACT_CONFLICT", "context-checkpoint.yml.current_stage_objective", "stage objective may select active work but may not supersede or exclude product tasks")
            result.summary["current_stage_objective"] = stage
        constraints = checkpoint.get("temporary_operational_constraints")
        if _strings(constraints) is None:
            result.error("CHECKPOINT_PRODUCT_CONTRACT_CONFLICT", "context-checkpoint.yml.temporary_operational_constraints", "must be a string list")
        else:
            result.summary["temporary_operational_constraints"] = constraints

    covered = [task_id for task_id, row in coverage_rows.items() if row.get("coverage_status") == "covered"]
    deferred = [task_id for task_id, row in coverage_rows.items() if row.get("coverage_status") == "deferred"]
    rejected = [task_id for task_id, row in coverage_rows.items() if row.get("coverage_status") == "rejected"]
    result.summary.update({
        "covered_stable_task_ids": covered,
        "uncovered_stable_task_ids": [item for item in active if item not in covered],
        "deferred_stable_task_ids": deferred,
        "rejected_stable_task_ids": rejected,
        "task_contract_drift": "matches" if not any(issue.code in {"TASK_CONTRACT_MISSING", "TASK_CONTRACT_INVALID", "TASK_CONTRACT_SNAPSHOT_MISMATCH"} for issue in result.errors) else "mismatch",
        "status": "valid" if result.ok else ("legacy-contract-review-required" if result.summary["legacy_contract_review_required"] else "invalid"),
    })
    return result

from __future__ import annotations

import contextlib
import io
import json
import os
import tempfile
import unittest
from pathlib import Path

import yaml

from audit_candidate_disclosure import (
    DisclosureInputError,
    audit_candidate_disclosure,
    main,
)


class CandidateDisclosureAuditTests(unittest.TestCase):
    def setUp(self):
        self.tempdir = tempfile.TemporaryDirectory()
        self.root = Path(self.tempdir.name)
        self.candidate = self.root / "candidate"
        self.candidate.mkdir()
        self.manifest = self.root / "private-sources.yml"
        self.identities = {
            "title": "Velvet Quasar Methods",
            "author": ["Nora Exampleton"],
            "publisher": "Imaginary Meridian Press",
            "isbn": "978-1-23-456789-0",
            "edition": "Synthetic third edition",
            "series_title": "Nightglass Inquiry Series",
            "local_path": "sources/raw/Velvet-Quasar-Methods.docx",
        }
        self._write_manifest()

    def tearDown(self):
        self.tempdir.cleanup()

    def _write_manifest(self, **overrides):
        source = {
            "id": "source-opaque-7f31",
            "type": "book-docx",
            "source_role": "primary-book",
            "provenance_role": "method-source",
            **self.identities,
            **overrides,
            "checksum": "sha256:" + "a" * 64,
            "locator_strategy": {"primary": "ooxml-block"},
        }
        self.manifest.write_text(
            yaml.safe_dump({"schema_version": 2, "sources": [source]}, sort_keys=False),
            encoding="utf-8",
        )

    def _write_candidate(self, relative: str, content: str) -> Path:
        path = self.candidate / relative
        path.parent.mkdir(parents=True, exist_ok=True)
        path.write_text(content, encoding="utf-8")
        return path

    def _identity_findings(self, report, category=None):
        findings = [
            finding
            for finding in report.get("findings", [])
            if finding["code"] == "IDENTITY_DISCLOSURE"
        ]
        if category is not None:
            findings = [
                finding
                for finding in findings
                if finding["identity_category"] == category
            ]
        return findings

    def test_passes_source_neutral_candidate_and_exempt_metadata_words(self):
        self._write_manifest(local_path="sources/raw/source-opaque-7f31.docx")
        self._write_candidate(
            "evals/task-cases.json",
            json.dumps(
                {
                    "title": "Synthetic audit task",
                    "source_id": "source-opaque-7f31",
                    "checksum": "sha256:" + "b" * 64,
                    "locator": "section-4",
                    "reviewer": "human-delegate",
                }
            ),
        )
        self._write_candidate(
            "SKILL.md",
            "The authorizing Gate checks lifecycle state.\n"
            "Source files are read as immutable bytes for deterministic validation.\n",
        )
        first = audit_candidate_disclosure(self.candidate, self.manifest)
        second = audit_candidate_disclosure(self.candidate, self.manifest)
        self.assertEqual(first, second)
        self.assertTrue(first["ok"])
        self.assertEqual("pass", first["status"])
        self.assertEqual([], first["findings"])

    def test_source_field_references_cannot_echo_identity_bearing_keys(self):
        forbidden_key_part = "ObsidianSyntheticKey"
        source = {
            "id": "source-opaque-key-test",
            "type": "book-docx",
            "source_role": "primary-book",
            "title": forbidden_key_part,
            f"{forbidden_key_part}_author": "Alice Exampleton",
        }
        self.manifest.write_text(
            yaml.safe_dump({"schema_version": 2, "sources": [source]}, sort_keys=False),
            encoding="utf-8",
        )
        self._write_candidate("SKILL.md", "Alice Exampleton supplies the example.\n")
        report = audit_candidate_disclosure(self.candidate, self.manifest)
        self.assertTrue(self._identity_findings(report, "author"))
        self.assertNotIn(
            forbidden_key_part.casefold(),
            json.dumps(report, ensure_ascii=False).casefold(),
        )

    def test_detects_series_like_book_collection_field(self):
        self._write_manifest(book_collection="Synthetic Aurora Collection")
        self._write_candidate(
            "SKILL.md", "The Synthetic Aurora Collection label is forbidden.\n"
        )
        report = audit_candidate_disclosure(self.candidate, self.manifest)
        self.assertTrue(self._identity_findings(report, "series"))

    def test_ignores_unrelated_policy_paths_and_target_identity(self):
        method_record = {
            "id": "method-source-1",
            **self.identities,
            "type": "book-docx",
            "source_role": "primary-book",
        }
        policy_record = {
            "id": "policy-source-1",
            "type": "local-policy-bundle",
            "source_role": "controlling-requirements",
            "title": "Synthetic Project Policy",
            "local_path": "docs/workflow.md",
        }
        target_record = {
            "id": "target-source-1",
            "type": "paper-pdf",
            "source_role": "eval-target",
            "provenance_role": "target-material",
            "title": "Synthetic Target Paper",
        }
        self.manifest.write_text(
            yaml.safe_dump(
                {"schema_version": 2, "sources": [method_record, policy_record, target_record]},
                sort_keys=False,
            ),
            encoding="utf-8",
        )
        self._write_candidate(
            "docs/workflow.md",
            "Review Synthetic Project Policy and Synthetic Target Paper as task inputs.\n",
        )
        report = audit_candidate_disclosure(self.candidate, self.manifest)
        self.assertTrue(report["ok"])
        self.assertEqual([], report["findings"])

    def test_detects_title_without_echoing_value(self):
        self._write_candidate("SKILL.md", "Apply Velvet Quasar Methods to the input.\n")
        report = audit_candidate_disclosure(self.candidate, self.manifest)
        findings = self._identity_findings(report, "title")
        self.assertTrue(findings)
        self.assertIn("sources[0].title", findings[0]["source_field_refs"])
        self.assertNotIn(self.identities["title"].casefold(), json.dumps(report).casefold())

    def test_detects_author(self):
        self._write_candidate("references/policy.md", "Nora Exampleton defines the method.\n")
        report = audit_candidate_disclosure(self.candidate, self.manifest)
        findings = self._identity_findings(report, "author")
        self.assertTrue(findings)
        self.assertIn("sources[0].author", findings[0]["source_field_refs"])

    def test_detects_publisher(self):
        self._write_candidate("references/provenance.md", "Issued by Imaginary Meridian Press.\n")
        report = audit_candidate_disclosure(self.candidate, self.manifest)
        self.assertTrue(self._identity_findings(report, "publisher"))

    def test_detects_isbn_with_punctuation_variation(self):
        self._write_candidate("references/id.md", "Catalog number: 978 1 23 456789 0.\n")
        report = audit_candidate_disclosure(self.candidate, self.manifest)
        findings = self._identity_findings(report, "isbn")
        self.assertTrue(findings)
        self.assertEqual("compact", findings[0]["match_mode"])

    def test_extra_terms_cover_alias_forms_and_named_case_without_echo(self):
        extra = self.root / "extra-terms.txt"
        forbidden = [
            "星幕译称",                 # synthetic translation
            "Xingmu Synthetic",        # synthetic transliteration
            "star-curtain-protocol",   # synthetic slug
            "霁光丛系",                 # synthetic series alias
            "Project Nightglass Case", # synthetic named case
        ]
        extra.write_text(
            "# synthetic aliases only\n" + "\n".join(forbidden) + "\n",
            encoding="utf-8",
        )
        self._write_candidate(
            "SKILL.md",
            "Apply 星幕译称 and Xingmu Synthetic.\n"
            "Route star-curtain-protocol through 霁光丛系.\n"
            "Use Project Nightglass Case as the worked example.\n",
        )
        report = audit_candidate_disclosure(
            self.candidate, self.manifest, extra_terms_file=extra
        )
        findings = self._identity_findings(report, "extra-term")
        self.assertEqual(5, len(findings))
        refs = {
            reference
            for finding in findings
            for reference in finding["source_field_refs"]
        }
        self.assertEqual(
            {f"extra_terms[line:{line}]" for line in range(2, 7)},
            refs,
        )
        serialized = json.dumps(report, ensure_ascii=False).casefold()
        compact_serialized = "".join(
            character for character in serialized if character.isalnum()
        )
        for value in forbidden:
            with self.subTest(value_type="synthetic-extra-term"):
                self.assertNotIn(value.casefold(), serialized)
                compact_value = "".join(
                    character
                    for character in value.casefold()
                    if character.isalnum()
                )
                self.assertNotIn(compact_value, compact_serialized)

    def test_detects_two_character_cjk_author_with_compact_variation(self):
        author = "霁岚"
        self._write_manifest(author=[author])
        self._write_candidate("SKILL.md", "合成署名：霁-岚。\n")
        report = audit_candidate_disclosure(self.candidate, self.manifest)
        findings = self._identity_findings(report, "author")
        self.assertTrue(findings)
        self.assertEqual("compact", findings[0]["match_mode"])
        self.assertNotIn(author, json.dumps(report, ensure_ascii=False))

    def test_detects_three_character_cjk_author_with_compact_variation(self):
        author = "霁岚舟"
        self._write_manifest(author=[author])
        self._write_candidate("SKILL.md", "合成署名：霁 岚 舟。\n")
        report = audit_candidate_disclosure(self.candidate, self.manifest)
        findings = self._identity_findings(report, "author")
        self.assertTrue(findings)
        self.assertEqual("compact", findings[0]["match_mode"])
        self.assertNotIn(author, json.dumps(report, ensure_ascii=False))

    def test_detects_filename_leak_and_redacts_candidate_location(self):
        leaked = "Copper Nebula Handbook"
        self._write_manifest(
            title="Unrelated Synthetic Title",
            local_path="sources/raw/Copper-Nebula-Handbook.pdf",
        )
        self._write_candidate("references/Copper-Nebula-Handbook.md", "Neutral body.\n")
        report = audit_candidate_disclosure(self.candidate, self.manifest)
        path_findings = [
            finding
            for finding in self._identity_findings(report)
            if finding["candidate_location"]["kind"] == "path"
        ]
        self.assertTrue(path_findings)
        self.assertTrue(
            any(
                finding["identity_category"] in {"path-basename", "path-stem"}
                for finding in path_findings
            )
        )
        self.assertEqual("<redacted>", path_findings[0]["candidate_location"]["path"])
        self.assertRegex(
            path_findings[0]["candidate_location"]["path_sha256"],
            r"^sha256:[0-9a-f]{64}$",
        )
        serialized = json.dumps(report).casefold()
        self.assertNotIn(leaked.casefold(), serialized)
        self.assertNotIn("copper-nebula-handbook", serialized)

    def test_detects_generic_attribution_phrase_but_not_authorizing(self):
        self._write_candidate(
            "SKILL.md",
            "The authorizing Gate remains source-neutral.\n"
            "According to the author, the framework has three stages.\n",
        )
        report = audit_candidate_disclosure(self.candidate)
        attribution = [
            finding
            for finding in report["findings"]
            if finding["code"] == "GENERIC_SOURCE_ATTRIBUTION"
        ]
        self.assertEqual(1, len(attribution))
        self.assertEqual(2, attribution[0]["candidate_location"]["line"])

    def test_target_material_author_attribution_is_not_method_source_leak(self):
        self._write_candidate(
            "SKILL.md",
            "先记录论文作者提出的主张，再区分论文作者解释与分析者判断。\n",
        )
        report = audit_candidate_disclosure(self.candidate, self.manifest)
        attribution = [
            finding
            for finding in report["findings"]
            if finding["code"] == "GENERIC_SOURCE_ATTRIBUTION"
        ]
        self.assertEqual([], attribution)

    def test_cli_never_echoes_forbidden_identity(self):
        secret = "Obsidian Zephyr Codex"
        self._write_manifest(title=secret, local_path="sources/raw/Obsidian-Zephyr-Codex.docx")
        self._write_candidate("Obsidian-Zephyr-Codex.md", f"Use {secret} here.\n")
        stdout = io.StringIO()
        stderr = io.StringIO()
        with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr):
            status = main(
                [str(self.candidate), "--sources-manifest", str(self.manifest)]
            )
        self.assertEqual(1, status)
        self.assertEqual("", stderr.getvalue())
        output = stdout.getvalue()
        parsed = json.loads(output)
        self.assertEqual("findings", parsed["status"])
        self.assertNotIn(secret.casefold(), output.casefold())
        compact_output = "".join(character for character in output.casefold() if character.isalnum())
        self.assertNotIn("obsidianzephyrcodex", compact_output)

    def test_fails_closed_on_symlink_and_cache_artifacts(self):
        target = self._write_candidate("references/target.md", "Neutral.\n")
        link = self.candidate / "linked.md"
        try:
            os.symlink(target, link)
        except (OSError, NotImplementedError) as exc:
            self.skipTest(f"symlinks unavailable: {exc}")
        with self.assertRaises(DisclosureInputError) as caught:
            audit_candidate_disclosure(self.candidate, self.manifest)
        self.assertEqual("CANDIDATE_SYMLINK", caught.exception.code)
        link.unlink()

        (self.candidate / "__pycache__").mkdir()
        with self.assertRaises(DisclosureInputError) as caught:
            audit_candidate_disclosure(self.candidate, self.manifest)
        self.assertEqual("CANDIDATE_CACHE_ARTIFACT", caught.exception.code)

    def test_fails_closed_on_invalid_utf8_text(self):
        (self.candidate / "SKILL.md").write_bytes(b"\xff\xfe\x00")
        with self.assertRaises(DisclosureInputError) as caught:
            audit_candidate_disclosure(self.candidate, self.manifest)
        self.assertEqual("CANDIDATE_BINARY_UNAUDITED", caught.exception.code)

    def test_cli_blocks_binary_artifact_without_echoing_content_or_identity_path(self):
        secret = "Synthetic Cover Identity"
        self._write_manifest(title=secret)
        binary_path = self.candidate / "Synthetic-Cover-Identity.png"
        binary_path.write_bytes(b"\x89PNG\r\n\x1a\n\x00" + secret.encode("utf-8"))
        stdout = io.StringIO()
        stderr = io.StringIO()
        with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr):
            status = main(
                [str(self.candidate), "--sources-manifest", str(self.manifest)]
            )
        self.assertEqual(2, status)
        self.assertEqual("", stderr.getvalue())
        report = json.loads(stdout.getvalue())
        self.assertEqual("CANDIDATE_BINARY_UNAUDITED", report["input_error"]["code"])
        serialized = stdout.getvalue().casefold()
        self.assertNotIn(secret.casefold(), serialized)
        self.assertNotIn("synthetic-cover-identity", serialized)
        compact_serialized = "".join(
            character for character in serialized if character.isalnum()
        )
        self.assertNotIn("syntheticcoveridentity", compact_serialized)


    def test_detects_wrapped_identity_and_attribution_phrase(self):
        self._write_candidate(
            "SKILL.md",
            "Apply Velvet\nQuasar Methods to the target.\n"
            "According to the\nauthor, reuse the framework.\n",
        )
        report = audit_candidate_disclosure(self.candidate, self.manifest)
        title_findings = self._identity_findings(report, "title")
        attribution_findings = [
            finding
            for finding in report["findings"]
            if finding["code"] == "GENERIC_SOURCE_ATTRIBUTION"
        ]
        self.assertTrue(title_findings)
        self.assertTrue(attribution_findings)
        self.assertEqual(1, title_findings[0]["candidate_location"]["line"])
        self.assertEqual(3, attribution_findings[0]["candidate_location"]["line"])

    def test_ascii_compact_match_preserves_word_boundaries(self):
        self._write_manifest(title="Signal")
        self._write_candidate("SKILL.md", "Analyze signaling dynamics.\n")
        neutral_report = audit_candidate_disclosure(self.candidate, self.manifest)
        self.assertFalse(self._identity_findings(neutral_report, "title"))

        self._write_candidate("SKILL.md", "Apply Sig-nal to the task.\n")
        disclosure_report = audit_candidate_disclosure(self.candidate, self.manifest)
        findings = self._identity_findings(disclosure_report, "title")
        self.assertTrue(findings)
        self.assertEqual("compact", findings[0]["match_mode"])

    def test_cache_artifact_rejection_is_case_insensitive(self):
        cache_dir = self.candidate / "__PYCACHE__"
        cache_dir.mkdir()
        with self.assertRaises(DisclosureInputError) as caught:
            audit_candidate_disclosure(self.candidate, self.manifest)
        self.assertEqual("CANDIDATE_CACHE_ARTIFACT", caught.exception.code)
        cache_dir.rmdir()

        cache_file = self.candidate / "artifact.PYC"
        cache_file.write_bytes(b"synthetic cache")
        with self.assertRaises(DisclosureInputError) as caught:
            audit_candidate_disclosure(self.candidate, self.manifest)
        self.assertEqual("CANDIDATE_CACHE_ARTIFACT", caught.exception.code)


if __name__ == "__main__":
    unittest.main()

from __future__ import annotations

import contextlib
import hashlib
import io
import os
import tempfile
import threading
import unittest
from pathlib import Path
from unittest import mock

import hash_candidate_tree as hash_candidate_tree_module
from hash_candidate_tree import (
    TREE_HASH_PREFIX,
    CandidateTreeError,
    candidate_tree_sha256,
    main,
)


class CandidateTreeHashTests(unittest.TestCase):
    def setUp(self):
        self.tempdir = tempfile.TemporaryDirectory()
        self.root = Path(self.tempdir.name)
        self.candidate_path = "candidates/test-skill"
        self.candidate = self.root / self.candidate_path
        (self.candidate / "references").mkdir(parents=True)
        (self.candidate / "SKILL.md").write_bytes(b"skill-body\n")
        (self.candidate / "references" / "policy.md").write_bytes(b"policy\x00bytes\n")

    def tearDown(self):
        self.tempdir.cleanup()

    @staticmethod
    def manual_hash(files):
        digest = hashlib.sha256()
        digest.update(TREE_HASH_PREFIX)
        digest.update(len(files).to_bytes(8, "big"))
        for relative_path, content in sorted(
            files, key=lambda item: item[0].encode("utf-8")
        ):
            path_bytes = relative_path.encode("utf-8")
            digest.update(b"F\0")
            digest.update(len(path_bytes).to_bytes(8, "big"))
            digest.update(path_bytes)
            digest.update(len(content).to_bytes(8, "big"))
            digest.update(content)
        return f"sha256:{digest.hexdigest()}"

    def test_hash_is_deterministic_prefixed_and_exactly_framed(self):
        first = candidate_tree_sha256(self.root, self.candidate_path)
        second = candidate_tree_sha256(self.root, self.candidate_path)
        expected = self.manual_hash([
            ("SKILL.md", b"skill-body\n"),
            ("references/policy.md", b"policy\x00bytes\n"),
        ])
        self.assertEqual(expected, first)
        self.assertEqual(first, second)
        self.assertRegex(first, r"^sha256:[0-9a-f]{64}$")

    def test_content_and_relative_path_changes_change_hash(self):
        original = candidate_tree_sha256(self.root, self.candidate_path)
        policy = self.candidate / "references" / "policy.md"
        policy.write_bytes(b"changed\n")
        content_changed = candidate_tree_sha256(self.root, self.candidate_path)
        self.assertNotEqual(original, content_changed)

        policy.write_bytes(b"policy\x00bytes\n")
        policy.rename(self.candidate / "references" / "renamed.md")
        path_changed = candidate_tree_sha256(self.root, self.candidate_path)
        self.assertNotEqual(original, path_changed)

    def test_rejects_file_mutated_after_its_read_while_hashing_continues(self):
        target = self.candidate / "SKILL.md"
        mutation_requested = threading.Event()
        mutation_finished = threading.Event()
        original_read = hash_candidate_tree_module._read_stable_file

        def mutate_after_read():
            self.assertTrue(mutation_requested.wait(timeout=5))
            target.write_bytes(b"changed after the stable read\n")
            mutation_finished.set()

        def read_then_wait_for_mutation(record):
            content = original_read(record)
            if record.relative_path == "SKILL.md":
                mutation_requested.set()
                self.assertTrue(mutation_finished.wait(timeout=5))
            return content

        worker = threading.Thread(target=mutate_after_read)
        worker.start()
        try:
            with mock.patch.object(
                hash_candidate_tree_module,
                "_read_stable_file",
                side_effect=read_then_wait_for_mutation,
            ):
                with self.assertRaises(CandidateTreeError) as caught:
                    candidate_tree_sha256(self.root, self.candidate_path)
            self.assertEqual("CANDIDATE_TREE_CHANGED", caught.exception.code)
            self.assertEqual("SKILL.md", caught.exception.path)
        finally:
            mutation_requested.set()
            worker.join(timeout=5)
            self.assertFalse(worker.is_alive())

    def test_cli_outputs_only_prefixed_hash_on_success(self):
        stdout = io.StringIO()
        stderr = io.StringIO()
        with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr):
            status = main([str(self.root), self.candidate_path])
        self.assertEqual(0, status)
        self.assertEqual("", stderr.getvalue())
        self.assertEqual(
            candidate_tree_sha256(self.root, self.candidate_path),
            stdout.getvalue().strip(),
        )

    def test_rejects_absolute_parent_and_noncanonical_paths(self):
        for unsafe in (
            "/tmp/test-skill",
            "../test-skill",
            "candidates/../test-skill",
            "candidates//test-skill",
            "candidates/./test-skill",
            "candidates\\test-skill",
        ):
            with self.subTest(candidate_path=unsafe):
                with self.assertRaises(CandidateTreeError) as caught:
                    candidate_tree_sha256(self.root, unsafe)
                self.assertEqual("CANDIDATE_PATH_INVALID", caught.exception.code)

    def test_rejects_symlink_without_following_it(self):
        link = self.candidate / "linked-policy.md"
        try:
            os.symlink(self.candidate / "references" / "policy.md", link)
        except (OSError, NotImplementedError) as exc:
            self.skipTest(f"symlinks unavailable: {exc}")
        with self.assertRaises(CandidateTreeError) as caught:
            candidate_tree_sha256(self.root, self.candidate_path)
        self.assertEqual("CANDIDATE_TREE_SYMLINK", caught.exception.code)

    def test_rejects_python_cache_directory_and_pyc_file(self):
        cache = self.candidate / "__pycache__"
        cache.mkdir()
        with self.assertRaises(CandidateTreeError) as caught:
            candidate_tree_sha256(self.root, self.candidate_path)
        self.assertEqual("CANDIDATE_CACHE_ARTIFACT", caught.exception.code)
        cache.rmdir()

        (self.candidate / "compiled.pyc").write_bytes(b"bytecode")
        with self.assertRaises(CandidateTreeError) as caught:
            candidate_tree_sha256(self.root, self.candidate_path)
        self.assertEqual("CANDIDATE_CACHE_ARTIFACT", caught.exception.code)


if __name__ == "__main__":
    unittest.main()

from __future__ import annotations

import contextlib
import copy
import hashlib
import io
import json
import shutil
import sys
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch

import yaml

SCRIPT_DIR = Path(__file__).resolve().parent
sys.path.insert(0, str(SCRIPT_DIR))

from hash_candidate_tree import CandidateTreeError, candidate_tree_sha256
from inspect_distillation_state import (
    StateInspectionError,
    inspect_distillation_state,
    main,
)
from test_validate_distillation import (
    FIXTURE_CANDIDATE_PATH,
    FIXTURE_SKILL,
    FIXTURE_TASK_DEFINITION,
    FIXTURE_TRIGGER_DEFINITION,
    set_rule_pending,
    sync_task_contract_snapshot,
    sync_current_gate3_approval_snapshot,
    valid_documents,
    valid_sources_manifest,
)
from validate_distillation import validate_distillation as run_validator


class DistillationStateInspectionTests(unittest.TestCase):
    def setUp(self):
        self.tempdir = tempfile.TemporaryDirectory()
        self.root = Path(self.tempdir.name)
        self.candidate = self.root / FIXTURE_CANDIDATE_PATH
        self.candidate.mkdir(parents=True)
        (self.candidate / "SKILL.md").write_text(FIXTURE_SKILL, encoding="utf-8")
        eval_dir = self.candidate / "evals"
        eval_dir.mkdir()
        (eval_dir / "trigger-cases.json").write_text(
            json.dumps(FIXTURE_TRIGGER_DEFINITION, ensure_ascii=False, indent=2) + "\n",
            encoding="utf-8",
        )
        (eval_dir / "task-cases.json").write_text(
            json.dumps(FIXTURE_TASK_DEFINITION, ensure_ascii=False, indent=2) + "\n",
            encoding="utf-8",
        )
        self.documents = valid_documents()
        self._refresh_recorded_hash()
        self.manifest = self.root / "sources.yml"
        self.manifest.write_text(
            yaml.safe_dump(valid_sources_manifest(), allow_unicode=True, sort_keys=False),
            encoding="utf-8",
        )
        self.write_documents()

    def tearDown(self):
        self.tempdir.cleanup()

    def _refresh_recorded_hash(self):
        materializations = self.documents["gate-decisions.yml"]["materializations"]
        if materializations:
            candidate_hash = candidate_tree_sha256(
                self.root, FIXTURE_CANDIDATE_PATH
            )
            materializations[0]["candidate_hash"] = candidate_hash
            materializations[0]["quick_validation"]["candidate_hash"] = candidate_hash

    def write_documents(self, *, sync_approval_snapshot=True):
        sync_task_contract_snapshot(self.documents, self.root)
        for name in (
            "evidence-ledger.yml",
            "concept-map.yml",
            "capability-rules.yml",
        ):
            document = self.documents[name]
            (self.root / name).write_text(
                yaml.safe_dump(document, allow_unicode=True, sort_keys=False),
                encoding="utf-8",
            )
        if sync_approval_snapshot:
            sync_current_gate3_approval_snapshot(self.documents, self.root)
        for name, document in self.documents.items():
            if name in {
                "evidence-ledger.yml",
                "concept-map.yml",
                "capability-rules.yml",
                "task-contract.yml",
                "task-coverage.yml",
            }:
                continue
            (self.root / name).write_text(
                yaml.safe_dump(document, allow_unicode=True, sort_keys=False),
                encoding="utf-8",
            )

    def inspect(self, *, with_manifest=True):
        return inspect_distillation_state(
            self.root,
            "candidate-001",
            FIXTURE_CANDIDATE_PATH,
            self.manifest if with_manifest else None,
        )

    def assert_fixture_valid(self):
        report = run_validator(self.root)
        self.assertTrue(report.ok, [item.as_dict() for item in report.errors])

    def test_exact_match_is_only_gate4_eligible_not_behavior_pass(self):
        self.assert_fixture_valid()
        report = self.inspect()
        self.assertEqual("gate4-eligible", report["activation_state"])
        self.assertEqual(["materialization-001"], report["materializations"]["matching"])
        self.assertFalse(report["truth_assessed"])
        self.assertFalse(report["behavior_effectiveness_assessed"])
        self.assertFalse(report["gate4_accepted"])
        self.assertEqual(0, report["completed_pass_eval_count"])
        snapshot = report["current_gate3"]["approval_snapshot"]
        self.assertTrue(snapshot["present"])
        self.assertEqual(FIXTURE_CANDIDATE_PATH, snapshot["candidate_path"])
        self.assertEqual(
            report["candidate"]["computed_hash"], snapshot["candidate_hash"]
        )
        self.assertTrue(
            report["current_gate3"]["approval_snapshot_matches_candidate"]
        )
        product = report["authoritative_product_contract"]
        self.assertEqual("matches", product["task_contract_drift"])
        self.assertEqual(["stable-task-001"], product["active_stable_task_ids"])
        self.assertEqual("gate4-eligible", report["continuation_mode"])

    def test_legacy_gate3_reports_review_repair_only(self):
        gate3 = self.documents["gate-decisions.yml"]["gate_decisions"][2]
        gate3["approval_snapshot"] = {
            key: value for key, value in gate3["approval_snapshot"].items()
            if key in {"contract", "candidate_path", "candidate_hash", "governance_hashes"}
        }
        gate3["approval_snapshot"]["contract"] = "gate3-approval-snapshot:v1"
        self.write_documents(sync_approval_snapshot=False)
        report = self.inspect()
        self.assertEqual("invalid", report["activation_state"])
        self.assertEqual("review-repair-only", report["continuation_mode"])
        self.assertIn("review", report["allowed_actions"])

    def test_checkpoint_cannot_supersede_product_contract(self):
        gate1 = self.documents["gate-decisions.yml"]["gate_decisions"][0]
        snapshot = gate1["task_contract_snapshot"]
        checkpoint = {
            "schema_version": 1,
            "distillation_id": "validator-test-v1",
            "checkpoint_id": "checkpoint-001",
            "created_at": "2026-08-07",
            "product_contract_anchor": {
                "path": snapshot["task_contract_path"],
                "sha256": snapshot["task_contract_hash"],
                "task_contract_id": snapshot["task_contract_id"],
                "contract_version": snapshot["contract_version"],
                "active_stable_task_ids": snapshot["active_stable_task_ids"],
            },
            "current_stage_objective": {
                "gate": "gate-3",
                "statement": "Exclude the external task after compression.",
                "stable_task_ids": ["stable-task-001"],
                "supersedes_product_contract": True,
                "excluded_stable_task_ids": ["stable-task-001"],
            },
            "temporary_operational_constraints": ["Review only."],
        }
        (self.root / "context-checkpoint.yml").write_text(
            yaml.safe_dump(checkpoint, sort_keys=False), encoding="utf-8"
        )
        report = self.inspect()
        self.assertEqual("invalid", report["activation_state"])
        self.assertEqual("review-repair-only", report["continuation_mode"])
        self.assertTrue(any(
            item["code"] == "CHECKPOINT_PRODUCT_CONTRACT_CONFLICT"
            for item in report["blockers"]
        ))

    def test_report_and_cli_output_are_deterministic(self):
        first = json.dumps(self.inspect(), ensure_ascii=False, indent=2, sort_keys=True)
        second = json.dumps(self.inspect(), ensure_ascii=False, indent=2, sort_keys=True)
        self.assertEqual(first, second)

        outputs = []
        for _ in range(2):
            stdout = io.StringIO()
            with contextlib.redirect_stdout(stdout):
                status = main([
                    str(self.root),
                    "--candidate-id", "candidate-001",
                    "--candidate-path", FIXTURE_CANDIDATE_PATH,
                    "--sources-manifest", str(self.manifest),
                ])
            self.assertEqual(0, status)
            outputs.append(stdout.getvalue())
        self.assertEqual(outputs[0], outputs[1])

    def test_pending_revise_or_rejected_gate3_is_review_only(self):
        for decision in ("pending", "revise", "rejected"):
            with self.subTest(decision=decision):
                documents = valid_documents()
                set_rule_pending(documents)
                gate3 = documents["gate-decisions.yml"]["gate_decisions"][2]
                gate3["decision"] = decision
                if decision == "pending":
                    gate3.update({
                        "reviewer_type": None,
                        "reviewer": None,
                        "decided_at": None,
                    })
                else:
                    gate3.update({
                        "reviewer_type": "user",
                        "reviewer": "project-user",
                        "decided_at": "2026-08-04",
                        "rationale": f"Gate 3 {decision} for the synthetic fixture.",
                    })
                self.documents = documents
                self.write_documents()
                self.assert_fixture_valid()
                self.assertEqual("review-only", self.inspect()["activation_state"])

    def test_approval_without_materialization_requires_materialization(self):
        self.documents["gate-decisions.yml"]["materializations"] = []
        self.write_documents()
        self.assert_fixture_valid()
        report = self.inspect()
        self.assertEqual("materialization-required", report["activation_state"])
        self.assertEqual([], report["materializations"]["matching"])

    def test_missing_tree_after_approval_is_invalid_even_before_materialization(self):
        self.documents["gate-decisions.yml"]["materializations"] = []
        self.write_documents()
        shutil.rmtree(self.candidate)
        report = self.inspect()
        self.assertEqual("invalid", report["activation_state"])
        self.assertIsNone(report["candidate"]["computed_hash"])
        self.assertEqual("not-yet-materialized", report["candidate"]["tree_status"])
        self.assertIn(
            "APPROVAL_SNAPSHOT_CANDIDATE_HASH_MISMATCH",
            {item["code"] for item in report["blockers"]},
        )

    def test_invalidated_or_legacy_materialization_is_historical_only(self):
        materialization = self.documents["gate-decisions.yml"]["materializations"][0]
        for status in ("invalidated", "legacy-quarantined"):
            with self.subTest(status=status):
                materialization["status"] = status
                self.write_documents()
                self.assert_fixture_valid()
                report = self.inspect()
                self.assertEqual("materialization-required", report["activation_state"])
                self.assertEqual([], report["materializations"]["matching"])

    def test_candidate_byte_change_never_remains_eligible(self):
        (self.candidate / "SKILL.md").write_text(
            FIXTURE_SKILL + "\nChanged.\n", encoding="utf-8"
        )
        report = self.inspect()
        self.assertEqual("invalid", report["activation_state"])
        self.assertIn(
            "MATERIALIZATION_CANDIDATE_HASH_MISMATCH",
            {item["code"] for item in report["blockers"]},
        )
        self.assertIn(
            "APPROVAL_SNAPSHOT_CANDIDATE_HASH_MISMATCH",
            {item["code"] for item in report["blockers"]},
        )

    def test_missing_or_malformed_approval_snapshot_is_invalid(self):
        gate3 = self.documents["gate-decisions.yml"]["gate_decisions"][2]
        for snapshot in (None, [], {"contract": "wrong"}):
            with self.subTest(snapshot=repr(snapshot)):
                self.documents = valid_documents()
                self._refresh_recorded_hash()
                self.write_documents()
                gate3 = self.documents["gate-decisions.yml"]["gate_decisions"][2]
                if snapshot is None:
                    gate3.pop("approval_snapshot", None)
                else:
                    gate3["approval_snapshot"] = snapshot
                self.write_documents(sync_approval_snapshot=False)
                report = self.inspect()
                self.assertEqual("invalid", report["activation_state"])
                self.assertIn(
                    "APPROVAL_SNAPSHOT_INVALID",
                    {item["code"] for item in report["blockers"]},
                )

    def test_cli_candidate_path_must_exactly_match_approval_snapshot(self):
        alternative_path = "alternate/review-book-task"
        alternative = self.root / alternative_path
        shutil.copytree(self.candidate, alternative)
        report = inspect_distillation_state(
            self.root,
            "candidate-001",
            alternative_path,
            self.manifest,
        )
        self.assertEqual("invalid", report["activation_state"])
        self.assertIn(
            "APPROVAL_SNAPSHOT_CANDIDATE_PATH_MISMATCH",
            {item["code"] for item in report["blockers"]},
        )

    def test_failed_quick_validation_is_invalid(self):
        quick = self.documents["gate-decisions.yml"]["materializations"][0][
            "quick_validation"
        ]
        quick["status"] = "fail"
        self.write_documents()
        report = self.inspect()
        self.assertEqual("invalid", report["activation_state"])
        self.assertIn(
            "MATERIALIZATION_QUICK_VALIDATION_REQUIRED",
            {item["code"] for item in report["blockers"]},
        )

    def test_gate4_eligibility_requires_explicit_manifest(self):
        report = self.inspect(with_manifest=False)
        self.assertEqual("invalid", report["activation_state"])
        self.assertIn(
            "SOURCES_MANIFEST_REQUIRED_FOR_GATE4",
            {item["code"] for item in report["blockers"]},
        )
        self.assertFalse(report["validation"]["sources_manifest_provided"])

    def test_governance_hash_drift_is_invalid(self):
        evidence_path = self.root / "evidence-ledger.yml"
        evidence_path.write_bytes(evidence_path.read_bytes() + b"\n")
        report = self.inspect()
        self.assertEqual("invalid", report["activation_state"])
        self.assertIn(
            "GATE3_APPROVAL_GOVERNANCE_HASH_MISMATCH",
            {item["code"] for item in report["blockers"]},
        )

    def test_completed_materialization_must_match_approval_snapshot(self):
        alternative_path = "alternate/review-book-task"
        shutil.copytree(self.candidate, self.root / alternative_path)
        materialization = self.documents["gate-decisions.yml"]["materializations"][0]
        materialization["candidate_path"] = alternative_path
        self.write_documents()
        report = self.inspect()
        self.assertEqual("invalid", report["activation_state"])
        self.assertIn(
            "MATERIALIZATION_APPROVAL_PATH_MISMATCH",
            {item["code"] for item in report["blockers"]},
        )

    def test_nonreview_lifecycle_never_routes_to_gate4(self):
        candidate = self.documents["capability-rules.yml"]["skill_candidates"][0]
        for lifecycle in ("draft", "rejected", "deprecated"):
            with self.subTest(lifecycle=lifecycle):
                candidate["lifecycle"] = lifecycle
                self.write_documents()
                report = self.inspect()
                self.assertEqual("invalid", report["activation_state"])
                self.assertIn(
                    "CANDIDATE_LIFECYCLE_NOT_REVIEW",
                    {item["code"] for item in report["blockers"]},
                )

    def test_malformed_rule_ids_returns_invalid_instead_of_crashing(self):
        self.documents["gate-decisions.yml"]["materializations"][0]["rule_ids"] = [
            ["rule-001"]
        ]
        self.write_documents()
        report = self.inspect()
        self.assertEqual("invalid", report["activation_state"])
        self.assertIn(
            "approved-rule-set-mismatch",
            report["materializations"]["stale_completed"][0]["reasons"],
        )

    def test_duplicate_yaml_key_is_rejected(self):
        gate_path = self.root / "gate-decisions.yml"
        text = gate_path.read_text(encoding="utf-8")
        text = text.replace(
            "  decision: approved-for-eval\n",
            "  decision: rejected\n  decision: approved-for-eval\n",
            1,
        )
        gate_path.write_text(text, encoding="utf-8")
        with self.assertRaises(StateInspectionError) as caught:
            self.inspect()
        self.assertEqual("YAML_PARSE", caught.exception.code)

    def test_governance_change_during_validation_fails_closed(self):
        original = run_validator

        def mutate_then_validate(root, manifest):
            set_rule_pending(self.documents)
            self.write_documents()
            return original(root, manifest)

        with patch(
            "inspect_distillation_state.validate_distillation",
            side_effect=mutate_then_validate,
        ):
            report = self.inspect()
        self.assertEqual("invalid", report["activation_state"])
        self.assertIn(
            "INSPECTION_SNAPSHOT_CHANGED",
            {item["code"] for item in report["blockers"]},
        )

    def test_write_restore_cannot_mix_routing_and_validator_snapshots(self):
        original = run_validator
        tracked_paths = [self.root / name for name in self.documents]
        original_bytes = {path: path.read_bytes() for path in tracked_paths}

        def switch_to_b_validate_then_restore(root, manifest):
            extra = copy.deepcopy(
                self.documents["evidence-ledger.yml"]["evidence"][0]
            )
            extra["evidence_id"] = "ev-002"
            self.documents["evidence-ledger.yml"]["evidence"].append(extra)
            self.write_documents()
            try:
                return original(root, manifest)
            finally:
                for path, payload in original_bytes.items():
                    path.write_bytes(payload)

        with patch(
            "inspect_distillation_state.validate_distillation",
            side_effect=switch_to_b_validate_then_restore,
        ):
            report = self.inspect()
        self.assertEqual("gate4-eligible", report["activation_state"])
        self.assertEqual(1, report["validation"]["metrics"]["evidence_count"])

    def test_external_manifest_write_restore_uses_frozen_bytes(self):
        original = run_validator
        with tempfile.TemporaryDirectory() as external_dir:
            manifest = Path(external_dir) / "manifests" / "sources.yml"
            manifest.parent.mkdir()
            manifest.write_text(
                yaml.safe_dump(
                    valid_sources_manifest(), allow_unicode=True, sort_keys=False
                ),
                encoding="utf-8",
            )
            original_bytes = manifest.read_bytes()

            def switch_manifest_then_restore(root, snapshot_manifest):
                changed = valid_sources_manifest()
                changed["sources"][0]["id"] = "different-book"
                manifest.write_text(
                    yaml.safe_dump(changed, allow_unicode=True, sort_keys=False),
                    encoding="utf-8",
                )
                try:
                    return original(root, snapshot_manifest)
                finally:
                    manifest.write_bytes(original_bytes)

            with patch(
                "inspect_distillation_state.validate_distillation",
                side_effect=switch_manifest_then_restore,
            ):
                report = inspect_distillation_state(
                    self.root,
                    "candidate-001",
                    FIXTURE_CANDIDATE_PATH,
                    manifest,
                )
        self.assertEqual("gate4-eligible", report["activation_state"])
        self.assertTrue(report["validation"]["ok"])

    def test_markdown_source_write_restore_uses_frozen_bytes(self):
        original = run_validator
        source_text = "# Policy\n\n## Review Gate\n\nStable policy text.\n"
        evidence = self.documents["evidence-ledger.yml"]["evidence"][0]
        evidence["locator"] = {
            "source_id": "book-001",
            "locator_type": "markdown-section",
            "anchor": "docs/policy.md:5#review-gate",
            "content_hash": hashlib.sha256(
                b"Stable policy text."
            ).hexdigest()[:12],
        }
        evidence["raw_text"] = "Stable policy text."
        evidence["normalized_text"] = "Stable policy text."
        self.write_documents()

        with tempfile.TemporaryDirectory() as external_dir:
            project = Path(external_dir) / "project"
            manifest = project / "manifests" / "sources.yml"
            source = project / "docs" / "policy.md"
            manifest.parent.mkdir(parents=True)
            source.parent.mkdir(parents=True)
            manifest_data = valid_sources_manifest()
            manifest_data["sources"][0].update({
                "local_path": "docs/policy.md",
                "related_local_paths": [],
            })
            manifest.write_text(
                yaml.safe_dump(manifest_data, allow_unicode=True, sort_keys=False),
                encoding="utf-8",
            )
            source.write_text(source_text, encoding="utf-8")
            original_source = source.read_bytes()

            def switch_source_then_restore(root, snapshot_manifest):
                source.write_text("# Policy\n\nChanged text.\n", encoding="utf-8")
                try:
                    return original(root, snapshot_manifest)
                finally:
                    source.write_bytes(original_source)

            with patch(
                "inspect_distillation_state.validate_distillation",
                side_effect=switch_source_then_restore,
            ):
                report = inspect_distillation_state(
                    self.root,
                    "candidate-001",
                    FIXTURE_CANDIDATE_PATH,
                    manifest,
                )
        self.assertEqual("gate4-eligible", report["activation_state"])
        self.assertTrue(report["validation"]["ok"])

    def test_old_gate_or_rule_mismatch_is_invalid(self):
        variants = (
            ("gate3_decision_id", "gate-decision-old"),
            ("rule_ids", ["rule-missing"]),
        )
        for field, value in variants:
            with self.subTest(field=field):
                self.documents = valid_documents()
                self._refresh_recorded_hash()
                self.documents["gate-decisions.yml"]["materializations"][0][field] = value
                self.write_documents()
                self.assertEqual("invalid", self.inspect()["activation_state"])

    def test_duplicate_exact_materializations_are_ambiguous(self):
        duplicate = copy.deepcopy(
            self.documents["gate-decisions.yml"]["materializations"][0]
        )
        duplicate["materialization_id"] = "materialization-002"
        self.documents["gate-decisions.yml"]["materializations"].append(duplicate)
        self.write_documents()
        self.assert_fixture_valid()
        report = self.inspect()
        self.assertEqual("invalid", report["activation_state"])
        self.assertIn(
            "AMBIGUOUS_COMPLETED_MATERIALIZATION",
            {item["code"] for item in report["blockers"]},
        )

    def test_unknown_candidate_and_unsafe_path_fail_closed(self):
        with self.assertRaises(StateInspectionError) as caught:
            inspect_distillation_state(
                self.root, "candidate-missing", FIXTURE_CANDIDATE_PATH
            )
        self.assertEqual("CANDIDATE_SELECTION_ERROR", caught.exception.code)

        with self.assertRaises(CandidateTreeError):
            inspect_distillation_state(
                self.root, "candidate-001", "../review-book-task"
            )

    def test_candidate_path_must_end_with_candidate_name(self):
        wrong = self.root / "candidates" / "different-name"
        wrong.mkdir()
        (wrong / "SKILL.md").write_text(FIXTURE_SKILL, encoding="utf-8")
        with self.assertRaises(StateInspectionError) as caught:
            inspect_distillation_state(
                self.root, "candidate-001", "candidates/different-name"
            )
        self.assertEqual("CANDIDATE_PATH_NAME_MISMATCH", caught.exception.code)


if __name__ == "__main__":
    unittest.main()

#!/usr/bin/env python3
"""Small standard-library regression test for normalize_docx_source.py."""

from __future__ import annotations

import base64
import hashlib
import json
import tempfile
import unittest
import zipfile
from pathlib import Path

from normalize_docx_source import NormalizationError, normalize_docx


ONE_PIXEL_PNG = base64.b64decode(
    "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNk+A8AAQUBAScY42YAAAAASUVORK5CYII="
)

DOCUMENT_XML = """<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"
 xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"
 xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main">
 <w:body>
  <w:p><w:pPr><w:pStyle w:val="Heading1"/></w:pPr><w:r><w:t>第一章</w:t></w:r></w:p>
  <w:p><w:r><w:t xml:space="preserve">Cafe\u0301   test</w:t></w:r></w:p>
  <w:p><w:r><w:drawing><a:blip r:embed="rId5"/></w:drawing></w:r></w:p>
  <w:tbl>
   <w:tr><w:tc><w:p><w:r><w:t>A</w:t></w:r></w:p></w:tc><w:tc><w:p><w:r><w:t>B</w:t></w:r></w:p></w:tc></w:tr>
   <w:tr><w:tc><w:p><w:r><w:t>C</w:t></w:r></w:p></w:tc><w:tc><w:p><w:r><w:t>D</w:t></w:r></w:p></w:tc></w:tr>
  </w:tbl>
  <w:sectPr/>
 </w:body>
</w:document>
"""

STYLES_XML = """<?xml version="1.0" encoding="UTF-8"?>
<w:styles xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main">
 <w:style w:type="paragraph" w:styleId="Heading1">
  <w:name w:val="heading 1"/><w:pPr><w:outlineLvl w:val="0"/></w:pPr>
 </w:style>
</w:styles>
"""

RELS_XML = """<?xml version="1.0" encoding="UTF-8"?>
<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">
 <Relationship Id="rId5" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/image" Target="media/image1.png"/>
</Relationships>
"""

CORE_XML = """<?xml version="1.0" encoding="UTF-8"?>
<cp:coreProperties xmlns:cp="http://schemas.openxmlformats.org/package/2006/metadata/core-properties"
 xmlns:dc="http://purl.org/dc/elements/1.1/">
 <dc:title>测试图书</dc:title><dc:creator>测试作者</dc:creator>
</cp:coreProperties>
"""


def sha256(path: Path) -> str:
    return hashlib.sha256(path.read_bytes()).hexdigest()


def create_fixture(path: Path) -> None:
    with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_DEFLATED) as package:
        package.writestr("word/document.xml", DOCUMENT_XML)
        package.writestr("word/styles.xml", STYLES_XML)
        package.writestr("word/_rels/document.xml.rels", RELS_XML)
        package.writestr("word/media/image1.png", ONE_PIXEL_PNG)
        package.writestr("docProps/core.xml", CORE_XML)


class NormalizeDocxSourceTest(unittest.TestCase):
    def test_normalizes_deterministically_and_preserves_source_and_media(self) -> None:
        with tempfile.TemporaryDirectory() as temporary:
            root = Path(temporary)
            source = root / "fixture.docx"
            first = root / "first"
            second = root / "second"
            create_fixture(source)
            source_before = sha256(source)

            summary = normalize_docx(source, first, "book-test-001")
            normalize_docx(source, second, "book-test-001")

            self.assertEqual(source_before, sha256(source))
            self.assertTrue(summary["source_unchanged"])
            self.assertEqual(summary["block_count"], 4)
            self.assertEqual(summary["heading_count"], 1)
            self.assertEqual(summary["table_count"], 1)
            self.assertEqual(summary["image_asset_count"], 1)
            self.assertEqual((first / "media/image1.png").read_bytes(), ONE_PIXEL_PNG)

            records = [json.loads(line) for line in (first / "blocks.jsonl").read_text(encoding="utf-8").splitlines()]
            self.assertEqual(records[1]["raw_text"], "Cafe\u0301   test")
            self.assertEqual(records[1]["normalized_text"], "Caf\u00e9 test")
            self.assertEqual(records[3]["normalized_text"], "A\tB\nC\tD")
            self.assertEqual(records[2]["locator"]["heading_path"], ["第一章"])
            self.assertEqual(records[2]["locator"]["figure_ids"], ["figure-000001"])
            self.assertEqual(records[3]["locator"]["table_id"], "table-000001")

            first_files = {
                path.relative_to(first).as_posix(): path.read_bytes()
                for path in first.rglob("*") if path.is_file()
            }
            second_files = {
                path.relative_to(second).as_posix(): path.read_bytes()
                for path in second.rglob("*") if path.is_file()
            }
            self.assertEqual(first_files, second_files)

    def test_refuses_to_overwrite_nonempty_output(self) -> None:
        with tempfile.TemporaryDirectory() as temporary:
            root = Path(temporary)
            source = root / "fixture.docx"
            output = root / "output"
            create_fixture(source)
            output.mkdir()
            (output / "keep.txt").write_text("do not overwrite", encoding="utf-8")

            with self.assertRaises(NormalizationError):
                normalize_docx(source, output, "book-test-001")
            self.assertEqual((output / "keep.txt").read_text(encoding="utf-8"), "do not overwrite")


if __name__ == "__main__":
    unittest.main()

from __future__ import annotations

import hashlib
import json
import subprocess
import tempfile
import unittest
from pathlib import Path

import yaml

from ocr_book_source import (
    OCRRunError,
    run_docx_bundle,
    run_scanned_pdf,
)
from verify_ocr_locators import verify


TSV = (
    "level\tpage_num\tblock_num\tpar_num\tline_num\tword_num\tleft\ttop\twidth\theight\tconf\ttext\n"
    "5\t1\t1\t1\t1\t1\t10\t20\t30\t12\t91.5\tAlpha\n"
)
EMPTY_TSV = "level\tpage_num\tblock_num\tpar_num\tline_num\tword_num\tleft\ttop\twidth\theight\tconf\ttext\n"


class FakeRunner:
    def __init__(self, *, tsv=TSV, fail_item=False):
        self.tsv = tsv
        self.plain_text = "" if tsv == EMPTY_TSV else "Alpha\r\n"
        self.fail_item = fail_item
        self.calls = []

    def __call__(self, args, **kwargs):
        self.calls.append((list(args), dict(kwargs)))
        self.assert_safe(kwargs)
        if args[:2] == ["tesseract", "--version"]:
            return subprocess.CompletedProcess(args, 0, "tesseract 5.3.0\n", "")
        if args[:2] == ["tesseract", "--list-langs"]:
            return subprocess.CompletedProcess(args, 0, "List of available languages (2):\neng\nchi_sim\n", "")
        if args[:2] == ["pdftoppm", "-v"]:
            return subprocess.CompletedProcess(args, 0, "", "pdftoppm version 24.01\n")
        if args[0] == "pdfinfo":
            return subprocess.CompletedProcess(args, 0, "Pages:          2\n", "")
        if args[0] == "pdftoppm":
            Path(args[-1]).with_suffix(".png").write_bytes(
                b"page-" + args[args.index("-f") + 1].encode()
            )
            return subprocess.CompletedProcess(args, 0, "", "")
        if args[0] == "tesseract":
            if self.fail_item:
                return subprocess.CompletedProcess(args, 1, "", "synthetic OCR failure")
            if args[-1] == "tsv":
                return subprocess.CompletedProcess(args, 0, self.tsv, "")
            return subprocess.CompletedProcess(args, 0, self.plain_text, "")
        raise AssertionError(args)

    @staticmethod
    def assert_safe(kwargs):
        if kwargs.get("shell") is not False:
            raise AssertionError("OCR runner must never use a shell")


def all_tools(_name):
    return "/usr/bin/fake"


class OCRBookSourceTests(unittest.TestCase):
    def setUp(self):
        self.tempdir = tempfile.TemporaryDirectory()
        self.root = Path(self.tempdir.name)
        self.source_id = "book-ocr-001"
        self.raw_source = self.root / "raw" / "book.docx"
        self.raw_source.parent.mkdir()
        self.raw_source.write_bytes(b"synthetic-docx-source")
        self.source_hash = "sha256:" + hashlib.sha256(self.raw_source.read_bytes()).hexdigest()
        self.manifest = self.root / "sources.yml"
        self.write_manifest("book-docx")
        self.bundle = self.root / "normalized" / self.source_id
        (self.bundle / "media").mkdir(parents=True)
        self.image = self.bundle / "media" / "image.png"
        self.image.write_bytes(b"synthetic-image")
        image_hash = hashlib.sha256(self.image.read_bytes()).hexdigest()
        (self.bundle / "media-map.yml").write_text(yaml.safe_dump({
            "schema_version": 1,
            "source_id": self.source_id,
            "assets": [{
                "asset_id": "media-001", "extracted_path": "media/image.png",
                "sha256_extracted": image_hash,
            }],
            "occurrences": [
                {"occurrence_id": "occ-001", "figure_id": "figure-001", "asset_id": "media-001"},
                {"occurrence_id": "occ-002", "figure_id": "figure-002", "asset_id": "media-001"},
            ],
        }, sort_keys=False), encoding="utf-8")
        self.refresh_normalized_checksums()

    def tearDown(self):
        self.tempdir.cleanup()

    def write_manifest(self, source_type):
        coverage = "all-pages" if source_type == "book-pdf-scan" else "all-images"
        self.manifest.write_text(yaml.safe_dump({
            "schema_version": 2,
            "sources": [{
                "id": self.source_id, "type": source_type,
                "source_role": "primary-book", "provenance_role": "method-source",
                "privacy": "private", "checksum": self.source_hash,
                "local_path": self.raw_source.relative_to(self.root).as_posix(),
                "ocr_policy": {
                    "required": True, "coverage": coverage,
                    "execution_mode": "local-only", "engine": "tesseract",
                    "languages": ["eng"],
                    **({"renderer": "poppler-pdftoppm", "dpi": 300} if source_type == "book-pdf-scan" else {}),
                },
            }],
        }, sort_keys=False), encoding="utf-8")

    def set_manifest_languages(self, languages):
        document = yaml.safe_load(self.manifest.read_text(encoding="utf-8"))
        document["sources"][0]["ocr_policy"]["languages"] = list(languages)
        self.manifest.write_text(yaml.safe_dump(document, sort_keys=False), encoding="utf-8")

    def refresh_normalized_checksums(self):
        generated = []
        for path in (self.bundle / "media-map.yml", self.image):
            generated.append({
                "path": path.relative_to(self.bundle).as_posix(),
                "byte_size": path.stat().st_size,
                "sha256": hashlib.sha256(path.read_bytes()).hexdigest(),
            })
        digest = self.source_hash.removeprefix("sha256:")
        (self.bundle / "checksums.yml").write_text(yaml.safe_dump({
            "schema_version": 1, "source_id": self.source_id,
            "source": {
                "sha256_before": digest, "sha256_after": digest,
                "sha256_unchanged": True,
            },
            "generated_files": generated,
        }, sort_keys=False), encoding="utf-8")

    def test_docx_all_occurrences_are_bound_and_duplicate_asset_runs_once(self):
        self.set_manifest_languages(["chi_sim", "eng"])
        before = {
            path.relative_to(self.root).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest()
            for path in (self.raw_source, self.image, self.bundle / "media-map.yml", self.bundle / "checksums.yml")
        }
        runner = FakeRunner()
        output = self.root / "ocr" / self.source_id
        summary = run_docx_bundle(
            self.bundle, output, source_id=self.source_id,
            sources_manifest=self.manifest, languages=["chi_sim", "eng"],
            runner=runner, which=all_tools,
        )
        self.assertEqual("completed", summary["status"])
        self.assertEqual(2, summary["coverage"]["occurrence_count"])
        self.assertEqual(1, summary["coverage"]["attempted_items"])
        tesseract_calls = [call for call, _ in runner.calls if call[0] == "tesseract" and call[1] not in {"--version", "--list-langs"}]
        self.assertEqual(2, len(tesseract_calls))
        record = json.loads((output / "ocr-results.jsonl").read_text().strip())
        self.assertEqual(["occ-001", "occ-002"], record["context"]["media_occurrence_ids"])
        self.assertEqual("Alpha\r\n", record["raw_text"])
        self.assertEqual("Alpha\n", record["normalized_text"])
        after = {
            path.relative_to(self.root).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest()
            for path in (self.raw_source, self.image, self.bundle / "media-map.yml", self.bundle / "checksums.yml")
        }
        self.assertEqual(before, after)

    def test_empty_ocr_is_recorded_not_skipped(self):
        output = self.root / "empty"
        summary = run_docx_bundle(
            self.bundle, output, source_id=self.source_id,
            sources_manifest=self.manifest, languages=["eng"],
            runner=FakeRunner(tsv=EMPTY_TSV), which=all_tools,
        )
        self.assertEqual(1, summary["coverage"]["empty"])
        record = json.loads((output / "ocr-results.jsonl").read_text().strip())
        self.assertEqual("empty", record["status"])
        self.assertIn("ocr-empty", record["quality_flags"])

    def test_ocr_failure_writes_blocked_bundle_and_fails_closed(self):
        output = self.root / "failed"
        with self.assertRaises(OCRRunError) as caught:
            run_docx_bundle(
                self.bundle, output, source_id=self.source_id,
                sources_manifest=self.manifest, languages=["eng"],
                runner=FakeRunner(fail_item=True), which=all_tools,
            )
        self.assertEqual("OCR_COVERAGE_INCOMPLETE", caught.exception.code)
        self.assertEqual("blocked", yaml.safe_load((output / "ocr-manifest.yml").read_text())["status"])

    def test_missing_engine_and_nonempty_output_are_stable_errors(self):
        with self.assertRaises(OCRRunError) as caught:
            run_docx_bundle(
                self.bundle, self.root / "missing", source_id=self.source_id,
                sources_manifest=self.manifest, languages=["eng"],
                runner=FakeRunner(), which=lambda _name: None,
            )
        self.assertEqual("OCR_ENGINE_UNAVAILABLE", caught.exception.code)
        output = self.root / "nonempty"; output.mkdir(); (output / "keep").write_text("x")
        with self.assertRaises(OCRRunError) as caught:
            run_docx_bundle(
                self.bundle, output, source_id=self.source_id,
                sources_manifest=self.manifest, languages=["eng"],
                runner=FakeRunner(), which=all_tools,
            )
        self.assertEqual("OCR_OUTPUT_NOT_EMPTY", caught.exception.code)

    def test_missing_language_pack_and_pdf_renderer_are_stable_errors(self):
        class EnglishOnly(FakeRunner):
            def __call__(self, args, **kwargs):
                if args[:2] == ["tesseract", "--list-langs"]:
                    self.calls.append((list(args), dict(kwargs)))
                    self.assert_safe(kwargs)
                    return subprocess.CompletedProcess(args, 0, "List of available languages (1):\neng\n", "")
                return super().__call__(args, **kwargs)
        self.set_manifest_languages(["chi_sim"])
        with self.assertRaises(OCRRunError) as caught:
            run_docx_bundle(
                self.bundle, self.root / "missing-language", source_id=self.source_id,
                sources_manifest=self.manifest, languages=["chi_sim"],
                runner=EnglishOnly(), which=all_tools,
            )
        self.assertEqual("OCR_LANGUAGE_MISSING", caught.exception.code)

        pdf = self.root / "missing-renderer.pdf"; pdf.write_bytes(b"pdf")
        self.source_hash = "sha256:" + hashlib.sha256(pdf.read_bytes()).hexdigest()
        self.write_manifest("book-pdf-scan")
        with self.assertRaises(OCRRunError) as caught:
            run_scanned_pdf(
                pdf, self.root / "missing-renderer", source_id=self.source_id,
                sources_manifest=self.manifest, languages=["eng"],
                runner=FakeRunner(),
                which=lambda name: None if name == "pdftoppm" else "/usr/bin/fake",
            )
        self.assertEqual("PDF_RENDERER_UNAVAILABLE", caught.exception.code)

    def test_source_policy_language_mismatch_fails_before_ocr(self):
        with self.assertRaises(OCRRunError) as caught:
            run_docx_bundle(
                self.bundle, self.root / "policy-mismatch",
                source_id=self.source_id, sources_manifest=self.manifest,
                languages=["chi_sim"], runner=FakeRunner(), which=all_tools,
            )
        self.assertEqual("OCR_SOURCE_POLICY_MISMATCH", caught.exception.code)

    def test_normalized_media_hash_drift_fails_closed(self):
        self.image.write_bytes(b"drift-after-normalization")
        with self.assertRaises(OCRRunError) as caught:
            run_docx_bundle(
                self.bundle, self.root / "normalized-drift",
                source_id=self.source_id, sources_manifest=self.manifest,
                languages=["eng"], runner=FakeRunner(), which=all_tools,
            )
        self.assertEqual("OCR_NORMALIZED_BUNDLE_HASH_MISMATCH", caught.exception.code)

    def test_unsupported_image_is_recorded_failed_not_skipped(self):
        unsupported = self.image.with_suffix(".emf")
        self.image.rename(unsupported)
        self.image = unsupported
        media_map = yaml.safe_load((self.bundle / "media-map.yml").read_text())
        media_map["assets"][0]["extracted_path"] = "media/image.emf"
        media_map["assets"][0]["sha256_extracted"] = hashlib.sha256(
            unsupported.read_bytes()
        ).hexdigest()
        (self.bundle / "media-map.yml").write_text(
            yaml.safe_dump(media_map, sort_keys=False), encoding="utf-8"
        )
        self.refresh_normalized_checksums()
        output = self.root / "unsupported"
        with self.assertRaises(OCRRunError) as caught:
            run_docx_bundle(
                self.bundle, output, source_id=self.source_id,
                sources_manifest=self.manifest, languages=["eng"],
                runner=FakeRunner(), which=all_tools,
            )
        self.assertEqual("OCR_COVERAGE_INCOMPLETE", caught.exception.code)
        record = json.loads((output / "ocr-results.jsonl").read_text().strip())
        self.assertEqual("failed", record["status"])
        self.assertEqual("OCR_IMAGE_FORMAT_UNSUPPORTED", record["error_code"])

    def test_unbound_docx_occurrence_blocks_complete_coverage(self):
        media_map = yaml.safe_load((self.bundle / "media-map.yml").read_text())
        media_map["occurrences"][1]["asset_id"] = "missing-asset"
        (self.bundle / "media-map.yml").write_text(
            yaml.safe_dump(media_map, sort_keys=False), encoding="utf-8"
        )
        self.refresh_normalized_checksums()
        output = self.root / "unbound-occurrence"
        with self.assertRaises(OCRRunError) as caught:
            run_docx_bundle(
                self.bundle, output, source_id=self.source_id,
                sources_manifest=self.manifest, languages=["eng"],
                runner=FakeRunner(), which=all_tools,
            )
        self.assertEqual("OCR_COVERAGE_INCOMPLETE", caught.exception.code)
        manifest = yaml.safe_load((output / "ocr-manifest.yml").read_text())
        self.assertEqual(["occ-002"], manifest["coverage"]["unbound_occurrence_ids"])

    def test_scanned_pdf_renders_every_page_at_fixed_300_dpi(self):
        pdf = self.root / "book.pdf"; pdf.write_bytes(b"synthetic-pdf")
        self.raw_source = pdf
        self.source_hash = "sha256:" + hashlib.sha256(pdf.read_bytes()).hexdigest()
        self.write_manifest("book-pdf-scan")
        self.set_manifest_languages(["chi_sim", "eng"])
        runner = FakeRunner(); output = self.root / "pdf-ocr"
        summary = run_scanned_pdf(
            pdf, output, source_id=self.source_id, sources_manifest=self.manifest,
            languages=["chi_sim", "eng"], runner=runner, which=all_tools,
        )
        self.assertEqual(2, summary["coverage"]["attempted_items"])
        render_calls = [call for call, _ in runner.calls if call[0] == "pdftoppm" and call[1] != "-v"]
        self.assertEqual(2, len(render_calls))
        self.assertTrue(all(call[call.index("-r") + 1] == "300" for call in render_calls))
        self.assertEqual(self.source_hash, "sha256:" + hashlib.sha256(pdf.read_bytes()).hexdigest())

    def test_pdf_render_failure_still_records_every_page(self):
        class FailFirstPage(FakeRunner):
            def __call__(self, args, **kwargs):
                if (
                    args[0] == "pdftoppm" and args[1] != "-v"
                    and args[args.index("-f") + 1] == "1"
                ):
                    self.calls.append((list(args), dict(kwargs)))
                    self.assert_safe(kwargs)
                    return subprocess.CompletedProcess(args, 1, "", "synthetic render failure")
                return super().__call__(args, **kwargs)

        pdf = self.root / "render-failure.pdf"; pdf.write_bytes(b"synthetic-pdf")
        self.raw_source = pdf
        self.source_hash = "sha256:" + hashlib.sha256(pdf.read_bytes()).hexdigest()
        self.write_manifest("book-pdf-scan")
        output = self.root / "render-failure"
        with self.assertRaises(OCRRunError) as caught:
            run_scanned_pdf(
                pdf, output, source_id=self.source_id,
                sources_manifest=self.manifest, languages=["eng"],
                runner=FailFirstPage(), which=all_tools,
            )
        self.assertEqual("OCR_COVERAGE_INCOMPLETE", caught.exception.code)
        records = [json.loads(line) for line in (output / "ocr-results.jsonl").read_text().splitlines()]
        self.assertEqual(2, len(records))
        self.assertEqual("PDF_RENDER_FAILED", records[0]["error_code"])
        self.assertEqual("completed", records[1]["status"])

    def test_pdf_verifier_recomputes_source_and_full_page_count(self):
        pdf = self.root / "verified.pdf"; pdf.write_bytes(b"synthetic-pdf")
        self.raw_source = pdf
        self.source_hash = "sha256:" + hashlib.sha256(pdf.read_bytes()).hexdigest()
        self.write_manifest("book-pdf-scan")
        output_root = self.root / "pdf-verified"
        run_scanned_pdf(
            pdf, output_root / self.source_id, source_id=self.source_id,
            sources_manifest=self.manifest, languages=["eng"],
            runner=FakeRunner(), which=all_tools,
        )
        distillation = self.root / "pdf-distillation"; distillation.mkdir()
        (distillation / "evidence-ledger.yml").write_text(
            yaml.safe_dump({"schema_version": 1, "evidence": [], "claims": []}),
            encoding="utf-8",
        )
        report = verify(
            distillation, output_root, self.manifest,
            runner=FakeRunner(), which=all_tools,
        )
        self.assertTrue(report.ok, [item.as_dict() for item in report.errors])
        pdf.write_bytes(b"source-drift")
        report = verify(
            distillation, output_root, self.manifest,
            runner=FakeRunner(), which=all_tools,
        )
        self.assertFalse(report.ok)
        self.assertIn("OCR_SOURCE_CHECKSUM_MISMATCH", {item.code for item in report.errors})

    def test_verifier_resolves_region_and_detects_hash_drift(self):
        output = self.root / "ocr-verified" / self.source_id
        run_docx_bundle(
            self.bundle, output, source_id=self.source_id,
            sources_manifest=self.manifest, languages=["eng"],
            runner=FakeRunner(), which=all_tools,
        )
        record = json.loads((output / "ocr-results.jsonl").read_text().strip())
        region = record["regions"][0]
        distillation = self.root / "distillation"; distillation.mkdir()
        (distillation / "evidence-ledger.yml").write_text(yaml.safe_dump({
            "schema_version": 1, "distillation_id": "ocr-test",
            "evidence": [{
                "evidence_id": "ev-ocr-001", "source_id": self.source_id,
                "normalized_text": region["text"],
                "locator": {
                    "source_id": self.source_id, "locator_type": "ocr-region",
                    "anchor": "figure-001#region-001", "carrier": "docx-image",
                    "image_sha256": record["image_sha256"],
                    "ocr_run_id": f"ocr-{self.source_id}-v1",
                    "ocr_record_id": record["ocr_record_id"],
                    "region_id": region["region_id"], "bbox_px": region["bbox_px"],
                    "figure_id": "figure-001", "media_occurrence_id": "occ-001",
                    "content_hash": hashlib.sha256(region["text"].encode()).hexdigest()[:12],
                },
            }], "claims": [],
        }, sort_keys=False), encoding="utf-8")
        report = verify(
            distillation,
            self.root / "ocr-verified",
            self.manifest,
            self.bundle,
        )
        self.assertTrue(report.ok, [item.as_dict() for item in report.errors])
        image_path = output / record["image_path"]
        image_path.write_bytes(b"drift")
        report = verify(
            distillation,
            self.root / "ocr-verified",
            self.manifest,
            self.bundle,
        )
        self.assertFalse(report.ok)
        self.assertIn("OCR_BUNDLE_HASH_MISMATCH", {item.code for item in report.errors})


if __name__ == "__main__":
    unittest.main()

from __future__ import annotations

import copy
import hashlib
import json
import os
import sys
import tempfile
import unittest
from pathlib import Path

import yaml

SCRIPT_DIR = Path(__file__).resolve().parent
sys.path.insert(0, str(SCRIPT_DIR))

from hash_candidate_tree import CandidateTreeError, candidate_tree_sha256, main as hash_main
from validate_distillation import (
    APPROVAL_SNAPSHOT_GOVERNANCE_FILES,
    DistillationInputError,
    GATE3_APPROVAL_SNAPSHOT_CONTRACT,
    validate_distillation,
)


FIXTURE_CANDIDATE_PATH = "candidates/review-book-task"
FIXTURE_SKILL = (
    "---\n"
    "name: review-book-task\n"
    "description: Test candidate.\n"
    "---\n\n"
    "# Review book task\n\n"
    "Apply the bounded, reviewed method and stop when source context is missing.\n"
)
FIXTURE_DIMENSIONS = [
    "source-layer-separation",
    "traceability",
    "scope-discipline",
    "stopping-behavior",
    "output-usability",
]
FIXTURE_FATAL_FAILURES = [
    "reads without Gate approval",
    "overwrites existing records",
]
FIXTURE_TRIGGER_DEFINITION = {
    "schema_version": 2,
    "skill_name": "review-book-task",
    "should_trigger": [
        {
            "case_id": f"trigger-{index}",
            "prompt": f"Trigger request {index}.",
            "expected_reason": "This request uses the reviewed bounded method.",
            "stable_task_ids": ["stable-task-001"],
            "input_type_ids": ["book-material"],
            "positive_example_ids": ["positive-example-001"],
            "negative_example_ids": [],
        }
        for index in range(1, 4)
    ],
    "should_not_trigger": [
        {
            "case_id": f"nontrigger-{index}",
            "prompt": f"Adjacent request {index}.",
            "expected_reason": "This request is outside the reviewed task boundary.",
            "stable_task_ids": ["stable-task-001"],
            "input_type_ids": ["book-material"],
            "positive_example_ids": [],
            "negative_example_ids": ["negative-example-001"],
        }
        for index in range(1, 4)
    ],
}
FIXTURE_TASK_DEFINITION = {
    "schema_version": 2,
    "skill_name": "review-book-task",
    "comparison_protocol": {
        "required": True,
        "baseline": "Run without loading the candidate.",
        "with_skill": "Run with only the candidate and allowed fixture.",
        "leakage_control": "Withhold expected answers and isolate contexts.",
        "human_review_dimensions": FIXTURE_DIMENSIONS,
        "rubric": {
            "rubric_id": "rubric-001",
            "score_min": 0,
            "score_max_per_dimension": 2,
            "pass_threshold": 8,
            "dimensions": [
                {"dimension_id": item, "description": item.replace("-", " ")}
                for item in FIXTURE_DIMENSIONS
            ],
            "fatal_failures": [
                {"failure_id": f"fatal-{index:03d}", "description": item}
                for index, item in enumerate(FIXTURE_FATAL_FAILURES, 1)
            ],
        },
    },
    "tasks": [
        {
            "case_id": f"task-{index}",
            "title": f"Representative task {index}",
            "holdout": index == 1,
            "rubric_id": "rubric-001",
            "input_profile": "A bounded local source fixture.",
            "request": f"Representative task {index}.",
            "expected_behaviors": ["Stay within the reviewed source scope."],
            "failure_signals": ["Invents unsupported source content."],
            "stable_task_ids": ["stable-task-001"],
            "input_type_ids": ["book-material"],
            "positive_example_ids": ["positive-example-001"],
            "negative_example_ids": ["negative-example-001"],
        }
        for index in range(1, 4)
    ],
}


def valid_documents():
    gate_rule_rationale = "Rule accepted for bounded evaluation."
    candidate_hash = f"sha256:{'0' * 64}"
    accepted_history = [
        {
            "from": None,
            "to": "candidate",
            "decided_by": "project-agent",
            "decided_at": "2026-08-04",
            "rationale": "Initial extraction for review.",
        },
        {
            "from": "candidate",
            "to": "accepted",
            "decided_by": "human-reviewer",
            "decided_at": "2026-08-04",
            "rationale": "Accepted after traceability review.",
        },
    ]
    evidence = {
        "evidence_id": "ev-001",
        "source_id": "book-001",
        "locator": {
            "source_id": "book-001",
            "locator_type": "ooxml-block",
            "heading_path": ["Chapter 1"],
            "ooxml_block_index": 12,
            "content_hash": "abc123def456",
        },
        "evidence_type": "text",
        "raw_text": "Source text.",
        "normalized_text": "Source text.",
        "capture_mode": "excerpt",
        "extraction_confidence": "high",
        "limitations": [],
        "quality_flags": [],
        "status": "accepted",
        "status_history": copy.deepcopy(accepted_history),
    }
    claim = {
        "claim_id": "cl-001",
        "statement": "The book states a bounded method.",
        "claim_type": "method",
        "source_position": "book-assertion",
        "evidence_ids": ["ev-001"],
        "transformation": "T1",
        "scope": ["example task"],
        "limitations": [],
        "importance": "important",
        "status": "accepted",
        "human_decision": None,
        "status_history": copy.deepcopy(accepted_history),
    }
    relation = {
        "relation_id": "rel-001",
        "subject": "method",
        "predicate": "requires",
        "object": "context",
        "qualifiers": ["within the stated scope"],
        "claim_ids": ["cl-001"],
        "evidence_ids": ["ev-001"],
        "relation_status": "explicit",
        "status": "accepted",
        "status_history": copy.deepcopy(accepted_history),
    }
    rule = {
        "rule_id": "rule-001",
        "stable_task_ids": ["stable-task-001"],
        "trigger": {"task": "apply the method", "signals": ["book-based request"]},
        "required_context": ["source material"],
        "checks": ["check scope"],
        "action": ["apply only the supported step"],
        "output": ["auditable result"],
        "stop_conditions": ["missing source context"],
        "claim_ids": ["cl-001"],
        "relation_ids": ["rel-001"],
        "transformation": "T4",
        "status": "accepted",
        "human_decision": {
            "decision": "accepted",
            "reviewer_type": "user",
            "reviewer": "project-user",
            "decided_at": "2026-08-04",
            "rationale": gate_rule_rationale,
            "gate_decision_id": "gate-decision-003",
        },
        "status_history": [
            {
                "from": None,
                "to": "candidate",
                "decided_by": "project-agent",
                "decided_at": "2026-08-04",
                "rationale": "Initial extraction for review.",
            },
            {
                "from": "candidate",
                "to": "accepted",
                "decided_by": "project-user",
                "decided_at": "2026-08-04",
                "rationale": gate_rule_rationale,
            },
        ],
        "semantic_support": {
            "checks": [{"item_index": 0, "claim_ids": ["cl-001"], "relation_ids": []}],
            "action": [{"item_index": 0, "claim_ids": ["cl-001"], "relation_ids": []}],
            "output": [{"item_index": 0, "claim_ids": ["cl-001"], "relation_ids": []}],
            "stop_conditions": [{
                "item_index": 0, "claim_ids": ["cl-001"], "relation_ids": []
            }],
        },
    }
    candidate = {
        "candidate_id": "candidate-001",
        "name": "review-book-task",
        "stable_task": "Apply the reviewed method to a bounded input.",
        "stable_task_ids": ["stable-task-001"],
        "trigger_case_ids": ["trigger-1", "trigger-2", "trigger-3"],
        "nontrigger_case_ids": ["nontrigger-1", "nontrigger-2", "nontrigger-3"],
        "task_case_ids": ["task-1", "task-2", "task-3"],
        "should_trigger": ["request one", "request two", "request three"],
        "should_not_trigger": ["adjacent one", "adjacent two", "adjacent three"],
        "inputs": ["source material"],
        "outputs": ["auditable result"],
        "rule_ids": ["rule-001"],
        "stop_conditions": ["source context missing"],
        "risks": ["overgeneralization"],
        "lifecycle": "review",
        "lifecycle_history": [
            {
                "from": None,
                "to": "draft",
                "decided_by": "project-agent",
                "decided_at": "2026-08-04",
                "rationale": "Candidate specification initialized.",
            },
            {
                "from": "draft",
                "to": "review",
                "decided_by": "project-agent",
                "decided_at": "2026-08-04",
                "rationale": "Candidate prepared for human review.",
            },
        ],
    }
    task_contract = {
        "schema_version": 1,
        "distillation_id": "validator-test-v1",
        "task_contract_id": "task-contract-001",
        "contract_version": 1,
        "status": "frozen",
        "product_goal": "Apply one reviewed source-contained method without task drift.",
        "audience": ["test reviewer"],
        "input_types": [{
            "input_type_id": "book-material",
            "description": "Registered source-book material.",
            "provenance_role": "method-source",
        }],
        "stable_tasks": [{
            "stable_task_id": "stable-task-001",
            "statement": "Apply the reviewed method to bounded book material.",
            "task_mode": "source-contained",
            "required_input_types": ["book-material"],
            "required_outputs": ["auditable-result"],
            "non_negotiable_constraints": ["Preserve traceability."],
            "positive_examples": [{
                "example_id": "positive-example-001",
                "input_type_id": "book-material",
                "statement": "Apply the method to this registered source.",
            }],
            "negative_examples": [{
                "example_id": "negative-example-001",
                "input_type_id": "book-material",
                "statement": "Produce an unsupported summary.",
            }],
            "acceptance_question_ids": ["acceptance-001"],
            "provenance_requirements": {
                "required_output_layers": ["method-source-evidence"],
                "target_source_role": None,
                "missing_target_evidence": "stop",
                "forbidden_transfers": [],
            },
            "status": "active",
        }],
        "exclusions": [{"exclusion_id": "exclusion-001", "statement": "Unreviewed deployment."}],
        "acceptance_questions": [{
            "acceptance_question_id": "acceptance-001",
            "question": "Is the result traceable?",
        }],
    }
    task_coverage = {
        "schema_version": 1,
        "distillation_id": "validator-test-v1",
        "status": "review-ready",
        "task_contract": {},
        "coverage": [{
            "stable_task_id": "stable-task-001",
            "coverage_status": "covered",
            "candidate_ids": ["candidate-001"],
            "capability_rule_ids": ["rule-001"],
            "trigger_case_ids": ["trigger-1"],
            "nontrigger_case_ids": ["nontrigger-1"],
            "task_eval_case_ids": ["task-1"],
            "holdout_case_ids": ["task-1"],
            "rubric_dimension_ids": ["traceability"],
            "gate1_decision_id": "gate-decision-001",
            "rationale": "Complete synthetic coverage.",
        }],
    }
    return {
        "task-contract.yml": task_contract,
        "task-coverage.yml": task_coverage,
        "evidence-ledger.yml": {
            "schema_version": 1, "distillation_id": "validator-test-v1",
            "evidence": [evidence], "claims": [claim]
        },
        "concept-map.yml": {
            "schema_version": 1, "distillation_id": "validator-test-v1",
            "relations": [relation]
        },
        "capability-rules.yml": {
            "schema_version": 1,
            "distillation_id": "validator-test-v1",
            "capability_rules": [rule],
            "skill_candidates": [candidate],
        },
        "gate-decisions.yml": {
            "schema_version": 1,
            "distillation_id": "validator-test-v1",
            "gate_decisions": [
                {
                    "decision_id": "gate-decision-001",
                    "sequence": 1,
                    "supersedes": None,
                    "is_current": True,
                    "gate": "gate-1",
                    "candidate_id": None,
                    "decision": "approved",
                    "scope": ["validator-test-v1"],
                    "reviewer_type": "user",
                    "reviewer": "project-user",
                    "decided_at": "2026-08-04",
                    "rationale": "Requirements frozen for the test fixture.",
                    "conditions": [],
                    "eval_run_ids": [],
                    "rule_decisions": [],
                    "stable_task_decisions": [{
                        "stable_task_id": "stable-task-001",
                        "decision": "active",
                        "rationale": "Task approved as active.",
                    }],
                    "task_contract_snapshot": {},
                },
                {
                    "decision_id": "gate-decision-002",
                    "sequence": 2,
                    "supersedes": None,
                    "is_current": True,
                    "gate": "gate-2",
                    "candidate_id": None,
                    "decision": "approved",
                    "scope": ["book-001"],
                    "reviewer_type": "user",
                    "reviewer": "project-user",
                    "decided_at": "2026-08-04",
                    "rationale": "Source structure approved for bounded extraction.",
                    "conditions": [],
                    "eval_run_ids": [],
                    "rule_decisions": [],
                },
                {
                    "decision_id": "gate-decision-003",
                    "sequence": 3,
                    "supersedes": None,
                    "is_current": True,
                    "gate": "gate-3",
                    "candidate_id": "candidate-001",
                    "decision": "approved-for-eval",
                    "scope": ["candidate-001"],
                    "reviewer_type": "user",
                    "reviewer": "project-user",
                    "decided_at": "2026-08-04",
                    "rationale": "Approved for bounded evaluation.",
                    "conditions": [],
                    "eval_run_ids": [],
                    "rule_decisions": [{
                        "rule_id": "rule-001",
                        "decision": "accepted",
                        "rationale": gate_rule_rationale,
                    }],
                    "approval_snapshot": {
                        "contract": GATE3_APPROVAL_SNAPSHOT_CONTRACT,
                        "candidate_path": FIXTURE_CANDIDATE_PATH,
                        "candidate_hash": candidate_hash,
                        "governance_hashes": {
                            filename: candidate_hash
                            for filename in APPROVAL_SNAPSHOT_GOVERNANCE_FILES
                        },
                    },
                },
            ],
            "materializations": [{
                "materialization_id": "materialization-001",
                "candidate_id": "candidate-001",
                "gate3_decision_id": "gate-decision-003",
                "status": "completed",
                "candidate_path": FIXTURE_CANDIDATE_PATH,
                "candidate_hash": candidate_hash,
                "materialized_at": "2026-08-04",
                "rule_ids": ["rule-001"],
                "quick_validation": {
                    "status": "pass",
                    "validator": "quick_validate.py",
                    "validated_at": "2026-08-04",
                    "candidate_hash": candidate_hash,
                },
            }],
        },
        "eval-runs.yml": {
            "schema_version": 1,
            "distillation_id": "validator-test-v1",
            "eval_runs": [],
        },
    }


def valid_sources_manifest(source_role="primary-book"):
    return {
        "schema_version": 2,
        "sources": [{
            "id": "book-001",
            "source_role": source_role,
            "provenance_role": "method-source",
            "privacy": "private",
            "allow_public_quotes": False,
        }],
    }


def sync_task_contract_snapshot(documents, root):
    contract = documents["task-contract.yml"]
    contract_path = root / "task-contract.yml"
    contract_path.write_text(
        yaml.safe_dump(contract, allow_unicode=True, sort_keys=False),
        encoding="utf-8",
    )
    contract_hash = f"sha256:{hashlib.sha256(contract_path.read_bytes()).hexdigest()}"
    coverage = documents["task-coverage.yml"]
    coverage["task_contract"] = {
        "path": "task-contract.yml",
        "sha256": contract_hash,
        "task_contract_id": contract["task_contract_id"],
        "contract_version": contract["contract_version"],
    }
    (root / "task-coverage.yml").write_text(
        yaml.safe_dump(coverage, allow_unicode=True, sort_keys=False),
        encoding="utf-8",
    )
    gate1 = next(
        item for item in documents["gate-decisions.yml"]["gate_decisions"]
        if item.get("gate") == "gate-1" and item.get("is_current") is True
    )
    gate1["task_contract_snapshot"] = {
        "contract": "gate1-task-contract-snapshot:v1",
        "task_contract_path": "task-contract.yml",
        "task_contract_hash": contract_hash,
        "task_contract_id": contract["task_contract_id"],
        "contract_version": contract["contract_version"],
        "active_stable_task_ids": [
            item["stable_task_id"] for item in contract["stable_tasks"]
            if item["status"] == "active"
        ],
    }


def sync_current_gate3_approval_snapshot(
    documents,
    root,
    candidate_path=FIXTURE_CANDIDATE_PATH,
):
    """Create a valid synthetic user-approval snapshot from files on disk."""
    candidate_hash = candidate_tree_sha256(root, candidate_path)
    governance_hashes = {
        filename: f"sha256:{hashlib.sha256((root / filename).read_bytes()).hexdigest()}"
        for filename in APPROVAL_SNAPSHOT_GOVERNANCE_FILES
    }
    contract = documents["task-contract.yml"]
    gate1 = next(
        item for item in documents["gate-decisions.yml"]["gate_decisions"]
        if item.get("gate") == "gate-1" and item.get("is_current") is True
    )
    contract_hash = f"sha256:{hashlib.sha256((root / 'task-contract.yml').read_bytes()).hexdigest()}"
    coverage_hash = f"sha256:{hashlib.sha256((root / 'task-coverage.yml').read_bytes()).hexdigest()}"
    for decision in documents["gate-decisions.yml"]["gate_decisions"]:
        if (
            decision.get("gate") == "gate-3"
            and decision.get("is_current") is True
            and decision.get("decision") == "approved-for-eval"
        ):
            decision["approval_snapshot"] = {
                "contract": GATE3_APPROVAL_SNAPSHOT_CONTRACT,
                "candidate_path": candidate_path,
                "candidate_hash": candidate_hash,
                "governance_hashes": governance_hashes,
                "current_gate1_decision_id": gate1["decision_id"],
                "task_contract": {
                    "path": "task-contract.yml",
                    "sha256": contract_hash,
                    "task_contract_id": contract["task_contract_id"],
                    "contract_version": contract["contract_version"],
                    "active_stable_task_ids": gate1["task_contract_snapshot"]["active_stable_task_ids"],
                },
                "task_coverage": {
                    "path": "task-coverage.yml",
                    "sha256": coverage_hash,
                },
                "candidate_stable_task_ids": documents["capability-rules.yml"]["skill_candidates"][0]["stable_task_ids"],
            }


def set_rule_pending(documents):
    rule = documents["capability-rules.yml"]["capability_rules"][0]
    rule["status"] = "candidate"
    rule.pop("status_history", None)
    rule["human_decision"] = {
        "decision": "pending",
        "reviewer_type": None,
        "reviewer": None,
        "decided_at": None,
        "rationale": "",
        "gate_decision_id": None,
    }
    gate3 = next(
        item for item in documents["gate-decisions.yml"]["gate_decisions"]
        if item["gate"] == "gate-3"
    )
    gate3.update({
        "decision": "pending",
        "reviewer_type": None,
        "reviewer": None,
        "decided_at": None,
        "rationale": "Awaiting explicit rule decisions.",
        "rule_decisions": [],
    })
    documents["gate-decisions.yml"]["materializations"] = []


def reject_rule_at_gate3(documents):
    rule = documents["capability-rules.yml"]["capability_rules"][0]
    rationale = "Rule rejected by the Gate 3 user review."
    rule["status"] = "rejected"
    rule["human_decision"] = {
        "decision": "rejected",
        "reviewer_type": "user",
        "reviewer": "project-user",
        "decided_at": "2026-08-04",
        "rationale": rationale,
        "gate_decision_id": "gate-decision-003",
    }
    rule["status_history"] = [
        {
            "from": None, "to": "candidate", "decided_by": "project-agent",
            "decided_at": "2026-08-04", "rationale": "Initial extraction for review.",
        },
        {
            "from": "candidate", "to": "rejected", "decided_by": "project-user",
            "decided_at": "2026-08-04", "rationale": rationale,
        },
    ]
    gate3 = documents["gate-decisions.yml"]["gate_decisions"][2]
    gate3["rule_decisions"] = [{
        "rule_id": "rule-001", "decision": "rejected", "rationale": rationale,
    }]


def accept_candidate(documents, root):
    candidate = documents["capability-rules.yml"]["skill_candidates"][0]
    materialization_hash = documents["gate-decisions.yml"]["materializations"][0][
        "candidate_hash"
    ]
    candidate["lifecycle"] = "accepted"
    candidate["lifecycle_history"].append({
        "from": "review",
        "to": "accepted",
        "decided_by": "human-reviewer",
        "decided_at": "2026-08-04",
        "rationale": "Accepted after the completed Gate 4 evaluation.",
    })
    eval_runs = []
    for run_number, (case_type, case_number) in enumerate(
        (
            ("trigger", 1), ("trigger", 2), ("trigger", 3),
            ("nontrigger", 1), ("nontrigger", 2), ("nontrigger", 3),
            ("task", 1), ("task", 2), ("task", 3),
        ),
        start=1,
    ):
        token = f"{case_type}-{case_number}"
        eval_run_id = f"eval-run-{run_number:03d}"
        if case_type == "trigger":
            case_record = FIXTURE_TRIGGER_DEFINITION["should_trigger"][case_number - 1]
        elif case_type == "nontrigger":
            case_record = FIXTURE_TRIGGER_DEFINITION["should_not_trigger"][case_number - 1]
        else:
            case_record = FIXTURE_TASK_DEFINITION["tasks"][case_number - 1]
        case_hash = "sha256:" + hashlib.sha256(json.dumps(
            case_record,
            ensure_ascii=False,
            sort_keys=True,
            separators=(",", ":"),
            allow_nan=False,
        ).encode("utf-8")).hexdigest()
        request = case_record.get("request", case_record.get("prompt"))
        holdout = bool(case_record.get("holdout", False))
        fixture_path = f"fixtures/{token}.json"
        baseline_path = f"outputs/{token}-baseline.json"
        with_skill_path = f"outputs/{token}-with-skill.json"
        leakage_controls = {
            "expected_answer_withheld": True,
            "context_isolated": True,
            "context_differences": ["candidate loaded only in with_skill"],
            "exceptions": [],
        }
        fixture = {
            "schema_version": 1,
            "fixture_id": f"fixture-{run_number:03d}",
            "case_type": case_type,
            "case_id": token,
            "case_definition_hash": case_hash,
            "request": request,
            "source_ids": ["book-001"],
            "holdout": holdout,
            "input_payload": {"source_excerpt": f"bounded input for {token}"},
            "leakage_controls": {
                "expected_answer_withheld": True,
                "context_isolated": True,
                "exceptions": [],
            },
        }
        baseline_output = {
            "schema_version": 1,
            "eval_run_id": eval_run_id,
            "case_id": token,
            "condition": "baseline",
            "response": f"baseline response for {token}",
        }
        with_skill_output = {
            "schema_version": 1,
            "eval_run_id": eval_run_id,
            "case_id": token,
            "condition": "with_skill",
            "response": f"with-Skill response for {token}",
        }
        artifact_payloads = {
            fixture_path: (json.dumps(fixture, sort_keys=True) + "\n").encode(),
            baseline_path: (json.dumps(baseline_output, sort_keys=True) + "\n").encode(),
            with_skill_path: (json.dumps(with_skill_output, sort_keys=True) + "\n").encode(),
        }
        for relative_path, payload in artifact_payloads.items():
            artifact_path = root / relative_path
            artifact_path.parent.mkdir(parents=True, exist_ok=True)
            artifact_path.write_bytes(payload)
        eval_runs.append({
            "eval_run_id": eval_run_id,
            "candidate_id": "candidate-001",
            "materialization_id": "materialization-001",
            "case_type": case_type,
            "case_id": token,
            "case_definition_hash": case_hash,
            "status": "completed",
            "outcome": "pass",
            "fixture_id": f"fixture-{run_number:03d}",
            "fixture_path": fixture_path,
            "fixture_hash": hashlib.sha256(artifact_payloads[fixture_path]).hexdigest(),
            "source_ids": ["book-001"],
            "holdout": holdout,
            "rule_ids": ["rule-001"],
            "candidate_hash": materialization_hash,
            "execution_environment": {"model": "test-model", "tools": []},
            "baseline_output_path": baseline_path,
            "baseline_output_hash": hashlib.sha256(
                artifact_payloads[baseline_path]
            ).hexdigest(),
            "with_skill_output_path": with_skill_path,
            "with_skill_output_hash": hashlib.sha256(
                artifact_payloads[with_skill_path]
            ).hexdigest(),
            "rubric_id": "rubric-001",
            "dimension_scores": {
                FIXTURE_DIMENSIONS[0]: 2,
                FIXTURE_DIMENSIONS[1]: 2,
                FIXTURE_DIMENSIONS[2]: 2,
                FIXTURE_DIMENSIONS[3]: 2,
                FIXTURE_DIMENSIONS[4]: 1,
            },
            "fatal_failures_observed": [],
            "leakage_controls": leakage_controls,
            "score": 9,
            "max_score": 10,
            "pass_threshold": 8,
            "reviewer_type": "human-delegate",
            "reviewer": "evaluation-reviewer",
            "completed_at": "2026-08-04",
            "limitations": [],
        })
    documents["eval-runs.yml"]["eval_runs"] = eval_runs
    documents["gate-decisions.yml"]["gate_decisions"].append(
        {
            "decision_id": "gate-decision-004",
            "sequence": 4,
            "supersedes": None,
            "is_current": True,
            "gate": "gate-4",
            "candidate_id": "candidate-001",
            "decision": "accepted",
            "scope": ["candidate-001"],
            "reviewer_type": "user",
            "reviewer": "project-user",
            "decided_at": "2026-08-04",
            "rationale": "Accepted after reviewing the completed passing run.",
            "conditions": [],
            "eval_run_ids": [item["eval_run_id"] for item in eval_runs],
            "rule_decisions": [],
        },
    )


def add_valid_correction(documents):
    evidence = documents["evidence-ledger.yml"]["evidence"][0]
    claim = documents["evidence-ledger.yml"]["claims"][0]
    claim["correction_ids"] = ["correction-001"]
    documents["correction-overlay.yml"] = {
        "schema_version": 1,
        "distillation_id": "validator-test-v1",
        "overlay_id": "correction-overlay-001",
        "source_id": "book-001",
        "policy": {
            "source_remains_read_only": True,
            "normalized_text_must_remain_semantically_unchanged": True,
        },
        "corrections": [{
            "correction_id": "correction-001",
            "evidence_id": "ev-001",
            "locator": copy.deepcopy(evidence["locator"]),
            "issue_type": "ocr-risk",
            "raw_value": "Source text.",
            "proposed_value": "Corrected source text.",
            "basis": "Human comparison with a reliable local carrier.",
            "status": "accepted",
            "human_decision": {
                "decision": "revised",
                "reviewer_type": "human-delegate",
                "reviewer": "human-reviewer",
                "decided_at": "2026-08-04",
                "rationale": "Verified against the reliable carrier.",
            },
            "applies_to_claim_ids": ["cl-001"],
            "resolved_quality_flags": ["ocr-risk"],
            "resulting_value": "Corrected source text.",
        }],
    }


class DistillationValidatorTests(unittest.TestCase):
    def setUp(self):
        self.tempdir = tempfile.TemporaryDirectory()
        self.root = Path(self.tempdir.name)
        self.write_candidate_contract()
        self.documents = self.fresh_documents()
        self.write_documents()

    def write_candidate_contract(self):
        candidate_dir = self.root / FIXTURE_CANDIDATE_PATH
        candidate_dir.mkdir(parents=True, exist_ok=True)
        (candidate_dir / "SKILL.md").write_text(FIXTURE_SKILL, encoding="utf-8")
        eval_dir = candidate_dir / "evals"
        eval_dir.mkdir(exist_ok=True)
        (eval_dir / "trigger-cases.json").write_text(
            json.dumps(FIXTURE_TRIGGER_DEFINITION, ensure_ascii=False, indent=2) + "\n",
            encoding="utf-8",
        )
        (eval_dir / "task-cases.json").write_text(
            json.dumps(FIXTURE_TASK_DEFINITION, ensure_ascii=False, indent=2) + "\n",
            encoding="utf-8",
        )

    def tearDown(self):
        self.tempdir.cleanup()

    def fresh_documents(self):
        documents = valid_documents()
        materialization = documents["gate-decisions.yml"]["materializations"][0]
        candidate_hash = candidate_tree_sha256(self.root, FIXTURE_CANDIDATE_PATH)
        materialization["candidate_hash"] = candidate_hash
        materialization["quick_validation"]["candidate_hash"] = candidate_hash
        return documents

    def sync_candidate_hash(self):
        materialization = self.documents["gate-decisions.yml"]["materializations"][0]
        candidate_hash = candidate_tree_sha256(self.root, FIXTURE_CANDIDATE_PATH)
        materialization["candidate_hash"] = candidate_hash
        materialization["quick_validation"]["candidate_hash"] = candidate_hash

    def write_documents(self, *, sync_approval_snapshot=True):
        # The approval snapshot hashes these exact serialized bytes. Write the
        # three frozen governance documents first, then synthesize the Gate 3
        # decision, then write the remaining records.
        sync_task_contract_snapshot(self.documents, self.root)
        for name in APPROVAL_SNAPSHOT_GOVERNANCE_FILES:
            document = self.documents[name]
            (self.root / name).write_text(
                yaml.safe_dump(document, allow_unicode=True, sort_keys=False),
                encoding="utf-8",
            )
        if sync_approval_snapshot:
            sync_current_gate3_approval_snapshot(self.documents, self.root)
        for name, document in self.documents.items():
            if name in APPROVAL_SNAPSHOT_GOVERNANCE_FILES or name in {
                "task-contract.yml", "task-coverage.yml"
            }:
                continue
            (self.root / name).write_text(
                yaml.safe_dump(document, allow_unicode=True, sort_keys=False),
                encoding="utf-8",
            )

    def write_manifest(self, source_role="primary-book"):
        manifest = self.root / "sources.yml"
        manifest.write_text(
            yaml.safe_dump(
                valid_sources_manifest(source_role), allow_unicode=True, sort_keys=False
            ),
            encoding="utf-8",
        )
        return manifest

    def configure_markdown_policy(
        self,
        *,
        repository_path="docs/policy.md",
        local_path=None,
        related_local_paths=None,
        source_text="# Policy\n\n## Review Gate（Gate 3）\n\nStable policy text.\n",
        raw_text="Stable policy text.",
        normalized_text=None,
        anchor=None,
        write_source=True,
    ):
        normalized_text = raw_text if normalized_text is None else normalized_text
        local_path = repository_path if local_path is None else local_path
        anchor = anchor or f"{repository_path}:5#review-gate"
        evidence = self.documents["evidence-ledger.yml"]["evidence"][0]
        evidence["locator"] = {
            "source_id": "book-001",
            "locator_type": "markdown-section",
            "anchor": anchor,
            "content_hash": hashlib.sha256(
                normalized_text.encode("utf-8")
            ).hexdigest()[:12],
        }
        evidence["raw_text"] = raw_text
        evidence["normalized_text"] = normalized_text
        if write_source:
            source_path = self.root / repository_path
            source_path.parent.mkdir(parents=True, exist_ok=True)
            source_path.write_text(source_text, encoding="utf-8")

        manifest_data = valid_sources_manifest()
        source = manifest_data["sources"][0]
        source["local_path"] = local_path
        source["related_local_paths"] = list(related_local_paths or [])
        manifest = self.root / "manifests" / "sources.yml"
        manifest.parent.mkdir(parents=True, exist_ok=True)
        manifest.write_text(
            yaml.safe_dump(manifest_data, allow_unicode=True, sort_keys=False),
            encoding="utf-8",
        )
        self.write_documents()
        return manifest

    def error_codes(self):
        return {issue.code for issue in validate_distillation(self.root).errors}

    def configure_method_transfer(self, *, include_candidate_provenance=True,
                                  external_holdout=True,
                                  cover_required_input=True):
        contract = self.documents["task-contract.yml"]
        contract["input_types"].append({
            "input_type_id": "external-target-material",
            "description": "Independent unfamiliar target material.",
            "provenance_role": "target-material",
        })
        task = contract["stable_tasks"][0]
        task.update({
            "task_mode": "method-transfer",
            "required_input_types": ["external-target-material"],
            "positive_examples": [{
                "example_id": "positive-example-001",
                "input_type_id": "external-target-material",
                "statement": "Apply the method to unfamiliar target material.",
            }],
            "negative_examples": [{
                "example_id": "negative-example-001",
                "input_type_id": "external-target-material",
                "statement": "Treat a method-source case as a target fact.",
            }],
            "provenance_requirements": {
                "required_output_layers": [
                    "method-source-evidence", "target-material-evidence",
                    "analogy-hypothesis",
                ],
                "target_source_role": "target-material",
                "missing_target_evidence": "stop",
                "forbidden_transfers": ["method-source-fact-as-target-fact"],
            },
        })
        candidate = self.documents["capability-rules.yml"]["skill_candidates"][0]
        if include_candidate_provenance:
            candidate["provenance_contract"] = {
                "output_layers": [
                    "method-source-evidence", "target-material-evidence",
                    "analogy-hypothesis",
                ],
                "missing_target_evidence": "stop",
            }
        trigger_path = self.root / FIXTURE_CANDIDATE_PATH / "evals" / "trigger-cases.json"
        trigger_definition = json.loads(trigger_path.read_text(encoding="utf-8"))
        for case in (
            trigger_definition["should_trigger"]
            + trigger_definition["should_not_trigger"]
        ):
            case["input_type_ids"] = ["external-target-material"]
        trigger_path.write_text(
            json.dumps(trigger_definition, ensure_ascii=False, indent=2) + "\n",
            encoding="utf-8",
        )
        task_path = self.root / FIXTURE_CANDIDATE_PATH / "evals" / "task-cases.json"
        definition = json.loads(task_path.read_text(encoding="utf-8"))
        rubric = definition["comparison_protocol"]["rubric"]
        rubric["dimensions"].extend([
            {"dimension_id": "provenance-layer-separation", "description": "Separate provenance layers."},
            {"dimension_id": "anti-forced-analogy", "description": "Avoid forced analogy."},
        ])
        rubric["fatal_failures"].append({
            "failure_id": "method-source-fact-as-target-fact",
            "description": "Method-source fact is presented as target fact.",
        })
        rubric["pass_threshold"] = 10
        for case in definition["tasks"]:
            case["input_type_ids"] = ["external-target-material"]
        if not cover_required_input:
            definition["tasks"][0]["input_type_ids"] = ["book-material"]
        if external_holdout:
            definition["tasks"][0]["holdout_contract"] = {
                "target_source_ids": ["target-001"],
                "target_source_hashes": {"target-001": f"sha256:{'1' * 64}"},
                "used_for_rule_extraction": False,
                "unfamiliarity_dimensions": ["domain"],
                "isolation": "Independent synthetic target fixture.",
            }
        task_path.write_text(
            json.dumps(definition, ensure_ascii=False, indent=2) + "\n",
            encoding="utf-8",
        )
        coverage = self.documents["task-coverage.yml"]["coverage"][0]
        coverage["rubric_dimension_ids"] = [
            "traceability", "provenance-layer-separation", "anti-forced-analogy"
        ]
        self.sync_candidate_hash()
        self.write_documents()
        manifest = valid_sources_manifest()
        manifest["sources"][0]["provenance_role"] = "method-source"
        manifest["sources"].append({
            "id": "target-001", "source_role": "eval-target",
            "provenance_role": "target-material", "privacy": "private",
            "allow_public_quotes": False, "checksum": f"sha256:{'1' * 64}",
        })
        path = self.root / "method-sources.yml"
        path.write_text(yaml.safe_dump(manifest, sort_keys=False), encoding="utf-8")
        return path

    def test_valid_complete_fixture_passes(self):
        report = validate_distillation(self.root)
        self.assertTrue(report.ok, [item.as_dict() for item in report.errors])
        self.assertEqual(1.0, report.metrics["accepted_rule_traceability"])
        self.assertTrue(report.metrics["accepted_rule_traceability_applicable"])
        self.assertEqual(1.0, report.metrics["important_claim_locator_coverage"])
        self.assertFalse(report.as_dict()["truth_assessed"])
        self.assertFalse(report.as_dict()["behavior_effectiveness_assessed"])

    def test_task_contract_hash_drift_fails(self):
        with (self.root / "task-contract.yml").open("a", encoding="utf-8") as handle:
            handle.write("# drift\n")
        codes = self.error_codes()
        self.assertIn("TASK_CONTRACT_SNAPSHOT_MISMATCH", codes)
        self.assertIn("GATE3_TASK_CONTRACT_MISMATCH", codes)

    def test_task_contract_version_must_match_immutable_filename(self):
        self.documents["task-contract.yml"]["contract_version"] = 2
        self.write_documents()
        self.assertIn("TASK_CONTRACT_INVALID", self.error_codes())

    def test_execution_capability_rejects_unknown_input_type(self):
        contract = self.documents["task-contract.yml"]
        contract["execution_capability"] = {
            "input_handling": [{
                "input_type_id": "not-declared",
                "carrier": "pdf-text",
                "modality_strategy": "unimodal-text",
                "degradation_rule": "Fall back to text and state the limitation.",
            }],
        }
        self.write_documents()
        self.assertIn("TASK_CONTRACT_INVALID", self.error_codes())

    def test_execution_capability_rejects_bad_modality(self):
        contract = self.documents["task-contract.yml"]
        contract["execution_capability"] = {
            "input_handling": [{
                "input_type_id": "book-material",
                "carrier": "pdf-text",
                "modality_strategy": "vision-only",
                "degradation_rule": "Fall back to text and state the limitation.",
            }],
        }
        self.write_documents()
        self.assertIn("TASK_CONTRACT_INVALID", self.error_codes())

    def test_execution_capability_valid_when_complete(self):
        contract = self.documents["task-contract.yml"]
        contract["execution_capability"] = {
            "input_handling": [{
                "input_type_id": "book-material",
                "carrier": "pdf-text",
                "modality_strategy": "unimodal-text",
                "degradation_rule": "Fall back to text and state the limitation.",
            }],
            "notes": "Text-only analysis of registered source material.",
        }
        self.write_documents()
        report = validate_distillation(self.root)
        self.assertTrue(report.ok, [item.as_dict() for item in report.errors])

    def test_gate1_duplicate_stable_task_decision_fails(self):
        gate1 = self.documents["gate-decisions.yml"]["gate_decisions"][0]
        gate1["stable_task_decisions"].append(
            copy.deepcopy(gate1["stable_task_decisions"][0])
        )
        self.write_documents()
        self.assertIn("TASK_CONTRACT_SNAPSHOT_MISMATCH", self.error_codes())

    def test_eval_v2_task_requires_positive_and_negative_example_ids(self):
        path = self.root / FIXTURE_CANDIDATE_PATH / "evals" / "task-cases.json"
        definition = json.loads(path.read_text(encoding="utf-8"))
        definition["tasks"][0].pop("negative_example_ids")
        path.write_text(json.dumps(definition, indent=2) + "\n", encoding="utf-8")
        self.sync_candidate_hash()
        self.write_documents()
        codes = self.error_codes()
        self.assertTrue(
            {"MATERIALIZATION_EVAL_DEFINITION_INVALID", "TASK_COVERAGE_INVALID"}
            & codes
        )

    def test_candidate_must_register_complete_eval_case_ids(self):
        candidate = self.documents["capability-rules.yml"]["skill_candidates"][0]
        candidate["trigger_case_ids"] = candidate["trigger_case_ids"][:-1]
        self.write_documents()
        self.assertIn("CANDIDATE_TASK_MISMATCH", self.error_codes())

    def test_active_task_requires_complete_coverage(self):
        self.documents["task-coverage.yml"]["coverage"][0]["task_eval_case_ids"] = []
        self.write_documents()
        self.assertIn("STABLE_TASK_UNCOVERED", self.error_codes())

    def test_candidate_unknown_or_nonactive_task_fails(self):
        self.documents["capability-rules.yml"]["skill_candidates"][0]["stable_task_ids"] = ["unknown-task"]
        self.write_documents()
        self.assertIn("STABLE_TASK_UNKNOWN", self.error_codes())

    def test_gate3_v2_requires_contract_and_coverage_bindings(self):
        gate3 = self.documents["gate-decisions.yml"]["gate_decisions"][2]
        gate3["approval_snapshot"].pop("task_coverage", None)
        self.write_documents(sync_approval_snapshot=False)
        self.assertIn("GATE3_TASK_CONTRACT_MISMATCH", self.error_codes())

    def test_current_gate3_v1_is_legacy_review_required(self):
        gate3 = self.documents["gate-decisions.yml"]["gate_decisions"][2]
        gate3["approval_snapshot"] = {
            key: value for key, value in gate3["approval_snapshot"].items()
            if key in {"contract", "candidate_path", "candidate_hash", "governance_hashes"}
        }
        gate3["approval_snapshot"]["contract"] = "gate3-approval-snapshot:v1"
        self.write_documents(sync_approval_snapshot=False)
        self.assertIn("LEGACY_TASK_CONTRACT_REVIEW_REQUIRED", self.error_codes())

    def test_method_transfer_requires_candidate_provenance_contract(self):
        manifest = self.configure_method_transfer(include_candidate_provenance=False)
        codes = {item.code for item in validate_distillation(self.root, manifest).errors}
        self.assertIn("METHOD_TRANSFER_PROVENANCE_REQUIRED", codes)

    def test_method_transfer_requires_external_target_holdout(self):
        manifest = self.configure_method_transfer(external_holdout=False)
        codes = {item.code for item in validate_distillation(self.root, manifest).errors}
        self.assertIn("METHOD_TRANSFER_EXTERNAL_HOLDOUT_REQUIRED", codes)

    def test_method_transfer_holdout_must_cover_required_input_type(self):
        manifest = self.configure_method_transfer(cover_required_input=False)
        codes = {item.code for item in validate_distillation(self.root, manifest).errors}
        self.assertIn("METHOD_TRANSFER_EXTERNAL_HOLDOUT_REQUIRED", codes)

    def test_method_transfer_holdout_requires_isolation_record(self):
        manifest = self.configure_method_transfer()
        task_path = self.root / FIXTURE_CANDIDATE_PATH / "evals" / "task-cases.json"
        definition = json.loads(task_path.read_text(encoding="utf-8"))
        definition["tasks"][0]["holdout_contract"].pop("isolation")
        task_path.write_text(
            json.dumps(definition, ensure_ascii=False, indent=2) + "\n",
            encoding="utf-8",
        )
        self.sync_candidate_hash()
        self.write_documents()
        codes = {item.code for item in validate_distillation(self.root, manifest).errors}
        self.assertIn("METHOD_TRANSFER_EXTERNAL_HOLDOUT_REQUIRED", codes)

    def test_method_example_positive_case_cannot_be_nontrigger(self):
        trigger_path = self.root / FIXTURE_CANDIDATE_PATH / "evals" / "trigger-cases.json"
        definition = json.loads(trigger_path.read_text(encoding="utf-8"))
        definition["should_not_trigger"][0]["negative_example_ids"] = ["positive-example-001"]
        trigger_path.write_text(json.dumps(definition, indent=2) + "\n", encoding="utf-8")
        self.sync_candidate_hash()
        self.write_documents()
        self.assertIn("CANDIDATE_TASK_MISMATCH", self.error_codes())

    def test_accepted_ocr_evidence_requires_region_locator_and_human_review(self):
        evidence = self.documents["evidence-ledger.yml"]["evidence"][0]
        evidence["capture_mode"] = "ocr"
        evidence["ocr_review"] = {
            "decision": "accepted", "reviewer_type": "human-delegate",
            "reviewer": "ocr-reviewer", "decided_at": "2026-08-07",
            "rationale": "Compared with the source image.",
        }
        evidence["locator"] = {
            "source_id": "book-001", "locator_type": "ocr-region",
            "anchor": "figure-001#region-001", "content_hash": "abc123def456",
            "carrier": "docx-image", "image_sha256": f"sha256:{'2' * 64}",
            "ocr_run_id": "ocr-run-001", "ocr_record_id": "ocr-record-001",
            "region_id": "ocr-region-001", "bbox_px": [1, 2, 30, 10],
            "figure_id": "figure-001", "media_occurrence_id": "occurrence-001",
        }
        self.write_documents()
        report = validate_distillation(self.root)
        self.assertTrue(report.ok, [item.as_dict() for item in report.errors])
        evidence.pop("ocr_review")
        self.write_documents()
        self.assertIn("OCR_EVIDENCE_REVIEW_REQUIRED", self.error_codes())

    def test_validation_is_read_only(self):
        def snapshot():
            result = {}
            for path in sorted(self.root.rglob("*")):
                relative = path.relative_to(self.root).as_posix()
                if path.is_symlink():
                    result[relative] = ("symlink", os.readlink(path))
                elif path.is_dir():
                    result[relative] = ("directory", None)
                else:
                    result[relative] = (
                        "file", hashlib.sha256(path.read_bytes()).hexdigest()
                    )
            return result

        before = snapshot()
        validate_distillation(self.root)
        after = snapshot()
        self.assertEqual(before, after)

    def test_rejects_duplicate_ids_across_record_types(self):
        self.documents["concept-map.yml"]["relations"][0]["relation_id"] = "cl-001"
        self.write_documents()
        self.assertIn("DUPLICATE_ID", self.error_codes())

    def test_rejects_missing_foreign_key(self):
        self.documents["evidence-ledger.yml"]["claims"][0]["evidence_ids"] = ["ev-missing"]
        self.write_documents()
        self.assertIn("MISSING_FOREIGN_KEY", self.error_codes())

    def test_rejects_invalid_status(self):
        self.documents["evidence-ledger.yml"]["evidence"][0]["status"] = "verified"
        self.write_documents()
        self.assertIn("INVALID_STATUS", self.error_codes())

    def test_rejects_mismatched_distillation_id(self):
        self.documents["concept-map.yml"]["distillation_id"] = "another-v1"
        self.write_documents()
        self.assertIn("DISTILLATION_ID_MISMATCH", self.error_codes())

    def test_candidate_t4_may_have_pending_decision(self):
        set_rule_pending(self.documents)
        self.write_documents()
        report = validate_distillation(self.root)
        self.assertFalse(any(item.code.startswith("T34_DECISION") for item in report.errors))

    def test_accepted_t4_requires_completed_human_decision(self):
        rule = self.documents["capability-rules.yml"]["capability_rules"][0]
        rule["human_decision"] = {
            "decision": "pending", "reviewer_type": None, "reviewer": None,
            "decided_at": None, "rationale": "", "gate_decision_id": None,
        }
        self.write_documents()
        codes = self.error_codes()
        self.assertIn("T34_DECISION_NOT_ACCEPTED", codes)
        self.assertIn("T34_DECISION_INCOMPLETE", codes)

    def test_accepted_rule_rejects_needs_verification_evidence(self):
        self.documents["evidence-ledger.yml"]["evidence"][0]["status"] = "needs-verification"
        self.write_documents()
        self.assertIn("BLOCKED_DEPENDENCY", self.error_codes())

    def test_accepted_rule_rejects_blocking_quality_flag(self):
        self.documents["evidence-ledger.yml"]["evidence"][0]["quality_flags"] = [
            "nonblocking-looking-note"
        ]
        self.write_documents()
        self.assertIn("BLOCKED_DEPENDENCY", self.error_codes())

    def test_schema_version_must_be_integer_one(self):
        for invalid in (True, "1", 1.0, 2):
            with self.subTest(invalid=invalid):
                self.documents = self.fresh_documents()
                self.documents["concept-map.yml"]["schema_version"] = invalid
                self.write_documents()
                self.assertIn("SCHEMA_VERSION", self.error_codes())

    def test_rejects_id_surrounding_whitespace(self):
        self.documents["evidence-ledger.yml"]["evidence"][0]["evidence_id"] = " ev-001"
        self.write_documents()
        self.assertIn("INVALID_ID", self.error_codes())

    def test_rule_trigger_and_output_must_be_nonempty(self):
        rule = self.documents["capability-rules.yml"]["capability_rules"][0]
        rule["trigger"] = {"task": "  ", "signals": []}
        rule["output"] = ["  "]
        self.write_documents()
        empty_paths = {
            issue.path for issue in validate_distillation(self.root).errors
            if issue.code == "EMPTY_FIELD"
        }
        self.assertIn("capability-rules.yml.capability_rules[0].trigger", empty_paths)
        self.assertIn("capability-rules.yml.capability_rules[0].output", empty_paths)

    def test_rejects_duplicate_trigger_boundaries(self):
        candidate = self.documents["capability-rules.yml"]["skill_candidates"][0]
        candidate["should_trigger"] = ["Same", " same ", "third"]
        candidate["should_not_trigger"] = ["one", "one", "three"]
        self.write_documents()
        codes = self.error_codes()
        self.assertIn("DUPLICATE_TRIGGER_CASE", codes)
        self.assertIn("DUPLICATE_NONTRIGGER_CASE", codes)

    def test_rejects_trigger_nontrigger_overlap(self):
        candidate = self.documents["capability-rules.yml"]["skill_candidates"][0]
        candidate["should_not_trigger"][0] = " REQUEST ONE "
        self.write_documents()
        self.assertIn("TRIGGER_BOUNDARY_OVERLAP", self.error_codes())

    def test_skill_candidates_allowed_only_in_capability_rules(self):
        self.documents["concept-map.yml"]["skill_candidates"] = []
        self.write_documents()
        self.assertIn("MISPLACED_SKILL_CANDIDATES", self.error_codes())

    def test_rejects_plural_locator_fields(self):
        evidence = self.documents["evidence-ledger.yml"]["evidence"][0]
        evidence["locators"] = [copy.deepcopy(evidence["locator"])]
        evidence["related_locators"] = [copy.deepcopy(evidence["locator"])]
        self.write_documents()
        issues = [
            item for item in validate_distillation(self.root).errors
            if item.code == "FORBIDDEN_LOCATOR_FIELD"
        ]
        self.assertEqual(2, len(issues))

    def test_every_accepted_record_requires_status_history(self):
        cases = [
            ("evidence-ledger.yml", "evidence"),
            ("evidence-ledger.yml", "claims"),
            ("concept-map.yml", "relations"),
            ("capability-rules.yml", "capability_rules"),
        ]
        for filename, key in cases:
            with self.subTest(filename=filename, key=key):
                self.documents = self.fresh_documents()
                self.documents[filename][key][0].pop("status_history")
                self.write_documents()
                self.assertIn("STATUS_HISTORY_REQUIRED", self.error_codes())

    def test_every_non_draft_candidate_requires_lifecycle_history(self):
        for lifecycle in ("review", "accepted", "deployed", "deprecated", "rejected"):
            with self.subTest(lifecycle=lifecycle):
                self.documents = self.fresh_documents()
                candidate = self.documents["capability-rules.yml"]["skill_candidates"][0]
                candidate["lifecycle"] = lifecycle
                candidate.pop("lifecycle_history")
                self.write_documents()
                self.assertIn("LIFECYCLE_HISTORY_REQUIRED", self.error_codes())

    def test_accepted_candidate_requires_accepted_rules(self):
        candidate = self.documents["capability-rules.yml"]["skill_candidates"][0]
        candidate["lifecycle"] = "accepted"
        candidate["lifecycle_history"].append({
            "from": "review",
            "to": "accepted",
            "decided_by": "human-reviewer",
            "decided_at": "2026-08-04",
            "rationale": "Accepted for the test.",
        })
        rule = self.documents["capability-rules.yml"]["capability_rules"][0]
        rule["status"] = "candidate"
        rule.pop("status_history")
        rule["human_decision"] = {
            "decision": "pending", "reviewer_type": None, "reviewer": None,
            "decided_at": None, "rationale": "", "gate_decision_id": None,
        }
        self.write_documents()
        self.assertIn("CANDIDATE_RULE_NOT_ACCEPTED", self.error_codes())

    def test_accepted_candidate_requires_at_least_one_rule(self):
        candidate = self.documents["capability-rules.yml"]["skill_candidates"][0]
        candidate["lifecycle"] = "accepted"
        candidate["lifecycle_history"].append({
            "from": "review", "to": "accepted", "decided_by": "human-reviewer",
            "decided_at": "2026-08-04", "rationale": "Accepted for the test."
        })
        candidate["rule_ids"] = []
        self.write_documents()
        self.assertIn("CANDIDATE_RULES_REQUIRED", self.error_codes())

    def test_no_accepted_rules_reports_not_applicable(self):
        set_rule_pending(self.documents)
        self.write_documents()
        report = validate_distillation(self.root)
        self.assertIsNone(report.metrics["accepted_rule_traceability"])
        self.assertFalse(report.metrics["accepted_rule_traceability_applicable"])

    def test_project_policy_is_valid_source_position(self):
        self.documents["evidence-ledger.yml"]["claims"][0]["source_position"] = "project-policy"
        self.write_documents()
        self.assertNotIn("INVALID_ENUM", self.error_codes())

    def test_sources_manifest_is_opt_in(self):
        report = validate_distillation(self.root)
        self.assertNotIn("UNKNOWN_SOURCE_ID", {item.code for item in report.errors})
        manifest = self.root / "sources.yml"
        manifest_data = valid_sources_manifest()
        manifest_data["sources"][0]["id"] = "another-book"
        manifest.write_text(
            yaml.safe_dump(manifest_data),
            encoding="utf-8",
        )
        report = validate_distillation(self.root, manifest)
        self.assertIn("UNKNOWN_SOURCE_ID", {item.code for item in report.errors})

    def test_sources_manifest_accepts_known_source(self):
        manifest = self.write_manifest()
        report = validate_distillation(self.root, manifest)
        self.assertTrue(report.ok, [item.as_dict() for item in report.errors])

    def test_markdown_locator_resolves_allowed_path_heading_and_hash(self):
        manifest = self.configure_markdown_policy()
        report = validate_distillation(self.root, manifest)
        self.assertTrue(report.ok, [item.as_dict() for item in report.errors])
        self.assertEqual([], [item.as_dict() for item in report.warnings])
        self.assertEqual(1, report.metrics["markdown_locator_count"])
        self.assertEqual(1, report.metrics["markdown_locator_resolved"])
        self.assertTrue(report.metrics["markdown_locator_resolution_applicable"])

    def test_markdown_locator_accepts_related_path_and_line_range(self):
        manifest = self.configure_markdown_policy(
            repository_path="docs/related.md",
            local_path="docs/policy.md",
            related_local_paths=["docs/related.md"],
            anchor="docs/related.md:5-5#Review-Gate",
        )
        report = validate_distillation(self.root, manifest)
        self.assertTrue(report.ok, [item.as_dict() for item in report.errors])

    def test_markdown_locator_accepts_line_only_anchor(self):
        manifest = self.configure_markdown_policy(anchor="docs/policy.md:5")
        report = validate_distillation(self.root, manifest)
        self.assertTrue(report.ok, [item.as_dict() for item in report.

from __future__ import annotations

import hashlib
import json
import os
import sys
import tempfile
import unittest
from pathlib import Path

import yaml

SCRIPT_DIR = Path(__file__).resolve().parent
sys.path.insert(0, str(SCRIPT_DIR))

from verify_normalized_locators import verify


class NormalizedLocatorTests(unittest.TestCase):
    def setUp(self):
        self.tempdir = tempfile.TemporaryDirectory()
        self.root = Path(self.tempdir.name)
        self.distillation = self.root / "distillation"
        self.normalized_root = self.root / "normalized"
        self.bundle = self.normalized_root / "book-001"
        self.manifest = self.root / "manifests" / "sources.yml"
        self.distillation.mkdir()
        self.bundle.mkdir(parents=True)
        self.manifest.parent.mkdir()
        self.source_sha = "a" * 64
        text = "Source text."
        full_hash = hashlib.sha256(text.encode()).hexdigest()
        short_hash = full_hash[:12]
        ledger = {
            "schema_version": 1,
            "distillation_id": "test-v1",
            "evidence": [{
                "evidence_id": "ev-001",
                "source_id": "book-001",
                "locator": {
                    "source_id": "book-001",
                    "locator_type": "ooxml-block",
                    "heading_path": ["Chapter 1"],
                    "ooxml_block_index": 1,
                    "content_hash": short_hash,
                },
                "raw_text": text,
                "normalized_text": text,
            }],
            "claims": [],
        }
        (self.distillation / "evidence-ledger.yml").write_text(
            yaml.safe_dump(ledger, sort_keys=False), encoding="utf-8"
        )
        block = {
            "schema_version": 1,
            "source_id": "book-001",
            "ooxml_block_index": 1,
            "locator": {
                "source_id": "book-001",
                "heading_path": ["Chapter 1"],
                "ooxml_block_index": 1,
                "content_hash": short_hash,
            },
            "heading_path": ["Chapter 1"],
            "raw_text": text,
            "normalized_text": text,
            "text_sha256": full_hash,
            "short_content_hash": short_hash,
            "figure_ids": [],
        }
        (self.bundle / "blocks.jsonl").write_text(
            json.dumps(block) + "\n", encoding="utf-8"
        )
        for filename in ("structure.yml", "media-map.yml", "normalization-log.yml"):
            (self.bundle / filename).write_text(
                yaml.safe_dump({"schema_version": 1, "source_id": "book-001"}),
                encoding="utf-8",
            )
        self._write_checksums()
        self._write_manifest()

    def tearDown(self):
        self.tempdir.cleanup()

    def _write_manifest(self, checksum=None):
        document = {
            "schema_version": 2,
            "sources": [{
                "id": "book-001",
                "checksum": checksum or f"sha256:{self.source_sha}",
            }],
        }
        self.manifest.write_text(yaml.safe_dump(document), encoding="utf-8")

    def _write_checksums(self, *, after=None):
        generated = []
        for filename in (
            "blocks.jsonl", "structure.yml", "media-map.yml", "normalization-log.yml"
        ):
            content = (self.bundle / filename).read_bytes()
            generated.append({
                "path": filename,
                "byte_size": len(content),
                "sha256": hashlib.sha256(content).hexdigest(),
            })
        checksums = {
            "schema_version": 1,
            "source_id": "book-001",
            "source": {
                "sha256_before": self.source_sha,
                "sha256_after": self.source_sha if after is None else after,
                "sha256_unchanged": after is None,
            },
            "generated_files": generated,
        }
        (self.bundle / "checksums.yml").write_text(
            yaml.safe_dump(checksums, sort_keys=False), encoding="utf-8"
        )

    def _verify(self):
        return verify(self.distillation, self.normalized_root, self.manifest)

    def test_resolves_valid_locator_excerpt_and_bundle_integrity(self):
        report = self._verify()
        self.assertTrue(report.ok, [item.as_dict() for item in report.errors])
        self.assertEqual(1, report.checked)
        self.assertEqual(1, report.integrity_verified_sources)
        self.assertEqual(1, report.manifest_checksum_verified_sources)

    def test_rejects_hash_and_excerpt_drift(self):
        ledger = yaml.safe_load((self.distillation / "evidence-ledger.yml").read_text())
        ledger["evidence"][0]["locator"]["content_hash"] = "ffffffffffff"
        ledger["evidence"][0]["raw_text"] = "Invented text."
        (self.distillation / "evidence-ledger.yml").write_text(
            yaml.safe_dump(ledger, sort_keys=False), encoding="utf-8"
        )
        report = self._verify()
        self.assertFalse(report.ok)
        self.assertEqual(
            {"CONTENT_HASH_MISMATCH", "EVIDENCE_TEXT_MISMATCH"},
            {item.code for item in report.errors},
        )

    def test_rejects_source_checksum_change(self):
        self._write_checksums(after="b" * 64)
        report = self._verify()
        self.assertIn("SOURCE_CHECKSUM_CHANGED", {item.code for item in report.errors})

    def test_rejects_joint_ledger_and_bundle_tamper_without_checksum_update(self):
        invented = "Invented replacement text."
        full_hash = hashlib.sha256(invented.encode()).hexdigest()
        short_hash = full_hash[:12]
        ledger = yaml.safe_load((self.distillation / "evidence-ledger.yml").read_text())
        evidence = ledger["evidence"][0]
        evidence["raw_text"] = invented
        evidence["normalized_text"] = invented
        evidence["locator"]["content_hash"] = short_hash
        (self.distillation / "evidence-ledger.yml").write_text(
            yaml.safe_dump(ledger, sort_keys=False), encoding="utf-8"
        )
        block = json.loads((self.bundle / "blocks.jsonl").read_text())
        block["raw_text"] = invented
        block["normalized_text"] = invented
        block["text_sha256"] = full_hash
        block["short_content_hash"] = short_hash
        block["locator"]["content_hash"] = short_hash
        (self.bundle / "blocks.jsonl").write_text(json.dumps(block) + "\n")
        report = self._verify()
        self.assertFalse(report.ok)
        self.assertIn(
            "BUNDLE_GENERATED_FILE_MISMATCH", {item.code for item in report.errors}
        )

    def test_rejects_manifest_source_checksum_mismatch(self):
        self._write_manifest(checksum="sha256:" + "c" * 64)
        report = self._verify()
        self.assertIn(
            "MANIFEST_SOURCE_CHECKSUM_MISMATCH", {item.code for item in report.errors}
        )

    def test_rejects_unsafe_source_id_bundle_traversal(self):
        ledger = yaml.safe_load((self.distillation / "evidence-ledger.yml").read_text())
        ledger["evidence"][0]["source_id"] = "../outside"
        ledger["evidence"][0]["locator"]["source_id"] = "../outside"
        (self.distillation / "evidence-ledger.yml").write_text(
            yaml.safe_dump(ledger, sort_keys=False), encoding="utf-8"
        )
        manifest = yaml.safe_load(self.manifest.read_text())
        manifest["sources"][0]["id"] = "../outside"
        self.manifest.write_text(yaml.safe_dump(manifest), encoding="utf-8")
        report = self._verify()
        self.assertFalse(report.ok)
        self.assertIn("BUNDLE_READ", {item.code for item in report.errors})

    def test_rejects_symlinked_bundle_file(self):
        target = self.root / "outside.jsonl"
        target.write_text((self.bundle / "blocks.jsonl").read_text(), encoding="utf-8")
        (self.bundle / "blocks.jsonl").unlink()
        try:
            os.symlink(target, self.bundle / "blocks.jsonl")
        except OSError as exc:
            self.skipTest(f"symlinks unavailable: {exc}")
        report = self._verify()
        self.assertFalse(report.ok)
        self.assertIn("BUNDLE_GENERATED_FILE_READ", {item.code for item in report.errors})


if __name__ == "__main__":
    unittest.main()

#!/usr/bin/env python3
"""Validate distillation YAML structure and traceability, not knowledge truth.

The validator is intentionally read-only. It checks five required governance
YAML files plus a conditionally required correction overlay and never claims
that their scientific or historical contents are correct, current, or
externally accepted.
"""

from __future__ import annotations

import argparse
import contextlib
import hashlib
import json
import os
import re
import stat
import sys
import tempfile
import unicodedata
from dataclasses import dataclass, field
from pathlib import Path, PurePosixPath
from typing import Any, Iterator, Iterable, Mapping, Sequence

# Running this validator must not mutate a materialized candidate by creating
# bytecode for its local helper import, even when the caller forgets ``-B``.
sys.dont_write_bytecode = True

from hash_candidate_tree import (
    CandidateTreeError,
    candidate_tree_sha256,
    canonical_candidate_path,
)
from task_contracts import inspect_task_governance

try:
    import yaml
except ImportError as exc:  # pragma: no cover - exercised only in minimal runtimes
    raise SystemExit(
        "PyYAML is required to parse YAML. Do not install it without the user's approval."
    ) from exc


REQUIRED_FILES = {
    "evidence-ledger.yml": ("evidence", "claims"),
    "concept-map.yml": ("relations",),
    "capability-rules.yml": ("capability_rules",),
    "gate-decisions.yml": ("gate_decisions", "materializations"),
    "eval-runs.yml": ("eval_runs",),
}
RECORD_STATUSES = {
    "candidate", "accepted", "reference-only", "needs-verification", "rejected"
}
SKILL_LIFECYCLES = {
    "draft", "review", "accepted", "deployed", "deprecated", "rejected"
}
TRANSFORMATIONS = {"T0", "T1", "T2", "T3", "T4"}
RELATION_STATUSES = {"explicit", "implicit", "inferred"}
HUMAN_DECISIONS = {"pending", "accepted", "revised", "rejected"}
EVIDENCE_TYPES = {
    "text", "definition", "figure", "figure-caption", "table", "case",
    "footnote", "author-summary", "quoted-source", "formula",
}
CAPTURE_MODES = {"quote", "excerpt", "paraphrase", "visual-observation", "ocr"}
CONFIDENCE_LEVELS = {"high", "medium", "low"}
CLAIM_TYPES = {
    "definition", "principle", "empirical-claim", "relation", "distinction",
    "method", "heuristic", "misconception", "limitation", "controversy",
    "analogy", "historical-development",
}
SOURCE_POSITIONS = {
    "book-assertion", "author-view", "teaching-simplification", "quoted-source",
    "project-policy", "distiller-synthesis", "task-transfer",
}
IMPORTANCE_LEVELS = {"important", "supporting"}
ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]*$")
SKILL_NAME_RE = re.compile(r"^[a-z0-9]+(?:-[a-z0-9]+)*$")
CONTENT_HASH_RE = re.compile(r"^(?:sha256:)?[0-9a-fA-F]{12,64}$")
TREE_HASH_RE = re.compile(r"^sha256:[0-9a-f]{64}$")
FULL_SHA256_RE = re.compile(r"^(?:sha256:)?(?P<digest>[0-9a-fA-F]{64})$")
ANCHOR_LINE_RE = re.compile(r"^(?P<start>[1-9][0-9]*)(?:-(?P<end>[1-9][0-9]*))?$")
ATX_HEADING_RE = re.compile(r"^\s{0,3}(?P<marks>#{1,6})[ \t]+(?P<title>.+?)\s*$")
FENCE_RE = re.compile(r"^\s{0,3}(?P<marker>`{3,}|~{3,})")
TRAILING_PAREN_RE = re.compile(r"\s*[（(][^（）()]*[）)]\s*$")
PLACEHOLDER_RE = re.compile(
    r"\{\{[^{}]+\}\}|(?:^|[\s._:-])(?:todo|tbd|placeholder|replace[-_ ]?me)(?:$|[\s._:-])",
    re.IGNORECASE,
)
EXAMPLE_IDENTIFIER_RE = re.compile(
    r"(?:^|[._:-])(?:example|sample)(?:$|[._:-])", re.IGNORECASE
)

GATES = {"gate-1", "gate-2", "gate-3", "gate-4", "gate-5"}
GATE_DECISIONS = {
    "pending", "approved", "approved-with-conditions", "approved-for-eval",
    "accepted", "revise", "rejected", "blocked",
}
GATE_DECISIONS_BY_GATE = {
    "gate-1": {"pending", "approved", "approved-with-conditions", "revise", "rejected", "blocked"},
    "gate-2": {"pending", "approved", "approved-with-conditions", "revise", "rejected", "blocked"},
    "gate-3": {"pending", "approved-for-eval", "revise", "rejected", "blocked"},
    "gate-4": {"pending", "accepted", "revise", "rejected", "blocked"},
    "gate-5": {"pending", "approved", "approved-with-conditions", "revise", "rejected", "blocked"},
}
HUMAN_REVIEWER_TYPES = {"user", "human-delegate"}
EVAL_STATUSES = {"planned", "blocked", "completed", "invalidated"}
EVAL_OUTCOMES = {None, "pass", "fail", "inconclusive"}
EVAL_CASE_TYPES = {"trigger", "nontrigger", "task"}
SEMANTIC_SUPPORT_KEYS = ("checks", "action", "output", "stop_conditions")
RULE_GATE_DECISIONS = {"accepted", "revised", "rejected"}
MATERIALIZATION_STATUSES = {
    "planned", "completed", "failed", "invalidated", "legacy-quarantined"
}
QUICK_VALIDATION_STATUSES = {"not-run", "pass", "fail"}
POSITIVE_GATE_DECISIONS = {
    "gate-1": {"approved", "approved-with-conditions"},
    "gate-2": {"approved", "approved-with-conditions"},
    "gate-3": {"approved-for-eval"},
    "gate-4": {"accepted"},
    "gate-5": {"approved", "approved-with-conditions"},
}

GATE3_APPROVAL_SNAPSHOT_CONTRACT = "gate3-approval-snapshot:v2"
GATE3_APPROVAL_SNAPSHOT_LEGACY_CONTRACT = "gate3-approval-snapshot:v1"
APPROVAL_SNAPSHOT_GOVERNANCE_FILES = (
    "evidence-ledger.yml",
    "concept-map.yml",
    "capability-rules.yml",
)

RECORD_TRANSITIONS = {
    "candidate": {"accepted", "reference-only", "needs-verification", "rejected"},
    "accepted": {"candidate"},
    "reference-only": {"candidate"},
    "needs-verification": {"candidate"},
    "rejected": {"candidate"},
}
LIFECYCLE_TRANSITIONS = {
    "draft": {"review", "rejected"},
    "review": {"accepted", "rejected"},
    "accepted": {"deployed", "deprecated"},
    "deployed": {"deprecated"},
    "deprecated": set(),
    "rejected": set(),
}


class DistillationInputError(RuntimeError):
    """Raised when required input files cannot be read or parsed."""

    def __init__(self, code: str, path: Path, message: str):
        super().__init__(message)
        self.code = code
        self.path = path
        self.message = message


class _DuplicateYamlKeyError(yaml.YAMLError):
    """Raised when a YAML mapping contains the same key more than once."""


class _UniqueKeySafeLoader(yaml.SafeLoader):
    """SafeLoader variant that rejects duplicate mapping keys."""


def _construct_unique_mapping(
    loader: _UniqueKeySafeLoader,
    node: yaml.nodes.MappingNode,
    deep: bool = False,
) -> dict[Any, Any]:
    loader.flatten_mapping(node)
    mapping: dict[Any, Any] = {}
    for key_node, value_node in node.value:
        key = loader.construct_object(key_node, deep=deep)
        try:
            duplicate = key in mapping
        except TypeError as exc:
            raise yaml.constructor.ConstructorError(
                "while constructing a mapping",
                node.start_mark,
                "found an unhashable mapping key",
                key_node.start_mark,
            ) from exc
        if duplicate:
            location = f"line {key_node.start_mark.line + 1}, column {key_node.start_mark.column + 1}"
            raise _DuplicateYamlKeyError(f"duplicate mapping key {key!r} at {location}")
        mapping[key] = loader.construct_object(value_node, deep=deep)
    return mapping


_UniqueKeySafeLoader.add_constructor(
    yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
    _construct_unique_mapping,
)


class _ArtifactFileError(RuntimeError):
    """A candidate or evaluation artifact cannot be read safely."""

    def __init__(self, code: str, message: str):
        super().__init__(message)
        self.code = code
        self.message = message


@dataclass(frozen=True)
class Issue:
    code: str
    path: str
    message: str

    def as_dict(self) -> dict[str, str]:
        return {"code": self.code, "path": self.path, "message": self.message}


@dataclass
class ValidationReport:
    root: str
    errors: list[Issue] = field(default_factory=list)
    warnings: list[Issue] = field(default_factory=list)
    metrics: dict[str, Any] = field(default_factory=dict)

    @property
    def ok(self) -> bool:
        return not self.errors

    def add_error(self, code: str, path: str, message: str) -> None:
        self.errors.append(Issue(code, path, message))

    def add_warning(self, code: str, path: str, message: str) -> None:
        self.warnings.append(Issue(code, path, message))

    def as_dict(self) -> dict[str, Any]:
        return {
            "validator_scope": "structure-and-traceability-only",
            "truth_assessed": False,
            "behavior_effectiveness_assessed": False,
            "root": self.root,
            "ok": self.ok,
            "errors": [item.as_dict() for item in self.errors],
            "warnings": [item.as_dict() for item in self.warnings],
            "metrics": self.metrics,
        }


@dataclass(frozen=True)
class MarkdownAnchor:
    repository_path: str
    start_line: int
    end_line: int
    heading: str | None


@dataclass(frozen=True)
class MarkdownSection:
    start_line: int
    end_line: int
    level: int
    title: str
    slug_variants: frozenset[str]


@dataclass(frozen=True)
class TextOccurrence:
    start_offset: int
    end_offset: int
    start_line: int
    end_line: int


@dataclass(frozen=True)
class EvalCaseContract:
    case_type: str
    case_id: str
    request: str
    definition_hash: str
    holdout: bool
    prompt_leakage_terms: tuple[str, ...] = ()


@dataclass(frozen=True)
class EvalRubricContract:
    rubric_id: str
    dimensions: tuple[str, ...]
    score_min: float
    score_max_per_dimension: float
    pass_threshold: float
    fatal_failures: frozenset[str]

    @property
    def max_score(self) -> float:
        return len(self.dimensions) * self.score_max_per_dimension


@dataclass(frozen=True)
class CandidateEvalContract:
    """Versioned cases and rubric parsed from one materialized candidate tree."""

    case_ids: Mapping[str, frozenset[str]]
    task_holdout: Mapping[str, bool]
    cases: Mapping[str, Mapping[str, EvalCaseContract]]
    rubric: EvalRubricContract | None
    definitions_valid: bool


def _load_yaml(path: Path) -> dict[str, Any]:
    if not path.is_file():
        raise DistillationInputError("MISSING_FILE", path, "required YAML file is missing")
    try:
        data = yaml.load(
            path.read_text(encoding="utf-8"),
            Loader=_UniqueKeySafeLoader,
        )
    except (OSError, UnicodeError) as exc:
        raise DistillationInputError("READ_ERROR", path, str(exc)) from exc
    except _DuplicateYamlKeyError as exc:
        raise DistillationInputError("YAML_DUPLICATE_KEY", path, str(exc)) from exc
    except (yaml.YAMLError, RecursionError) as exc:
        raise DistillationInputError("YAML_PARSE", path, str(exc)) from exc
    if not isinstance(data, dict):
        raise DistillationInputError("ROOT_TYPE", path, "YAML root must be a mapping")
    return data


def _canonical_relative_artifact_path(value: Any) -> str:
    if not isinstance(value, str) or not value or value != value.strip():
        raise _ArtifactFileError(
            "PATH_INVALID",
            "artifact path must be a non-empty string without surrounding whitespace",
        )
    if "\\" in value or "\x00" in value:
        raise _ArtifactFileError(
            "PATH_INVALID",
            "artifact path must use POSIX separators and contain no NUL byte",
        )
    raw_parts = value.split("/")
    if any(part in {"", ".", ".."} for part in raw_parts):
        raise _ArtifactFileError(
            "PATH_INVALID",
            "artifact path must be canonical and contain no empty, '.' or '..' components",
        )
    relative = PurePosixPath(value)
    if relative.is_absolute() or not relative.parts or relative.as_posix() != value:
        raise _ArtifactFileError(
            "PATH_INVALID",
            "artifact path must be a canonical relative POSIX path",
        )
    try:
        value.encode("utf-8", errors="strict")
    except UnicodeEncodeError as exc:
        raise _ArtifactFileError(
            "PATH_INVALID", "artifact path must be valid UTF-8"
        ) from exc
    return value


def _same_filesystem_entry(before: os.stat_result, after: os.stat_result) -> bool:
    same_identity = (
        before.st_dev == after.st_dev
        and before.st_ino == after.st_ino
        and stat.S_IFMT(before.st_mode) == stat.S_IFMT(after.st_mode)
    )
    if not same_identity:
        return False
    # Directory size/mtime changes when unrelated siblings are created. Identity
    # and no-follow checks are sufficient for path components; ordinary file
    # bytes still require stable size and mtime.
    if stat.S_ISDIR(before.st_mode):
        return True
    return before.st_size == after.st_size and before.st_mtime_ns == after.st_mtime_ns


def _read_distillation_artifact(root: Path, relative_path: Any) -> tuple[str, bytes]:
    """Read one root-confined regular file without accepting symlink components."""
    canonical = _canonical_relative_artifact_path(relative_path)
    try:
        resolved_root = root.resolve(strict=True)
    except OSError as exc:
        raise _ArtifactFileError(
            "ROOT_INVALID", f"distillation root cannot be resolved: {exc}"
        ) from exc
    try:
        if not stat.S_ISDIR(resolved_root.stat().st_mode):
            raise _ArtifactFileError("ROOT_INVALID", "distillation root is not a directory")
    except OSError as exc:
        raise _ArtifactFileError(
            "ROOT_INVALID", f"distillation root cannot be inspected: {exc}"
        ) from exc

    components: list[tuple[Path, os.stat_result]] = []
    current = resolved_root
    parts = PurePosixPath(canonical).parts
    for index, part in enumerate(parts):
        current = current / part
        try:
            entry_stat = current.lstat()
        except FileNotFoundError as exc:
            raise _ArtifactFileError("MISSING", "artifact file does not exist") from exc
        except OSError as exc:
            raise _ArtifactFileError(
                "READ_ERROR", f"artifact path cannot be inspected: {exc}"
            ) from exc
        if stat.S_ISLNK(entry_stat.st_mode):
            raise _ArtifactFileError("SYMLINK", "artifact path contains a symlink")
        is_last = index == len(parts) - 1
        if not is_last and not stat.S_ISDIR(entry_stat.st_mode):
            raise _ArtifactFileError(
                "NOT_REGULAR", "an intermediate artifact path component is not a directory"
            )
        if is_last and not stat.S_ISREG(entry_stat.st_mode):
            raise _ArtifactFileError("NOT_REGULAR", "artifact is not an ordinary file")
        components.append((current, entry_stat))

    flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0)
    try:
        descriptor = os.open(current, flags)
    except OSError as exc:
        raise _ArtifactFileError(
            "READ_ERROR", f"artifact file cannot be opened safely: {exc}"
        ) from exc
    try:
        opened_stat = os.fstat(descriptor)
        if not stat.S_ISREG(opened_stat.st_mode):
            raise _ArtifactFileError("NOT_REGULAR", "artifact is not an ordinary file")
        if not _same_filesystem_entry(components[-1][1], opened_stat):
            raise _ArtifactFileError("CHANGED", "artifact changed before it was read")
        chunks: list[bytes] = []
        while True:
            chunk = os.read(descriptor, 1024 * 1024)
            if not chunk:
                break
            chunks.append(chunk)
        final_stat = os.fstat(descriptor)
    except OSError as exc:
        raise _ArtifactFileError(
            "READ_ERROR", f"artifact file cannot be read: {exc}"
        ) from exc
    finally:
        os.close(descriptor)
    if not _same_filesystem_entry(opened_stat, final_stat):
        raise _ArtifactFileError("CHANGED", "artifact changed while it was read")
    content = b"".join(chunks)
    if len(content) != final_stat.st_size:
        raise _ArtifactFileError(
            "CHANGED", "artifact bytes do not match the stable file length"
        )
    for component, original_stat in components:
        try:
            current_stat = component.lstat()
        except OSError as exc:
            raise _ArtifactFileError(
                "CHANGED", f"artifact path changed after reading: {exc}"
            ) from exc
        if stat.S_ISLNK(current_stat.st_mode) or not _same_filesystem_entry(
            original_stat, current_stat
        ):
            raise _ArtifactFileError("CHANGED", "artifact path changed while it was read")
    return canonical, content


@contextlib.contextmanager
def distillation_read_snapshot(root: Path | str) -> Iterator[Path]:
    """Yield a private byte snapshot of a distillation tree.

    Every ordinary file is copied from one stable, no-follow read. Symlinks and
    non-regular entries fail closed. The immutable copy lets routing, parsing,
    artifact validation, and candidate hashing all observe the same bytes even
    if another process rewrites and restores the source tree concurrently.
    """
    source = Path(root)
    try:
        resolved_source = source.resolve(strict=True)
        source_stat = resolved_source.lstat()
    except OSError as exc:
        raise DistillationInputError(
            "DISTILLATION_SNAPSHOT_INVALID",
            source,
            f"distillation root cannot be inspected: {exc}",
        ) from exc
    if not stat.S_ISDIR(source_stat.st_mode):
        raise DistillationInputError(
            "DISTILLATION_SNAPSHOT_INVALID",
            source,
            "distillation root must be a directory",
        )

    with tempfile.TemporaryDirectory(prefix="distillation-read-snapshot-") as tempdir:
        snapshot_root = Path(tempdir) / "distillation"
        snapshot_root.mkdir()
        directories: list[tuple[Path, os.stat_result, str]] = []

        def capture(directory: Path, relative: PurePosixPath | None) -> None:
            display = "." if relative is None else relative.as_posix()
            try:
                directory_stat = directory.lstat()
            except OSError as exc:
                raise DistillationInputError(
                    "DISTILLATION_SNAPSHOT_CHANGED",
                    directory,
                    f"directory cannot be inspected: {exc}",
                ) from exc
            if stat.S_ISLNK(directory_stat.st_mode) or not stat.S_ISDIR(
                directory_stat.st_mode
            ):
                raise DistillationInputError(
                    "DISTILLATION_SNAPSHOT_INVALID",
                    directory,
                    "snapshot traversal accepts only ordinary directories",
                )
            directories.append((directory, directory_stat, display))
            destination_directory = (
                snapshot_root
                if relative is None
                else snapshot_root.joinpath(*relative.parts)
            )
            try:
                entries = sorted(os.scandir(directory), key=lambda item: item.name)
            except OSError as exc:
                raise DistillationInputError(
                    "DISTILLATION_SNAPSHOT_READ_ERROR",
                    directory,
                    f"directory cannot be enumerated: {exc}",
                ) from exc
            for entry in entries:
                relative_entry = (
                    PurePosixPath(entry.name)
                    if relative is None
                    else relative / entry.name
                )
                relative_string = relative_entry.as_posix()
                try:
                    entry_stat = entry.stat(follow_symlinks=False)
                except OSError as exc:
                    raise DistillationInputError(
                        "DISTILLATION_SNAPSHOT_READ_ERROR",
                        Path(entry.path),
                        f"entry cannot be inspected: {exc}",
                    ) from exc
                destination = destination_directory / entry.name
                if stat.S_ISLNK(entry_stat.st_mode):
                    # Preserve the link itself without reading its target. Any
                    # referenced governance/artifact/candidate path will then
                    # be rejected by its existing no-follow validation. Map an
                    # in-root target into the private snapshot; replace an
                    # out-of-root target with a self-loop so parsing cannot
                    # accidentally follow it outside the snapshot.
                    try:
                        raw_target = os.readlink(entry.path)
                        source_target = Path(
                            os.path.abspath(
                                raw_target
                                if os.path.isabs(raw_target)
                                else os.path.join(directory, raw_target)
                            )
                        )
                        try:
                            target_relative = source_target.relative_to(resolved_source)
                        except ValueError:
                            snapshot_target = entry.name
                        else:
                            snapshot_target = str(snapshot_root / target_relative)
                        os.symlink(snapshot_target, destination)
                    except OSError as exc:
                        raise DistillationInputError(
                            "DISTILLATION_SNAPSHOT_WRITE_ERROR",
                            destination,
                            f"symlink metadata cannot be snapshotted: {exc}",
                        ) from exc
                    continue
                if stat.S_ISDIR(entry_stat.st_mode):
                    destination.mkdir()
                    capture(Path(entry.path), relative_entry)
                    continue
                if not stat.S_ISREG(entry_stat.st_mode):
                    raise DistillationInputError(
                        "DISTILLATION_SNAPSHOT_NON_REGULAR",
                        Path(entry.path),
                        "only ordinary files and directories may enter a validation snapshot",
                    )
                try:
                    _, content = _read_distillation_artifact(
                        resolved_source, relative_string
                    )
                    destination.write_bytes(content)
                except _ArtifactFileError as exc:
                    raise DistillationInputError(
                        "DISTILLATION_SNAPSHOT_CHANGED",
                        Path(entry.path),
                        f"{exc.code}: {exc.message}",
                    ) from exc
                except OSError as exc:
                    raise DistillationInputError(
                        "DISTILLATION_SNAPSHOT_WRITE_ERROR",
                        destination,
                        str(exc),
                    ) from exc

        capture(resolved_source, None)
        for directory, original_stat, display in directories:
            try:
                final_stat = directory.lstat()
            except OSError as exc:
                raise DistillationInputError(
                    "DISTILLATION_SNAPSHOT_CHANGED",
                    directory,
                    f"directory changed after capture: {exc}",
                ) from exc
            if not _same_filesystem_entry(original_stat, final_stat):
                raise DistillationInputError(
                    "DISTILLATION_SNAPSHOT_CHANGED",
                    directory,
                    f"directory {display!r} changed during snapshot capture",
                )
        yield snapshot_root


def _read_stable_external_file(path: Path | str) -> tuple[Path, bytes]:
    """Read one absolute ordinary file while rejecting every symlink component."""
    requested = Path(path)
    absolute = Path(os.path.abspath(requested))
    parts = absolute.parts
    if not parts:
        raise _ArtifactFileError("PATH_INVALID", "external file path is empty")
    current = Path(parts[0])
    components: list[tuple[Path, os.stat_result]] = []
    for index, part in enumerate(parts[1:], start=1):
        current = current / part
        try:
            entry_stat = current.lstat()
        except FileNotFoundError as exc:
            raise _ArtifactFileError("MISSING", "external file does not exist") from exc
        except OSError as exc:
            raise _ArtifactFileError(
                "READ_ERROR", f"external path cannot be inspected: {exc}"
            ) from exc
        if stat.S_ISLNK(entry_stat.st_mode):
            raise _ArtifactFileError("SYMLINK", "external path contains a symlink")
        is_last = index == len(parts) - 1
        if not is_last and not stat.S_ISDIR(entry_stat.st_mode):
            raise _ArtifactFileError(
                "NOT_REGULAR", "an intermediate external path is not a directory"
            )
        if is_last and not stat.S_ISREG(entry_stat.st_mode):
            raise _ArtifactFileError("NOT_REGULAR", "external input is not an ordinary file")
        components.append((current, entry_stat))

    if not components:
        raise _ArtifactFileError("NOT_REGULAR", "filesystem root is not a file")
    flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0)
    try:
        descriptor = os.open(absolute, flags)
    except OSError as exc:
        raise _ArtifactFileError(
            "READ_ERROR", f"external file cannot be opened safely: {exc}"
        ) from exc
    try:
        opened_stat = os.fstat(descriptor)
        if not stat.S_ISREG(opened_stat.st_mode):
            raise _ArtifactFileError("NOT_REGULAR", "external input is not an ordinary file")
        if not _same_filesystem_entry(components[-1][1], opened_stat):
            raise _ArtifactFileError("CHANGED", "external file changed before reading")
        chunks: list[bytes] = []
        while True:
            chunk = os.read(descriptor, 1024 * 1024)
            if not chunk:
                break
            chunks.append(chunk)
        final_stat = os.fstat(descriptor)
    except OSError as exc:
        raise _ArtifactFileError(
            "READ_ERROR", f"external file cannot be read: {exc}"
        ) from exc
    finally:
        os.close(descriptor)
    if not _same_filesystem_entry(opened_stat, final_stat):
        raise _ArtifactFileError("CHANGED", "external file changed while being read")
    content = b"".join(chunks)
    if len(content) != final_stat.st_size:
        raise _ArtifactFileError("CHANGED", "external file length changed while reading")
    for component, original_stat in components:
        try:
            current_stat = component.lstat()
        except OSError as exc:
            raise _ArtifactFileError(
                "CHANGED", f"external path changed after reading: {exc}"
            ) from exc
        if stat.S_ISLNK(current_stat.st_mode) or not _same_filesystem_entry(
            original_stat, current_stat
        ):
            raise _ArtifactFileError("CHANGED", "external path changed while being read")
    return absolute, content


def _anchor_repository_path(value: Any) -> str | None:
    if not isinstance(value, str):
        return None
    without_heading = value.rsplit("#", 1)[0]
    repository_path, separator, _line_value = without_heading.rpartition(":")
    if not separator:
        return None
    return _canonical_repository_path(repository_path)


def snapshot_validation_inputs(
    distillation_snapshot_root: Path,
    sources_manifest: Path | str | None,
) -> Path | None:
    """Freeze the explicit manifest and every Markdown locator file it can read."""
    if sources_manifest is None:
        return None
    try:
        manifest_path, manifest_bytes = _read_stable_external_file(sources_manifest)
    except _ArtifactFileError as exc:
        raise DistillationInputError(
            f"SOURCES_MANIFEST_{exc.code}", Path(sources_manifest), exc.message
        ) from exc

    input_root = Path(
        tempfile.mkdtemp(prefix=".validation-inputs-", dir=distillation_snapshot_root)
    )
    project_root = input_root / "project"
    snapshot_manifest = project_root / "manifests" / "sources.yml"
    snapshot_manifest.parent.mkdir(parents=True)
    snapshot_manifest.write_bytes(manifest_bytes)

    manifest_document = _load_yaml(snapshot_manifest)
    evidence_document = _load_yaml(distillation_snapshot_root / "evidence-ledger.yml")
    sources = {
        record.get("id"): record
        for record in _records(manifest_document, "sources")
        if _is_nonempty_string(record.get("id"))
    }
    original_project_root = (
        manifest_path.parent.parent
        if manifest_path.parent.name == "manifests"
        else manifest_path.parent
    )
    paths_to_freeze: set[str] = set()
    for evidence in _records(evidence_document, "evidence"):
        locator = evidence.get("locator")
        if not isinstance(locator, dict) or locator.get("locator_type") != "markdown-section":
            continue
        source_id = evidence.get("source_id")
        source = sources.get(source_id) if _is_nonempty_string(source_id) else None
        if source is None:
            continue
        repository_path = _anchor_repository_path(locator.get("anchor"))
        if repository_path is not None and repository_path in _allowed_source_paths(source):
            paths_to_freeze.add(repository_path)

    for repository_path in sorted(paths_to_freeze):
        source_path = original_project_root.joinpath(
            *PurePosixPath(repository_path).parts
        )
        try:
            _absolute_source, source_bytes = _read_stable_external_file(source_path)
        except _ArtifactFileError as exc:
            if exc.code == "MISSING":
                continue
            raise DistillationInputError(
                f"LOCATOR_SOURCE_{exc.code}", source_path, exc.message
            ) from exc
        destination = project_root.joinpath(*PurePosixPath(repository_path).parts)
        destination.parent.mkdir(parents=True, exist_ok=True)
        destination.write_bytes(source_bytes)
    return snapshot_manifest


def _normalized_full_sha256(value: Any) -> str | None:
    if not isinstance(value, str):
        return None
    match = FULL_SHA256_RE.fullmatch(value)
    return None if match is None else match.group("digest").lower()


def _load_json_object_strict(content: bytes) -> dict[str, Any]:
    try:
        text = content.decode("utf-8")
    except UnicodeDecodeError as exc:
        raise ValueError(f"JSON is not UTF-8: {exc}") from exc

    def unique_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
        result: dict[str, Any] = {}
        for key, value in pairs:
            if key in result:
                raise ValueError(f"duplicate JSON object key {key!r}")
            result[key] = value
        return result

    def reject_non_json_constant(value: str) -> None:
        raise ValueError(f"non-JSON numeric constant {value!r}")

    try:
        value = json.loads(
            text,
            object_pairs_hook=unique_object,
            parse_constant=reject_non_json_constant,
        )
    except (json.JSONDecodeError, ValueError, RecursionError) as exc:
        raise ValueError(f"invalid JSON: {exc}") from exc
    if not isinstance(value, dict):
        raise ValueError("JSON root must be an object")
    return value


def _string_items(value: Any) -> list[str]:
    """Return only string members of a list, without ever hashing bad values."""
    if not isinstance(value, list):
        return []
    return [item for item in value if isinstance(item, str)]


def _string_set(value: Any) -> set[str]:
    return set(_string_items(value))


def _is_string_list(value: Any, *, nonempty: bool = False) -> bool:
    return (
        isinstance(value, list)
        and (not nonempty or bool(value))
        and all(_is_nonempty_string(item) for item in value)
    )


def _safe_member(value: Any, container: Any) -> bool:
    """Membership test that treats unhashable malformed values as non-members."""
    try:
        return value in container
    except (TypeError, ValueError):
        return False


def _candidate_file_bytes(
    report: ValidationReport,
    root: Path,
    relative_path: str,
    report_path: str,
    *,
    kind: str,
) -> bytes | None:
    try:
        _, content = _read_distillation_artifact(root, relative_path)
    except _ArtifactFileError as exc:
        code = (
            f"MATERIALIZATION_{kind}_MISSING"
            if exc.code == "MISSING"
            else f"MATERIALIZATION_{kind}_INVALID"
        )
        report.add_error(code, report_path, f"{exc.code}: {exc.message}")
        return None
    return content


def _validate_materialized_skill_file(
    report: ValidationReport,
    root: Path,
    candidate_path: str,
    candidate_name: Any,
    materialization_path: str,
) -> bool:
    relative_path = f"{candidate_path}/SKILL.md"
    report_path = f"{materialization_path}.candidate_path[{relative_path}]"
    content = _candidate_file_bytes(
        report,
        root,
        relative_path,
        report_path,
        kind="SKILL_FILE",
    )
    if content is None:
        return False
    try:
        text = content.decode("utf-8")
    except UnicodeDecodeError as exc:
        report.add_error(
            "MATERIALIZATION_SKILL_FILE_INVALID",
            report_path,
            f"SKILL.md must be UTF-8: {exc}",
        )
        return False

    lines = text.splitlines()
    if not lines or lines[0] != "---":
        report.add_error(
            "MATERIALIZATION_SKILL_FRONTMATTER_INVALID",
            report_path,
            "SKILL.md must start with an exact '---' frontmatter delimiter",
        )
        return False
    try:
        closing_index = lines.index("---", 1)
    except ValueError:
        report.add_error(
            "MATERIALIZATION_SKILL_FRONTMATTER_INVALID",
            report_path,
            "SKILL.md frontmatter must have an exact closing '---' delimiter",
        )
        return False
    if not any(line.strip() for line in lines[closing_index + 1 :]):
        report.add_error(
            "MATERIALIZATION_SKILL_BODY_INVALID",
            report_path,
            "SKILL.md must contain a non-empty body after frontmatter",
        )

    frontmatter_text = "\n".join(lines[1:closing_index])
    try:
        frontmatter = yaml.load(frontmatter_text, Loader=_UniqueKeySafeLoader)
    except _DuplicateYamlKeyError as exc:
        report.add_error(
            "MATERIALIZATION_SKILL_FRONTMATTER_INVALID",
            report_path,
            f"duplicate frontmatter key: {exc}",
        )
        return False
    except (yaml.YAMLError, RecursionError) as exc:
        report.add_error(
            "MATERIALIZATION_SKILL_FRONTMATTER_INVALID",
            report_path,
            f"frontmatter is not valid YAML: {exc}",
        )
        return False
    if not isinstance(frontmatter, dict):
        report.add_error(
            "MATERIALIZATION_SKILL_FRONTMATTER_INVALID",
            report_path,
            "frontmatter must be a mapping",
        )
        return False
    allowed_keys = {"name", "description"}
    if set(frontmatter) != allowed_keys:
        missing = sorted(allowed_keys - set(frontmatter))
        extra = sorted(str(key) for key in set(frontmatter) - allowed_keys)
        details: list[str] = []
        if missing:
            details.append(f"missing {missing!r}")
        if extra:
            details.append(f"extra {extra!r}")
        report.add_error(
            "MATERIALIZATION_SKILL_FRONTMATTER_INVALID",
            report_path,
            "frontmatter must contain only name and description"
            + (f" ({'; '.join(details)})" if details else ""),
        )
        return False
    name = frontmatter.get("name")
    description = frontmatter.get("description")
    valid = True
    if (
        not _is_nonempty_string(name)
        or name != name.strip()
        or not SKILL_NAME_RE.fullmatch(name)
        or len(name) > 64
    ):
        report.add_error(
            "MATERIALIZATION_SKILL_FRONTMATTER_INVALID",
            report_path,
            "frontmatter name must be a canonical hyphen-case Skill name",
        )
        valid = False
    if not _is_nonempty_string(description):
        report.add_error(
            "MATERIALIZATION_SKILL_FRONTMATTER_INVALID",
            report_path,
            "frontmatter description must be a non-empty string",
        )
        valid = False
    path_name = PurePosixPath(candidate_path).name
    if _is_nonempty_string(name) and name != path_name:
        report.add_error(
            "MATERIALIZATION_SKILL_NAME_MISMATCH",
            report_path,
            f"frontmatter name {name!r} must match candidate directory {path_name!r}",
        )
        valid = False
    if (
        _is_nonempty_string(name)
        and _is_nonempty_string(candidate_name)
        and name != candidate_name
    ):
        report.add_error(
            "MATERIALIZATION_SKILL_NAME_MISMATCH",
            report_path,
            f"frontmatter name {name!r} must match linked candidate name {candidate_name!r}",
        )
        valid = False
    return valid


def _canonical_json_sha256(value: Mapping[str, Any]) -> str:
    payload = json.dumps(
        value,
        ensure_ascii=False,
        sort_keys=True,
        separators=(",", ":"),
        allow_nan=False,
    ).encode("utf-8")
    return f"sha256:{hashlib.sha256(payload).hexdigest()}"


def _parse_eval_definition(
    report: ValidationReport,
    root: Path,
    candidate_path: str,
    candidate_name: str,
    materialization_path: str,
    *,
    filename: str,
    list_keys: Sequence[str],
) -> tuple[
    dict[str, frozenset[str]],
    dict[str, bool],
    dict[str, dict[str, EvalCaseContract]],
    EvalRubricContract | None,
    bool,
]:
    relative_path = f"{candidate_path}/evals/{filename}"
    report_path = f"{materialization_path}.candidate_path[{relative_path}]"
    content = _candidate_file_bytes(
        report,
        root,
        relative_path,
        report_path,
        kind="EVAL_DEFINITION",
    )
    empty_ids = {key: frozenset() for key in list_keys}
    empty_cases = {case_type: {} for case_type in EVAL_CASE_TYPES}
    if content is None:
        return empty_ids, {}, empty_cases, None, False
    try:
        definition = _load_json_object_strict(content)
    except ValueError as exc:
        report.add_error(
            "MATERIALIZATION_EVAL_DEFINITION_INVALID",
            report_path,
            str(exc),
        )
        return empty_ids, {}, empty_cases, None, False

    valid = True
    definition_schema = definition.get("schema_version")
    if (
        not isinstance(definition_schema, int)
        or isinstance(definition_schema, bool)
        or definition_schema not in {1, 2}
    ):
        report.add_error(
            "MATERIALIZATION_EVAL_DEFINITION_INVALID",
            f"{report_path}.schema_version",
            "schema_version must be integer 1 (legacy) or 2 (task contract)",
        )
        valid = False
    if definition.get("skill_name") != candidate_name:
        report.add_error(
            "MATERIALIZATION_EVAL_DEFINITION_INVALID",
            f"{report_path}.skill_name",
            f"skill_name must equal {candidate_name!r}",
        )
        valid = False

    case_type_by_key = {
        "should_trigger": "trigger",
        "should_not_trigger": "nontrigger",
        "tasks": "task",
    }
    case_ids: dict[str, frozenset[str]] = {}
    task_holdout: dict[str, bool] = {}
    cases: dict[str, dict[str, EvalCaseContract]] = {
        case_type: {} for case_type in EVAL_CASE_TYPES
    }
    for list_key in list_keys:
        records = definition.get(list_key)
        if not isinstance(records, list) or len(records) < 3:
            report.add_error(
                "MATERIALIZATION_EVAL_DEFINITION_INVALID",
                f"{report_path}.{list_key}",
                "must be a list containing at least three case definitions",
            )
            case_ids[list_key] = frozenset()
            valid = False
            continue
        seen: set[str] = set()
        case_type = case_type_by_key[list_key]
        for index, record in enumerate(records):
            case_path = f"{report_path}.{list_key}[{index}]"
            if not isinstance(record, dict):
                report.add_error(
                    "MATERIALIZATION_EVAL_DEFINITION_INVALID",
                    case_path,
                    "case definition must be an object",
                )
                valid = False
                continue
            _validate_no_placeholder_deep(report, record, case_path)
            case_id = record.get("case_id")
            if (
                not _is_nonempty_string(case_id)
                or case_id != case_id.strip()
                or not ID_RE.fullmatch(case_id)
            ):
                report.add_error(
                    "MATERIALIZATION_EVAL_DEFINITION_INVALID",
                    f"{case_path}.case_id",
                    "case_id must be a stable non-empty ID",
                )
                valid = False
                continue
            if case_id in seen:
                report.add_error(
                    "MATERIALIZATION_EVAL_DEFINITION_INVALID",
                    f"{case_path}.case_id",
                    f"duplicate case_id {case_id!r}",
                )
                valid = False
                continue
            seen.add(case_id)
            request_key = "request" if case_type == "task" else "prompt"
            request = record.get(request_key)
            if not _is_nonempty_string(request):
                report.add_error(
                    "MATERIALIZATION_EVAL_DEFINITION_INVALID",
                    f"{case_path}.{request_key}",
                    f"{request_key} must be a non-empty string",
                )
                valid = False
                continue
            holdout = False
            if case_type == "task":
                holdout_value = record.get("holdout")
                if not isinstance(holdout_value, bool):
                    report.add_error(
                        "MATERIALIZATION_EVAL_DEFINITION_INVALID",
                        f"{case_path}.holdout",
                        "task case holdout must be a boolean",
                    )
                    valid = False
                    continue
                holdout = holdout_value
                task_holdout[case_id] = holdout
                for field_name in ("title", "input_profile", "rubric_id"):
                    if not _is_nonempty_string(record.get(field_name)):
                        report.add_error(
                            "MATERIALIZATION_EVAL_DEFINITION_INVALID",
                            f"{case_path}.{field_name}",
                            f"task case {field_name} must be a non-empty string",
                        )
                        valid = False
                for field_name in ("expected_behaviors", "failure_signals"):
                    if not _is_string_list(record.get(field_name), nonempty=True):
                        report.add_error(
                            "MATERIALIZATION_EVAL_DEFINITION_INVALID",
                            f"{case_path}.{field_name}",
                            f"task case {field_name} must be a non-empty string list",
                        )
                        valid = False
            elif not _is_nonempty_string(record.get("expected_reason")):
                report.add_error(
                    "MATERIALIZATION_EVAL_DEFINITION_INVALID",
                    f"{case_path}.expected_reason",
                    "trigger boundary case requires a non-empty expected_reason",
                )
                valid = False
            if definition_schema == 2:
                for field_name in ("stable_task_ids", "input_type_ids"):
                    if not _is_string_list(record.get(field_name), nonempty=True):
                        report.add_error(
                            "MATERIALIZATION_EVAL_DEFINITION_INVALID",
                            f"{case_path}.{field_name}",
                            f"schema v2 case requires a unique non-empty {field_name} list",
                        )
                        valid = False
                for field_name, must_be_nonempty in (
                    (
                        "positive_example_ids",
                        case_type in {"trigger", "task"},
                    ),
                    (
                        "negative_example_ids",
                        case_type in {"nontrigger", "task"},
                    ),
                ):
                    values = record.get(field_name)
                    if (
                        not _is_string_list(values, nonempty=must_be_nonempty)
                        or len(_string_items(values)) != len(set(_string_items(values)))
                        or (
                            case_type == "trigger"
                            and field_name == "negative_example_ids"
                            and values != []
                        )
                        or (
                            case_type == "nontrigger"
                            and field_name == "positive_example_ids"
                            and values != []
                        )
                    ):
                        report.add_error(
                            "MATERIALIZATION_EVAL_DEFINITION_INVALID",
                            f"{case_path}.{field_name}",
                            "schema v2 requires explicit unique polarity-separated example IDs",
                        )
                        valid = False
            leakage_terms: list[str] = []
            if case_type == "task":
                for field_name in ("expected_behaviors", "failure_signals"):
                    values = record.get(field_name)
                    if isinstance(values, list):
                        leakage_terms.extend(
                            item for item in values if isinstance(item, str) and item.strip()
                        )
            else:
                reason = record.get("expected_reason")
                if isinstance(reason, str) and reason.strip():
                    leakage_terms.append(reason)
            cases[case_type][case_id] = EvalCaseContract(
                case_type=case_type,
                case_id=case_id,
                request=request,
                definition_hash=_canonical_json_sha256(record),
                holdout=holdout,
                prompt_leakage_terms=tuple(leakage_terms),
            )
        case_ids[list_key] = frozenset(seen)

    rubric: EvalRubricContract | None = None
    if "tasks" in list_keys:
        if not any(task_holdout.values()):
            report.add_error(
                "MATERIALIZATION_EVAL_DEFINITION_INVALID",
                f"{report_path}.tasks",
                "task definitions must include at least one holdout case",
            )
            valid = False
        protocol = definition.get("comparison_protocol")
        if not isinstance(protocol, dict):
            report.add_error(
                "MATERIALIZATION_EVAL_DEFINITION_INVALID",
                f"{report_path}.comparison_protocol",
                "task definitions require a comparison_protocol mapping",
            )
            valid = False
        else:
            if protocol.get("required") is not True:
                report.add_error(
                    "MATERIALIZATION_EVAL_DEFINITION_INVALID",
                    f"{report_path}.comparison_protocol.required",
                    "comparison protocol must be explicitly required",
                )
                valid = False
            for field_name in ("baseline", "with_skill", "leakage_control"):
                if not _is_nonempty_string(protocol.get(field_name)):
                    report.add_error(
                        "MATERIALIZATION_EVAL_DEFINITION_INVALID",
                        f"{report_path}.comparison_protocol.{field_name}",
                        f"{field_name} must be a non-empty string",
                    )
                    valid = False
            dimensions = protocol.get("human_review_dimensions")
            if definition_schema == 1 and (
                not _is_string_list(dimensions, nonempty=True)
                or len(dimensions) != len(set(_string_items(dimensions)))
            ):
                report.add_error(
                    "MATERIALIZATION_EVAL_DEFINITION_INVALID",
                    f"{report_path}.comparison_protocol.human_review_dimensions",
                    "human review dimensions must be a unique non-empty string list",
                )
                valid = False
                dimensions = []
            rubric_record = protocol.get("rubric")
            if not isinstance(rubric_record, dict):
                report.add_error(
                    "MATERIALIZATION_EVAL_DEFINITION_INVALID",
                    f"{report_path}.comparison_protocol.rubric",
                    "comparison protocol requires a structured rubric",
                )
                valid = False
            else:
                rubric_id = rubric_record.get("rubric_id")
                score_min = rubric_record.get("score_min")
                score_max = rubric_record.get("score_max_per_dimension")
                threshold = rubric_record.get("pass_threshold")
                fatal_failures = rubric_record.get("fatal_failures")
                if definition_schema == 2:
                    dimension_records = rubric_record.get("dimensions")
                    if not isinstance(dimension_records, list) or not dimension_records:
                        dimensions = []
                    else:
                        dimensions = [
                            item.get("dimension_id")
                            for item in dimension_records
                            if isinstance(item, dict)
                            and _is_nonempty_string(item.get("dimension_id"))
                            and _is_nonempty_string(item.get("description"))
                        ]
                        if len(dimensions) != len(dimension_records) or len(dimensions) != len(set(dimensions)):
                            dimensions = []
                    if not isinstance(fatal_failures, list):
                        fatal_failures = []
                    else:
                        fatal_failures = [
                            item.get("failure_id")
                            for item in fatal_failures
                            if isinstance(item, dict)
                            and _is_nonempty_string(item.get("failure_id"))
                            and _is_nonempty_string(item.get("description"))
                        ]
                rubric_shape_ok = True
                if not _is_nonempty_string(rubric_id) or not ID_RE.fullmatch(rubric_id):
                    rubric_shape_ok = False
                if not _is_number(score_min) or not _is_number(score_max) or score_max <= score_min:
                    rubric_shape_ok = False
                max_score = len(_string_items(dimensions)) * score_max if _is_number(score_max) else None
                if (
                    not _is_number(threshold)
                    or max_score is None
                    or not score_min <= threshold <= max_score
                ):
                    rubric_shape_ok = False
                if not _is_string_list(fatal_failures, nonempty=True) or not dimensions:
                    rubric_shape_ok = False
                if not rubric_shape_ok:
                    report.add_error(
                        "MATERIALIZATION_EVAL_DEFINITION_INVALID",
                        f"{report_path}.comparison_protocol.rubric",
                        "rubric requires ID, numeric score bounds/threshold, and fatal failures",
                    )
                    valid = False
                else:
                    rubric = EvalRubricContract(
                        rubric_id=rubric_id,
                        dimensions=tuple(dimensions),
                        score_min=float(score_min),
                        score_max_per_dimension=float(score_max),
                        pass_threshold=float(threshold),
                        fatal_failures=frozenset(fatal_failures),
                    )
                    for case_id, case in cases["task"].items():
                        raw_case = next(
                            item for item in definition["tasks"]
                            if isinstance(item, dict) and item.get("case_id") == case_id
                        )
                        if raw_case.get("rubric_id") != rubric.rubric_id:
                            report.add_error(
                                "MATERIALIZATION_EVAL_DEFINITION_INVALID",
                                f"{report_path}.tasks[{case_id}].rubric_id",
                                "task rubric_id must match comparison_protocol rubric",
                            )
                            valid = False

    return case_ids, task_holdout, cases, rubric, valid


def _validate_candidate_contract(
    report: ValidationReport,
    root: Path,
    candidate_path: str,
    candidate_name: Any,
    materialization_path: str,
) -> CandidateEvalContract:
    expected_name = (
        candidate_name
        if _is_nonempty_string(candidate_name)
        else PurePosixPath(candidate_path).name
    )
    _validate_materialized_skill_file(
        report, root, candidate_path, candidate_name, materialization_path
    )
    trigger_cases, _, trigger_contracts, _, trigger_valid = _parse_eval_definition(
        report,
        root,
        candidate_path,
        expected_name,
        materialization_path,
        filename="trigger-cases.json",
        list_keys=("should_trigger", "should_not_trigger"),
    )
    task_cases, task_holdout, task_contracts, rubric, task_valid = _parse_eval_definition(
        report,
        root,
        candidate_path,
        expected_name,
        materialization_path,
        filename="task-cases.json",
        list_keys=("tasks",),
    )
    case_ids = {
        "trigger": trigger_cases.get("should_trigger", frozenset()),
        "nontrigger": trigger_cases.get("should_not_trigger", frozenset()),
        "task": task_cases.get("tasks", frozenset()),
    }
    definitions_valid = trigger_valid and task_valid and rubric is not None
    case_types = tuple(case_ids)
    for left_index, left_type in enumerate(case_types):
        for right_type in case_types[left_index + 1 :]:
            overlap = case_ids[left_type] & case_ids[right_type]
            if overlap:
                report.add_error(
                    "MATERIALIZATION_EVAL_DEFINITION_INVALID",
                    f"{materialization_path}.candidate_path",
                    "case IDs must be unique across eval case types; overlap: "
                    + ", ".join(sorted(overlap)),
                )
                definitions_valid = False
    merged_cases = {case_type: {} for case_type in EVAL_CASE_TYPES}
    for source in (trigger_contracts, task_contracts):
        for case_type, records in source.items():
            merged_cases[case_type].update(records)
    return CandidateEvalContract(
        case_ids={key: frozenset(value) for key, value in case_ids.items()},
        task_holdout=dict(task_holdout),
        cases=merged_cases,
        rubric=rubric,
        definitions_valid=definitions_valid,
    )

def _is_nonempty_string(value: Any) -> bool:
    return isinstance(value, str) and bool(value.strip())


def _is_nonempty_value(value: Any) -> bool:
    """Return whether a string/container contains at least one usable value."""
    pending = [value]
    seen: set[int] = set()
    while pending:
        item = pending.pop()
        if isinstance(item, str):
            if item.strip():
                return True
            continue
        if not isinstance(item, (list, dict)):
            continue
        identity = id(item)
        if identity in seen:
            continue
        seen.add(identity)
        pending.extend(item if isinstance(item, list) else item.values())
    return False


def _is_number(value: Any) -> bool:
    return isinstance(value, (int, float)) and not isinstance(value, bool)


def _validate_no_placeholder(
    report: ValidationReport,
    value: Any,
    path: str,
    *,
    identifier: bool = False,
) -> None:
    if not isinstance(value, str):
        return
    if PLACEHOLDER_RE.search(value) or (identifier and EXAMPLE_IDENTIFIER_RE.search(value)):
        report.add_error(
            "PLACEHOLDER_VALUE", path, "replace template/example placeholder before validation"
        )


def _validate_no_placeholder_deep(
    report: ValidationReport,
    value: Any,
    path: str,
) -> None:
    """Reject template markers anywhere inside a strict JSON value."""
    pending: list[tuple[Any, str]] = [(value, path)]
    while pending:
        item, item_path = pending.pop()
        if isinstance(item, str):
            _validate_no_placeholder(report, item, item_path)
        elif isinstance(item, list):
            pending.extend(
                (child, f"{item_path}[{index}]")
                for index, child in enumerate(item)
            )
        elif isinstance(item, dict):
            for key, child in item.items():
                key_label = key if isinstance(key, str) else repr(key)
                if isinstance(key, str):
                    _validate_no_placeholder(report, key, f"{item_path}.<key>")
                pending.append((child, f"{item_path}.{key_label}"))


def _validate_content_hash(report: ValidationReport, value: Any, path: str) -> bool:
    if not isinstance(value, str) or not CONTENT_HASH_RE.fullmatch(value):
        report.add_error(
            "CONTENT_HASH_INVALID",
            path,
            "must be 12-64 hexadecimal characters, optionally prefixed with sha256:",
        )
        return False
    return True


def _require_fields(
    report: ValidationReport,
    record: Mapping[str, Any],
    fields: Iterable[str],
    path: str,
) -> None:
    for key in fields:
        if key not in record:
            report.add_error("MISSING_FIELD", f"{path}.{key}", "required field is missing")


def _require_list(
    report: ValidationReport,
    record: Mapping[str, Any],
    key: str,
    path: str,
    *,
    nonempty: bool = False,
) -> list[Any]:
    value = record.get(key)
    if not isinstance(value, list):
        report.add_error("FIELD_TYPE", f"{path}.{key}", "must be a list")
        return []
    if nonempty and not value:
        report.add_error("EMPTY_LIST", f"{path}.{key}", "must not be empty")
    return value


def _validate_string_list(
    report: ValidationReport,
    record: Mapping[str, Any],
    key: str,
    path: str,
    *,
    nonempty: bool = False,
    min_items: int = 0,
) -> list[str]:
    values = _require_list(report, record, key, path, nonempty=nonempty)
    result: list[str] = []
    for index, value in enumerate(values):
        if not _is_nonempty_string(value):
            report.add_error(
                "FIELD_TYPE", f"{path}.{key}[{index}]", "must be a non-empty string"
            )
        else:
            result.append(value)
    if len(values) < min_items:
        report.add_error(
            "MIN_ITEMS", f"{path}.{key}", f"must contain at least {min_items} items"
        )
    return result


def _validate_id(report: ValidationReport, value: Any, path: str) -> str | None:
    if not _is_nonempty_string(value):
        report.add_error("INVALID_ID", path, "ID must be a non-empty string")
        return None
    if value != value.strip():
        report.add_error("INVALID_ID", path, "ID must not contain leading or trailing whitespace")
        return None

#!/usr/bin/env python3
"""Resolve OOXML evidence locators against integrity-bound DOCX bundles.

The checker is read-only. It rejects self-asserted or drifted normalization
bundles, binds their recorded source checksum to the explicit source manifest,
and then verifies each OOXML locator and inline excerpt. It does not judge
whether a source statement is true.
"""

from __future__ import annotations

import argparse
import hashlib
import json
import os
import re
import stat
import sys
from dataclasses import dataclass, field
from pathlib import Path, PurePosixPath
from typing import Any, Mapping, Sequence

try:
    import yaml
except ImportError as exc:  # pragma: no cover
    raise SystemExit(
        "PyYAML is required. Do not install it without the user's approval."
    ) from exc


FULL_SHA256_RE = re.compile(r"^(?:sha256:)?(?P<digest>[0-9a-fA-F]{64})$")
REQUIRED_GENERATED_FILES = {
    "blocks.jsonl",
    "structure.yml",
    "media-map.yml",
    "normalization-log.yml",
}


class _UniqueKeySafeLoader(yaml.SafeLoader):
    """Safe YAML loader that rejects duplicate mapping keys."""


def _construct_unique_mapping(loader, node, deep=False):
    loader.flatten_mapping(node)
    result = {}
    for key_node, value_node in node.value:
        key = loader.construct_object(key_node, deep=deep)
        try:
            duplicate = key in result
        except TypeError as exc:
            raise yaml.constructor.ConstructorError(
                "while constructing a mapping",
                node.start_mark,
                "found an unhashable mapping key",
                key_node.start_mark,
            ) from exc
        if duplicate:
            raise yaml.constructor.ConstructorError(
                "while constructing a mapping",
                node.start_mark,
                f"duplicate mapping key {key!r}",
                key_node.start_mark,
            )
        result[key] = loader.construct_object(value_node, deep=deep)
    return result


_UniqueKeySafeLoader.add_constructor(
    yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
    _construct_unique_mapping,
)


@dataclass(frozen=True)
class Issue:
    code: str
    path: str
    message: str

    def as_dict(self) -> dict[str, str]:
        return {"code": self.code, "path": self.path, "message": self.message}


@dataclass
class Report:
    distillation_dir: str
    normalized_root: str
    sources_manifest: str
    checked: int = 0
    skipped: int = 0
    integrity_verified_sources: int = 0
    manifest_checksum_verified_sources: int = 0
    errors: list[Issue] = field(default_factory=list)

    @property
    def ok(self) -> bool:
        return not self.errors

    def error(self, code: str, path: str, message: str) -> None:
        self.errors.append(Issue(code, path, message))

    def as_dict(self) -> dict[str, Any]:
        return {
            "checker_scope": "integrity-bound-normalized-locator-resolution",
            "truth_assessed": False,
            "ok": self.ok,
            "distillation_dir": self.distillation_dir,
            "normalized_root": self.normalized_root,
            "sources_manifest": self.sources_manifest,
            "checked_ooxml_evidence": self.checked,
            "skipped_non_ooxml_evidence": self.skipped,
            "integrity_verified_sources": self.integrity_verified_sources,
            "manifest_checksum_verified_sources": self.manifest_checksum_verified_sources,
            "errors": [item.as_dict() for item in self.errors],
        }


def _same_entry(before: os.stat_result, after: os.stat_result) -> bool:
    same_identity = (
        before.st_dev == after.st_dev
        and before.st_ino == after.st_ino
        and stat.S_IFMT(before.st_mode) == stat.S_IFMT(after.st_mode)
    )
    if not same_identity:
        return False
    if stat.S_ISDIR(before.st_mode):
        return True
    return before.st_size == after.st_size and before.st_mtime_ns == after.st_mtime_ns


def _read_stable_regular_file(path: Path) -> bytes:
    """Read one ordinary file without following symlinks or accepting drift."""
    absolute = Path(os.path.abspath(path))
    components: list[tuple[Path, os.stat_result]] = []
    current = Path(absolute.anchor)
    for part in absolute.parts[1:]:
        current = current / part
        try:
            entry_stat = current.lstat()
        except OSError as exc:
            raise RuntimeError(f"cannot inspect {current}: {exc}") from exc
        if stat.S_ISLNK(entry_stat.st_mode):
            raise RuntimeError(f"symlink path component is forbidden: {current}")
        components.append((current, entry_stat))
    if not components or not stat.S_ISREG(components[-1][1].st_mode):
        raise RuntimeError(f"path is not an ordinary file: {absolute}")

    flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0)
    try:
        descriptor = os.open(absolute, flags)
    except OSError as exc:
        raise RuntimeError(f"cannot open ordinary file safely: {absolute}: {exc}") from exc
    try:
        opened_stat = os.fstat(descriptor)
        if not stat.S_ISREG(opened_stat.st_mode) or not _same_entry(
            components[-1][1], opened_stat
        ):
            raise RuntimeError(f"file changed before reading: {absolute}")
        chunks = []
        while True:
            chunk = os.read(descriptor, 1024 * 1024)
            if not chunk:
                break
            chunks.append(chunk)
        final_stat = os.fstat(descriptor)
    finally:
        os.close(descriptor)
    content = b"".join(chunks)
    if not _same_entry(opened_stat, final_stat) or len(content) != final_stat.st_size:
        raise RuntimeError(f"file changed while reading: {absolute}")
    for component, original_stat in components:
        try:
            current_stat = component.lstat()
        except OSError as exc:
            raise RuntimeError(f"path changed after reading: {component}: {exc}") from exc
        if stat.S_ISLNK(current_stat.st_mode) or not _same_entry(
            original_stat, current_stat
        ):
            raise RuntimeError(f"path changed while reading: {component}")
    return content


def _load_yaml(path: Path) -> Mapping[str, Any]:
    try:
        content = _read_stable_regular_file(path)
        data = yaml.load(content.decode("utf-8"), Loader=_UniqueKeySafeLoader)
    except (UnicodeError, yaml.YAMLError, RecursionError, RuntimeError) as exc:
        raise RuntimeError(f"cannot load YAML {path}: {exc}") from exc
    if not isinstance(data, dict):
        raise RuntimeError(f"YAML root must be a mapping: {path}")
    return data


def _load_json_object_strict(line: str, path: Path, line_number: int) -> Mapping[str, Any]:
    def unique_object(pairs):
        result = {}
        for key, value in pairs:
            if key in result:
                raise ValueError(f"duplicate object key {key!r}")
            result[key] = value
        return result

    def reject_constant(value):
        raise ValueError(f"non-JSON numeric constant {value!r}")

    try:
        record = json.loads(
            line, object_pairs_hook=unique_object, parse_constant=reject_constant
        )
    except (json.JSONDecodeError, ValueError, RecursionError) as exc:
        raise RuntimeError(f"invalid JSON at {path}:{line_number}: {exc}") from exc
    if not isinstance(record, dict):
        raise RuntimeError(f"block must be a mapping at {path}:{line_number}")
    return record


def _normalized_sha256(value: Any) -> str | None:
    if not isinstance(value, str):
        return None
    match = FULL_SHA256_RE.fullmatch(value)
    return None if match is None else match.group("digest").lower()


def _canonical_bundle_path(value: Any) -> str:
    if not isinstance(value, str) or not value or value != value.strip():
        raise RuntimeError("bundle path must be a non-empty canonical string")
    if "\\" in value or "\x00" in value:
        raise RuntimeError("bundle path must use POSIX separators and contain no NUL")
    parts = value.split("/")
    if any(part in {"", ".", ".."} for part in parts):
        raise RuntimeError("bundle path contains an empty, '.' or '..' component")
    path = PurePosixPath(value)
    if path.is_absolute() or path.as_posix() != value:
        raise RuntimeError("bundle path must be canonical and relative")
    return value


def _load_sources_manifest(path: Path) -> dict[str, Mapping[str, Any]]:
    document = _load_yaml(path)
    records = document.get("sources")
    if not isinstance(records, list):
        raise RuntimeError("sources manifest must contain a sources list")
    result = {}
    for index, record in enumerate(records):
        if not isinstance(record, dict):
            raise RuntimeError(f"sources[{index}] must be a mapping")
        source_id = record.get("id")
        if not isinstance(source_id, str) or not source_id.strip():
            raise RuntimeError(f"sources[{index}].id must be a non-empty string")
        if source_id in result:
            raise RuntimeError(f"duplicate source ID in manifest: {source_id!r}")
        result[source_id] = record
    return result


def _bundle_dir(normalized_root: Path, source_id: str) -> Path:
    if (
        not source_id
        or source_id != source_id.strip()
        or "\\" in source_id
        or "/" in source_id
        or source_id in {".", ".."}
    ):
        raise RuntimeError(f"unsafe source_id for normalized bundle lookup: {source_id!r}")
    direct = normalized_root / source_id
    if direct.is_dir():
        return direct
    if normalized_root.name == source_id and normalized_root.is_dir():
        return normalized_root
    return direct


def _load_blocks(bundle: Path, source_id: str) -> dict[int, Mapping[str, Any]]:
    blocks_path = bundle / "blocks.jsonl"
    try:
        content = _read_stable_regular_file(blocks_path).decode("utf-8")
    except (UnicodeError, RuntimeError) as exc:
        raise RuntimeError(f"cannot read blocks.jsonl for {source_id}: {exc}") from exc
    blocks = {}
    for line_number, line in enumerate(content.splitlines(), 1):
        if not line.strip():
            continue
        record = _load_json_object_strict(line, blocks_path, line_number)
        if record.get("source_id") != source_id:
            raise RuntimeError(
                f"source_id mismatch at {blocks_path}:{line_number}: {record.get('source_id')!r}"
            )
        index = record.get("ooxml_block_index")
        if not isinstance(index, int) or isinstance(index, bool) or index < 1:
            raise RuntimeError(f"invalid block index at {blocks_path}:{line_number}")
        if index in blocks:
            raise RuntimeError(f"duplicate block index {index} in {blocks_path}")
        normalized_text = record.get("normalized_text")
        if not isinstance(normalized_text, str):
            raise RuntimeError(f"normalized_text must be a string at {blocks_path}:{line_number}")
        full_hash = hashlib.sha256(normalized_text.encode("utf-8")).hexdigest()
        short_hash = full_hash[:12]
        locator = record.get("locator")
        if (
            record.get("text_sha256") != full_hash
            or record.get("short_content_hash") != short_hash
            or not isinstance(locator, dict)
            or locator.get("content_hash") != short_hash
        ):
            raise RuntimeError(
                f"block content hashes do not match normalized_text at {blocks_path}:{line_number}"
            )
        blocks[index] = record
    return blocks


def _verify_bundle_integrity(report, bundle, source_id, manifest_source):
    checksums_path = bundle / "checksums.yml"
    try:
        checksums = _load_yaml(checksums_path)
    except Exception as exc:
        report.error("BUNDLE_CHECKSUMS", str(checksums_path), str(exc))
        return False
    if checksums.get("source_id") != source_id:
        report.error(
            "BUNDLE_SOURCE_MISMATCH",
            str(checksums_path),
            "checksums source_id does not match evidence source_id",
        )
        return False

    source = checksums.get("source")
    before = _normalized_sha256(source.get("sha256_before")) if isinstance(source, dict) else None
    after = _normalized_sha256(source.get("sha256_after")) if isinstance(source, dict) else None
    manifest_checksum = _normalized_sha256(manifest_source.get("checksum"))
    integrity_ok = True
    if (
        not isinstance(source, dict)
        or source.get("sha256_unchanged") is not True
        or before is None
        or after is None
        or before != after
    ):
        report.error(
            "SOURCE_CHECKSUM_CHANGED",
            str(checksums_path),
            "normalization report must contain equal complete before/after source SHA-256 values",
        )
        integrity_ok = False
    if manifest_checksum is None:
        report.error(
            "MANIFEST_SOURCE_CHECKSUM_MISSING",
            str(checksums_path),
            "manifest source checksum must be a complete SHA-256",
        )
        integrity_ok = False
    elif before is not None and manifest_checksum != before:
        report.error(
            "MANIFEST_SOURCE_CHECKSUM_MISMATCH",
            str(checksums_path),
            "normalization source checksum does not match the explicit source manifest",
        )
        integrity_ok = False
    else:
        report.manifest_checksum_verified_sources += 1

    generated_files = checksums.get("generated_files")
    if not isinstance(generated_files, list):
        report.error(
            "BUNDLE_GENERATED_FILES_INVALID",
            str(checksums_path),
            "checksums.yml must contain a generated_files list",
        )
        return False
    recorded_paths = set()
    for index, record in enumerate(generated_files):
        entry_path = f"{checksums_path}.generated_files[{index}]"
        if not isinstance(record, dict):
            report.error("BUNDLE_GENERATED_FILE_INVALID", entry_path, "entry must be a mapping")
            integrity_ok = False
            continue
        try:
            relative = _canonical_bundle_path(record.get("path"))
        except RuntimeError as exc:
            report.error("BUNDLE_GENERATED_FILE_INVALID", entry_path, str(exc))
            integrity_ok = False
            continue
        if relative in recorded_paths:
            report.error("BUNDLE_GENERATED_FILE_DUPLICATE", entry_path, f"duplicate path {relative!r}")
            integrity_ok = False
            continue
        recorded_paths.add(relative)
        expected = _normalized_sha256(record.get("sha256"))
        if expected is None:
            report.error("BUNDLE_GENERATED_FILE_HASH_INVALID", entry_path, "sha256 must be complete")
            integrity_ok = False
            continue
        try:
            content = _read_stable_regular_file(bundle.joinpath(*PurePosixPath(relative).parts))
        except RuntimeError as exc:
            report.error("BUNDLE_GENERATED_FILE_READ", entry_path, str(exc))
            integrity_ok = False
            continue
        actual = hashlib.sha256(content).hexdigest()
        if actual != expected or record.get("byte_size") != len(content):
            report.error(
                "BUNDLE_GENERATED_FILE_MISMATCH",
                entry_path,
                "recorded byte_size/SHA-256 does not match the current bundle file",
            )
            integrity_ok = False
    missing = sorted(REQUIRED_GENERATED_FILES - recorded_paths)
    if missing:
        report.error(
            "BUNDLE_GENERATED_FILES_INCOMPLETE",
            str(checksums_path),
            "missing required generated-file hashes: " + ", ".join(missing),
        )
        integrity_ok = False
    if integrity_ok:
        report.integrity_verified_sources += 1
    return integrity_ok


def verify(distillation_dir, normalized_root, sources_manifest):
    distillation_dir = Path(distillation_dir)
    normalized_root = Path(normalized_root)
    sources_manifest = Path(sources_manifest)
    report = Report(
        str(distillation_dir.resolve()),
        str(normalized_root.resolve()),
        str(sources_manifest.resolve()),
    )
    ledger = _load_yaml(distillation_dir / "evidence-ledger.yml")
    manifest_sources = _load_sources_manifest(sources_manifest)
    evidence = ledger.get("evidence")
    if not isinstance(evidence, list):
        raise RuntimeError("evidence-ledger.yml must contain an evidence list")

    blocks_by_source = {}
    failed_sources = set()
    for evidence_index, item in enumerate(evidence):
        path = f"evidence-ledger.yml.evidence[{evidence_index}]"
        if not isinstance(item, dict):
            report.error("EVIDENCE_TYPE", path, "evidence must be a mapping")
            continue
        locator = item.get("locator")
        if not isinstance(locator, dict) or locator.get("locator_type") != "ooxml-block":
            report.skipped += 1
            continue
        report.checked += 1
        source_id = item.get("source_id")
        if not isinstance(source_id, str) or not source_id.strip():
            report.error("SOURCE_ID", f"{path}.source_id", "must be a non-empty string")
            continue
        manifest_source = manifest_sources.get(source_id)
        if manifest_source is None:
            report.error("UNKNOWN_SOURCE_ID", f"{path}.source_id", "source is absent from manifest")
            failed_sources.add(source_id)
            continue
        if source_id not in blocks_by_source and source_id not in failed_sources:
            try:
                bundle = _bundle_dir(normalized_root, source_id)
                if not _verify_bundle_integrity(report, bundle, source_id, manifest_source):
                    failed_sources.add(source_id)
                    continue
                blocks_by_source[source_id] = _load_blocks(bundle, source_id)
            except Exception as exc:
                failed_sources.add(source_id)
                report.error("BUNDLE_READ", str(normalized_root / source_id), str(exc))
        if source_id in failed_sources:
            continue
        block_index = locator.get("ooxml_block_index")
        block = blocks_by_source[source_id].get(block_index)
        if block is None:
            report.error(
                "LOCATOR_NOT_FOUND",
                f"{path}.locator.ooxml_block_index",
                f"block {block_index!r} is absent from the normalized bundle",
            )
            continue
        expected_hash = block.get("short_content_hash")
        if str(locator.get("content_hash", "")).lower() != str(expected_hash).lower():
            report.error(
                "CONTENT_HASH_MISMATCH",
                f"{path}.locator.content_hash",
                f"recorded {locator.get('content_hash')!r}, bundle has {expected_hash!r}",
            )
        detected_heading = locator.get("detected_heading_path", locator.get("heading_path"))
        if detected_heading != block.get("heading_path"):
            report.error(
                "HEADING_PATH_MISMATCH",
                f"{path}.locator.heading_path",
                "detected heading path does not match the normalized block",
            )
        for field_name in ("raw_text", "normalized_text"):
            excerpt = item.get(field_name)
            block_text = block.get(field_name)
            if isinstance(excerpt, str) and excerpt and (
                not isinstance(block_text, str) or excerpt not in block_text
            ):
                report.error(
                    "EVIDENCE_TEXT_MISMATCH",
                    f"{path}.{field_name}",
                    f"inline {field_name} is not present in the resolved block",
                )
        figure_ids = []
        if isinstance(locator.get("figure_id"), str):
            figure_ids.append(locator["figure_id"])
        if isinstance(locator.get("figure_ids"), list):
            figure_ids.extend(value for value in locator["figure_ids"] if isinstance(value, str))
        block_figures = set(block.get("figure_ids", []))
        for figure_id in figure_ids:
            if figure_id not in block_figures:
                report.error(
                    "FIGURE_NOT_IN_BLOCK",
                    f"{path}.locator",
                    f"figure {figure_id!r} is not associated with the resolved block",
                )
    return report


def main(argv: Sequence[str] | None = None) -> int:
    parser = argparse.ArgumentParser(
        description="Resolve OOXML evidence locators against integrity-bound DOCX bundles."
    )
    parser.add_argument("distillation_dir")
    parser.add_argument("normalized_root")
    parser.add_argument(
        "--sources-manifest",
        required=True,
        help="Explicit source manifest whose checksum must match each normalized bundle",
    )
    args = parser.parse_args(argv)
    try:
        report = verify(args.distillation_dir, args.normalized_root, args.sources_manifest)
    except Exception as exc:
        print(json.dumps({
            "checker_scope": "integrity-bound-normalized-locator-resolution",
            "truth_assessed": False,
            "ok": False,
            "input_error": str(exc),
        }, ensure_ascii=False, indent=2))
        return 2
    print(json.dumps(report.as_dict(), ensure_ascii=False, indent=2, sort_keys=True))
    return 0 if report.ok else 1


if __name__ == "__main__":
    sys.exit(main())

#!/usr/bin/env python3
"""Read-only integrity and locator verification for private OCR bundles."""

from __future__ import annotations

import argparse
import hashlib
import json
import os
import re
import shutil
import stat
import subprocess
import sys
import unicodedata
from dataclasses import dataclass, field
from pathlib import Path, PurePosixPath
from typing import Any, Mapping, Sequence

try:
    import yaml
except ImportError as exc:  # pragma: no cover
    raise SystemExit("PyYAML is required; do not install it without approval.") from exc


@dataclass(frozen=True)
class Issue:
    code: str
    path: str
    message: str

    def as_dict(self) -> dict[str, str]:
        return {"code": self.code, "path": self.path, "message": self.message}


@dataclass
class Report:
    checked_ocr_evidence: int = 0
    verified_sources: int = 0
    errors: list[Issue] = field(default_factory=list)

    @property
    def ok(self) -> bool:
        return not self.errors

    def error(self, code: str, path: str, message: str) -> None:
        self.errors.append(Issue(code, path, message))

    def as_dict(self) -> dict[str, Any]:
        return {
            "checker_scope": "integrity-bound-ocr-locator-resolution",
            "truth_assessed": False,
            "ocr_correctness_assessed": False,
            "ok": self.ok,
            "checked_ocr_evidence": self.checked_ocr_evidence,
            "verified_sources": self.verified_sources,
            "errors": [item.as_dict() for item in self.errors],
        }


def _same_entry(before: os.stat_result, after: os.stat_result) -> bool:
    same_identity = (
        before.st_dev == after.st_dev
        and before.st_ino == after.st_ino
        and stat.S_IFMT(before.st_mode) == stat.S_IFMT(after.st_mode)
    )
    if not same_identity:
        return False
    if stat.S_ISDIR(before.st_mode):
        # Ancestor directory metadata can change when unrelated siblings are
        # created; inode and type preserve the no-follow path identity.
        return True
    return before.st_size == after.st_size and before.st_mtime_ns == after.st_mtime_ns


def _safe_file(path: Path) -> bytes:
    absolute = Path(os.path.abspath(path))
    current = Path(absolute.anchor)
    before: list[tuple[Path, os.stat_result]] = []
    for part in absolute.parts[1:]:
        current /= part
        item = current.lstat()
        if stat.S_ISLNK(item.st_mode):
            raise RuntimeError(f"symlink is forbidden: {current}")
        before.append((current, item))
    if not before or not stat.S_ISREG(before[-1][1].st_mode):
        raise RuntimeError(f"not an ordinary file: {absolute}")
    flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0)
    fd = os.open(absolute, flags)
    try:
        opened = os.fstat(fd)
        chunks = []
        while True:
            chunk = os.read(fd, 1024 * 1024)
            if not chunk:
                break
            chunks.append(chunk)
        final = os.fstat(fd)
    finally:
        os.close(fd)
    if not _same_entry(before[-1][1], opened) or not _same_entry(opened, final):
        raise RuntimeError(f"file changed while reading: {absolute}")
    payload = b"".join(chunks)
    if len(payload) != final.st_size:
        raise RuntimeError(f"file length changed: {absolute}")
    for component, original in before:
        now = component.lstat()
        if stat.S_ISLNK(now.st_mode) or not _same_entry(original, now):
            raise RuntimeError(f"path changed while reading: {component}")
    return payload


def _load_yaml(path: Path) -> Mapping[str, Any]:
    value = yaml.safe_load(_safe_file(path).decode("utf-8"))
    if not isinstance(value, dict):
        raise RuntimeError("YAML root must be a mapping")
    return value


def _load_results(path: Path) -> list[Mapping[str, Any]]:
    results = []
    for number, line in enumerate(_safe_file(path).decode("utf-8").splitlines(), 1):
        if not line.strip():
            continue
        value = json.loads(line)
        if not isinstance(value, dict):
            raise RuntimeError(f"result line {number} is not an object")
        results.append(value)
    return results


def _sha(payload: bytes) -> str:
    return "sha256:" + hashlib.sha256(payload).hexdigest()


def _normalize_text(value: str) -> str:
    return unicodedata.normalize(
        "NFC", value.replace("\r\n", "\n").replace("\r", "\n")
    )


def _canonical_relative(value: Any) -> str | None:
    if (
        not isinstance(value, str)
        or not value
        or value != value.strip()
        or "\\" in value
        or "\x00" in value
    ):
        return None
    path = PurePosixPath(value)
    if path.is_absolute() or path.as_posix() != value or any(
        part in {"", ".", ".."} for part in path.parts
    ):
        return None
    return value


def _project_root(manifest_path: Path) -> Path:
    return (
        manifest_path.parent.parent
        if manifest_path.parent.name == "manifests"
        else manifest_path.parent
    )


def _pdf_page_count(
    source: Path,
    *,
    runner=subprocess.run,
    which=shutil.which,
) -> int:
    if which("pdfinfo") is None:
        raise RuntimeError("PDF_RENDERER_UNAVAILABLE: pdfinfo is not installed")
    completed = runner(
        ["pdfinfo", str(source)],
        check=False,
        capture_output=True,
        text=True,
        shell=False,
    )
    if completed.returncode != 0:
        raise RuntimeError(
            "PDF_INPUT_INVALID: "
            + (completed.stderr.strip() or "pdfinfo failed")
        )
    match = re.search(r"(?m)^Pages:\s*([1-9][0-9]*)\s*$", completed.stdout)
    if match is None:
        raise RuntimeError("PDF_INPUT_INVALID: pdfinfo returned no positive page count")
    return int(match.group(1))


def _bundle_path(root: Path, source_id: str) -> Path:
    if not source_id or source_id != source_id.strip() or "/" in source_id or "\\" in source_id or source_id in {".", ".."}:
        raise RuntimeError(f"unsafe source ID {source_id!r}")
    direct = root / source_id
    return direct if direct.is_dir() else root


def verify(
    distillation_dir: Path | str,
    ocr_root: Path | str,
    sources_manifest: Path | str,
    normalized_root: Path | str | None = None,
    *,
    runner=subprocess.run,
    which=shutil.which,
) -> Report:
    """Recompute live source, bundle, coverage, image, text, region and locator bindings."""
    distillation = Path(distillation_dir)
    ocr_root = Path(ocr_root)
    manifest_path = Path(sources_manifest)
    normalized_root = Path(normalized_root) if normalized_root is not None else None
    report = Report()
    try:
        ledger = _load_yaml(distillation / "evidence-ledger.yml")
        manifest_document = _load_yaml(manifest_path)
    except Exception as exc:
        report.error("OCR_INPUT_INVALID", str(distillation), str(exc))
        return report

    raw_sources = manifest_document.get("sources")
    if not isinstance(raw_sources, list) or any(not isinstance(item, dict) for item in raw_sources):
        report.error("OCR_INPUT_INVALID", str(manifest_path), "sources must be a mapping list")
        return report
    sources: dict[str, Mapping[str, Any]] = {}
    for source in raw_sources:
        source_id = source.get("id")
        if not isinstance(source_id, str) or not source_id or source_id in sources:
            report.error("OCR_INPUT_INVALID", str(manifest_path), "source IDs must be unique non-empty strings")
            return report
        sources[source_id] = source

    evidence = [
        item for item in ledger.get("evidence", [])
        if isinstance(item, dict)
        and isinstance(item.get("locator"), dict)
        and item["locator"].get("locator_type") == "ocr-region"
    ]
    by_source: dict[str, list[Mapping[str, Any]]] = {}
    for item in evidence:
        by_source.setdefault(str(item.get("source_id")), []).append(item)
    required_sources = {
        source_id for source_id, source in sources.items()
        if source.get("type") in {"book-docx", "book-pdf-scan"}
        and isinstance(source.get("ocr_policy"), dict)
        and source["ocr_policy"].get("required") is True
    }

    for source_id in sorted(set(by_source) | required_sources):
        source = sources.get(source_id)
        if source is None:
            report.error("OCR_SOURCE_UNKNOWN", source_id, "source is missing from manifest")
            continue
        source_errors_before = len(report.errors)
        bundle = _bundle_path(ocr_root, source_id)
        try:
            ocr_manifest = _load_yaml(bundle / "ocr-manifest.yml")
            checksums = _load_yaml(bundle / "checksums.yml")
            records = _load_results(bundle / "ocr-results.jsonl")
        except Exception as exc:
            report.error("OCR_BUNDLE_INVALID", str(bundle), str(exc))
            continue

        # The manifest path is project-relative; recompute the original bytes now.
        local_path = _canonical_relative(source.get("local_path"))
        source_path = None
        source_payload = None
        if local_path is not None:
            source_path = _project_root(manifest_path).joinpath(*PurePosixPath(local_path).parts)
            try:
                source_payload = _safe_file(source_path)
            except Exception as exc:
                report.error("OCR_SOURCE_READ_ERROR", str(source_path), str(exc))
        else:
            report.error("OCR_SOURCE_PATH_REQUIRED", source_id, "a canonical local_path is required")
        live_hash = _sha(source_payload) if source_payload is not None else None
        if (
            live_hash != source.get("checksum")
            or ocr_manifest.get("source_id") != source_id
            or ocr_manifest.get("source_sha256") != source.get("checksum")
            or checksums.get("source_id") != source_id
            or checksums.get("source_sha256") != source.get("checksum")
        ):
            report.error("OCR_SOURCE_CHECKSUM_MISMATCH", str(bundle), "live source and both bundle manifests must bind the same SHA-256")

        # Recompute every listed derivative and reject unlisted extra files.
        generated = checksums.get("generated_files")
        if not isinstance(generated, list):
            report.error("OCR_BUNDLE_INVALID", str(bundle / "checksums.yml"), "generated_files is required")
            continue
        recorded_paths: set[str] = set()
        for index, entry in enumerate(generated):
            relative = _canonical_relative(entry.get("path")) if isinstance(entry, dict) else None
            if relative is None or relative in recorded_paths:
                report.error("OCR_BUNDLE_INVALID", f"checksums.generated_files[{index}]", "invalid/duplicate generated path")
                continue
            recorded_paths.add(relative)
            try:
                payload = _safe_file(bundle.joinpath(*PurePosixPath(relative).parts))
            except Exception as exc:
                report.error("OCR_BUNDLE_INVALID", relative, str(exc))
                continue
            if _sha(payload) != entry.get("sha256") or len(payload) != entry.get("size"):
                report.error("OCR_BUNDLE_HASH_MISMATCH", relative, "generated file hash/size drift")
        actual_paths: set[str] = set()
        try:
            for path in bundle.rglob("*"):
                if path.is_symlink():
                    raise RuntimeError(f"symlink is forbidden: {path}")
                if path.is_file() and path.name != "checksums.yml":
                    actual_paths.add(path.relative_to(bundle).as_posix())
        except Exception as exc:
            report.error("OCR_BUNDLE_INVALID", str(bundle), str(exc))
        if actual_paths != recorded_paths or not {"ocr-manifest.yml", "ocr-results.jsonl"}.issubset(recorded_paths):
            report.error("OCR_BUNDLE_INVALID", str(bundle / "checksums.yml"), "generated file inventory is incomplete or contains extras")

        # Recompute record/image/text/region integrity before resolving locators.
        record_map: dict[str, Mapping[str, Any]] = {}
        item_map: dict[str, Mapping[str, Any]] = {}
        statuses = {"completed": 0, "empty": 0, "failed": 0}
        for index, record in enumerate(records):
            record_id = record.get("ocr_record_id")
            item_id = record.get("item_id")
            if (
                not isinstance(record_id, str) or not isinstance(item_id, str)
                or record_id in record_map or item_id in item_map
            ):
                report.error("OCR_BUNDLE_INVALID", f"ocr-results.jsonl[{index}]", "record/item IDs must be unique strings")
                continue
            record_map[record_id] = record
            item_map[item_id] = record
            status_value = record.get("status")
            if status_value not in statuses:
                report.error("OCR_BUNDLE_INVALID", record_id, "invalid status")
                continue
            statuses[status_value] += 1
            raw_text = record.get("raw_text")
            normalized_text = record.get("normalized_text")
            if (
                not isinstance(raw_text, str) or not isinstance(normalized_text, str)
                or record.get("raw_text_sha256") != _sha(str(raw_text).encode("utf-8"))
                or normalized_text != _normalize_text(raw_text)
                or record.get("text_sha256") != _sha(str(normalized_text).encode("utf-8"))
            ):
                report.error("OCR_TEXT_HASH_MISMATCH", record_id, "raw/normalized text or hash drift")
            image_path = _canonical_relative(record.get("image_path"))
            if image_path is None:
                if status_value != "failed" or record.get("image_sha256") is not None:
                    report.error("OCR_IMAGE_HASH_MISMATCH", record_id, "non-failed record has no bound image")
            else:
                try:
                    image_payload = _safe_file(bundle.joinpath(*PurePosixPath(image_path).parts))
                except Exception as exc:
                    report.error("OCR_IMAGE_HASH_MISMATCH", record_id, str(exc))
                else:
                    if _sha(image_payload) != record.get("image_sha256"):
                        report.error("OCR_IMAGE_HASH_MISMATCH", record_id, "image/page hash drift")
            regions = record.get("regions")
            if not isinstance(regions, list):
                report.error("OCR_BUNDLE_INVALID", record_id, "regions must be a list")
                continue
            region_ids: set[str] = set()
            for region_index, region in enumerate(regions):
                bbox = region.get("bbox_px") if isinstance(region, dict) else None
                region_id = region.get("region_id") if isinstance(region, dict) else None
                text_value = region.get("text") if isinstance(region, dict) else None
                if (
                    not isinstance(region_id, str) or region_id in region_ids
                    or not isinstance(bbox, list) or len(bbox) != 4
                    or any(not isinstance(value, int) or isinstance(value, bool) for value in bbox)
                    or bbox[0] < 0 or bbox[1] < 0 or bbox[2] < 1 or bbox[3] < 1
                    or not isinstance(text_value, str)
                    or region.get("text_sha256") != _sha(str(text_value).encode("utf-8"))
                    or not isinstance(region.get("confidence_raw"), str)
                ):
                    report.error("OCR_REGION_HASH_MISMATCH", f"{record_id}.regions[{region_index}]", "invalid bbox/text/confidence/hash")
                else:
                    region_ids.add(region_id)
            if status_value == "empty" and (regions or str(normalized_text).strip()):
                report.error("OCR_BUNDLE_INVALID", record_id, "empty record contains OCR output")
            if status_value == "failed" and not isinstance(record.get("error_code"), str):
                report.error("OCR_BUNDLE_INVALID", record_id, "failed record lacks stable error_code")

        coverage = ocr_manifest.get("coverage")
        recomputed = {"discovered_items": len(records), "attempted_items": len(records), **statuses}
        if (
            not isinstance(coverage, dict)
            or any(coverage.get(key) != value for key, value in recomputed.items())
            or coverage.get("unbound_occurrence_ids") != []
            or coverage.get("complete") is not True
            or ocr_manifest.get("status") != "completed"
            or statuses["failed"] != 0
        ):
            report.error("OCR_COVERAGE_INCOMPLETE", str(bundle / "ocr-manifest.yml"), "all images/pages must be accounted for without failures")

        binding = ocr_manifest.get("input_binding")
        if not isinstance(binding, dict):
            report.error("OCR_BUNDLE_INVALID", str(bundle / "ocr-manifest.yml"), "input_binding is required")
            binding = {}
        carrier = ocr_manifest.get("carrier")
        raw_policy = source.get("ocr_policy")
        policy = raw_policy if isinstance(raw_policy, dict) else {}
        engine = ocr_manifest.get("engine")
        if (
            not isinstance(raw_policy, dict)
            or not isinstance(engine, dict)
            or policy.get("required") is not True
            or policy.get("execution_mode") != "local-only"
            or policy.get("engine") != "tesseract"
            or engine.get("name") != "tesseract"
            or engine.get("languages") != policy.get("languages")
        ):
            report.error("OCR_SOURCE_POLICY_MISMATCH", source_id, "source and OCR engine/language policy drift")
        if source.get("type") == "book-docx":
            if carrier != "docx-image" or policy.get("coverage") != "all-images":
                report.error("OCR_BUNDLE_INVALID", source_id, "DOCX source requires docx-image carrier")
            if normalized_root is None:
                report.error("OCR_NORMALIZED_BUNDLE_REQUIRED", source_id, "DOCX verification requires --normalized-root")
            else:
                normalized_bundle = _bundle_path(normalized_root, source_id)
                try:
                    media_bytes = _safe_file(normalized_bundle / "media-map.yml")
                    normalized_checksum_bytes = _safe_file(normalized_bundle / "checksums.yml")
                    media_map = yaml.safe_load(media_bytes.decode("utf-8"))
                    normalized_checksums = yaml.safe_load(normalized_checksum_bytes.decode("utf-8"))
                    if not isinstance(media_map, dict) or not isinstance(normalized_checksums, dict):
                        raise RuntimeError("normalized YAML roots must be mappings")
                except Exception as exc:
                    report.error("OCR_NORMALIZED_BUNDLE_INVALID", str(normalized_bundle), str(exc))
                else:
                    normalized_source = normalized_checksums.get("source")
                    before = str(normalized_source.get("sha256_before") or "") if isinstance(normalized_source, dict) else ""
                    before = before if before.startswith("sha256:") else "sha256:" + before
                    if (
                        before != source.get("checksum")
                        or not isinstance(normalized_source, dict)
                        or normalized_source.get("sha256_unchanged") is not True
                        or binding.get("media_map_sha256") != _sha(media_bytes)
                        or binding.get("normalization_checksums_sha256") != _sha(normalized_checksum_bytes)
                    ):
                        report.error("OCR_NORMALIZED_BUNDLE_HASH_MISMATCH", str(normalized_bundle), "normalization/source binding drift")
                    assets = media_map.get("assets")
                    occurrences = media_map.get("occurrences")
                    if (
                        not isinstance(assets, list) or any(not isinstance(item, dict) for item in assets)
                        or not isinstance(occurrences, list) or any(not isinstance(item, dict) for item in occurrences)
                    ):
                        report.error("OCR_NORMALIZED_BUNDLE_INVALID", str(normalized_bundle), "assets/occurrences must be mapping lists")
                    else:
                        by_asset: dict[str, list[Mapping[str, Any]]] = {}
                        for occurrence in occurrences:
                            by_asset.setdefault(str(occurrence.get("asset_id")), []).append(occurrence)
                        expected_assets: set[str] = set()
                        expected_media_paths: set[str] = set()
                        for asset in assets:
                            asset_id = asset.get("asset_id")
                            relative = _canonical_relative(asset.get("extracted_path"))
                            if not isinstance(asset_id, str) or not asset_id or asset_id in expected_assets or relative is None:
                                report.error("OCR_NORMALIZED_BUNDLE_INVALID", source_id, "invalid/duplicate asset identity")
                                continue
                            expected_assets.add(asset_id)
                            expected_media_paths.add(relative)
                            record = item_map.get(asset_id)
                            if record is None:
                                report.error("OCR_COVERAGE_INCOMPLETE", asset_id, "normalized asset has no OCR record")
                                continue
                            try:
                                media_payload = _safe_file(normalized_bundle.joinpath(*PurePosixPath(relative).parts))
                            except Exception as exc:
                                report.error("OCR_IMAGE_HASH_MISMATCH", asset_id, str(exc))
                                continue
                            context = record.get("context") if isinstance(record.get("context"), dict) else {}
                            expected_occurrences = [str(item.get("occurrence_id")) for item in by_asset.get(asset_id, [])]
                            expected_figures = [str(item.get("figure_id")) for item in by_asset.get(asset_id, [])]
                            if (
                                record.get("image_sha256") != _sha(media_payload)
                                or context.get("media_occurrence_ids") != expected_occurrences
                                or context.get("figure_ids") != expected_figures
                            ):
                                report.error("OCR_COVERAGE_INCOMPLETE", asset_id, "media/occurrence binding drift")
                        if set(item_map) != expected_assets or coverage.get("occurrence_count") != len(occurrences):
                            report.error("OCR_COVERAGE_INCOMPLETE", source_id, "OCR asset/occurrence set differs from normalized media map")
                        normalized_generated = normalized_checksums.get("generated_files")
                        generated_media_paths = {
                            str(entry.get("path"))
                            for entry in normalized_generated
                            if isinstance(entry, dict)
                            and isinstance(entry.get("path"), str)
                            and str(entry.get("path")).startswith("media/")
                        } if isinstance(normalized_generated, list) else set()
                        if (
                            not isinstance(normalized_generated, list)
                            or expected_media_paths != generated_media_paths
                        ):
                            report.error(
                                "OCR_NORMALIZED_BUNDLE_INVALID",
                                source_id,
                                "normalized checksums/media-map asset coverage differs",
                            )
        elif source.get("type") == "book-pdf-scan":
            if (
                carrier != "pdf-page"
                or policy.get("coverage") != "all-pages"
                or policy.get("renderer") != "poppler-pdftoppm"
                or policy.get("dpi") != 300
            ):
                report.error("OCR_BUNDLE_INVALID", source_id, "scanned PDF requires pdf-page carrier")
            page_count = None
            if source_path is not None:
                try:
                    page_count = _pdf_page_count(source_path, runner=runner, which=which)
                except Exception as exc:
                    report.error("PDF_PAGE_COUNT_UNVERIFIED", str(source_path), str(exc))
            if page_count is not None:
                expected_pages = {f"page-{number:06d}" for number in range(1, page_count + 1)}
                renderer_contract = ocr_manifest.get("renderer")
                if (
                    set(item_map) != expected_pages
                    or binding.get("pdfinfo_page_count") != page_count
                    or not isinstance(renderer_contract, dict)
                    or renderer_contract.get("dpi") != 300
                    or renderer_contract.get("format") != "png"
                ):
                    report.error("OCR_COVERAGE_INCOMPLETE", source_id, "live PDF page set/300-DPI binding drift")
                for number in range(1, page_count + 1):
                    record = item_map.get(f"page-{number:06d}")
                    context = record.get("context") if isinstance(record, dict) and isinstance(record.get("context"), dict) else {}
                    if context.get("page_number") != number or context.get("dpi") != 300:
                        report.error("OCR_COVERAGE_INCOMPLETE", f"page-{number:06d}", "page identity/DPI drift")
        else:
            report.error("OCR_SOURCE_MANIFEST_INVALID", source_id, "unsupported OCR source type")

        for item in by_source.get(source_id, []):
            report.checked_ocr_evidence += 1
            locator = item["locator"]
            record = record_map.get(locator.get("ocr_record_id"))
            if record is None:
                report.error("OCR_LOCATOR_UNRESOLVED", str(item.get("evidence_id")), "OCR record does not exist")
                continue
            if (
                record.get("status") != "completed"
                or record.get("image_sha256") != locator.get("image_sha256")
                or locator.get("source_id") != source_id
                or locator.get("carrier") != carrier
                or locator.get("ocr_run_id") != ocr_manifest.get("ocr_run_id")
            ):
                report.error("OCR_LOCATOR_UNRESOLVED", str(item.get("evidence_id")), "record/source/run/carrier/image binding mismatch")
                continue
            context = record.get("context") if isinstance(record.get("context"), dict) else {}
            if carrier == "docx-image" and (
                locator.get("media_occurrence_id") not in context.get("media_occurrence_ids", [])
                or locator.get("figure_id") not in context.get("figure_ids", [])
            ):
                report.error("OCR_LOCATOR_UNRESOLVED", str(item.get("evidence_id")), "DOCX figure occurrence mismatch")
                continue
            if carrier == "pdf-page" and locator.get("page_number") != context.get("page_number"):
                report.error("OCR_LOCATOR_UNRESOLVED", str(item.get("evidence_id")), "PDF page mismatch")
                continue
            regions = {
                region.get("region_id"): region for region in record.get("regions", [])
                if isinstance(region, dict)
            }
            region = regions.get(locator.get("region_id"))
            normalized = item.get("normalized_text")
            expected_hash = hashlib.sha256(str(normalized).encode("utf-8")).hexdigest()
            locator_hash = str(locator.get("content_hash", "")).removeprefix("sha256:")
            if (
                region is None
                or region.get("bbox_px") != locator.get("bbox_px")
                or region.get("text") != normalized
                or region.get("text_sha256") != _sha(str(normalized).encode("utf-8"))
                or not locator_hash
                or not expected_hash.startswith(locator_hash)
            ):
                report.error("OCR_LOCATOR_UNRESOLVED", str(item.get("evidence_id")), "bbox/text/content hash mismatch")
        if len(report.errors) == source_errors_before:
            report.verified_sources += 1
    return report


def main(argv: Sequence[str] | None = None) -> int:
    parser = argparse.ArgumentParser(description="Verify private OCR bundle integrity and ocr-region evidence locators.")
    parser.add_argument("distillation_dir")
    parser.add_argument("ocr_root")
    parser.add_argument("--sources-manifest", required=True)
    parser.add_argument(
        "--normalized-root",
        help="Required for DOCX OCR: root containing the integrity-bound normalized bundle.",
    )
    args = parser.parse_args(argv)
    report = verify(
        args.distillation_dir,
        args.ocr_root,
        args.sources_manifest,
        args.normalized_root,
    )
    print(json.dumps(report.as_dict(), ensure_ascii=False, indent=2, sort_keys=True))
    return 0 if report.ok else 1


if __name__ == "__main__":
    sys.exit(main())