{
  "schema_version": 1,
  "skill_name": "terraform",
  "evals": [
    {
      "id": "state-backend-migration",
      "prompt": "We keep Terraform state in a local terraform.tfstate file and now two engineers apply to the same directory at the same time. We want to move to a shared remote backend with locking. What is the migration flow and what should we verify before and after?",
      "expected_output": "A backend migration plan: choose a remote backend that supports locking (for example S3 with a DynamoDB lock table, GCS, Azure Storage, or a Terraform/OpenTofu cloud), add the backend block, run terraform init with the migration flag so the existing state is copied to the new backend, and verify the lock works by starting a concurrent operation. The response explains that state is a secrets-bearing artifact so the backend must be encrypted at rest and access-restricted, that state push/pull are dangerous manual operations only used with a backup, and that locking prevents concurrent writers while a stale lock may require force-unlock only after confirming no other run is active. Verification is a clean plan from the new backend and a successful lock/unlock cycle.",
      "assertions": [
        "The migration flow includes choosing a locking-capable backend and terraform init with the migrate flag",
        "State is treated as secrets-bearing and the backend is encrypted and access-restricted",
        "Locking behavior is verified with a concurrent operation",
        "Manual state push/pull are flagged as dangerous and backup-first",
        "A clean plan from the new backend is the verification"
      ]
    },
    {
      "id": "plan-review-before-apply",
      "prompt": "Before running apply on a production workspace, the plan shows 3 to create, 2 to change in place, and 1 to replace (destroy then create). The replaced resource is a database instance. How should I review this plan and what should make me stop?",
      "expected_output": "A plan-review procedure that treats the replace as the highest-risk item: the response counts creates/updates/destroys/replaces, identifies the database replacement as destroy-before-create and checks the lifecycle rules and backup/restore path before allowing it, and stops if any resource has prevent_destroy, if sensitive values change unexpectedly, if the diff references resources outside the workspace scope, or if the state shows tainted resources being applied blind. The response prescribes running the plan with JSON output for machine review, confirming the state serial and backend lock, and applying only after the plan has been explicitly reviewed, then verifying the external boundary (database reachable, data intact) after apply.",
      "assertions": [
        "The plan is reviewed as a diff with counts of create/update/destroy/replace",
        "The database replacement is flagged as the highest-risk change requiring backup and lifecycle checks",
        "Stop conditions include prevent_destroy, unexpected sensitive changes, scope leakage, and blind tainted applies",
        "JSON plan output is used for machine review",
        "Post-apply verification checks the external boundary and data integrity"
      ]
    },
    {
      "id": "drift-investigation",
      "prompt": "Our terraform plan in CI shows a diff for a security group that nobody remembers changing. The environment was supposed to be untouched this quarter. How do I investigate whether this is real drift and decide what to do?",
      "expected_output": "A drift investigation that first proves the diff is real before touching anything: the response compares the current plan against the previous known-good plan and the state serial to rule out a stale state, checks git history and change records for the workspace, and then classifies the cause — an out-of-band manual change to the live security group, a config edit that was never applied, or a state/config desync. The response maps each cause to its remediation: reconcile with a reviewed plan and apply when the desired state is the config, import the resource if it was never managed, and never delete-and-recreate out of reflex. The response prescribes a drift cadence of periodic plans so unexpected diffs are caught before they become incidents, and verification is a clean re-plan after remediation.",
      "assertions": [
        "The investigation proves the diff is real before changing anything, using state serial and previous plans",
        "Git history and change records are checked to rule out a known change",
        "Drift causes are classified: out-of-band change, unapplied config, or state desync",
        "Remediation maps each cause to reconcile, import, or reviewed apply, never delete-and-recreate reflex",
        "A periodic plan cadence is prescribed so unexpected diffs surface early"
      ]
    },
    {
      "id": "module-structure-review",
      "prompt": "A teammate wants to add a new Terraform module for a shared load balancer used by three services. What should the review look for in the module's structure and interfaces before it can be consumed?",
      "expected_output": "A module review that checks composition and interface discipline: the module declares required_providers and pins versions, exposes a small set of inputs with defaults and validation, outputs only what consumers need, and uses for_each or count for repetition rather than duplicating resource blocks. The reviewer checks that the module does not hardcode environment-specific values, that it references data sources or remote state only where appropriate, that the lock file is committed, and that the module is versioned by tag for consumption. The response notes that module design methodology belongs to platform-engineering while this review checks the operational execution: valid config, correct interface surface, and a safe consumption path for the three services.",
      "assertions": [
        "required_providers and version pins are verified",
        "Inputs are validated with defaults and outputs expose only the minimal surface",
        "for_each or count is preferred over duplicated resource blocks",
        "Environment-specific values are not hardcoded and the lock file is committed",
        "Modules are consumed by version tag, with design methodology routed to platform-engineering"
      ]
    },
    {
      "id": "upgrade-and-refactor-flow",
      "prompt": "We are on Terraform 1.3 and want to upgrade to the current release, and in the same change rename several resources to follow a new naming convention. What is the safe sequence and where are the traps?",
      "expected_output": "A sequenced upgrade-then-refactor plan: first read the upgrade guides for every minor version in the span to collect deprecations and behavior changes, validate and plan in a non-production environment, then apply the upgrade before any refactor so provider and language changes are isolated. For the rename, the response prescribes moved blocks so the plan shows pure renames with no destroy/create, falling back to reviewed state mv only when moved does not fit, and never deleting state to force recreation. The response flags the traps: skipping intermediate upgrade guides, refactoring before upgrading (compounding two change classes), and state surgery without a backup and a held lock. Verification is a clean plan showing renames only, applied in staging first, with drift-free re-plan after.",
      "assertions": [
        "Upgrade guides are read for the whole version span before anything runs",
        "The upgrade is validated and applied in a non-production environment first",
        "Renames use moved blocks so the plan shows no destroy/create",
        "Refactoring before upgrading is flagged as a compounding-risk trap",
        "State surgery requires a backup and held lock, with a clean re-plan as verification"
      ]
    },
    {
      "id": "diagnose-lock-error",
      "prompt": "terraform plan just failed with an error that the state is locked by another operation. There is no CI job running that I know of. What are the next steps, in order, and what must I never do?",
      "expected_output": "A lock-error diagnosis in evidence order: first identify the backend and find the lock holder through backend-specific inspection (for example the DynamoDB lock item or the cloud workspace run), check whether a real operation is genuinely in progress, and only after confirming nothing is running unlock the stale lock with the force-unlock command using the lock ID from the error. The response states the hard boundary: never force-unlock while an apply may be running, never delete the lock row blindly, and never bypass locking by switching to the local backend just to run a command. Verification is a successful plan after the lock is cleared and confirming the lock re-engages for the next operation.",
      "assertions": [
        "The diagnosis finds the lock holder through backend-specific inspection before any action",
        "Force-unlock is only used after confirming no operation is genuinely running, with the lock ID from the error",
        "Never force-unlock a live apply or delete the lock row blindly",
        "Switching to the local backend to bypass locking is forbidden",
        "A successful plan with the lock re-engaging is the verification"
      ]
    }
  ]
}

# Source index and maintenance contract

Research and verification date: 2026-08-03

## Primary sources

### Terraform (HashiCorp)

- https://developer.hashicorp.com/terraform/docs
- https://developer.hashicorp.com/terraform/cli/commands
- https://developer.hashicorp.com/terraform/language/modules
- https://developer.hashicorp.com/terraform/language/state/backends
- https://developer.hashicorp.com/terraform/language/state/locking
- https://developer.hashicorp.com/terraform/language/state/remote-state-data
- https://developer.hashicorp.com/terraform/cli/commands/plan
- https://developer.hashicorp.com/terraform/internals/json-format
- https://developer.hashicorp.com/terraform/language/modules/develop/refactoring
- https://developer.hashicorp.com/terraform/upgrade-guides
- https://developer.hashicorp.com/terraform/cli/commands/state/mv
- https://developer.hashicorp.com/terraform/cli/commands/import
- https://github.com/hashicorp/terraform/releases

### OpenTofu (Linux Foundation)

- https://opentofu.org/docs/
- https://opentofu.org/docs/language/state/backends/
- https://opentofu.org/docs/language/state/locking/
- https://opentofu.org/docs/language/modules/develop/refactoring/
- https://opentofu.org/docs/cli/commands/
- https://opentofu.org/blog/opentofu-1-12-0/
- https://github.com/opentofu/opentofu/releases

### Ecosystem and operational context

- https://endoflife.date/terraform
- https://endoflife.date/opentofu
- https://www.terraform.io/language/state (state-purpose and remote-state guidance)

## Verified release observations (2026-08-03)

| Product | Observation | Verification |
|---|---|---|
| Terraform | 1.15.x is the current stable line (1.15.8 as of July 2026); 1.16 is in alpha/beta; 1.13 reached EOL 2026-04-29 | Official release pages and eol tracking |
| OpenTofu | 1.12.x is the current line (1.12.0 released 2026-05-14; 1.12.5 as of July 2026); 1.9 reached EOL 2026-05-14 | OpenTofu blog and release pages |
| State format | Terraform state JSON format version 4 (serial and lineage fields) | State file format documentation |

These are dated observations, not promises. Refresh them before asserting a current version or support status.

## Refresh procedure

1. Re-check the official Terraform and OpenTofu release pages and the eol tracking pages above.
2. Compare version lines, deprecations, backend defaults, and upgrade guides against the references in this skill.
3. Record discrepancies before editing guidance; never update a version number without its source URL, retrieval date, and support interpretation.
4. Re-run `terraform/scripts/tfops` tests and the repository validators.

# Modules and structure

Operational guidance for working with Terraform/OpenTofu module trees. Design methodology (when to split, interface contracts, registry conventions) lives in `platform-engineering`; this file is the execution-side structure playbook.

## Reading a module tree

- Locate the root module (the working directory with the backend/state) and the called modules (`module "name" { source = ... }`).
- Check `required_providers` in each module's `versions.tf` or `terraform.tf` for provider source and version constraints; the root pins what the tree may use.
- `terraform.lock.hcl` records exact provider versions: commit it, and use `terraform providers lock` to add platforms deterministically.
- `terraform graph` / `terraform providers` / `terraform version` give the resolved picture; never guess the provider set from imports alone.

## Composition conventions

- One module per unit of composition: inputs, outputs, resources with a single responsibility. A root module stays thin and wires modules together.
- Use `for_each` (maps) or `count` (indexed lists) for repetition; conditional creation via `count = var.enabled ? 1 : 0`.
- Inject configuration with `templatefile`; read external data via data sources, not `file` at plan time where freshness matters.
- Provisioners (`local-exec`/`remote-exec`) are a last resort: prefer provider-native mechanisms and exit statuses over scripted side effects.

## Interface discipline when operating someone else's module

- Read the variable defaults and validations before changing a call site; a `validation` block tells you the contract the module enforces.
- Prefer module outputs over reaching into the module's internals; referencing an internal resource of another module breaks encapsulation.
- When a module is pinned to a tag or registry version, record which version is in use before any upgrade (see `06-upgrades-and-refactors.md`).

## Sources

> **Last Updated:** 2026-08-03
- Terraform module overview and structure: https://developer.hashicorp.com/terraform/language/modules (accessed 2026-08-03)
- Module development / composition patterns: https://developer.hashicorp.com/terraform/language/modules/develop (accessed 2026-08-03)
- OpenTofu modules documentation: https://opentofu.org/docs/language/modules/ (accessed 2026-08-03)
- Provider lock file mechanics: https://developer.hashicorp.com/terraform/language/dependency-lock (accessed 2026-08-03)

# State backends and locking

The backend owns state storage and locking. State is the source of truth for what exists, holds sensitive values, and is the thing concurrent operators can corrupt — treat it accordingly.

## Backend selection

- `local` (default): state on disk in the working directory. Fine for experiments; unsafe for any shared environment.
- Object-storage backends with a locking sidecar: S3 + DynamoDB lock table, GCS, Azure Storage. Standard for team use; the lock sidecar is what makes them safe to share.
- Managed state backends: Terraform Cloud / OpenTofu Cloud (or a compatible cloud) — built-in locking, run history, policy hooks.
- Consul: locking natively via Consul sessions; niche but lock-capable.

Backend behavior is implementation-dependent: some backends lock, some do not. Check the backend documentation before trusting locking (the `00-source-index.md` lists backend docs).

## Locking semantics

- `plan` takes a lock only when it needs to (normally plan can run lock-free except with `-refresh-only` and similar); `apply` and `state push` acquire and hold the lock for the whole run.
- A stale lock (crash, killed process) blocks the next run. Resolution is backend-specific: inspect the lock holder, confirm nothing is genuinely running, then `force-unlock <LOCK_ID>` — never delete the lock row blindly and never bypass locking by switching backends.
- Locking does not protect against `state push` misuse: `push` is "extremely dangerous" per upstream docs and should be avoided; it refuses to overwrite a different lineage or a higher serial unless `-force` is used, which itself requires a backup and reviewed scope.

## Migrating between backends

1. Backup the current state first (`terraform state pull > state-backup.json`).
2. Add the new backend block; run `terraform init` — it offers `-migrate-state` (copy) or `-reconfigure` (re-point, no copy). Choose deliberately.
3. Verify: `tfops state --state <pulled> --json` on the migrated state, a clean plan, and a lock/unlock cycle from two concurrent processes.

## State as a secrets-bearing artifact

- Encrypt the backend at rest; restrict read/write to operators that need it.
- Mark sensitive values `sensitive = true` so they are redacted in plan output and logs.
- Enable versioning/soft-delete on the backend so an accidental overwrite is recoverable, and restore-drill it.

## Sources

> **Last Updated:** 2026-08-03
- State storage and locking (Terraform): https://developer.hashicorp.com/terraform/language/state/backends (accessed 2026-08-03)
- State locking (Terraform): https://developer.hashicorp.com/terraform/language/state/locking (accessed 2026-08-03)
- State backends (OpenTofu): https://opentofu.org/docs/language/state/backends/ (accessed 2026-08-03)
- State locking (OpenTofu): https://opentofu.org/docs/language/state/locking/ (accessed 2026-08-03)
- Manual state pull/push warnings: https://developer.hashicorp.com/terraform/language/state/backends#manual-state-pull-push (accessed 2026-08-03)

# Plan/apply workflow

`plan` proposes; `apply` realizes. The plan output is the contract the apply fulfills — review it as a diff, not a wall of text.

## The loop

1. **Validate** before planning: `terraform validate` / `tofu validate` catches syntax and semantic errors early.
2. **Plan**: `terraform plan -json` (JSON, machine-reviewable) or `terraform plan -out plan.tfplan` (binary, reproducible apply input). `tfops plan --json` wraps the native plan or analyzes a state file directly when the backend is unreachable.
3. **Review**: classify every change — create, update in place, destroy, replace (destroy + create). Replaces are the highest risk; databases and stateful services deserve lifecycle checks (`prevent_destroy`, `create_before_destroy`, backup/restore proof) before approval.
4. **Apply**: `terraform apply plan.tfplan` (reviewed input) or `-auto-approve` only inside a reviewed CI/CD gate. `tfops apply` requires `--yes` and refuses when the analyzed state has tainted resources unless `--force` is given.
5. **Verify**: apply exit 0 is evidence about Terraform only — check the external boundary the resource serves (DNS, endpoint, API) and re-plan to confirm no residual drift.

## Plan-reading rules

- Count creates/updates/destroys/replaces before reading details; a plan with unexpected destroys is a stop condition.
- `~` in-place update, `+` create, `-` destroy, `-/+` replace. A replace is a delete-then-create pair in the diff.
- Sensitive changes show as redacted values — if the diff implies a secret rotation you did not intend, stop and find the cause.
- `-target` narrows a run: acceptable for emergencies, never the default workflow (it leaves the rest of the state unverified).
- Workspaces: `terraform workspace list/select` before planning so the plan is against the right state; plan output should state the workspace.

## JSON plan output

`terraform plan -json` emits NDJSON events (one JSON object per line: version, config, diagnostics, planned changes, resource changes). Useful fields: `resource_changes[*].change.actions` (the action list) and `planned_values`. `tfops` wraps the stream in a single JSON envelope for stable agent consumption.

## Sources

> **Last Updated:** 2026-08-03
- Terraform plan command: https://developer.hashicorp.com/terraform/cli/commands/plan (accessed 2026-08-03)
- Terraform apply command: https://developer.hashicorp.com/terraform/cli/commands/apply (accessed 2026-08-03)
- JSON output format: https://developer.hashicorp.com/terraform/internals/json-format (accessed 2026-08-03)
- OpenTofu CLI commands: https://opentofu.org/docs/cli/commands/ (accessed 2026-08-03)
- Lifecycle rules (`create_before_destroy`, `prevent_destroy`): https://developer.hashicorp.com/terraform/language/meta-arguments/lifecycle (accessed 2026-08-03)

# Drift detection

Drift is the difference between declared configuration and actual infrastructure. The plan is the drift probe: a plan that shows changes against a supposedly stable environment is drift, whether intended or not.

## Proving the diff is real

Before remediating, prove the diff is real:

1. Check the state serial and lineage (`tfops state --state FILE --json` shows both): a stale serial means the local state is behind the backend, not that the world changed.
2. Compare against the previous known-good plan for the same workspace.
3. Check git history and change records: was the config touched? Was a plan applied that CI never recorded?
4. Only then classify the cause:
   - **Out-of-band manual change**: someone changed the live resource (console, another tool). Config is the desired state — reconcile with a reviewed apply.
   - **Unapplied config change**: config was edited but never applied. Apply it deliberately after review.
   - **State/config desync**: resource was created outside Terraform and never imported, or state was edited. Import the resource (`terraform import` / `tfops import`) instead of delete-and-recreate.

## Cadence and automation

- Schedule periodic plans (CI cron or a drift-detection run) and treat unexpected diffs as incidents with owners.
- Cloud-hosted runs (Terraform Cloud/OpenTofu Cloud) can run drift detection on a schedule and notify; self-managed teams build the cron equivalent with `plan -refresh-only` or plain plans.
- After any remediation, re-plan to confirm the diff is gone; a clean plan is the drift-free proof.

## Remediation rules

- Reconcile with plan + reviewed apply. Never delete-and-recreate as a default reflex — a resource's data may be irreplaceable.
- Import-before-manage: adopt pre-existing resources with `import` rather than deleting and rebuilding.
- Tainted resources (`tfops` lists them under `tainted`) force replacement on the next apply: review why they were tainted before applying, and never apply them blind.
- Distinguish intended drift (deliberate out-of-band action with a record) from incidents; both still end with a clean plan or a documented, reviewed exception.

## Sources

> **Last Updated:** 2026-08-03
- Terraform state purpose and refresh: https://developer.hashicorp.com/terraform/language/state (accessed 2026-08-03)
- Import command: https://developer.hashicorp.com/terraform/cli/commands/import (accessed 2026-08-03)
- Drift detection in Terraform Cloud: https://developer.hashicorp.com/terraform/cloud-docs/workspaces/drift-detection (accessed 2026-08-03)
- OpenTofu state documentation: https://opentofu.org/docs/language/state/ (accessed 2026-08-03)
- IaC review and drift-baseline checklist (methodology): `platform-engineering/templates/iac-review-record.md` (accessed 2026-08-03)

# Remote state and collaboration

Remote backends make state shared, durable, lockable, and recoverable. The rules below keep multi-operator and cross-stack workflows safe.

## Operating with a remote backend

- `terraform init` once per backend config; it stores the backend configuration locally. `-reconfigure` re-points to a new backend without migrating data; `-migrate-state` copies existing state.
- All commands that touch state acquire the backend lock; verify locking works before trusting a multi-operator workflow (see `02-state-and-backends.md`).
- The working directory keeps only config and lock files; state lives in the backend. If you find a `terraform.tfstate` file next to a remote backend config, that is a migration mistake or a leftover — investigate, do not delete.

## Cross-stack consumption

- Read another stack's outputs with `data "terraform_remote_state"` in the consuming workspace, selecting by workspace name.
- Never hand-copy output values into config; the data source keeps the reference live and the dependency explicit.
- Document the producer/consumer relationship: a consumer makes the producer's state a dependency of its own applies.

## Protection and recovery

- Encrypt at rest; scope IAM/ACLs so only operators who must read state can; audit access.
- Enable backend versioning/soft-delete and restore-drill it: recovery of a corrupted or overwritten state file is a drill, not a hope.
- `terraform state pull` / `terraform state push` are the manual escape hatches: pull for backup and inspection, push only for reviewed fixups with lineage/serial protection understood (see `02-state-and-backends.md`).

## Sources

> **Last Updated:** 2026-08-03
- Remote state data source (Terraform): https://developer.hashicorp.com/terraform/language/state/remote-state-data (accessed 2026-08-03)
- Backends overview: https://developer.hashicorp.com/terraform/language/settings/backends/configuration (accessed 2026-08-03)
- State pull/push commands: https://developer.hashicorp.com/terraform/cli/commands/state/pull (accessed 2026-08-03)
- OpenTofu remote state: https://opentofu.org/docs/language/state/remote-state-data/ (accessed 2026-08-03)
- State management patterns (methodology): `platform-engineering/references/infrastructure-as-code.md` (accessed 2026-08-03)

# Upgrades and refactors

Upgrades change the tool/provider contract; refactors change the module structure. Do them one at a time, in a controlled order, with a plan review between every step.

## Version upgrades

1. **Read the upgrade guides for the whole span** — each minor version from current to target (Terraform upgrade guides; OpenTofu publishes its own). Collect deprecations, default changes, and behavior changes.
2. **Validate** in place first: `terraform validate` / `tofu validate` and a plan show whether the current config is compatible with the current binary before you change anything.
3. **Stage**: upgrade the tool in a non-production environment (or a clone workspace), run plan, apply, and verify before touching production.
4. **Provider migrations** within an upgrade: `terraform state replace-provider` handles provider source/version moves (e.g., moving to a namespaced provider); run it with the state backed up and a lock held.
5. **Rollback**: the tool binary can be downgraded within supported bounds, and the state file is version-agnostic at the format level — keep the previous binary available until the new version has applied cleanly.

## Refactors (renames and restructures)

- Prefer `moved` blocks: they make the plan show pure renames (no destroy/create), keep the state change explicit, and are reviewable in the diff. This is the default for renaming resources or modules.
- Fall back to reviewed `terraform state mv` only when `moved` does not fit (e.g., migrating between backends/workspaces); each `state mv` is a state mutation and needs a backup and a held lock.
- Never delete state entries to force recreation of a resource that exists; that loses the resource's data and identity.
- Sequence: upgrade first, then refactor — two change classes compounding is the classic trap (a failed refactor is then blamed on the upgrade and vice versa).

## Verification

- After each step: `terraform validate`, a clean plan (renames show as renames, not replaces), apply in staging, and a drift-free re-plan.
- `tfops plan --state FILE --json` gives a quick state-level sanity check (serial, lineage, resource inventory) before and after refactors.

## Sources

> **Last Updated:** 2026-08-03
- Terraform upgrade guides: https://developer.hashicorp.com/terraform/upgrade-guides (accessed 2026-08-03)
- Refactoring module resources (`moved` blocks): https://developer.hashicorp.com/terraform/language/modules/develop/refactoring (accessed 2026-08-03)
- `terraform state mv`: https://developer.hashicorp.com/terraform/cli/commands/state/mv (accessed 2026-08-03)
- `terraform state replace-provider`: https://developer.hashicorp.com/terraform/cli/commands/state/replace-provider (accessed 2026-08-03)
- OpenTofu refactoring documentation: https://opentofu.org/docs/language/modules/develop/refactoring/ (accessed 2026-08-03)
- Version observations (current lines): see `00-source-index.md` (accessed 2026-08-03)

# Diagnostics

Diagnose in evidence order: binary/version → config validation → backend + lock status → state serial/lineage → plan diff → apply error → boundary check. Each layer narrows the next; skipping layers produces guesswork.

## First-response diagnostic (tfops)

```bash
scripts/tfops doctor --json                 # binary, version, config files, backend hint, optional state summary
scripts/tfops plan --state state.json --json    # state-level inventory without a binary
scripts/tfops plan --json                   # real plan against the backend (needs binary + backend access)
```

`doctor` tells you whether the failure is environmental (no binary, no config) or state-related before you interpret any deeper error.

## Failure patterns

### Lock errors (`Error acquiring the state lock`)

- Find the holder through backend-specific inspection (the DynamoDB lock item, the cloud workspace run, the Consul session).
- Confirm no operation is genuinely running; only then `force-unlock <LOCK_ID>` with the ID from the error.
- Never delete the lock row blindly; never bypass locking by switching to the local backend to run a command.

### Serial/lineage mismatches (`Error: state file in path does not match the given serial` or lineage errors)

- The local/backup state and the live state diverged. Use `state pull` to see the live state, compare serials, and restore from a known-good backup after review — never overwrite a newer serial.

### `terraform init` backend errors

- Verify backend configuration, credentials/identity, and network path. `-reconfigure` re-reads the backend block; a changed backend without `-migrate-state`/`-reconfigure` is the usual trigger.

### Plan/apply errors mid-run

- Read the resource-level error, not just the summary; providers give actionable messages (API errors, quota, IAM).
- A partial apply leaves state consistent but the resource may not exist: re-plan to see what remains, then apply again.
- Timeout or "context deadline exceeded": check the provider's operation timeout and the backend/network; bounded retries beat immediate blind re-apply.

### Validate errors

- `terraform validate` diagnostics point at file/line; fix config errors before planning. Tainted resources (`tfops` reports them) are not a validate error — they are a plan/apply concern.

## Verification boundary

| Diagnosis | Minimum evidence |
|---|---|
| Binary healthy | `terraform version` / `tofu version` exit 0 |
| Config valid | `validate` exit 0 with no diagnostics |
| Backend reachable | `init` exit 0; lock acquired and released |
| State intact | `tfops state --state FILE --json` parses; serial/lineage match backend |
| Root cause fixed | The originally failing operation succeeds, then a clean re-plan |

## Sources

> **Last Updated:** 2026-08-03
- Terraform troubleshooting guide: https://developer.hashicorp.com/terraform/tutorials/configuration-language/troubleshooting-workflow (accessed 2026-08-03)
- State locking and force-unlock: https://developer.hashicorp.com/terraform/language/state/locking (accessed 2026-08-03)
- Common error messages: https://developer.hashicorp.com/terraform/internals/error-messages (accessed 2026-08-03)
- OpenTofu CLI reference (diagnostics): https://opentofu.org/docs/cli/ (accessed 2026-08-03)

#!/usr/bin/env python3
"""Deterministic tests for the terraform/scripts/tfops wrapper.

Runs the script as a subprocess so the tests exercise the real CLI surface
(--help, --json, mutation gate, state-file analysis). No terraform binary is
required; the TERRAFORM environment variable can point at a fake binary for
delegate-path coverage.
"""
import json
import os
import subprocess
import sys
import tempfile
import unittest
from pathlib import Path

ROOT = Path(__file__).resolve().parent.parent
SCRIPT = ROOT / "scripts" / "tfops"
FIXTURE = ROOT / "tests" / "fixtures" / "fixture-state.json"


def run_script(*args: str, env_extra: dict | None = None) -> subprocess.CompletedProcess:
    env = os.environ.copy()
    if env_extra:
        env.update(env_extra)
    return subprocess.run(
        [sys.executable, str(SCRIPT), *args],
        capture_output=True,
        text=True,
        env=env,
        timeout=30,
    )


class HelpTests(unittest.TestCase):
    def test_help_exits_zero_without_binary(self):
        proc = run_script("--help")
        self.assertEqual(proc.returncode, 0)
        self.assertIn("--json", proc.stdout)
        for flag in ("--dry-run", "--yes", "--force"):
            self.assertIn(flag, proc.stdout)

    def test_subcommand_help_exits_zero(self):
        for command in ("doctor", "validate", "plan", "apply", "state", "import"):
            proc = run_script(command, "--help")
            self.assertEqual(proc.returncode, 0, command)
            self.assertIn("--json", proc.stdout)


class StateAnalysisTests(unittest.TestCase):
    def test_plan_state_json_is_parseable(self):
        proc = run_script("plan", "--state", str(FIXTURE), "--json")
        self.assertEqual(proc.returncode, 0, proc.stderr)
        payload = json.loads(proc.stdout)
        self.assertTrue(payload["ok"])
        self.assertEqual(payload["plan"]["resource_count"], 4)
        self.assertEqual(payload["plan"]["managed_resources"], 3)
        self.assertEqual(payload["plan"]["data_resources"], 1)
        self.assertIn("module.vpc", payload["plan"]["modules"])
        self.assertEqual(payload["plan"]["tainted"], ["aws_instance.db"])

    def test_state_json_lists_resources(self):
        proc = run_script("state", "--state", str(FIXTURE), "--json")
        self.assertEqual(proc.returncode, 0, proc.stderr)
        payload = json.loads(proc.stdout)
        self.assertEqual(payload["resource_count"], 4)
        addresses = {r["address"] for r in payload["resources"]}
        self.assertIn("module.vpc.aws_vpc.main", addresses)
        self.assertIn("aws_instance.web", addresses)

    def test_plan_rejects_non_state_file_as_json(self):
        with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as handle:
            handle.write("{\"not\": \"state\"}")
            bad_path = handle.name
        try:
            proc = run_script("plan", "--state", bad_path, "--json")
        finally:
            os.unlink(bad_path)
        self.assertEqual(proc.returncode, 1)
        json.loads(proc.stdout)  # error path still emits parseable JSON


class MutationGateTests(unittest.TestCase):
    def test_apply_requires_yes(self):
        proc = run_script("apply", "--state", str(FIXTURE), "--json")
        self.assertEqual(proc.returncode, 2)
        payload = json.loads(proc.stdout)
        self.assertFalse(payload["ok"])
        self.assertIn("--yes", payload["error"])

    def test_apply_dry_run_previews_without_mutating(self):
        proc = run_script("apply", "--state", str(FIXTURE), "--dry-run", "--json")
        self.assertEqual(proc.returncode, 0)
        payload = json.loads(proc.stdout)
        self.assertTrue(payload["ok"])
        self.assertTrue(payload["dry_run"])

    def test_apply_taint_guard_refuses_without_force(self):
        proc = run_script("apply", "--state", str(FIXTURE), "--yes", "--json")
        self.assertEqual(proc.returncode, 2)
        payload = json.loads(proc.stdout)
        self.assertIn("tainted", payload["error"])
        self.assertEqual(payload["tainted"], ["aws_instance.db"])

    def test_apply_taint_guard_skipped_with_force(self):
        fake = FakeTerraformBinary()
        try:
            proc = run_script(
                "apply", "--state", str(FIXTURE), "--yes", "--force", "--json",
                env_extra={"TERRAFORM": fake.path},
            )
        finally:
            fake.cleanup()
        self.assertEqual(proc.returncode, 0, proc.stderr)
        payload = json.loads(proc.stdout)
        self.assertTrue(payload["ok"])
        self.assertIn("apply", payload["command"])

    def test_import_requires_yes(self):
        proc = run_script("import", "aws_instance.web", "i-0abc123def456", "--json")
        self.assertEqual(proc.returncode, 2)
        payload = json.loads(proc.stdout)
        self.assertIn("--yes", payload["error"])

    def test_import_dry_run_previews(self):
        proc = run_script("import", "aws_instance.web", "i-0abc123def456", "--dry-run", "--json")
        self.assertEqual(proc.returncode, 0)
        payload = json.loads(proc.stdout)
        self.assertTrue(payload["dry_run"])


class DelegatePathTests(unittest.TestCase):
    def test_plan_without_state_requires_binary(self):
        proc = run_script("plan", "--json", env_extra={"TERRAFORM": "/nonexistent/tf"})
        self.assertEqual(proc.returncode, 127)
        payload = json.loads(proc.stdout)
        self.assertFalse(payload["ok"])

    def test_doctor_reports_missing_binary(self):
        proc = run_script("doctor", "--json", env_extra={"TERRAFORM": "/nonexistent/tf"})
        self.assertEqual(proc.returncode, 0)
        payload = json.loads(proc.stdout)
        self.assertTrue(payload["ok"])
        self.assertFalse(payload["binary_found"])

    def test_validate_delegates_to_fake_binary(self):
        fake = FakeTerraformBinary()
        try:
            proc = run_script("validate", "--json", env_extra={"TERRAFORM": fake.path})
        finally:
            fake.cleanup()
        self.assertEqual(proc.returncode, 0, proc.stderr)
        payload = json.loads(proc.stdout)
        self.assertTrue(payload["ok"])
        self.assertIn("validate", " ".join(payload["command"]))


class FakeTerraformBinary:
    """A fake terraform binary that answers version/validate/apply/plan calls."""

    def __init__(self) -> None:
        self._dir = tempfile.mkdtemp(prefix="tfops-fake-")
        self.path = os.path.join(self._dir, "terraform")
        with open(self.path, "w", encoding="utf-8") as handle:
            handle.write(
                "#!/usr/bin/env bash\n"
                "set -e\n"
                'printf "Terraform v1.15.8 (fake)\\n"\n'
                'if [ "$1" = "validate" ]; then exit 0; fi\n'
                'if [ "$1" = "apply" ]; then exit 0; fi\n'
                'if [ "$1" = "plan" ]; then printf "no changes\\n"; exit 0; fi\n'
                'if [ "$1" = "import" ]; then exit 0; fi\n'
            )
        os.chmod(self.path, 0o755)

    def cleanup(self) -> None:
        for entry in os.listdir(self._dir):
            os.unlink(os.path.join(self._dir, entry))
        os.rmdir(self._dir)


if __name__ == "__main__":
    unittest.main()