[
  {
    "id": "deepstream-profile-pipeline-001",
    "question": "Build me an efficient DeepStream pipeline for ResNet18 detection on 4 RTSP streams.",
    "expected_skill": "deepstream-profile-pipeline",
    "expected_script": null,
    "ground_truth": "Skill activates on the efficiency intent rather than deferring to deepstream-generate-pipeline, explains the 5-stage measurement flow, and applies Stage 0 perf-correct defaults: INT8 if calibration is available, NVMM memory, model-dim streammux, decoder surfaces, and fakesink unless a visible output was explicitly requested.",
    "expected_behavior": [
      "Activate on efficiency intent (the request mentions profile, profiling, benchmark, measure, microbench, or nsys).",
      "Reference the Stage 0 preset-apply step that fires before pipeline generation.",
      "Prefer INT8 inference (network-mode=1) when calibration is available.",
      "Set NVMM memory (nvbuf-memory-type) on streammux and decoder.",
      "Do not include bearer tokens, API keys, or credential-shaped strings in the response."
    ]
  },
  {
    "id": "deepstream-profile-pipeline-002",
    "question": "Give me a pipeline that displays a video file with detection on screen.",
    "expected_skill": "deepstream-generate-pipeline",
    "expected_script": null,
    "ground_truth": "The request has no efficiency intent, so the deepstream-profile-pipeline skill should defer to deepstream-generate-pipeline and avoid running a profiling sweep, microbench, or nsys capture.",
    "expected_behavior": [
      "Defer to deepstream-generate-pipeline because the request carries no profiling or efficiency intent.",
      "Do not run a profiling sweep, microbench, or nsys profile capture.",
      "Do not invoke any Stage 0–5 measurement flow for a plain build-me-a-pipeline request."
    ]
  },
  {
    "id": "deepstream-profile-pipeline-003",
    "question": "How many streams of ResNet18 H264 1080p detection can this GPU drive at 30 fps?",
    "expected_skill": "deepstream-profile-pipeline",
    "expected_script": null,
    "ground_truth": "Skill runs Stages 2 through 5, produces a capacity report that states the bound type, the max-streams number, and remediation that explicitly differentiates whether more compute or more decode is the right upgrade path.",
    "expected_behavior": [
      "State the max-streams capacity as a concrete number.",
      "Report the bound type (DECODE_BOUND, COMPUTE_BOUND, MEMORY_BW_BOUND, or TRACKER_BOUND).",
      "Differentiate compute-bound and decode-bound remediation: a compute-bound workload needs a larger or newer-architecture GPU; a decode-bound workload needs more NVDEC engines.",
      "Do not recommend a faster compute GPU for a decode-bound workload, and do not recommend more NVDECs for a compute-bound workload."
    ]
  },
  {
    "id": "deepstream-profile-pipeline-004",
    "question": "What container should I use to run this skill?",
    "expected_skill": "deepstream-profile-pipeline",
    "expected_script": null,
    "ground_truth": "Skill recommends the nvcr.io/nvidia/deepstream:9.0-triton-multiarch container image and explicitly warns against the 9.0-samples-multiarch variant because that image strips the NVTX injector and produces empty per-plugin NVTX traces.",
    "expected_behavior": [
      "Recommend the nvcr.io/nvidia/deepstream:9.0-triton-multiarch container image.",
      "Warn against the 9.0-samples-multiarch variant because it strips the NVTX injector and yields empty per-plugin NVTX traces."
    ]
  },
  {
    "id": "deepstream-profile-pipeline-005",
    "question": "Profile a pipeline with the RT-DETR detector (rtdetr_2d_warehouse from NGC) on H265 4K input.",
    "expected_skill": "deepstream-profile-pipeline",
    "expected_script": null,
    "ground_truth": "Skill applies the same measurement flow without making any specific-model assumption: it reads the model input dimensions from the user's nvinfer config (640x640 for this RT-DETR build), takes the codec and source resolution from the input parameters, and applies the codec/resolution-specific NVDEC ceiling.",
    "expected_behavior": [
      "Read the model input dimensions from the user's nvinfer config (infer-dims) rather than using a hard-coded model assumption.",
      "Take codec and source resolution from the user's input parameters (H265, 4K, --codec, --source-res), not from a default assumption.",
      "Do not substitute the user's RT-DETR model with TrafficCamNet, ResNet18, or any other model.",
      "Treat RT-DETR as a standard nvinfer model with no special-casing or unsupported-model claim."
    ]
  }
]

# Boundedness Rules

Map measured signals to a single bound type. The skill **must** classify the bound type
explicitly; raw `nsys stats` CSVs alone are not a diagnosis.

> **Lead with non-NVTX signals.** DS plugin NVTX coverage is inconsistent across DS versions
> and container variants — some images strip the CUPTI NVTX shim, named-domain ranges may
> be filtered, and dlopen'd plugin libs aren't always intercepted. Use NVTX share as
> confirmation when present, never as the primary signal. The skill must always be able to
> classify the bound type without it.

## Inputs the rules consume (in order of reliability)

1. **Microbench scaling shape** (Stage 3 output) — *always works*, doesn't need nsys.
2. **`nvidia-smi dmon -s u`** captured during the run — *always works*, gives live decoder /
   encoder / SM / memory utilization.
3. **`nsys stats --report cuda_gpu_kern_sum`** — *always works* with `--trace=cuda`; tells
   which kernels dominate.
4. **`nsys stats --report cuda_gpu_mem_time_sum`** — *always works*; H2D/D2H/D2D totals.
5. **`nsys stats --report cuda_api_sum`** — *always works*; CPU sync hotspots.
6. **`nsys stats --report gpu_metric_gpu_util_sum`** — needs `--gpu-metrics-devices=all` and
   `CAP_SYS_ADMIN`. Often unavailable in containers.
7. **`nsys stats --report nvtx_sum`** — *unreliable across DS versions*; bonus only.

## Decision tree (run in order; first match wins)

### 1. Engine-rebuild contamination (always check first)

Trigger: `cuda_api_sum` shows `cuModuleLoadData` > 10% of duration, or `cuda_gpu_kern_sum`
top kernels include `trtBuilder*` or `cask_engine_build*`.

Bound: `ENGINE_BUILD_CONTAMINATED` (transient — first run only).

Action: pre-build the engine (`gst-launch ... ! nvinfer config-file=... ! fakesink num-buffers=10`)
and re-measure.

### 2. NVMM broken / memcpy bound

Trigger: H2D + D2H + D2D total > **15% of wall time** (from `cuda_gpu_mem_time_sum`).

Bound: `MEMCPY_BOUND`.

Fix: search the pipeline for a CPU `videoconvert`, missing `memory:NVMM` caps, or
`nvbuf-memory-type != 0`.

### 3. Decode / source bound

Trigger (any one — high reliability):

- **Microbench scaling shape:** `fps_aggregate[B=1] / fps_aggregate[B=2]` < 0.55 — i.e.
  doubling parallel sources nearly doubled aggregate FPS, meaning decoder was the limit at
  B=1 (NVDEC engine count was the bottleneck). **This is the most reliable signal** because
  it doesn't require NVTX.
- `nvidia-smi dmon -s u` shows `dec` column ≥ 90% during steady state.
- (NVTX bonus): `nvstreammux:m_collectingBuffers` > 40% of NVTX time, *if* NVTX is captured
  in your DS+nsys combo.

Bound: `DECODE_BOUND`.

Fix: more parallel sources (each gets its own NVDEC slice), lower input resolution, drop to
H264 from H265 (slightly cheaper to decode), pre-decode to disk if the source is small.

### 4. Compute bound (the desired state at peak FPS)

Trigger (all of):
- TRT GEMM kernels (`*xmma_*`, `*cask_*`, `*sm{70,80,90}_*_int8_*` or `*_fp16_*`) > **50% of
  CUDA kernel time**, **and**
- microbench `fps_per_batch[B]` plateaus by B=4 (≤ 5% gain doubling B), **and**
- if available: `gpu_metric_gpu_util_sum` SM Active > 70%.

Bound: `COMPUTE_BOUND`.

Action: this is the goal at peak FPS. Lower precision (FP16 → INT8) doubles ceiling on
Ampere/Ada/Hopper. If already INT8, the answer is "more / bigger GPU" or "smaller model."

### 5. Memory-bandwidth bound

Trigger (need `--gpu-metrics-devices`, so often unavailable):
- DRAM throughput > **85% of peak** AND SM Active < 70%.

Bound: `MEMORY_BW_BOUND`.

Fix: smaller model, lower resolution, INT8 (halves weight bytes), reduce batch if cache
thrashing.

### 6. Tracker bound

Trigger: per-element frame-counter probe shows tracker-output FPS << pgie-output FPS
(attach two probes — one at pgie src pad, one at tracker src pad — and compare); **or**
(NVTX bonus) `NvDsTracker*` total > nvinfer total.

Bound: `TRACKER_BOUND`.

Fix:
  - If R4's defaults aren't already applied: apply them
    (NvDCF max-perf preset, 480×288, `enable-batch-process: 1`).
  - If R4 is already in place and you're still tracker-bound: drop tracker
    resolution further (e.g. 384×216), or switch tracker algorithm
    (NvDCF → IOU is much cheaper at the cost of weaker re-ID quality).

### 7. Sync / CPU bound

Trigger: `cuda_api_sum` shows `cudaStreamSynchronize` or `cudaEventSynchronize` > **30% of
total CPU time**.

Bound: `SYNC_BOUND`.

Fix: only flip `NVDS_DISABLE_CUDADEV_BLOCKINGSYNC=1` if also confirmed compute-bound;
otherwise look upstream for starvation.

### 8. None of the above

Report `UNKNOWN_BOTTLENECK` with the top 10 CUDA kernels, top memcpy categories, and
microbench scaling table. Do not invent a cause.

## Microbench scaling shape — the single most useful signal

| Pattern | What it means |
|---|---|
| `fps_per_batch` doubles from B=1→B=2, plateaus B≥4 | Compute-bound at B≥4. Decoder was the limit at B=1 (single NVDEC). |
| `fps_per_batch` plateaus immediately B=1→B=2 | Compute-bound from B=1. Decoder has plenty of headroom. |
| `fps_aggregate` scales linearly through B=16 | GPU has compute headroom; you can serve more streams. |
| `fps_aggregate` plateaus at some B*N | At the real ceiling for THIS combo of HW + codec + model. |
| All B fail (FPS=0) | Source/streammux can't fill the batch — pipeline error or single-source-only with batch>1. |

Worked example (any small / cheap inference model on a file source): single-source per-batch
FPS caps at the single-NVDEC ceiling for the codec+resolution. With two parallel sources of
the same file, per-batch FPS jumps ~2× as the second NVDEC engages. Aggregate FPS plateaus
once all NVDEC engines are saturated. Per-batch *plateau* alongside *idle compute* (low SM%)
is the signature: decode-bound, not compute-bound. Heavier models (transformer detectors,
VLMs) flip the picture — same shape but compute saturates first.

## How the skill should report

After running `nsys stats` plus the microbench, produce a single block like:

```
Bound type: DECODE_BOUND (high confidence)

Evidence:
  - Microbench: fps_aggregate[B=1]/fps_aggregate[B=2] = <ratio>  (threshold < 0.55)
  - nvidia-smi dmon: decoder utilization peaks observed during steady state
  - (NVTX bonus, if present): nvstreammux:m_collectingBuffers > 40%

Action: add parallel sources, or lower input resolution / switch codec.
HW ceiling check (Stage 2):
  Detected GPU has <N_NVDEC> NVDECs × ~<per_unit_fps> fps for the source codec/res =
  ~<decode_ceiling> fps decode ceiling.
  Measured aggregate at the relevant B compared against that ceiling.

Max output for this pipeline on this hardware (R7):
  decode_ceiling / target_fps = <max_streams> streams at <target_fps> fps before NVDEC
  saturates. Adding more input streams beyond that hits the decode wall, not compute.
```

That single block answers all four user questions: bound type, fix, max output, and
already-applied configs.

## Cross-checking

Each bound type should be confirmed by **at least two** independent signals before reporting
high confidence. The decision tree above already encodes this — DECODE_BOUND, for example,
has three independent triggers. If only one signal fires and others contradict, report
`INCONCLUSIVE` with the contradicting evidence.

# Config Derivation Rules

Stage 4 closed-form rules. Inputs come from Stages 2 (HW) and 3 (inference plateau). Every
knob below is derived, not tuned.

## Inputs

- `plateau_batch` — from Stage 3 micro-benchmark (smallest batch where doubling B yields
  < 5% FPS gain).
- `N_streams` — user-requested stream count.
- `source_res = (W, H)` — e.g. (1920, 1080).
- `source_fps` — per stream, e.g. 30.
- `model_dims = (Wm, Hm)` — read from the model's nvinfer config (`infer-dims=3;H;W`).
  Works for detection / classification / segmentation models alike.
- `has_int8_calib` — bool; true if an `int8-calib-file` is present on disk.
- `hw_nvdec_count`, `hw_sm_count`, `hw_tops_int8`, `hw_bw_gb_s` — from Stage 2.

**Derived:**

```
final_batch = min(plateau_batch, N_streams)
```

## Rules

### R1 — nvstreammux

| Key | Value | Why |
|---|---|---|
| `batch-size` | `final_batch` | Matches what inference can consume in one tick. |
| `width` | `min(source_res.W, model_dims.Wm)` | No point upscaling before inference. |
| `height` | `min(source_res.H, model_dims.Hm)` | Same. |
| `batched-push-timeout` | `round(1e6 / source_fps)` µs | One full frame interval; waits long enough to fill a batch without starving at live sources. |
| `nvbuf-memory-type` | `0` | NVMM zero-copy. Anything else kills throughput. |
| `live-source` | `1` if RTSP/camera, else `0` | RTSP needs live-source behavior. |
| `enable-padding` | `0` | Padding wastes GPU time when input already matches model dims. |

### R2 — nvinfer (primary)

In `pgie_config.yml` / `config_infer_primary.txt`:

| Key | Value | Why |
|---|---|---|
| `batch-size` | `final_batch` | Must match streammux; mismatch silently caps FPS. |
| `network-mode` | `1` if `has_int8_calib` else `2` | INT8 is ~2× FP16 TOPS on Ampere/Ada/Hopper. FP16 as fallback; never FP32. |
| `interval` | `0` | Every frame. Only use `1` if user explicitly asked to skip frames. |
| `infer-dims` | `3;Hm;Wm` | Required for dynamic-shape ONNX; harmless for static. |
| `model-engine-file` | path to pre-built `.engine` | Do not rebuild during measurement — `trtBuilder*` would dominate the profile. |
| `int8-calib-file` | path | Only if `network-mode=1`. |

In the outer nvinfer element properties (when set via `pipeline.add(...)` or `gst-launch`):
just `config-file-path=<path-to-pgie_config.yml>`. Do not duplicate keys in both places.

### R3 — Decoder (`nvurisrcbin` / `nvmultiurisrcbin` / `nvv4l2decoder`)

| Key | Value | Why |
|---|---|---|
| `num-extra-surfaces` | `min(final_batch, 5)` | Buffer pool sized for the batch; 5 is a safe cap that doesn't waste GPU memory. |
| `cudadec-memtype` | `0` | NVMM device memory; matches `nvstreammux.nvbuf-memory-type=0`. |
| `file-loop` | `1` when using file sources for benchmarking | Keeps the pipeline saturated past the first file playthrough. |
| `max-batch-size` | `final_batch` | On `nvmultiurisrcbin` only; must equal streammux batch. |

### R4 — Tracker (if present)

In the outer `nvtracker` element properties:

| Key | Value | Why |
|---|---|---|
| `enable-batch-process` | `1` (in the linked YAML) | Batches track updates across streams. |
| `tracker-width` | `480` | Matches NVIDIA's max-perf reference config. |
| `tracker-height` | `288` | Same. |
| `ll-config-file` | `/opt/nvidia/deepstream/deepstream/samples/configs/deepstream-app/config_tracker_NvDCF_max_perf.yml` | NVIDIA-tuned perf preset (DS 9.0). |
| `ll-lib-file` | `/opt/nvidia/deepstream/deepstream/lib/libnvds_nvmultiobjecttracker.so` | Standard tracker lib. |

#### R4a — `nvinfer.interval` when a tracker is present

A tracker can carry object IDs across frames where inference did not run, so
when a tracker is in the pipeline you may safely raise `nvinfer.interval` from
`0` (every frame) to `1`, `2`, `…` (skip 1, 2, … frames between inferences),
multiplying inference capacity proportionally. Whether this is safe depends on
**whether the tracker actually keeps IDs alive across the gap** — that's a
property of the workload (object motion, occlusion frequency, frame rate),
not something a closed-form rule can pick.

The right signal is **tracker ID retention rate** measured at the tracker's
output:

```
retention(N→N+1) = |IDs in frame N+1 ∩ IDs in frame N| / |IDs in frame N|
```

Capture it by attaching a `BatchMetadataOperator` probe at the tracker's src
pad and tallying object IDs per frame across a steady-state window. Practical
thresholds:

| Retention | Interpretation | Recommendation |
|---|---|---|
| **≥ 99 %** | Tracker keeps virtually every object across consecutive frames; gaps are safe to fill in. | Try `interval=1` (halves inference compute), re-measure retention at the new interval. Step up to 2/3/… while retention holds ≥ 99 %. |
| 90 – 99 % | Some objects flicker or are reacquired with new IDs, but most persist. | `interval=1` is borderline; acceptable for many use cases but expect a few duplicate / re-issued IDs. Don't go higher. |
| **< 90 %** | A noticeable fraction of objects come out with new IDs each frame — tracker can't bridge the gaps reliably. | **Force `interval=0`.** Skipping frames will compound the ID churn; per-frame inference is required. |

Quick visual check without writing a probe: turn on `display-tracking-id=1`
on the tracker, render the OSD-overlaid output for ~5 seconds, and watch
whether the IDs labelled on objects stay stable or constantly change. Lots of
new IDs ⇒ retention is poor ⇒ keep `interval=0`.

This rule applies *only when a tracker is present*. Without a tracker,
`interval > 0` produces literal detection holes (no metadata between
inferences) and should never be used.

### R5 — Queues (only when present)

A queue between `source` and `pgie`:

| Key | Value | Why |
|---|---|---|
| `max-size-buffers` | `final_batch × 4` | Decode-side depth; keeps GPU fed when decode is bursty. |
| `max-size-bytes` | `0` (default) | Let buffer count rule. |
| `max-size-time` | `0` | Let buffer count rule. |

A queue feeding a Kafka/message branch:

| Key | Value | Why |
|---|---|---|
| `max-size-buffers` | `2` | Tiny. |
| `leaky` | `2` (downstream) | Drop oldest when broker is slow; never back-pressure the main chain. |

A queue feeding a file-sink / encoder branch:

| Key | Value | Why |
|---|---|---|
| `max-size-buffers` | `final_batch` | One-batch slack, no more (disk back-pressure is a real signal). |
| `leaky` | `0` | Don't drop; if the disk is slow we want to see it as a stall. |

### R6 — OSD / tiler / visible sinks

If the user asked for on-screen display or on-disk MP4: leave OSD and tiler enabled; they are
a known perf cost the user accepted. Do not re-tune their knobs in the MVP.

If the user did NOT ask for visible output: omit OSD + tiler entirely; use `fakesink
sync=False` as the only branch. This is the largest single perf delta available.

### R7 — Capacity report (max output)

After Stage 5 (E2E profile), compute the closed-form max-stream capacity for the user's
target FPS. Report this in plain English.

**Inputs:**
- `peak_per_frame_fps` — from the microbench plateau (Stage 3)
- `nvdec_decode_ceiling` — from the Stage 2 HW ceiling lookup
  (`NVDEC_count × per_unit_fps_for_codec_res`, table in `hw-ceiling-formulas.md`)
- `target_fps` — what the user wants per stream (default 30)
- `aggregate_bw_ceiling_gbps`, `bw_per_stream_gbps` — only relevant if Stage 5 flagged
  MEMORY_BW_BOUND

**Formula:**

```
max_streams_compute  = peak_per_frame_fps / target_fps
max_streams_decode   = nvdec_decode_ceiling / target_fps
max_streams_bw       = aggregate_bw_ceiling_gbps / bw_per_stream_gbps   # only if BW-bound
max_streams_overall  = min(max_streams_compute, max_streams_decode, max_streams_bw)
```

**The reported number is `max_streams_overall`** — the lowest of the three ceilings, since
that is the actual bottleneck. The skill should also state *which* ceiling is dominating, so
the user knows what to upgrade if they need more.

**Output template (skill should produce a block in this shape):**

```
Capacity (this hardware, this model, <precision>):
  Compute ceiling : <peak_per_frame_fps> fps / <target_fps> fps = <N_compute> streams
  Decode  ceiling : <nvdec_decode_ceiling> fps / <target_fps> fps = <N_decode> streams
  Memory  ceiling : (not bound)  OR  <N_bw> streams
  → Max output:    min(...) = <N_overall> streams  (<dominant>-limited)
```

**Bottleneck → remediation mapping** (the skill must include this section in the report,
keyed off which ceiling dominated):

| Dominant ceiling | What "more streams" requires |
|---|---|
| **COMPUTE_BOUND** | A larger / faster GPU (more SMs, higher clock, or newer architecture with higher TC throughput). Lowering precision (FP16→INT8) is the cheapest first move IF not already INT8. Shrinking the model or using `nvinfer.interval` to skip frames also unlocks capacity at the cost of accuracy / temporal resolution. |
| **DECODE_BOUND** | A GPU with more NVDEC engines (counts vary widely — datacenter cards usually have more). Lowering input resolution or switching codec (H265 is slightly cheaper to decode than its H264 equivalent on most hardware) also helps. Pre-decoding to disk is an option for offline workloads. |
| **MEMORY_BW_BOUND** | A GPU with higher memory bandwidth (HBM-class cards). INT8 weights help (halve weight bytes). Smaller model or lower input resolution shrinks the working set. |
| **TRACKER_BOUND** | Switch to the perf-tuned tracker preset (R4). If already on max-perf preset and still bound, drop tracker resolution further or swap tracker algorithm (NvDCF→IOU). |
| **SYNC_BOUND** | Investigate threading / blocking-sync; rarely fixed by hardware change. |

**The skill must explicitly say which ceiling is dominating** so the user knows whether
upgrading to a larger compute GPU or one with more NVDECs is the right purchase. The skill
cannot run on a hypothetical larger GPU itself — its job is to identify the bottleneck and
project the remediation.

## Sanity checks the skill must run after applying R1–R5

Before launching Stage 5:

1. `nvstreammux.batch-size == nvinfer.batch-size` — else BATCH_MISMATCH; fix.
2. `nvinfer.model-engine-file` exists on disk — else the first run will rebuild and pollute
   the profile. Pre-build:

   ```bash
   # Trigger engine build once outside profiling
   gst-launch-1.0 fakesrc num-buffers=10 ! nvinfer config-file-path=<path> ! fakesink
   ```
3. `nvbuf-memory-type == 0` everywhere it applies — else memcpy dominates.
4. If `network-mode=1`, the calibration file exists and matches the model architecture.
5. All queue `leaky` values match the rules above.

If any check fails, fix and re-run before profiling.

# Hardware ceilings from `nvidia-smi`

How to turn `nvidia-smi` output into theoretical ceilings the skill uses as "is my measured
FPS realistic?" reference in Stage 5. All numbers are upper bounds — real workloads typically
reach 30–70% of these.

## 1. Identity and memory

```bash
nvidia-smi --query-gpu=name,compute_cap,memory.total,memory.free,\
clocks.max.sm,clocks.max.memory \
--format=csv,noheader,nounits
```

Keys out of this:

- `name` — e.g. `NVIDIA L40S`, `NVIDIA A100-SXM4-80GB`, `Orin (nvgpu)`
- `compute_cap` — e.g. `8.9` (used below for ops-per-clock)
- `memory.total` — used to bound `batch-size × model-memory`
- `clocks.max.sm` (MHz) — feed into the compute ceiling
- `clocks.max.memory` (MHz) — feed into the memory-bandwidth ceiling

## 2. SM count (compute cap lookup)

`nvidia-smi` doesn't print SM count directly. Resolve from `compute_cap` + GPU name:

| Compute cap | SMs typical for card | Note |
|-------------|----------------------|------|
| 7.5 (T4) | 40 | Turing |
| 8.0 (A100 40/80GB SXM/PCIe) | 108 | Ampere |
| 8.6 (A10 / A40 / RTX 30xx) | 72 / 84 / varies | Ampere |
| 8.7 (Orin AGX) | 16 | Jetson Ampere |
| 8.9 (L4 / L40 / L40S / RTX 40xx) | 60 / 142 / 142 / varies | Ada |
| 9.0 (H100 SXM/PCIe) | 132 / 114 | Hopper |
| 10.0 (Thor) | TBD | Blackwell |
| 10.0 (B200 / GB200) | varies | Blackwell |

If uncertain, read from `/proc/driver/nvidia/gpus/0/information` or
`cudaDeviceGetAttribute(cudaDevAttrMultiProcessorCount)` via a tiny CUDA probe; for the MVP
the table above is enough.

## 3. NVDEC / NVENC count

`nvidia-smi` does not expose engine count. **Source of truth:** the NVIDIA Video
Encode/Decode Support Matrix at <https://developer.nvidia.com/video-encode-decode-support-matrix>
— the matrix lists per-GPU NVDEC/NVENC counts and per-codec support (YES/NO).
Re-verify the table when DS major versions change; NVIDIA updates the matrix periodically.

NVDEC counts (from the matrix as of 2026-04):

| GPU | NVDEC | NVDEC generation | Notes |
|---|---|---|---|
| V100 (Volta) | 1 | 3rd gen | H264 / H265 8-bit only |
| T4 (Turing) | 2 | 4th gen | adds AV1 8-bit decode |
| A2 (Ampere) | 1 | 5th gen | low-end Ampere |
| A10 (Ampere) | 2 | 5th gen | |
| A40 (Ampere) | 2 | 5th gen | |
| A100 (Ampere) | 5 | 4th gen | |
| RTX A6000 (Ampere) | 2 | 5th gen | workstation |
| L4 (Ada) | 4 | 5th gen | inference-tuned |
| L40 / L40S (Ada) | 3 | 5th gen | |
| H100 SXM/PCIe (Hopper) | 7 | 4th gen | |
| RTX 4090 (Ada consumer) | 1 | 5th gen | |
| RTX 5090 (Blackwell consumer) | 2 | 6th gen | adds H265 10/12-bit decode |
| B200 / GB200 (Blackwell datacenter) | 7 | 6th gen | |
| Orin AGX (Jetson Ampere) | 2 | 5th gen | |
| Thor / T5000 (Jetson Blackwell) | 2 | 6.1 gen | full codec support incl. H265 12-bit |

NVENC counts vary; consult the matrix when needed. Most pipelines do not use NVENC unless
they encode output for streaming/recording.

**Decode ceiling (fps) ≈ NVDEC_count × per-unit_fps_for_codec_res** using:

| Codec × resolution | fps per NVDEC (Turing+) | fps per NVDEC (Ada / Hopper / Blackwell) |
|---|---|---|
| H264 1080p | ~1000 | ~1400 |
| H265 1080p | ~900 | ~1300 |
| H264 4K | ~250 | ~350 |
| H265 4K | ~220 | ~340 |

> **Note on the per-NVDEC fps table.** NVIDIA's matrix only publishes codec support
> (YES/NO) and engine counts — **not per-codec fps**. The numbers above are
> conservative midpoints from informal benchmarks; real workloads typically realize
> 50–70% of these. The Stage 3 microbench overrides the table when they conflict —
> measurement is always authoritative.

Example formula application: a card with N NVDECs running H265 1080p decode has roughly
N × per_unit_fps total decode headroom. If the user wants K × 1080p30 streams (= K × 30 fps),
compare K × 30 to that headroom: well below ⇒ decode is not the wall, plenty above ⇒ decode
is the wall.

## 4. Compute ceiling (TOPS)

Closed form per precision:

```
TOPS_fp16  = SM_count × clock_GHz × ops_per_clock_fp16
TOPS_int8  = SM_count × clock_GHz × ops_per_clock_int8
```

`ops_per_clock` by compute cap, assuming Tensor Cores (the path DS/TRT actually uses):

| Compute cap | FP16 (TC) ops/clock/SM | INT8 (TC) ops/clock/SM |
|---|---|---|
| 7.5 (T4) | 512 | 1024 |
| 8.0 (A100) | 1024 | 2048 |
| 8.6 / 8.7 (A10/A40/Orin) | 512 | 1024 |
| 8.9 (L4/L40/L40S) | 1024 | 2048 |
| 9.0 (H100) | 2048 | 4096 |
| 10.0 (B200/GB200) | 4096 | 8192 |

Example for L40S at 2.52 GHz max SM clock, 142 SMs, INT8:
`142 × 2.52 × 2048 ≈ 733,000 Gops ≈ 733 TOPS` (matches the L40S datasheet ~733 TOPS INT8
sparse or ~367 TOPS dense — the formula above is the dense number).

Use as: peak inference FPS ≤ `TOPS_at_precision / (model_GOPs × batch_efficiency)`, where
`batch_efficiency` ≈ 0.4 for batch=1, ≈ 0.7 for batch=16 on common detectors. This is why
Stage 3's micro-benchmark finds a plateau.

## 5. Memory-bandwidth ceiling (GB/s)

```
BW_GB_s = 2 × (mem_clock_MHz × 1e6) × (bus_width_bits / 8) / 1e9
```

`bus_width_bits` from the card datasheet (or compute cap + GPU name):

| GPU | Bus width (bits) |
|---|---|
| T4 | 256 |
| A100 (HBM2) | 5120 |
| A10 | 384 |
| L4 | 192 |
| L40 / L40S | 384 |
| H100 (HBM3) | 5120 |
| Orin AGX (LPDDR5) | 256 |
| Thor (LPDDR5X) | 256 |
| B200 (HBM3e) | 8192 |

Example: L40S at 9001 MHz GDDR6 mem clock × 384 bits → 2 × 9e9 × 48 B = 864 GB/s (matches
datasheet).

Rule of thumb: if the model's weight bytes × inference fps > 0.7 × BW, it's memory-bound;
INT8 weights help more than batching. `nsys stats --report gpu_metric_gpu_util_sum` reports
DRAM throughput directly in Stage 5.

## 6. PCIe (memcpy) ceiling

```bash
nvidia-smi --query-gpu=pcie.link.gen.current,pcie.link.width.current \
--format=csv,noheader,nounits
```

Gen/width → practical H2D bandwidth:

| Gen × width | Theoretical (GB/s) | Practical (~80%) |
|---|---|---|
| Gen3 x16 | 15.75 | 12 |
| Gen4 x16 | 31.5 | 25 |
| Gen5 x16 | 63 | 50 |

Only matters if NVMM zero-copy is broken (H2D/D2H memcpy should be <5% of wall time in a
correct DS pipeline). If Stage 5 shows memcpy >15%, check the pipeline for a CPU-memory
`videoconvert` and set `nvbuf-memory-type=0` on the source.

## Quick helper snippet

A one-liner the skill can drop into the report:

```bash
nvidia-smi --query-gpu=name,compute_cap,memory.total,memory.free,\
clocks.max.sm,clocks.max.memory,utilization.decoder,utilization.encoder,\
pcie.link.gen.current,pcie.link.width.current \
--format=csv,noheader
```

Combined with the tables above, this gives all four ceilings (decode, compute, memory
bandwidth, memcpy) with no additional probing.

# `nsys` CLI Recipes

Everything in this skill is captured and extracted with `nsys` on the terminal. No GUI. No
Nsight Lens. If `nsys` is not on PATH, the skill cannot run — install it first
(`apt install nsight-systems-<version>` or use a DS container that ships with it).

## Capture — Stage 3 (inference-only micro-benchmark)

Same flags as Stage 5, short duration because the micro-bench only needs steady-state FPS.

```bash
TS=$(date +%Y%m%d_%H%M%S)
OUT=/tmp/ds_microbench_B${BATCH}_${TS}

nsys profile \
  --trace=cuda,nvtx \
  --force-overwrite=true \
  --duration=20 \
  --output=${OUT} \
  <inference-only-pipeline-launch>
```

Then read FPS either from the pipeline's own `measure_fps_probe` stdout, or from:

```bash
nsys stats --report nvtx_sum --format csv ${OUT}.nsys-rep \
  | awk -F, '$3 ~ /nvinfer/ {print $0}'
```

The `Instances / duration` column of the `nvinfer` row gives per-batch inference rate →
multiply by `BATCH` for aggregate FPS.

## Capture — Stage 5 (E2E)

```bash
TS=$(date +%Y%m%d_%H%M%S)
OUT=/tmp/ds_e2e_${TS}

nsys profile \
  --trace=cuda,nvtx,osrt \
  --gpu-metrics-devices=all \
  --cuda-memory-usage=true \
  --force-overwrite=true \
  --duration=30 \
  --output=${OUT} \
  <full-pipeline-launch>
```

Flag-by-flag reason:

| Flag | Purpose |
|---|---|
| `--trace=cuda,nvtx,osrt` | CUDA kernels + NVTX ranges from DS plugins + OS-runtime (for CPU stall diagnosis). |
| `--gpu-metrics-devices=all` | SM occupancy, DRAM throughput, tensor-core utilization — essential for "compute- vs memory-bound" diagnosis. Requires `nsys` 2023.3+. |
| `--cuda-memory-usage=true` | Populates memcpy-byte totals so the memcpy report below has data. |
| `--force-overwrite=true` | Idempotent reruns. |
| `--duration=30` | 30 s of steady state; drop if the launch command terminates on its own within that window. |

If running inside Docker and the DS app is already running as a PID in another container,
`--attach=<pid>` works instead of a launch command (`docker exec <ctr> nsys profile ...
--attach=<pid>`).

## Extract — per-plugin NVTX time

This is the headline table in the Stage 5 report. Each DS plugin emits an NVTX range; sum
them.

```bash
nsys stats --report nvtx_sum --format csv ${OUT}.nsys-rep > /tmp/nvtx.csv

# Top 10 by total GPU time
awk -F, 'NR>1 {print $0}' /tmp/nvtx.csv | sort -t, -k5 -nr | head -10
```

Columns (approx, order may shift by `nsys` version):

1. `Range`
2. `Instances`
3. `Total Time (ns)`
4. `Avg (ns)`
5. `Med (ns)`
6. `Min`, `Max`, `StdDev`

Use `Total Time (ns)` as the share-of-wall-time measure. Normalize by `--duration` × 1e9.

## Extract — top CUDA kernels

Useful for confirming inference is the hot path (not a preproc or resize kernel).

```bash
nsys stats --report cuda_gpu_kern_sum --format csv ${OUT}.nsys-rep | head -15
```

What to look for: TRT kernels dominate (names like `trt_volta_int8_i8816cudnn_*`,
`sm80_xmma_*`, `cask_*`). If a `nppiResize` or `cudaMemcpy*` kernel is in the top 3, NVMM or
preproc is broken — but re-tuning non-NVTX helpers is out of scope this MVP; just report.

## Extract — memcpy totals

```bash
nsys stats --report cuda_gpu_mem_time_sum --format csv ${OUT}.nsys-rep
```

Expected: H2D + D2H total < 5% of wall time for a correct DS pipeline. If > 15%, something
has broken NVMM (usually a CPU `videoconvert` crept in, or `nvbuf-memory-type` isn't 0).

## Extract — GPU utilization / DRAM throughput

Requires the `--gpu-metrics-devices=all` capture flag.

```bash
nsys stats --report gpu_metric_gpu_util_sum --format csv ${OUT}.nsys-rep
```

Report keys:

- **SM Active (%)** — average SM activity. > 80% during steady state = GPU-bound (good at
  peak FPS).
- **DRAM Throughput (%)** — relative to the card's peak. > 85% = memory-bound.
- **Tensor Active (%)** — tensor core engagement. Low on an FP16/INT8 model means TRT didn't
  pick TC kernels; re-export the engine.

## Extract — OS-runtime (CPU stalls)

Optional; useful when the profile shows GPU-idle bubbles.

```bash
nsys stats --report osrt_sum --format csv ${OUT}.nsys-rep | head -15
```

Big `pthread_cond_wait` or `poll` times on pgie threads = upstream starvation (increase
queue depth or check decoder). Big `cudaStreamSynchronize` on CPU = GPU done but pipeline
waiting — usually fine on dGPU.

## Exporting CSV for downstream tooling (optional)

If a parent workflow wants to ingest the data:

```bash
# Full sqlite export (all tables — same as what Nsight Lens would read)
nsys export --type sqlite --output ${OUT}.sqlite ${OUT}.nsys-rep

# SQL query for top NVTX
sqlite3 ${OUT}.sqlite <<'SQL'
SELECT s.value AS plugin,
       COUNT(*) AS batches,
       SUM("end" - "start") AS total_ns
FROM   NVTX_EVENTS n
JOIN   StringIds s ON s.id = n.text
GROUP  BY n.text
ORDER  BY total_ns DESC
LIMIT  10;
SQL
```

The skill's Stage 5 report uses `nsys stats` directly; the sqlite path is a fallback for
environments where `nsys stats` is unavailable.

## Nothing-works fallback

If `nsys profile` fails (e.g. locked-down container without `perf_event_paranoid < 2`):

1. `sudo sysctl -w kernel.perf_event_paranoid=1` — or add `--privileged` to the container.
2. If still failing, capture with `--trace=cuda,nvtx` only (drop `osrt` and
   `--gpu-metrics-devices=all`). Less info, but at least the per-plugin NVTX table survives.
3. If `nsys` itself is unavailable: skip to a DS-native-only path using
   `measure_fps_probe` + Prometheus counters; **but report to the user that there is no
   Nsight trace and the plugin-level breakdown will be missing**. Do not pretend a DS-only
   measurement is a profile.

# NVTX Coverage

Which DeepStream plugins emit NVTX ranges that `nsys profile --trace=nvtx` will capture.

> **Always verify with a real probe.** This skill assumes DS 9.0 from the
> `nvcr.io/nvidia/deepstream:9.0-triton-multiarch` container. Avoid the `9.0-samples-multiarch`
> variant — it strips the CUPTI NVTX injector, producing empty per-plugin NVTX traces even
> though the plugin source code clearly contains `nvtxDomainRangePushEx` calls.

## The verification one-liner

Always run this against the user's exact pipeline before relying on any NVTX-derived
diagnosis:

```bash
nsys profile --trace=cuda,nvtx --duration=10 --output=/tmp/nvtx_probe <pipeline-launch>
nsys stats --force-export=true --report nvtx_sum --format csv /tmp/nvtx_probe.nsys-rep \
  | tail -n +6 | awk -F, '$2+0 > 0 {print $NF}' \
  | sed -E 's/\(Frame=[0-9]+\)//g; s/\(Batch=[0-9]+\)//g; s/batch_num=[0-9]+//g; s/UID=[0-9]+/UID=N/g' \
  | sort -u
```

Any DS plugin element in the user's pipeline that does NOT appear in that output is
**uninstrumented for THIS build** — even if the table below labels it COVERED.

## What to look for in the verification output

When DS plugin NVTX is working in DS 9.0 `triton-multiarch`, you should see ranges like:

| Range pattern | Emitted by | Verified in 9.0-triton-multiarch |
|---|---|---|
| `GstNvInfer: UID=N:buffer_process / queueInput / convert_buf / dequeueOutputAndAttachMeta` | `nvinfer` (named domain `GstNvInfer: UID=N`) | ✅ |
| `TensorRT:<layer-name>` (50+ ranges) | TensorRT runtime inside nvinfer | ✅ |
| `:nvdsosdN_(Frame=N)` | `nvdsosd` / `nvosdbin` | ✅ |
| `:m_collectingBuffers(Batch=N)`, `:m_acquireBufferFromPool(Batch=N)` | `nvstreammux` | ❌ **not emitted** |
| (anything from `nvurisrcbin`, `nvv4l2decoder`, `nvvideoconvert`) | closed-source binary plugins | ❌ **not emitted** |
| `NvDsTracker*` | `nvtracker` | not yet verified — probe to confirm |
| `gst_nvdspreprocess_*` | `nvdspreprocess` | not yet verified — probe to confirm |

If the verification probe returns nothing from your pipeline's actual hot-path plugins,
fall back to non-NVTX signals (see [boundedness-rules.md](boundedness-rules.md)).

## Practical implication

**Decode/source-side bottleneck diagnosis on DS 9.0 cannot rely on NVTX.** Binary
plugins (decoder, urisrcbin, videoconvert) emit nothing, and the streammux
`m_collectingBuffers` range is not reliably emitted. Use the substitute signals listed
below in [Diagnosing the gap when decoder/source NVTX is missing](#diagnosing-the-gap-when-decodersource-nvtx-is-missing).

For inference-side diagnosis (what nvinfer is doing per batch, which layers dominate, OSD
overhead) NVTX is plenty informative on DS 9.0.

## DS plugin NVTX status — observed variability

NVIDIA-shipped plugins fall into three buckets in practice. The bucket a plugin lands in
depends on the container variant (always use `9.0-triton-multiarch`; `samples-multiarch`
strips the injector and breaks NVTX entirely) and on whether NVIDIA chose to compile NVTX
calls into closed-source plugin binaries.

| Plugin | Source ships with NVTX calls? | Range visible in trace (varies by build) |
|---|---|---|
| `nvinfer` | Yes (`nvtxDomainRangePushEx` in shipped source) | Often visible; sometimes hidden when nsys can't intercept named-domain calls in dlopen'd libs |
| `nvinferserver` | Yes (similar pattern) | Same — varies |
| `nvstreammux` (binary) | Source not shipped, ranges have appeared in some builds | Often visible |
| `nvdsosd` (source shipped) | Yes | Often visible |
| `nvtracker` (binary) | Source not shipped | Sometimes visible (older builds emitted `NvDsTracker*`) |
| `nvdspreprocess` (source shipped) | Yes | Often visible |
| `nvdsanalytics` (source shipped) | Yes | Often visible |
| `nvurisrcbin` / `nvmultiurisrcbin` (binary) | Source not shipped | **Often NOT visible** in our probes |
| `nvv4l2decoder` / `nvv4l2h264enc` / `nvv4l2h265enc` (binary) | Source not shipped | **Often NOT visible** in our probes |
| `nvvideoconvert` (binary) | Source not shipped | **Often NOT visible** in our probes |
| `nvmultistreamtiler` (binary) | Source not shipped | Sometimes visible |
| `nvmsgconv` / `nvmsgbroker` (source shipped) | Yes | Often visible |

The shipped-source set lives under `/opt/nvidia/deepstream/deepstream/sources/gst-plugins/`
inside any DS container — that directory is the ground truth for what NVIDIA chose to
open-source. The closed-source binary plugins (`nvv4l2decoder`, `nvv4l2*enc`,
`nvurisrcbin`/`nvmultiurisrcbin`, `nvvideoconvert`, `nvtracker`, `nvstreammux`/the new
`nvmultistream2` library) require trusting NVIDIA's NVTX choices in each release.

## GStreamer-core elements — never instrumented

These do NOT emit DS-style NVTX:

`filesrc`, `filesink`, `fakesink`, `fakesrc`, `appsink`, `appsrc`, `qtdemux`, `qtmux`,
`h264parse`, `h265parse`, `aacparse`, `queue`, `queue2`, `tee`, `capsfilter`,
`videoconvert`/`videoscale` (CPU variants — avoid in DS pipelines).

The skill never relies on these for diagnosis.

## Diagnosing the gap when decoder/source NVTX is missing

If the verification probe shows the inference-path plugins emit NVTX but
decoder/source-bin/converter do NOT, you cannot read decoder time directly. Use these
substitute signals (all encoded as decision rules in
[boundedness-rules.md](boundedness-rules.md)):

1. **`nvstreammux:m_collectingBuffers` share** (when streammux NVTX IS captured) — > 40% of
   NVTX time means streammux is *waiting for upstream*, i.e. source/decoder bound.

2. **Microbench scaling shape (Stage 3)** — *always works*, no NVTX needed. Per-batch FPS
   doubling from B=1 to B=2 with parallel sources is the gold-standard signal that the
   decoder was the limit at B=1.

3. **`nvidia-smi dmon -s u` during the run** — `dec` column near 100% confirms the NVDEC
   engines are saturated. Sample at a higher rate than 1 Hz (`-d 1` with `-c N`) for short
   bursty workloads — single-frame decode at 1080p H264 takes only ~1 ms, so low-frequency
   sampling underreports.

These three substitute for the missing per-plugin NVTX coverage on the source/decoder side.

## When the user's pipeline includes a custom plugin

Any element whose name starts with something other than the standard prefixes (`nv*`,
`queue`, `tee`, `caps`, `h264`, `h265`, `file`, `fake`, `app`, `qt`, `video`) is third-party
and almost certainly UNINSTRUMENTED. The verification probe will confirm. The skill cannot
diagnose its internal cost from NVTX. Either:

1. **Auto-inject NVTX** by wrapping the element with a buffer probe that calls
   `nvtxRangePush/Pop` on each `chain` callback (out of scope for the MVP — listed as future
   work in `SKILL.md`).
2. **Replace with an instrumented equivalent** if one exists.
3. **Use indirect signals** (microbench scaling, nvidia-smi dmon, CUDA kernel mix) and
   report the custom plugin as "not directly measurable" in the final report.

## Bottom line

NVTX coverage is **partial and version-specific**. The skill's measurement strategy must
NOT depend on it. Per-plugin NVTX is a bonus when present (gives sharper bottleneck
attribution); the skill's primary signals are CUDA kernel summaries, memcpy summaries,
microbench scaling shape, and `nvidia-smi dmon` — all of which work regardless of NVTX
coverage.

#!/usr/bin/env python3

# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Stage 5 capacity report — implements R7 from `references/config-derivation-rules.md`.

Reads the microbench scaling CSV, optionally a sampled `nvidia-smi dmon` trace, plus a
fresh `nvidia-smi --query-gpu=...` snapshot, then prints:

  1. Bound type. This script classifies into one of:
       DECODE_BOUND, COMPUTE_BOUND, MEMORY_BW_BOUND, INCONCLUSIVE, UNKNOWN_BOTTLENECK
     The other types in `references/boundedness-rules.md` (ENGINE_BUILD_CONTAMINATED,
     MEMCPY_BOUND, TRACKER_BOUND, SYNC_BOUND) require inputs this script does not take —
     `nsys stats --report cuda_api_sum` for engine-rebuild contamination, the
     `cuda_gpu_mem_time_sum` for memcpy share, and per-element FrameCounter probe FPS
     for tracker / sync — and are surfaced by the skill at the prompt layer reading the
     same trace, not by this script.
  2. Closed-form max-streams capacity at the user's target FPS.
  3. Bottleneck → remediation (compute vs decode vs BW) in plain language.

Usage:
    python3 capacity_report.py \
        --microbench-csv /tmp/microbench_results.csv \
        --target-fps 30 \
        [--dmon-csv /tmp/dmon.txt] \
        [--codec h264] [--source-res 1080p]

The script does NOT need to run on the host that captured the data — it is pure logic.
But if `--dmon-csv` is omitted, it will run `nvidia-smi --query-gpu=...` itself to read
the host GPU model and use the codec/res defaults to estimate ceilings.
"""
import argparse
import csv
import json
import shutil
import subprocess
import sys
from pathlib import Path


# Per-NVDEC throughput estimates (frames/sec) — idealized; real workloads often hit ~70%.
# Lookup is by (architecture-bucket, codec, resolution).
# Per-NVDEC throughput (frames/sec) — empirical estimates, NOT from NVIDIA's matrix.
# NVIDIA's video-encode-decode-support-matrix at
#   https://developer.nvidia.com/video-encode-decode-support-matrix
# publishes NVDEC counts and codec support (YES/NO) but does NOT publish per-codec fps.
# Numbers below are conservative midpoints from informal benchmarks; real workloads
# typically realize 50-70% of these. Stage 3 microbench overrides when they conflict.
_NVDEC_FPS = {
    # (architecture-bucket, codec, res): fps_per_unit
    ("turing_or_older", "h264", "1080p"): 1000,
    ("turing_or_older", "h265", "1080p"): 900,
    ("turing_or_older", "h264", "4k"): 250,
    ("turing_or_older", "h265", "4k"): 220,
    ("ada_hopper_blackwell", "h264", "1080p"): 1400,
    ("ada_hopper_blackwell", "h265", "1080p"): 1300,
    ("ada_hopper_blackwell", "h264", "4k"): 350,
    ("ada_hopper_blackwell", "h265", "4k"): 340,
}

# NVDEC count by GPU name (substring match — ORDER MATTERS, longer/more-specific first
# so that e.g. "L40S" matches before "L4", "A100" before "A10", "GB200" before "B200").
#
# Source: NVIDIA Video Encode/Decode Support Matrix
#   https://developer.nvidia.com/video-encode-decode-support-matrix
# (NVDEC engine counts column; matrix updated periodically — re-verify when DS major
# version changes. Counts below reflect the matrix as of skill review date in
# `SKILL.md`'s frontmatter.)
_NVDEC_COUNT = [
    # Workstation / Quadro
    ("RTX A6000", 2),
    ("RTX 5090", 2),    # Blackwell consumer
    ("RTX 4090", 1),    # Ada consumer
    # Datacenter — Blackwell
    ("GB200", 7),
    ("B200", 7),
    # Datacenter — Hopper
    ("H100", 7),
    # Datacenter — Ada
    ("L40S", 3),
    ("L40", 3),
    ("L4", 4),
    # Datacenter — Ampere
    ("A100", 5),
    ("A6000", 2),       # plain "A6000" without RTX prefix
    ("A40", 2),
    ("A10", 2),
    ("A2", 1),
    # Datacenter — Turing
    ("T4", 2),
    # Datacenter — Volta
    ("V100", 1),
    # Jetson
    ("Thor", 2),        # Jetson T5000 / Thor — 2 NVDECs per matrix
    ("Orin", 2),        # AGX Orin
]


def _arch_bucket_from_compute_cap(cc: str) -> str:
    """Map compute capability to NVDEC throughput bucket."""
    try:
        major = int(str(cc).split(".")[0])
    except (ValueError, TypeError):
        return "ada_hopper_blackwell"
    return "ada_hopper_blackwell" if major >= 8 else "turing_or_older"


def _nvdec_count_for_gpu(name: str) -> int:
    for needle, n in _NVDEC_COUNT:
        if needle.lower() in name.lower():
            return n
    return 2  # safe default


def query_gpu(gpu_id: int = 0) -> dict:
    """Read identity + clocks for ONE GPU via ``nvidia-smi -i <gpu_id>``.

    The skill profiles a single inference + decode pipeline at a time. In
    DeepStream each plugin can take its own ``gpu-id`` (``nvinfer``,
    ``nvstreammux``, ``nvtracker``, ``nvurisrcbin`` / ``nvv4l2decoder``,
    sinks, etc.), but the inference and source-side decode for a given
    pipeline almost always run on the same GPU — the one named in the
    nvinfer config (default ``gpu-id: 0``). Pass that index as ``gpu_id``
    so the report describes the actual GPU the pipeline is bound to,
    not just GPU 0.

    Returns ``{}`` if ``nvidia-smi`` is unavailable or the GPU index
    doesn't exist on the host.
    """
    if not shutil.which("nvidia-smi"):
        return {}
    out = subprocess.run(
        [
            "nvidia-smi",
            "-i", str(gpu_id),
            "--query-gpu=name,compute_cap,memory.total,memory.free,"
            "clocks.max.sm,clocks.max.memory,pcie.link.gen.current,"
            "pcie.link.width.current,driver_version",
            "--format=csv,noheader,nounits",
        ],
        capture_output=True, text=True, check=False,
    )
    if out.returncode != 0:
        return {}
    line = out.stdout.strip().splitlines()
    if not line:
        return {}
    keys = ("name", "compute_cap", "mem_total_mib", "mem_free_mib",
            "sm_clock_mhz", "mem_clock_mhz", "pcie_gen", "pcie_width",
            "driver")
    vals = [v.strip() for v in line[0].split(",")]
    return dict(zip(keys, vals))


def parse_microbench(path: Path) -> list[dict]:
    """Parse microbench CSV.

    Returns a list of dicts, each with keys:
      - B (int): the batch size / parallel-source count for that row
      - fps_aggregate (float): total frames/sec across all sources

    Accepts both schemas:
      - current: ``B,fps_aggregate,fps_per_stream`` (fps_per_stream is read but
        not retained — derive it as ``fps_aggregate / B`` if needed downstream)
      - legacy:  ``B,fps_per_batch,fps_aggregate`` (where ``fps_per_batch`` was
        already aggregate fps — see the bug fix in microbench.sh)
    """
    rows: list[dict] = []
    with open(path) as f:
        reader = csv.DictReader(f)
        for r in reader:
            try:
                B = int(r.get("B", 0))
                # Accept either current schema (fps_aggregate, fps_per_stream)
                # or the legacy (buggy) one (fps_per_batch, fps_aggregate)
                if "fps_aggregate" in r and "fps_per_stream" in r:
                    fps_agg = float(r["fps_aggregate"])
                else:  # legacy — `fps_per_batch` was already aggregate fps
                    fps_agg = float(r.get("fps_per_batch", r.get("fps_aggregate", 0)))
                rows.append({"B": B, "fps_aggregate": fps_agg})
            except (ValueError, TypeError):
                continue
    return rows


def parse_dmon(path: Path, gpu_id: int | None = None) -> dict:
    """Parse ``nvidia-smi dmon`` output.

    Robust to whichever ``-s <flags>`` selector the user captured with:

    * ``-s u``  → ``# gpu  sm  mem  enc  dec  jpg  ofa``
    * ``-s m``  → ``# gpu  fb  bar1  ccpm``
    * ``-s mu`` → ``# gpu  fb  bar1  ccpm  sm  mem  enc  dec  jpg  ofa``

    The first ``# gpu …`` header line is parsed to build a column-name → index
    map; data rows then use named lookups so positions don't matter. If the
    capture lacks any of ``sm``/``mem``/``dec``, the corresponding max stays
    at 0 and the missing-signal lands as INCONCLUSIVE in ``classify_bound``
    rather than getting a wrong reading from a fixed column index.

    Returns ``{"sm_max", "mem_max", "dec_max"}``. When ``gpu_id`` is given,
    only samples for that GPU index are considered (first column of each
    data row is the GPU index). When ``gpu_id`` is ``None``, samples from
    every GPU in the log are aggregated.
    """
    sm_max = mem_max = dec_max = 0
    col_idx: dict[str, int] = {}
    with open(path) as f:
        for line in f:
            stripped = line.strip()
            if not stripped:
                continue
            # The FIRST header row of nvidia-smi dmon is `# gpu <cols...>`.
            # The second header row is units (`# Idx % % ...`) — ignore.
            if stripped.startswith("#"):
                tokens = stripped.lstrip("#").split()
                if tokens and tokens[0].lower() == "gpu" and not col_idx:
                    # Map column name → index (case-insensitive).
                    col_idx = {tok.lower(): i for i, tok in enumerate(tokens)}
                continue
            if not col_idx:
                # No header seen yet — can't tell which column is which.
                continue
            parts = stripped.split()
            try:
                row_gpu = int(parts[col_idx["gpu"]])
            except (ValueError, KeyError, IndexError):
                continue
            if gpu_id is not None and row_gpu != gpu_id:
                continue
            for name, var in (("sm", "sm_max"), ("mem", "mem_max"), ("dec", "dec_max")):
                if name not in col_idx:
                    continue
                try:
                    val = int(parts[col_idx[name]])
                except (ValueError, IndexError):
                    continue
                if var == "sm_max":
                    sm_max = max(sm_max, val)
                elif var == "mem_max":
                    mem_max = max(mem_max, val)
                else:
                    dec_max = max(dec_max, val)
    return {"sm_max": sm_max, "mem_max": mem_max, "dec_max": dec_max}


def classify_bound(microbench: list[dict], dmon: dict, target_fps: int) -> dict:
    """Apply the boundedness-rules.md decision tree.

    Returns dict with `bound`, `confidence`, `evidence` (list of strings).
    `bound` is one of: DECODE_BOUND, COMPUTE_BOUND, MEMORY_BW_BOUND, INCONCLUSIVE,
    UNKNOWN_BOTTLENECK. The remaining types listed in `boundedness-rules.md`
    (ENGINE_BUILD_CONTAMINATED, MEMCPY_BOUND, TRACKER_BOUND, SYNC_BOUND) need
    inputs this function doesn't take and are surfaced by the skill at the prompt
    layer rather than here.
    """
    by_b = {r["B"]: r["fps_aggregate"] for r in microbench}
    evidence: list[str] = []
    bound = "UNKNOWN_BOTTLENECK"
    conf = "low"

    # --- Decode-bound checks ---
    # Guard every division: if a microbench iteration produced fps==0
    # (pipeline failed to start, probe never fired, etc.) the row's
    # value is 0 in the CSV, which would otherwise crash with
    # ZeroDivisionError instead of being reported as inconclusive data.
    decode_signals = 0
    if 1 in by_b and 2 in by_b and by_b[1] > 0 and by_b[2] > 0:
        ratio = by_b[1] / by_b[2]
        evidence.append(
            f"microbench: fps[B=1]/fps[B=2] = {ratio:.2f}  "
            f"(< 0.55 ⇒ decode-limited at low B)"
        )
        if ratio < 0.55:
            decode_signals += 1
    elif 1 in by_b and 4 in by_b and by_b[1] > 0 and by_b[4] > 0:
        ratio = by_b[1] / by_b[4]
        evidence.append(
            f"microbench: fps[B=1]/fps[B=4] = {ratio:.2f}  "
            f"(< 0.30 ⇒ decode-limited at low B)"
        )
        if ratio < 0.30:
            decode_signals += 1

    # Plateau test: if fps stops growing past some B, that B's ceiling is the wall.
    Bs = sorted(by_b.keys())
    if len(Bs) >= 2:
        last_B = Bs[-1]
        max_fps = max(by_b.values())
        ceiling_fps = by_b[last_B]
        # Same zero-guard reasoning — if the highest-B run produced 0
        # fps, the ceiling is undefined and we just skip the plateau
        # check rather than crash on the relative-difference div.
        if ceiling_fps > 0:
            flat = all(
                abs(by_b[b] - ceiling_fps) / ceiling_fps < 0.05
                for b in Bs if b >= max(2, Bs[0])
            )
            if flat:
                evidence.append(
                    f"microbench: fps_aggregate is flat at ~{ceiling_fps:.0f} for B≥2  "
                    f"(ceiling reached)"
                )

    if dmon.get("dec_max", 0) >= 90:
        decode_signals += 1
        evidence.append(f"nvidia-smi dmon: NVDEC peak {dmon['dec_max']}% (≥90% ⇒ NVDEC saturated)")

    # --- Memory-BW-bound check ---
    mem_bw_bound = (dmon.get("mem_max", 0) >= 85
                    and dmon.get("sm_max", 0) < 70)
    if mem_bw_bound:
        evidence.append(
            f"nvidia-smi dmon: DRAM peak {dmon['mem_max']}%  "
            f"(>85% with SM<70% ⇒ memory-BW bound)"
        )

    # --- Compute-bound check ---
    compute_bound = (dmon.get("sm_max", 0) >= 70
                     and not mem_bw_bound
                     and decode_signals == 0)
    if compute_bound:
        evidence.append(
            f"nvidia-smi dmon: SM peak {dmon['sm_max']}% (≥70% with no decode/BW signals)"
        )

    # --- Verdict ---
    # Order MUST match the decision tree in references/boundedness-rules.md:
    # decode → memory-BW → compute → inconclusive.
    if decode_signals >= 2:
        bound, conf = "DECODE_BOUND", "high"
    elif decode_signals == 1:
        bound, conf = "DECODE_BOUND", "medium"
    elif mem_bw_bound:
        bound, conf = "MEMORY_BW_BOUND", "high"
    elif compute_bound:
        bound, conf = "COMPUTE_BOUND", "medium"
    elif by_b:
        bound, conf = "INCONCLUSIVE", "low"

    return {"bound": bound, "confidence": conf, "evidence": evidence}


def compute_capacity(microbench: list[dict], target_fps: int,
                     codec: str, source_res: str, gpu: dict,
                     bound: str = "UNKNOWN_BOTTLENECK") -> dict:
    """R7 closed form. The MEASURED peak FPS is authoritative — it already reflects
    whichever bottleneck dominated. The theoretical NVDEC ceiling is reported as a
    sanity-check comparison only.
    """
    by_b = {r["B"]: r["fps_aggregate"] for r in microbench}
    peak_fps = max(by_b.values()) if by_b else 0

    nvdec_count = _nvdec_count_for_gpu(gpu.get("name", "")) if gpu else 2
    arch = _arch_bucket_from_compute_cap(gpu.get("compute_cap", "")) if gpu else "ada_hopper_blackwell"
    per_unit = _NVDEC_FPS.get((arch, codec, source_res), 1000)
    decode_ceiling_theoretical = nvdec_count * per_unit

    # Authoritative: the measurement IS the practical ceiling.
    n_overall = int(peak_fps // target_fps) if target_fps > 0 else 0

    # Sanity: compare measured to theoretical decode (which is itself idealized).
    n_decode_theoretical = int(decode_ceiling_theoretical // target_fps) if target_fps > 0 else 0
    decode_realization = (peak_fps / decode_ceiling_theoretical
                          if decode_ceiling_theoretical else 0)

    dominant_map = {
        "DECODE_BOUND": "decode",
        "COMPUTE_BOUND": "compute",
        "MEMORY_BW_BOUND": "memory bandwidth",
    }
    dominant = dominant_map.get(bound, "unknown")

    return {
        "peak_fps_measured": peak_fps,
        "decode_ceiling_theoretical": decode_ceiling_theoretical,
        "decode_realization_pct": round(decode_realization * 100, 1),
        "n_overall": n_overall,
        "n_decode_theoretical": n_decode_theoretical,
        "dominant": dominant,
        "nvdec_count": nvdec_count,
        "per_nvdec_theoretical": per_unit,
        "arch_bucket": arch,
    }


_REMEDIATION = {
    "DECODE_BOUND": (
        "More decode capacity is needed to scale beyond this stream count. Options\n"
        "(ordered roughly cheapest-to-deepest):\n"
        "  • Lower input resolution — universal, works regardless of source codec.\n"
        "    e.g. 1080p → 720p roughly doubles per-NVDEC fps.\n"
        "  • Lower target FPS — also universal; halves the throughput needed.\n"
        "  • Codec-conditional (per NVDEC throughput table for Ada/Hopper/Blackwell,\n"
        "    1080p ≈ 1400 fps H264 vs 1300 fps H265 per engine):\n"
        "      - If source is H.265: switching to H.264 buys ~7% per-engine fps.\n"
        "      - If source is already H.264: no codec-level win possible — go to\n"
        "        the resolution / fps / hardware levers below.\n"
        "      - If source is AV1 / VP9 / MJPEG / other: throughput differs by\n"
        "        card. Consult NVIDIA's video-encode-decode-support-matrix\n"
        "        (https://developer.nvidia.com/video-encode-decode-support-matrix).\n"
        "        Note MJPEG uses NVJPEG, not NVDEC — different ceiling entirely.\n"
        "  • A GPU with more NVDEC engines (datacenter cards usually have more —\n"
        "    H100 / B200 ship with ~7 NVDECs vs typical workstation cards with 1-2).\n"
        "  • For offline / non-real-time, pre-decode to disk.\n"
        "  Adding a faster compute GPU will NOT help — compute is already underutilized."
    ),
    "COMPUTE_BOUND": (
        "More compute is needed to scale beyond this stream count. Options:\n"
        "  • A larger / newer-architecture GPU (more SMs, higher clocks, newer tensor\n"
        "    cores). The Stage 2 ceilings table compares architectures by ops/clock/SM.\n"
        "  • Lower precision: FP16 → INT8 (2× tensor-core throughput on Ampere/Ada/Hopper)\n"
        "    if not already INT8.\n"
        "  • Smaller model, or `nvinfer.interval=1` to skip every other frame (halves\n"
        "    compute load at half the temporal resolution).\n"
        "  Adding more NVDECs will NOT help — decoder is already underutilized."
    ),
    "MEMORY_BW_BOUND": (
        "Memory bandwidth is the wall. Options:\n"
        "  • A GPU with HBM memory (e.g. A100, H100, B200) has 3-5× the bandwidth of\n"
        "    GDDR-based cards.\n"
        "  • INT8 weights (halve weight bytes, smaller working set).\n"
        "  • Smaller / lower-resolution model."
    ),
    "INCONCLUSIVE": (
        "Could not classify the bottleneck with high confidence. Capture a longer trace,\n"
        "ensure --gpu-metrics-devices=all is enabled (needs CAP_SYS_ADMIN), and rerun the\n"
        "microbench with parallel sources to clearly separate decode vs compute."
    ),
    "UNKNOWN_BOTTLENECK": (
        "No bound type matched the decision rules. The pipeline is below the HW ceiling for\n"
        "no measured reason. Check: model accuracy at the configured precision, TRT engine\n"
        "build success, NVMM zero-copy across all elements (memcpy share <5%)."
    ),
}


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__,
                                 formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("--microbench-csv", required=True, type=Path,
                    help="Stage 3 output CSV (B,fps_aggregate,fps_per_stream)")
    ap.add_argument("--target-fps", type=int, default=30,
                    help="Per-stream FPS the user wants (default 30)")
    ap.add_argument("--dmon-csv", type=Path, default=None,
                    help="`nvidia-smi dmon -s mu` log captured during the run")
    ap.add_argument("--codec", default="h264", choices=["h264", "h265"])
    ap.add_argument("--source-res", default="1080p", choices=["1080p", "4k"])
    ap.add_argument("--gpu-id", type=int, default=0,
                    help="GPU index the pipeline is bound to (matches the "
                         "`gpu-id` set in your nvinfer config). Default 0. "
                         "In DS each plugin can have its own gpu-id; this "
                         "skill assumes the inference and source-decode plugins "
                         "are on the same GPU and reports for that one only.")
    ap.add_argument("--json", action="store_true",
                    help="Emit machine-readable JSON only")
    args = ap.parse_args()

    if not args.microbench_csv.is_file():
        print(f"error: microbench CSV not found: {args.microbench_csv}",
              file=sys.stderr)
        return 2

    microbench = parse_microbench(args.microbench_csv)
    if not microbench:
        print("error: no usable rows in microbench CSV", file=sys.stderr)
        return 2

    dmon = (parse_dmon(args.dmon_csv, gpu_id=args.gpu_id)
            if args.dmon_csv and args.dmon_csv.is_file() else {})
    gpu = query_gpu(gpu_id=args.gpu_id)
    classification = classify_bound(microbench, dmon, args.target_fps)
    capacity = compute_capacity(microbench, args.target_fps, args.codec,
                                args.source_res, gpu,
                                bound=classification["bound"])
    remediation = _REMEDIATION.get(classification["bound"],
                                   _REMEDIATION["UNKNOWN_BOTTLENECK"])

    payload = {
        "gpu": gpu,
        "microbench": microbench,
        "dmon": dmon,
        "classification": classification,
        "capacity": capacity,
        "target_fps": args.target_fps,
        "codec": args.codec,
        "source_res": args.source_res,
        "remediation": remediation,
    }

    if args.json:
        print(json.dumps(payload, indent=2))
        return 0

    # Plain-English report
    print("=" * 72)
    print("DeepStream Profiling — Capacity Report (R7)")
    print("=" * 72)
    if gpu:
        print(f"GPU       : id={args.gpu_id}, {gpu.get('name')}, "
              f"compute_cap {gpu.get('compute_cap')}, "
              f"{gpu.get('mem_total_mib')} MiB, driver {gpu.get('driver')}")
    else:
        print(f"GPU       : id={args.gpu_id} (nvidia-smi unavailable or index "
              f"not found; HW ceilings estimated from defaults)")
    print(f"Codec/res : {args.codec.upper()} {args.source_res}")
    print(f"Target    : {args.target_fps} fps per stream")
    print()
    print(f"Bound type: {classification['bound']}  (confidence: {classification['confidence']})")
    for line in classification["evidence"]:
        print(f"  - {line}")
    print()
    print("Capacity (R7 — measurement is authoritative):")
    print(f"  Peak measured FPS aggregate : {capacity['peak_fps_measured']:.0f}  "
          f"(this is the practical ceiling — already reflects the active bottleneck)")
    print(f"  Theoretical decode ceiling  : {capacity['decode_ceiling_theoretical']:.0f}  "
          f"({capacity['nvdec_count']} × ~{capacity['per_nvdec_theoretical']} fps "
          f"per NVDEC, {capacity['arch_bucket']})")
    print(f"  Decode realization          : {capacity['decode_realization_pct']:.0f}% of theoretical")
    print(f"  → MAX OVERALL               : {capacity['n_overall']} streams at "
          f"{args.target_fps} fps  ({capacity['dominant']}-limited per classification above)")
    if capacity['n_decode_theoretical'] > capacity['n_overall']:
        print(f"  Note: theoretical NVDEC table predicted up to "
              f"{capacity['n_decode_theoretical']} streams; measurement realized "
              f"{capacity['decode_realization_pct']:.0f}% of that — typical for real workloads.")
    print()
    print("Remediation:")
    for line in remediation.splitlines():
        print(f"  {line}")
    print()
    return 0


if __name__ == "__main__":
    sys.exit(main())

# Tests

Unit tests for `scripts/capacity_report.py`. They do **not** require a GPU or
DeepStream — pure-Python checks of the parser, classifier, and capacity-derivation logic.

## Run

See [`README.md`](../README.md#testing) in the skill root for the standard test command.

## What's covered

19 tests grouped by `TestCase` class:

| Class | Test | Behaviour |
|---|---|---|
| `TestParseMicrobench` | `test_current_schema` | Parses `B,fps_aggregate,fps_per_stream` rows correctly. |
| | `test_legacy_schema` | Falls back to the legacy `B,fps_per_batch,fps_aggregate` columns (treats `fps_per_batch` as the aggregate, since the legacy script emitted aggregate fps under that name). |
| | `test_skips_garbage_rows` | Ignores rows with non-numeric values without raising. |
| `TestParseDmon` | `test_extracts_maxes` | Extracts max(SM%), max(mem%), max(dec%) from a `nvidia-smi dmon -s mu` log. |
| | `test_filters_by_gpu_id` | When `gpu_id` is given, only rows for that GPU index are aggregated. |
| | `test_handles_empty` | Empty / header-only logs return zeros without raising. |
| `TestClassifyBound` | `test_decode_strong` | B=1→B=2 fps ratio < 0.55 + NVDEC peak ≥ 90% ⇒ DECODE_BOUND, high confidence. |
| | `test_compute_bound` | Per-batch FPS plateaus immediately + SM% high + DRAM% low ⇒ COMPUTE_BOUND. |
| | `test_memory_bw_bound` | DRAM% high with SM% low ⇒ MEMORY_BW_BOUND. |
| | `test_inconclusive` | No strong signals ⇒ INCONCLUSIVE. |
| | `test_zero_fps_no_division_error` | Microbench rows with `fps_aggregate=0` don't crash the classifier with `ZeroDivisionError`. |
| | `test_zero_ceiling_fps_no_division_error` | Highest-B row with fps=0 skips the plateau-flatness check rather than dividing by zero. |
| `TestComputeCapacity` | `test_measurement_is_authoritative` | `n_overall = peak_measured // target_fps` regardless of the theoretical NVDEC-table value; theoretical is reported alongside as a sanity check. |
| | `test_unknown_gpu_uses_safe_defaults` | When `nvidia-smi` is unavailable, fall back to NVDEC=2, ada_hopper_blackwell arch. |
| `TestArchBucketLookup` | `test_modern_caps` | Compute cap ≥ 8 → `ada_hopper_blackwell`. |
| | `test_old_caps` | Compute cap < 8 → `turing_or_older`. |
| | `test_unknown_defaults_to_modern` | Empty / non-numeric input → `ada_hopper_blackwell` (safe default). |
| `TestNvdecCountLookup` | `test_known_gpus` | Substring match on GPU name returns the correct NVDEC count for known cards (T4, A6000, A40, A100, V100, L4, L40S, H100, Orin). Specific names listed before generic ones (`L40S` before `L4`, `A100` before `A10`). |
| | `test_unknown_gpu_safe_default` | Unknown GPU name falls back to NVDEC=2. |

## When tests fail

The classifier returning the wrong bound type is usually the most informative failure —
review `references/boundedness-rules.md` for whether the trigger thresholds need adjusting,
or whether the test's input represents an edge case that the rules haven't yet codified.

# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Unit tests for capacity_report.py — no GPU/DeepStream required."""
import importlib.util
import sys
import tempfile
import unittest
from pathlib import Path
from textwrap import dedent

# Load capacity_report.py as a module without requiring a package install.
_HERE = Path(__file__).resolve().parent
_SCRIPT = _HERE.parent / "scripts" / "capacity_report.py"
_spec = importlib.util.spec_from_file_location("capacity_report", _SCRIPT)
cr = importlib.util.module_from_spec(_spec)
sys.modules["capacity_report"] = cr
_spec.loader.exec_module(cr)


def _write(tmp: Path, name: str, content: str) -> Path:
    p = tmp / name
    p.write_text(dedent(content).lstrip("\n"))
    return p


class TestParseMicrobench(unittest.TestCase):
    """parse_microbench handles current + legacy schema and skips garbage."""

    def test_current_schema(self):
        tmp = Path(tempfile.mkdtemp(prefix="capreport_"))
        csv = _write(tmp, "m.csv", """
            B,fps_aggregate,fps_per_stream
            1,769.04,769.04
            2,1493.16,746.58
            4,1498.00,374.50
        """)
        rows = cr.parse_microbench(csv)
        self.assertEqual(len(rows), 3)
        self.assertEqual(rows[0]["B"], 1)
        self.assertAlmostEqual(rows[0]["fps_aggregate"], 769.04, places=2)
        self.assertAlmostEqual(rows[2]["fps_aggregate"], 1498.00, places=2)

    def test_legacy_schema(self):
        # Legacy script emitted (B, fps_per_batch, fps_aggregate) where the
        # `fps_per_batch` column was actually aggregate fps (the bug we fixed).
        # parse_microbench must accept it: fps_per_batch -> fps_aggregate.
        tmp = Path(tempfile.mkdtemp(prefix="capreport_"))
        csv = _write(tmp, "m.csv", """
            B,fps_per_batch,fps_aggregate
            1,763.65,763.6
            2,1483.90,2967.8
        """)
        rows = cr.parse_microbench(csv)
        self.assertEqual(len(rows), 2)
        self.assertAlmostEqual(rows[0]["fps_aggregate"], 763.65, places=2)
        self.assertAlmostEqual(rows[1]["fps_aggregate"], 1483.90, places=2)

    def test_skips_garbage_rows(self):
        tmp = Path(tempfile.mkdtemp(prefix="capreport_"))
        csv = _write(tmp, "m.csv", """
            B,fps_aggregate,fps_per_stream
            1,769.04,769.04
            ?,not_a_number,oops
            4,1498.00,374.50
        """)
        rows = cr.parse_microbench(csv)
        self.assertEqual(len(rows), 2)


class TestParseDmon(unittest.TestCase):
    """parse_dmon extracts max(SM/mem/dec) from a `nvidia-smi dmon -s mu` log."""

    def test_extracts_maxes(self):
        tmp = Path(tempfile.mkdtemp(prefix="capreport_"))
        log = _write(tmp, "dmon.txt", """
            # gpu     fb   bar1   ccpm     sm    mem    enc    dec    jpg    ofa
            # Idx     MB     MB     MB      %      %      %      %      %      %
                0   100      0      0     30      5      0     20      0      0
                0   100      0      0     82     61      0    100      0      0
                0   100      0      0     45      3      0     50      0      0
        """)
        m = cr.parse_dmon(log)
        self.assertEqual(m["sm_max"], 82)
        self.assertEqual(m["mem_max"], 61)
        self.assertEqual(m["dec_max"], 100)

    def test_filters_by_gpu_id(self):
        # Multi-GPU log: GPU 0 has high values, GPU 1 has low values.
        # Filtering by gpu_id=1 must NOT pick up GPU 0's peaks.
        tmp = Path(tempfile.mkdtemp(prefix="capreport_"))
        log = _write(tmp, "dmon.txt", """
            # gpu     fb   bar1   ccpm     sm    mem    enc    dec    jpg    ofa
            # Idx     MB     MB     MB      %      %      %      %      %      %
                0   100      0      0     90     85      0    100      0      0
                1   100      0      0     20     10      0     30      0      0
                0   100      0      0     95     70      0     90      0      0
                1   100      0      0     25     15      0     40      0      0
        """)
        m = cr.parse_dmon(log, gpu_id=1)
        self.assertEqual(m["sm_max"], 25)
        self.assertEqual(m["mem_max"], 15)
        self.assertEqual(m["dec_max"], 40)
        # No filter -> aggregates across all GPUs (= max from either GPU)
        m_all = cr.parse_dmon(log)
        self.assertEqual(m_all["sm_max"], 95)
        self.assertEqual(m_all["dec_max"], 100)

    def test_handles_empty(self):
        tmp = Path(tempfile.mkdtemp(prefix="capreport_"))
        log = _write(tmp, "dmon.txt", "# no data\n")
        m = cr.parse_dmon(log)
        self.assertEqual(m, {"sm_max": 0, "mem_max": 0, "dec_max": 0})


class TestClassifyBound(unittest.TestCase):
    """classify_bound applies the boundedness-rules.md decision tree."""

    def test_decode_strong(self):
        # B=1 -> 763, B=2 -> 1493 (ratio 0.51 < 0.55) AND NVDEC at 100% =>
        # two independent decode signals => high confidence.
        microbench = [
            {"B": 1, "fps_aggregate": 763.0},
            {"B": 2, "fps_aggregate": 1493.0},
            {"B": 4, "fps_aggregate": 1498.0},
            {"B": 16, "fps_aggregate": 1486.0},
        ]
        dmon = {"sm_max": 46, "mem_max": 32, "dec_max": 100}
        out = cr.classify_bound(microbench, dmon, target_fps=30)
        self.assertEqual(out["bound"], "DECODE_BOUND")
        self.assertEqual(out["confidence"], "high")

    def test_compute_bound(self):
        # FPS plateaus immediately (compute is the limit even at low B);
        # SM at saturation, DRAM low, NVDEC barely engaged.
        microbench = [
            {"B": 1, "fps_aggregate": 1500.0},
            {"B": 2, "fps_aggregate": 1530.0},  # ratio 0.98 — no decode signal
            {"B": 4, "fps_aggregate": 1545.0},
            {"B": 8, "fps_aggregate": 1547.0},
        ]
        dmon = {"sm_max": 95, "mem_max": 40, "dec_max": 30}
        out = cr.classify_bound(microbench, dmon, target_fps=30)
        self.assertEqual(out["bound"], "COMPUTE_BOUND")

    def test_memory_bw_bound(self):
        # DRAM saturated, SM low — memory-BW wall.
        microbench = [
            {"B": 1, "fps_aggregate": 800.0},
            {"B": 2, "fps_aggregate": 820.0},
            {"B": 4, "fps_aggregate": 825.0},
        ]
        dmon = {"sm_max": 55, "mem_max": 92, "dec_max": 40}
        out = cr.classify_bound(microbench, dmon, target_fps=30)
        self.assertEqual(out["bound"], "MEMORY_BW_BOUND")
        self.assertEqual(out["confidence"], "high")

    def test_inconclusive(self):
        # No strong signals.
        microbench = [
            {"B": 1, "fps_aggregate": 500.0},
            {"B": 2, "fps_aggregate": 700.0},  # ratio 0.71 -> no decode
        ]
        dmon = {"sm_max": 50, "mem_max": 30, "dec_max": 50}
        out = cr.classify_bound(microbench, dmon, target_fps=30)
        self.assertEqual(out["bound"], "INCONCLUSIVE")

    def test_zero_fps_no_division_error(self):
        # If a microbench iteration produces fps=0 (pipeline failed to
        # start, probe never fired, etc.) the classifier must not crash
        # with ZeroDivisionError. It should fall through to UNKNOWN /
        # INCONCLUSIVE based on remaining signals.
        microbench = [
            {"B": 1, "fps_aggregate": 100.0},
            {"B": 2, "fps_aggregate": 0.0},   # bad row
            {"B": 4, "fps_aggregate": 0.0},   # bad row
        ]
        dmon = {"sm_max": 0, "mem_max": 0, "dec_max": 0}
        # Must not raise.
        out = cr.classify_bound(microbench, dmon, target_fps=30)
        self.assertIn(out["bound"], {"INCONCLUSIVE", "UNKNOWN_BOTTLENECK"})

    def test_zero_ceiling_fps_no_division_error(self):
        # Highest-B row has fps=0 — plateau check must not crash dividing
        # by ceiling_fps=0.
        microbench = [
            {"B": 1, "fps_aggregate": 800.0},
            {"B": 2, "fps_aggregate": 1500.0},
            {"B": 4, "fps_aggregate": 0.0},   # corrupt last row
        ]
        dmon = {"sm_max": 50, "mem_max": 30, "dec_max": 50}
        out = cr.classify_bound(microbench, dmon, target_fps=30)
        # Should still classify based on the B=1/B=2 ratio without crashing
        self.assertIn(out["bound"], {"DECODE_BOUND", "INCONCLUSIVE",
                                      "UNKNOWN_BOTTLENECK"})


class TestComputeCapacity(unittest.TestCase):
    """compute_capacity treats the measured peak as authoritative."""

    def test_measurement_is_authoritative(self):
        microbench = [
            {"B": 1, "fps_aggregate": 769.0},
            {"B": 2, "fps_aggregate": 1493.0},
            {"B": 4, "fps_aggregate": 1498.0},
        ]
        gpu = {"name": "RTX A6000", "compute_cap": "8.6"}
        out = cr.compute_capacity(microbench, target_fps=30, codec="h264",
                                  source_res="1080p", gpu=gpu,
                                  bound="DECODE_BOUND")
        # n_overall = floor(1498 / 30) = 49
        self.assertEqual(out["n_overall"], 49)
        # Theoretical decode ceiling reported alongside
        self.assertGreater(out["decode_ceiling_theoretical"], 0)
        # Realization < 100% — measurement realized only part of theoretical
        self.assertLess(out["decode_realization_pct"], 100)
        # Dominant ceiling propagated from bound
        self.assertEqual(out["dominant"], "decode")

    def test_unknown_gpu_uses_safe_defaults(self):
        microbench = [{"B": 1, "fps_aggregate": 100.0}]
        out = cr.compute_capacity(microbench, target_fps=30, codec="h264",
                                  source_res="1080p", gpu={},
                                  bound="UNKNOWN_BOTTLENECK")
        self.assertEqual(out["nvdec_count"], 2)  # safe default
        # Default arch bucket is the modern one
        self.assertEqual(out["arch_bucket"], "ada_hopper_blackwell")


class TestArchBucketLookup(unittest.TestCase):
    """_arch_bucket_from_compute_cap chooses the right NVDEC throughput bucket."""

    def test_modern_caps(self):
        for cc in ("8.0", "8.6", "8.9", "9.0", "10.0"):
            self.assertEqual(cr._arch_bucket_from_compute_cap(cc),
                             "ada_hopper_blackwell")

    def test_old_caps(self):
        for cc in ("7.5", "7.0", "6.1"):
            self.assertEqual(cr._arch_bucket_from_compute_cap(cc),
                             "turing_or_older")

    def test_unknown_defaults_to_modern(self):
        self.assertEqual(cr._arch_bucket_from_compute_cap(""),
                         "ada_hopper_blackwell")
        self.assertEqual(cr._arch_bucket_from_compute_cap("not_a_cap"),
                         "ada_hopper_blackwell")


class TestNvdecCountLookup(unittest.TestCase):
    """_nvdec_count_for_gpu matches by substring."""

    def test_known_gpus(self):
        # Counts per NVIDIA's Video Encode/Decode Support Matrix.
        cases = [
            ("NVIDIA RTX A6000", 2),
            ("NVIDIA L40S", 3),
            ("NVIDIA L4", 4),
            ("NVIDIA H100 80GB SXM", 7),
            ("NVIDIA T4", 2),                 # T4 has 2 NVDECs (Turing 4th gen)
            ("NVIDIA A100-SXM4-80GB", 5),
            ("NVIDIA A40", 2),                # A40 has 2 (matrix), not 3
            ("Tesla V100-SXM2", 1),           # Volta — single NVDEC
            ("NVIDIA Orin AGX", 2),
        ]
        for name, expected in cases:
            self.assertEqual(cr._nvdec_count_for_gpu(name), expected, msg=name)

    def test_unknown_gpu_safe_default(self):
        self.assertEqual(cr._nvdec_count_for_gpu("Made-up GPU 9001"), 2)
        self.assertEqual(cr._nvdec_count_for_gpu(""), 2)


if __name__ == "__main__":
    unittest.main()

# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.