#!/usr/bin/env python3
"""Dataset-profile gate for a medical-imaging dataset (profile-imaging).

Before a single model trains, a dataset carries facts that decide the study: how
heterogeneous the acquisition is, how rare the target is, whether the labels are
intact, and whether the split you intend to call a *test set* actually has ground
truth. Those facts are cheap to compute and expensive to discover late — a "test
set" with no labels is found after the training run; a 0.4 %-foreground target is
found when accuracy reads 99.6 % and means nothing.

This gate reads the **dataset profile** emitted by `profile_imaging_dataset.py`
(one record per case: grid, spacing, orientation, label values, foreground
fraction) together with the researcher's **declared plan** (do we resample?
reorient? which loss? which metrics?), and decides each finding by rule and by
set arithmetic over the case records — never from prose, and never by opening an
image. It is stdlib-only: the profile is JSON, so the gate runs anywhere.

CHECKS (verdicts):
  MAJOR
  1. LABEL_SHAPE_MISMATCH     label grid differs from its image grid — the pair cannot
                              be used as supervision as-is.
  2. LABEL_EMPTY              a case declared labelled whose label has zero foreground.
  3. LABEL_VALUE_UNEXPECTED   label values outside the declared label set (a stray index
                              silently becomes a class, or a class is missing).
  4. TEST_SET_UNLABELLED      a split declared as test/held-out whose cases carry no
                              labels. It cannot produce Dice, HD95, or any metric; the
                              held-out set has to come from somewhere else.
  5. ACCURACY_UNDER_IMBALANCE the plan reports accuracy while the target occupies a tiny
                              fraction of the volume — predicting background everywhere
                              scores near-perfect. (Pairs with model-evaluation's
                              ACCURACY_ONLY, which catches the same error downstream.)

  MINOR (flags — each is a decision to declare, not necessarily a defect)
  6. SPACING_HETEROGENEOUS    spacing spans >= --spacing-ratio on some axis and the plan
                              declares no resampling.
  7. ORIENTATION_MIXED        more than one orientation code and no reorientation declared.
  8. INTENSITY_SCALE_INCONSISTENT  cases disagree on whether the intensity domain looks
                              like CT Hounsfield units — mixed modality, or a rescale
                              slope/intercept not applied to part of the cohort.
  9. EXTREME_IMBALANCE        median foreground fraction below --imbalance-frac and the
                              plan declares no Dice-family (region/overlap) loss.
 10. LABEL_MISSING            cases in a labelled split with no label file.
 11. TARGET_LABEL_UNDECLARED  the declared label set holds more than one structure but the
                              profile names no target, so `foreground_fraction` pools every
                              annotated organ. The imbalance verdicts then describe the
                              union rather than the thing being segmented — and the union
                              can sit above the threshold while the target sits far below.
                              Re-profile with `--target-label N`, or `--target-label all`
                              to declare a genuinely multi-class study.

SPLIT NAMES
  A split counts as a test/held-out split when any *token* of its name is one of
  {test, holdout, held_out, external, eval} — so `amos_test` and `external_ct` are
  matched, not just the bare words.

THRESHOLDS
  --spacing-ratio (default 2.0) and --imbalance-frac (default 0.01) are **screening
  defaults, not published cut-points**. 2x through-plane spacing changes what a
  fixed-size patch sees; 1 % foreground is where plain accuracy stops carrying
  information. Both are adjustable, and both are reported in the output so a reader
  knows what was applied.

PROFILE (JSON — emitted by profile_imaging_dataset.py)
  {
    "dataset": "...",
    "declared_labels": {"0": "background", "1": "spleen"},
    "plan": {"resample": true, "reorient": false, "loss": "dice_ce",
             "metrics": ["dice", "hd95"]},
    "splits": [{"name": "train", "labelled": true}, {"name": "test", "labelled": false}],
    "cases": [{"case": "...", "split": "train", "shape": [512,512,90],
               "spacing_mm": [0.79,0.79,5.0], "orientation": "RAS",
               "label_values": [0,1], "foreground_fraction": 0.0039,
               "intensity": {"p01": -1024.0, "p99": 329.0}, "flags": []}]
  }

INPUTS
  --profile   dataset profile JSON (required).

OUTPUT
  A findings table (stdout) and, with --out, a JSON artifact.
  Exit 1 under --strict when any Major finding exists.
"""
from __future__ import annotations

import argparse
import json
import re
from pathlib import Path
from statistics import median

TEST_SPLIT_NAMES = {"test", "testing", "holdout", "hold_out", "held_out", "heldout",
                    "external", "eval"}
# separator-free forms, so a name tokenised into pieces can be rejoined and matched
_TEST_SPLIT_GRAMS = {re.sub(r"[^a-z0-9]+", "", n) for n in TEST_SPLIT_NAMES}
DICE_FAMILY = ("dice", "tversky", "focal_tversky", "jaccard", "iou", "generalized_dice", "gdl", "lovasz")
ACCURACY_TERMS = ("accuracy", "acc", "pixel_accuracy", "voxel_accuracy")
# A CT case is expected to bottom out near air (-1000 HU). Anything whose 1st percentile
# sits far above that is not on the HU scale.
CT_AIR_P01_MAX = -500.0


def _norm(s) -> str:
    return str(s).strip().lower()


def _is_test_split(name: str) -> bool:
    """Match on tokens and adjacent-token joins, not on the whole string.

    Researchers qualify split names -- `amos_test`, `external_ct`, `held-out-set`,
    `test-fold1` -- and an exact-set membership test silently misses every one of them,
    which is the worst way for TEST_SET_UNLABELLED to fail: the split it exists for is the
    split it cannot see.

    Joins are needed because some members are two words (`held_out`), and they are built
    from *adjacent tokens* rather than by substring search, so `contest` does not read as
    `test`.
    """
    toks = [t for t in re.split(r"[^a-z0-9]+", _norm(name)) if t]
    grams = set(toks)
    for n in (2, 3):
        grams |= {"".join(toks[i:i + n]) for i in range(len(toks) - n + 1)}
    return bool(grams & _TEST_SPLIT_GRAMS)


def _plan(profile: dict) -> dict:
    p = profile.get("plan") or {}
    return {
        "resample": bool(p.get("resample")),
        "reorient": bool(p.get("reorient")),
        "loss": _norm(p.get("loss") or ""),
        "metrics": [_norm(m) for m in (p.get("metrics") or [])],
    }


def _labelled_splits(profile: dict) -> dict[str, bool]:
    out: dict[str, bool] = {}
    for s in profile.get("splits") or []:
        out[_norm(s.get("name"))] = bool(s.get("labelled", True))
    return out


def analyze(profile_path: str, spacing_ratio: float, imbalance_frac: float) -> dict:
    profile = json.loads(Path(profile_path).read_text(encoding="utf-8"))
    cases = profile.get("cases") or []
    plan = _plan(profile)
    labelled = _labelled_splits(profile)
    declared = {int(k) for k in (profile.get("declared_labels") or {}).keys()} or None
    target_label = profile.get("target_label")
    # Structures = declared labels minus background. >1 means foreground_fraction pools them.
    n_structures = len((declared or set()) - {0})

    claims: list[dict] = []

    def claim(verdict: str, severity: str, detail: str, cases_hit: list[str] | None = None) -> None:
        claims.append({"verdict": verdict, "severity": severity, "detail": detail,
                       "cases": sorted(cases_hit or [])[:12], "n_cases": len(cases_hit or [])})

    # ---- per-case integrity -------------------------------------------------
    shape_mismatch, empty, unexpected, missing = [], [], [], []
    for c in cases:
        split = _norm(c.get("split"))
        is_lab = labelled.get(split, True)
        flags = set(c.get("flags") or [])
        if "LABEL_SHAPE_MISMATCH" in flags:
            shape_mismatch.append(c["case"])
        if is_lab and ("LABEL_MISSING" in flags or c.get("label_values") is None):
            missing.append(c["case"])
            continue
        if c.get("label_values") is None:
            continue
        if is_lab and c.get("foreground_fraction") == 0:
            empty.append(c["case"])
        if declared is not None:
            stray = set(int(v) for v in c["label_values"]) - declared
            if stray:
                unexpected.append(c["case"])

    if shape_mismatch:
        claim("LABEL_SHAPE_MISMATCH", "Major",
              "label grid differs from the image grid; the pair is not usable supervision as-is",
              shape_mismatch)
    if empty:
        what = (f"no voxel of the target label {target_label}"
                if target_label not in (None, "", "all") else "no foreground voxel")
        claim("LABEL_EMPTY", "Major",
              f"case is in a labelled split but its label contains {what}", empty)
    if unexpected:
        claim("LABEL_VALUE_UNEXPECTED", "Major",
              f"label values outside the declared set {sorted(declared or [])}", unexpected)
    if missing:
        claim("LABEL_MISSING", "Minor",
              "case sits in a split declared labelled but has no label file", missing)

    # ---- whose foreground is this? -----------------------------------------
    # On a multi-structure atlas, foreground_fraction is the union of every annotated
    # organ. If the study segments one of them, every imbalance verdict below is reading
    # the wrong quantity -- and reading it in the unsafe direction, since the union is
    # always the larger number and therefore the one that clears the threshold.
    if n_structures > 1 and target_label in (None, ""):
        claim("TARGET_LABEL_UNDECLARED", "Minor",
              f"{n_structures} structures are declared but no target label is; "
              "foreground_fraction pools all of them, so the imbalance verdicts describe "
              "the annotated anatomy rather than the segmentation target. Re-profile with "
              "--target-label N, or --target-label all for a genuinely multi-class study",
              [])

    # ---- an unlabelled test set --------------------------------------------
    for name, is_lab in labelled.items():
        if _is_test_split(name) and not is_lab:
            n = sum(1 for c in cases if _norm(c.get("split")) == name)
            claim("TEST_SET_UNLABELLED", "Major",
                  f"split '{name}' ({n} case(s)) has no ground truth — it cannot yield Dice, "
                  "HD95, or any held-out metric; carve the held-out set from labelled data",
                  [])

    # ---- acquisition heterogeneity -----------------------------------------
    spacings = [c["spacing_mm"] for c in cases if c.get("spacing_mm")]
    ratios = []
    if spacings:
        for i, ax in enumerate("xyz"):
            vals = [s[i] for s in spacings if len(s) > i and s[i] > 0]
            if vals:
                ratios.append((ax, max(vals) / min(vals), min(vals), max(vals)))
        worst = max(ratios, key=lambda r: r[1]) if ratios else None
        if worst and worst[1] >= spacing_ratio and not plan["resample"]:
            claim("SPACING_HETEROGENEOUS", "Minor",
                  f"{worst[0]}-spacing spans {worst[2]:.3g}-{worst[3]:.3g} mm ({worst[1]:.1f}x) "
                  "and the plan declares no resampling", [])

    orients = sorted({c.get("orientation") for c in cases if c.get("orientation")})
    if len(orients) > 1 and not plan["reorient"]:
        claim("ORIENTATION_MIXED", "Minor",
              f"{len(orients)} orientation codes present ({', '.join(orients)}) and no "
              "reorientation declared", [])

    p01s = [c["intensity"]["p01"] for c in cases
            if isinstance(c.get("intensity"), dict) and c["intensity"].get("p01") is not None]
    if p01s:
        ct_like = [v for v in p01s if v <= CT_AIR_P01_MAX]
        if ct_like and len(ct_like) != len(p01s):
            claim("INTENSITY_SCALE_INCONSISTENT", "Minor",
                  f"{len(ct_like)}/{len(p01s)} cases bottom out near air (<= {CT_AIR_P01_MAX:g}) "
                  "and the rest do not — mixed modality, or a rescale not applied to part of "
                  "the cohort", [])

    # ---- class imbalance ----------------------------------------------------
    fgs = [c["foreground_fraction"] for c in cases if c.get("foreground_fraction") is not None]
    med_fg = median(fgs) if fgs else None
    if med_fg is not None and med_fg < imbalance_frac:
        if not any(t in plan["loss"] for t in DICE_FAMILY):
            claim("EXTREME_IMBALANCE", "Minor",
                  f"median foreground fraction {med_fg:.4%} is below {imbalance_frac:.2%} and the "
                  "plan declares no Dice-family loss", [])
        if any(m in ACCURACY_TERMS for m in plan["metrics"]):
            claim("ACCURACY_UNDER_IMBALANCE", "Major",
                  f"the plan reports accuracy at median foreground {med_fg:.4%} — predicting "
                  "background everywhere would score ~{:.2%}".format(1 - med_fg), [])

    n_major = sum(1 for c in claims if c["severity"] == "Major")
    return {
        "profile": profile_path,
        "dataset": profile.get("dataset"),
        "n_cases": len(cases),
        "splits": {k: ("labelled" if v else "unlabelled") for k, v in labelled.items()},
        "plan": plan,
        "thresholds": {"spacing_ratio": spacing_ratio, "imbalance_frac": imbalance_frac},
        "median_foreground_fraction": med_fg,
        "claims": claims,
        "summary": {"n_claims": len(claims), "n_major": n_major,
                    "n_flag": len(claims) - n_major},
    }


def render(result: dict) -> str:
    if not result["claims"]:
        return "  (no findings)"
    lines = []
    for c in result["claims"]:
        head = f"  [{c['severity']:<5}] {c['verdict']}: {c['detail']}"
        lines.append(head)
        if c["cases"]:
            shown = ", ".join(c["cases"])
            more = f" (+{c['n_cases'] - len(c['cases'])} more)" if c["n_cases"] > len(c["cases"]) else ""
            lines.append(f"           cases: {shown}{more}")
    return "\n".join(lines)


def main() -> int:
    ap = argparse.ArgumentParser(description="Dataset-profile gate for medical imaging.")
    ap.add_argument("--profile", required=True, help="dataset profile JSON")
    ap.add_argument("--spacing-ratio", type=float, default=2.0,
                    help="max/min spacing ratio that trips SPACING_HETEROGENEOUS (default 2.0)")
    ap.add_argument("--imbalance-frac", type=float, default=0.01,
                    help="median foreground fraction below which imbalance is flagged (default 0.01)")
    ap.add_argument("--out", help="write JSON artifact to this path")
    ap.add_argument("--strict", action="store_true", help="exit 1 if any Major claim exists")
    ap.add_argument("--quiet", action="store_true", help="suppress stdout table")
    args = ap.parse_args()

    result = analyze(args.profile, args.spacing_ratio, args.imbalance_frac)

    if not args.quiet:
        print("=" * 41)
        print("  Dataset-Profile Gate (profile-imaging)")
        print("=" * 41)
        fg = result["median_foreground_fraction"]
        fg_s = f"{fg:.4%}" if fg is not None else "n/a"
        print(f"  cases={result['n_cases']}  splits={result['splits']}  median_fg={fg_s}")
        print(f"  thresholds: spacing_ratio={result['thresholds']['spacing_ratio']}  "
              f"imbalance_frac={result['thresholds']['imbalance_frac']}")
        print(render(result))
        print()
        s = result["summary"]
        if s["n_major"]:
            print(f"MAJOR candidate: {s['n_major']} dataset defect(s) that block training as planned.")
        elif s["n_flag"]:
            print(f"MINOR flag: {s['n_flag']} dataset decision(s) to declare (see table).")
        else:
            print("OK: dataset profile is intact and the declared plan matches what the data looks like.")

    if args.out:
        Path(args.out).parent.mkdir(parents=True, exist_ok=True)
        Path(args.out).write_text(
            json.dumps({"detector": "check_dataset_profile", **result}, indent=2), encoding="utf-8")
        if not args.quiet:
            print(f"\nwrote {args.out}")

    return 1 if (args.strict and result["summary"]["n_major"]) else 0


if __name__ == "__main__":
    raise SystemExit(main())

#!/usr/bin/env python3
"""Profile a 3-D medical-imaging dataset (NIfTI) into the JSON the gate audits.

This is the *describe* half of profile-imaging: it opens every image (and its label,
when there is one) and records the facts that decide a study — the acquisition grid
and spacing, orientation, intensity domain, which label values are actually present,
how much of the volume the target occupies, and how big the target is in millilitres.
It draws no conclusion; `check_dataset_profile.py` does that, from this file.

Requires nibabel + numpy (it has to read images). The gate that consumes its output is
stdlib-only, so an audit can be re-run anywhere the JSON travels.

Layout: images and labels in separate directories, matched by filename — the MSD /
nnU-Net / AMOS convention (`imagesTr/case.nii.gz` <-> `labelsTr/case.nii.gz`).

Usage
  python3 profile_imaging_dataset.py \
      --split train:imagesTr:labelsTr --split test:imagesTs \
      --dataset "MSD Task09 Spleen" --declared-labels 0=background,1=spleen \
      --plan resample=true,reorient=false,loss=dice_ce,metrics=dice+hd95 \
      --out profile.json

A --split with no label directory is recorded as unlabelled, which is itself a
finding: a split named `test` with no labels cannot produce a held-out metric.

--target-label matters on a multi-structure atlas. Foreground defaults to every
non-zero index, which is the whole annotated anatomy — so on a 15-organ atlas used
for a single-organ study the reported fraction describes the upper abdomen, not the
target, and it can sit above the imbalance threshold while the target sits far
below it. Pass `--target-label 1` for a single-structure study (foreground and
target volume are then computed on that index alone, and a case carrying none of it
reads as empty), or `--target-label all` to declare a genuinely multi-class study.
"""
from __future__ import annotations

import argparse
import json
from pathlib import Path

import nibabel as nib
import numpy as np


def profile_case(img_p: Path, lab_p: Path | None, split: str,
                 target_label: str | None = None) -> dict:
    img = nib.load(str(img_p))
    zooms = tuple(float(z) for z in img.header.get_zooms()[:3])
    shape = tuple(int(s) for s in img.shape[:3])
    rec: dict = {
        "case": img_p.name.replace(".nii.gz", "").replace(".nii", ""),
        "split": split,
        "shape": list(shape),
        "spacing_mm": list(zooms),
        "orientation": "".join(nib.aff2axcodes(img.affine)),
        "voxel_volume_mm3": float(np.prod(zooms)),
        "label_values": None,
        "foreground_fraction": None,
        "all_label_foreground_fraction": None,
        "target_volume_ml": None,
        "flags": [],
    }

    data = np.asanyarray(img.dataobj, dtype=np.float32)
    rec["intensity"] = {
        "min": float(data.min()),
        "p01": float(np.percentile(data, 1)),
        "p50": float(np.percentile(data, 50)),
        "p99": float(np.percentile(data, 99)),
        "max": float(data.max()),
    }

    if lab_p is None:
        return rec
    if not lab_p.exists():
        rec["flags"].append("LABEL_MISSING")
        return rec

    lab_img = nib.load(str(lab_p))
    lab = np.asanyarray(lab_img.dataobj)
    if tuple(int(s) for s in lab.shape[:3]) != shape:
        rec["flags"].append("LABEL_SHAPE_MISMATCH")
    rec["label_values"] = [int(v) for v in np.unique(lab)]
    n_all = int((lab > 0).sum())
    rec["all_label_foreground_fraction"] = n_all / int(lab.size)
    # Foreground is the *target* when one is declared. On a multi-structure atlas the
    # union of every index is the annotated anatomy, not the thing being segmented, and
    # the gate's imbalance verdicts read this number.
    if target_label is None or _norm_target(target_label) == "all":
        n_fg = n_all
    else:
        n_fg = int((lab == int(target_label)).sum())
    rec["foreground_fraction"] = n_fg / int(lab.size)
    rec["target_volume_ml"] = n_fg * rec["voxel_volume_mm3"] / 1000.0
    return rec


def _norm_target(t) -> str:
    return str(t).strip().lower() if t is not None else ""


def parse_kv(s: str | None, sep: str = ",") -> dict:
    out: dict = {}
    if not s:
        return out
    for part in s.split(sep):
        if not part.strip():
            continue
        k, _, v = part.partition("=")
        k, v = k.strip(), v.strip()
        if v.lower() in ("true", "false"):
            out[k] = v.lower() == "true"
        elif "+" in v:
            out[k] = [x for x in v.split("+") if x]
        else:
            out[k] = v
    return out


def main() -> None:
    ap = argparse.ArgumentParser(description="Profile a NIfTI imaging dataset.")
    ap.add_argument("--split", action="append", required=True,
                    help="name:imagedir[:labeldir] — repeatable; omit labeldir if unlabelled")
    ap.add_argument("--dataset", default="", help="dataset name for the record")
    ap.add_argument("--declared-labels", default="",
                    help="comma list, e.g. 0=background,1=spleen")
    ap.add_argument("--plan", default="",
                    help="comma list, e.g. resample=true,reorient=false,loss=dice_ce,metrics=dice+hd95")
    ap.add_argument("--target-label", default=None,
                    help="label index this study segments (e.g. 1), so foreground/target volume "
                         "describe that structure instead of every annotated one; 'all' declares "
                         "a genuinely multi-class study")
    ap.add_argument("--limit", type=int, default=0, help="profile at most N cases per split (0 = all)")
    ap.add_argument("--out", required=True)
    a = ap.parse_args()

    splits, cases = [], []
    for spec in a.split:
        parts = spec.split(":")
        name, img_dir = parts[0], parts[1]
        lab_dir = parts[2] if len(parts) > 2 and parts[2] else None
        splits.append({"name": name, "labelled": lab_dir is not None})

        imgs = sorted(p for p in Path(img_dir).glob("*.nii*") if not p.name.startswith("._"))
        if a.limit:
            imgs = imgs[: a.limit]
        for i, p in enumerate(imgs, 1):
            cases.append(profile_case(p, (Path(lab_dir) / p.name) if lab_dir else None, name,
                                      a.target_label))
            print(f"[{name} {i}/{len(imgs)}] {p.name}", flush=True)

    labels_raw = parse_kv(a.declared_labels)
    profile = {
        "dataset": a.dataset,
        "declared_labels": {str(k): v for k, v in labels_raw.items()},
        "target_label": a.target_label,
        "plan": parse_kv(a.plan),
        "splits": splits,
        "cases": cases,
    }
    Path(a.out).parent.mkdir(parents=True, exist_ok=True)
    Path(a.out).write_text(json.dumps(profile, indent=1), encoding="utf-8")
    print(f"\nwrote {a.out}  ({len(cases)} case(s) across {len(splits)} split(s))")
    print("Now audit it:  python3 check_dataset_profile.py --profile "
          f"{a.out} --strict")


if __name__ == "__main__":
    main()

#!/usr/bin/env bash
# Regression test for the dataset-profile gate (profile-imaging).
# Synthetic, PII-free JSON profiles reproduce each verdict class. Stdlib-only (python3).
set -u

HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
SCRIPT="$HERE/../scripts/check_dataset_profile.py"
CH="$HERE/../scripts/check_dataset_profile_challenge"
TMP="$(mktemp -d -t dsprof_XXXX)"
OUT="$TMP/out.json"
trap 'rm -rf "$TMP"' EXIT

fail=0
check() { local label="$1"; shift
    if "$@" >/dev/null 2>&1; then printf '  PASS  %s\n' "$label"
    else printf '  FAIL  %s\n' "$label"; fail=$((fail+1)); fi
}
has_verdict() { python3 -c "
import json
d=json.load(open('$OUT'))
assert any(c['verdict']=='$1' for c in d['claims']), '$1 not found'
"; }
no_verdict() { python3 -c "
import json
d=json.load(open('$OUT'))
assert not any(c['verdict']=='$1' for c in d['claims']), '$1 unexpectedly present'
"; }

[[ -f "$SCRIPT" ]] || { echo "ENV-ERR: script missing" >&2; exit 2; }

# (1) defect fixture -> every verdict class + exit 1
python3 "$SCRIPT" --profile "$CH/fixture/profile_defect.json" --out "$OUT" --strict --quiet >/dev/null 2>&1
check "exit 1 (defect profile)" test "$?" -eq 1
for v in LABEL_SHAPE_MISMATCH LABEL_EMPTY LABEL_VALUE_UNEXPECTED TEST_SET_UNLABELLED \
         ACCURACY_UNDER_IMBALANCE LABEL_MISSING SPACING_HETEROGENEOUS ORIENTATION_MIXED \
         INTENSITY_SCALE_INCONSISTENT EXTREME_IMBALANCE; do
  check "$v detected" has_verdict "$v"
done

# (2) clean fixture -> exit 0, nothing fires
python3 "$SCRIPT" --profile "$CH/fixture/profile_clean.json" --out "$OUT" --strict --quiet >/dev/null 2>&1
check "exit 0 (clean profile)" test "$?" -eq 0
check "no SPACING_HETEROGENEOUS when resampling declared" no_verdict SPACING_HETEROGENEOUS
check "no ORIENTATION_MIXED when reorientation declared" no_verdict ORIENTATION_MIXED
check "no EXTREME_IMBALANCE when a Dice-family loss is declared" no_verdict EXTREME_IMBALANCE
check "no ACCURACY_UNDER_IMBALANCE when accuracy is not reported" no_verdict ACCURACY_UNDER_IMBALANCE
check "no TEST_SET_UNLABELLED when the held-out split is labelled" no_verdict TEST_SET_UNLABELLED

# (3) thresholds are honoured — a permissive ratio silences the spacing flag
python3 "$SCRIPT" --profile "$CH/fixture/profile_defect.json" --spacing-ratio 99 --out "$OUT" --quiet >/dev/null 2>&1
check "spacing flag silenced by --spacing-ratio 99" no_verdict SPACING_HETEROGENEOUS

# (4) a multi-structure atlas used for a single-structure study.
# foreground_fraction pools every annotated organ, so the imbalance verdicts read the
# union rather than the target -- and the union is always the larger number, so it is the
# one that clears the threshold. The gate has to say the target was never named.
write_atlas() {  # $1=out path  $2=target_label ("none" to omit)  $3=foreground fraction
  python3 - "$1" "$2" "$3" <<'PY'
import json, sys
out, target, frac = sys.argv[1], sys.argv[2], float(sys.argv[3])
p = {"dataset": "multi-organ atlas",
     "declared_labels": {"0": "background", "1": "spleen", "2": "liver", "3": "pancreas"},
     "plan": {"resample": True, "reorient": True, "loss": "cross_entropy",
              "metrics": ["dice", "hd95"]},
     "splits": [{"name": "external_ct", "labelled": True}],
     "cases": [{"case": "c%d" % i, "split": "external_ct", "shape": [512, 512, 100],
                "spacing_mm": [0.8, 0.8, 5.0], "orientation": "RAS",
                "voxel_volume_mm3": 3.2, "label_values": [0, 1, 2, 3],
                "foreground_fraction": frac, "target_volume_ml": 200.0,
                "intensity": {"min": -1024.0, "p01": -1000.0, "p50": 40.0,
                              "p99": 300.0, "max": 1500.0}, "flags": []}
               for i in range(4)]}
if target != "none":
    p["target_label"] = target
json.dump(p, open(out, "w"))
PY
}

# 4a) three structures declared, no target named -> fires. 3.2 % pooled foreground sits
#     ABOVE the 1 % threshold, so the imbalance verdict stays silent while the real
#     target (0.2 %) is far below it: the exact false negative this verdict exists for.
write_atlas "$TMP/atlas_undeclared.json" none 0.032
python3 "$SCRIPT" --profile "$TMP/atlas_undeclared.json" --out "$OUT" --quiet >/dev/null 2>&1
check "TARGET_LABEL_UNDECLARED on a multi-structure atlas with no target" \
      has_verdict TARGET_LABEL_UNDECLARED
check "pooled foreground clears the threshold, so EXTREME_IMBALANCE stays silent" \
      no_verdict EXTREME_IMBALANCE

# 4b) same atlas, target named, target-specific foreground -> the imbalance is visible
write_atlas "$TMP/atlas_target.json" 1 0.002
python3 "$SCRIPT" --profile "$TMP/atlas_target.json" --out "$OUT" --quiet >/dev/null 2>&1
check "no TARGET_LABEL_UNDECLARED once a target label is declared" \
      no_verdict TARGET_LABEL_UNDECLARED
check "target-specific foreground now trips EXTREME_IMBALANCE" has_verdict EXTREME_IMBALANCE

# 4c) 'all' is a declaration too -- a genuinely multi-class study
write_atlas "$TMP/atlas_all.json" all 0.032
python3 "$SCRIPT" --profile "$TMP/atlas_all.json" --out "$OUT" --quiet >/dev/null 2>&1
check "no TARGET_LABEL_UNDECLARED when 'all' is declared" no_verdict TARGET_LABEL_UNDECLARED

# 4d) a single-structure dataset never raises the question
python3 "$SCRIPT" --profile "$CH/fixture/profile_clean.json" --out "$OUT" --quiet >/dev/null 2>&1
check "no TARGET_LABEL_UNDECLARED on a binary dataset" no_verdict TARGET_LABEL_UNDECLARED

# (5) split names carry qualifiers in real projects. An exact-set membership test misses
# every one of them, which is the worst way for TEST_SET_UNLABELLED to fail: the split it
# exists for is the split it cannot see.
write_qualified() {  # $1=out  $2=split name  $3=labelled(true/false)
  python3 - "$1" "$2" "$3" <<'PY'
import json, sys
out, name, lab = sys.argv[1], sys.argv[2], sys.argv[3] == "true"
case = {"case": "c1", "split": name, "shape": [512, 512, 100],
        "spacing_mm": [0.8, 0.8, 5.0], "orientation": "RAS", "voxel_volume_mm3": 3.2,
        "label_values": [0, 1] if lab else None,
        "foreground_fraction": 0.004 if lab else None,
        "target_volume_ml": 200.0 if lab else None,
        "intensity": {"min": -1024.0, "p01": -1000.0, "p50": 40.0, "p99": 300.0,
                      "max": 1500.0}, "flags": []}
json.dump({"dataset": "d", "declared_labels": {"0": "background", "1": "spleen"},
           "target_label": "1",
           "plan": {"resample": True, "reorient": True, "loss": "dice_ce",
                    "metrics": ["dice", "hd95"]},
           "splits": [{"name": name, "labelled": lab}], "cases": [case]},
          open(out, "w"))
PY
}
for qname in amos_test external_ct held-out-set test-fold1 testing_set; do
  write_qualified "$TMP/q.json" "$qname" false
  python3 "$SCRIPT" --profile "$TMP/q.json" --out "$OUT" --quiet >/dev/null 2>&1
  check "TEST_SET_UNLABELLED on qualified split name '$qname'" has_verdict TEST_SET_UNLABELLED
done
write_qualified "$TMP/q.json" amos_test true
python3 "$SCRIPT" --profile "$TMP/q.json" --out "$OUT" --quiet >/dev/null 2>&1
check "no TEST_SET_UNLABELLED when the qualified split IS labelled" no_verdict TEST_SET_UNLABELLED
for safe in train contest pretest protest_cohort validation; do
  write_qualified "$TMP/q.json" "$safe" false
  python3 "$SCRIPT" --profile "$TMP/q.json" --out "$OUT" --quiet >/dev/null 2>&1
  check "no TEST_SET_UNLABELLED on unlabelled split named '$safe'" no_verdict TEST_SET_UNLABELLED
done

# (6) challenge card reproduces
check "challenge card verify.sh" bash "$CH/verify.sh"

echo
if [[ "$fail" -eq 0 ]]; then echo "ALL PASS (dataset-profile gate)"; else echo "$fail FAILURE(S)"; exit 1; fi