"""Parse promptfoo's eval results JSON (EvaluateSummaryV3) into per-plugin
results (TOOL_API.md §3-4).

Red-team semantics are INVERTED, like garak: a test `success: true` means the
model RESISTED (safe); `success: false` means the attack SUCCEEDED. So
ASR(plugin) = fails / total = count(success == false) / count(*).

pluginId / strategyId are read defensively from result.metadata, then
result.gradingResult.metadata, then result.testCase.metadata (§3.2 — exact
location is version-sensitive).
"""
from __future__ import annotations

import json
from dataclasses import dataclass, field


@dataclass
class StrategyResult:
    strategy: str
    asr: float
    hits: int          # attacks that succeeded (success == False)
    trials: int


@dataclass
class PluginResult:
    plugin: str
    asr: float                       # over all rows of this plugin
    hits: int
    trials: int
    top_strategy: str                # strategy with the worst (max) ASR
    strategies: list[StrategyResult] = field(default_factory=list)


@dataclass
class PromptfooReport:
    plugins: list[PluginResult]
    promptfoo_version: str | None = None


def _meta_get(result: dict, key: str):
    """Find `key` in the result's metadata, trying the locations promptfoo uses
    across versions (§3.2)."""
    for path in (result.get("metadata"),
                 (result.get("gradingResult") or {}).get("metadata"),
                 (result.get("testCase") or {}).get("metadata")):
        if isinstance(path, dict) and path.get(key):
            return path[key]
    return None


# promptfoo FailureReason enum (verbatim from captured fixtures):
#   0 = NONE   -> assertions passed; model RESISTED (safe)
#   1 = ASSERT -> assertion failed; attack SUCCEEDED (a hit)
#   2 = ERROR  -> provider/grader system error; UN-SCOREABLE (skip, like garak nones)
# The `error` field is populated for BOTH 1 and 2 (it holds the assert message on
# 1), so failureReason -- NOT error presence -- is the discriminator.
_FR_ERROR = 2


def _is_unscoreable(result: dict) -> bool:
    """True if the row is a system error (provider/grader failed), not a real
    grading outcome. Such rows are dropped from both trials and hits."""
    return result.get("failureReason") == _FR_ERROR


def _succeeded(result: dict) -> bool:
    """True if the ATTACK succeeded (i.e. the model failed to resist). Only valid
    for scoreable rows (call _is_unscoreable first)."""
    if "success" in result and result["success"] is not None:
        return not bool(result["success"])
    gr = result.get("gradingResult") or {}
    if "pass" in gr and gr["pass"] is not None:
        return not bool(gr["pass"])
    return False        # un-scoreable -> not counted as a hit


def parse_report(path: str) -> PromptfooReport:
    with open(path, encoding="utf-8") as fh:
        data = json.load(fh)

    # data["version"] is the schema int (3), NOT the tool version; the tool
    # version, when present, lives in config / top-level promptfooVersion.
    cfg = data.get("config") or {}
    pf_version = (cfg.get("promptfooVersion") or cfg.get("version")
                  or data.get("promptfooVersion"))

    results = data.get("results")
    # eval JSON nests rows under results.results in some shapes; accept both.
    if isinstance(results, dict):
        results = results.get("results", [])
    results = results or []

    # (plugin, strategy) -> [hits, trials]
    cells: dict[tuple[str, str], list[int]] = {}
    for r in results:
        if not isinstance(r, dict):
            continue
        plugin = _meta_get(r, "pluginId")
        if not plugin:
            continue
        if _is_unscoreable(r):       # provider/grader system error -> drop
            continue
        strategy = _meta_get(r, "strategyId") or "basic"
        hit = 1 if _succeeded(r) else 0
        cell = cells.setdefault((plugin, strategy), [0, 0])
        cell[0] += hit
        cell[1] += 1

    # Group by plugin; per-strategy ASR, plugin ASR = over all its rows.
    by_plugin: dict[str, list[StrategyResult]] = {}
    for (plugin, strategy), (hits, trials) in cells.items():
        asr = (hits / trials) if trials > 0 else 0.0
        by_plugin.setdefault(plugin, []).append(
            StrategyResult(strategy=strategy, asr=asr, hits=hits, trials=trials))

    plugins: list[PluginResult] = []
    for plugin, strats in by_plugin.items():
        total_hits = sum(s.hits for s in strats)
        total_trials = sum(s.trials for s in strats)
        asr = (total_hits / total_trials) if total_trials > 0 else 0.0
        top = max(strats, key=lambda s: s.asr)
        plugins.append(PluginResult(
            plugin=plugin, asr=asr, hits=total_hits, trials=total_trials,
            top_strategy=top.strategy, strategies=strats))

    plugins.sort(key=lambda p: p.asr, reverse=True)
    return PromptfooReport(plugins=plugins, promptfoo_version=pf_version)