fix(gooddata-eval): let --gate decide a what-if item, not pass@K alone

packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py

@@ -275,6 +275,7 @@ def _dispatch_agentic(
             question=item.question,
             expected_output=eo if isinstance(eo, dict) else {},
             k=k,
+            gate=gate,
             agent_id=agent_id,
             **lf_kw,
         )

packages/gooddata-eval/src/gooddata_eval/core/agentic/what_if.py

@@ -25,6 +25,14 @@
 from dataclasses import dataclass, field
 from typing import Any
 
+from gooddata_eval.core.agentic._gate import (
+    DEFAULT_GATE,
+    EvalGate,
+    gate_failure_note,
+    gate_passed,
+    log_gate_scores,
+    stamp_gate_metadata,
+)
 from gooddata_eval.core.agentic._trace_linker import (
     RunIdentity,
     RunTraceContext,
@@ -463,6 +471,7 @@ def evaluate_agentic_what_if(
     k: int = _DEFAULT_K,
     max_iterations: int = _DEFAULT_MAX_ITERATIONS,
     initial_conversation_id: str | None = None,
+    gate: EvalGate = DEFAULT_GATE,
     agent_id: str | None = None,
     langfuse: object | None = None,
     dataset_item_id: str = "",
@@ -493,6 +502,8 @@ def evaluate_agentic_what_if(
         window_end = utc_now()
 
         def _write_scores(ctx: RunTraceContext) -> None:
+            stamp_gate_metadata(ctx.run_metadata, k=len(summary.run_results), gate=gate)
+
             for run_idx, run in enumerate(summary.run_results):
                 pt = ctx.trace(run.conversation_id)
                 ev = run.evaluation
@@ -520,6 +531,7 @@ def _write_scores(ctx: RunTraceContext) -> None:
                 with ctx.observe(pt, run_idx) as tid:
                     for score_name, value in strict_checks.items():
                         ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN")
+                    log_gate_scores(ctx, tid, gate=gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k)
                     ctx.quality(
                         tid,
                         strict_checks=strict_checks,
@@ -564,9 +576,10 @@ def _write_scores(ctx: RunTraceContext) -> None:
     detail = _detail(best)
     runs_passed = sum(1 for r in summary.run_results if r.evaluation.strict_pass)
 
-    if not summary.pass_at_k:
+    if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k):
+        gate_note = gate_failure_note(gate, runs_passed, len(summary.run_results))
         message = (
-            f"What-if assertion failed. strict_pass={ev.strict_pass} "
+            f"What-if assertion failed. {gate_note} strict_pass={ev.strict_pass} "
             f"(triggered={ev.triggered}, executed={ev.executed}, success={ev.success}, "
             f"turn_completed={ev.turn_completed}, metric_correct={ev.metric_correct}, "
             f"maql_correct={ev.maql_correct}, scenario_count_correct={ev.scenario_count_correct}, "

packages/gooddata-eval/tests/test_agentic_what_if.py

@@ -362,6 +362,8 @@ def __init__(self):
         self.scores: dict[str, float] = {}
         # Not named `quality`: the method below would overwrite itself on first call.
         self.quality_call: dict = {}
+        # stamp_gate_metadata writes the gate and K here, the way every other kind does.
+        self.run_metadata: dict = {}
 
     def trace(self, _conversation_id):
         return None
@@ -437,3 +439,50 @@ def test_cost_is_reported_even_when_the_tool_was_never_reached():
     ev.triggered hid that and understated what the item cost."""
     ctx = _scored(_EXPECTED, calls=[])
     assert "cost_usd" in ctx.quality_call
+
+
+# ── the gate decides the verdict, not pass@K alone ──────────────────────────
+#
+# run_agentic_what_if computed pass^K and the evaluator never read it, so `--gate power`
+# on a flaky item passed on the strength of one good run out of K -- the precise case the
+# gate exists to catch.
+
+
+def _two_runs(gate=None):
+    """One passing run then one failing one, which pass@K clears and pass^K must not."""
+    client = MagicMock()
+    client.create_conversation.side_effect = ["conv-1", "conv-2"]
+    client.send_message.side_effect = [_chat(_pair()), *[_chat([])] * 8]
+    kwargs = {"gate": gate} if gate is not None else {}
+    with (
+        patch(f"{_MODULE}.ChatClient", return_value=client),
+        patch(f"{_MODULE}.generate_simulated_what_if_response", return_value="adjust revenue by 10%"),
+    ):
+        return evaluate_agentic_what_if(
+            host="http://h",
+            token="tok",
+            workspace_id="ws1",
+            question="What if revenue rose 10%?",
+            expected_output=_EXPECTED,
+            k=2,
+            **kwargs,
+        )
+
+
+def test_power_gate_fails_an_item_where_only_some_runs_passed():
+    with pytest.raises(WhatIfAssertionError) as excinfo:
+        _two_runs(gate="power")
+    assert "pass^2" in str(excinfo.value)
+    assert excinfo.value.runs_passed == 1
+    assert excinfo.value.runs_effective == 2
+
+
+def test_any_gate_still_passes_the_same_item():
+    """The default is unchanged: one passing run out of K clears pass@K."""
+    outcome = _two_runs(gate="any")
+    assert outcome.runs_passed == 1
+    assert outcome.runs_effective == 2
+
+
+def test_the_default_gate_is_pass_at_k():
+    assert _two_runs().runs_passed == 1