diff --git a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py index 20885a85c..2169ed1e6 100644 --- a/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py +++ b/packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py @@ -275,6 +275,7 @@ def _dispatch_agentic( question=item.question, expected_output=eo if isinstance(eo, dict) else {}, k=k, + gate=gate, agent_id=agent_id, **lf_kw, ) diff --git a/packages/gooddata-eval/src/gooddata_eval/core/agentic/what_if.py b/packages/gooddata-eval/src/gooddata_eval/core/agentic/what_if.py index 9a01b1575..0ee3d925f 100644 --- a/packages/gooddata-eval/src/gooddata_eval/core/agentic/what_if.py +++ b/packages/gooddata-eval/src/gooddata_eval/core/agentic/what_if.py @@ -25,6 +25,14 @@ from dataclasses import dataclass, field from typing import Any +from gooddata_eval.core.agentic._gate import ( + DEFAULT_GATE, + EvalGate, + gate_failure_note, + gate_passed, + log_gate_scores, + stamp_gate_metadata, +) from gooddata_eval.core.agentic._trace_linker import ( RunIdentity, RunTraceContext, @@ -463,6 +471,7 @@ def evaluate_agentic_what_if( k: int = _DEFAULT_K, max_iterations: int = _DEFAULT_MAX_ITERATIONS, initial_conversation_id: str | None = None, + gate: EvalGate = DEFAULT_GATE, agent_id: str | None = None, langfuse: object | None = None, dataset_item_id: str = "", @@ -493,6 +502,8 @@ def evaluate_agentic_what_if( window_end = utc_now() def _write_scores(ctx: RunTraceContext) -> None: + stamp_gate_metadata(ctx.run_metadata, k=len(summary.run_results), gate=gate) + for run_idx, run in enumerate(summary.run_results): pt = ctx.trace(run.conversation_id) ev = run.evaluation @@ -520,6 +531,7 @@ def _write_scores(ctx: RunTraceContext) -> None: with ctx.observe(pt, run_idx) as tid: for score_name, value in strict_checks.items(): ctx.score(tid, name=score_name, value=float(value), data_type="BOOLEAN") + log_gate_scores(ctx, tid, gate=gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k) ctx.quality( tid, strict_checks=strict_checks, @@ -564,9 +576,10 @@ def _write_scores(ctx: RunTraceContext) -> None: detail = _detail(best) runs_passed = sum(1 for r in summary.run_results if r.evaluation.strict_pass) - if not summary.pass_at_k: + if not gate_passed(gate, pass_at_k=summary.pass_at_k, pass_power_k=summary.pass_power_k): + gate_note = gate_failure_note(gate, runs_passed, len(summary.run_results)) message = ( - f"What-if assertion failed. strict_pass={ev.strict_pass} " + f"What-if assertion failed. {gate_note} strict_pass={ev.strict_pass} " f"(triggered={ev.triggered}, executed={ev.executed}, success={ev.success}, " f"turn_completed={ev.turn_completed}, metric_correct={ev.metric_correct}, " f"maql_correct={ev.maql_correct}, scenario_count_correct={ev.scenario_count_correct}, " diff --git a/packages/gooddata-eval/tests/test_agentic_what_if.py b/packages/gooddata-eval/tests/test_agentic_what_if.py index 07da79ca8..2f45aea4f 100644 --- a/packages/gooddata-eval/tests/test_agentic_what_if.py +++ b/packages/gooddata-eval/tests/test_agentic_what_if.py @@ -362,6 +362,8 @@ def __init__(self): self.scores: dict[str, float] = {} # Not named `quality`: the method below would overwrite itself on first call. self.quality_call: dict = {} + # stamp_gate_metadata writes the gate and K here, the way every other kind does. + self.run_metadata: dict = {} def trace(self, _conversation_id): return None @@ -437,3 +439,50 @@ def test_cost_is_reported_even_when_the_tool_was_never_reached(): ev.triggered hid that and understated what the item cost.""" ctx = _scored(_EXPECTED, calls=[]) assert "cost_usd" in ctx.quality_call + + +# ── the gate decides the verdict, not pass@K alone ────────────────────────── +# +# run_agentic_what_if computed pass^K and the evaluator never read it, so `--gate power` +# on a flaky item passed on the strength of one good run out of K -- the precise case the +# gate exists to catch. + + +def _two_runs(gate=None): + """One passing run then one failing one, which pass@K clears and pass^K must not.""" + client = MagicMock() + client.create_conversation.side_effect = ["conv-1", "conv-2"] + client.send_message.side_effect = [_chat(_pair()), *[_chat([])] * 8] + kwargs = {"gate": gate} if gate is not None else {} + with ( + patch(f"{_MODULE}.ChatClient", return_value=client), + patch(f"{_MODULE}.generate_simulated_what_if_response", return_value="adjust revenue by 10%"), + ): + return evaluate_agentic_what_if( + host="http://h", + token="tok", + workspace_id="ws1", + question="What if revenue rose 10%?", + expected_output=_EXPECTED, + k=2, + **kwargs, + ) + + +def test_power_gate_fails_an_item_where_only_some_runs_passed(): + with pytest.raises(WhatIfAssertionError) as excinfo: + _two_runs(gate="power") + assert "pass^2" in str(excinfo.value) + assert excinfo.value.runs_passed == 1 + assert excinfo.value.runs_effective == 2 + + +def test_any_gate_still_passes_the_same_item(): + """The default is unchanged: one passing run out of K clears pass@K.""" + outcome = _two_runs(gate="any") + assert outcome.runs_passed == 1 + assert outcome.runs_effective == 2 + + +def test_the_default_gate_is_pass_at_k(): + assert _two_runs().runs_passed == 1