From cc120931033fdfb6e85b3947c95607999f8b2924 Mon Sep 17 00:00:00 2001 From: Adriana Knight Date: Tue, 27 Oct 2026 15:28:26 -0400 Subject: [PATCH] fix(evaluations): send criterion errors as errorMessage and skip criteria for rows that did not generate A criterion ERROR event now carries only a top-level errorMessage, with "Unknown error" as the fallback. The nested error object, whose code was a string cause name ingest never reads, is no longer sent, and errorCode stays omitted until integer codes are assigned. The cause names remain on the internal result, where the evaluate span still reports them as error.type. An error whose message is empty now falls back to "Unknown error" instead of sending the repr of the internal error mapping. A row whose generation did not complete is no longer scored: no judge or scorer runs for it, it takes no concurrency slot, and no criterion event is tracked for it. Its generation ERROR event already settles the row as failed, so a criterion event for it would be unaccounted for. The generation_incomplete cause is removed. Co-Authored-By: Claude Opus 5.5 --- packages/client/README.md | 2 +- .../evaluations/events.py | 2 - .../evaluations/runner.py | 39 ++-- packages/client/tests/test_evaluations_run.py | 188 ++++++++++++++++-- 4 files changed, 192 insertions(+), 39 deletions(-) diff --git a/packages/client/README.md b/packages/client/README.md index 89ba2d07..cfc20434 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -196,7 +196,7 @@ A tool result is now judge-prompt input. It stays literal for the same reason th **Judges are independent AI Configs, so handlers are routed per judge.** A judge may resolve to a different provider or mode than `generation`. Each judge selects its handler from `handler` or `handlers` by its provider and mode, with the same rule as generation. There is no mode fallback: a messages-mode judge never runs on an agent handler. A judge's prompt must also match its handler's mode. Judges get no tools: the harness passes an empty tool map and removes `tools` from the judge config. -Judges are resolved through flag delivery, and handlers are matched to them, **before** any evaluation records are created — a missing judge, a judge that no handler covers, or a judge whose prompt does not match its handler fails the run up front rather than after the generation spend. One error lists every judge with a problem. After that point a criterion failure never aborts the run: an unparseable judge response, an out-of-range score, a raising handler or scorer, and a row whose generation errored each become a per-criterion `ERROR` event with a cause code (`invalid_judge_output`, `invalid_score`, `handler_raised`, `scorer_raised`, `generation_incomplete`) and a top-level `errorMessage`. Event *delivery* is different: the backend needs one result per `(row, criterion)` to finish row accounting, so if tracking a criterion event fails, every remaining result is still attempted and flushed and then `run()` raises — rather than polling to its timeout with the cause hidden. +Judges are resolved through flag delivery, and handlers are matched to them, **before** any evaluation records are created — a missing judge, a judge that no handler covers, or a judge whose prompt does not match its handler fails the run up front rather than after the generation spend. One error lists every judge with a problem. After that point a criterion failure never aborts the run: an unparseable judge response, an out-of-range score, and a raising handler or scorer each become a per-criterion `ERROR` event carrying a top-level `errorMessage` that describes the failure. A row whose generation errored is not scored at all: no judge or scorer runs for it and no criterion event is sent, because its generation `ERROR` event already records the row as failed. Event *delivery* is different: the backend needs one result per `(row, criterion)` to finish row accounting, so if tracking a criterion event fails, every remaining result is still attempted and flushed and then `run()` raises — rather than polling to its timeout with the cause hidden. The client uses **lazy initialization**: importing the package does not connect to LaunchDarkly. The singleton is created automatically on the first API call that needs it (`config().invoke()`, `graph().invoke()`, `resolve_graph()`, etc.), as long as `LD_SDK_KEY` is set in the environment. diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/events.py b/packages/client/src/launchdarkly_ai_server/evaluations/events.py index 9418ff45..07c00247 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/events.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/events.py @@ -52,7 +52,6 @@ class CriterionEventPayload: evaluation_version: int | None = None score: float | None = None reason: str | None = None - error: dict[str, Any] | None = None error_message: str | None = None def to_track_payload(self) -> dict[str, Any]: @@ -76,7 +75,6 @@ def to_track_payload(self) -> dict[str, Any]: "latencyMs": self.latency_ms, "score": self.score, "reason": self.reason, - "error": self.error, "errorMessage": self.error_message, } return {key: value for key, value in payload.items() if value is not None} diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py index d9e2f9dd..0aed09eb 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py @@ -1002,13 +1002,6 @@ async def _run_scorer_for_result( "kind": "scorer", "started_at": started.isoformat().replace("+00:00", "Z"), } - if row.get("status") != "COMPLETE": - return self._criterion_error_result( - base, - started_clock, - "generation_incomplete", - "generation did not complete", - ) dataset_row = DatasetRow( row_index=row["row_index"], input=row.get("input"), @@ -1072,13 +1065,6 @@ async def _run_ld_judge_for_result( "variation_key": resolved.variation_key, "version": resolved.version, } - if row.get("status") != "COMPLETE": - return self._criterion_error_result( - base, - started_clock, - "generation_incomplete", - "generation did not complete", - ) # The config is passed unrendered: the handler owns the single # parse_template pass, so ``{{...}}`` sequences inside generated output # or dataset values are never re-expanded into the judge prompt. @@ -1182,7 +1168,7 @@ async def _run_criteria_for_results( evaluation_run_id: str, row_spans: Mapping[int, Any] | None = None, ) -> list[dict[str, Any]]: - """Run every (row, criterion) pair, bounded by the run's concurrency. + """Run every (generated row, criterion) pair, bounded by the run's concurrency. Each ``evaluate`` span parents under its row's span in ``row_spans``. """ @@ -1202,10 +1188,6 @@ async def run_one( ) -> dict[str, Any]: await controller.acquire() try: - if row.get("status") != "COMPLETE": - # A row that did not generate is never scored, so there is - # no judge or scorer call for an ``evaluate`` span to wrap. - return await score(row, criterion) row_index = row.get("row_index") index = row_index if isinstance(row_index, int) else 0 with bind_evaluation_row(evaluation_run_id, index): @@ -1226,9 +1208,17 @@ async def run_one( finally: controller.release() + # A row that did not generate gets no criterion result at all: its + # generation ERROR event already settled it as failed, and a criterion + # event for it would be unaccounted for in the backend's row counts. + generated = [row for row in rows if row.get("status") == "COMPLETE"] return list( await asyncio.gather( - *(run_one(row, criterion) for row in rows for criterion in criteria) + *( + run_one(row, criterion) + for row in generated + for criterion in criteria + ) ) ) @@ -1280,11 +1270,15 @@ def _emit_evaluation_events( input_tokens=normalized_usage["input"], output_tokens=normalized_usage["output"], ) + # The cause code stays internal: ingest reads only a top-level + # integer ``errorCode`` and ``errorMessage``, and no integers are + # assigned to criterion causes yet, so only the message is sent. error = result.get("error") error_message: str | None = None if result["status"] == "ERROR": - if isinstance(error, Mapping) and error.get("message"): - error_message = str(error["message"]) + if isinstance(error, Mapping): + message = error.get("message") + error_message = str(message) if message else "Unknown error" else: error_message = str(error) if error else "Unknown error" common_payload: dict[str, Any] = { @@ -1306,7 +1300,6 @@ def _emit_evaluation_events( "latency_ms": result["latency_ms"], "score": result.get("score"), "reason": result.get("reason"), - "error": error, "error_message": error_message, } payload_model: CriterionEventPayload diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index 1436b901..3bce31a7 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -13,6 +13,12 @@ from launchdarkly_ai_server import NativeTool, create_handler from launchdarkly_ai_server.evaluations.api import HttpResponse +from launchdarkly_ai_server.evaluations.runner import EvaluationsRunner +from launchdarkly_ai_server.evaluations.types import ( + DatasetRef, + EvaluationRef, + EvaluationRunRef, +) from launchdarkly_ai_server.experimental.evaluations import ( AIConfig, DatasetRow, @@ -2108,6 +2114,25 @@ def judge_run_transport(*, summary: dict[str, Any] | None = None) -> SequencedTr ) +def capture_criterion_results( + monkeypatch: pytest.MonkeyPatch, +) -> list[dict[str, Any]]: + """Record the internal criterion results, which carry the cause code. + + The cause never reaches the wire, so this is where a test tells causes apart. + """ + captured: list[dict[str, Any]] = [] + original = EvaluationsRunner._run_criteria_for_results + + async def spy(self: EvaluationsRunner, *args: Any, **kwargs: Any) -> Any: + results = await original(self, *args, **kwargs) + captured.extend(results) + return results + + monkeypatch.setattr(EvaluationsRunner, "_run_criteria_for_results", spy) + return captured + + def accuracy_judge_variation(monkeypatch: pytest.MonkeyPatch) -> None: async def fake_extract_variation( key: str, context: dict[str, Any] @@ -2167,6 +2192,7 @@ async def test_bad_judge_output_emits_error_event_instead_of_crashing( ) -> None: transport = judge_run_transport() accuracy_judge_variation(monkeypatch) + criterion_results = capture_criterion_results(monkeypatch) evals = init_evaluations( project_key="proj", api_key="token", sdk_key="sdk-key", transport=transport ) @@ -2193,14 +2219,97 @@ async def handler( events = [call.args[2] for call in stub_sdk_client.track.call_args_list] judge_event = next(event for event in events if event.get("kind") == "judge") assert judge_event["status"] == "ERROR" - assert judge_event["error"]["code"] == expected_code # The bad value is shown as JSON, not a Python repr, as in every SDK. - assert judge_event["error"]["message"] == expected_message - assert judge_event["errorMessage"] == judge_event["error"]["message"] + assert judge_event["errorMessage"] == expected_message + # The cause is internal taxonomy: ingest reads only a top-level integer + # errorCode, none is assigned yet, and a nested error object is never sent. + assert "error" not in judge_event + assert "errorCode" not in judge_event + assert [result["error"]["code"] for result in criterion_results] == [expected_code] assert "score" not in judge_event stub_sdk_client.flush.assert_awaited() +@pytest.mark.parametrize( + ("fn", "expected_code", "expected_message"), + [ + ( + lambda row, output: 1 / 0, + "scorer_raised", + "scorer fn raised: division by zero", + ), + ( + lambda row, output: 3, + "invalid_score", + "scorer fn score must be between 0 and 1, got 3", + ), + ], +) +@pytest.mark.asyncio +async def test_scorer_error_event_sends_only_a_top_level_error_message( + monkeypatch: pytest.MonkeyPatch, + stub_sdk_client: MagicMock, + fn: Callable[[Any, Any], Any], + expected_code: str, + expected_message: str, +) -> None: + transport = judge_run_transport() + criterion_results = capture_criterion_results(monkeypatch) + evals = init_evaluations( + project_key="proj", api_key="token", sdk_key="sdk-key", transport=transport + ) + + async def handler(*args: Any) -> dict[str, Any]: + return {"output": "generated"} + + await evals.run( + key="support-qa", + dataset="golden", + handler=tagged(handler), + generation={"provider": "OpenAI", "model": "gpt-4o", "mode": "completion"}, + criteria=[Scorer(name="exact", fn=fn)], + ) + + events = [call.args[2] for call in stub_sdk_client.track.call_args_list] + scorer_event = next(event for event in events if event.get("kind") == "scorer") + assert scorer_event["status"] == "ERROR" + assert scorer_event["errorMessage"] == expected_message + assert "error" not in scorer_event + assert "errorCode" not in scorer_event + assert [result["error"]["code"] for result in criterion_results] == [expected_code] + + +def test_criterion_error_without_a_message_falls_back_to_unknown_error() -> None: + client = MagicMock() + runner = EvaluationsRunner(MagicMock()) + runner._emit_evaluation_events( + client, + project_key="proj", + evaluation=EvaluationRef(id="evaluation-id", key="support-qa", version=3), + evaluation_run=EvaluationRunRef( + id="run-id", evaluation_id="evaluation-id", state="RUNNING" + ), + dataset=DatasetRef(id="dataset-id", key="golden"), + results=[ + { + "row_index": 0, + "criterion_type": "scorer:exact", + "kind": "scorer", + "status": "ERROR", + "error": {"code": "scorer_raised", "message": ""}, + "started_at": "2026-01-01T00:00:00Z", + "evaluated_at": "2026-01-01T00:00:00Z", + "latency_ms": 0, + } + ], + ) + + payload = client.track.call_args.args[2] + assert payload["errorMessage"] == "Unknown error" + assert "error" not in payload + assert "errorCode" not in payload + + @pytest.mark.asyncio async def test_generated_placeholders_are_not_expanded_into_judge_prompt( monkeypatch: pytest.MonkeyPatch, @@ -2403,17 +2512,46 @@ async def handler(*args: object) -> dict[str, Any]: @pytest.mark.asyncio -async def test_errored_generation_row_emits_generation_incomplete_criterion_event( +async def test_a_row_that_did_not_generate_is_never_scored( monkeypatch: pytest.MonkeyPatch, stub_sdk_client: MagicMock, ) -> None: - transport = judge_run_transport( - summary={"statusCounts": {"total": 1, "passed": 0, "error": 1, "pending": 0}} + """Its generation ERROR event is the whole story for the row. + + LaunchDarkly counts a failed row as soon as its generation event lands, so + a criterion result for it would be unaccounted for. The other row is + scored as usual. + """ + transport = SequencedTransport( + [ + response(200, {"id": "dataset-id", "name": "golden"}), + response( + 200, + dataset_page( + [ + {"rowIndex": 0, "input": "fails"}, + {"rowIndex": 1, "input": "works"}, + ], + total=2, + ), + ), + response(201, {"id": "evaluation-id", "name": "support-qa", "version": 3}), + response( + 201, + {"id": "run-id", "evaluationId": "evaluation-id", "state": "PENDING"}, + ), + response( + 200, + {"statusCounts": {"total": 2, "passed": 1, "error": 1, "pending": 0}}, + ), + ] ) accuracy_judge_variation(monkeypatch) evals = init_evaluations( project_key="proj", api_key="token", sdk_key="sdk-key", transport=transport ) + judged: list[str | None] = [] + scored: list[int] = [] async def handler( config: dict[str, Any], @@ -2422,22 +2560,46 @@ async def handler( variables: dict[str, Any], ) -> dict[str, Any]: if "Judge" in prompt_text(config): - raise AssertionError("judges must not run for errored generations") - raise RuntimeError("provider unavailable") + judged.append(variables["input"]) + return {"output": '{"score": 1, "reasoning": "ok"}'} + if user_input == "fails": + raise RuntimeError("provider unavailable") + return {"output": "generated"} + + def scorer(row: Any, output: Any) -> float: + scored.append(row.row_index) + return 1.0 result = await evals.run( key="support-qa", dataset="golden", handler=tagged(handler), generation={"provider": "OpenAI", "model": "gpt-4o", "mode": "completion"}, - criteria=[Judge(key="$ld:ai:judge:accuracy")], + criteria=[ + Judge(key="$ld:ai:judge:accuracy"), + Scorer(name="exact", fn=scorer), + ], ) assert result.passed is False - events = [call.args[2] for call in stub_sdk_client.track.call_args_list] - judge_event = next(event for event in events if event.get("kind") == "judge") - assert judge_event["status"] == "ERROR" - assert judge_event["error"]["code"] == "generation_incomplete" + assert judged == ["works"] + assert scored == [1] + tracked = [ + (call.args[0], call.args[2]) for call in stub_sdk_client.track.call_args_list + ] + generation = [p for name, p in tracked if name.endswith(":generation")] + criterion = [p for name, p in tracked if name.endswith(":criterion")] + assert [(p["rowIndex"], p["status"]) for p in generation] == [ + (0, "ERROR"), + (1, "COMPLETE"), + ] + assert sorted((p["rowIndex"], p["kind"]) for p in criterion) == [ + (1, "judge"), + (1, "scorer"), + ] + assert all(p["status"] == "COMPLETE" for p in criterion) + stub_sdk_client.flush.assert_awaited() + assert transport.requests[-1]["url"].endswith("/summary") @pytest.mark.asyncio