diff --git a/packages/client/README.md b/packages/client/README.md index 89ba2d07..cfc20434 100644 --- a/packages/client/README.md +++ b/packages/client/README.md @@ -196,7 +196,7 @@ A tool result is now judge-prompt input. It stays literal for the same reason th **Judges are independent AI Configs, so handlers are routed per judge.** A judge may resolve to a different provider or mode than `generation`. Each judge selects its handler from `handler` or `handlers` by its provider and mode, with the same rule as generation. There is no mode fallback: a messages-mode judge never runs on an agent handler. A judge's prompt must also match its handler's mode. Judges get no tools: the harness passes an empty tool map and removes `tools` from the judge config. -Judges are resolved through flag delivery, and handlers are matched to them, **before** any evaluation records are created — a missing judge, a judge that no handler covers, or a judge whose prompt does not match its handler fails the run up front rather than after the generation spend. One error lists every judge with a problem. After that point a criterion failure never aborts the run: an unparseable judge response, an out-of-range score, a raising handler or scorer, and a row whose generation errored each become a per-criterion `ERROR` event with a cause code (`invalid_judge_output`, `invalid_score`, `handler_raised`, `scorer_raised`, `generation_incomplete`) and a top-level `errorMessage`. Event *delivery* is different: the backend needs one result per `(row, criterion)` to finish row accounting, so if tracking a criterion event fails, every remaining result is still attempted and flushed and then `run()` raises — rather than polling to its timeout with the cause hidden. +Judges are resolved through flag delivery, and handlers are matched to them, **before** any evaluation records are created — a missing judge, a judge that no handler covers, or a judge whose prompt does not match its handler fails the run up front rather than after the generation spend. One error lists every judge with a problem. After that point a criterion failure never aborts the run: an unparseable judge response, an out-of-range score, and a raising handler or scorer each become a per-criterion `ERROR` event carrying a top-level `errorMessage` that describes the failure. A row whose generation errored is not scored at all: no judge or scorer runs for it and no criterion event is sent, because its generation `ERROR` event already records the row as failed. Event *delivery* is different: the backend needs one result per `(row, criterion)` to finish row accounting, so if tracking a criterion event fails, every remaining result is still attempted and flushed and then `run()` raises — rather than polling to its timeout with the cause hidden. The client uses **lazy initialization**: importing the package does not connect to LaunchDarkly. The singleton is created automatically on the first API call that needs it (`config().invoke()`, `graph().invoke()`, `resolve_graph()`, etc.), as long as `LD_SDK_KEY` is set in the environment. diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/events.py b/packages/client/src/launchdarkly_ai_server/evaluations/events.py index 9418ff45..07c00247 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/events.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/events.py @@ -52,7 +52,6 @@ class CriterionEventPayload: evaluation_version: int | None = None score: float | None = None reason: str | None = None - error: dict[str, Any] | None = None error_message: str | None = None def to_track_payload(self) -> dict[str, Any]: @@ -76,7 +75,6 @@ def to_track_payload(self) -> dict[str, Any]: "latencyMs": self.latency_ms, "score": self.score, "reason": self.reason, - "error": self.error, "errorMessage": self.error_message, } return {key: value for key, value in payload.items() if value is not None} diff --git a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py index d9e2f9dd..0aed09eb 100644 --- a/packages/client/src/launchdarkly_ai_server/evaluations/runner.py +++ b/packages/client/src/launchdarkly_ai_server/evaluations/runner.py @@ -1002,13 +1002,6 @@ async def _run_scorer_for_result( "kind": "scorer", "started_at": started.isoformat().replace("+00:00", "Z"), } - if row.get("status") != "COMPLETE": - return self._criterion_error_result( - base, - started_clock, - "generation_incomplete", - "generation did not complete", - ) dataset_row = DatasetRow( row_index=row["row_index"], input=row.get("input"), @@ -1072,13 +1065,6 @@ async def _run_ld_judge_for_result( "variation_key": resolved.variation_key, "version": resolved.version, } - if row.get("status") != "COMPLETE": - return self._criterion_error_result( - base, - started_clock, - "generation_incomplete", - "generation did not complete", - ) # The config is passed unrendered: the handler owns the single # parse_template pass, so ``{{...}}`` sequences inside generated output # or dataset values are never re-expanded into the judge prompt. @@ -1182,7 +1168,7 @@ async def _run_criteria_for_results( evaluation_run_id: str, row_spans: Mapping[int, Any] | None = None, ) -> list[dict[str, Any]]: - """Run every (row, criterion) pair, bounded by the run's concurrency. + """Run every (generated row, criterion) pair, bounded by the run's concurrency. Each ``evaluate`` span parents under its row's span in ``row_spans``. """ @@ -1202,10 +1188,6 @@ async def run_one( ) -> dict[str, Any]: await controller.acquire() try: - if row.get("status") != "COMPLETE": - # A row that did not generate is never scored, so there is - # no judge or scorer call for an ``evaluate`` span to wrap. - return await score(row, criterion) row_index = row.get("row_index") index = row_index if isinstance(row_index, int) else 0 with bind_evaluation_row(evaluation_run_id, index): @@ -1226,9 +1208,17 @@ async def run_one( finally: controller.release() + # A row that did not generate gets no criterion result at all: its + # generation ERROR event already settled it as failed, and a criterion + # event for it would be unaccounted for in the backend's row counts. + generated = [row for row in rows if row.get("status") == "COMPLETE"] return list( await asyncio.gather( - *(run_one(row, criterion) for row in rows for criterion in criteria) + *( + run_one(row, criterion) + for row in generated + for criterion in criteria + ) ) ) @@ -1280,11 +1270,15 @@ def _emit_evaluation_events( input_tokens=normalized_usage["input"], output_tokens=normalized_usage["output"], ) + # The cause code stays internal: ingest reads only a top-level + # integer ``errorCode`` and ``errorMessage``, and no integers are + # assigned to criterion causes yet, so only the message is sent. error = result.get("error") error_message: str | None = None if result["status"] == "ERROR": - if isinstance(error, Mapping) and error.get("message"): - error_message = str(error["message"]) + if isinstance(error, Mapping): + message = error.get("message") + error_message = str(message) if message else "Unknown error" else: error_message = str(error) if error else "Unknown error" common_payload: dict[str, Any] = { @@ -1306,7 +1300,6 @@ def _emit_evaluation_events( "latency_ms": result["latency_ms"], "score": result.get("score"), "reason": result.get("reason"), - "error": error, "error_message": error_message, } payload_model: CriterionEventPayload diff --git a/packages/client/tests/test_evaluations_run.py b/packages/client/tests/test_evaluations_run.py index 1436b901..3bce31a7 100644 --- a/packages/client/tests/test_evaluations_run.py +++ b/packages/client/tests/test_evaluations_run.py @@ -13,6 +13,12 @@ from launchdarkly_ai_server import NativeTool, create_handler from launchdarkly_ai_server.evaluations.api import HttpResponse +from launchdarkly_ai_server.evaluations.runner import EvaluationsRunner +from launchdarkly_ai_server.evaluations.types import ( + DatasetRef, + EvaluationRef, + EvaluationRunRef, +) from launchdarkly_ai_server.experimental.evaluations import ( AIConfig, DatasetRow, @@ -2108,6 +2114,25 @@ def judge_run_transport(*, summary: dict[str, Any] | None = None) -> SequencedTr ) +def capture_criterion_results( + monkeypatch: pytest.MonkeyPatch, +) -> list[dict[str, Any]]: + """Record the internal criterion results, which carry the cause code. + + The cause never reaches the wire, so this is where a test tells causes apart. + """ + captured: list[dict[str, Any]] = [] + original = EvaluationsRunner._run_criteria_for_results + + async def spy(self: EvaluationsRunner, *args: Any, **kwargs: Any) -> Any: + results = await original(self, *args, **kwargs) + captured.extend(results) + return results + + monkeypatch.setattr(EvaluationsRunner, "_run_criteria_for_results", spy) + return captured + + def accuracy_judge_variation(monkeypatch: pytest.MonkeyPatch) -> None: async def fake_extract_variation( key: str, context: dict[str, Any] @@ -2167,6 +2192,7 @@ async def test_bad_judge_output_emits_error_event_instead_of_crashing( ) -> None: transport = judge_run_transport() accuracy_judge_variation(monkeypatch) + criterion_results = capture_criterion_results(monkeypatch) evals = init_evaluations( project_key="proj", api_key="token", sdk_key="sdk-key", transport=transport ) @@ -2193,14 +2219,97 @@ async def handler( events = [call.args[2] for call in stub_sdk_client.track.call_args_list] judge_event = next(event for event in events if event.get("kind") == "judge") assert judge_event["status"] == "ERROR" - assert judge_event["error"]["code"] == expected_code # The bad value is shown as JSON, not a Python repr, as in every SDK. - assert judge_event["error"]["message"] == expected_message - assert judge_event["errorMessage"] == judge_event["error"]["message"] + assert judge_event["errorMessage"] == expected_message + # The cause is internal taxonomy: ingest reads only a top-level integer + # errorCode, none is assigned yet, and a nested error object is never sent. + assert "error" not in judge_event + assert "errorCode" not in judge_event + assert [result["error"]["code"] for result in criterion_results] == [expected_code] assert "score" not in judge_event stub_sdk_client.flush.assert_awaited() +@pytest.mark.parametrize( + ("fn", "expected_code", "expected_message"), + [ + ( + lambda row, output: 1 / 0, + "scorer_raised", + "scorer fn raised: division by zero", + ), + ( + lambda row, output: 3, + "invalid_score", + "scorer fn score must be between 0 and 1, got 3", + ), + ], +) +@pytest.mark.asyncio +async def test_scorer_error_event_sends_only_a_top_level_error_message( + monkeypatch: pytest.MonkeyPatch, + stub_sdk_client: MagicMock, + fn: Callable[[Any, Any], Any], + expected_code: str, + expected_message: str, +) -> None: + transport = judge_run_transport() + criterion_results = capture_criterion_results(monkeypatch) + evals = init_evaluations( + project_key="proj", api_key="token", sdk_key="sdk-key", transport=transport + ) + + async def handler(*args: Any) -> dict[str, Any]: + return {"output": "generated"} + + await evals.run( + key="support-qa", + dataset="golden", + handler=tagged(handler), + generation={"provider": "OpenAI", "model": "gpt-4o", "mode": "completion"}, + criteria=[Scorer(name="exact", fn=fn)], + ) + + events = [call.args[2] for call in stub_sdk_client.track.call_args_list] + scorer_event = next(event for event in events if event.get("kind") == "scorer") + assert scorer_event["status"] == "ERROR" + assert scorer_event["errorMessage"] == expected_message + assert "error" not in scorer_event + assert "errorCode" not in scorer_event + assert [result["error"]["code"] for result in criterion_results] == [expected_code] + + +def test_criterion_error_without_a_message_falls_back_to_unknown_error() -> None: + client = MagicMock() + runner = EvaluationsRunner(MagicMock()) + runner._emit_evaluation_events( + client, + project_key="proj", + evaluation=EvaluationRef(id="evaluation-id", key="support-qa", version=3), + evaluation_run=EvaluationRunRef( + id="run-id", evaluation_id="evaluation-id", state="RUNNING" + ), + dataset=DatasetRef(id="dataset-id", key="golden"), + results=[ + { + "row_index": 0, + "criterion_type": "scorer:exact", + "kind": "scorer", + "status": "ERROR", + "error": {"code": "scorer_raised", "message": ""}, + "started_at": "2026-01-01T00:00:00Z", + "evaluated_at": "2026-01-01T00:00:00Z", + "latency_ms": 0, + } + ], + ) + + payload = client.track.call_args.args[2] + assert payload["errorMessage"] == "Unknown error" + assert "error" not in payload + assert "errorCode" not in payload + + @pytest.mark.asyncio async def test_generated_placeholders_are_not_expanded_into_judge_prompt( monkeypatch: pytest.MonkeyPatch, @@ -2403,17 +2512,46 @@ async def handler(*args: object) -> dict[str, Any]: @pytest.mark.asyncio -async def test_errored_generation_row_emits_generation_incomplete_criterion_event( +async def test_a_row_that_did_not_generate_is_never_scored( monkeypatch: pytest.MonkeyPatch, stub_sdk_client: MagicMock, ) -> None: - transport = judge_run_transport( - summary={"statusCounts": {"total": 1, "passed": 0, "error": 1, "pending": 0}} + """Its generation ERROR event is the whole story for the row. + + LaunchDarkly counts a failed row as soon as its generation event lands, so + a criterion result for it would be unaccounted for. The other row is + scored as usual. + """ + transport = SequencedTransport( + [ + response(200, {"id": "dataset-id", "name": "golden"}), + response( + 200, + dataset_page( + [ + {"rowIndex": 0, "input": "fails"}, + {"rowIndex": 1, "input": "works"}, + ], + total=2, + ), + ), + response(201, {"id": "evaluation-id", "name": "support-qa", "version": 3}), + response( + 201, + {"id": "run-id", "evaluationId": "evaluation-id", "state": "PENDING"}, + ), + response( + 200, + {"statusCounts": {"total": 2, "passed": 1, "error": 1, "pending": 0}}, + ), + ] ) accuracy_judge_variation(monkeypatch) evals = init_evaluations( project_key="proj", api_key="token", sdk_key="sdk-key", transport=transport ) + judged: list[str | None] = [] + scored: list[int] = [] async def handler( config: dict[str, Any], @@ -2422,22 +2560,46 @@ async def handler( variables: dict[str, Any], ) -> dict[str, Any]: if "Judge" in prompt_text(config): - raise AssertionError("judges must not run for errored generations") - raise RuntimeError("provider unavailable") + judged.append(variables["input"]) + return {"output": '{"score": 1, "reasoning": "ok"}'} + if user_input == "fails": + raise RuntimeError("provider unavailable") + return {"output": "generated"} + + def scorer(row: Any, output: Any) -> float: + scored.append(row.row_index) + return 1.0 result = await evals.run( key="support-qa", dataset="golden", handler=tagged(handler), generation={"provider": "OpenAI", "model": "gpt-4o", "mode": "completion"}, - criteria=[Judge(key="$ld:ai:judge:accuracy")], + criteria=[ + Judge(key="$ld:ai:judge:accuracy"), + Scorer(name="exact", fn=scorer), + ], ) assert result.passed is False - events = [call.args[2] for call in stub_sdk_client.track.call_args_list] - judge_event = next(event for event in events if event.get("kind") == "judge") - assert judge_event["status"] == "ERROR" - assert judge_event["error"]["code"] == "generation_incomplete" + assert judged == ["works"] + assert scored == [1] + tracked = [ + (call.args[0], call.args[2]) for call in stub_sdk_client.track.call_args_list + ] + generation = [p for name, p in tracked if name.endswith(":generation")] + criterion = [p for name, p in tracked if name.endswith(":criterion")] + assert [(p["rowIndex"], p["status"]) for p in generation] == [ + (0, "ERROR"), + (1, "COMPLETE"), + ] + assert sorted((p["rowIndex"], p["kind"]) for p in criterion) == [ + (1, "judge"), + (1, "scorer"), + ] + assert all(p["status"] == "COMPLETE" for p in criterion) + stub_sdk_client.flush.assert_awaited() + assert transport.requests[-1]["url"].endswith("/summary") @pytest.mark.asyncio