diff --git a/inference_gateway/config.py b/inference_gateway/config.py index 8cbc383a..b76cead8 100644 --- a/inference_gateway/config.py +++ b/inference_gateway/config.py @@ -55,6 +55,12 @@ logger.fatal("MAX_COST_PER_EVALUATION_RUN_USD is not set in .env") MAX_COST_PER_EVALUATION_RUN_USD = float(MAX_COST_PER_EVALUATION_RUN_USD) +MAX_INFERENCE_ERRORS_PER_EVALUATION_RUN = os.getenv("MAX_INFERENCE_ERRORS_PER_EVALUATION_RUN") +if not MAX_INFERENCE_ERRORS_PER_EVALUATION_RUN: + MAX_INFERENCE_ERRORS_PER_EVALUATION_RUN = 5 + logger.warning(f"MAX_INFERENCE_ERRORS_PER_EVALUATION_RUN is not set in .env, defaulting to {MAX_INFERENCE_ERRORS_PER_EVALUATION_RUN}") +MAX_INFERENCE_ERRORS_PER_EVALUATION_RUN = int(MAX_INFERENCE_ERRORS_PER_EVALUATION_RUN) + USE_CHUTES = os.getenv("USE_CHUTES") if not USE_CHUTES: diff --git a/inference_gateway/error_hash_map.py b/inference_gateway/error_hash_map.py new file mode 100644 index 00000000..06078f3e --- /dev/null +++ b/inference_gateway/error_hash_map.py @@ -0,0 +1,50 @@ +# Tracks the number of platform-side inference errors per evaluation run. +# When the count exceeds a configured threshold the run is flagged as a +# platform error so the agent is not penalized unfairly. + +import time + +from uuid import UUID +from pydantic import BaseModel + + + +ERROR_HASH_MAP_CLEANUP_INTERVAL_SECONDS = 60 # 1 minute + +class ErrorHashMapEntry(BaseModel): + inference_errors: int + last_accessed_at: float + +class ErrorHashMap: + def __init__(self): + self.error_hash_map = {} + self.last_cleanup_at = time.time() + + + + def _cleanup(self): + now = time.time() + if now - self.last_cleanup_at > ERROR_HASH_MAP_CLEANUP_INTERVAL_SECONDS: + self.error_hash_map = {k: v for k, v in self.error_hash_map.items() if now - v.last_accessed_at < ERROR_HASH_MAP_CLEANUP_INTERVAL_SECONDS} + self.last_cleanup_at = now + + + + def get_inference_errors(self, uuid: UUID) -> int: + self._cleanup() + + if uuid in self.error_hash_map: + self.error_hash_map[uuid].last_accessed_at = time.time() + return self.error_hash_map[uuid].inference_errors + else: + return 0 + + def add_inference_error(self, uuid: UUID): + self._cleanup() + + if uuid in self.error_hash_map: + entry = self.error_hash_map[uuid] + entry.inference_errors += 1 + entry.last_accessed_at = time.time() + else: + self.error_hash_map[uuid] = ErrorHashMapEntry(inference_errors=1, last_accessed_at=time.time()) diff --git a/inference_gateway/main.py b/inference_gateway/main.py index 4eb7ea72..4ef615c9 100644 --- a/inference_gateway/main.py +++ b/inference_gateway/main.py @@ -11,6 +11,7 @@ import inference_gateway.config as config import utils.logger as logger from inference_gateway.cost_hash_map import CostHashMap +from inference_gateway.error_hash_map import ErrorHashMap from inference_gateway.models import ( EmbeddingModelInfo, EmbeddingRequest, @@ -103,6 +104,18 @@ async def wrapper(*args, **kwargs): cost_hash_map = CostHashMap() +error_hash_map = ErrorHashMap() + + +# Platform-side error codes that are not the agent's fault. 4xx errors like +# 400/404/422 are excluded because those indicate bad agent requests (wrong +# model, invalid format, etc). 429 is excluded because it means the cost +# limit was intentionally reached. +PLATFORM_ERROR_CODES = {500, 502, 503, 504, -1} + + +def is_platform_error(status_code: int) -> bool: + return status_code in PLATFORM_ERROR_CODES # NOTE ADAM: inference@main.py -> Handles HTTP exceptions and database @@ -149,6 +162,17 @@ async def inference(request: InferenceRequest) -> InferenceResponse: detail=f"The evaluation run with ID {request.evaluation_run_id} has reached or exceeded the evaluation run cost limit of {config.MAX_COST_PER_EVALUATION_RUN_USD} USD (current cost: {cost} USD).", ) + # Make sure the evaluation run has not had too many platform-side inference errors + inference_errors = error_hash_map.get_inference_errors(request.evaluation_run_id) + if inference_errors >= config.MAX_INFERENCE_ERRORS_PER_EVALUATION_RUN: + logger.warning( + f"Blocking inference for run {request.evaluation_run_id}: too many platform errors ({inference_errors}/{config.MAX_INFERENCE_ERRORS_PER_EVALUATION_RUN})" + ) + raise HTTPException( + status_code=503, + detail=f"The evaluation run with ID {request.evaluation_run_id} has had too many platform-side inference errors ({inference_errors} errors, limit is {config.MAX_INFERENCE_ERRORS_PER_EVALUATION_RUN}).", + ) + # Make sure we support the model for inference provider = get_provider_that_supports_model_for_inference(request.model) if not provider: @@ -189,6 +213,14 @@ async def inference(request: InferenceRequest) -> InferenceResponse: if response.status_code == 200: return InferenceResponse(content=response.content, tool_calls=response.tool_calls) else: + # Track platform errors (provider failures, not agent mistakes) + if is_platform_error(response.status_code): + error_hash_map.add_inference_error(request.evaluation_run_id) + error_count = error_hash_map.get_inference_errors(request.evaluation_run_id) + logger.warning( + f"Platform inference error for run {request.evaluation_run_id}: status {response.status_code} (error {error_count}/{config.MAX_INFERENCE_ERRORS_PER_EVALUATION_RUN})" + ) + raise HTTPException(status_code=response.status_code, detail=response.error_message) @@ -224,6 +256,17 @@ async def embedding(request: EmbeddingRequest) -> EmbeddingResponse: detail=f"The evaluation run with ID {request.evaluation_run_id} has reached or exceeded the evaluation run cost limit of {config.MAX_COST_PER_EVALUATION_RUN_USD} USD (current cost: {cost} USD).", ) + # Make sure the evaluation run has not had too many platform-side inference errors + inference_errors = error_hash_map.get_inference_errors(request.evaluation_run_id) + if inference_errors >= config.MAX_INFERENCE_ERRORS_PER_EVALUATION_RUN: + logger.warning( + f"Blocking embedding for run {request.evaluation_run_id}: too many platform errors ({inference_errors}/{config.MAX_INFERENCE_ERRORS_PER_EVALUATION_RUN})" + ) + raise HTTPException( + status_code=503, + detail=f"The evaluation run with ID {request.evaluation_run_id} has had too many platform-side inference errors ({inference_errors} errors, limit is {config.MAX_INFERENCE_ERRORS_PER_EVALUATION_RUN}).", + ) + # Make sure we support the model for embedding provider = get_provider_that_supports_model_for_embedding(request.model) if not provider: @@ -256,6 +299,14 @@ async def embedding(request: EmbeddingRequest) -> EmbeddingResponse: if response.status_code == 200: return EmbeddingResponse(embedding=response.embedding) else: + # Track platform errors (provider failures, not agent mistakes) + if is_platform_error(response.status_code): + error_hash_map.add_inference_error(request.evaluation_run_id) + error_count = error_hash_map.get_inference_errors(request.evaluation_run_id) + logger.warning( + f"Platform embedding error for run {request.evaluation_run_id}: status {response.status_code} (error {error_count}/{config.MAX_INFERENCE_ERRORS_PER_EVALUATION_RUN})" + ) + raise HTTPException(status_code=response.status_code, detail=response.error_message) @@ -263,6 +314,8 @@ class UsageResponse(BaseModel): used_cost_usd: float remaining_cost_usd: float max_cost_usd: float + inference_errors: int + max_inference_errors: int @app.get("/api/usage") @@ -273,6 +326,8 @@ async def usage(evaluation_run_id: UUID) -> UsageResponse: used_cost_usd=used_cost_usd, remaining_cost_usd=config.MAX_COST_PER_EVALUATION_RUN_USD - used_cost_usd, max_cost_usd=config.MAX_COST_PER_EVALUATION_RUN_USD, + inference_errors=error_hash_map.get_inference_errors(evaluation_run_id), + max_inference_errors=config.MAX_INFERENCE_ERRORS_PER_EVALUATION_RUN, ) diff --git a/models/evaluation_run.py b/models/evaluation_run.py index d2e539aa..c450842c 100644 --- a/models/evaluation_run.py +++ b/models/evaluation_run.py @@ -65,6 +65,10 @@ def __new__(cls, code: int, message: str): 3040, "The platform was restarted while the evaluation run was running the evaluation", ) + PLATFORM_TOO_MANY_INFERENCE_ERRORS = ( + 3050, + "Too many platform-side inference errors occurred during the evaluation run", + ) def get_error_message(self) -> str: return self.message diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/test_inference_error_tracking.py b/tests/test_inference_error_tracking.py new file mode 100644 index 00000000..f09766e0 --- /dev/null +++ b/tests/test_inference_error_tracking.py @@ -0,0 +1,371 @@ +""" +Tests for platform-side inference error tracking. +Covers: + - ErrorHashMap error counting and cleanup + - Non-halting error code classification + - Inference gateway threshold enforcement via the /api/inference endpoint + - Usage endpoint reporting error counts +""" + +import os +import time +import pytest +from uuid import uuid4 + +from inference_gateway.error_hash_map import ErrorHashMap, ERROR_HASH_MAP_CLEANUP_INTERVAL_SECONDS + + + +# --------------------------------------------------------------------------- +# Unit tests: ErrorHashMap +# --------------------------------------------------------------------------- + +class TestErrorHashMap: + def setup_method(self): + self.ehm = ErrorHashMap() + self.run_id = uuid4() + + def test_get_inference_errors_returns_zero_for_unknown_run(self): + assert self.ehm.get_inference_errors(uuid4()) == 0 + + def test_add_inference_error_creates_entry(self): + self.ehm.add_inference_error(self.run_id) + assert self.ehm.get_inference_errors(self.run_id) == 1 + + def test_add_inference_error_increments(self): + for _ in range(7): + self.ehm.add_inference_error(self.run_id) + assert self.ehm.get_inference_errors(self.run_id) == 7 + + def test_separate_runs_tracked_independently(self): + run_a = uuid4() + run_b = uuid4() + + self.ehm.add_inference_error(run_a) + self.ehm.add_inference_error(run_a) + self.ehm.add_inference_error(run_b) + + assert self.ehm.get_inference_errors(run_a) == 2 + assert self.ehm.get_inference_errors(run_b) == 1 + + def test_cleanup_removes_stale_entries(self): + self.ehm.add_inference_error(self.run_id) + assert self.ehm.get_inference_errors(self.run_id) == 1 + + # Simulate the entry going stale + self.ehm.error_hash_map[self.run_id].last_accessed_at = time.time() - ERROR_HASH_MAP_CLEANUP_INTERVAL_SECONDS - 1 + self.ehm.last_cleanup_at = time.time() - ERROR_HASH_MAP_CLEANUP_INTERVAL_SECONDS - 1 + + # Next access triggers cleanup, entry is gone + assert self.ehm.get_inference_errors(self.run_id) == 0 + + + +# --------------------------------------------------------------------------- +# Unit tests: platform error classification +# +# We define the expected set here to avoid importing inference_gateway.main, +# which triggers the config import chain and requires env vars. +# --------------------------------------------------------------------------- + +EXPECTED_PLATFORM_ERROR_CODES = {500, 502, 503, 504, -1} + +class TestPlatformErrorClassification: + def test_server_errors_are_platform_errors(self): + for code in [500, 502, 503, 504]: + assert code in EXPECTED_PLATFORM_ERROR_CODES, f"Expected {code} to be a platform error" + + def test_internal_error_is_platform_error(self): + assert -1 in EXPECTED_PLATFORM_ERROR_CODES + + def test_client_errors_are_not_platform_errors(self): + for code in [400, 404, 422, 429]: + assert code not in EXPECTED_PLATFORM_ERROR_CODES, f"Expected {code} to not be a platform error" + + def test_success_is_not_platform_error(self): + assert 200 not in EXPECTED_PLATFORM_ERROR_CODES + + + +# --------------------------------------------------------------------------- +# Unit tests: EvaluationRunErrorCode +# --------------------------------------------------------------------------- + +class TestEvaluationRunErrorCode: + def test_new_error_code_exists(self): + from models.evaluation_run import EvaluationRunErrorCode + code = EvaluationRunErrorCode.PLATFORM_TOO_MANY_INFERENCE_ERRORS + assert code.value == 3050 + assert code.is_platform_error() + assert not code.is_agent_error() + assert not code.is_validator_error() + + def test_error_message(self): + from models.evaluation_run import EvaluationRunErrorCode + msg = EvaluationRunErrorCode.PLATFORM_TOO_MANY_INFERENCE_ERRORS.get_error_message() + assert "inference errors" in msg.lower() + + + +# --------------------------------------------------------------------------- +# Integration tests: inference gateway endpoints +# +# These require the full app to be importable. They set minimal env vars +# to satisfy the config module, then mock the providers. +# --------------------------------------------------------------------------- + +def _set_minimal_gateway_env(): + """Set the bare minimum env vars so inference_gateway.config can load.""" + defaults = { + "HOST": "0.0.0.0", + "PORT": "9999", + "USE_DATABASE": "false", + "MAX_COST_PER_EVALUATION_RUN_USD": "10.0", + "MAX_INFERENCE_ERRORS_PER_EVALUATION_RUN": "5", + "USE_CHUTES": "false", + "USE_TARGON": "false", + "USE_OPENROUTER": "false", + "TEST_INFERENCE_MODELS": "false", + "TEST_EMBEDDING_MODELS": "false", + } + for key, val in defaults.items(): + os.environ.setdefault(key, val) + + +# Guard: only define integration tests if we can import the app +_can_import_app = False +try: + _set_minimal_gateway_env() + + # The config fatals if no provider is enabled, so we need to patch + # the fatal check. We do this by temporarily setting one provider. + os.environ["USE_OPENROUTER"] = "true" + os.environ["OPENROUTER_BASE_URL"] = "http://localhost:9999" + os.environ["OPENROUTER_API_KEY"] = "test-key" + os.environ["OPENROUTER_WEIGHT"] = "1" + + from inference_gateway.main import app, cost_hash_map as global_cost_hash_map, error_hash_map as global_error_hash_map, is_platform_error, PLATFORM_ERROR_CODES + from inference_gateway.models import InferenceResult, EmbeddingResult + _can_import_app = True +except Exception: + pass + + +if _can_import_app: + import pytest_asyncio + from unittest.mock import AsyncMock, patch, MagicMock + from httpx import ASGITransport, AsyncClient + + @pytest_asyncio.fixture + async def client(): + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://test") as c: + yield c + + def _mock_provider(status_code=200, for_embedding=False): + """Create a mock provider that returns the given status code.""" + provider = MagicMock() + provider.name = "MockProvider" + provider.is_model_supported_for_inference.return_value = True + provider.is_model_supported_for_embedding.return_value = True + + inference_result = InferenceResult( + status_code=status_code, + content="hello" if status_code == 200 else None, + error_message="provider error" if status_code != 200 else None, + tool_calls=[], + num_input_tokens=10, + num_output_tokens=5, + cost_usd=0.001 + ) + provider.inference = AsyncMock(return_value=inference_result) + + embedding_result = EmbeddingResult( + status_code=status_code, + embedding=[0.1, 0.2, 0.3] if status_code == 200 else None, + error_message="provider error" if status_code != 200 else None, + num_input_tokens=10, + cost_usd=0.0005 + ) + provider.embedding = AsyncMock(return_value=embedding_result) + + return provider + + + @pytest.mark.asyncio + class TestInferenceGatewayErrorTracking: + def setup_method(self): + global_cost_hash_map.cost_hash_map = {} + global_cost_hash_map.last_cleanup_at = time.time() + global_error_hash_map.error_hash_map = {} + global_error_hash_map.last_cleanup_at = time.time() + + async def test_platform_error_increments_counter(self, client): + run_id = str(uuid4()) + mock_provider = _mock_provider(status_code=500) + + with patch("inference_gateway.main.get_provider_that_supports_model_for_inference", return_value=mock_provider), \ + patch("inference_gateway.main.config") as mock_config: + mock_config.USE_DATABASE = False + mock_config.MAX_COST_PER_EVALUATION_RUN_USD = 10.0 + mock_config.MAX_INFERENCE_ERRORS_PER_EVALUATION_RUN = 5 + + response = await client.post("/api/inference", json={ + "evaluation_run_id": run_id, + "model": "test-model", + "temperature": 0.5, + "messages": [{"role": "user", "content": "test"}] + }) + + assert response.status_code == 500 + from uuid import UUID + assert global_error_hash_map.get_inference_errors(UUID(run_id)) == 1 + + async def test_non_platform_error_does_not_increment_counter(self, client): + run_id = str(uuid4()) + mock_provider = _mock_provider(status_code=422) + + with patch("inference_gateway.main.get_provider_that_supports_model_for_inference", return_value=mock_provider), \ + patch("inference_gateway.main.config") as mock_config: + mock_config.USE_DATABASE = False + mock_config.MAX_COST_PER_EVALUATION_RUN_USD = 10.0 + mock_config.MAX_INFERENCE_ERRORS_PER_EVALUATION_RUN = 5 + + response = await client.post("/api/inference", json={ + "evaluation_run_id": run_id, + "model": "test-model", + "temperature": 0.5, + "messages": [{"role": "user", "content": "test"}] + }) + + assert response.status_code == 422 + from uuid import UUID + assert global_error_hash_map.get_inference_errors(UUID(run_id)) == 0 + + async def test_threshold_blocks_further_requests(self, client): + run_id = str(uuid4()) + from uuid import UUID + run_uuid = UUID(run_id) + + # Pre-fill the error count to the limit + for _ in range(5): + global_error_hash_map.add_inference_error(run_uuid) + + mock_provider = _mock_provider(status_code=200) + + with patch("inference_gateway.main.get_provider_that_supports_model_for_inference", return_value=mock_provider), \ + patch("inference_gateway.main.config") as mock_config: + mock_config.USE_DATABASE = True + mock_config.CHECK_EVALUATION_RUNS = True + mock_config.MAX_COST_PER_EVALUATION_RUN_USD = 10.0 + mock_config.MAX_INFERENCE_ERRORS_PER_EVALUATION_RUN = 5 + + with patch("inference_gateway.main.get_evaluation_run_status_by_id", new_callable=AsyncMock) as mock_status: + from models.evaluation_run import EvaluationRunStatus + mock_status.return_value = EvaluationRunStatus.running_agent + + response = await client.post("/api/inference", json={ + "evaluation_run_id": run_id, + "model": "test-model", + "temperature": 0.5, + "messages": [{"role": "user", "content": "test"}] + }) + + assert response.status_code == 503 + assert "too many platform-side inference errors" in response.json()["detail"].lower() + + async def test_usage_endpoint_reports_errors(self, client): + run_id = str(uuid4()) + from uuid import UUID + run_uuid = UUID(run_id) + + global_error_hash_map.add_inference_error(run_uuid) + global_error_hash_map.add_inference_error(run_uuid) + global_cost_hash_map.add_cost(run_uuid, 0.05) + + with patch("inference_gateway.main.config") as mock_config: + mock_config.MAX_COST_PER_EVALUATION_RUN_USD = 10.0 + mock_config.MAX_INFERENCE_ERRORS_PER_EVALUATION_RUN = 5 + + response = await client.get(f"/api/usage?evaluation_run_id={run_id}") + + assert response.status_code == 200 + data = response.json() + assert data["inference_errors"] == 2 + assert data["max_inference_errors"] == 5 + assert data["used_cost_usd"] == 0.05 + + async def test_usage_endpoint_zero_errors(self, client): + """A run with no errors should report zero.""" + run_id = str(uuid4()) + + with patch("inference_gateway.main.config") as mock_config: + mock_config.MAX_COST_PER_EVALUATION_RUN_USD = 10.0 + mock_config.MAX_INFERENCE_ERRORS_PER_EVALUATION_RUN = 5 + + response = await client.get(f"/api/usage?evaluation_run_id={run_id}") + + assert response.status_code == 200 + data = response.json() + assert data["inference_errors"] == 0 + assert data["used_cost_usd"] == 0.0 + + async def test_separate_runs_do_not_interfere(self, client): + """Errors for one run must not affect another run's count.""" + run_a = str(uuid4()) + run_b = str(uuid4()) + mock_provider = _mock_provider(status_code=502) + + with patch("inference_gateway.main.get_provider_that_supports_model_for_inference", return_value=mock_provider), \ + patch("inference_gateway.main.config") as mock_config: + mock_config.USE_DATABASE = False + mock_config.MAX_COST_PER_EVALUATION_RUN_USD = 10.0 + mock_config.MAX_INFERENCE_ERRORS_PER_EVALUATION_RUN = 5 + + # Hit run_a three times + for _ in range(3): + await client.post("/api/inference", json={ + "evaluation_run_id": run_a, + "model": "test-model", + "temperature": 0.5, + "messages": [{"role": "user", "content": "test"}] + }) + + # Hit run_b once + await client.post("/api/inference", json={ + "evaluation_run_id": run_b, + "model": "test-model", + "temperature": 0.5, + "messages": [{"role": "user", "content": "test"}] + }) + + from uuid import UUID + assert global_error_hash_map.get_inference_errors(UUID(run_a)) == 3 + assert global_error_hash_map.get_inference_errors(UUID(run_b)) == 1 + + async def test_embedding_platform_error_increments_counter(self, client): + """Embedding endpoint should also track platform errors.""" + run_id = str(uuid4()) + mock_provider = _mock_provider(status_code=503) + + with patch("inference_gateway.main.get_provider_that_supports_model_for_embedding", return_value=mock_provider), \ + patch("inference_gateway.main.config") as mock_config: + mock_config.USE_DATABASE = False + mock_config.MAX_COST_PER_EVALUATION_RUN_USD = 10.0 + mock_config.MAX_INFERENCE_ERRORS_PER_EVALUATION_RUN = 5 + + response = await client.post("/api/embedding", json={ + "evaluation_run_id": run_id, + "model": "test-model", + "input": "hello world" + }) + + assert response.status_code == 503 + from uuid import UUID + assert global_error_hash_map.get_inference_errors(UUID(run_id)) == 1 + + async def test_constants_match_expected(self): + """Verify the actual constants match what we test against.""" + assert PLATFORM_ERROR_CODES == EXPECTED_PLATFORM_ERROR_CODES + assert is_platform_error(500) + assert not is_platform_error(400) diff --git a/validator/main.py b/validator/main.py index 971f3090..1c9cc2b9 100644 --- a/validator/main.py +++ b/validator/main.py @@ -235,6 +235,28 @@ async def _run_evaluation_run(evaluation_run_id: UUID, problem_name: str, agent_ f"Finished running agent for problem {problem_name}: {len(patch.splitlines())} lines of patch, {len(agent_logs.splitlines())} lines of agent logs" ) + # Check if the agent was affected by platform-side inference errors. + # If the inference gateway saw too many non-halting errors for this + # run, the agent never had a fair chance, so we bail out early and + # mark this as a platform error instead of scoring a bad patch. + try: + async with httpx.AsyncClient(timeout=10.0) as client: + usage_response = await client.get(f"{config.RIDGES_INFERENCE_GATEWAY_URL}/api/usage?evaluation_run_id={evaluation_run_id}") + if usage_response.status_code == 200: + usage = usage_response.json() + inference_errors = usage.get("inference_errors", 0) + max_inference_errors = usage.get("max_inference_errors", float("inf")) + if inference_errors >= max_inference_errors: + raise EvaluationRunException( + EvaluationRunErrorCode.PLATFORM_TOO_MANY_INFERENCE_ERRORS, + f"{EvaluationRunErrorCode.PLATFORM_TOO_MANY_INFERENCE_ERRORS.get_error_message()}: {inference_errors} inference errors (limit: {max_inference_errors})", + extra={"agent_logs": truncate_logs_if_required(agent_logs)} + ) + except EvaluationRunException: + raise + except Exception as e: + logger.warning(f"Failed to check inference error count for evaluation run {evaluation_run_id}: {e}") + # Move from running_agent -> initializing_eval await update_evaluation_run( evaluation_run_id, @@ -285,7 +307,7 @@ async def _run_evaluation_run(evaluation_run_id: UUID, problem_name: str, agent_ evaluation_run_id, problem_name, EvaluationRunStatus.error, - {"error_code": e.error_code.value, "error_message": e.error_message}, + {"error_code": e.error_code.value, "error_message": e.error_message, **(e.extra or {})}, ) except Exception as e: