From d0569b5a643727f15cd1eaef7fe76ef11dd86c9c Mon Sep 17 00:00:00 2001 From: Steve Liu Date: Wed, 12 Nov 2025 13:25:10 -0800 Subject: [PATCH 1/4] test --- .../distro/aws_opentelemetry_configurator.py | 31 ++++++------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_configurator.py b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_configurator.py index 41fa542db..9dfe81df2 100644 --- a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_configurator.py +++ b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_configurator.py @@ -95,7 +95,7 @@ OTEL_EXPORTER_OTLP_TRACES_ENDPOINT = "OTEL_EXPORTER_OTLP_TRACES_ENDPOINT" OTEL_EXPORTER_OTLP_LOGS_ENDPOINT = "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT" OTEL_EXPORTER_OTLP_LOGS_HEADERS = "OTEL_EXPORTER_OTLP_LOGS_HEADERS" -CODE_CORRELATION_ENABLED_CONFIG = "OTEL_AWS_CODE_CORRELATION_ENABLED" +OTEL_AWS_ENHANCED_CODE_ATTRIBUTES = "OTEL_AWS_ENHANCED_CODE_ATTRIBUTES" XRAY_SERVICE = "xray" LOGS_SERIVCE = "logs" @@ -473,7 +473,7 @@ def _customize_logs_exporter(log_exporter: LogExporter) -> LogExporter: def _customize_span_processors(provider: TracerProvider, resource: Resource) -> None: - if get_code_correlation_enabled_status() is True: + if is_enhanced_code_attributes() is True: # pylint: disable=import-outside-toplevel from amazon.opentelemetry.distro.code_correlation import CodeAttributesSpanProcessor @@ -622,32 +622,21 @@ def _is_application_signals_runtime_enabled(): ) -def get_code_correlation_enabled_status() -> Optional[bool]: +def is_enhanced_code_attributes() -> bool: """ - Get the code correlation enabled status from environment variable. + Get the enhanced code attributes enabled status from environment variable. Returns: - True if OTEL_AWS_CODE_CORRELATION_ENABLED is set to 'true' - False if OTEL_AWS_CODE_CORRELATION_ENABLED is set to 'false' - None if OTEL_AWS_CODE_CORRELATION_ENABLED is not set (default state) + True if OTEL_AWS_ENHANCED_CODE_ATTRIBUTES is set to 'true' + else False """ - env_value = os.environ.get(CODE_CORRELATION_ENABLED_CONFIG) - - if env_value is None: - return None # Default state - environment variable not set + env_value = os.environ.get(OTEL_AWS_ENHANCED_CODE_ATTRIBUTES, "false") env_value_lower = env_value.strip().lower() if env_value_lower == "true": return True - if env_value_lower == "false": - return False - # Invalid value, treat as default and log warning - _logger.warning( - "Invalid value for %s: %s. Expected 'true' or 'false'. Using default.", - CODE_CORRELATION_ENABLED_CONFIG, - env_value, - ) - return None + + return False def _is_lambda_environment(): @@ -907,4 +896,4 @@ def _create_aws_otlp_exporter(endpoint: str, service: str, region: str): # pylint: disable=broad-exception-caught except Exception as errors: _logger.error("Failed to create AWS OTLP exporter: %s", errors) - return None + return None \ No newline at end of file From 952af6546c983bfdb32009460c7bb4337529ac94 Mon Sep 17 00:00:00 2001 From: Steve Liu Date: Tue, 2 Dec 2025 12:08:49 -0800 Subject: [PATCH 2/4] fix botocore malformed response handling for converse stream operations --- .../distro/patches/_botocore_patches.py | 21 ++++++++++++++++++- .../patches/test_instrumentation_patch.py | 10 +++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_botocore_patches.py b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_botocore_patches.py index a415f6148..603988ecc 100644 --- a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_botocore_patches.py +++ b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_botocore_patches.py @@ -334,9 +334,10 @@ def patched_extract_tool_calls( tool_calls.append(tool_call) return tool_calls - # TODO: The following code is to patch a bedrock bug that was fixed in + # TODO: The following code is to patch bedrock bugs that were fixed in # opentelemetry-instrumentation-botocore==0.60b0 in: # https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3875 + # https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3990 # Remove this code once we've bumped opentelemetry-instrumentation-botocore dependency to 0.60b0 def patched_process_anthropic_claude_chunk(self, chunk): # pylint: disable=too-many-return-statements,too-many-branches @@ -412,12 +413,30 @@ def patched_process_anthropic_claude_chunk(self, chunk): self._stream_done_callback(self._response) return + def patched_from_converse(cls, response: dict[str, Any], capture_content: bool) -> bedrock_utils._Choice: + # be defensive about malformed responses, refer to #3958 for more context + output = response.get("output", {}) + orig_message = output.get("message", {}) + if role := orig_message.get("role"): + message = {"role": role} + else: + # amazon.titan does not serialize the role + message = {} + + if tool_calls := bedrock_utils.extract_tool_calls(orig_message, capture_content): + message["tool_calls"] = tool_calls + elif capture_content: + message["content"] = orig_message["content"] + + return cls(message, response["stopReason"], index=0) + bedrock_utils.ConverseStreamWrapper.__init__ = patched_init bedrock_utils.ConverseStreamWrapper._process_event = patched_process_event bedrock_utils.InvokeModelWithResponseStreamWrapper._process_anthropic_claude_chunk = ( patched_process_anthropic_claude_chunk ) bedrock_utils.extract_tool_calls = patched_extract_tool_calls + bedrock_utils._Choice.from_converse = classmethod(patched_from_converse) # END The OpenTelemetry Authors code diff --git a/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/patches/test_instrumentation_patch.py b/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/patches/test_instrumentation_patch.py index e27f881aa..5880e8491 100644 --- a/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/patches/test_instrumentation_patch.py +++ b/aws-opentelemetry-distro/tests/amazon/opentelemetry/distro/patches/test_instrumentation_patch.py @@ -250,6 +250,7 @@ def _test_patched_botocore_instrumentation(self): self._test_patched_process_anthropic_claude_chunk({"location": "Seattle"}, {"location": "Seattle"}) self._test_patched_process_anthropic_claude_chunk(None, None) self._test_patched_process_anthropic_claude_chunk({}, {}) + self._test_patched_from_converse_with_malformed_response() # Bedrock Agent Runtime self.assertTrue("bedrock-agent-runtime" in _KNOWN_EXTENSIONS) @@ -645,6 +646,15 @@ def _test_patched_extract_tool_calls(self): result = bedrock_utils.extract_tool_calls(message_with_type_tool_use, True) self.assertEqual(len(result), 1) + def _test_patched_from_converse_with_malformed_response(self): + """Test patched from_converse handles malformed response missing output key""" + malformed_response = {"stopReason": "end_turn"} + choice = bedrock_utils._Choice.from_converse(malformed_response, capture_content=False) + + self.assertEqual(choice.finish_reason, "end_turn") + self.assertEqual(choice.message, {}) + self.assertEqual(choice.index, 0) + def _test_patched_process_anthropic_claude_chunk( self, input_value: Dict[str, str], expected_output: Dict[str, str] ): From a3109c8c0f7c9f517e0dbb6c537d4918ec9088b8 Mon Sep 17 00:00:00 2001 From: Steve Liu Date: Tue, 2 Dec 2025 12:15:12 -0800 Subject: [PATCH 3/4] fix botocore malformed response handling for converse stream operations --- CHANGELOG.md | 2 ++ .../opentelemetry/distro/aws_opentelemetry_configurator.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9daaa7803..a086e69e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,4 +23,6 @@ If your change does not need a CHANGELOG entry, add the "skip changelog" label t ([#522](https://github.com/aws-observability/aws-otel-python-instrumentation/pull/522)) - Support credentials provider name for BedrockAgentCore Identity ([#534](https://github.com/aws-observability/aws-otel-python-instrumentation/pull/534)) +- [PATCH] Add safety check for bedrock ConverseStream responses + ([#547](https://github.com/aws-observability/aws-otel-python-instrumentation/pull/547)) diff --git a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_configurator.py b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_configurator.py index c97662cfd..9924fdfcd 100644 --- a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_configurator.py +++ b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/aws_opentelemetry_configurator.py @@ -897,4 +897,4 @@ def _create_aws_otlp_exporter(endpoint: str, service: str, region: str): # pylint: disable=broad-exception-caught except Exception as errors: _logger.error("Failed to create AWS OTLP exporter: %s", errors) - return None \ No newline at end of file + return None From 982ee2d016e56fbb02cef00fe29d301047821b99 Mon Sep 17 00:00:00 2001 From: Steve Liu Date: Tue, 2 Dec 2025 14:36:13 -0800 Subject: [PATCH 4/4] defensive content checking --- .../amazon/opentelemetry/distro/patches/_botocore_patches.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_botocore_patches.py b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_botocore_patches.py index 603988ecc..22af87af5 100644 --- a/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_botocore_patches.py +++ b/aws-opentelemetry-distro/src/amazon/opentelemetry/distro/patches/_botocore_patches.py @@ -425,8 +425,8 @@ def patched_from_converse(cls, response: dict[str, Any], capture_content: bool) if tool_calls := bedrock_utils.extract_tool_calls(orig_message, capture_content): message["tool_calls"] = tool_calls - elif capture_content: - message["content"] = orig_message["content"] + elif capture_content and (content := orig_message.get("content")): + message["content"] = content return cls(message, response["stopReason"], index=0)