From a00a3de0e90f4bfb3c9c003e74d4ae5628d17512 Mon Sep 17 00:00:00 2001 From: Lukas Hering Date: Sun, 30 Nov 2025 22:07:36 -0500 Subject: [PATCH 1/5] opentelemetry-instrumentation-aiohttp-client: add support to capture custom headers --- .../aiohttp_client/__init__.py | 174 +++++++++++++++++- .../src/opentelemetry/util/http/__init__.py | 6 + 2 files changed, 179 insertions(+), 1 deletion(-) diff --git a/instrumentation/opentelemetry-instrumentation-aiohttp-client/src/opentelemetry/instrumentation/aiohttp_client/__init__.py b/instrumentation/opentelemetry-instrumentation-aiohttp-client/src/opentelemetry/instrumentation/aiohttp_client/__init__.py index 5db778a06a..5a16cd452c 100644 --- a/instrumentation/opentelemetry-instrumentation-aiohttp-client/src/opentelemetry/instrumentation/aiohttp_client/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-aiohttp-client/src/opentelemetry/instrumentation/aiohttp_client/__init__.py @@ -98,6 +98,97 @@ def response_hook(span: Span, params: typing.Union[ will exclude requests such as ``https://site/client/123/info`` and ``https://site/xyz/healthcheck``. +Capture HTTP request and response headers +***************************************** +You can configure the agent to capture specified HTTP headers as span attributes, according to the +`semantic conventions `_. + +Request headers +*************** +To capture HTTP request headers as span attributes, set the environment variable +``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_CLIENT_REQUEST`` to a comma delimited list of HTTP header names. + +For example using the environment variable, +:: + + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_CLIENT_REQUEST="content-type,custom_request_header" + +will extract ``content-type`` and ``custom_request_header`` from the request headers and add them as span attributes. + +Request header names in aiohttp are case-insensitive. So, giving the header name as ``CUStom-Header`` in the environment +variable will capture the header named ``custom-header``. + +Regular expressions may also be used to match multiple headers that correspond to the given pattern. For example: +:: + + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_CLIENT_REQUEST="Accept.*,X-.*" + +Would match all request headers that start with ``Accept`` and ``X-``. + +To capture all request headers, set ``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_CLIENT_REQUEST`` to ``".*"``. +:: + + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_CLIENT_REQUEST=".*" + +The name of the added span attribute will follow the format ``http.request.header.`` where ```` +is the normalized HTTP header name (lowercase, with ``-`` replaced by ``_``). The value of the attribute will be a +single item list containing all the header values. + +For example: +``http.request.header.custom_request_header = ["", ""]`` + +Response headers +**************** +To capture HTTP response headers as span attributes, set the environment variable +``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_CLIENT_RESPONSE`` to a comma delimited list of HTTP header names. + +For example using the environment variable, +:: + + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_CLIENT_RESPONSE="content-type,custom_response_header" + +will extract ``content-type`` and ``custom_response_header`` from the response headers and add them as span attributes. + +Response header names in aiohttp are case-insensitive. So, giving the header name as ``CUStom-Header`` in the environment +variable will capture the header named ``custom-header``. + +Regular expressions may also be used to match multiple headers that correspond to the given pattern. For example: +:: + + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_CLIENT_RESPONSE="Content.*,X-.*" + +Would match all response headers that start with ``Content`` and ``X-``. + +To capture all response headers, set ``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_CLIENT_RESPONSE`` to ``".*"``. +:: + + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_CLIENT_RESPONSE=".*" + +The name of the added span attribute will follow the format ``http.response.header.`` where ```` +is the normalized HTTP header name (lowercase, with ``-`` replaced by ``_``). The value of the attribute will be a +list containing the header values. + +For example: +``http.response.header.custom_response_header = ["", ""]`` + +Sanitizing headers +****************** +In order to prevent storing sensitive data such as personally identifiable information (PII), session keys, passwords, +etc, set the environment variable ``OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS`` +to a comma delimited list of HTTP header names to be sanitized. + +Regexes may be used, and all header names will be matched in a case-insensitive manner. + +For example using the environment variable, +:: + + export OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS=".*session.*,set-cookie" + +will replace the value of headers such as ``session-id`` and ``set-cookie`` with ``[REDACTED]`` in the span. + +Note: + The environment variable names used to capture HTTP headers are still experimental, and thus are subject to change. + API --- """ @@ -105,7 +196,7 @@ def response_hook(span: Span, params: typing.Union[ import types import typing from timeit import default_timer -from typing import Collection +from typing import Callable, Collection, Mapping from urllib.parse import urlparse import aiohttp @@ -150,7 +241,14 @@ def response_hook(span: Span, params: typing.Union[ from opentelemetry.trace import Span, SpanKind, TracerProvider, get_tracer from opentelemetry.trace.status import Status, StatusCode from opentelemetry.util.http import ( + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_CLIENT_REQUEST, + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_CLIENT_RESPONSE, + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS, + SanitizeValue, + get_custom_headers, get_excluded_urls, + normalise_request_header_name, + normalise_response_header_name, redact_url, sanitize_method, ) @@ -206,6 +304,35 @@ def _set_http_status_code_attribute( ) +def _get_custom_header_attributes( + headers: Mapping[str, str | list[str]] | None, + captured_headers: list[str] | None, + sensitive_headers: list[str] | None, + normalize_function: Callable[[str], str], +) -> dict[str, list[str]]: + """Extract and sanitize HTTP headers for span attributes. + + Args: + headers: The HTTP headers to process, either from a request or response. + Can be None if no headers are available. + captured_headers: List of header regexes to capture as span attributes. + If None or empty, no headers will be captured. + sensitive_headers: List of header regexes whose values should be sanitized + (redacted). If None, no sanitization is applied. + normalize_function: Function to normalize header names. + + Returns: + Dictionary of normalized header attribute names to their values + as lists of strings. + """ + if not headers or not captured_headers: + return {} + sanitize: SanitizeValue = SanitizeValue(sensitive_headers or ()) + return sanitize.sanitize_header_values( + headers, captured_headers, normalize_function + ) + + # pylint: disable=too-many-locals # pylint: disable=too-many-statements def create_trace_config( @@ -215,6 +342,9 @@ def create_trace_config( tracer_provider: TracerProvider = None, meter_provider: MeterProvider = None, sem_conv_opt_in_mode: _StabilityMode = _StabilityMode.DEFAULT, + captured_request_headers: list[str] | None = None, + captured_response_headers: list[str] | None = None, + sensitive_headers: list[str] | None = None, ) -> aiohttp.TraceConfig: """Create an aiohttp-compatible trace configuration. @@ -243,6 +373,15 @@ def create_trace_config( :param Callable response_hook: Optional callback that can modify span name and response params. :param tracer_provider: optional TracerProvider from which to get a Tracer :param meter_provider: optional Meter provider to use + :param captured_request_headers: List of HTTP request header regexes to capture as + span attributes. Header names matching these patterns will be added as span + attributes with the format ``http.request.header.``. + :param captured_response_headers: List of HTTP response header regexes to capture as + span attributes. Header names matching these patterns will be added as span + attributes with the format ``http.response.header.``. + :param sensitive_headers: List of HTTP header regexes whose values should be + sanitized (redacted) when captured. Header values matching these patterns + will be replaced with ``[REDACTED]``. :return: An object suitable for use with :py:class:`aiohttp.ClientSession`. :rtype: :py:class:`aiohttp.TraceConfig` @@ -385,6 +524,15 @@ async def on_request_start( except ValueError: pass + span_attributes.update( + _get_custom_header_attributes( + params.headers, + captured_request_headers, + sensitive_headers, + normalise_request_header_name, + ) + ) + trace_config_ctx.span = trace_config_ctx.tracer.start_span( request_span_name, kind=SpanKind.CLIENT, attributes=span_attributes ) @@ -415,6 +563,15 @@ async def on_request_end( sem_conv_opt_in_mode, ) + trace_config_ctx.span.set_attributes( + _get_custom_header_attributes( + params.headers, + captured_response_headers, + sensitive_headers, + normalise_response_header_name, + ) + ) + _end_trace(trace_config_ctx) async def on_request_exception( @@ -475,6 +632,9 @@ def _instrument( typing.Sequence[aiohttp.TraceConfig] ] = None, sem_conv_opt_in_mode: _StabilityMode = _StabilityMode.DEFAULT, + captured_request_headers: list[str] | None = None, + captured_response_headers: list[str] | None = None, + sensitive_headers: list[str] | None = None, ): """Enables tracing of all ClientSessions @@ -496,6 +656,9 @@ def instrumented_init(wrapped, instance, args, kwargs): tracer_provider=tracer_provider, meter_provider=meter_provider, sem_conv_opt_in_mode=sem_conv_opt_in_mode, + captured_request_headers=captured_request_headers, + captured_response_headers=captured_response_headers, + sensitive_headers=sensitive_headers, ) trace_config._is_instrumented_by_opentelemetry = True client_trace_configs.append(trace_config) @@ -560,6 +723,15 @@ def _instrument(self, **kwargs): response_hook=kwargs.get("response_hook"), trace_configs=kwargs.get("trace_configs"), sem_conv_opt_in_mode=_sem_conv_opt_in_mode, + captured_request_headers=get_custom_headers( + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_CLIENT_REQUEST + ), + captured_response_headers=get_custom_headers( + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_CLIENT_RESPONSE + ), + sensitive_headers=get_custom_headers( + OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS + ), ) def _uninstrument(self, **kwargs): diff --git a/util/opentelemetry-util-http/src/opentelemetry/util/http/__init__.py b/util/opentelemetry-util-http/src/opentelemetry/util/http/__init__.py index e23e03dede..fcb5eaa289 100644 --- a/util/opentelemetry-util-http/src/opentelemetry/util/http/__init__.py +++ b/util/opentelemetry-util-http/src/opentelemetry/util/http/__init__.py @@ -48,6 +48,12 @@ OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE = ( "OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SERVER_RESPONSE" ) +OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_CLIENT_REQUEST = ( + "OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_CLIENT_REQUEST" +) +OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_CLIENT_RESPONSE = ( + "OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_CLIENT_RESPONSE" +) OTEL_PYTHON_INSTRUMENTATION_HTTP_CAPTURE_ALL_METHODS = ( "OTEL_PYTHON_INSTRUMENTATION_HTTP_CAPTURE_ALL_METHODS" From 48c5fc58f5e69598d42946fb2f141c87a4fbc12d Mon Sep 17 00:00:00 2001 From: Lukas Hering Date: Mon, 1 Dec 2025 01:04:51 -0500 Subject: [PATCH 2/5] update unit tests --- .../aiohttp_client/__init__.py | 28 +- .../tests/test_aiohttp_client_integration.py | 395 ++++++++++++++++++ 2 files changed, 412 insertions(+), 11 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-aiohttp-client/src/opentelemetry/instrumentation/aiohttp_client/__init__.py b/instrumentation/opentelemetry-instrumentation-aiohttp-client/src/opentelemetry/instrumentation/aiohttp_client/__init__.py index 5a16cd452c..02278c8adc 100644 --- a/instrumentation/opentelemetry-instrumentation-aiohttp-client/src/opentelemetry/instrumentation/aiohttp_client/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-aiohttp-client/src/opentelemetry/instrumentation/aiohttp_client/__init__.py @@ -305,9 +305,9 @@ def _set_http_status_code_attribute( def _get_custom_header_attributes( - headers: Mapping[str, str | list[str]] | None, - captured_headers: list[str] | None, - sensitive_headers: list[str] | None, + headers: typing.Optional[Mapping[str, typing.Union[str, list[str]]]], + captured_headers: typing.Optional[list[str]], + sensitive_headers: typing.Optional[list[str]], normalize_function: Callable[[str], str], ) -> dict[str, list[str]]: """Extract and sanitize HTTP headers for span attributes. @@ -342,9 +342,9 @@ def create_trace_config( tracer_provider: TracerProvider = None, meter_provider: MeterProvider = None, sem_conv_opt_in_mode: _StabilityMode = _StabilityMode.DEFAULT, - captured_request_headers: list[str] | None = None, - captured_response_headers: list[str] | None = None, - sensitive_headers: list[str] | None = None, + captured_request_headers: typing.Optional[list[str]] = None, + captured_response_headers: typing.Optional[list[str]] = None, + sensitive_headers: typing.Optional[list[str]] = None, ) -> aiohttp.TraceConfig: """Create an aiohttp-compatible trace configuration. @@ -526,7 +526,10 @@ async def on_request_start( span_attributes.update( _get_custom_header_attributes( - params.headers, + { + key: params.headers.getall(key) + for key in params.headers.keys() + }, captured_request_headers, sensitive_headers, normalise_request_header_name, @@ -565,7 +568,10 @@ async def on_request_end( trace_config_ctx.span.set_attributes( _get_custom_header_attributes( - params.headers, + { + key: params.response.headers.getall(key) + for key in params.response.headers.keys() + }, captured_response_headers, sensitive_headers, normalise_response_header_name, @@ -632,9 +638,9 @@ def _instrument( typing.Sequence[aiohttp.TraceConfig] ] = None, sem_conv_opt_in_mode: _StabilityMode = _StabilityMode.DEFAULT, - captured_request_headers: list[str] | None = None, - captured_response_headers: list[str] | None = None, - sensitive_headers: list[str] | None = None, + captured_request_headers: typing.Optional[list[str]] = None, + captured_response_headers: typing.Optional[list[str]] = None, + sensitive_headers: typing.Optional[list[str]] = None, ): """Enables tracing of all ClientSessions diff --git a/instrumentation/opentelemetry-instrumentation-aiohttp-client/tests/test_aiohttp_client_integration.py b/instrumentation/opentelemetry-instrumentation-aiohttp-client/tests/test_aiohttp_client_integration.py index f9a7325eaf..a2b3493c0d 100644 --- a/instrumentation/opentelemetry-instrumentation-aiohttp-client/tests/test_aiohttp_client_integration.py +++ b/instrumentation/opentelemetry-instrumentation-aiohttp-client/tests/test_aiohttp_client_integration.py @@ -126,6 +126,12 @@ def _assert_metrics(self, num_metrics: int = 1): self.assertEqual(len(metrics), num_metrics) return metrics + def _assert_single_span(self): + """Assert that exactly one span was created and return it.""" + spans = self.memory_exporter.get_finished_spans() + self.assertEqual(1, len(spans)) + return spans[0] + @staticmethod def _http_request( trace_config, @@ -881,6 +887,395 @@ async def timeout_handler(request): duration_dp_attributes, ) + def test_custom_request_headers_captured(self): + """Test that specified request headers are captured as span attributes.""" + trace_config = aiohttp_client.create_trace_config( + captured_request_headers=["X-Custom-Header", "X-Another-Header"] + ) + + self._http_request( + trace_config=trace_config, + url="/test-path", + status_code=200, + headers={ + "X-Custom-Header": "custom-value", + "X-Another-Header": "another-value", + "X-Excluded-Header": "excluded-value", + }, + ) + + span = self._assert_single_span() + self.assertEqual( + span.attributes["http.request.header.x_custom_header"], + ("custom-value",), + ) + self.assertEqual( + span.attributes["http.request.header.x_another_header"], + ("another-value",), + ) + self.assertNotIn( + "http.request.header.x_excluded_header", span.attributes + ) + + def test_custom_response_headers_captured(self): + """Test that specified response headers are captured as span attributes.""" + trace_config = aiohttp_client.create_trace_config( + captured_response_headers=["X-Custom-Header", "X-Another-Header"] + ) + + def custom_handler(_request): + headers = { + "X-Custom-Header": "custom-value", + "X-Another-Header": "another-value", + "X-Excluded-Header": "excluded-value", + } + return aiohttp.web.Response(status=200, text="OK", headers=headers) + + self._http_request( + trace_config=trace_config, + url="/test-path", + status_code=200, + request_handler=custom_handler, + ) + + span = self._assert_single_span() + self.assertEqual( + span.attributes["http.response.header.x_custom_header"], + ("custom-value",), + ) + self.assertEqual( + span.attributes["http.response.header.x_another_header"], + ("another-value",), + ) + self.assertNotIn( + "http.response.header.x_excluded_header", span.attributes + ) + + def test_custom_headers_not_captured_when_not_configured(self): + """Test that headers are not captured when env vars are not set.""" + trace_config = aiohttp_client.create_trace_config() + + def custom_handler(_request): + return aiohttp.web.Response( + status=200, + text="OK", + headers={"X-Response-Header": "response-value"}, + ) + + self._http_request( + trace_config=trace_config, + url="/test-path", + status_code=200, + request_handler=custom_handler, + headers={"X-Request-Header": "request-value"}, + ) + + span = self._assert_single_span() + self.assertNotIn( + "http.request.header.x_request_header", span.attributes + ) + self.assertNotIn( + "http.response.header.x_response_header", span.attributes + ) + + def test_sensitive_headers_sanitized(self): + """Test that sensitive header values are redacted.""" + trace_config = aiohttp_client.create_trace_config( + captured_request_headers=["Authorization", "X-Api-Key"], + captured_response_headers=["Set-Cookie", "X-Secret"], + sensitive_headers=[ + "Authorization", + "X-Api-Key", + "Set-Cookie", + "X-Secret", + ], + ) + + def custom_handler(_request): + response_headers = { + "Set-Cookie": "session=abc123", + "X-Secret": "secret", + } + return aiohttp.web.Response( + status=200, text="OK", headers=response_headers + ) + + self._http_request( + trace_config=trace_config, + url="/test-path", + status_code=200, + request_handler=custom_handler, + headers={ + "Authorization": "Bearer secret-token", + "X-Api-Key": "secret-key", + }, + ) + + span = self._assert_single_span() + self.assertEqual( + span.attributes["http.request.header.authorization"], + ("[REDACTED]",), + ) + self.assertEqual( + span.attributes["http.request.header.x_api_key"], + ("[REDACTED]",), + ) + self.assertEqual( + span.attributes["http.response.header.set_cookie"], + ("[REDACTED]",), + ) + self.assertEqual( + span.attributes["http.response.header.x_secret"], + ("[REDACTED]",), + ) + + def test_custom_headers_with_regex(self): + """Test that header capture works with regex patterns.""" + trace_config = aiohttp_client.create_trace_config( + captured_request_headers=["X-Custom-Request-.*"], + captured_response_headers=["X-Custom-Response-.*"], + ) + + def custom_handler(_request): + response_headers = { + "X-Custom-Response-A": "value-A", + "X-Custom-Response-B": "value-B", + "X-Other-Response-Header": "other-value", + } + return aiohttp.web.Response( + status=200, text="OK", headers=response_headers + ) + + self._http_request( + trace_config=trace_config, + url="/test-path", + status_code=200, + request_handler=custom_handler, + headers={ + "X-Custom-Request-One": "value-one", + "X-Custom-Request-Two": "value-two", + "X-Other-Request-Header": "other-value", + }, + ) + + span = self._assert_single_span() + self.assertEqual( + span.attributes["http.request.header.x_custom_request_one"], + ("value-one",), + ) + self.assertEqual( + span.attributes["http.request.header.x_custom_request_two"], + ("value-two",), + ) + self.assertNotIn( + "http.request.header.x_other_request_header", span.attributes + ) + self.assertEqual( + span.attributes["http.response.header.x_custom_response_a"], + ("value-A",), + ) + self.assertEqual( + span.attributes["http.response.header.x_custom_response_b"], + ("value-B",), + ) + self.assertNotIn( + "http.response.header.x_other_response_header", span.attributes + ) + + def test_custom_headers_case_insensitive(self): + """Test that header capture is case-insensitive.""" + trace_config = aiohttp_client.create_trace_config( + captured_request_headers=["x-request-header"], + captured_response_headers=["x-response-header"], + ) + + def custom_handler(_request): + response_headers = {"X-ReSPoNse-HeaDER": "custom-value"} + return aiohttp.web.Response( + status=200, text="OK", headers=response_headers + ) + + self._http_request( + trace_config=trace_config, + url="/test-path", + status_code=200, + request_handler=custom_handler, + headers={"X-ReQuESt-HeaDER": "custom-value"}, + ) + + span = self._assert_single_span() + self.assertEqual( + span.attributes["http.request.header.x_request_header"], + ("custom-value",), + ) + self.assertEqual( + span.attributes["http.response.header.x_response_header"], + ("custom-value",), + ) + + def test_multi_value_request_headers(self): + """Test that headers with multiple values are captured correctly.""" + trace_config = aiohttp_client.create_trace_config( + captured_request_headers=["X-Multi-Value"] + ) + + self._http_request( + trace_config=trace_config, + url="/test-path", + status_code=200, + headers=(("X-Multi-Value", "value1"), ("X-Multi-Value", "value2")), + ) + + span = self._assert_single_span() + self.assertEqual( + span.attributes["http.request.header.x_multi_value"], + ("value1", "value2"), + ) + + def test_standard_http_headers_captured(self): + """Test that standard HTTP headers can be captured.""" + trace_config = aiohttp_client.create_trace_config( + captured_request_headers=["Content-Type", "Accept"], + captured_response_headers=["Content-Type", "Server"], + ) + + def custom_handler(_request): + response_headers = { + "Content-Type": "text/plain", + "Server": "TestServer/1.0", + } + return aiohttp.web.Response( + status=200, body=b"OK", headers=response_headers + ) + + self._http_request( + trace_config=trace_config, + url="/test-path", + status_code=200, + request_handler=custom_handler, + headers={ + "Content-Type": "application/json", + "Accept": "application/json", + }, + ) + + span = self._assert_single_span() + self.assertEqual( + span.attributes["http.request.header.content_type"], + ("application/json",), + ) + self.assertEqual( + span.attributes["http.request.header.accept"], + ("application/json",), + ) + self.assertEqual( + span.attributes["http.response.header.content_type"], + ("text/plain",), + ) + self.assertEqual( + span.attributes["http.response.header.server"], + ("TestServer/1.0",), + ) + + def test_capture_all_request_headers(self): + """Test that all request headers can be captured with .* pattern.""" + trace_config = aiohttp_client.create_trace_config( + captured_request_headers=[".*"] + ) + + def custom_handler(_request): + return aiohttp.web.Response(status=200, text="OK") + + self._http_request( + trace_config=trace_config, + url="/test-path", + status_code=200, + request_handler=custom_handler, + headers={ + "X-Header-One": "value1", + "X-Header-Two": "value2", + "X-Header-Three": "value3", + }, + ) + + span = self._assert_single_span() + self.assertEqual( + span.attributes["http.request.header.x_header_one"], + ("value1",), + ) + self.assertEqual( + span.attributes["http.request.header.x_header_two"], + ("value2",), + ) + self.assertEqual( + span.attributes["http.request.header.x_header_three"], + ("value3",), + ) + + def test_capture_all_response_headers(self): + """Test that all response headers can be captured with .* pattern.""" + trace_config = aiohttp_client.create_trace_config( + captured_response_headers=[".*"] + ) + + def custom_handler(_request): + response_headers = { + "X-Response-One": "value1", + "X-Response-Two": "value2", + "X-Response-Three": "value3", + } + return aiohttp.web.Response( + status=200, text="OK", headers=response_headers + ) + + self._http_request( + trace_config=trace_config, + url="/test-path", + status_code=200, + request_handler=custom_handler, + ) + + span = self._assert_single_span() + self.assertEqual( + span.attributes["http.response.header.x_response_one"], + ("value1",), + ) + self.assertEqual( + span.attributes["http.response.header.x_response_two"], + ("value2",), + ) + self.assertEqual( + span.attributes["http.response.header.x_response_three"], + ("value3",), + ) + + def test_sanitize_with_regex_pattern(self): + """Test that sanitization works with regex patterns.""" + trace_config = aiohttp_client.create_trace_config( + captured_request_headers=["X-Test.*"], + sensitive_headers=[".*secret.*"], + ) + + self._http_request( + trace_config=trace_config, + url="/test-path", + status_code=200, + headers={ + "X-Test": "normal-value", + "X-Test-Secret": "secret-value", + }, + ) + + span = self._assert_single_span() + self.assertEqual( + span.attributes["http.request.header.x_test"], + ("normal-value",), + ) + self.assertEqual( + span.attributes["http.request.header.x_test_secret"], + ("[REDACTED]",), + ) + class TestAioHttpClientInstrumentor(TestBase): URL = "/test-path" From 8b160bfa605c8218b9a75dafb2d1bef55bd81378 Mon Sep 17 00:00:00 2001 From: Lukas Hering Date: Mon, 1 Dec 2025 01:08:52 -0500 Subject: [PATCH 3/5] update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9cbbe7aa3b..9f8926812e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ([#3967](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3967)) - `opentelemetry-instrumentation-redis`: add missing copyright header for opentelemetry-instrumentation-redis ([#3976](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3976)) +- `opentelemetry-instrumentation-aiohttp-client`: add ability to capture custom headers + ([#3988](https://github.com/open-telemetry/opentelemetry-python-contrib/pull/3988)) ### Fixed From dd5bf24faad8721b4f192d4113f61cb7648822b7 Mon Sep 17 00:00:00 2001 From: Lukas Hering Date: Mon, 1 Dec 2025 20:58:13 -0500 Subject: [PATCH 4/5] add 'get_custom_header_attributes' helper to opentelemetry-util-http --- .../aiohttp_client/__init__.py | 38 ++----------------- .../src/opentelemetry/util/http/__init__.py | 29 ++++++++++++++ 2 files changed, 33 insertions(+), 34 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-aiohttp-client/src/opentelemetry/instrumentation/aiohttp_client/__init__.py b/instrumentation/opentelemetry-instrumentation-aiohttp-client/src/opentelemetry/instrumentation/aiohttp_client/__init__.py index 02278c8adc..a2c1ceb919 100644 --- a/instrumentation/opentelemetry-instrumentation-aiohttp-client/src/opentelemetry/instrumentation/aiohttp_client/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-aiohttp-client/src/opentelemetry/instrumentation/aiohttp_client/__init__.py @@ -196,7 +196,7 @@ def response_hook(span: Span, params: typing.Union[ import types import typing from timeit import default_timer -from typing import Callable, Collection, Mapping +from typing import Collection from urllib.parse import urlparse import aiohttp @@ -244,13 +244,12 @@ def response_hook(span: Span, params: typing.Union[ OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_CLIENT_REQUEST, OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_CLIENT_RESPONSE, OTEL_INSTRUMENTATION_HTTP_CAPTURE_HEADERS_SANITIZE_FIELDS, - SanitizeValue, get_custom_headers, get_excluded_urls, normalise_request_header_name, normalise_response_header_name, redact_url, - sanitize_method, + sanitize_method, get_custom_header_attributes, ) _UrlFilterT = typing.Optional[typing.Callable[[yarl.URL], str]] @@ -304,35 +303,6 @@ def _set_http_status_code_attribute( ) -def _get_custom_header_attributes( - headers: typing.Optional[Mapping[str, typing.Union[str, list[str]]]], - captured_headers: typing.Optional[list[str]], - sensitive_headers: typing.Optional[list[str]], - normalize_function: Callable[[str], str], -) -> dict[str, list[str]]: - """Extract and sanitize HTTP headers for span attributes. - - Args: - headers: The HTTP headers to process, either from a request or response. - Can be None if no headers are available. - captured_headers: List of header regexes to capture as span attributes. - If None or empty, no headers will be captured. - sensitive_headers: List of header regexes whose values should be sanitized - (redacted). If None, no sanitization is applied. - normalize_function: Function to normalize header names. - - Returns: - Dictionary of normalized header attribute names to their values - as lists of strings. - """ - if not headers or not captured_headers: - return {} - sanitize: SanitizeValue = SanitizeValue(sensitive_headers or ()) - return sanitize.sanitize_header_values( - headers, captured_headers, normalize_function - ) - - # pylint: disable=too-many-locals # pylint: disable=too-many-statements def create_trace_config( @@ -525,7 +495,7 @@ async def on_request_start( pass span_attributes.update( - _get_custom_header_attributes( + get_custom_header_attributes( { key: params.headers.getall(key) for key in params.headers.keys() @@ -567,7 +537,7 @@ async def on_request_end( ) trace_config_ctx.span.set_attributes( - _get_custom_header_attributes( + get_custom_header_attributes( { key: params.response.headers.getall(key) for key in params.response.headers.keys() diff --git a/util/opentelemetry-util-http/src/opentelemetry/util/http/__init__.py b/util/opentelemetry-util-http/src/opentelemetry/util/http/__init__.py index fcb5eaa289..00ac611b34 100644 --- a/util/opentelemetry-util-http/src/opentelemetry/util/http/__init__.py +++ b/util/opentelemetry-util-http/src/opentelemetry/util/http/__init__.py @@ -257,6 +257,35 @@ def get_custom_headers(env_var: str) -> list[str]: return [] +def get_custom_header_attributes( + headers: Mapping[str, str | list[str]] | None, + captured_headers: list[str] | None, + sensitive_headers: list[str] | None, + normalize_function: Callable[[str], str], +) -> dict[str, list[str]]: + """Extract and sanitize HTTP headers for span attributes. + + Args: + headers: The HTTP headers to process, either from a request or response. + Can be None if no headers are available. + captured_headers: List of header regexes to capture as span attributes. + If None or empty, no headers will be captured. + sensitive_headers: List of header regexes whose values should be sanitized + (redacted). If None, no sanitization is applied. + normalize_function: Function to normalize header names. + + Returns: + Dictionary of normalized header attribute names to their values + as lists of strings. + """ + if not headers or not captured_headers: + return {} + sanitize: SanitizeValue = SanitizeValue(sensitive_headers or ()) + return sanitize.sanitize_header_values( + headers, captured_headers, normalize_function + ) + + def _parse_active_request_count_attrs(req_attrs): active_requests_count_attrs = { key: req_attrs[key] From 9870a3dc3714551cc1331295067c1fa7f552ac97 Mon Sep 17 00:00:00 2001 From: Lukas Hering Date: Mon, 1 Dec 2025 21:00:57 -0500 Subject: [PATCH 5/5] fix formatting issues --- .../instrumentation/aiohttp_client/__init__.py | 3 ++- .../src/opentelemetry/util/http/__init__.py | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/instrumentation/opentelemetry-instrumentation-aiohttp-client/src/opentelemetry/instrumentation/aiohttp_client/__init__.py b/instrumentation/opentelemetry-instrumentation-aiohttp-client/src/opentelemetry/instrumentation/aiohttp_client/__init__.py index a2c1ceb919..ca32acda93 100644 --- a/instrumentation/opentelemetry-instrumentation-aiohttp-client/src/opentelemetry/instrumentation/aiohttp_client/__init__.py +++ b/instrumentation/opentelemetry-instrumentation-aiohttp-client/src/opentelemetry/instrumentation/aiohttp_client/__init__.py @@ -249,7 +249,8 @@ def response_hook(span: Span, params: typing.Union[ normalise_request_header_name, normalise_response_header_name, redact_url, - sanitize_method, get_custom_header_attributes, + sanitize_method, + get_custom_header_attributes, ) _UrlFilterT = typing.Optional[typing.Callable[[yarl.URL], str]] diff --git a/util/opentelemetry-util-http/src/opentelemetry/util/http/__init__.py b/util/opentelemetry-util-http/src/opentelemetry/util/http/__init__.py index 00ac611b34..ee3752a7f1 100644 --- a/util/opentelemetry-util-http/src/opentelemetry/util/http/__init__.py +++ b/util/opentelemetry-util-http/src/opentelemetry/util/http/__init__.py @@ -258,10 +258,10 @@ def get_custom_headers(env_var: str) -> list[str]: def get_custom_header_attributes( - headers: Mapping[str, str | list[str]] | None, - captured_headers: list[str] | None, - sensitive_headers: list[str] | None, - normalize_function: Callable[[str], str], + headers: Mapping[str, str | list[str]] | None, + captured_headers: list[str] | None, + sensitive_headers: list[str] | None, + normalize_function: Callable[[str], str], ) -> dict[str, list[str]]: """Extract and sanitize HTTP headers for span attributes.