Initial Hybrid Agent Trace implementation

newrelic/config.py

@@ -691,6 +691,7 @@ def _process_configuration(section):
     _process_setting(section, "instrumentation.middleware.django.enabled", "getboolean", None)
     _process_setting(section, "instrumentation.middleware.django.exclude", "get", _map_inc_excl_middleware)
     _process_setting(section, "instrumentation.middleware.django.include", "get", _map_inc_excl_middleware)
+    _process_setting(section, "otel_bridge.enabled", "getboolean", None)
 
 
 # Loading of configuration from specified file and for specified
@@ -4367,6 +4368,15 @@ def _process_module_builtin_defaults():
         "pyzeebe.worker.job_executor", "newrelic.hooks.external_pyzeebe", "instrument_pyzeebe_worker_job_executor"
     )
 
+    # Hybrid Agent Hooks
+    _process_module_definition(
+        "opentelemetry.trace", "newrelic.hooks.hybridagent_opentelemetry", "instrument_trace_api"
+    )
+
+    _process_module_definition(
+        "opentelemetry.instrumentation.utils", "newrelic.hooks.hybridagent_opentelemetry", "instrument_utils"
+    )
+
 
 def _process_module_entry_points():
     try:

newrelic/core/config.py

@@ -534,6 +534,10 @@ class EventHarvestConfigHarvestLimitSettings(Settings):
     nested = True
 
 
+class OtelBridgeSettings(Settings):
+    pass
+
+
 _settings = TopLevelSettings()
 _settings.agent_limits = AgentLimitsSettings()
 _settings.application_logging = ApplicationLoggingSettings()
@@ -620,6 +624,7 @@ class EventHarvestConfigHarvestLimitSettings(Settings):
 _settings.instrumentation.middleware = InstrumentationMiddlewareSettings()
 _settings.instrumentation.middleware.django = InstrumentationDjangoMiddlewareSettings()
 _settings.message_tracer = MessageTracerSettings()
+_settings.otel_bridge = OtelBridgeSettings()
 _settings.process_host = ProcessHostSettings()
 _settings.rum = RumSettings()
 _settings.serverless_mode = ServerlessModeSettings()
@@ -1267,6 +1272,7 @@ def default_otlp_host(host):
 _settings.azure_operator.enabled = _environ_as_bool("NEW_RELIC_AZURE_OPERATOR_ENABLED", default=False)
 _settings.package_reporting.enabled = _environ_as_bool("NEW_RELIC_PACKAGE_REPORTING_ENABLED", default=True)
 _settings.ml_insights_events.enabled = _environ_as_bool("NEW_RELIC_ML_INSIGHTS_EVENTS_ENABLED", default=False)
+_settings.otel_bridge.enabled = _environ_as_bool("NEW_RELIC_OTEL_BRIDGE_ENABLED", default=False)
 
 
 def global_settings():

newrelic/core/otlp_utils.py

@@ -116,8 +116,9 @@ def create_key_values_from_iterable(iterable):
     return list(filter(lambda i: i is not None, (create_key_value(key, value) for key, value in iterable)))
 
 
-def create_resource(attributes=None, attach_apm_entity=True):
-    attributes = attributes or {"instrumentation.provider": "newrelic-opentelemetry-python-ml"}
+def create_resource(attributes=None, attach_apm_entity=True, hybrid_bridge=False):
+    instrumentation_provider = "newrelic-opentelemetry-bridge" if hybrid_bridge else "newrelic-opentelemetry-python-ml"
+    attributes = attributes or {"instrumentation.provider": instrumentation_provider}
     if attach_apm_entity:
         metadata = get_service_linking_metadata()
         attributes.update(metadata)

newrelic/hooks/hybridagent_opentelemetry.py

@@ -0,0 +1,172 @@
+# Copyright 2010 New Relic, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+
+from newrelic.api.application import application_instance
+from newrelic.api.time_trace import add_custom_span_attribute, current_trace
+from newrelic.api.transaction import Sentinel, current_transaction
+from newrelic.common.object_wrapper import wrap_function_wrapper
+from newrelic.common.signature import bind_args
+from newrelic.core.config import global_settings
+
+_logger = logging.getLogger(__name__)
+_TRACER_PROVIDER = None
+
+###########################################
+#   Trace Instrumentation
+###########################################
+
+
+def wrap_set_tracer_provider(wrapped, instance, args, kwargs):
+    settings = global_settings()
+
+    if not settings.otel_bridge.enabled:
+        return wrapped(*args, **kwargs)
+
+    global _TRACER_PROVIDER
+
+    if _TRACER_PROVIDER is None:
+        bound_args = bind_args(wrapped, args, kwargs)
+        tracer_provider = bound_args.get("tracer_provider")
+        _TRACER_PROVIDER = tracer_provider
+    else:
+        _logger.warning("TracerProvider has already been set.")
+
+
+def wrap_get_tracer_provider(wrapped, instance, args, kwargs):
+    settings = global_settings()
+
+    if not settings.otel_bridge.enabled:
+        return wrapped(*args, **kwargs)
+
+    # This needs to act as a singleton, like the agent instance.
+    # We should initialize the agent here as well, if there is
+    # not an instance already.
+    application = application_instance(activate=False)
+    if not application or (application and not application.active):
+        application_instance().activate()
+
+    global _TRACER_PROVIDER
+
+    if _TRACER_PROVIDER is None:
+        from newrelic.api.opentelemetry import TracerProvider
+
+        hybrid_agent_tracer_provider = TracerProvider("hybrid_agent_tracer_provider")
+        _TRACER_PROVIDER = hybrid_agent_tracer_provider
+    return _TRACER_PROVIDER
+
+
+def wrap_get_current_span(wrapped, instance, args, kwargs):
+    transaction = current_transaction()
+    trace = current_trace()
+
+    # If a NR trace does not exist (aside from the Sentinel
+    # trace), return the original function's result.
+    if not transaction or isinstance(trace, Sentinel):
+        return wrapped(*args, **kwargs)
+
+    # Do not allow the wrapper to continue if
+    # the Hybrid Agent setting is not enabled
+    settings = transaction.settings or global_settings()
+
+    if not settings.otel_bridge.enabled:
+        return wrapped(*args, **kwargs)
+
+    # If a NR trace does exist, check to see if the current
+    # OTel span corresponds to the current NR trace.  If so,
+    # return the original function's result.
+    span = wrapped(*args, **kwargs)
+
+    if span.get_span_context().span_id == int(trace.guid, 16):
+        return span
+
+    # If the current OTel span does not match the current NR
+    # trace, this means that a NR trace was created either
+    # manually or through the NR agent.  Either way, the OTel
+    # API was not used to create a span object.  The Hybrid
+    # Agent's Span object creates a NR trace but since the NR
+    # trace has already been created, we just need a symbolic
+    # OTel span to represent it the span object.  A LazySpan
+    # will be created.  It will effectively be a NonRecordingSpan
+    # with the ability to add custom attributes.
+
+    from opentelemetry import trace as otel_api_trace
+
+    class LazySpan(otel_api_trace.NonRecordingSpan):
+        def set_attribute(self, key, value):
+            add_custom_span_attribute(key, value)
+
+        def set_attributes(self, attributes):
+            for key, value in attributes.items():
+                add_custom_span_attribute(key, value)
+
+    otel_tracestate_headers = None
+
+    span_context = otel_api_trace.SpanContext(
+        trace_id=int(transaction.trace_id, 16),
+        span_id=int(trace.guid, 16),
+        is_remote=span.get_span_context().is_remote,
+        trace_flags=otel_api_trace.TraceFlags(span.get_span_context().trace_flags),
+        trace_state=otel_api_trace.TraceState(otel_tracestate_headers),
+    )
+
+    return LazySpan(span_context)
+
+
+def wrap_start_internal_or_server_span(wrapped, instance, args, kwargs):
+    # We want to take the NR version of the context_carrier
+    # and put that into the attributes.  Keep the original
+    # context_carrier intact.
+
+    # Do not allow the wrapper to continue if
+    # the Hybrid Agent setting is not enabled
+    settings = global_settings()
+
+    if not settings.otel_bridge.enabled:
+        return wrapped(*args, **kwargs)
+
+    bound_args = bind_args(wrapped, args, kwargs)
+    context_carrier = bound_args.get("context_carrier")
+    attributes = bound_args.get("attributes", {})
+
+    if context_carrier:
+        if ("HTTP_HOST" in context_carrier) or ("http_version" in context_carrier):
+            # This is an HTTP request (WSGI, ASGI, or otherwise)
+            nr_environ = context_carrier.copy()
+            attributes["nr.http.headers"] = nr_environ
+
+        else:
+            nr_headers = context_carrier.copy()
+            attributes["nr.nonhttp.headers"] = nr_headers
+
+        bound_args["attributes"] = attributes
+
+    return wrapped(**bound_args)
+
+
+def instrument_trace_api(module):
+    if hasattr(module, "set_tracer_provider"):
+        wrap_function_wrapper(module, "set_tracer_provider", wrap_set_tracer_provider)
+
+    if hasattr(module, "get_tracer_provider"):
+        wrap_function_wrapper(module, "get_tracer_provider", wrap_get_tracer_provider)
+
+    if hasattr(module, "get_current_span"):
+        wrap_function_wrapper(module, "get_current_span", wrap_get_current_span)
+
+
+def instrument_utils(module):
+    if hasattr(module, "_start_internal_or_server_span"):
+        wrap_function_wrapper(module, "_start_internal_or_server_span", wrap_start_internal_or_server_span)

tests/hybridagent_opentelemetry/conftest.py

@@ -0,0 +1,40 @@
+# Copyright 2010 New Relic, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+from opentelemetry import trace
+from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture
+
+from newrelic.api.opentelemetry import TracerProvider
+
+_default_settings = {
+    "package_reporting.enabled": False,  # Turn off package reporting for testing as it causes slow downs.
+    "transaction_tracer.explain_threshold": 0.0,
+    "transaction_tracer.transaction_threshold": 0.0,
+    "transaction_tracer.stack_trace_threshold": 0.0,
+    "debug.log_data_collector_payloads": True,
+    "debug.record_transaction_failure": True,
+    "otel_bridge.enabled": True,
+}
+
+collector_agent_registration = collector_agent_registration_fixture(
+    app_name="Python Agent Test (Hybrid Agent)", default_settings=_default_settings
+)
+
+@pytest.fixture(scope="session")
+def tracer():
+    trace_provider = TracerProvider()
+    trace.set_tracer_provider(trace_provider)
+
+    return trace.get_tracer(__name__)

tests/hybridagent_opentelemetry/test_hybrid_cross_agent.py

@@ -0,0 +1,112 @@
+# Copyright 2010 New Relic, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from opentelemetry import trace as otel_api_trace
+from testing_support.validators.validate_error_event_attributes import validate_error_event_attributes
+from testing_support.validators.validate_span_events import validate_span_events
+from testing_support.validators.validate_transaction_count import validate_transaction_count
+from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics
+
+from newrelic.api.application import application_instance
+from newrelic.api.background_task import BackgroundTask
+from newrelic.api.function_trace import FunctionTrace
+from newrelic.api.time_trace import current_trace
+from newrelic.api.transaction import current_transaction
+
+
+# Does not create segment without a transaction
+@validate_transaction_count(0)
+def test_does_not_create_segment_without_a_transaction(tracer):
+    with tracer.start_as_current_span(name="Bar", kind=otel_api_trace.SpanKind.INTERNAL):
+        # The OpenTelmetry span should not be created
+        assert otel_api_trace.get_current_span() == otel_api_trace.INVALID_SPAN
+
+        # There should be no transaction
+        assert not current_transaction()
+
+
+# Creates OpenTelemetry segment in a transaction
+@validate_transaction_metrics(name="Foo", background_task=True)
+@validate_span_events(
+    exact_intrinsics={"name": "Function/Bar", "category": "generic"}, expected_intrinsics=("parentId",)
+)
+@validate_span_events(exact_intrinsics={"name": "Function/Foo", "category": "generic", "nr.entryPoint": True})
+def test_creates_opentelemetry_segment_in_a_transaction(tracer):
+    application = application_instance(activate=False)
+
+    with BackgroundTask(application, name="Foo"):
+        with tracer.start_as_current_span(name="Bar", kind=otel_api_trace.SpanKind.INTERNAL):
+            # OpenTelemetry API and New Relic API report the same traceId
+            assert otel_api_trace.get_current_span().get_span_context().trace_id == int(
+                current_transaction()._trace_id, 16
+            )
+
+            # OpenTelemetry API and New Relic API report the same spanId
+            assert otel_api_trace.get_current_span().get_span_context().span_id == int(current_trace().guid, 16)
+
+
+# Creates New Relic span as child of OpenTelemetry span
+@validate_transaction_metrics(name="Foo", background_task=True)
+@validate_span_events(
+    exact_intrinsics={"name": "Function/Baz", "category": "generic"}, expected_intrinsics=("parentId",)
+)
+@validate_span_events(
+    exact_intrinsics={"name": "Function/Bar", "category": "generic"}, expected_intrinsics=("parentId",)
+)
+@validate_span_events(exact_intrinsics={"name": "Function/Foo", "category": "generic"})
+def test_creates_new_relic_span_as_child_of_open_telemetry_span(tracer):
+    application = application_instance(activate=False)
+
+    with BackgroundTask(application, name="Foo"):
+        with tracer.start_as_current_span(name="Bar", kind=otel_api_trace.SpanKind.INTERNAL):
+            with FunctionTrace(name="Baz"):
+                # OpenTelemetry API and New Relic API report the same traceId
+                assert otel_api_trace.get_current_span().get_span_context().trace_id == int(
+                    current_transaction().trace_id, 16
+                )
+
+                # OpenTelemetry API and New Relic API report the same spanId
+                assert otel_api_trace.get_current_span().get_span_context().span_id == int(current_trace().guid, 16)
+
+
+# OpenTelemetry API can add custom attributes to spans
+@validate_transaction_metrics(name="Foo", background_task=True)
+@validate_span_events(exact_intrinsics={"name": "Function/Baz"}, exact_users={"spanNumber": 2})
+@validate_span_events(exact_intrinsics={"name": "Function/Bar"}, exact_users={"spanNumber": 1})
+def test_opentelemetry_api_can_add_custom_attributes_to_spans(tracer):
+    application = application_instance(activate=False)
+
+    with BackgroundTask(application, name="Foo"):
+        with tracer.start_as_current_span(name="Bar", kind=otel_api_trace.SpanKind.INTERNAL):
+            with FunctionTrace(name="Baz"):
+                otel_api_trace.get_current_span().set_attribute("spanNumber", 2)
+
+            otel_api_trace.get_current_span().set_attribute("spanNumber", 1)
+
+
+# OpenTelemetry API can record errors
+@validate_transaction_metrics(name="Foo", background_task=True)
+@validate_error_event_attributes(
+    exact_attrs={"agent": {}, "intrinsic": {"error.message": "Test exception message"}, "user": {}}
+)
+@validate_span_events(exact_intrinsics={"name": "Function/Bar"})
+def test_opentelemetry_api_can_record_errors(tracer):
+    application = application_instance(activate=False)
+
+    with BackgroundTask(application, name="Foo"):
+        with tracer.start_as_current_span(name="Bar", kind=otel_api_trace.SpanKind.INTERNAL):
+            try:
+                raise Exception("Test exception message")
+            except Exception as e:
+                otel_api_trace.get_current_span().record_exception(e)