feat: [traceloop] fix evals for llm calls

instrumentation-genai/opentelemetry-instrumentation-langchain/examples/multi_agent_travel_planner/traceloop/.env

@@ -11,7 +11,7 @@ OPENAI_API_KEY=sk-YOUR_API_KEY
 # OTEL_EXPORTER_OTLP_PROTOCOL=grpc
 
 # Traces will use this service.name
-OTEL_SERVICE_NAME=travel-plan-tl
+OTEL_SERVICE_NAME=travel-planner-tl
 
 OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318
 OTEL_SEMCONV_STABILITY_OPT_IN=gen_ai_latest_experimental
@@ -20,4 +20,6 @@ OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT_MODE=SPAN_AND_EVENT
 OTEL_INSTRUMENTATION_GENAI_EMITTERS=span_metric_event,splunk
 OTEL_INSTRUMENTATION_GENAI_EMITTERS_EVALUATION=replace-category:SplunkEvaluationResults
 OTEL_INSTRUMENTATION_GENAI_EVALS_RESULTS_AGGREGATION=true
-OTEL_INSTRUMENTATION_GENAI_DEBUG=true
+OTEL_INSTRUMENTATION_GENAI_DEBUG=false
+
+DEEPEVAL_TELEMETRY_OPT_OUT="YES"

instrumentation-genai/opentelemetry-instrumentation-langchain/examples/multi_agent_travel_planner/traceloop/Dockerfile.traceloop → instrumentation-genai/opentelemetry-instrumentation-langchain/examples/multi_agent_travel_planner/traceloop/Dockerfile.lc

@@ -18,17 +18,18 @@ COPY util/opentelemetry-util-genai-evals /app/opentelemetry-util-genai-evals
 COPY util/opentelemetry-util-genai-evals-deepeval /app/opentelemetry-util-genai-evals-deepeval
 COPY util/opentelemetry-util-genai-emitters-splunk /app/opentelemetry-util-genai-emitters-splunk
 
-# Set working directory to the example
-WORKDIR /app/opentelemetry-instrumentation-langchain/examples/multi_agent_travel_planner
+# Set working directory to the traceloop example
+WORKDIR /app/opentelemetry-instrumentation-langchain/examples/multi_agent_travel_planner/traceloop
 
-# Install Python dependencies from requirements.traceloop.txt (excluding local -e packages)
-# First, create a temporary requirements file without the local editable packages
-RUN grep -v "^-e \.\." requirements.traceloop.txt > /tmp/requirements_external.txt && \
+# Install Python dependencies from requirements.traceloop.txt (excluding local -e packages and git+ssh)
+# First, create a temporary requirements file without the local editable packages and SSH git dependencies
+RUN grep -v "^-e" requirements.traceloop.txt | grep -v "git+ssh" > /tmp/requirements_external.txt && \
     pip install --no-cache-dir -r /tmp/requirements_external.txt && \
     rm /tmp/requirements_external.txt
 
 # Install the local packages in editable mode
 # The Traceloop translator will enable zero-code instrumentation via .pth file
+# splunk-otel-instrumentation-langchain IS installed
 RUN cd /app/opentelemetry-util-genai && \
     pip install --no-cache-dir --no-deps -e . && \
     cd /app/opentelemetry-util-genai-evals && \

instrumentation-genai/opentelemetry-instrumentation-langchain/examples/multi_agent_travel_planner/traceloop/Dockerfile.tl

@@ -0,0 +1,48 @@
+FROM python:3.12-slim
+
+WORKDIR /app
+
+# Install git for pip dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    git \
+    && rm -rf /var/lib/apt/lists/*
+
+# Copy required packages (build context is repo root)
+COPY util/opentelemetry-util-genai-traceloop-translator /app/opentelemetry-util-genai-traceloop-translator
+COPY util/opentelemetry-util-genai /app/opentelemetry-util-genai
+COPY util/opentelemetry-util-genai-evals /app/opentelemetry-util-genai-evals
+COPY util/opentelemetry-util-genai-evals-deepeval /app/opentelemetry-util-genai-evals-deepeval
+COPY util/opentelemetry-util-genai-emitters-splunk /app/opentelemetry-util-genai-emitters-splunk
+COPY instrumentation-genai/opentelemetry-instrumentation-langchain/examples/multi_agent_travel_planner /app/opentelemetry-instrumentation-langchain/examples/multi_agent_travel_planner
+
+WORKDIR /app/opentelemetry-instrumentation-langchain/examples/multi_agent_travel_planner/traceloop
+
+# Install Python dependencies (excluding local -e packages and git+ssh dependencies)
+RUN pip install --no-cache-dir -r requirements.traceloop.txt
+
+# Install local utility packages with updated dependencies
+RUN cd /app/opentelemetry-util-genai-traceloop-translator && pip install --no-cache-dir --no-deps -e . && \
+    cd /app/opentelemetry-util-genai && pip install --no-cache-dir -e . && \
+    cd /app/opentelemetry-util-genai-evals && pip install --no-cache-dir -e . && \
+    cd /app/opentelemetry-util-genai-evals-deepeval && pip install --no-cache-dir -e . && \
+    cd /app/opentelemetry-util-genai-emitters-splunk && pip install --no-cache-dir -e .
+
+# Verify packages are installed correctly
+RUN python3 -c "from opentelemetry.util.genai.handler import get_telemetry_handler; print('✓ GenAI handler available')" && \
+    python3 -c "from opentelemetry.util.genai.evals import create_evaluation_manager; print('✓ Evaluation manager available')" && \
+    python3 -c "import opentelemetry.util.genai.emitters.splunk; print('✓ Splunk emitters available')" && \
+    python3 -c "import opentelemetry.util.evaluator.deepeval; print('✓ Deepeval evaluator module available')" && \
+    python3 -c "import deepeval; print('✓ Deepeval SDK installed')" && \
+    python3 -c "from opentelemetry.util.genai.traceloop import enable_traceloop_translator; print('✓ Traceloop translator available')"
+
+# Make the script executable
+RUN chmod +x main_traceloop.py
+
+# Set default environment variables
+ENV OTEL_PYTHON_LOG_CORRELATION=true \
+    OTEL_PYTHON_LOG_LEVEL=info \
+    OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf \
+    PYTHONUNBUFFERED=1
+
+# Run the Traceloop version
+CMD ["python3", "main_traceloop.py"]

instrumentation-genai/opentelemetry-instrumentation-langchain/examples/multi_agent_travel_planner/trip-planner-cronjob.yaml → instrumentation-genai/opentelemetry-instrumentation-langchain/examples/multi_agent_travel_planner/traceloop/cronjob-tl-lc.yaml

@@ -7,13 +7,12 @@ metadata:
     app: trip-planner-tl
     component: telemetry
   annotations:
-    description: "Multi-agent trip planner with Traceloop translator and GenAI evaluations (Deepeval telemetry disabled)"
-    git-commit: "20251107-164153"
+    description: "Multi-agent trip planner with Traceloop translator and GenAI evaluations. splunk-otel-instrumentation-langchain is installed"
+    git-commit: "1b4045e"
 spec:
-  # Run every 10 minutes from Nov 7 (Fri) through Nov 10 (Mon) 10 AM PST
-  # Covers: Fri Nov 7 (now) through Mon Nov 10 morning
-  # NOTE: Manually suspend on Monday Nov 10 at 10 AM PST
-  schedule: "*/10 * 7-10 11 *"
+  # Run every 30 minutes from 8 AM to 5 PM PST on weekdays (Monday-Friday)
+  # Offset from travel-planner-tl by 45 minutes (runs at :15 and :45)
+  schedule: "15,45 8-17 * * 1-5"
   timeZone: "America/Los_Angeles"
   suspend: false
 
@@ -38,7 +37,7 @@ spec:
           containers:
             - name: trip-planner-traceloop
               # Multi-platform image (amd64, arm64) with git commit hash tag
-              image: admehra621/trip-planner-tl:20251107-164153
+              image: admehra621/trip-planner-tl-lc:1b4045e
               imagePullPolicy: Always
 
               env:
@@ -48,7 +47,7 @@ spec:
 
                 # === OpenTelemetry Resource Attributes ===
                 - name: OTEL_RESOURCE_ATTRIBUTES
-                  value: "deployment.environment=o11y-inframon-ai,git.commit.id=20251107-164153"
+                  value: "deployment.environment=o11y-inframon-ai,git.commit.id=1b4045e"
 
                 # === Service name for telemetry ===
                 - name: OTEL_SERVICE_NAME

instrumentation-genai/opentelemetry-instrumentation-langchain/examples/multi_agent_travel_planner/traceloop/cronjob-tl.yaml

@@ -7,8 +7,8 @@ metadata:
     app: travel-planner-tl
     component: telemetry
   annotations:
-    description: "Multi-agent travel planner with Traceloop translator and GenAI evaluations (Deepeval telemetry disabled)"
-    git-commit: "2b4a41a"
+    description: "Multi-agent travel planner with Traceloop translator and GenAI evaluations. splunk-otel-instrumentation-langchain is NOT installed"
+    git-commit: "4896ced"
 spec:
   # Run every 30 minutes from 8 AM to 5 PM PST on weekdays (Monday-Friday)
   schedule: "*/30 8-17 * * 1-5"
@@ -36,7 +36,7 @@ spec:
           containers:
             - name: travel-planner-traceloop
               # Multi-platform image (amd64, arm64) with git commit hash tag
-              image: admehra621/travel-planner-tl:2b4a41a
+              image: admehra621/travel-planner-tl:4896ced
               imagePullPolicy: Always
 
               env:
@@ -46,7 +46,7 @@ spec:
 
                 # === OpenTelemetry Resource Attributes ===
                 - name: OTEL_RESOURCE_ATTRIBUTES
-                  value: "deployment.environment=o11y-inframon-ai,git.commit.id=2b4a41a"
+                  value: "deployment.environment=o11y-inframon-ai,git.commit.id=4896ced"
 
                 # === Service name for telemetry ===
                 - name: OTEL_SERVICE_NAME
@@ -64,7 +64,7 @@ spec:
 
                 # === Deepeval Telemetry Opt-Out ===
                 - name: DEEPEVAL_TELEMETRY_OPT_OUT
-                  value: "1"
+                  value: "YES"
 
                 # === GenAI Content Capture ===
                 - name: OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT

instrumentation-genai/opentelemetry-instrumentation-langchain/examples/multi_agent_travel_planner/traceloop/requirements.traceloop.txt

@@ -78,40 +78,40 @@ opentelemetry-exporter-otlp-proto-common @ git+https://github.com/open-telemetry
 opentelemetry-exporter-otlp-proto-grpc @ git+https://github.com/open-telemetry/opentelemetry-python.git@1f68134481c46e476a127b61a36dc69889275d15#subdirectory=exporter/opentelemetry-exporter-otlp-proto-grpc
 opentelemetry-exporter-otlp-proto-http==1.38.0
 opentelemetry-instrumentation @ git+https://github.com/open-telemetry/opentelemetry-python-contrib.git@071f68697432e7e57b31238a5998dc4c1325855c#subdirectory=opentelemetry-instrumentation
-opentelemetry-instrumentation-alephalpha==0.47.5
-opentelemetry-instrumentation-anthropic==0.47.5
-opentelemetry-instrumentation-bedrock==0.47.5
-opentelemetry-instrumentation-chromadb==0.47.5
-opentelemetry-instrumentation-cohere==0.47.5
-opentelemetry-instrumentation-crewai==0.47.5
-opentelemetry-instrumentation-google-generativeai==0.47.5
-opentelemetry-instrumentation-groq==0.47.5
-opentelemetry-instrumentation-haystack==0.47.5
-opentelemetry-instrumentation-lancedb==0.47.5
-opentelemetry-instrumentation-llamaindex==0.47.5
+opentelemetry-instrumentation-alephalpha==0.48.0
+opentelemetry-instrumentation-anthropic==0.48.0
+opentelemetry-instrumentation-bedrock==0.48.0
+opentelemetry-instrumentation-chromadb==0.48.0
+opentelemetry-instrumentation-cohere==0.48.0
+opentelemetry-instrumentation-crewai==0.48.0
+opentelemetry-instrumentation-google-generativeai==0.48.0
+opentelemetry-instrumentation-groq==0.48.0
+opentelemetry-instrumentation-haystack==0.48.0
+opentelemetry-instrumentation-lancedb==0.48.0
+opentelemetry-instrumentation-llamaindex==0.48.0
 opentelemetry-instrumentation-logging==0.59b0
-opentelemetry-instrumentation-marqo==0.47.5
-opentelemetry-instrumentation-mcp==0.47.5
-opentelemetry-instrumentation-milvus==0.47.5
-opentelemetry-instrumentation-mistralai==0.47.5
-opentelemetry-instrumentation-ollama==0.47.5
-opentelemetry-instrumentation-openai==0.47.5
-opentelemetry-instrumentation-openai-agents==0.47.5
-opentelemetry-instrumentation-pinecone==0.47.5
-opentelemetry-instrumentation-qdrant==0.47.5
+opentelemetry-instrumentation-marqo==0.48.0
+opentelemetry-instrumentation-mcp==0.48.0
+opentelemetry-instrumentation-milvus==0.48.0
+opentelemetry-instrumentation-mistralai==0.48.0
+opentelemetry-instrumentation-ollama==0.48.0
+opentelemetry-instrumentation-openai==0.48.0
+opentelemetry-instrumentation-openai-agents==0.48.0
+opentelemetry-instrumentation-pinecone==0.48.0
+opentelemetry-instrumentation-qdrant==0.48.0
 opentelemetry-instrumentation-redis==0.59b0
-opentelemetry-instrumentation-replicate==0.47.5
+opentelemetry-instrumentation-replicate==0.48.0
 opentelemetry-instrumentation-requests==0.59b0
-opentelemetry-instrumentation-sagemaker==0.47.5
+opentelemetry-instrumentation-sagemaker==0.48.0
 opentelemetry-instrumentation-sqlalchemy==0.59b0
 opentelemetry-instrumentation-threading==0.59b0
-opentelemetry-instrumentation-together==0.47.5
-opentelemetry-instrumentation-transformers==0.47.5
+opentelemetry-instrumentation-together==0.48.0
+opentelemetry-instrumentation-transformers==0.48.0
 opentelemetry-instrumentation-urllib3==0.59b0
-opentelemetry-instrumentation-vertexai==0.47.5
-opentelemetry-instrumentation-watsonx==0.47.5
-opentelemetry-instrumentation-weaviate==0.47.5
-opentelemetry-instrumentation-writer==0.47.5
+opentelemetry-instrumentation-vertexai==0.48.0
+opentelemetry-instrumentation-watsonx==0.48.0
+opentelemetry-instrumentation-weaviate==0.48.0
+opentelemetry-instrumentation-writer==0.48.0
 opentelemetry-proto @ git+https://github.com/open-telemetry/opentelemetry-python.git@1f68134481c46e476a127b61a36dc69889275d15#subdirectory=opentelemetry-proto
 opentelemetry-sdk @ git+https://github.com/open-telemetry/opentelemetry-python.git@1f68134481c46e476a127b61a36dc69889275d15#subdirectory=opentelemetry-sdk
 opentelemetry-semantic-conventions @ git+https://github.com/open-telemetry/opentelemetry-python.git@1f68134481c46e476a127b61a36dc69889275d15#subdirectory=opentelemetry-semantic-conventions
@@ -170,18 +170,18 @@ sphinxcontrib-jquery==4.1
 sphinxcontrib-jsmath==1.0.1
 sphinxcontrib-qthelp==2.0.0
 sphinxcontrib-serializinghtml==2.0.0
--e git+ssh://git@github.com/signalfx/splunk-otel-python-contrib.git@2b4a41abac7725b53c82f480833c874aca0af072#egg=splunk_otel_genai_evals_deepeval&subdirectory=util/opentelemetry-util-genai-evals-deepeval
--e git+ssh://git@github.com/signalfx/splunk-otel-python-contrib.git@2b4a41abac7725b53c82f480833c874aca0af072#egg=splunk_otel_instrumentation_langchain&subdirectory=instrumentation-genai/opentelemetry-instrumentation-langchain
--e git+ssh://git@github.com/signalfx/splunk-otel-python-contrib.git@2b4a41abac7725b53c82f480833c874aca0af072#egg=splunk_otel_util_genai&subdirectory=util/opentelemetry-util-genai
--e git+ssh://git@github.com/signalfx/splunk-otel-python-contrib.git@2b4a41abac7725b53c82f480833c874aca0af072#egg=splunk_otel_util_genai_evals&subdirectory=util/opentelemetry-util-genai-evals
--e git+ssh://git@github.com/signalfx/splunk-otel-python-contrib.git@2b4a41abac7725b53c82f480833c874aca0af072#egg=splunk_otel_util_genai_translator_traceloop&subdirectory=util/opentelemetry-util-genai-traceloop-translator
+# splunk-otel-genai-evals-deepeval
+splunk-otel-instrumentation-langchain
+splunk-otel-util-genai
+splunk-otel-util-genai-evals
+splunk-otel-genai-emitters-splunk
 tabulate==0.9.0
 tenacity==9.1.2
 tiktoken==0.12.0
 tokenizers==0.22.1
 tomlkit==0.13.3
 tqdm==4.67.1
-traceloop-sdk==0.47.5
+traceloop-sdk==0.48.0
 typer==0.20.0
 typer-slim==0.20.0
 typing-inspection==0.4.2

util/opentelemetry-util-genai-evals-deepeval/pyproject.toml

@@ -27,7 +27,7 @@ classifiers = [
 dependencies = [
   "splunk-otel-util-genai>=0.1.4",
   "splunk-otel-util-genai-evals>=0.1.4",
-  "deepeval>=3.7.0",
+  "deepeval>=3.3.9,<3.8.0",
   "openai>=1.0.0",
 ]
 

util/opentelemetry-util-genai-evals-deepeval/src/opentelemetry/util/evaluator/deepeval.py

@@ -82,8 +82,9 @@ def genai_debug_log(*_a: Any, **_k: Any) -> None:  # type: ignore
 # it does not emit extra spans or events when running inside the GenAI
 # instrumentation stack. Users can re-enable it by explicitly setting
 # ``DEEPEVAL_TELEMETRY_OPT_OUT`` to ``0`` before importing this module.
+# "YES" works with deepeval>=3.3.9,<3.8.0
 if os.environ.get("DEEPEVAL_TELEMETRY_OPT_OUT") is None:
-    os.environ["DEEPEVAL_TELEMETRY_OPT_OUT"] = "1"
+    os.environ["DEEPEVAL_TELEMETRY_OPT_OUT"] = "YES"
 
 
 @dataclass(frozen=True)

util/opentelemetry-util-genai-traceloop-translator/src/opentelemetry/util/genai/processor/__init__.py

@@ -14,7 +14,6 @@
 
 """Traceloop span processor and transformation utilities."""
 
-from .filtering_span_processor import FilteringSpanProcessor
 from .traceloop_span_processor import TraceloopSpanProcessor
 
-__all__ = ["TraceloopSpanProcessor", "FilteringSpanProcessor"]
+__all__ = ["TraceloopSpanProcessor"]