From ce5f3ff9f8461843d642f983bcab244f0bb316f0 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Wed, 12 Nov 2025 13:25:55 +0000 Subject: [PATCH 01/34] feat(llma): Add OpenTelemetry traces ingestion foundation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements the core infrastructure for ingesting OpenTelemetry traces and converting them to PostHog AI events for LLM Analytics. ## TypeScript (Plugin-server) **Location**: `plugin-server/src/llm-analytics/otel/` - **validation.ts**: Industry-standard OTel limits and validation functions - Max 1000 spans per request (aligns with OTel SDK defaults) - Max 100KB per attribute value (generous for LLM content) - Max 128 attributes/events/links per span - **types.ts**: TypeScript types for OTel structures and AI events - OTelSpan, SpanStatus, SpanEvent, etc. - AIEventProperties mapping - Transformer version tracking (v1.0.0) - **conventions/posthog-native.ts**: PostHog-native attributes (posthog.ai.*) - Highest priority in waterfall pattern - Direct mapping to PostHog AI event properties - **conventions/genai.ts**: GenAI semantic conventions (gen_ai.*) - OpenTelemetry standard for LLM observability - Fallback when PostHog-native attributes not present - Supports prompt/completion in multiple formats - **transformer.ts**: Core span → AI event transformation logic - Waterfall pattern: PostHog > GenAI > OTel built-ins - Validates spans against limits - Calculates latency, determines event type - Handles baggage for session context - Maps to $ai_generation, $ai_span, $ai_trace events ## Python (API Endpoint) **Location**: `products/llm_analytics/backend/api/otel/` - **ingestion.py**: OTLP/HTTP endpoint stub - Endpoint: POST /api/projects/:project_id/ai/otel/v1/traces - Auth: PersonalAPIKeyAuthentication - Content-Type: application/x-protobuf - Returns 200 with placeholder response (parser TODO) **URL Routing**: `posthog/urls.py` - Registered at `/api/projects//ai/otel/v1/traces` - CSRF exempt (external OTel exporters don't have CSRF tokens) ## Next Steps 1. Implement protobuf parser (opentelemetry-proto package) 2. Connect transformer to capture pipeline 3. Add unit tests for transformer 4. Add integration tests for endpoint 5. Test end-to-end with OTel SDK 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .../llm-analytics/otel/conventions/genai.ts | 222 +++++++++++++ .../otel/conventions/posthog-native.ts | 147 +++++++++ plugin-server/src/llm-analytics/otel/index.ts | 23 ++ .../src/llm-analytics/otel/transformer.ts | 305 ++++++++++++++++++ plugin-server/src/llm-analytics/otel/types.ts | 220 +++++++++++++ .../src/llm-analytics/otel/validation.ts | 154 +++++++++ posthog/urls.py | 3 + .../backend/api/otel/__init__.py | 1 + .../backend/api/otel/ingestion.py | 227 +++++++++++++ 9 files changed, 1302 insertions(+) create mode 100644 plugin-server/src/llm-analytics/otel/conventions/genai.ts create mode 100644 plugin-server/src/llm-analytics/otel/conventions/posthog-native.ts create mode 100644 plugin-server/src/llm-analytics/otel/index.ts create mode 100644 plugin-server/src/llm-analytics/otel/transformer.ts create mode 100644 plugin-server/src/llm-analytics/otel/types.ts create mode 100644 plugin-server/src/llm-analytics/otel/validation.ts create mode 100644 products/llm_analytics/backend/api/otel/__init__.py create mode 100644 products/llm_analytics/backend/api/otel/ingestion.py diff --git a/plugin-server/src/llm-analytics/otel/conventions/genai.ts b/plugin-server/src/llm-analytics/otel/conventions/genai.ts new file mode 100644 index 000000000000..302e8db53911 --- /dev/null +++ b/plugin-server/src/llm-analytics/otel/conventions/genai.ts @@ -0,0 +1,222 @@ +/** + * OpenTelemetry GenAI semantic conventions. + * + * Based on: https://opentelemetry.io/docs/specs/semconv/gen-ai/ + * + * These are standard attributes defined by the OpenTelemetry community + * for generative AI workloads. + * + * Example usage in OTel SDK: + * ```python + * from opentelemetry.semconv.ai import SpanAttributes + * + * span.set_attribute(SpanAttributes.GEN_AI_SYSTEM, "openai") + * span.set_attribute(SpanAttributes.GEN_AI_REQUEST_MODEL, "gpt-4") + * span.set_attribute(SpanAttributes.GEN_AI_OPERATION_NAME, "chat") + * ``` + */ +import type { AttributeValue, ExtractedAttributes, OTelSpan } from '../types' + +/** + * Get attribute value + */ +function getAttr(span: OTelSpan, key: string): AttributeValue | undefined { + return span.attributes[key] +} + +/** + * Get string attribute + */ +function getString(span: OTelSpan, key: string): string | undefined { + const value = getAttr(span, key) + return typeof value === 'string' ? value : undefined +} + +/** + * Get number attribute + */ +function getNumber(span: OTelSpan, key: string): number | undefined { + const value = getAttr(span, key) + return typeof value === 'number' ? value : undefined +} + +/** + * Parse JSON attribute if it's a string + */ +function parseJSON(value: AttributeValue | undefined): any { + if (typeof value === 'string') { + try { + return JSON.parse(value) + } catch { + return value + } + } + return value +} + +/** + * Extract model name from either request or response + */ +function extractModel(span: OTelSpan): string | undefined { + return getString(span, 'gen_ai.request.model') || getString(span, 'gen_ai.response.model') +} + +/** + * Extract provider/system + */ +function extractProvider(span: OTelSpan): string | undefined { + return getString(span, 'gen_ai.system') +} + +/** + * Extract token usage + */ +function extractTokenUsage(span: OTelSpan): { + input_tokens?: number + output_tokens?: number + cache_read_tokens?: number + cache_write_tokens?: number +} { + return { + input_tokens: getNumber(span, 'gen_ai.usage.input_tokens'), + output_tokens: getNumber(span, 'gen_ai.usage.output_tokens'), + // Cache tokens might not be standardized yet, but support if present + cache_read_tokens: getNumber(span, 'gen_ai.usage.cache_read_input_tokens'), + cache_write_tokens: getNumber(span, 'gen_ai.usage.cache_creation_input_tokens'), + } +} + +/** + * Extract cost + */ +function extractCost(span: OTelSpan): { + input_cost_usd?: number + output_cost_usd?: number + total_cost_usd?: number +} { + const totalCost = getNumber(span, 'gen_ai.usage.cost') + + return { + input_cost_usd: getNumber(span, 'gen_ai.usage.input_cost'), + output_cost_usd: getNumber(span, 'gen_ai.usage.output_cost'), + total_cost_usd: totalCost, + } +} + +/** + * Extract model parameters + */ +function extractModelParams(span: OTelSpan): { + temperature?: number + max_tokens?: number +} { + return { + temperature: getNumber(span, 'gen_ai.request.temperature'), + max_tokens: + getNumber(span, 'gen_ai.request.max_tokens') || getNumber(span, 'gen_ai.request.max_completion_tokens'), + } +} + +/** + * Extract content (prompt/completion) + * GenAI conventions support multiple formats: + * 1. JSON string: gen_ai.prompt = '[{"role": "user", "content": "..."}]' + * 2. Flattened: gen_ai.prompt.0.role, gen_ai.prompt.0.content + * 3. Simple string: gen_ai.prompt = "text" + */ +function extractContent(span: OTelSpan): { + prompt?: string | object + completion?: string | object + input?: string + output?: string +} { + const prompt = getAttr(span, 'gen_ai.prompt') + const completion = getAttr(span, 'gen_ai.completion') + + return { + prompt: parseJSON(prompt), + completion: parseJSON(completion), + // Some implementations use these simpler names + input: getString(span, 'gen_ai.input'), + output: getString(span, 'gen_ai.output'), + } +} + +/** + * Extract GenAI semantic convention attributes from span. + * Returns undefined for missing attributes. + */ +export function extractGenAIAttributes(span: OTelSpan): ExtractedAttributes { + const tokens = extractTokenUsage(span) + const cost = extractCost(span) + const params = extractModelParams(span) + const content = extractContent(span) + + return { + // Core + model: extractModel(span), + provider: extractProvider(span), + operation_name: getString(span, 'gen_ai.operation.name'), + + // Tokens + ...tokens, + + // Cost + ...cost, + + // Parameters + ...params, + + // Content + ...content, + } +} + +/** + * Check if span has any GenAI semantic convention attributes + */ +export function hasGenAIAttributes(span: OTelSpan): boolean { + return Object.keys(span.attributes).some((key) => key.startsWith('gen_ai.')) +} + +/** + * List of supported GenAI semantic convention attributes + */ +export const SUPPORTED_GENAI_ATTRIBUTES = [ + // Core + 'gen_ai.system', + 'gen_ai.request.model', + 'gen_ai.response.model', + 'gen_ai.operation.name', + + // Usage + 'gen_ai.usage.input_tokens', + 'gen_ai.usage.output_tokens', + 'gen_ai.usage.cache_read_input_tokens', + 'gen_ai.usage.cache_creation_input_tokens', + + // Cost + 'gen_ai.usage.cost', + 'gen_ai.usage.input_cost', + 'gen_ai.usage.output_cost', + + // Request params + 'gen_ai.request.temperature', + 'gen_ai.request.max_tokens', + 'gen_ai.request.max_completion_tokens', + 'gen_ai.request.top_p', + 'gen_ai.request.top_k', + 'gen_ai.request.frequency_penalty', + 'gen_ai.request.presence_penalty', + 'gen_ai.request.stop_sequences', + + // Content + 'gen_ai.prompt', + 'gen_ai.completion', + 'gen_ai.input', + 'gen_ai.output', + + // Response + 'gen_ai.response.id', + 'gen_ai.response.finish_reasons', +] as const diff --git a/plugin-server/src/llm-analytics/otel/conventions/posthog-native.ts b/plugin-server/src/llm-analytics/otel/conventions/posthog-native.ts new file mode 100644 index 000000000000..eb8ce638b003 --- /dev/null +++ b/plugin-server/src/llm-analytics/otel/conventions/posthog-native.ts @@ -0,0 +1,147 @@ +/** + * PostHog-native OpenTelemetry attribute conventions. + * + * These attributes have the highest priority and follow the pattern: + * posthog.ai.* + * + * Example usage in OTel SDK: + * ```python + * span.set_attribute("posthog.ai.model", "gpt-4") + * span.set_attribute("posthog.ai.provider", "openai") + * span.set_attribute("posthog.ai.session_id", "sess_123") + * ``` + */ +import type { AttributeValue, ExtractedAttributes, OTelSpan } from '../types' + +/** + * PostHog-native attribute namespace + */ +const POSTHOG_PREFIX = 'posthog.ai.' + +/** + * Get attribute with PostHog prefix + */ +function getPostHogAttr(span: OTelSpan, key: string): AttributeValue | undefined { + return span.attributes[`${POSTHOG_PREFIX}${key}`] +} + +/** + * Get string attribute + */ +function getString(span: OTelSpan, key: string): string | undefined { + const value = getPostHogAttr(span, key) + return typeof value === 'string' ? value : undefined +} + +/** + * Get number attribute + */ +function getNumber(span: OTelSpan, key: string): number | undefined { + const value = getPostHogAttr(span, key) + return typeof value === 'number' ? value : undefined +} + +/** + * Get boolean attribute + */ +function getBoolean(span: OTelSpan, key: string): boolean | undefined { + const value = getPostHogAttr(span, key) + return typeof value === 'boolean' ? value : undefined +} + +/** + * Extract PostHog-native attributes from span. + * Returns undefined for missing attributes (not null). + */ +export function extractPostHogNativeAttributes(span: OTelSpan): ExtractedAttributes { + return { + // Core identifiers + model: getString(span, 'model'), + provider: getString(span, 'provider'), + trace_id: getString(span, 'trace_id'), + span_id: getString(span, 'span_id'), + parent_id: getString(span, 'parent_id'), + session_id: getString(span, 'session_id'), + + // Token usage + input_tokens: getNumber(span, 'input_tokens'), + output_tokens: getNumber(span, 'output_tokens'), + cache_read_tokens: getNumber(span, 'cache_read_tokens'), + cache_write_tokens: getNumber(span, 'cache_write_tokens'), + + // Cost + input_cost_usd: getNumber(span, 'input_cost_usd'), + output_cost_usd: getNumber(span, 'output_cost_usd'), + request_cost_usd: getNumber(span, 'request_cost_usd'), + total_cost_usd: getNumber(span, 'total_cost_usd'), + + // Operation + operation_name: getString(span, 'operation_name'), + + // Content + input: getString(span, 'input'), + output: getString(span, 'output'), + prompt: getPostHogAttr(span, 'prompt'), + completion: getPostHogAttr(span, 'completion'), + + // Model parameters + temperature: getNumber(span, 'temperature'), + max_tokens: getNumber(span, 'max_tokens'), + stream: getBoolean(span, 'stream'), + + // Error tracking + is_error: getBoolean(span, 'is_error'), + error_message: getString(span, 'error_message'), + } +} + +/** + * Check if span has any PostHog-native attributes + */ +export function hasPostHogAttributes(span: OTelSpan): boolean { + return Object.keys(span.attributes).some((key) => key.startsWith(POSTHOG_PREFIX)) +} + +/** + * List of supported PostHog-native attributes for documentation + */ +export const SUPPORTED_POSTHOG_ATTRIBUTES = [ + // Core + 'posthog.ai.model', + 'posthog.ai.provider', + 'posthog.ai.trace_id', + 'posthog.ai.span_id', + 'posthog.ai.parent_id', + 'posthog.ai.session_id', + 'posthog.ai.generation_id', + + // Tokens + 'posthog.ai.input_tokens', + 'posthog.ai.output_tokens', + 'posthog.ai.cache_read_tokens', + 'posthog.ai.cache_write_tokens', + + // Cost + 'posthog.ai.input_cost_usd', + 'posthog.ai.output_cost_usd', + 'posthog.ai.request_cost_usd', + 'posthog.ai.total_cost_usd', + + // Operation + 'posthog.ai.operation_name', + + // Content + 'posthog.ai.input', + 'posthog.ai.output', + 'posthog.ai.prompt', + 'posthog.ai.completion', + + // Parameters + 'posthog.ai.temperature', + 'posthog.ai.max_tokens', + 'posthog.ai.stream', + + // Error + 'posthog.ai.is_error', + 'posthog.ai.error_message', +] as const diff --git a/plugin-server/src/llm-analytics/otel/index.ts b/plugin-server/src/llm-analytics/otel/index.ts new file mode 100644 index 000000000000..09ade8e4d7bd --- /dev/null +++ b/plugin-server/src/llm-analytics/otel/index.ts @@ -0,0 +1,23 @@ +/** + * OpenTelemetry traces ingestion for PostHog LLM Analytics. + * + * This module provides transformation of OpenTelemetry spans to PostHog AI events. + * It supports both PostHog-native and GenAI semantic conventions. + * + * Architecture: + * 1. Python API endpoint receives OTLP protobuf HTTP requests + * 2. Python parses protobuf and creates PostHog events + * 3. Events go through standard ingestion pipeline (Kafka) + * 4. Plugin-server processes AI events (cost calculation, normalization) + * + * This TypeScript code can be used for: + * - Documentation of the transformation logic + * - Testing transformation behavior + * - Future: Direct TypeScript-based ingestion if needed + */ + +export { transformSpanToAIEvent, spanUsesKnownConventions, OTEL_TRANSFORMER_VERSION } from './transformer' +export { extractPostHogNativeAttributes, hasPostHogAttributes } from './conventions/posthog-native' +export { extractGenAIAttributes, hasGenAIAttributes } from './conventions/genai' +export { OTEL_LIMITS } from './validation' +export type * from './types' diff --git a/plugin-server/src/llm-analytics/otel/transformer.ts b/plugin-server/src/llm-analytics/otel/transformer.ts new file mode 100644 index 000000000000..2d558b02cf57 --- /dev/null +++ b/plugin-server/src/llm-analytics/otel/transformer.ts @@ -0,0 +1,305 @@ +/** + * Core OTel span to PostHog AI event transformer. + * + * Transforms OpenTelemetry spans into PostHog AI events using a waterfall + * pattern for attribute extraction: + * 1. PostHog native attributes (highest priority) + * 2. GenAI semantic conventions (fallback) + * 3. OTel span built-ins (trace_id, span_id, etc.) + */ +import { extractGenAIAttributes, hasGenAIAttributes } from './conventions/genai' +import { extractPostHogNativeAttributes, hasPostHogAttributes } from './conventions/posthog-native' +import type { + AIEvent, + AIEventProperties, + AIEventType, + Baggage, + ExtractedAttributes, + InstrumentationScope, + OTEL_TRANSFORMER_VERSION, + OTelSpan, + Resource, + SpanStatusCode, +} from './types' +import { + type ValidationError, + validateAttributeCount, + validateAttributeValue, + validateEventCount, + validateLinkCount, + validateSpanName, +} from './validation' + +export { OTEL_TRANSFORMER_VERSION } from './types' + +/** + * Transform a single OTel span to PostHog AI event. + */ +export function transformSpanToAIEvent( + span: OTelSpan, + resource: Resource, + scope: InstrumentationScope, + baggage?: Baggage +): { event: AIEvent; errors: ValidationError[] } { + const errors: ValidationError[] = [] + + // Validate span + const validation = validateSpan(span) + errors.push(...validation) + + // Extract attributes using waterfall pattern + const posthogAttrs = extractPostHogNativeAttributes(span) + const genaiAttrs = extractGenAIAttributes(span) + + // Merge with precedence: PostHog > GenAI + const mergedAttrs: ExtractedAttributes = { + ...genaiAttrs, + ...posthogAttrs, // PostHog overrides GenAI + } + + // Build AI event properties + const properties = buildEventProperties(span, mergedAttrs, resource, scope, baggage) + + // Determine event type + const eventType = determineEventType(span, mergedAttrs) + + // Calculate timestamp and latency + const timestamp = calculateTimestamp(span) + + // Get distinct_id (from resource attributes or default) + const distinct_id = extractDistinctId(resource, baggage) + + const event: AIEvent = { + event: eventType, + distinct_id, + timestamp, + properties, + } + + return { event, errors } +} + +/** + * Validate span against limits + */ +function validateSpan(span: OTelSpan): ValidationError[] { + const errors: ValidationError[] = [] + + // Validate span name + const nameError = validateSpanName(span.name) + if (nameError) errors.push(nameError) + + // Validate attribute count + const attrCountError = validateAttributeCount(Object.keys(span.attributes).length) + if (attrCountError) errors.push(attrCountError) + + // Validate event count + const eventCountError = validateEventCount(span.events?.length || 0) + if (eventCountError) errors.push(eventCountError) + + // Validate link count + const linkCountError = validateLinkCount(span.links?.length || 0) + if (linkCountError) errors.push(linkCountError) + + // Validate attribute values + for (const [key, value] of Object.entries(span.attributes)) { + const valueError = validateAttributeValue(key, value) + if (valueError) errors.push(valueError) + } + + return errors +} + +/** + * Build PostHog AI event properties from extracted attributes + */ +function buildEventProperties( + span: OTelSpan, + attrs: ExtractedAttributes, + resource: Resource, + scope: InstrumentationScope, + baggage?: Baggage +): AIEventProperties { + // Core identifiers (prefer extracted, fallback to span built-ins) + const trace_id = attrs.trace_id || span.trace_id + const span_id = attrs.span_id || span.span_id + const parent_id = attrs.parent_id || span.parent_span_id + + // Session ID (prefer extracted, fallback to baggage) + const session_id = attrs.session_id || baggage?.session_id || baggage?.['posthog.session_id'] + + // Calculate latency + const latency = calculateLatency(span) + + // Detect error from span status + const is_error = attrs.is_error !== undefined ? attrs.is_error : span.status.code === SpanStatusCode.ERROR + const error_message = attrs.error_message || (is_error ? span.status.message : undefined) + + // Build base properties + const properties: AIEventProperties = { + // Core IDs + $ai_trace_id: trace_id, + $ai_span_id: span_id, + ...(parent_id && { $ai_parent_id: parent_id }), + ...(session_id && { $ai_session_id: session_id }), + + // Model info + ...(attrs.model && { $ai_model: attrs.model }), + ...(attrs.provider && { $ai_provider: attrs.provider }), + + // Tokens + ...(attrs.input_tokens !== undefined && { $ai_input_tokens: attrs.input_tokens }), + ...(attrs.output_tokens !== undefined && { $ai_output_tokens: attrs.output_tokens }), + ...(attrs.cache_read_tokens !== undefined && { $ai_cache_read_tokens: attrs.cache_read_tokens }), + ...(attrs.cache_write_tokens !== undefined && { $ai_cache_write_tokens: attrs.cache_write_tokens }), + + // Cost + ...(attrs.input_cost_usd !== undefined && { $ai_input_cost_usd: attrs.input_cost_usd }), + ...(attrs.output_cost_usd !== undefined && { $ai_output_cost_usd: attrs.output_cost_usd }), + ...(attrs.total_cost_usd !== undefined && { $ai_total_cost_usd: attrs.total_cost_usd }), + + // Timing + ...(latency !== undefined && { $ai_latency: latency }), + + // Error + ...(is_error && { $ai_is_error: is_error }), + ...(error_message && { $ai_error_message: error_message }), + + // Model parameters + ...(attrs.temperature !== undefined && { $ai_temperature: attrs.temperature }), + ...(attrs.max_tokens !== undefined && { $ai_max_tokens: attrs.max_tokens }), + ...(attrs.stream !== undefined && { $ai_stream: attrs.stream }), + + // Content (if present and not too large) + ...(attrs.input && { $ai_input: stringifyContent(attrs.input) }), + ...(attrs.output && { $ai_output_choices: stringifyContent(attrs.output) }), + ...(attrs.prompt && { $ai_input: stringifyContent(attrs.prompt) }), + ...(attrs.completion && { $ai_output_choices: stringifyContent(attrs.completion) }), + + // Metadata + $ai_otel_transformer_version: '1.0.0', + $ai_otel_span_kind: span.kind.toString(), + $ai_otel_status_code: span.status.code.toString(), + + // Resource attributes (service name, etc.) + ...(resource.attributes['service.name'] && { + $ai_service_name: resource.attributes['service.name'] as string, + }), + + // Instrumentation scope + $ai_instrumentation_scope_name: scope.name, + ...(scope.version && { $ai_instrumentation_scope_version: scope.version }), + } + + // Add remaining span attributes (not already mapped) + const mappedKeys = new Set([ + 'posthog.ai.model', + 'posthog.ai.provider', + 'gen_ai.system', + 'gen_ai.request.model', + 'gen_ai.response.model', + 'gen_ai.operation.name', + 'gen_ai.usage.input_tokens', + 'gen_ai.usage.output_tokens', + 'gen_ai.prompt', + 'gen_ai.completion', + 'service.name', + ]) + + for (const [key, value] of Object.entries(span.attributes)) { + if (!mappedKeys.has(key) && !key.startsWith('posthog.ai.') && !key.startsWith('gen_ai.')) { + // Add unmapped attributes with prefix + properties[`otel.${key}`] = value + } + } + + return properties +} + +/** + * Determine AI event type from span + */ +function determineEventType(span: OTelSpan, attrs: ExtractedAttributes): AIEventType { + const opName = attrs.operation_name?.toLowerCase() + + // Check operation name + if (opName === 'chat' || opName === 'completion') { + return '$ai_generation' + } else if (opName === 'embedding' || opName === 'embeddings') { + return '$ai_embedding' + } + + // Check if span is root (no parent) + if (!span.parent_span_id) { + return '$ai_trace' + } + + // Default to generic span + return '$ai_span' +} + +/** + * Calculate timestamp from span start time + */ +function calculateTimestamp(span: OTelSpan): string { + const nanos = BigInt(span.start_time_unix_nano) + const millis = Number(nanos / BigInt(1_000_000)) + return new Date(millis).toISOString() +} + +/** + * Calculate latency in seconds from span start/end time + */ +function calculateLatency(span: OTelSpan): number | undefined { + if (!span.end_time_unix_nano) return undefined + + const startNanos = BigInt(span.start_time_unix_nano) + const endNanos = BigInt(span.end_time_unix_nano) + const durationNanos = endNanos - startNanos + + // Convert to seconds + return Number(durationNanos) / 1_000_000_000 +} + +/** + * Extract distinct_id from resource or baggage + */ +function extractDistinctId(resource: Resource, baggage?: Baggage): string { + // Try resource attributes + const userId = + resource.attributes['user.id'] || + resource.attributes['enduser.id'] || + resource.attributes['posthog.distinct_id'] + + if (typeof userId === 'string') { + return userId + } + + // Try baggage + if (baggage?.user_id) { + return baggage.user_id + } + if (baggage?.distinct_id) { + return baggage.distinct_id + } + + // Default to anonymous + return 'anonymous' +} + +/** + * Stringify content (handles objects and strings) + */ +function stringifyContent(content: any): string { + if (typeof content === 'string') { + return content + } + return JSON.stringify(content) +} + +/** + * Check if span uses PostHog or GenAI conventions + */ +export function spanUsesKnownConventions(span: OTelSpan): boolean { + return hasPostHogAttributes(span) || hasGenAIAttributes(span) +} diff --git a/plugin-server/src/llm-analytics/otel/types.ts b/plugin-server/src/llm-analytics/otel/types.ts new file mode 100644 index 000000000000..2d164d168afc --- /dev/null +++ b/plugin-server/src/llm-analytics/otel/types.ts @@ -0,0 +1,220 @@ +/** + * TypeScript types for OpenTelemetry trace ingestion. + * + * These types represent the subset of OTLP trace data we care about + * for converting to PostHog AI events. + */ + +/** + * Simplified OTel Span representation. + * Maps to opentelemetry.proto.trace.v1.Span + */ +export interface OTelSpan { + trace_id: string // hex-encoded 16-byte trace ID + span_id: string // hex-encoded 8-byte span ID + parent_span_id?: string // hex-encoded 8-byte parent span ID + name: string + kind: SpanKind + start_time_unix_nano: string // nanoseconds since epoch + end_time_unix_nano: string // nanoseconds since epoch + attributes: Record + events: SpanEvent[] + links: SpanLink[] + status: SpanStatus +} + +/** + * Span kind enum + */ +export enum SpanKind { + UNSPECIFIED = 0, + INTERNAL = 1, + SERVER = 2, + CLIENT = 3, + PRODUCER = 4, + CONSUMER = 5, +} + +/** + * Attribute value types + */ +export type AttributeValue = string | number | boolean | string[] | number[] + +/** + * Span event + */ +export interface SpanEvent { + time_unix_nano: string + name: string + attributes: Record +} + +/** + * Span link + */ +export interface SpanLink { + trace_id: string + span_id: string + attributes: Record +} + +/** + * Span status + */ +export interface SpanStatus { + code: SpanStatusCode + message?: string +} + +export enum SpanStatusCode { + UNSET = 0, + OK = 1, + ERROR = 2, +} + +/** + * Resource represents the entity producing telemetry. + */ +export interface Resource { + attributes: Record +} + +/** + * Instrumentation scope (library/tracer info) + */ +export interface InstrumentationScope { + name: string + version?: string + attributes?: Record +} + +/** + * Baggage context propagation + */ +export interface Baggage { + [key: string]: string +} + +/** + * Parsed OTLP trace request + */ +export interface ParsedOTLPRequest { + spans: OTelSpan[] + resource: Resource + scope: InstrumentationScope + baggage?: Baggage +} + +/** + * PostHog AI event properties (subset relevant for OTel mapping) + */ +export interface AIEventProperties { + // Core identifiers + $ai_trace_id: string + $ai_span_id: string + $ai_parent_id?: string + $ai_session_id?: string + $ai_generation_id?: string + + // Model info + $ai_model?: string + $ai_provider?: string + + // Token usage + $ai_input_tokens?: number + $ai_output_tokens?: number + $ai_cache_read_tokens?: number + $ai_cache_write_tokens?: number + + // Cost + $ai_input_cost_usd?: number + $ai_output_cost_usd?: number + $ai_total_cost_usd?: number + + // Timing + $ai_latency?: number // seconds + + // Error tracking + $ai_is_error?: boolean + $ai_error_message?: string + + // Model parameters + $ai_temperature?: number + $ai_max_tokens?: number + $ai_stream?: boolean + + // Content (may be URLs to blob storage) + $ai_input?: string + $ai_output_choices?: string + + // Metadata + $ai_otel_transformer_version: string + $ai_otel_span_kind?: string + $ai_otel_status_code?: string + + // Additional properties + [key: string]: AttributeValue | undefined +} + +/** + * PostHog AI event + */ +export interface AIEvent { + event: AIEventType + distinct_id: string + timestamp: string // ISO 8601 + properties: AIEventProperties +} + +/** + * AI event types + */ +export type AIEventType = '$ai_generation' | '$ai_embedding' | '$ai_span' | '$ai_trace' | '$ai_metric' | '$ai_feedback' + +/** + * Transformer version + */ +export const OTEL_TRANSFORMER_VERSION = '1.0.0' + +/** + * Extracted attributes from conventions + */ +export interface ExtractedAttributes { + // Core + model?: string + provider?: string + trace_id?: string + span_id?: string + parent_id?: string + session_id?: string + + // Tokens + input_tokens?: number + output_tokens?: number + cache_read_tokens?: number + cache_write_tokens?: number + + // Cost + input_cost_usd?: number + output_cost_usd?: number + request_cost_usd?: number + total_cost_usd?: number + + // Operation + operation_name?: string // chat, completion, embedding + + // Content + input?: string + output?: string + prompt?: string | object + completion?: string | object + + // Parameters + temperature?: number + max_tokens?: number + stream?: boolean + + // Error + is_error?: boolean + error_message?: string +} diff --git a/plugin-server/src/llm-analytics/otel/validation.ts b/plugin-server/src/llm-analytics/otel/validation.ts new file mode 100644 index 000000000000..d13d52c1ed7b --- /dev/null +++ b/plugin-server/src/llm-analytics/otel/validation.ts @@ -0,0 +1,154 @@ +/** + * OpenTelemetry ingestion validation constants and limits. + * + * These limits align with: + * - OpenTelemetry SDK defaults + * - Industry standard observability platforms (Jaeger, New Relic, Datadog) + * - PostHog infrastructure constraints + */ + +export const OTEL_LIMITS = { + /** + * Maximum number of spans per OTLP export request. + * Most OTel SDKs batch export around 512 spans by default. + */ + MAX_SPANS_PER_REQUEST: 1000, + + /** + * Maximum number of attributes per span. + * Aligns with OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT default. + */ + MAX_ATTRIBUTES_PER_SPAN: 128, + + /** + * Maximum number of events per span. + * Aligns with OTEL_SPAN_EVENT_COUNT_LIMIT default. + */ + MAX_EVENTS_PER_SPAN: 128, + + /** + * Maximum number of links per span. + * Aligns with OTEL_SPAN_LINK_COUNT_LIMIT default. + */ + MAX_LINKS_PER_SPAN: 128, + + /** + * Maximum length of an attribute value in bytes. + * Set generously to 100KB to accommodate LLM prompts/completions. + * Can be extended to blob storage in future if needed. + */ + MAX_ATTRIBUTE_VALUE_LENGTH: 100_000, + + /** + * Maximum length of span name. + * Aligns with OTel recommendations. + */ + MAX_SPAN_NAME_LENGTH: 1024, + + /** + * Maximum length of resource attributes. + */ + MAX_RESOURCE_ATTRIBUTES: 64, +} as const + +export interface ValidationError { + field: string + value: number + limit: number + message: string +} + +export interface ValidationResult { + valid: boolean + errors: ValidationError[] +} + +/** + * Validate attribute value length. + */ +export function validateAttributeValue(key: string, value: unknown): ValidationError | null { + if (typeof value === 'string' && value.length > OTEL_LIMITS.MAX_ATTRIBUTE_VALUE_LENGTH) { + return { + field: `attribute.${key}`, + value: value.length, + limit: OTEL_LIMITS.MAX_ATTRIBUTE_VALUE_LENGTH, + message: `Attribute '${key}' exceeds ${OTEL_LIMITS.MAX_ATTRIBUTE_VALUE_LENGTH} bytes (${value.length} bytes). Consider reducing payload size.`, + } + } + return null +} + +/** + * Validate span name length. + */ +export function validateSpanName(name: string): ValidationError | null { + if (name.length > OTEL_LIMITS.MAX_SPAN_NAME_LENGTH) { + return { + field: 'span.name', + value: name.length, + limit: OTEL_LIMITS.MAX_SPAN_NAME_LENGTH, + message: `Span name exceeds ${OTEL_LIMITS.MAX_SPAN_NAME_LENGTH} characters (${name.length} characters).`, + } + } + return null +} + +/** + * Validate number of attributes in span. + */ +export function validateAttributeCount(attributeCount: number): ValidationError | null { + if (attributeCount > OTEL_LIMITS.MAX_ATTRIBUTES_PER_SPAN) { + return { + field: 'span.attributes', + value: attributeCount, + limit: OTEL_LIMITS.MAX_ATTRIBUTES_PER_SPAN, + message: `Span has ${attributeCount} attributes, maximum is ${OTEL_LIMITS.MAX_ATTRIBUTES_PER_SPAN}.`, + } + } + return null +} + +/** + * Validate number of events in span. + */ +export function validateEventCount(eventCount: number): ValidationError | null { + if (eventCount > OTEL_LIMITS.MAX_EVENTS_PER_SPAN) { + return { + field: 'span.events', + value: eventCount, + limit: OTEL_LIMITS.MAX_EVENTS_PER_SPAN, + message: `Span has ${eventCount} events, maximum is ${OTEL_LIMITS.MAX_EVENTS_PER_SPAN}.`, + } + } + return null +} + +/** + * Validate number of links in span. + */ +export function validateLinkCount(linkCount: number): ValidationError | null { + if (linkCount > OTEL_LIMITS.MAX_LINKS_PER_SPAN) { + return { + field: 'span.links', + value: linkCount, + limit: OTEL_LIMITS.MAX_LINKS_PER_SPAN, + message: `Span has ${linkCount} links, maximum is ${OTEL_LIMITS.MAX_LINKS_PER_SPAN}.`, + } + } + return null +} + +/** + * Validate total span count in request. + */ +export function validateSpanCount(spanCount: number): ValidationError | null { + if (spanCount > OTEL_LIMITS.MAX_SPANS_PER_REQUEST) { + return { + field: 'request.spans', + value: spanCount, + limit: OTEL_LIMITS.MAX_SPANS_PER_REQUEST, + message: `Request contains ${spanCount} spans, maximum is ${OTEL_LIMITS.MAX_SPANS_PER_REQUEST}. Configure batch size in your OTel SDK (e.g., OTEL_BSP_MAX_EXPORT_BATCH_SIZE).`, + } + } + return null +} diff --git a/posthog/urls.py b/posthog/urls.py index e9a95d8a3a44..338258908ab5 100644 --- a/posthog/urls.py +++ b/posthog/urls.py @@ -47,6 +47,7 @@ from posthog.temporal.codec_server import decode_payloads from products.early_access_features.backend.api import early_access_features +from products.llm_analytics.backend.api.otel.ingestion import otel_traces_endpoint from .utils import opt_slash_path, render_template from .views import ( @@ -168,6 +169,8 @@ def authorize_and_redirect(request: HttpRequest) -> HttpResponse: # ee *ee_urlpatterns, # api + # OpenTelemetry traces ingestion for LLM Analytics + path("api/projects//ai/otel/v1/traces", csrf_exempt(otel_traces_endpoint)), path("api/environments//progress/", progress), path("api/environments//query//progress/", progress), path("api/environments//query//progress", progress), diff --git a/products/llm_analytics/backend/api/otel/__init__.py b/products/llm_analytics/backend/api/otel/__init__.py new file mode 100644 index 000000000000..1cffc22cfd30 --- /dev/null +++ b/products/llm_analytics/backend/api/otel/__init__.py @@ -0,0 +1 @@ +# OpenTelemetry traces ingestion for PostHog LLM Analytics diff --git a/products/llm_analytics/backend/api/otel/ingestion.py b/products/llm_analytics/backend/api/otel/ingestion.py new file mode 100644 index 000000000000..8f5b0e8c9efc --- /dev/null +++ b/products/llm_analytics/backend/api/otel/ingestion.py @@ -0,0 +1,227 @@ +""" +OpenTelemetry traces ingestion API endpoint. + +Accepts OTLP/HTTP (protobuf) format traces and converts them to PostHog AI events. + +Endpoint: POST /api/projects/:project_id/ai/otel/v1/traces +Content-Type: application/x-protobuf +Authorization: Bearer +""" + +from typing import Any + +from django.http import HttpRequest + +import structlog +from drf_spectacular.utils import extend_schema +from rest_framework import status +from rest_framework.decorators import api_view, authentication_classes, permission_classes +from rest_framework.exceptions import ValidationError +from rest_framework.permissions import IsAuthenticated +from rest_framework.response import Response + +from posthog.auth import PersonalAPIKeyAuthentication +from posthog.models import Team + +logger = structlog.get_logger(__name__) + +# OpenTelemetry limits (aligned with plugin-server validation) +OTEL_LIMITS = { + "MAX_SPANS_PER_REQUEST": 1000, + "MAX_ATTRIBUTES_PER_SPAN": 128, + "MAX_EVENTS_PER_SPAN": 128, + "MAX_LINKS_PER_SPAN": 128, + "MAX_ATTRIBUTE_VALUE_LENGTH": 100_000, # 100KB + "MAX_SPAN_NAME_LENGTH": 1024, +} + + +@extend_schema( + description=""" + OpenTelemetry traces ingestion endpoint for LLM Analytics. + + Accepts OTLP/HTTP (protobuf) format traces following the OpenTelemetry Protocol specification. + Converts OTel spans to PostHog AI events using PostHog-native and GenAI semantic conventions. + + Supported conventions: + - PostHog native: posthog.ai.* attributes (highest priority) + - GenAI semantic conventions: gen_ai.* attributes (fallback) + + Example OTel SDK configuration: + ```python + from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter + + exporter = OTLPSpanExporter( + endpoint="https://app.posthog.com/api/projects/{project_id}/ai/otel/v1/traces", + headers={"Authorization": "Bearer phc_your_api_key"} + ) + ``` + + Rate limits and quotas apply as per normal PostHog event ingestion. + """, + request={"application/x-protobuf": bytes}, + responses={ + 200: {"description": "Traces accepted for processing"}, + 400: {"description": "Invalid OTLP format or validation errors"}, + 401: {"description": "Authentication failed"}, + 413: {"description": "Request too large (exceeds span/attribute limits)"}, + }, +) +@api_view(["POST"]) +@authentication_classes([PersonalAPIKeyAuthentication]) +@permission_classes([IsAuthenticated]) +def otel_traces_endpoint(request: HttpRequest, project_id: int) -> Response: + """ + Process OTLP trace export requests. + + This endpoint: + 1. Validates authentication and project access + 2. Parses OTLP protobuf payload + 3. Validates against size/span limits + 4. Transforms OTel spans to PostHog AI events + 5. Routes events to capture pipeline for ingestion + """ + + # Verify team access + try: + team = Team.objects.get(id=project_id, organization=request.user.current_organization) + except Team.DoesNotExist: + return Response( + {"error": "Project not found or access denied"}, + status=status.HTTP_404_NOT_FOUND, + ) + + # Check content type + content_type = request.content_type or "" + if "protobuf" not in content_type and "octet-stream" not in content_type: + return Response( + { + "error": f"Invalid content type: {content_type}. Expected application/x-protobuf or application/octet-stream" + }, + status=status.HTTP_400_BAD_REQUEST, + ) + + # Get raw protobuf body + protobuf_data = request.body + + if not protobuf_data: + return Response( + {"error": "Empty request body"}, + status=status.HTTP_400_BAD_REQUEST, + ) + + logger.info( + "otel_traces_received", + team_id=team.id, + content_length=len(protobuf_data), + content_type=content_type, + ) + + try: + # Parse OTLP protobuf (TODO: implement parser) + # For now, return a placeholder response + # parsed_request = parse_otlp_trace_request(protobuf_data) + + # Validate request + # validation_errors = validate_otlp_request(parsed_request) + # if validation_errors: + # return Response( + # {"errors": validation_errors}, + # status=status.HTTP_400_BAD_REQUEST, + # ) + + # Transform spans to AI events + # events = transform_spans_to_ai_events(parsed_request, team) + + # Route to capture pipeline + # capture_events(events, team) + + return Response( + { + "status": "success", + "message": "Traces accepted for processing (stub implementation)", + # "spans_received": len(parsed_request.spans), + # "events_created": len(events), + }, + status=status.HTTP_200_OK, + ) + + except ValidationError as e: + logger.warning( + "otel_traces_validation_error", + team_id=team.id, + error=str(e), + ) + return Response( + {"error": str(e)}, + status=status.HTTP_400_BAD_REQUEST, + ) + + except Exception as e: + logger.error( + "otel_traces_processing_error", + team_id=team.id, + error=str(e), + exc_info=True, + ) + return Response( + {"error": "Internal server error processing traces"}, + status=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + + +def parse_otlp_trace_request(protobuf_data: bytes) -> dict[str, Any]: + """ + Parse OTLP ExportTraceServiceRequest from protobuf bytes. + + TODO: Implement using opentelemetry-proto package + + Returns dict with: + - resource_spans: list of ResourceSpans + - spans: flattened list of Span objects + """ + raise NotImplementedError("OTLP protobuf parsing not yet implemented") + + +def validate_otlp_request(parsed_request: dict[str, Any]) -> list[dict[str, Any]]: + """ + Validate OTLP request against limits. + + Returns list of validation errors (empty if valid). + """ + errors = [] + + # TODO: Implement validation + # - Check span count + # - Check attribute counts + # - Check attribute value sizes + # - Check span name lengths + + return errors + + +def transform_spans_to_ai_events(parsed_request: dict[str, Any], team: Team) -> list[dict[str, Any]]: + """ + Transform OTel spans to PostHog AI events. + + Uses waterfall pattern for attribute extraction: + 1. PostHog native (posthog.ai.*) + 2. GenAI semantic conventions (gen_ai.*) + """ + # TODO: Implement transformation + # - Extract attributes using conventions + # - Determine event type ($ai_generation, $ai_span, etc.) + # - Build event properties + # - Calculate latency + # - Handle baggage for session context + + raise NotImplementedError("Span transformation not yet implemented") + + +def capture_events(events: list[dict[str, Any]], team: Team) -> None: + """ + Route transformed events to PostHog capture pipeline. + + TODO: Use capture_internal or direct Kafka ingestion + """ + raise NotImplementedError("Event capture not yet implemented") From 285563db5b03b93eb86581eb9be3a61ae800eacb Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Wed, 12 Nov 2025 13:44:29 +0000 Subject: [PATCH 02/34] feat(llma): Implement OTLP protobuf parser for OTel traces ingestion Step 1: Parse OTLP/HTTP protobuf payloads - Add opentelemetry-proto==1.38.0 dependency - Create parser.py with full OTLP ExportTraceServiceRequest parsing - Parses spans, resource attributes, instrumentation scope - Handles all OTel value types (string, int, double, bool, array, kvlist, bytes) - Parses baggage from HTTP headers for session context - Integrate parser into ingestion.py endpoint - Add validation against industry-standard limits - Check span count, attribute count/size, event count, link count - Provide actionable error messages with limit guidance Next: Transform parsed spans to PostHog AI events (Step 2) --- .../backend/api/otel/ingestion.py | 136 ++++++++++--- .../llm_analytics/backend/api/otel/parser.py | 185 ++++++++++++++++++ pyproject.toml | 1 + 3 files changed, 296 insertions(+), 26 deletions(-) create mode 100644 products/llm_analytics/backend/api/otel/parser.py diff --git a/products/llm_analytics/backend/api/otel/ingestion.py b/products/llm_analytics/backend/api/otel/ingestion.py index 8f5b0e8c9efc..74a359d688fe 100644 --- a/products/llm_analytics/backend/api/otel/ingestion.py +++ b/products/llm_analytics/backend/api/otel/ingestion.py @@ -23,6 +23,8 @@ from posthog.auth import PersonalAPIKeyAuthentication from posthog.models import Team +from .parser import parse_baggage_header, parse_otlp_request + logger = structlog.get_logger(__name__) # OpenTelemetry limits (aligned with plugin-server validation) @@ -118,30 +120,45 @@ def otel_traces_endpoint(request: HttpRequest, project_id: int) -> Response: ) try: - # Parse OTLP protobuf (TODO: implement parser) - # For now, return a placeholder response - # parsed_request = parse_otlp_trace_request(protobuf_data) + # Parse baggage from headers (for session context) + baggage_header = request.headers.get("baggage") + baggage = parse_baggage_header(baggage_header) if baggage_header else {} - # Validate request - # validation_errors = validate_otlp_request(parsed_request) - # if validation_errors: - # return Response( - # {"errors": validation_errors}, - # status=status.HTTP_400_BAD_REQUEST, - # ) + # Parse OTLP protobuf + parsed_request = parse_otlp_trace_request(protobuf_data) - # Transform spans to AI events - # events = transform_spans_to_ai_events(parsed_request, team) + logger.info( + "otel_traces_parsed", + team_id=team.id, + spans_count=len(parsed_request["spans"]), + has_baggage=bool(baggage), + ) - # Route to capture pipeline + # Validate request + validation_errors = validate_otlp_request(parsed_request) + if validation_errors: + logger.warning( + "otel_traces_validation_failed", + team_id=team.id, + errors=validation_errors, + ) + return Response( + {"error": "Validation failed", "details": validation_errors}, + status=status.HTTP_400_BAD_REQUEST, + ) + + # Transform spans to AI events (TODO: Step 2) + # events = transform_spans_to_ai_events(parsed_request, baggage, team) + + # Route to capture pipeline (TODO: Step 3) # capture_events(events, team) return Response( { "status": "success", - "message": "Traces accepted for processing (stub implementation)", - # "spans_received": len(parsed_request.spans), - # "events_created": len(events), + "message": "Traces parsed and validated successfully", + "spans_received": len(parsed_request["spans"]), + # "events_created": len(events), # Will add in step 3 }, status=status.HTTP_200_OK, ) @@ -174,13 +191,12 @@ def parse_otlp_trace_request(protobuf_data: bytes) -> dict[str, Any]: """ Parse OTLP ExportTraceServiceRequest from protobuf bytes. - TODO: Implement using opentelemetry-proto package - Returns dict with: - - resource_spans: list of ResourceSpans - - spans: flattened list of Span objects + - spans: list of parsed span dicts + - resource: dict of resource attributes + - scope: dict of instrumentation scope info """ - raise NotImplementedError("OTLP protobuf parsing not yet implemented") + return parse_otlp_request(protobuf_data) def validate_otlp_request(parsed_request: dict[str, Any]) -> list[dict[str, Any]]: @@ -190,12 +206,80 @@ def validate_otlp_request(parsed_request: dict[str, Any]) -> list[dict[str, Any] Returns list of validation errors (empty if valid). """ errors = [] + spans = parsed_request.get("spans", []) + + # Check span count + if len(spans) > OTEL_LIMITS["MAX_SPANS_PER_REQUEST"]: + errors.append( + { + "field": "request.spans", + "value": len(spans), + "limit": OTEL_LIMITS["MAX_SPANS_PER_REQUEST"], + "message": f"Request contains {len(spans)} spans, maximum is {OTEL_LIMITS['MAX_SPANS_PER_REQUEST']}. Configure batch size in your OTel SDK (e.g., OTEL_BSP_MAX_EXPORT_BATCH_SIZE).", + } + ) - # TODO: Implement validation - # - Check span count - # - Check attribute counts - # - Check attribute value sizes - # - Check span name lengths + # Validate each span + for i, span in enumerate(spans): + # Check span name length + span_name = span.get("name", "") + if len(span_name) > OTEL_LIMITS["MAX_SPAN_NAME_LENGTH"]: + errors.append( + { + "field": f"span[{i}].name", + "value": len(span_name), + "limit": OTEL_LIMITS["MAX_SPAN_NAME_LENGTH"], + "message": f"Span name exceeds {OTEL_LIMITS['MAX_SPAN_NAME_LENGTH']} characters.", + } + ) + + # Check attribute count + attributes = span.get("attributes", {}) + if len(attributes) > OTEL_LIMITS["MAX_ATTRIBUTES_PER_SPAN"]: + errors.append( + { + "field": f"span[{i}].attributes", + "value": len(attributes), + "limit": OTEL_LIMITS["MAX_ATTRIBUTES_PER_SPAN"], + "message": f"Span has {len(attributes)} attributes, maximum is {OTEL_LIMITS['MAX_ATTRIBUTES_PER_SPAN']}.", + } + ) + + # Check attribute value sizes + for key, value in attributes.items(): + if isinstance(value, str) and len(value) > OTEL_LIMITS["MAX_ATTRIBUTE_VALUE_LENGTH"]: + errors.append( + { + "field": f"span[{i}].attributes.{key}", + "value": len(value), + "limit": OTEL_LIMITS["MAX_ATTRIBUTE_VALUE_LENGTH"], + "message": f"Attribute '{key}' exceeds {OTEL_LIMITS['MAX_ATTRIBUTE_VALUE_LENGTH']} bytes ({len(value)} bytes). Consider reducing payload size.", + } + ) + + # Check event count + events = span.get("events", []) + if len(events) > OTEL_LIMITS["MAX_EVENTS_PER_SPAN"]: + errors.append( + { + "field": f"span[{i}].events", + "value": len(events), + "limit": OTEL_LIMITS["MAX_EVENTS_PER_SPAN"], + "message": f"Span has {len(events)} events, maximum is {OTEL_LIMITS['MAX_EVENTS_PER_SPAN']}.", + } + ) + + # Check link count + links = span.get("links", []) + if len(links) > OTEL_LIMITS["MAX_LINKS_PER_SPAN"]: + errors.append( + { + "field": f"span[{i}].links", + "value": len(links), + "limit": OTEL_LIMITS["MAX_LINKS_PER_SPAN"], + "message": f"Span has {len(links)} links, maximum is {OTEL_LIMITS['MAX_LINKS_PER_SPAN']}.", + } + ) return errors diff --git a/products/llm_analytics/backend/api/otel/parser.py b/products/llm_analytics/backend/api/otel/parser.py new file mode 100644 index 000000000000..cb3281b542a4 --- /dev/null +++ b/products/llm_analytics/backend/api/otel/parser.py @@ -0,0 +1,185 @@ +""" +OTLP protobuf parser for OpenTelemetry traces. + +Parses ExportTraceServiceRequest protobuf messages and extracts spans, +resource attributes, and instrumentation scope information. +""" + +from typing import Any + +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest +from opentelemetry.proto.common.v1.common_pb2 import AnyValue, KeyValue +from opentelemetry.proto.trace.v1.trace_pb2 import Span + + +def parse_otlp_request(protobuf_data: bytes) -> dict[str, Any]: + """ + Parse OTLP ExportTraceServiceRequest from protobuf bytes. + + Returns a dict with: + - spans: list of parsed span dicts + - resource: dict of resource attributes + - scope: dict of instrumentation scope info + """ + request = ExportTraceServiceRequest() + request.ParseFromString(protobuf_data) + + parsed_spans = [] + resource_attrs = {} + scope_info = {} + + # OTLP structure: resource_spans -> scope_spans -> spans + for resource_spans in request.resource_spans: + # Extract resource attributes (service.name, etc.) + if resource_spans.HasField("resource"): + resource_attrs = parse_attributes(resource_spans.resource.attributes) + + # Iterate through scope spans + for scope_spans in resource_spans.scope_spans: + # Extract instrumentation scope + if scope_spans.HasField("scope"): + scope_info = { + "name": scope_spans.scope.name, + "version": scope_spans.scope.version if scope_spans.scope.version else None, + "attributes": parse_attributes(scope_spans.scope.attributes) + if scope_spans.scope.attributes + else {}, + } + + # Parse each span + for span in scope_spans.spans: + parsed_span = parse_span(span) + parsed_spans.append(parsed_span) + + return { + "spans": parsed_spans, + "resource": resource_attrs, + "scope": scope_info, + } + + +def parse_span(span: Span) -> dict[str, Any]: + """ + Parse a single OTLP span into a dict. + """ + return { + "trace_id": span.trace_id.hex(), + "span_id": span.span_id.hex(), + "parent_span_id": span.parent_span_id.hex() if span.parent_span_id else None, + "name": span.name, + "kind": span.kind, + "start_time_unix_nano": str(span.start_time_unix_nano), + "end_time_unix_nano": str(span.end_time_unix_nano), + "attributes": parse_attributes(span.attributes), + "events": [parse_span_event(event) for event in span.events], + "links": [parse_span_link(link) for link in span.links], + "status": parse_span_status(span.status), + } + + +def parse_span_event(event) -> dict[str, Any]: + """ + Parse a span event. + """ + return { + "time_unix_nano": str(event.time_unix_nano), + "name": event.name, + "attributes": parse_attributes(event.attributes), + } + + +def parse_span_link(link) -> dict[str, Any]: + """ + Parse a span link. + """ + return { + "trace_id": link.trace_id.hex(), + "span_id": link.span_id.hex(), + "attributes": parse_attributes(link.attributes), + } + + +def parse_span_status(status) -> dict[str, Any]: + """ + Parse span status. + """ + return { + "code": status.code, + "message": status.message if status.message else None, + } + + +def parse_attributes(attributes: list[KeyValue]) -> dict[str, Any]: + """ + Parse OpenTelemetry attributes (key-value pairs) into a dict. + + Handles different value types: string, int, double, bool, array, kvlist. + """ + result = {} + + for kv in attributes: + key = kv.key + value = parse_any_value(kv.value) + result[key] = value + + return result + + +def parse_any_value(value: AnyValue) -> Any: + """ + Parse an AnyValue protobuf type into a Python value. + + AnyValue can be: + - string_value + - int_value + - double_value + - bool_value + - array_value (list of AnyValue) + - kvlist_value (dict of key-value pairs) + - bytes_value + """ + # Check which field is set + which = value.WhichOneof("value") + + if which == "string_value": + return value.string_value + elif which == "int_value": + return value.int_value + elif which == "double_value": + return value.double_value + elif which == "bool_value": + return value.bool_value + elif which == "array_value": + return [parse_any_value(item) for item in value.array_value.values] + elif which == "kvlist_value": + return parse_attributes(value.kvlist_value.values) + elif which == "bytes_value": + return value.bytes_value.hex() + else: + # Unknown or unset + return None + + +def parse_baggage_header(baggage_header: str | None) -> dict[str, str]: + """ + Parse OTel baggage from HTTP header. + + Baggage format: key1=value1,key2=value2,... + + Example: session_id=abc123,user_id=user_456 + """ + if not baggage_header: + return {} + + baggage = {} + + # Split by comma + items = baggage_header.split(",") + + for item in items: + item = item.strip() + if "=" in item: + key, value = item.split("=", 1) + baggage[key.strip()] = value.strip() + + return baggage diff --git a/pyproject.toml b/pyproject.toml index a8422c2f7844..0669c98edca5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,6 +83,7 @@ dependencies = [ "nh3==0.2.14", "numpy~=2.1.0", "openai==1.109.1", + "opentelemetry-proto==1.38.0", "openpyxl==3.1.2", "orjson==3.10.15", "pandas~=2.2.2", From fd625955ff33bc73e7f923dc70802eb6b03146a9 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Wed, 12 Nov 2025 13:59:54 +0000 Subject: [PATCH 03/34] feat(llma): Implement span-to-AI-event transformer for OTel traces Step 2: Transform parsed OTel spans to PostHog AI events - Create conventions extractors: - posthog_native.py: Extract posthog.ai.* attributes (highest priority) - genai.py: Extract gen_ai.* attributes (fallback) - Create transformer.py with waterfall pattern: - Merge attributes with precedence: PostHog > GenAI > OTel built-ins - Build AI event properties with proper mapping - Determine event type ($ai_generation, $ai_embedding, $ai_span, $ai_trace) - Calculate timestamps and latency from nano timestamps - Extract distinct_id from resource/baggage with 'anonymous' fallback - Integrate transformer into ingestion.py endpoint: - Transform all spans to AI events - Log transformation results Events are ready for capture pipeline (Step 3). Transformation features: - Supports PostHog native and GenAI semantic conventions - Handles token usage, cost, model parameters - Preserves OTel metadata (span kind, status, scope) - Maps content (input/output, prompt/completion) - Adds unmapped attributes with 'otel.' prefix --- .../backend/api/otel/conventions/__init__.py | 7 + .../backend/api/otel/conventions/genai.py | 76 +++++ .../api/otel/conventions/posthog_native.py | 88 ++++++ .../backend/api/otel/ingestion.py | 35 ++- .../backend/api/otel/transformer.py | 276 ++++++++++++++++++ 5 files changed, 469 insertions(+), 13 deletions(-) create mode 100644 products/llm_analytics/backend/api/otel/conventions/__init__.py create mode 100644 products/llm_analytics/backend/api/otel/conventions/genai.py create mode 100644 products/llm_analytics/backend/api/otel/conventions/posthog_native.py create mode 100644 products/llm_analytics/backend/api/otel/transformer.py diff --git a/products/llm_analytics/backend/api/otel/conventions/__init__.py b/products/llm_analytics/backend/api/otel/conventions/__init__.py new file mode 100644 index 000000000000..06235538a984 --- /dev/null +++ b/products/llm_analytics/backend/api/otel/conventions/__init__.py @@ -0,0 +1,7 @@ +""" +OpenTelemetry semantic conventions for LLM traces. + +Supports: +- PostHog native: posthog.ai.* attributes (highest priority) +- GenAI: gen_ai.* attributes (fallback) +""" diff --git a/products/llm_analytics/backend/api/otel/conventions/genai.py b/products/llm_analytics/backend/api/otel/conventions/genai.py new file mode 100644 index 000000000000..4ba8b346b7ac --- /dev/null +++ b/products/llm_analytics/backend/api/otel/conventions/genai.py @@ -0,0 +1,76 @@ +""" +GenAI semantic conventions for OpenTelemetry. + +Implements the GenAI semantic conventions (gen_ai.*) as fallback +when PostHog-native attributes are not present. + +Reference: https://opentelemetry.io/docs/specs/semconv/gen-ai/ +""" + +from typing import Any + + +def has_genai_attributes(span: dict[str, Any]) -> bool: + """Check if span uses GenAI semantic conventions.""" + attributes = span.get("attributes", {}) + return any(key.startswith("gen_ai.") for key in attributes.keys()) + + +def extract_genai_attributes(span: dict[str, Any]) -> dict[str, Any]: + """ + Extract GenAI semantic convention attributes from span. + + GenAI conventions use `gen_ai.*` prefix and are fallback + when PostHog-native attributes are not present. + """ + attributes = span.get("attributes", {}) + result: dict[str, Any] = {} + + # Model (prefer request, fallback to response, then system) + model = ( + attributes.get("gen_ai.request.model") + or attributes.get("gen_ai.response.model") + or attributes.get("gen_ai.model") + ) + if model is not None: + result["model"] = model + + # Provider (from gen_ai.system) + if (system := attributes.get("gen_ai.system")) is not None: + result["provider"] = system + + # Operation name + if (operation_name := attributes.get("gen_ai.operation.name")) is not None: + result["operation_name"] = operation_name + + # Token usage + if (input_tokens := attributes.get("gen_ai.usage.input_tokens")) is not None: + result["input_tokens"] = input_tokens + if (output_tokens := attributes.get("gen_ai.usage.output_tokens")) is not None: + result["output_tokens"] = output_tokens + + # Content (prompt and completion) + if (prompt := attributes.get("gen_ai.prompt")) is not None: + result["prompt"] = prompt + if (completion := attributes.get("gen_ai.completion")) is not None: + result["completion"] = completion + + # Model parameters + if (temperature := attributes.get("gen_ai.request.temperature")) is not None: + result["temperature"] = temperature + if (max_tokens := attributes.get("gen_ai.request.max_tokens")) is not None: + result["max_tokens"] = max_tokens + if (top_p := attributes.get("gen_ai.request.top_p")) is not None: + result["top_p"] = top_p + if (frequency_penalty := attributes.get("gen_ai.request.frequency_penalty")) is not None: + result["frequency_penalty"] = frequency_penalty + if (presence_penalty := attributes.get("gen_ai.request.presence_penalty")) is not None: + result["presence_penalty"] = presence_penalty + + # Response metadata + if (finish_reasons := attributes.get("gen_ai.response.finish_reasons")) is not None: + result["finish_reasons"] = finish_reasons + if (response_id := attributes.get("gen_ai.response.id")) is not None: + result["response_id"] = response_id + + return result diff --git a/products/llm_analytics/backend/api/otel/conventions/posthog_native.py b/products/llm_analytics/backend/api/otel/conventions/posthog_native.py new file mode 100644 index 000000000000..d8fe30eb0881 --- /dev/null +++ b/products/llm_analytics/backend/api/otel/conventions/posthog_native.py @@ -0,0 +1,88 @@ +""" +PostHog-native OpenTelemetry conventions. + +Attributes with `posthog.ai.*` prefix have highest priority in the waterfall. +""" + +from typing import Any + + +def has_posthog_attributes(span: dict[str, Any]) -> bool: + """Check if span uses PostHog native conventions.""" + attributes = span.get("attributes", {}) + return any(key.startswith("posthog.ai.") for key in attributes.keys()) + + +def extract_posthog_native_attributes(span: dict[str, Any]) -> dict[str, Any]: + """ + Extract PostHog-native attributes from span. + + PostHog-native convention uses `posthog.ai.*` prefix. + This takes highest priority in the waterfall pattern. + """ + attributes = span.get("attributes", {}) + result: dict[str, Any] = {} + + # Helper to get attribute with prefix + def get_attr(key: str) -> Any: + return attributes.get(f"posthog.ai.{key}") + + # Core identifiers + if (model := get_attr("model")) is not None: + result["model"] = model + if (provider := get_attr("provider")) is not None: + result["provider"] = provider + if (trace_id := get_attr("trace_id")) is not None: + result["trace_id"] = trace_id + if (span_id := get_attr("span_id")) is not None: + result["span_id"] = span_id + if (parent_id := get_attr("parent_id")) is not None: + result["parent_id"] = parent_id + if (session_id := get_attr("session_id")) is not None: + result["session_id"] = session_id + if (generation_id := get_attr("generation_id")) is not None: + result["generation_id"] = generation_id + + # Token usage + if (input_tokens := get_attr("input_tokens")) is not None: + result["input_tokens"] = input_tokens + if (output_tokens := get_attr("output_tokens")) is not None: + result["output_tokens"] = output_tokens + if (cache_read_tokens := get_attr("cache_read_tokens")) is not None: + result["cache_read_tokens"] = cache_read_tokens + if (cache_write_tokens := get_attr("cache_write_tokens")) is not None: + result["cache_write_tokens"] = cache_write_tokens + + # Cost + if (input_cost_usd := get_attr("input_cost_usd")) is not None: + result["input_cost_usd"] = input_cost_usd + if (output_cost_usd := get_attr("output_cost_usd")) is not None: + result["output_cost_usd"] = output_cost_usd + if (total_cost_usd := get_attr("total_cost_usd")) is not None: + result["total_cost_usd"] = total_cost_usd + + # Operation + if (operation_name := get_attr("operation_name")) is not None: + result["operation_name"] = operation_name + + # Content + if (input_content := get_attr("input")) is not None: + result["input"] = input_content + if (output_content := get_attr("output")) is not None: + result["output"] = output_content + + # Model parameters + if (temperature := get_attr("temperature")) is not None: + result["temperature"] = temperature + if (max_tokens := get_attr("max_tokens")) is not None: + result["max_tokens"] = max_tokens + if (stream := get_attr("stream")) is not None: + result["stream"] = stream + + # Error tracking + if (is_error := get_attr("is_error")) is not None: + result["is_error"] = is_error + if (error_message := get_attr("error_message")) is not None: + result["error_message"] = error_message + + return result diff --git a/products/llm_analytics/backend/api/otel/ingestion.py b/products/llm_analytics/backend/api/otel/ingestion.py index 74a359d688fe..3a4135ed59c2 100644 --- a/products/llm_analytics/backend/api/otel/ingestion.py +++ b/products/llm_analytics/backend/api/otel/ingestion.py @@ -24,6 +24,7 @@ from posthog.models import Team from .parser import parse_baggage_header, parse_otlp_request +from .transformer import transform_span_to_ai_event logger = structlog.get_logger(__name__) @@ -147,8 +148,14 @@ def otel_traces_endpoint(request: HttpRequest, project_id: int) -> Response: status=status.HTTP_400_BAD_REQUEST, ) - # Transform spans to AI events (TODO: Step 2) - # events = transform_spans_to_ai_events(parsed_request, baggage, team) + # Transform spans to AI events + events = transform_spans_to_ai_events(parsed_request, baggage) + + logger.info( + "otel_traces_transformed", + team_id=team.id, + events_created=len(events), + ) # Route to capture pipeline (TODO: Step 3) # capture_events(events, team) @@ -156,9 +163,9 @@ def otel_traces_endpoint(request: HttpRequest, project_id: int) -> Response: return Response( { "status": "success", - "message": "Traces parsed and validated successfully", + "message": "Traces transformed successfully", "spans_received": len(parsed_request["spans"]), - # "events_created": len(events), # Will add in step 3 + "events_created": len(events), }, status=status.HTTP_200_OK, ) @@ -284,7 +291,7 @@ def validate_otlp_request(parsed_request: dict[str, Any]) -> list[dict[str, Any] return errors -def transform_spans_to_ai_events(parsed_request: dict[str, Any], team: Team) -> list[dict[str, Any]]: +def transform_spans_to_ai_events(parsed_request: dict[str, Any], baggage: dict[str, str]) -> list[dict[str, Any]]: """ Transform OTel spans to PostHog AI events. @@ -292,14 +299,16 @@ def transform_spans_to_ai_events(parsed_request: dict[str, Any], team: Team) -> 1. PostHog native (posthog.ai.*) 2. GenAI semantic conventions (gen_ai.*) """ - # TODO: Implement transformation - # - Extract attributes using conventions - # - Determine event type ($ai_generation, $ai_span, etc.) - # - Build event properties - # - Calculate latency - # - Handle baggage for session context - - raise NotImplementedError("Span transformation not yet implemented") + spans = parsed_request.get("spans", []) + resource = parsed_request.get("resource", {}) + scope = parsed_request.get("scope", {}) + + events = [] + for span in spans: + event = transform_span_to_ai_event(span, resource, scope, baggage) + events.append(event) + + return events def capture_events(events: list[dict[str, Any]], team: Team) -> None: diff --git a/products/llm_analytics/backend/api/otel/transformer.py b/products/llm_analytics/backend/api/otel/transformer.py new file mode 100644 index 000000000000..478d63b4a5ce --- /dev/null +++ b/products/llm_analytics/backend/api/otel/transformer.py @@ -0,0 +1,276 @@ +""" +Core OTel span to PostHog AI event transformer. + +Transforms OpenTelemetry spans into PostHog AI events using a waterfall +pattern for attribute extraction: +1. PostHog native attributes (highest priority) +2. GenAI semantic conventions (fallback) +3. OTel span built-ins (trace_id, span_id, etc.) +""" + +import json +from datetime import UTC, datetime +from typing import Any + +from .conventions.genai import extract_genai_attributes, has_genai_attributes +from .conventions.posthog_native import extract_posthog_native_attributes, has_posthog_attributes + +OTEL_TRANSFORMER_VERSION = "1.0.0" + +# Span status codes (from OpenTelemetry spec) +SPAN_STATUS_UNSET = 0 +SPAN_STATUS_OK = 1 +SPAN_STATUS_ERROR = 2 + + +def transform_span_to_ai_event( + span: dict[str, Any], + resource: dict[str, Any], + scope: dict[str, Any], + baggage: dict[str, str] | None = None, +) -> dict[str, Any]: + """ + Transform a single OTel span to PostHog AI event. + + Args: + span: Parsed OTel span + resource: Resource attributes (service.name, etc.) + scope: Instrumentation scope info + baggage: Baggage context (session_id, etc.) + + Returns: + PostHog AI event dict with: + - event: Event type ($ai_generation, $ai_span, etc.) + - distinct_id: User identifier + - timestamp: ISO 8601 timestamp + - properties: AI event properties + """ + baggage = baggage or {} + + # Extract attributes using waterfall pattern + posthog_attrs = extract_posthog_native_attributes(span) + genai_attrs = extract_genai_attributes(span) + + # Merge with precedence: PostHog > GenAI + merged_attrs = {**genai_attrs, **posthog_attrs} + + # Build AI event properties + properties = build_event_properties(span, merged_attrs, resource, scope, baggage) + + # Determine event type + event_type = determine_event_type(span, merged_attrs) + + # Calculate timestamp + timestamp = calculate_timestamp(span) + + # Get distinct_id + distinct_id = extract_distinct_id(resource, baggage) + + return { + "event": event_type, + "distinct_id": distinct_id, + "timestamp": timestamp, + "properties": properties, + } + + +def build_event_properties( + span: dict[str, Any], + attrs: dict[str, Any], + resource: dict[str, Any], + scope: dict[str, Any], + baggage: dict[str, str], +) -> dict[str, Any]: + """Build PostHog AI event properties from extracted attributes.""" + attributes = span.get("attributes", {}) + status = span.get("status", {}) + + # Core identifiers (prefer extracted, fallback to span built-ins) + trace_id = attrs.get("trace_id") or span.get("trace_id") + span_id = attrs.get("span_id") or span.get("span_id") + parent_id = attrs.get("parent_id") or span.get("parent_span_id") + + # Session ID (prefer extracted, fallback to baggage) + session_id = attrs.get("session_id") or baggage.get("session_id") or baggage.get("posthog.session_id") + + # Calculate latency + latency = calculate_latency(span) + + # Detect error from span status + is_error = attrs.get("is_error") + if is_error is None: + is_error = status.get("code") == SPAN_STATUS_ERROR + error_message = attrs.get("error_message") + if error_message is None and is_error: + error_message = status.get("message") + + # Build base properties + properties: dict[str, Any] = { + # Core IDs + "$ai_trace_id": trace_id, + "$ai_span_id": span_id, + } + + # Optional core IDs + if parent_id: + properties["$ai_parent_id"] = parent_id + if session_id: + properties["$ai_session_id"] = session_id + if attrs.get("generation_id"): + properties["$ai_generation_id"] = attrs["generation_id"] + + # Model info + if attrs.get("model"): + properties["$ai_model"] = attrs["model"] + if attrs.get("provider"): + properties["$ai_provider"] = attrs["provider"] + + # Tokens + if attrs.get("input_tokens") is not None: + properties["$ai_input_tokens"] = attrs["input_tokens"] + if attrs.get("output_tokens") is not None: + properties["$ai_output_tokens"] = attrs["output_tokens"] + if attrs.get("cache_read_tokens") is not None: + properties["$ai_cache_read_tokens"] = attrs["cache_read_tokens"] + if attrs.get("cache_write_tokens") is not None: + properties["$ai_cache_write_tokens"] = attrs["cache_write_tokens"] + + # Cost + if attrs.get("input_cost_usd") is not None: + properties["$ai_input_cost_usd"] = attrs["input_cost_usd"] + if attrs.get("output_cost_usd") is not None: + properties["$ai_output_cost_usd"] = attrs["output_cost_usd"] + if attrs.get("total_cost_usd") is not None: + properties["$ai_total_cost_usd"] = attrs["total_cost_usd"] + + # Timing + if latency is not None: + properties["$ai_latency"] = latency + + # Error + if is_error: + properties["$ai_is_error"] = is_error + if error_message: + properties["$ai_error_message"] = error_message + + # Model parameters + if attrs.get("temperature") is not None: + properties["$ai_temperature"] = attrs["temperature"] + if attrs.get("max_tokens") is not None: + properties["$ai_max_tokens"] = attrs["max_tokens"] + if attrs.get("stream") is not None: + properties["$ai_stream"] = attrs["stream"] + + # Content (handle both direct input/output and prompt/completion) + content_input = attrs.get("input") or attrs.get("prompt") + if content_input: + properties["$ai_input"] = stringify_content(content_input) + + content_output = attrs.get("output") or attrs.get("completion") + if content_output: + properties["$ai_output_choices"] = stringify_content(content_output) + + # Metadata + properties["$ai_otel_transformer_version"] = OTEL_TRANSFORMER_VERSION + properties["$ai_otel_span_kind"] = str(span.get("kind", 0)) + properties["$ai_otel_status_code"] = str(status.get("code", 0)) + + # Resource attributes (service name, etc.) + if resource.get("service.name"): + properties["$ai_service_name"] = resource["service.name"] + + # Instrumentation scope + properties["$ai_instrumentation_scope_name"] = scope.get("name", "unknown") + if scope.get("version"): + properties["$ai_instrumentation_scope_version"] = scope["version"] + + # Add remaining span attributes (not already mapped) + mapped_keys = { + "posthog.ai.model", + "posthog.ai.provider", + "gen_ai.system", + "gen_ai.request.model", + "gen_ai.response.model", + "gen_ai.operation.name", + "gen_ai.usage.input_tokens", + "gen_ai.usage.output_tokens", + "gen_ai.prompt", + "gen_ai.completion", + "service.name", + } + + for key, value in attributes.items(): + if key not in mapped_keys and not key.startswith("posthog.ai.") and not key.startswith("gen_ai."): + # Add unmapped attributes with prefix + properties[f"otel.{key}"] = value + + return properties + + +def determine_event_type(span: dict[str, Any], attrs: dict[str, Any]) -> str: + """Determine AI event type from span.""" + op_name = attrs.get("operation_name", "").lower() + + # Check operation name + if op_name in ("chat", "completion"): + return "$ai_generation" + elif op_name in ("embedding", "embeddings"): + return "$ai_embedding" + + # Check if span is root (no parent) + if not span.get("parent_span_id"): + return "$ai_trace" + + # Default to generic span + return "$ai_span" + + +def calculate_timestamp(span: dict[str, Any]) -> str: + """Calculate timestamp from span start time.""" + start_nanos = int(span.get("start_time_unix_nano", 0)) + millis = start_nanos // 1_000_000 + return datetime.fromtimestamp(millis / 1000, tz=UTC).isoformat() + + +def calculate_latency(span: dict[str, Any]) -> float | None: + """Calculate latency in seconds from span start/end time.""" + end_nanos = span.get("end_time_unix_nano") + if not end_nanos: + return None + + start_nanos = int(span.get("start_time_unix_nano", 0)) + end_nanos = int(end_nanos) + duration_nanos = end_nanos - start_nanos + + # Convert to seconds + return duration_nanos / 1_000_000_000 + + +def extract_distinct_id(resource: dict[str, Any], baggage: dict[str, str]) -> str: + """Extract distinct_id from resource or baggage.""" + # Try resource attributes + user_id = resource.get("user.id") or resource.get("enduser.id") or resource.get("posthog.distinct_id") + + if user_id and isinstance(user_id, str): + return user_id + + # Try baggage + if baggage.get("user_id"): + return baggage["user_id"] + if baggage.get("distinct_id"): + return baggage["distinct_id"] + + # Default to anonymous + return "anonymous" + + +def stringify_content(content: Any) -> str: + """Stringify content (handles objects and strings).""" + if isinstance(content, str): + return content + return json.dumps(content) + + +def span_uses_known_conventions(span: dict[str, Any]) -> bool: + """Check if span uses PostHog or GenAI conventions.""" + return has_posthog_attributes(span) or has_genai_attributes(span) From c9b0e0ff2bc63f3b44b88e16aae99995ac4143c8 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Wed, 12 Nov 2025 14:02:38 +0000 Subject: [PATCH 04/34] feat(llma): Route OTel AI events to PostHog capture pipeline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 3: Send transformed events to capture-rs for ingestion - Implement capture_events() using capture_batch_internal: - Submit events concurrently for better performance - Use team's API token for authentication - Set event_source='otel_traces_ingestion' for observability - Disable person processing (AI events don't need it) - Wait for all futures and log any capture errors - Integrate capture into ingestion endpoint: - Call capture_events() after transformation - Add logging for capture success - Update response message to 'Traces ingested successfully' End-to-end flow complete: 1. Parse OTLP protobuf → 2. Transform to AI events → 3. Capture to PostHog Events are now fully ingested and will appear in PostHog as: - $ai_generation (chat/completion) - $ai_embedding (embeddings) - $ai_span (generic LLM operations) - $ai_trace (root spans) Next: Testing (unit tests, integration tests, e2e with real OTel SDK) --- .../backend/api/otel/ingestion.py | 45 ++++++++++++++++--- 1 file changed, 40 insertions(+), 5 deletions(-) diff --git a/products/llm_analytics/backend/api/otel/ingestion.py b/products/llm_analytics/backend/api/otel/ingestion.py index 3a4135ed59c2..3fca118ca630 100644 --- a/products/llm_analytics/backend/api/otel/ingestion.py +++ b/products/llm_analytics/backend/api/otel/ingestion.py @@ -20,6 +20,7 @@ from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response +from posthog.api.capture import capture_batch_internal from posthog.auth import PersonalAPIKeyAuthentication from posthog.models import Team @@ -157,13 +158,19 @@ def otel_traces_endpoint(request: HttpRequest, project_id: int) -> Response: events_created=len(events), ) - # Route to capture pipeline (TODO: Step 3) - # capture_events(events, team) + # Route to capture pipeline + capture_events(events, team) + + logger.info( + "otel_traces_captured", + team_id=team.id, + events_captured=len(events), + ) return Response( { "status": "success", - "message": "Traces transformed successfully", + "message": "Traces ingested successfully", "spans_received": len(parsed_request["spans"]), "events_created": len(events), }, @@ -315,6 +322,34 @@ def capture_events(events: list[dict[str, Any]], team: Team) -> None: """ Route transformed events to PostHog capture pipeline. - TODO: Use capture_internal or direct Kafka ingestion + Uses capture_batch_internal to submit events to capture-rs. + Events are submitted concurrently for better performance. """ - raise NotImplementedError("Event capture not yet implemented") + if not events: + return + + # Submit events to capture pipeline + futures = capture_batch_internal( + events=events, + event_source="otel_traces_ingestion", + token=team.api_token, + process_person_profile=False, # AI events don't need person processing + ) + + # Wait for all futures to complete and check for errors + errors = [] + for i, future in enumerate(futures): + try: + response = future.result() + if response.status_code not in (200, 201): + errors.append(f"Event {i}: HTTP {response.status_code}") + except Exception as e: + errors.append(f"Event {i}: {str(e)}") + + if errors: + logger.warning( + "otel_traces_capture_errors", + team_id=team.id, + error_count=len(errors), + errors=errors[:10], # Log first 10 errors + ) From 0806fef9155b7035ac700a8171141ebe65dfe8fd Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Wed, 12 Nov 2025 18:10:35 +0000 Subject: [PATCH 05/34] feat(llma): add Django proxy for OTel logs ingestion at /i/v1/logs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds local dev support for OpenTelemetry logs ingestion: - Expose capture-logs service on port 4318 in docker-compose - Add Django proxy view at /i/v1/logs that forwards to capture-logs:4318/v1/logs - Enables OpenAI instrumentation to send message content as OTel logs This allows OpenTelemetry SDKs to send logs to /i/v1/logs in local dev, which matches the production endpoint pattern used by PostHog logs product. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- docker-compose.base.yml | 2 ++ posthog/urls.py | 44 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/docker-compose.base.yml b/docker-compose.base.yml index 5882def3ebfa..465bc08153e6 100644 --- a/docker-compose.base.yml +++ b/docker-compose.base.yml @@ -264,6 +264,8 @@ services: KAFKA_HOSTS: kafka:9092 JWT_SECRET: '' KAFKA_TOPIC: logs_ingestion + ports: + - '4318:4318' networks: - otel_network - default diff --git a/posthog/urls.py b/posthog/urls.py index 338258908ab5..e67dfa7bd5a6 100644 --- a/posthog/urls.py +++ b/posthog/urls.py @@ -146,6 +146,48 @@ def authorize_and_redirect(request: HttpRequest) -> HttpResponse: ) +@csrf_exempt +def proxy_logs_to_capture_service(request: HttpRequest) -> HttpResponse: + """ + Proxy OTLP logs to the capture-logs Rust service. + + This allows OpenTelemetry SDKs to send logs to /i/v1/logs in local dev, + which then forwards to the capture-logs service. + + Note: Using Docker container IP directly since Django runs on host. + """ + import requests + + # Forward to capture-logs service (exposed on host port 4318) + capture_logs_url = "http://localhost:4318/v1/logs" + + try: + # Forward the request with all headers and body + response = requests.post( + capture_logs_url, + data=request.body, + headers={ + "Content-Type": request.META.get("CONTENT_TYPE", ""), + "Authorization": request.META.get("HTTP_AUTHORIZATION", ""), + }, + timeout=10, + ) + + # Return the response from capture-logs + return HttpResponse( + response.content, + status=response.status_code, + content_type=response.headers.get("Content-Type", "application/json"), + ) + except Exception: + logger.exception("Error proxying logs to capture service") + return HttpResponse( + b'{"error": "Failed to forward logs"}', + status=500, + content_type="application/json", + ) + + urlpatterns = [ path("api/schema/", SpectacularAPIView.as_view(), name="schema"), # Optional UI: @@ -171,6 +213,8 @@ def authorize_and_redirect(request: HttpRequest) -> HttpResponse: # api # OpenTelemetry traces ingestion for LLM Analytics path("api/projects//ai/otel/v1/traces", csrf_exempt(otel_traces_endpoint)), + # OpenTelemetry logs proxy to capture-logs service + path("i/v1/logs", proxy_logs_to_capture_service), path("api/environments//progress/", progress), path("api/environments//query//progress/", progress), path("api/environments//query//progress", progress), From 8cca85f8f4a7184c4a9e7101ebcb7db2ae292b34 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Thu, 20 Nov 2025 15:34:50 +0000 Subject: [PATCH 06/34] feat(llm-analytics): implement OTEL logs ingestion endpoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds logs ingestion to complement existing traces ingestion: - Add logs_parser.py for parsing OTLP LogRecord protobuf messages - Add logs_transformer.py to convert logs to PostHog AI events - Add otel_logs_endpoint in ingestion.py with validation - Add URL route at /api/projects/:id/ai/otel/v1/logs - Add OTEL_QUICKSTART.md documentation This enables full OTEL ingestion (traces + logs) for LLM Analytics, allowing users to send data from any OTEL-instrumented framework (Pydantic AI, LangChain, LlamaIndex, etc.) directly to PostHog. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- posthog/urls.py | 6 +- products/llm_analytics/OTEL_QUICKSTART.md | 374 ++++++++++++++++++ .../backend/api/otel/ingestion.py | 248 +++++++++++- .../backend/api/otel/logs_parser.py | 76 ++++ .../backend/api/otel/logs_transformer.py | 229 +++++++++++ 5 files changed, 928 insertions(+), 5 deletions(-) create mode 100644 products/llm_analytics/OTEL_QUICKSTART.md create mode 100644 products/llm_analytics/backend/api/otel/logs_parser.py create mode 100644 products/llm_analytics/backend/api/otel/logs_transformer.py diff --git a/posthog/urls.py b/posthog/urls.py index e67dfa7bd5a6..bab1a59761f1 100644 --- a/posthog/urls.py +++ b/posthog/urls.py @@ -47,7 +47,7 @@ from posthog.temporal.codec_server import decode_payloads from products.early_access_features.backend.api import early_access_features -from products.llm_analytics.backend.api.otel.ingestion import otel_traces_endpoint +from products.llm_analytics.backend.api.otel.ingestion import otel_logs_endpoint, otel_traces_endpoint from .utils import opt_slash_path, render_template from .views import ( @@ -213,7 +213,9 @@ def proxy_logs_to_capture_service(request: HttpRequest) -> HttpResponse: # api # OpenTelemetry traces ingestion for LLM Analytics path("api/projects//ai/otel/v1/traces", csrf_exempt(otel_traces_endpoint)), - # OpenTelemetry logs proxy to capture-logs service + # OpenTelemetry logs ingestion for LLM Analytics + path("api/projects//ai/otel/v1/logs", csrf_exempt(otel_logs_endpoint)), + # OpenTelemetry logs proxy to capture-logs service (legacy) path("i/v1/logs", proxy_logs_to_capture_service), path("api/environments//progress/", progress), path("api/environments//query//progress/", progress), diff --git a/products/llm_analytics/OTEL_QUICKSTART.md b/products/llm_analytics/OTEL_QUICKSTART.md new file mode 100644 index 000000000000..97897274f4d3 --- /dev/null +++ b/products/llm_analytics/OTEL_QUICKSTART.md @@ -0,0 +1,374 @@ +# OpenTelemetry Ingestion for PostHog LLM Analytics - Quickstart + +This guide shows how to configure OpenTelemetry SDKs to send LLM traces and logs to PostHog. + +## Overview + +PostHog LLM Analytics supports OpenTelemetry Protocol (OTLP) ingestion for: + +- **Traces**: Spans from LLM operations (chat, completions, embeddings) +- **Logs**: Log records containing message content (prompts/completions) + +## Endpoints + +### Base URL Pattern + +```text +{posthog_host}/api/projects/{project_id}/ai/otel/v1/ +``` + +### Specific Endpoints + +- **Traces**: `POST /api/projects/{project_id}/ai/otel/v1/traces` +- **Logs**: `POST /api/projects/{project_id}/ai/otel/v1/logs` + +### Authentication + +Use Personal API Key as Bearer token: + +```text +Authorization: Bearer {personal_api_key} +``` + +## Quick Start + +### 1. Get Credentials + +1. Find your Project ID in PostHog UI (Settings → Project) +2. Create a Personal API Key (Settings → Personal API Keys) + +### 2. Configure Environment + +```bash +export POSTHOG_PROJECT_ID=123 +export POSTHOG_PERSONAL_API_KEY=phc_... +export POSTHOG_HOST=https://app.posthog.com # or http://localhost:8000 for local +``` + +### 3. Configure OpenTelemetry SDK + +#### Python (OpenTelemetry SDK) + +```python +from opentelemetry import trace +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk.resources import Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor + +# Create resource +resource = Resource.create({ + "service.name": "my-llm-app", + "service.version": "1.0.0", +}) + +# Create tracer provider +tracer_provider = TracerProvider(resource=resource) + +# Configure OTLP exporter +traces_endpoint = f"{posthog_host}/api/projects/{project_id}/ai/otel/v1/traces" +trace_exporter = OTLPSpanExporter( + endpoint=traces_endpoint, + headers={"Authorization": f"Bearer {personal_api_key}"}, +) + +# Add batch processor +tracer_provider.add_span_processor(BatchSpanProcessor(trace_exporter)) + +# Set as global tracer +trace.set_tracer_provider(tracer_provider) +``` + +#### JavaScript/TypeScript (OpenTelemetry SDK) + +```typescript +import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http'; +import { Resource } from '@opentelemetry/resources'; +import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-base'; +import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node'; + +const provider = new NodeTracerProvider({ + resource: new Resource({ + 'service.name': 'my-llm-app', + 'service.version': '1.0.0', + }), +}); + +const exporter = new OTLPTraceExporter({ + url: `${posthogHost}/api/projects/${projectId}/ai/otel/v1/traces`, + headers: { + 'Authorization': `Bearer ${personalApiKey}`, + }, +}); + +provider.addSpanProcessor(new BatchSpanProcessor(exporter)); +provider.register(); +``` + +### 4. Instrument Your LLM SDK + +#### OpenAI Python + +```python +from opentelemetry.instrumentation.openai import OpenAIInstrumentor + +# Instrument OpenAI SDK +OpenAIInstrumentor().instrument() + +# Now use OpenAI as normal +import openai +response = openai.chat.completions.create( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "Hello!"}] +) +``` + +#### Anthropic (with LangChain) + +```python +from opentelemetry.instrumentation.langchain import LangChainInstrumentor + +# Instrument LangChain +LangChainInstrumentor().instrument() + +# Now use LangChain/Anthropic as normal +from langchain_anthropic import ChatAnthropic +llm = ChatAnthropic(model="claude-3-5-sonnet-20241022") +response = llm.invoke("Hello!") +``` + +## Supported Conventions + +### PostHog Native Attributes (Highest Priority) + +Use these for direct control over AI event properties: + +```python +span.set_attribute("posthog.ai.model", "gpt-4o-mini") +span.set_attribute("posthog.ai.provider", "openai") +span.set_attribute("posthog.ai.input_tokens", 100) +span.set_attribute("posthog.ai.output_tokens", 50) +span.set_attribute("posthog.ai.total_cost_usd", 0.0042) +span.set_attribute("posthog.ai.input", "What is 2+2?") +span.set_attribute("posthog.ai.output", "4") +``` + +### GenAI Semantic Conventions (Fallback) + +Standard OpenTelemetry GenAI conventions: + +```python +span.set_attribute("gen_ai.system", "openai") +span.set_attribute("gen_ai.request.model", "gpt-4o-mini") +span.set_attribute("gen_ai.operation.name", "chat") +span.set_attribute("gen_ai.usage.input_tokens", 100) +span.set_attribute("gen_ai.usage.output_tokens", 50) +span.set_attribute("gen_ai.prompt", "What is 2+2?") +span.set_attribute("gen_ai.completion", "4") +``` + +## Generated AI Events + +PostHog automatically creates these event types: + +- `$ai_generation`: Chat/completion operations +- `$ai_embedding`: Embedding operations +- `$ai_span`: Generic LLM spans +- `$ai_trace`: Root spans (traces) + +### Event Properties + +Events include these properties (when available): + +```python +{ + # Core IDs + "$ai_trace_id": "hex-string", + "$ai_span_id": "hex-string", + "$ai_parent_id": "hex-string", + "$ai_session_id": "session-123", + + # Model info + "$ai_model": "gpt-4o-mini", + "$ai_provider": "openai", + + # Tokens & Cost + "$ai_input_tokens": 100, + "$ai_output_tokens": 50, + "$ai_total_cost_usd": 0.0042, + + # Timing & Error + "$ai_latency": 1.234, + "$ai_is_error": false, + + # Content + "$ai_input": "...", + "$ai_output_choices": "...", + + # Metadata + "$ai_service_name": "my-app", + "$ai_otel_transformer_version": "1.0.0" +} +``` + +## Framework Examples + +### Pydantic AI + +```python +from pydantic_ai import Agent +from pydantic_ai.models.openai import OpenAIModel + +# Setup OpenTelemetry (see above) + +# Instrument OpenAI +from opentelemetry.instrumentation.openai import OpenAIInstrumentor +OpenAIInstrumentor().instrument() + +# Use Pydantic AI as normal +agent = Agent(OpenAIModel("gpt-4o-mini")) +result = await agent.run("Hello!") +``` + +### LangChain + +```python +from langchain_openai import ChatOpenAI + +# Setup OpenTelemetry (see above) + +# Instrument LangChain +from opentelemetry.instrumentation.langchain import LangChainInstrumentor +LangChainInstrumentor().instrument() + +# Use LangChain as normal +llm = ChatOpenAI(model="gpt-4o-mini") +response = llm.invoke("Hello!") +``` + +### LlamaIndex + +```python +from llama_index.core import VectorStoreIndex, SimpleDirectoryReader + +# Setup OpenTelemetry (see above) + +# Instrument OpenAI (used by LlamaIndex) +from opentelemetry.instrumentation.openai import OpenAIInstrumentor +OpenAIInstrumentor().instrument() + +# Use LlamaIndex as normal +documents = SimpleDirectoryReader("data").load_data() +index = VectorStoreIndex.from_documents(documents) +response = index.as_query_engine().query("What is...?") +``` + +## Validation Limits + +The ingestion endpoint enforces these limits: + +- **Traces:** + - Max 1000 spans per request + - Max 128 attributes per span + - Max 128 events per span + - Max 128 links per span + - Max 100KB per attribute value + - Max 1024 characters for span names + +- **Logs:** + - Max 1000 log records per request + - Max 128 attributes per log + - Max 100KB for log body + - Max 100KB per attribute value + +Configure your SDK's batch settings if you hit these limits: + +```python +# Python +processor = BatchSpanProcessor( + exporter, + max_export_batch_size=500, # Lower than 1000 +) +``` + +## Troubleshooting + +### No events appearing in PostHog + +1. **Check authentication:** + - Verify Personal API Key is valid + - Check for 401/403 responses in network logs + +2. **Check endpoint URL:** + - Ensure project ID is correct + - Verify host URL (no trailing slash) + +3. **Check OTLP exporter:** + - Verify protobuf content type is being sent + - Check for connection errors in SDK logs + +### Events missing properties + +1. **Check instrumentation:** + - Ensure SDK instrumentation is installed + - Verify instrumentation is called before SDK usage + +2. **Check conventions:** + - Use PostHog native (`posthog.ai.*`) or GenAI (`gen_ai.*`) attributes + - Verify attribute names are correct + +### High latency or timeouts + +1. **Check batch settings:** + - Reduce batch size if hitting limits + - Increase batch timeout for better throughput + +2. **Check network:** + - Verify PostHog host is reachable + - Check for proxy/firewall issues + +## Development & Testing + +### Local Development + +1. Start PostHog locally: + + ```bash + cd /path/to/posthog + ./bin/start + ``` + +2. Use local endpoint: + + ```text + http://localhost:8000/api/projects/{project_id}/ai/otel/v1/ + ``` + +### Testing with Console Exporter + +Add console exporter for debugging: + +```python +from opentelemetry.sdk.trace.export import ConsoleSpanExporter + +# Add console exporter alongside PostHog exporter +console_exporter = ConsoleSpanExporter() +tracer_provider.add_span_processor(BatchSpanProcessor(console_exporter)) +``` + +This will print spans to stdout for verification. + +## Next Steps + +1. **Verify ingestion:** Check PostHog LLM Analytics UI for events +2. **Add session tracking:** Set `$ai_session_id` for grouping traces +3. **Add custom attributes:** Enrich spans with app-specific data +4. **Monitor costs:** Track `$ai_total_cost_usd` per model/provider +5. **Set up alerts:** Create insights for errors, latency, costs + +## References + +- [OpenTelemetry Python SDK](https://opentelemetry.io/docs/instrumentation/python/) +- [OpenTelemetry JavaScript SDK](https://opentelemetry.io/docs/instrumentation/js/) +- [GenAI Semantic Conventions](https://opentelemetry.io/docs/specs/semconv/gen-ai/) +- [OTLP Specification](https://opentelemetry.io/docs/specs/otlp/) diff --git a/products/llm_analytics/backend/api/otel/ingestion.py b/products/llm_analytics/backend/api/otel/ingestion.py index 3fca118ca630..d9bd39260c17 100644 --- a/products/llm_analytics/backend/api/otel/ingestion.py +++ b/products/llm_analytics/backend/api/otel/ingestion.py @@ -1,9 +1,12 @@ """ -OpenTelemetry traces ingestion API endpoint. +OpenTelemetry traces and logs ingestion API endpoints. -Accepts OTLP/HTTP (protobuf) format traces and converts them to PostHog AI events. +Accepts OTLP/HTTP (protobuf) format traces and logs and converts them to PostHog AI events. + +Endpoints: +- POST /api/projects/:project_id/ai/otel/v1/traces +- POST /api/projects/:project_id/ai/otel/v1/logs -Endpoint: POST /api/projects/:project_id/ai/otel/v1/traces Content-Type: application/x-protobuf Authorization: Bearer """ @@ -24,6 +27,8 @@ from posthog.auth import PersonalAPIKeyAuthentication from posthog.models import Team +from .logs_parser import parse_otlp_logs_request +from .logs_transformer import transform_log_to_ai_event from .parser import parse_baggage_header, parse_otlp_request from .transformer import transform_span_to_ai_event @@ -37,6 +42,9 @@ "MAX_LINKS_PER_SPAN": 128, "MAX_ATTRIBUTE_VALUE_LENGTH": 100_000, # 100KB "MAX_SPAN_NAME_LENGTH": 1024, + "MAX_LOGS_PER_REQUEST": 1000, + "MAX_ATTRIBUTES_PER_LOG": 128, + "MAX_LOG_BODY_LENGTH": 100_000, # 100KB } @@ -353,3 +361,237 @@ def capture_events(events: list[dict[str, Any]], team: Team) -> None: error_count=len(errors), errors=errors[:10], # Log first 10 errors ) + + +@extend_schema( + description=""" + OpenTelemetry logs ingestion endpoint for LLM Analytics. + + Accepts OTLP/HTTP (protobuf) format logs following the OpenTelemetry Protocol specification. + Converts OTel log records to PostHog AI events. Logs from GenAI instrumentation typically + contain message content (prompts/completions) in the body field. + + Supported conventions: + - GenAI semantic conventions: gen_ai.* attributes + - Generic OTel log attributes + + Example OTel SDK configuration: + ```python + from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter + + exporter = OTLPLogExporter( + endpoint="https://app.posthog.com/api/projects/{project_id}/ai/otel/v1/logs", + headers={"Authorization": "Bearer phc_your_api_key"} + ) + ``` + + Rate limits and quotas apply as per normal PostHog event ingestion. + """, + request={"application/x-protobuf": bytes}, + responses={ + 200: {"description": "Logs accepted for processing"}, + 400: {"description": "Invalid OTLP format or validation errors"}, + 401: {"description": "Authentication failed"}, + 413: {"description": "Request too large (exceeds log/attribute limits)"}, + }, +) +@api_view(["POST"]) +@authentication_classes([PersonalAPIKeyAuthentication]) +@permission_classes([IsAuthenticated]) +def otel_logs_endpoint(request: HttpRequest, project_id: int) -> Response: + """ + Process OTLP logs export requests. + + This endpoint: + 1. Validates authentication and project access + 2. Parses OTLP protobuf payload + 3. Validates against size/log limits + 4. Transforms OTel log records to PostHog AI events + 5. Routes events to capture pipeline for ingestion + """ + + # Verify team access + try: + team = Team.objects.get(id=project_id, organization=request.user.current_organization) + except Team.DoesNotExist: + return Response( + {"error": "Project not found or access denied"}, + status=status.HTTP_404_NOT_FOUND, + ) + + # Check content type + content_type = request.content_type or "" + if "protobuf" not in content_type and "octet-stream" not in content_type: + return Response( + { + "error": f"Invalid content type: {content_type}. Expected application/x-protobuf or application/octet-stream" + }, + status=status.HTTP_400_BAD_REQUEST, + ) + + # Get raw protobuf body + protobuf_data = request.body + + if not protobuf_data: + return Response( + {"error": "Empty request body"}, + status=status.HTTP_400_BAD_REQUEST, + ) + + logger.info( + "otel_logs_received", + team_id=team.id, + content_length=len(protobuf_data), + content_type=content_type, + ) + + try: + # Parse OTLP protobuf + parsed_request = parse_otlp_logs_request(protobuf_data) + + logger.info( + "otel_logs_parsed", + team_id=team.id, + logs_count=len(parsed_request["logs"]), + ) + + # Validate request + validation_errors = validate_otlp_logs_request(parsed_request) + if validation_errors: + logger.warning( + "otel_logs_validation_failed", + team_id=team.id, + errors=validation_errors, + ) + return Response( + {"error": "Validation failed", "details": validation_errors}, + status=status.HTTP_400_BAD_REQUEST, + ) + + # Transform logs to AI events + events = transform_logs_to_ai_events(parsed_request) + + logger.info( + "otel_logs_transformed", + team_id=team.id, + events_created=len(events), + ) + + # Route to capture pipeline + capture_events(events, team) + + logger.info( + "otel_logs_captured", + team_id=team.id, + events_captured=len(events), + ) + + return Response( + { + "status": "success", + "message": "Logs ingested successfully", + "logs_received": len(parsed_request["logs"]), + "events_created": len(events), + }, + status=status.HTTP_200_OK, + ) + + except ValidationError as e: + logger.warning( + "otel_logs_validation_error", + team_id=team.id, + error=str(e), + ) + return Response( + {"error": str(e)}, + status=status.HTTP_400_BAD_REQUEST, + ) + + except Exception as e: + logger.error( + "otel_logs_processing_error", + team_id=team.id, + error=str(e), + exc_info=True, + ) + return Response( + {"error": "Internal server error processing logs"}, + status=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + + +def validate_otlp_logs_request(parsed_request: dict[str, Any]) -> list[dict[str, Any]]: + """ + Validate OTLP logs request against limits. + + Returns list of validation errors (empty if valid). + """ + errors = [] + logs = parsed_request.get("logs", []) + + # Check log count + if len(logs) > OTEL_LIMITS["MAX_LOGS_PER_REQUEST"]: + errors.append( + { + "field": "request.logs", + "value": len(logs), + "limit": OTEL_LIMITS["MAX_LOGS_PER_REQUEST"], + "message": f"Request contains {len(logs)} logs, maximum is {OTEL_LIMITS['MAX_LOGS_PER_REQUEST']}. Configure batch size in your OTel SDK.", + } + ) + + # Validate each log record + for i, log_record in enumerate(logs): + # Check attribute count + attributes = log_record.get("attributes", {}) + if len(attributes) > OTEL_LIMITS["MAX_ATTRIBUTES_PER_LOG"]: + errors.append( + { + "field": f"log[{i}].attributes", + "value": len(attributes), + "limit": OTEL_LIMITS["MAX_ATTRIBUTES_PER_LOG"], + "message": f"Log has {len(attributes)} attributes, maximum is {OTEL_LIMITS['MAX_ATTRIBUTES_PER_LOG']}.", + } + ) + + # Check body size + body = log_record.get("body") + if body and isinstance(body, str) and len(body) > OTEL_LIMITS["MAX_LOG_BODY_LENGTH"]: + errors.append( + { + "field": f"log[{i}].body", + "value": len(body), + "limit": OTEL_LIMITS["MAX_LOG_BODY_LENGTH"], + "message": f"Log body exceeds {OTEL_LIMITS['MAX_LOG_BODY_LENGTH']} bytes ({len(body)} bytes). Consider reducing payload size.", + } + ) + + # Check attribute value sizes + for key, value in attributes.items(): + if isinstance(value, str) and len(value) > OTEL_LIMITS["MAX_ATTRIBUTE_VALUE_LENGTH"]: + errors.append( + { + "field": f"log[{i}].attributes.{key}", + "value": len(value), + "limit": OTEL_LIMITS["MAX_ATTRIBUTE_VALUE_LENGTH"], + "message": f"Attribute '{key}' exceeds {OTEL_LIMITS['MAX_ATTRIBUTE_VALUE_LENGTH']} bytes ({len(value)} bytes).", + } + ) + + return errors + + +def transform_logs_to_ai_events(parsed_request: dict[str, Any]) -> list[dict[str, Any]]: + """ + Transform OTel log records to PostHog AI events. + """ + logs = parsed_request.get("logs", []) + resource = parsed_request.get("resource", {}) + scope = parsed_request.get("scope", {}) + + events = [] + for log_record in logs: + event = transform_log_to_ai_event(log_record, resource, scope) + events.append(event) + + return events diff --git a/products/llm_analytics/backend/api/otel/logs_parser.py b/products/llm_analytics/backend/api/otel/logs_parser.py new file mode 100644 index 000000000000..31a1d2a1d323 --- /dev/null +++ b/products/llm_analytics/backend/api/otel/logs_parser.py @@ -0,0 +1,76 @@ +""" +OTLP protobuf parser for OpenTelemetry logs. + +Parses ExportLogsServiceRequest protobuf messages and extracts log records, +resource attributes, and instrumentation scope information. +""" + +from typing import Any + +from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ExportLogsServiceRequest +from opentelemetry.proto.logs.v1.logs_pb2 import LogRecord + +from .parser import parse_any_value, parse_attributes + + +def parse_otlp_logs_request(protobuf_data: bytes) -> dict[str, Any]: + """ + Parse OTLP ExportLogsServiceRequest from protobuf bytes. + + Returns a dict with: + - logs: list of parsed log record dicts + - resource: dict of resource attributes + - scope: dict of instrumentation scope info + """ + request = ExportLogsServiceRequest() + request.ParseFromString(protobuf_data) + + parsed_logs = [] + resource_attrs = {} + scope_info = {} + + # OTLP structure: resource_logs -> scope_logs -> log_records + for resource_logs in request.resource_logs: + # Extract resource attributes (service.name, etc.) + if resource_logs.HasField("resource"): + resource_attrs = parse_attributes(resource_logs.resource.attributes) + + # Iterate through scope logs + for scope_logs in resource_logs.scope_logs: + # Extract instrumentation scope + if scope_logs.HasField("scope"): + scope_info = { + "name": scope_logs.scope.name, + "version": scope_logs.scope.version if scope_logs.scope.version else None, + "attributes": parse_attributes(scope_logs.scope.attributes) if scope_logs.scope.attributes else {}, + } + + # Parse each log record + for log_record in scope_logs.log_records: + parsed_log = parse_log_record(log_record) + parsed_logs.append(parsed_log) + + return { + "logs": parsed_logs, + "resource": resource_attrs, + "scope": scope_info, + } + + +def parse_log_record(log_record: LogRecord) -> dict[str, Any]: + """ + Parse a single OTLP log record into a dict. + """ + return { + "time_unix_nano": str(log_record.time_unix_nano), + "observed_time_unix_nano": str(log_record.observed_time_unix_nano) + if log_record.observed_time_unix_nano + else None, + "severity_number": log_record.severity_number, + "severity_text": log_record.severity_text if log_record.severity_text else None, + "body": parse_any_value(log_record.body) if log_record.HasField("body") else None, + "attributes": parse_attributes(log_record.attributes), + "trace_id": log_record.trace_id.hex() if log_record.trace_id else None, + "span_id": log_record.span_id.hex() if log_record.span_id else None, + "flags": log_record.flags if log_record.flags else None, + } diff --git a/products/llm_analytics/backend/api/otel/logs_transformer.py b/products/llm_analytics/backend/api/otel/logs_transformer.py new file mode 100644 index 000000000000..0141683fd249 --- /dev/null +++ b/products/llm_analytics/backend/api/otel/logs_transformer.py @@ -0,0 +1,229 @@ +""" +Core OTel log record to PostHog AI event transformer. + +Transforms OpenTelemetry log records into PostHog AI events. +Log records from GenAI instrumentation typically contain message content +(prompts/completions) in the body field. +""" + +import json +from datetime import UTC, datetime +from typing import Any + +OTEL_TRANSFORMER_VERSION = "1.0.0" + + +def transform_log_to_ai_event( + log_record: dict[str, Any], + resource: dict[str, Any], + scope: dict[str, Any], +) -> dict[str, Any]: + """ + Transform a single OTel log record to PostHog AI event. + + Args: + log_record: Parsed OTel log record + resource: Resource attributes (service.name, etc.) + scope: Instrumentation scope info + + Returns: + PostHog AI event dict with: + - event: Event type ($ai_generation, $ai_span, etc.) + - distinct_id: User identifier + - timestamp: ISO 8601 timestamp + - properties: AI event properties + """ + attributes = log_record.get("attributes", {}) + + # Build AI event properties + properties = build_event_properties(log_record, attributes, resource, scope) + + # Determine event type + event_type = determine_event_type(log_record, attributes) + + # Calculate timestamp + timestamp = calculate_timestamp(log_record) + + # Get distinct_id + distinct_id = extract_distinct_id(resource, attributes) + + return { + "event": event_type, + "distinct_id": distinct_id, + "timestamp": timestamp, + "properties": properties, + } + + +def build_event_properties( + log_record: dict[str, Any], + attributes: dict[str, Any], + resource: dict[str, Any], + scope: dict[str, Any], +) -> dict[str, Any]: + """Build PostHog AI event properties from log record.""" + + # Core identifiers (from log record) + trace_id = log_record.get("trace_id") + span_id = log_record.get("span_id") + + # Session ID (from attributes or resource) + session_id = attributes.get("session_id") or resource.get("session.id") + + # Extract message content from body + body = log_record.get("body") + message_content = stringify_content(body) if body else None + + # Build base properties + properties: dict[str, Any] = {} + + # Core IDs + if trace_id: + properties["$ai_trace_id"] = trace_id + if span_id: + properties["$ai_span_id"] = span_id + if session_id: + properties["$ai_session_id"] = session_id + + # Model info (from attributes) + if attributes.get("gen_ai.system"): + properties["$ai_provider"] = attributes["gen_ai.system"] + elif attributes.get("model.provider"): + properties["$ai_provider"] = attributes["model.provider"] + + if attributes.get("gen_ai.request.model"): + properties["$ai_model"] = attributes["gen_ai.request.model"] + elif attributes.get("gen_ai.response.model"): + properties["$ai_model"] = attributes["gen_ai.response.model"] + elif attributes.get("model.name"): + properties["$ai_model"] = attributes["model.name"] + + # Tokens (from attributes) + if attributes.get("gen_ai.usage.input_tokens") is not None: + properties["$ai_input_tokens"] = attributes["gen_ai.usage.input_tokens"] + if attributes.get("gen_ai.usage.output_tokens") is not None: + properties["$ai_output_tokens"] = attributes["gen_ai.usage.output_tokens"] + + # Message content + # Check for specific GenAI log attributes for prompts/completions + if attributes.get("gen_ai.prompt"): + properties["$ai_input"] = stringify_content(attributes["gen_ai.prompt"]) + elif attributes.get("message.content"): + # Some instrumentation sends content in attributes + properties["$ai_input"] = stringify_content(attributes["message.content"]) + elif message_content and attributes.get("event.name") == "gen_ai.content.prompt": + # If event name indicates this is a prompt + properties["$ai_input"] = message_content + + if attributes.get("gen_ai.completion"): + properties["$ai_output_choices"] = stringify_content(attributes["gen_ai.completion"]) + elif message_content and attributes.get("event.name") == "gen_ai.content.completion": + # If event name indicates this is a completion + properties["$ai_output_choices"] = message_content + + # If we have message content but haven't categorized it, store it generically + if message_content and "$ai_input" not in properties and "$ai_output_choices" not in properties: + properties["$ai_message"] = message_content + + # Severity + if log_record.get("severity_number"): + properties["$ai_log_severity_number"] = log_record["severity_number"] + if log_record.get("severity_text"): + properties["$ai_log_severity_text"] = log_record["severity_text"] + + # Metadata + properties["$ai_otel_transformer_version"] = OTEL_TRANSFORMER_VERSION + properties["$ai_otel_log_source"] = "logs" + + # Resource attributes (service name, etc.) + if resource.get("service.name"): + properties["$ai_service_name"] = resource["service.name"] + + # Instrumentation scope + properties["$ai_instrumentation_scope_name"] = scope.get("name", "unknown") + if scope.get("version"): + properties["$ai_instrumentation_scope_version"] = scope["version"] + + # Add remaining log attributes (not already mapped) + mapped_keys = { + "gen_ai.system", + "gen_ai.request.model", + "gen_ai.response.model", + "gen_ai.usage.input_tokens", + "gen_ai.usage.output_tokens", + "gen_ai.prompt", + "gen_ai.completion", + "model.provider", + "model.name", + "message.content", + "session_id", + "session.id", + "event.name", + "service.name", + } + + for key, value in attributes.items(): + if key not in mapped_keys and not key.startswith("gen_ai."): + # Add unmapped attributes with prefix + properties[f"otel.{key}"] = value + + return properties + + +def determine_event_type(log_record: dict[str, Any], attributes: dict[str, Any]) -> str: + """Determine AI event type from log record.""" + event_name = attributes.get("event.name", "").lower() + + # Check event name for GenAI events + if "prompt" in event_name or "input" in event_name: + return "$ai_generation" + elif "completion" in event_name or "output" in event_name or "response" in event_name: + return "$ai_generation" + elif "embedding" in event_name: + return "$ai_embedding" + + # Check operation name + op_name = attributes.get("gen_ai.operation.name", "").lower() + if op_name in ("chat", "completion"): + return "$ai_generation" + elif op_name in ("embedding", "embeddings"): + return "$ai_embedding" + + # Default to generic span + return "$ai_span" + + +def calculate_timestamp(log_record: dict[str, Any]) -> str: + """Calculate timestamp from log record time.""" + time_nanos = int(log_record.get("time_unix_nano", 0)) + if time_nanos == 0: + # Fallback to observed time if time is not set + time_nanos = int(log_record.get("observed_time_unix_nano", 0)) + + millis = time_nanos // 1_000_000 + return datetime.fromtimestamp(millis / 1000, tz=UTC).isoformat() + + +def extract_distinct_id(resource: dict[str, Any], attributes: dict[str, Any]) -> str: + """Extract distinct_id from resource or attributes.""" + # Try resource attributes + user_id = resource.get("user.id") or resource.get("enduser.id") or resource.get("posthog.distinct_id") + + if user_id and isinstance(user_id, str): + return user_id + + # Try log attributes + if attributes.get("user_id"): + return str(attributes["user_id"]) + if attributes.get("distinct_id"): + return str(attributes["distinct_id"]) + + # Default to anonymous + return "anonymous" + + +def stringify_content(content: Any) -> str: + """Stringify content (handles objects and strings).""" + if isinstance(content, str): + return content + return json.dumps(content) From 1076cfa83b1174ead06d83246dd21b300b9977ef Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Thu, 20 Nov 2025 16:25:36 +0000 Subject: [PATCH 07/34] fix(llma): Remove conflicting opentelemetry-proto version pin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The explicit opentelemetry-proto==1.38.0 dependency conflicts with opentelemetry-exporter-otlp-proto-grpc==1.33.1 (already in master), which requires proto==1.33.1. Removing the explicit pin lets the proto package be resolved as a transitive dependency at 1.33.1, which is compatible with our OTLP parser implementation. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 0669c98edca5..a8422c2f7844 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,7 +83,6 @@ dependencies = [ "nh3==0.2.14", "numpy~=2.1.0", "openai==1.109.1", - "opentelemetry-proto==1.38.0", "openpyxl==3.1.2", "orjson==3.10.15", "pandas~=2.2.2", From eaf55ef3631aed1ef50939487e33d6cbe2c5cffd Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Thu, 20 Nov 2025 18:01:49 +0000 Subject: [PATCH 08/34] feat(llm-analytics): Improve OTEL ingestion authentication and data structure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Changes authentication from personal API keys to project tokens and improves data structure quality to match SDK-based traces. ## Authentication Changes - Replace PersonalAPIKeyAuthentication with ProjectTokenAuthentication - Use project tokens (phc_...) instead of personal API keys (phx_...) - Support token in Authorization header or ?token= query parameter - Follows same pattern as logs ingestion for consistency ## Data Structure Improvements - Keep structured data as native JSON (arrays/objects) instead of JSON strings - Add $ai_span_name extraction from span.name field - Add $ai_tools extraction from llm.request.functions.* attributes - Parse tool definitions into structured array format matching SDK ## GenAI Conventions - Add llm.request.functions.* attribute mapping for tool definitions - Improve attribute extraction to handle nested tool parameters ## Technical Details These changes ensure OTEL-captured traces have the same data quality as SDK-captured traces: - $ai_input: Structured array (not JSON string) - $ai_output_choices: Structured array (not JSON string) - $ai_tools: Structured array with proper schema (name, description, input_schema) - $ai_span_name: Captured from span name This makes OTEL ingestion a first-class alternative to SDK instrumentation with identical data structure and UI rendering. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .../backend/api/otel/conventions/genai.py | 58 +++++++- .../backend/api/otel/ingestion.py | 138 +++++++++++++++--- .../backend/api/otel/transformer.py | 94 +++++++++++- 3 files changed, 257 insertions(+), 33 deletions(-) diff --git a/products/llm_analytics/backend/api/otel/conventions/genai.py b/products/llm_analytics/backend/api/otel/conventions/genai.py index 4ba8b346b7ac..62248d8f799f 100644 --- a/products/llm_analytics/backend/api/otel/conventions/genai.py +++ b/products/llm_analytics/backend/api/otel/conventions/genai.py @@ -7,6 +7,7 @@ Reference: https://opentelemetry.io/docs/specs/semconv/gen-ai/ """ +from collections import defaultdict from typing import Any @@ -16,6 +17,49 @@ def has_genai_attributes(span: dict[str, Any]) -> bool: return any(key.startswith("gen_ai.") for key in attributes.keys()) +def _extract_indexed_messages(attributes: dict[str, Any], prefix: str) -> list[dict[str, Any]] | None: + """ + Extract indexed message attributes like gen_ai.prompt.{N}.{field} into a list of message dicts. + + Args: + attributes: Span attributes dictionary + prefix: Message prefix (e.g., "gen_ai.prompt" or "gen_ai.completion") + + Returns: + List of message dicts with role, content, etc., or None if no messages found + """ + # Group attributes by index + messages_by_index: dict[int, dict[str, Any]] = defaultdict(dict) + + for key, value in attributes.items(): + if not key.startswith(f"{prefix}."): + continue + + # Parse: gen_ai.prompt.0.role -> index=0, field=role + parts = key[len(prefix) + 1 :].split(".", 1) + if len(parts) != 2: + continue + + try: + index = int(parts[0]) + field = parts[1] + messages_by_index[index][field] = value + except (ValueError, IndexError): + continue + + if not messages_by_index: + return None + + # Convert to sorted list of messages + messages = [] + for index in sorted(messages_by_index.keys()): + msg = messages_by_index[index] + if msg: # Only include non-empty messages + messages.append(msg) + + return messages if messages else None + + def extract_genai_attributes(span: dict[str, Any]) -> dict[str, Any]: """ Extract GenAI semantic convention attributes from span. @@ -50,9 +94,19 @@ def extract_genai_attributes(span: dict[str, Any]) -> dict[str, Any]: result["output_tokens"] = output_tokens # Content (prompt and completion) - if (prompt := attributes.get("gen_ai.prompt")) is not None: + # Try indexed messages first (gen_ai.prompt.0.role, gen_ai.prompt.0.content, etc.) + prompts = _extract_indexed_messages(attributes, "gen_ai.prompt") + if prompts: + result["prompt"] = prompts + # Fallback to direct gen_ai.prompt attribute + elif (prompt := attributes.get("gen_ai.prompt")) is not None: result["prompt"] = prompt - if (completion := attributes.get("gen_ai.completion")) is not None: + + completions = _extract_indexed_messages(attributes, "gen_ai.completion") + if completions: + result["completion"] = completions + # Fallback to direct gen_ai.completion attribute + elif (completion := attributes.get("gen_ai.completion")) is not None: result["completion"] = completion # Model parameters diff --git a/products/llm_analytics/backend/api/otel/ingestion.py b/products/llm_analytics/backend/api/otel/ingestion.py index d9bd39260c17..4c8b94387dfa 100644 --- a/products/llm_analytics/backend/api/otel/ingestion.py +++ b/products/llm_analytics/backend/api/otel/ingestion.py @@ -8,23 +8,26 @@ - POST /api/projects/:project_id/ai/otel/v1/logs Content-Type: application/x-protobuf -Authorization: Bearer +Authorization: Bearer + +Authentication uses project API token (phc_...), NOT personal API key. +Token can be provided via Authorization header or ?token= query parameter. """ -from typing import Any +import re +from typing import Any, Optional, Union from django.http import HttpRequest import structlog from drf_spectacular.utils import extend_schema -from rest_framework import status +from rest_framework import authentication, status from rest_framework.decorators import api_view, authentication_classes, permission_classes -from rest_framework.exceptions import ValidationError -from rest_framework.permissions import IsAuthenticated +from rest_framework.exceptions import AuthenticationFailed, ValidationError +from rest_framework.request import Request from rest_framework.response import Response from posthog.api.capture import capture_batch_internal -from posthog.auth import PersonalAPIKeyAuthentication from posthog.models import Team from .logs_parser import parse_otlp_logs_request @@ -48,6 +51,65 @@ } +class ProjectTokenAuthentication(authentication.BaseAuthentication): + """ + Authenticates using a project API token (phc_...). + + This is used for ingestion endpoints where a public project token is used + instead of a personal API key. Supports token in: + 1. Authorization header: Bearer + 2. Query parameter: ?token= + + Similar to logs ingestion pattern. + """ + + keyword = "Bearer" + + @classmethod + def find_token( + cls, + request: Union[HttpRequest, Request], + ) -> Optional[str]: + """Try to find project token in request and return it.""" + # Try Authorization header first + if "HTTP_AUTHORIZATION" in request.META: + authorization_match = re.match(rf"^{cls.keyword}\s+(\S.+)$", request.META["HTTP_AUTHORIZATION"]) + if authorization_match: + token = authorization_match.group(1).strip() + # Only accept project tokens (phc_...), not personal keys + if token.startswith("phc_"): + return token + return None + + # Try query parameter + if "token" in request.GET: + token = request.GET["token"] + if token.startswith("phc_"): + return token + + return None + + def authenticate(self, request: Union[HttpRequest, Request]) -> Optional[tuple[Any, Team]]: + token = self.find_token(request) + + if not token: + return None + + # Get the team from the project token + team = Team.objects.get_team_from_cache_or_token(token) + + if team is None: + raise AuthenticationFailed(detail="Invalid project token.") + + # Return team as the "user" for this authentication + # The team itself acts as the authenticated entity + return (team, token) + + @classmethod + def authenticate_header(cls, request) -> str: + return cls.keyword + + @extend_schema( description=""" OpenTelemetry traces ingestion endpoint for LLM Analytics. @@ -65,10 +127,14 @@ exporter = OTLPSpanExporter( endpoint="https://app.posthog.com/api/projects/{project_id}/ai/otel/v1/traces", - headers={"Authorization": "Bearer phc_your_api_key"} + headers={"Authorization": "Bearer phc_your_project_token"} ) ``` + Authentication: + - Use your project API token (starts with phc_...), NOT a personal API key + - Token can be provided via Authorization header (Bearer token) or ?token= query parameter + Rate limits and quotas apply as per normal PostHog event ingestion. """, request={"application/x-protobuf": bytes}, @@ -80,8 +146,8 @@ }, ) @api_view(["POST"]) -@authentication_classes([PersonalAPIKeyAuthentication]) -@permission_classes([IsAuthenticated]) +@authentication_classes([ProjectTokenAuthentication]) +@permission_classes([]) def otel_traces_endpoint(request: HttpRequest, project_id: int) -> Response: """ Process OTLP trace export requests. @@ -94,13 +160,23 @@ def otel_traces_endpoint(request: HttpRequest, project_id: int) -> Response: 5. Routes events to capture pipeline for ingestion """ - # Verify team access - try: - team = Team.objects.get(id=project_id, organization=request.user.current_organization) - except Team.DoesNotExist: + # Get authenticated team from request + # ProjectTokenAuthentication returns (team, token) tuple + if not hasattr(request, "user") or not isinstance(request.user, Team): + return Response( + { + "error": "Invalid authentication. Use project token (phc_...) in Authorization header or ?token= parameter." + }, + status=status.HTTP_401_UNAUTHORIZED, + ) + + team = request.user + + # Verify the team ID matches the project_id in URL + if team.id != project_id: return Response( - {"error": "Project not found or access denied"}, - status=status.HTTP_404_NOT_FOUND, + {"error": "Project ID in URL does not match authenticated project token"}, + status=status.HTTP_403_FORBIDDEN, ) # Check content type @@ -381,10 +457,14 @@ def capture_events(events: list[dict[str, Any]], team: Team) -> None: exporter = OTLPLogExporter( endpoint="https://app.posthog.com/api/projects/{project_id}/ai/otel/v1/logs", - headers={"Authorization": "Bearer phc_your_api_key"} + headers={"Authorization": "Bearer phc_your_project_token"} ) ``` + Authentication: + - Use your project API token (starts with phc_...), NOT a personal API key + - Token can be provided via Authorization header (Bearer token) or ?token= query parameter + Rate limits and quotas apply as per normal PostHog event ingestion. """, request={"application/x-protobuf": bytes}, @@ -396,8 +476,8 @@ def capture_events(events: list[dict[str, Any]], team: Team) -> None: }, ) @api_view(["POST"]) -@authentication_classes([PersonalAPIKeyAuthentication]) -@permission_classes([IsAuthenticated]) +@authentication_classes([ProjectTokenAuthentication]) +@permission_classes([]) def otel_logs_endpoint(request: HttpRequest, project_id: int) -> Response: """ Process OTLP logs export requests. @@ -410,13 +490,23 @@ def otel_logs_endpoint(request: HttpRequest, project_id: int) -> Response: 5. Routes events to capture pipeline for ingestion """ - # Verify team access - try: - team = Team.objects.get(id=project_id, organization=request.user.current_organization) - except Team.DoesNotExist: + # Get authenticated team from request + # ProjectTokenAuthentication returns (team, token) tuple + if not hasattr(request, "user") or not isinstance(request.user, Team): + return Response( + { + "error": "Invalid authentication. Use project token (phc_...) in Authorization header or ?token= parameter." + }, + status=status.HTTP_401_UNAUTHORIZED, + ) + + team = request.user + + # Verify the team ID matches the project_id in URL + if team.id != project_id: return Response( - {"error": "Project not found or access denied"}, - status=status.HTTP_404_NOT_FOUND, + {"error": "Project ID in URL does not match authenticated project token"}, + status=status.HTTP_403_FORBIDDEN, ) # Check content type diff --git a/products/llm_analytics/backend/api/otel/transformer.py b/products/llm_analytics/backend/api/otel/transformer.py index 478d63b4a5ce..c2c919b892fe 100644 --- a/products/llm_analytics/backend/api/otel/transformer.py +++ b/products/llm_analytics/backend/api/otel/transformer.py @@ -170,6 +170,10 @@ def build_event_properties( if content_output: properties["$ai_output_choices"] = stringify_content(content_output) + # Span name + if span.get("name"): + properties["$ai_span_name"] = span["name"] + # Metadata properties["$ai_otel_transformer_version"] = OTEL_TRANSFORMER_VERSION properties["$ai_otel_span_kind"] = str(span.get("kind", 0)) @@ -184,6 +188,11 @@ def build_event_properties( if scope.get("version"): properties["$ai_instrumentation_scope_version"] = scope["version"] + # Extract tool definitions from llm.request.functions.* attributes + tools = _extract_tools_from_attributes(attributes) + if tools: + properties["$ai_tools"] = tools + # Add remaining span attributes (not already mapped) mapped_keys = { "posthog.ai.model", @@ -201,22 +210,81 @@ def build_event_properties( for key, value in attributes.items(): if key not in mapped_keys and not key.startswith("posthog.ai.") and not key.startswith("gen_ai."): - # Add unmapped attributes with prefix - properties[f"otel.{key}"] = value + # Skip llm.request.functions.* as they're now in $ai_tools + if not key.startswith("llm.request.functions."): + # Add unmapped attributes with prefix + properties[f"otel.{key}"] = value return properties +def _extract_tools_from_attributes(attributes: dict[str, Any]) -> list[dict[str, Any]] | None: + """ + Extract tool definitions from llm.request.functions.* attributes. + + Converts flat attributes like: + llm.request.functions.0.name = "get_weather" + llm.request.functions.0.description = "Get weather" + llm.request.functions.0.parameters = "{...}" + + Into structured array: + [{"name": "get_weather", "description": "Get weather", "input_schema": {...}}] + """ + from collections import defaultdict + + tools_by_index: dict[int, dict[str, Any]] = defaultdict(dict) + + for key, value in attributes.items(): + if not key.startswith("llm.request.functions."): + continue + + # Parse: llm.request.functions.0.name -> index=0, field=name + parts = key[len("llm.request.functions.") :].split(".", 1) + if len(parts) != 2: + continue + + try: + index = int(parts[0]) + field = parts[1] + + # Map OTEL field names to PostHog SDK format + if field == "parameters": + # Parse JSON string to dict for input_schema + try: + tools_by_index[index]["input_schema"] = json.loads(value) if isinstance(value, str) else value + except (json.JSONDecodeError, TypeError): + tools_by_index[index]["input_schema"] = value + else: + tools_by_index[index][field] = value + + except (ValueError, IndexError): + continue + + if not tools_by_index: + return None + + # Convert to sorted list + return [tools_by_index[i] for i in sorted(tools_by_index.keys())] + + def determine_event_type(span: dict[str, Any], attrs: dict[str, Any]) -> str: """Determine AI event type from span.""" op_name = attrs.get("operation_name", "").lower() - # Check operation name + # Check operation name first (highest priority) if op_name in ("chat", "completion"): return "$ai_generation" elif op_name in ("embedding", "embeddings"): return "$ai_embedding" + # Check if span has LLM-specific attributes (provider, model, tokens) + # These indicate it's an actual LLM generation, not just a wrapper span + has_llm_attrs = bool( + attrs.get("provider") and attrs.get("model") and (attrs.get("input_tokens") is not None or attrs.get("prompt")) + ) + if has_llm_attrs: + return "$ai_generation" + # Check if span is root (no parent) if not span.get("parent_span_id"): return "$ai_trace" @@ -264,11 +332,23 @@ def extract_distinct_id(resource: dict[str, Any], baggage: dict[str, str]) -> st return "anonymous" -def stringify_content(content: Any) -> str: - """Stringify content (handles objects and strings).""" - if isinstance(content, str): +def stringify_content(content: Any) -> Any: + """ + Return content in appropriate format for PostHog properties. + + Keep structured data (lists/dicts) as-is for better UI rendering. + Only convert to JSON string if it's already a string (rare case). + """ + if isinstance(content, list | dict): return content - return json.dumps(content) + if isinstance(content, str): + # If it's already a JSON string, parse it to get structured data + try: + parsed = json.loads(content) + return parsed + except (json.JSONDecodeError, TypeError): + return content + return content def span_uses_known_conventions(span: dict[str, Any]) -> bool: From a0fad6c37bc601c25de375e307322d9e7930ad52 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Thu, 20 Nov 2025 18:14:09 +0000 Subject: [PATCH 09/34] Remove unused OTEL logs endpoint Only the traces endpoint is needed for AI instrumentation. Removed: - otel_logs_endpoint function and helpers - logs_parser.py and logs_transformer.py - logs endpoint URL route - log-related OTEL_LIMITS --- posthog/urls.py | 4 +- .../backend/api/otel/ingestion.py | 260 +----------------- .../backend/api/otel/logs_parser.py | 76 ----- .../backend/api/otel/logs_transformer.py | 229 --------------- 4 files changed, 4 insertions(+), 565 deletions(-) delete mode 100644 products/llm_analytics/backend/api/otel/logs_parser.py delete mode 100644 products/llm_analytics/backend/api/otel/logs_transformer.py diff --git a/posthog/urls.py b/posthog/urls.py index bab1a59761f1..3b5da2223bf7 100644 --- a/posthog/urls.py +++ b/posthog/urls.py @@ -47,7 +47,7 @@ from posthog.temporal.codec_server import decode_payloads from products.early_access_features.backend.api import early_access_features -from products.llm_analytics.backend.api.otel.ingestion import otel_logs_endpoint, otel_traces_endpoint +from products.llm_analytics.backend.api.otel.ingestion import otel_traces_endpoint from .utils import opt_slash_path, render_template from .views import ( @@ -213,8 +213,6 @@ def proxy_logs_to_capture_service(request: HttpRequest) -> HttpResponse: # api # OpenTelemetry traces ingestion for LLM Analytics path("api/projects//ai/otel/v1/traces", csrf_exempt(otel_traces_endpoint)), - # OpenTelemetry logs ingestion for LLM Analytics - path("api/projects//ai/otel/v1/logs", csrf_exempt(otel_logs_endpoint)), # OpenTelemetry logs proxy to capture-logs service (legacy) path("i/v1/logs", proxy_logs_to_capture_service), path("api/environments//progress/", progress), diff --git a/products/llm_analytics/backend/api/otel/ingestion.py b/products/llm_analytics/backend/api/otel/ingestion.py index 4c8b94387dfa..a0a5da34c295 100644 --- a/products/llm_analytics/backend/api/otel/ingestion.py +++ b/products/llm_analytics/backend/api/otel/ingestion.py @@ -1,11 +1,10 @@ """ -OpenTelemetry traces and logs ingestion API endpoints. +OpenTelemetry traces ingestion API endpoint. -Accepts OTLP/HTTP (protobuf) format traces and logs and converts them to PostHog AI events. +Accepts OTLP/HTTP (protobuf) format traces and converts them to PostHog AI events. -Endpoints: +Endpoint: - POST /api/projects/:project_id/ai/otel/v1/traces -- POST /api/projects/:project_id/ai/otel/v1/logs Content-Type: application/x-protobuf Authorization: Bearer @@ -30,8 +29,6 @@ from posthog.api.capture import capture_batch_internal from posthog.models import Team -from .logs_parser import parse_otlp_logs_request -from .logs_transformer import transform_log_to_ai_event from .parser import parse_baggage_header, parse_otlp_request from .transformer import transform_span_to_ai_event @@ -45,9 +42,6 @@ "MAX_LINKS_PER_SPAN": 128, "MAX_ATTRIBUTE_VALUE_LENGTH": 100_000, # 100KB "MAX_SPAN_NAME_LENGTH": 1024, - "MAX_LOGS_PER_REQUEST": 1000, - "MAX_ATTRIBUTES_PER_LOG": 128, - "MAX_LOG_BODY_LENGTH": 100_000, # 100KB } @@ -437,251 +431,3 @@ def capture_events(events: list[dict[str, Any]], team: Team) -> None: error_count=len(errors), errors=errors[:10], # Log first 10 errors ) - - -@extend_schema( - description=""" - OpenTelemetry logs ingestion endpoint for LLM Analytics. - - Accepts OTLP/HTTP (protobuf) format logs following the OpenTelemetry Protocol specification. - Converts OTel log records to PostHog AI events. Logs from GenAI instrumentation typically - contain message content (prompts/completions) in the body field. - - Supported conventions: - - GenAI semantic conventions: gen_ai.* attributes - - Generic OTel log attributes - - Example OTel SDK configuration: - ```python - from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter - - exporter = OTLPLogExporter( - endpoint="https://app.posthog.com/api/projects/{project_id}/ai/otel/v1/logs", - headers={"Authorization": "Bearer phc_your_project_token"} - ) - ``` - - Authentication: - - Use your project API token (starts with phc_...), NOT a personal API key - - Token can be provided via Authorization header (Bearer token) or ?token= query parameter - - Rate limits and quotas apply as per normal PostHog event ingestion. - """, - request={"application/x-protobuf": bytes}, - responses={ - 200: {"description": "Logs accepted for processing"}, - 400: {"description": "Invalid OTLP format or validation errors"}, - 401: {"description": "Authentication failed"}, - 413: {"description": "Request too large (exceeds log/attribute limits)"}, - }, -) -@api_view(["POST"]) -@authentication_classes([ProjectTokenAuthentication]) -@permission_classes([]) -def otel_logs_endpoint(request: HttpRequest, project_id: int) -> Response: - """ - Process OTLP logs export requests. - - This endpoint: - 1. Validates authentication and project access - 2. Parses OTLP protobuf payload - 3. Validates against size/log limits - 4. Transforms OTel log records to PostHog AI events - 5. Routes events to capture pipeline for ingestion - """ - - # Get authenticated team from request - # ProjectTokenAuthentication returns (team, token) tuple - if not hasattr(request, "user") or not isinstance(request.user, Team): - return Response( - { - "error": "Invalid authentication. Use project token (phc_...) in Authorization header or ?token= parameter." - }, - status=status.HTTP_401_UNAUTHORIZED, - ) - - team = request.user - - # Verify the team ID matches the project_id in URL - if team.id != project_id: - return Response( - {"error": "Project ID in URL does not match authenticated project token"}, - status=status.HTTP_403_FORBIDDEN, - ) - - # Check content type - content_type = request.content_type or "" - if "protobuf" not in content_type and "octet-stream" not in content_type: - return Response( - { - "error": f"Invalid content type: {content_type}. Expected application/x-protobuf or application/octet-stream" - }, - status=status.HTTP_400_BAD_REQUEST, - ) - - # Get raw protobuf body - protobuf_data = request.body - - if not protobuf_data: - return Response( - {"error": "Empty request body"}, - status=status.HTTP_400_BAD_REQUEST, - ) - - logger.info( - "otel_logs_received", - team_id=team.id, - content_length=len(protobuf_data), - content_type=content_type, - ) - - try: - # Parse OTLP protobuf - parsed_request = parse_otlp_logs_request(protobuf_data) - - logger.info( - "otel_logs_parsed", - team_id=team.id, - logs_count=len(parsed_request["logs"]), - ) - - # Validate request - validation_errors = validate_otlp_logs_request(parsed_request) - if validation_errors: - logger.warning( - "otel_logs_validation_failed", - team_id=team.id, - errors=validation_errors, - ) - return Response( - {"error": "Validation failed", "details": validation_errors}, - status=status.HTTP_400_BAD_REQUEST, - ) - - # Transform logs to AI events - events = transform_logs_to_ai_events(parsed_request) - - logger.info( - "otel_logs_transformed", - team_id=team.id, - events_created=len(events), - ) - - # Route to capture pipeline - capture_events(events, team) - - logger.info( - "otel_logs_captured", - team_id=team.id, - events_captured=len(events), - ) - - return Response( - { - "status": "success", - "message": "Logs ingested successfully", - "logs_received": len(parsed_request["logs"]), - "events_created": len(events), - }, - status=status.HTTP_200_OK, - ) - - except ValidationError as e: - logger.warning( - "otel_logs_validation_error", - team_id=team.id, - error=str(e), - ) - return Response( - {"error": str(e)}, - status=status.HTTP_400_BAD_REQUEST, - ) - - except Exception as e: - logger.error( - "otel_logs_processing_error", - team_id=team.id, - error=str(e), - exc_info=True, - ) - return Response( - {"error": "Internal server error processing logs"}, - status=status.HTTP_500_INTERNAL_SERVER_ERROR, - ) - - -def validate_otlp_logs_request(parsed_request: dict[str, Any]) -> list[dict[str, Any]]: - """ - Validate OTLP logs request against limits. - - Returns list of validation errors (empty if valid). - """ - errors = [] - logs = parsed_request.get("logs", []) - - # Check log count - if len(logs) > OTEL_LIMITS["MAX_LOGS_PER_REQUEST"]: - errors.append( - { - "field": "request.logs", - "value": len(logs), - "limit": OTEL_LIMITS["MAX_LOGS_PER_REQUEST"], - "message": f"Request contains {len(logs)} logs, maximum is {OTEL_LIMITS['MAX_LOGS_PER_REQUEST']}. Configure batch size in your OTel SDK.", - } - ) - - # Validate each log record - for i, log_record in enumerate(logs): - # Check attribute count - attributes = log_record.get("attributes", {}) - if len(attributes) > OTEL_LIMITS["MAX_ATTRIBUTES_PER_LOG"]: - errors.append( - { - "field": f"log[{i}].attributes", - "value": len(attributes), - "limit": OTEL_LIMITS["MAX_ATTRIBUTES_PER_LOG"], - "message": f"Log has {len(attributes)} attributes, maximum is {OTEL_LIMITS['MAX_ATTRIBUTES_PER_LOG']}.", - } - ) - - # Check body size - body = log_record.get("body") - if body and isinstance(body, str) and len(body) > OTEL_LIMITS["MAX_LOG_BODY_LENGTH"]: - errors.append( - { - "field": f"log[{i}].body", - "value": len(body), - "limit": OTEL_LIMITS["MAX_LOG_BODY_LENGTH"], - "message": f"Log body exceeds {OTEL_LIMITS['MAX_LOG_BODY_LENGTH']} bytes ({len(body)} bytes). Consider reducing payload size.", - } - ) - - # Check attribute value sizes - for key, value in attributes.items(): - if isinstance(value, str) and len(value) > OTEL_LIMITS["MAX_ATTRIBUTE_VALUE_LENGTH"]: - errors.append( - { - "field": f"log[{i}].attributes.{key}", - "value": len(value), - "limit": OTEL_LIMITS["MAX_ATTRIBUTE_VALUE_LENGTH"], - "message": f"Attribute '{key}' exceeds {OTEL_LIMITS['MAX_ATTRIBUTE_VALUE_LENGTH']} bytes ({len(value)} bytes).", - } - ) - - return errors - - -def transform_logs_to_ai_events(parsed_request: dict[str, Any]) -> list[dict[str, Any]]: - """ - Transform OTel log records to PostHog AI events. - """ - logs = parsed_request.get("logs", []) - resource = parsed_request.get("resource", {}) - scope = parsed_request.get("scope", {}) - - events = [] - for log_record in logs: - event = transform_log_to_ai_event(log_record, resource, scope) - events.append(event) - - return events diff --git a/products/llm_analytics/backend/api/otel/logs_parser.py b/products/llm_analytics/backend/api/otel/logs_parser.py deleted file mode 100644 index 31a1d2a1d323..000000000000 --- a/products/llm_analytics/backend/api/otel/logs_parser.py +++ /dev/null @@ -1,76 +0,0 @@ -""" -OTLP protobuf parser for OpenTelemetry logs. - -Parses ExportLogsServiceRequest protobuf messages and extracts log records, -resource attributes, and instrumentation scope information. -""" - -from typing import Any - -from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ExportLogsServiceRequest -from opentelemetry.proto.logs.v1.logs_pb2 import LogRecord - -from .parser import parse_any_value, parse_attributes - - -def parse_otlp_logs_request(protobuf_data: bytes) -> dict[str, Any]: - """ - Parse OTLP ExportLogsServiceRequest from protobuf bytes. - - Returns a dict with: - - logs: list of parsed log record dicts - - resource: dict of resource attributes - - scope: dict of instrumentation scope info - """ - request = ExportLogsServiceRequest() - request.ParseFromString(protobuf_data) - - parsed_logs = [] - resource_attrs = {} - scope_info = {} - - # OTLP structure: resource_logs -> scope_logs -> log_records - for resource_logs in request.resource_logs: - # Extract resource attributes (service.name, etc.) - if resource_logs.HasField("resource"): - resource_attrs = parse_attributes(resource_logs.resource.attributes) - - # Iterate through scope logs - for scope_logs in resource_logs.scope_logs: - # Extract instrumentation scope - if scope_logs.HasField("scope"): - scope_info = { - "name": scope_logs.scope.name, - "version": scope_logs.scope.version if scope_logs.scope.version else None, - "attributes": parse_attributes(scope_logs.scope.attributes) if scope_logs.scope.attributes else {}, - } - - # Parse each log record - for log_record in scope_logs.log_records: - parsed_log = parse_log_record(log_record) - parsed_logs.append(parsed_log) - - return { - "logs": parsed_logs, - "resource": resource_attrs, - "scope": scope_info, - } - - -def parse_log_record(log_record: LogRecord) -> dict[str, Any]: - """ - Parse a single OTLP log record into a dict. - """ - return { - "time_unix_nano": str(log_record.time_unix_nano), - "observed_time_unix_nano": str(log_record.observed_time_unix_nano) - if log_record.observed_time_unix_nano - else None, - "severity_number": log_record.severity_number, - "severity_text": log_record.severity_text if log_record.severity_text else None, - "body": parse_any_value(log_record.body) if log_record.HasField("body") else None, - "attributes": parse_attributes(log_record.attributes), - "trace_id": log_record.trace_id.hex() if log_record.trace_id else None, - "span_id": log_record.span_id.hex() if log_record.span_id else None, - "flags": log_record.flags if log_record.flags else None, - } diff --git a/products/llm_analytics/backend/api/otel/logs_transformer.py b/products/llm_analytics/backend/api/otel/logs_transformer.py deleted file mode 100644 index 0141683fd249..000000000000 --- a/products/llm_analytics/backend/api/otel/logs_transformer.py +++ /dev/null @@ -1,229 +0,0 @@ -""" -Core OTel log record to PostHog AI event transformer. - -Transforms OpenTelemetry log records into PostHog AI events. -Log records from GenAI instrumentation typically contain message content -(prompts/completions) in the body field. -""" - -import json -from datetime import UTC, datetime -from typing import Any - -OTEL_TRANSFORMER_VERSION = "1.0.0" - - -def transform_log_to_ai_event( - log_record: dict[str, Any], - resource: dict[str, Any], - scope: dict[str, Any], -) -> dict[str, Any]: - """ - Transform a single OTel log record to PostHog AI event. - - Args: - log_record: Parsed OTel log record - resource: Resource attributes (service.name, etc.) - scope: Instrumentation scope info - - Returns: - PostHog AI event dict with: - - event: Event type ($ai_generation, $ai_span, etc.) - - distinct_id: User identifier - - timestamp: ISO 8601 timestamp - - properties: AI event properties - """ - attributes = log_record.get("attributes", {}) - - # Build AI event properties - properties = build_event_properties(log_record, attributes, resource, scope) - - # Determine event type - event_type = determine_event_type(log_record, attributes) - - # Calculate timestamp - timestamp = calculate_timestamp(log_record) - - # Get distinct_id - distinct_id = extract_distinct_id(resource, attributes) - - return { - "event": event_type, - "distinct_id": distinct_id, - "timestamp": timestamp, - "properties": properties, - } - - -def build_event_properties( - log_record: dict[str, Any], - attributes: dict[str, Any], - resource: dict[str, Any], - scope: dict[str, Any], -) -> dict[str, Any]: - """Build PostHog AI event properties from log record.""" - - # Core identifiers (from log record) - trace_id = log_record.get("trace_id") - span_id = log_record.get("span_id") - - # Session ID (from attributes or resource) - session_id = attributes.get("session_id") or resource.get("session.id") - - # Extract message content from body - body = log_record.get("body") - message_content = stringify_content(body) if body else None - - # Build base properties - properties: dict[str, Any] = {} - - # Core IDs - if trace_id: - properties["$ai_trace_id"] = trace_id - if span_id: - properties["$ai_span_id"] = span_id - if session_id: - properties["$ai_session_id"] = session_id - - # Model info (from attributes) - if attributes.get("gen_ai.system"): - properties["$ai_provider"] = attributes["gen_ai.system"] - elif attributes.get("model.provider"): - properties["$ai_provider"] = attributes["model.provider"] - - if attributes.get("gen_ai.request.model"): - properties["$ai_model"] = attributes["gen_ai.request.model"] - elif attributes.get("gen_ai.response.model"): - properties["$ai_model"] = attributes["gen_ai.response.model"] - elif attributes.get("model.name"): - properties["$ai_model"] = attributes["model.name"] - - # Tokens (from attributes) - if attributes.get("gen_ai.usage.input_tokens") is not None: - properties["$ai_input_tokens"] = attributes["gen_ai.usage.input_tokens"] - if attributes.get("gen_ai.usage.output_tokens") is not None: - properties["$ai_output_tokens"] = attributes["gen_ai.usage.output_tokens"] - - # Message content - # Check for specific GenAI log attributes for prompts/completions - if attributes.get("gen_ai.prompt"): - properties["$ai_input"] = stringify_content(attributes["gen_ai.prompt"]) - elif attributes.get("message.content"): - # Some instrumentation sends content in attributes - properties["$ai_input"] = stringify_content(attributes["message.content"]) - elif message_content and attributes.get("event.name") == "gen_ai.content.prompt": - # If event name indicates this is a prompt - properties["$ai_input"] = message_content - - if attributes.get("gen_ai.completion"): - properties["$ai_output_choices"] = stringify_content(attributes["gen_ai.completion"]) - elif message_content and attributes.get("event.name") == "gen_ai.content.completion": - # If event name indicates this is a completion - properties["$ai_output_choices"] = message_content - - # If we have message content but haven't categorized it, store it generically - if message_content and "$ai_input" not in properties and "$ai_output_choices" not in properties: - properties["$ai_message"] = message_content - - # Severity - if log_record.get("severity_number"): - properties["$ai_log_severity_number"] = log_record["severity_number"] - if log_record.get("severity_text"): - properties["$ai_log_severity_text"] = log_record["severity_text"] - - # Metadata - properties["$ai_otel_transformer_version"] = OTEL_TRANSFORMER_VERSION - properties["$ai_otel_log_source"] = "logs" - - # Resource attributes (service name, etc.) - if resource.get("service.name"): - properties["$ai_service_name"] = resource["service.name"] - - # Instrumentation scope - properties["$ai_instrumentation_scope_name"] = scope.get("name", "unknown") - if scope.get("version"): - properties["$ai_instrumentation_scope_version"] = scope["version"] - - # Add remaining log attributes (not already mapped) - mapped_keys = { - "gen_ai.system", - "gen_ai.request.model", - "gen_ai.response.model", - "gen_ai.usage.input_tokens", - "gen_ai.usage.output_tokens", - "gen_ai.prompt", - "gen_ai.completion", - "model.provider", - "model.name", - "message.content", - "session_id", - "session.id", - "event.name", - "service.name", - } - - for key, value in attributes.items(): - if key not in mapped_keys and not key.startswith("gen_ai."): - # Add unmapped attributes with prefix - properties[f"otel.{key}"] = value - - return properties - - -def determine_event_type(log_record: dict[str, Any], attributes: dict[str, Any]) -> str: - """Determine AI event type from log record.""" - event_name = attributes.get("event.name", "").lower() - - # Check event name for GenAI events - if "prompt" in event_name or "input" in event_name: - return "$ai_generation" - elif "completion" in event_name or "output" in event_name or "response" in event_name: - return "$ai_generation" - elif "embedding" in event_name: - return "$ai_embedding" - - # Check operation name - op_name = attributes.get("gen_ai.operation.name", "").lower() - if op_name in ("chat", "completion"): - return "$ai_generation" - elif op_name in ("embedding", "embeddings"): - return "$ai_embedding" - - # Default to generic span - return "$ai_span" - - -def calculate_timestamp(log_record: dict[str, Any]) -> str: - """Calculate timestamp from log record time.""" - time_nanos = int(log_record.get("time_unix_nano", 0)) - if time_nanos == 0: - # Fallback to observed time if time is not set - time_nanos = int(log_record.get("observed_time_unix_nano", 0)) - - millis = time_nanos // 1_000_000 - return datetime.fromtimestamp(millis / 1000, tz=UTC).isoformat() - - -def extract_distinct_id(resource: dict[str, Any], attributes: dict[str, Any]) -> str: - """Extract distinct_id from resource or attributes.""" - # Try resource attributes - user_id = resource.get("user.id") or resource.get("enduser.id") or resource.get("posthog.distinct_id") - - if user_id and isinstance(user_id, str): - return user_id - - # Try log attributes - if attributes.get("user_id"): - return str(attributes["user_id"]) - if attributes.get("distinct_id"): - return str(attributes["distinct_id"]) - - # Default to anonymous - return "anonymous" - - -def stringify_content(content: Any) -> str: - """Stringify content (handles objects and strings).""" - if isinstance(content, str): - return content - return json.dumps(content) From 006105263c2130c9270a14b9a5632dc3f520b7e4 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Thu, 20 Nov 2025 18:23:07 +0000 Subject: [PATCH 10/34] Revert "Remove unused OTEL logs endpoint" This reverts commit 3b600bd22256b5f270a4f947d702b8e85742d281. --- posthog/urls.py | 4 +- .../backend/api/otel/ingestion.py | 260 +++++++++++++++++- .../backend/api/otel/logs_parser.py | 76 +++++ .../backend/api/otel/logs_transformer.py | 229 +++++++++++++++ 4 files changed, 565 insertions(+), 4 deletions(-) create mode 100644 products/llm_analytics/backend/api/otel/logs_parser.py create mode 100644 products/llm_analytics/backend/api/otel/logs_transformer.py diff --git a/posthog/urls.py b/posthog/urls.py index 3b5da2223bf7..bab1a59761f1 100644 --- a/posthog/urls.py +++ b/posthog/urls.py @@ -47,7 +47,7 @@ from posthog.temporal.codec_server import decode_payloads from products.early_access_features.backend.api import early_access_features -from products.llm_analytics.backend.api.otel.ingestion import otel_traces_endpoint +from products.llm_analytics.backend.api.otel.ingestion import otel_logs_endpoint, otel_traces_endpoint from .utils import opt_slash_path, render_template from .views import ( @@ -213,6 +213,8 @@ def proxy_logs_to_capture_service(request: HttpRequest) -> HttpResponse: # api # OpenTelemetry traces ingestion for LLM Analytics path("api/projects//ai/otel/v1/traces", csrf_exempt(otel_traces_endpoint)), + # OpenTelemetry logs ingestion for LLM Analytics + path("api/projects//ai/otel/v1/logs", csrf_exempt(otel_logs_endpoint)), # OpenTelemetry logs proxy to capture-logs service (legacy) path("i/v1/logs", proxy_logs_to_capture_service), path("api/environments//progress/", progress), diff --git a/products/llm_analytics/backend/api/otel/ingestion.py b/products/llm_analytics/backend/api/otel/ingestion.py index a0a5da34c295..4c8b94387dfa 100644 --- a/products/llm_analytics/backend/api/otel/ingestion.py +++ b/products/llm_analytics/backend/api/otel/ingestion.py @@ -1,10 +1,11 @@ """ -OpenTelemetry traces ingestion API endpoint. +OpenTelemetry traces and logs ingestion API endpoints. -Accepts OTLP/HTTP (protobuf) format traces and converts them to PostHog AI events. +Accepts OTLP/HTTP (protobuf) format traces and logs and converts them to PostHog AI events. -Endpoint: +Endpoints: - POST /api/projects/:project_id/ai/otel/v1/traces +- POST /api/projects/:project_id/ai/otel/v1/logs Content-Type: application/x-protobuf Authorization: Bearer @@ -29,6 +30,8 @@ from posthog.api.capture import capture_batch_internal from posthog.models import Team +from .logs_parser import parse_otlp_logs_request +from .logs_transformer import transform_log_to_ai_event from .parser import parse_baggage_header, parse_otlp_request from .transformer import transform_span_to_ai_event @@ -42,6 +45,9 @@ "MAX_LINKS_PER_SPAN": 128, "MAX_ATTRIBUTE_VALUE_LENGTH": 100_000, # 100KB "MAX_SPAN_NAME_LENGTH": 1024, + "MAX_LOGS_PER_REQUEST": 1000, + "MAX_ATTRIBUTES_PER_LOG": 128, + "MAX_LOG_BODY_LENGTH": 100_000, # 100KB } @@ -431,3 +437,251 @@ def capture_events(events: list[dict[str, Any]], team: Team) -> None: error_count=len(errors), errors=errors[:10], # Log first 10 errors ) + + +@extend_schema( + description=""" + OpenTelemetry logs ingestion endpoint for LLM Analytics. + + Accepts OTLP/HTTP (protobuf) format logs following the OpenTelemetry Protocol specification. + Converts OTel log records to PostHog AI events. Logs from GenAI instrumentation typically + contain message content (prompts/completions) in the body field. + + Supported conventions: + - GenAI semantic conventions: gen_ai.* attributes + - Generic OTel log attributes + + Example OTel SDK configuration: + ```python + from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter + + exporter = OTLPLogExporter( + endpoint="https://app.posthog.com/api/projects/{project_id}/ai/otel/v1/logs", + headers={"Authorization": "Bearer phc_your_project_token"} + ) + ``` + + Authentication: + - Use your project API token (starts with phc_...), NOT a personal API key + - Token can be provided via Authorization header (Bearer token) or ?token= query parameter + + Rate limits and quotas apply as per normal PostHog event ingestion. + """, + request={"application/x-protobuf": bytes}, + responses={ + 200: {"description": "Logs accepted for processing"}, + 400: {"description": "Invalid OTLP format or validation errors"}, + 401: {"description": "Authentication failed"}, + 413: {"description": "Request too large (exceeds log/attribute limits)"}, + }, +) +@api_view(["POST"]) +@authentication_classes([ProjectTokenAuthentication]) +@permission_classes([]) +def otel_logs_endpoint(request: HttpRequest, project_id: int) -> Response: + """ + Process OTLP logs export requests. + + This endpoint: + 1. Validates authentication and project access + 2. Parses OTLP protobuf payload + 3. Validates against size/log limits + 4. Transforms OTel log records to PostHog AI events + 5. Routes events to capture pipeline for ingestion + """ + + # Get authenticated team from request + # ProjectTokenAuthentication returns (team, token) tuple + if not hasattr(request, "user") or not isinstance(request.user, Team): + return Response( + { + "error": "Invalid authentication. Use project token (phc_...) in Authorization header or ?token= parameter." + }, + status=status.HTTP_401_UNAUTHORIZED, + ) + + team = request.user + + # Verify the team ID matches the project_id in URL + if team.id != project_id: + return Response( + {"error": "Project ID in URL does not match authenticated project token"}, + status=status.HTTP_403_FORBIDDEN, + ) + + # Check content type + content_type = request.content_type or "" + if "protobuf" not in content_type and "octet-stream" not in content_type: + return Response( + { + "error": f"Invalid content type: {content_type}. Expected application/x-protobuf or application/octet-stream" + }, + status=status.HTTP_400_BAD_REQUEST, + ) + + # Get raw protobuf body + protobuf_data = request.body + + if not protobuf_data: + return Response( + {"error": "Empty request body"}, + status=status.HTTP_400_BAD_REQUEST, + ) + + logger.info( + "otel_logs_received", + team_id=team.id, + content_length=len(protobuf_data), + content_type=content_type, + ) + + try: + # Parse OTLP protobuf + parsed_request = parse_otlp_logs_request(protobuf_data) + + logger.info( + "otel_logs_parsed", + team_id=team.id, + logs_count=len(parsed_request["logs"]), + ) + + # Validate request + validation_errors = validate_otlp_logs_request(parsed_request) + if validation_errors: + logger.warning( + "otel_logs_validation_failed", + team_id=team.id, + errors=validation_errors, + ) + return Response( + {"error": "Validation failed", "details": validation_errors}, + status=status.HTTP_400_BAD_REQUEST, + ) + + # Transform logs to AI events + events = transform_logs_to_ai_events(parsed_request) + + logger.info( + "otel_logs_transformed", + team_id=team.id, + events_created=len(events), + ) + + # Route to capture pipeline + capture_events(events, team) + + logger.info( + "otel_logs_captured", + team_id=team.id, + events_captured=len(events), + ) + + return Response( + { + "status": "success", + "message": "Logs ingested successfully", + "logs_received": len(parsed_request["logs"]), + "events_created": len(events), + }, + status=status.HTTP_200_OK, + ) + + except ValidationError as e: + logger.warning( + "otel_logs_validation_error", + team_id=team.id, + error=str(e), + ) + return Response( + {"error": str(e)}, + status=status.HTTP_400_BAD_REQUEST, + ) + + except Exception as e: + logger.error( + "otel_logs_processing_error", + team_id=team.id, + error=str(e), + exc_info=True, + ) + return Response( + {"error": "Internal server error processing logs"}, + status=status.HTTP_500_INTERNAL_SERVER_ERROR, + ) + + +def validate_otlp_logs_request(parsed_request: dict[str, Any]) -> list[dict[str, Any]]: + """ + Validate OTLP logs request against limits. + + Returns list of validation errors (empty if valid). + """ + errors = [] + logs = parsed_request.get("logs", []) + + # Check log count + if len(logs) > OTEL_LIMITS["MAX_LOGS_PER_REQUEST"]: + errors.append( + { + "field": "request.logs", + "value": len(logs), + "limit": OTEL_LIMITS["MAX_LOGS_PER_REQUEST"], + "message": f"Request contains {len(logs)} logs, maximum is {OTEL_LIMITS['MAX_LOGS_PER_REQUEST']}. Configure batch size in your OTel SDK.", + } + ) + + # Validate each log record + for i, log_record in enumerate(logs): + # Check attribute count + attributes = log_record.get("attributes", {}) + if len(attributes) > OTEL_LIMITS["MAX_ATTRIBUTES_PER_LOG"]: + errors.append( + { + "field": f"log[{i}].attributes", + "value": len(attributes), + "limit": OTEL_LIMITS["MAX_ATTRIBUTES_PER_LOG"], + "message": f"Log has {len(attributes)} attributes, maximum is {OTEL_LIMITS['MAX_ATTRIBUTES_PER_LOG']}.", + } + ) + + # Check body size + body = log_record.get("body") + if body and isinstance(body, str) and len(body) > OTEL_LIMITS["MAX_LOG_BODY_LENGTH"]: + errors.append( + { + "field": f"log[{i}].body", + "value": len(body), + "limit": OTEL_LIMITS["MAX_LOG_BODY_LENGTH"], + "message": f"Log body exceeds {OTEL_LIMITS['MAX_LOG_BODY_LENGTH']} bytes ({len(body)} bytes). Consider reducing payload size.", + } + ) + + # Check attribute value sizes + for key, value in attributes.items(): + if isinstance(value, str) and len(value) > OTEL_LIMITS["MAX_ATTRIBUTE_VALUE_LENGTH"]: + errors.append( + { + "field": f"log[{i}].attributes.{key}", + "value": len(value), + "limit": OTEL_LIMITS["MAX_ATTRIBUTE_VALUE_LENGTH"], + "message": f"Attribute '{key}' exceeds {OTEL_LIMITS['MAX_ATTRIBUTE_VALUE_LENGTH']} bytes ({len(value)} bytes).", + } + ) + + return errors + + +def transform_logs_to_ai_events(parsed_request: dict[str, Any]) -> list[dict[str, Any]]: + """ + Transform OTel log records to PostHog AI events. + """ + logs = parsed_request.get("logs", []) + resource = parsed_request.get("resource", {}) + scope = parsed_request.get("scope", {}) + + events = [] + for log_record in logs: + event = transform_log_to_ai_event(log_record, resource, scope) + events.append(event) + + return events diff --git a/products/llm_analytics/backend/api/otel/logs_parser.py b/products/llm_analytics/backend/api/otel/logs_parser.py new file mode 100644 index 000000000000..31a1d2a1d323 --- /dev/null +++ b/products/llm_analytics/backend/api/otel/logs_parser.py @@ -0,0 +1,76 @@ +""" +OTLP protobuf parser for OpenTelemetry logs. + +Parses ExportLogsServiceRequest protobuf messages and extracts log records, +resource attributes, and instrumentation scope information. +""" + +from typing import Any + +from opentelemetry.proto.collector.logs.v1.logs_service_pb2 import ExportLogsServiceRequest +from opentelemetry.proto.logs.v1.logs_pb2 import LogRecord + +from .parser import parse_any_value, parse_attributes + + +def parse_otlp_logs_request(protobuf_data: bytes) -> dict[str, Any]: + """ + Parse OTLP ExportLogsServiceRequest from protobuf bytes. + + Returns a dict with: + - logs: list of parsed log record dicts + - resource: dict of resource attributes + - scope: dict of instrumentation scope info + """ + request = ExportLogsServiceRequest() + request.ParseFromString(protobuf_data) + + parsed_logs = [] + resource_attrs = {} + scope_info = {} + + # OTLP structure: resource_logs -> scope_logs -> log_records + for resource_logs in request.resource_logs: + # Extract resource attributes (service.name, etc.) + if resource_logs.HasField("resource"): + resource_attrs = parse_attributes(resource_logs.resource.attributes) + + # Iterate through scope logs + for scope_logs in resource_logs.scope_logs: + # Extract instrumentation scope + if scope_logs.HasField("scope"): + scope_info = { + "name": scope_logs.scope.name, + "version": scope_logs.scope.version if scope_logs.scope.version else None, + "attributes": parse_attributes(scope_logs.scope.attributes) if scope_logs.scope.attributes else {}, + } + + # Parse each log record + for log_record in scope_logs.log_records: + parsed_log = parse_log_record(log_record) + parsed_logs.append(parsed_log) + + return { + "logs": parsed_logs, + "resource": resource_attrs, + "scope": scope_info, + } + + +def parse_log_record(log_record: LogRecord) -> dict[str, Any]: + """ + Parse a single OTLP log record into a dict. + """ + return { + "time_unix_nano": str(log_record.time_unix_nano), + "observed_time_unix_nano": str(log_record.observed_time_unix_nano) + if log_record.observed_time_unix_nano + else None, + "severity_number": log_record.severity_number, + "severity_text": log_record.severity_text if log_record.severity_text else None, + "body": parse_any_value(log_record.body) if log_record.HasField("body") else None, + "attributes": parse_attributes(log_record.attributes), + "trace_id": log_record.trace_id.hex() if log_record.trace_id else None, + "span_id": log_record.span_id.hex() if log_record.span_id else None, + "flags": log_record.flags if log_record.flags else None, + } diff --git a/products/llm_analytics/backend/api/otel/logs_transformer.py b/products/llm_analytics/backend/api/otel/logs_transformer.py new file mode 100644 index 000000000000..0141683fd249 --- /dev/null +++ b/products/llm_analytics/backend/api/otel/logs_transformer.py @@ -0,0 +1,229 @@ +""" +Core OTel log record to PostHog AI event transformer. + +Transforms OpenTelemetry log records into PostHog AI events. +Log records from GenAI instrumentation typically contain message content +(prompts/completions) in the body field. +""" + +import json +from datetime import UTC, datetime +from typing import Any + +OTEL_TRANSFORMER_VERSION = "1.0.0" + + +def transform_log_to_ai_event( + log_record: dict[str, Any], + resource: dict[str, Any], + scope: dict[str, Any], +) -> dict[str, Any]: + """ + Transform a single OTel log record to PostHog AI event. + + Args: + log_record: Parsed OTel log record + resource: Resource attributes (service.name, etc.) + scope: Instrumentation scope info + + Returns: + PostHog AI event dict with: + - event: Event type ($ai_generation, $ai_span, etc.) + - distinct_id: User identifier + - timestamp: ISO 8601 timestamp + - properties: AI event properties + """ + attributes = log_record.get("attributes", {}) + + # Build AI event properties + properties = build_event_properties(log_record, attributes, resource, scope) + + # Determine event type + event_type = determine_event_type(log_record, attributes) + + # Calculate timestamp + timestamp = calculate_timestamp(log_record) + + # Get distinct_id + distinct_id = extract_distinct_id(resource, attributes) + + return { + "event": event_type, + "distinct_id": distinct_id, + "timestamp": timestamp, + "properties": properties, + } + + +def build_event_properties( + log_record: dict[str, Any], + attributes: dict[str, Any], + resource: dict[str, Any], + scope: dict[str, Any], +) -> dict[str, Any]: + """Build PostHog AI event properties from log record.""" + + # Core identifiers (from log record) + trace_id = log_record.get("trace_id") + span_id = log_record.get("span_id") + + # Session ID (from attributes or resource) + session_id = attributes.get("session_id") or resource.get("session.id") + + # Extract message content from body + body = log_record.get("body") + message_content = stringify_content(body) if body else None + + # Build base properties + properties: dict[str, Any] = {} + + # Core IDs + if trace_id: + properties["$ai_trace_id"] = trace_id + if span_id: + properties["$ai_span_id"] = span_id + if session_id: + properties["$ai_session_id"] = session_id + + # Model info (from attributes) + if attributes.get("gen_ai.system"): + properties["$ai_provider"] = attributes["gen_ai.system"] + elif attributes.get("model.provider"): + properties["$ai_provider"] = attributes["model.provider"] + + if attributes.get("gen_ai.request.model"): + properties["$ai_model"] = attributes["gen_ai.request.model"] + elif attributes.get("gen_ai.response.model"): + properties["$ai_model"] = attributes["gen_ai.response.model"] + elif attributes.get("model.name"): + properties["$ai_model"] = attributes["model.name"] + + # Tokens (from attributes) + if attributes.get("gen_ai.usage.input_tokens") is not None: + properties["$ai_input_tokens"] = attributes["gen_ai.usage.input_tokens"] + if attributes.get("gen_ai.usage.output_tokens") is not None: + properties["$ai_output_tokens"] = attributes["gen_ai.usage.output_tokens"] + + # Message content + # Check for specific GenAI log attributes for prompts/completions + if attributes.get("gen_ai.prompt"): + properties["$ai_input"] = stringify_content(attributes["gen_ai.prompt"]) + elif attributes.get("message.content"): + # Some instrumentation sends content in attributes + properties["$ai_input"] = stringify_content(attributes["message.content"]) + elif message_content and attributes.get("event.name") == "gen_ai.content.prompt": + # If event name indicates this is a prompt + properties["$ai_input"] = message_content + + if attributes.get("gen_ai.completion"): + properties["$ai_output_choices"] = stringify_content(attributes["gen_ai.completion"]) + elif message_content and attributes.get("event.name") == "gen_ai.content.completion": + # If event name indicates this is a completion + properties["$ai_output_choices"] = message_content + + # If we have message content but haven't categorized it, store it generically + if message_content and "$ai_input" not in properties and "$ai_output_choices" not in properties: + properties["$ai_message"] = message_content + + # Severity + if log_record.get("severity_number"): + properties["$ai_log_severity_number"] = log_record["severity_number"] + if log_record.get("severity_text"): + properties["$ai_log_severity_text"] = log_record["severity_text"] + + # Metadata + properties["$ai_otel_transformer_version"] = OTEL_TRANSFORMER_VERSION + properties["$ai_otel_log_source"] = "logs" + + # Resource attributes (service name, etc.) + if resource.get("service.name"): + properties["$ai_service_name"] = resource["service.name"] + + # Instrumentation scope + properties["$ai_instrumentation_scope_name"] = scope.get("name", "unknown") + if scope.get("version"): + properties["$ai_instrumentation_scope_version"] = scope["version"] + + # Add remaining log attributes (not already mapped) + mapped_keys = { + "gen_ai.system", + "gen_ai.request.model", + "gen_ai.response.model", + "gen_ai.usage.input_tokens", + "gen_ai.usage.output_tokens", + "gen_ai.prompt", + "gen_ai.completion", + "model.provider", + "model.name", + "message.content", + "session_id", + "session.id", + "event.name", + "service.name", + } + + for key, value in attributes.items(): + if key not in mapped_keys and not key.startswith("gen_ai."): + # Add unmapped attributes with prefix + properties[f"otel.{key}"] = value + + return properties + + +def determine_event_type(log_record: dict[str, Any], attributes: dict[str, Any]) -> str: + """Determine AI event type from log record.""" + event_name = attributes.get("event.name", "").lower() + + # Check event name for GenAI events + if "prompt" in event_name or "input" in event_name: + return "$ai_generation" + elif "completion" in event_name or "output" in event_name or "response" in event_name: + return "$ai_generation" + elif "embedding" in event_name: + return "$ai_embedding" + + # Check operation name + op_name = attributes.get("gen_ai.operation.name", "").lower() + if op_name in ("chat", "completion"): + return "$ai_generation" + elif op_name in ("embedding", "embeddings"): + return "$ai_embedding" + + # Default to generic span + return "$ai_span" + + +def calculate_timestamp(log_record: dict[str, Any]) -> str: + """Calculate timestamp from log record time.""" + time_nanos = int(log_record.get("time_unix_nano", 0)) + if time_nanos == 0: + # Fallback to observed time if time is not set + time_nanos = int(log_record.get("observed_time_unix_nano", 0)) + + millis = time_nanos // 1_000_000 + return datetime.fromtimestamp(millis / 1000, tz=UTC).isoformat() + + +def extract_distinct_id(resource: dict[str, Any], attributes: dict[str, Any]) -> str: + """Extract distinct_id from resource or attributes.""" + # Try resource attributes + user_id = resource.get("user.id") or resource.get("enduser.id") or resource.get("posthog.distinct_id") + + if user_id and isinstance(user_id, str): + return user_id + + # Try log attributes + if attributes.get("user_id"): + return str(attributes["user_id"]) + if attributes.get("distinct_id"): + return str(attributes["distinct_id"]) + + # Default to anonymous + return "anonymous" + + +def stringify_content(content: Any) -> str: + """Stringify content (handles objects and strings).""" + if isinstance(content, str): + return content + return json.dumps(content) From d0141119e4b1293bcf4887513b0ae6c413a90697 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Thu, 20 Nov 2025 18:23:37 +0000 Subject: [PATCH 11/34] Restore OTEL logs endpoint to support both v1 and v2 instrumentation The v2 OpenAI instrumentation (opentelemetry-instrumentation-openai-v2) sends message content as separate log records via logger.emit(), while v1 sends everything as span attributes. To meet users where they are, we need to support both: - v1: Uses traces endpoint only (span attributes) - v2: Uses both traces (metadata) and logs (message content) endpoints Updated module docstring to clarify the difference and requirements. --- products/llm_analytics/backend/api/otel/ingestion.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/products/llm_analytics/backend/api/otel/ingestion.py b/products/llm_analytics/backend/api/otel/ingestion.py index 4c8b94387dfa..0dc474358942 100644 --- a/products/llm_analytics/backend/api/otel/ingestion.py +++ b/products/llm_analytics/backend/api/otel/ingestion.py @@ -3,9 +3,13 @@ Accepts OTLP/HTTP (protobuf) format traces and logs and converts them to PostHog AI events. +Supports both OpenAI instrumentation versions: +- v1 (opentelemetry-instrumentation-openai): Sends everything as trace span attributes +- v2 (opentelemetry-instrumentation-openai-v2): Sends metadata as spans, message content as logs + Endpoints: -- POST /api/projects/:project_id/ai/otel/v1/traces -- POST /api/projects/:project_id/ai/otel/v1/logs +- POST /api/projects/:project_id/ai/otel/v1/traces - Required for all instrumentation +- POST /api/projects/:project_id/ai/otel/v1/logs - Required for v2 instrumentation with message content Content-Type: application/x-protobuf Authorization: Bearer From bf048c6f29b46fddf87b438dd7b17b368da5bc10 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Thu, 20 Nov 2025 18:38:04 +0000 Subject: [PATCH 12/34] Fix v2 log event parsing to extract message content The v2 instrumentation sends structured log events with bodies like: - gen_ai.user.message: {"content": "..."} - gen_ai.assistant.message: {"content": "..."} or {"tool_calls": [...]} - gen_ai.choice: {"index": 0, "finish_reason": "stop", "message": {...}} Updated logs_transformer to properly extract: - Message content from body.content - Tool calls from body.tool_calls - Output content and metadata from choice events - Role and finish_reason from structured bodies This ensures v2 message content appears in PostHog traces. --- .../backend/api/otel/logs_transformer.py | 47 +++++++++++++------ 1 file changed, 33 insertions(+), 14 deletions(-) diff --git a/products/llm_analytics/backend/api/otel/logs_transformer.py b/products/llm_analytics/backend/api/otel/logs_transformer.py index 0141683fd249..95a04e0747ea 100644 --- a/products/llm_analytics/backend/api/otel/logs_transformer.py +++ b/products/llm_analytics/backend/api/otel/logs_transformer.py @@ -72,7 +72,6 @@ def build_event_properties( # Extract message content from body body = log_record.get("body") - message_content = stringify_content(body) if body else None # Build base properties properties: dict[str, Any] = {} @@ -104,26 +103,46 @@ def build_event_properties( if attributes.get("gen_ai.usage.output_tokens") is not None: properties["$ai_output_tokens"] = attributes["gen_ai.usage.output_tokens"] - # Message content - # Check for specific GenAI log attributes for prompts/completions + # Message content - handle v2 structured log events + # v2 instrumentation sends logs with event names like: + # - gen_ai.user.message (body: {"content": "..."}) + # - gen_ai.assistant.message (body: {"content": "..."} or {"tool_calls": [...]}) + # - gen_ai.choice (body: {"index": 0, "finish_reason": "stop", "message": {...}}) + + # Handle v2 message events + # v2 uses LogRecord.event_name (in attributes) and structured body + if isinstance(body, dict): + # v2 user/system messages: {"content": "..."} + if "content" in body and isinstance(body["content"], str): + content_text = body["content"] + # Determine if input or output based on context + # For now, store as generic message - the span will have role info + properties["$ai_message_content"] = content_text + + # v2 assistant messages with tool calls: {"tool_calls": [...]} + if "tool_calls" in body and isinstance(body["tool_calls"], list): + properties["$ai_tool_calls"] = body["tool_calls"] + + # v2 choice events: {"index": 0, "finish_reason": "stop", "message": {...}} + if "message" in body and isinstance(body["message"], dict): + message_obj = body["message"] + if "content" in message_obj: + properties["$ai_output_content"] = message_obj["content"] + if "tool_calls" in message_obj: + properties["$ai_tool_calls"] = message_obj["tool_calls"] + if "role" in message_obj: + properties["$ai_message_role"] = message_obj["role"] + if "finish_reason" in body: + properties["$ai_finish_reason"] = body["finish_reason"] + + # Fallback: legacy v1-style attributes if attributes.get("gen_ai.prompt"): properties["$ai_input"] = stringify_content(attributes["gen_ai.prompt"]) elif attributes.get("message.content"): - # Some instrumentation sends content in attributes properties["$ai_input"] = stringify_content(attributes["message.content"]) - elif message_content and attributes.get("event.name") == "gen_ai.content.prompt": - # If event name indicates this is a prompt - properties["$ai_input"] = message_content if attributes.get("gen_ai.completion"): properties["$ai_output_choices"] = stringify_content(attributes["gen_ai.completion"]) - elif message_content and attributes.get("event.name") == "gen_ai.content.completion": - # If event name indicates this is a completion - properties["$ai_output_choices"] = message_content - - # If we have message content but haven't categorized it, store it generically - if message_content and "$ai_input" not in properties and "$ai_output_choices" not in properties: - properties["$ai_message"] = message_content # Severity if log_record.get("severity_number"): From e311ee9ae2fcb5ba38831a95629a78dd17aab8d7 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Fri, 21 Nov 2025 00:27:14 +0000 Subject: [PATCH 13/34] fix(otel): align transformers with PostHog LLM Analytics schema MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix property names to match documented schema: - $ai_error_message → $ai_error - $ai_cache_read_tokens → $ai_cache_read_input_tokens - $ai_cache_write_tokens → $ai_cache_creation_input_tokens - Remove undocumented properties ($ai_message_content, $ai_output_content, etc.) - Properly structure $ai_input and $ai_output_choices as arrays - Add Redis-based event merger to combine v2 trace and log events - Support multiple log event accumulation (user + assistant messages) - Add bidirectional merge logic (logs-first or traces-first) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .../backend/api/otel/event_merger.py | 155 ++++++++++++++++++ .../backend/api/otel/ingestion.py | 21 ++- .../backend/api/otel/logs_transformer.py | 134 +++++++++++---- .../backend/api/otel/test_event_merger.py | 110 +++++++++++++ .../backend/api/otel/transformer.py | 78 ++++++++- 5 files changed, 452 insertions(+), 46 deletions(-) create mode 100644 products/llm_analytics/backend/api/otel/event_merger.py create mode 100644 products/llm_analytics/backend/api/otel/test_event_merger.py diff --git a/products/llm_analytics/backend/api/otel/event_merger.py b/products/llm_analytics/backend/api/otel/event_merger.py new file mode 100644 index 000000000000..1e06a96b108c --- /dev/null +++ b/products/llm_analytics/backend/api/otel/event_merger.py @@ -0,0 +1,155 @@ +""" +Event merger for combining OTEL log and trace events using Redis. + +V2 instrumentation sends traces (with metadata) and logs (with message content) +as separate HTTP requests. The order varies - sometimes traces first, sometimes logs first. + +This module uses Redis to cache and merge them bidirectionally with NO BLOCKING: +- First arrival (trace OR log): Cache in Redis, don't send +- Second arrival (log OR trace): Find cached partner, merge, and send + +This eliminates the thread starvation problem of blocking with time.sleep(). +""" + +import json +import logging +from typing import Any, Optional + +from posthog.redis import get_client + +logger = logging.getLogger(__name__) + +_CACHE_TTL = 60 # seconds - Redis will auto-expire after this + + +def cache_and_merge_properties( + trace_id: str, span_id: str, properties: dict[str, Any], is_trace: bool = True +) -> Optional[dict[str, Any]]: + """ + Cache properties and merge with any existing cached properties for the same span. + + Uses separate cache keys for traces and logs to properly accumulate multiple log events: + - Trace cache: otel_merge:trace:{trace_id}:{span_id} + - Logs cache: otel_merge:logs:{trace_id}:{span_id} + + Flow: + 1. Logs accumulate in logs cache (multiple logs merge together) + 2. When trace arrives, merge all accumulated logs with trace and send + 3. When log arrives after trace, merge immediately and send + + Args: + trace_id: Trace ID + span_id: Span ID + properties: Properties dict to cache/merge + is_trace: True if trace properties, False if log properties + + Returns: + - None if waiting for more data (cached, not sent) + - Merged properties if ready to send (send to capture) + """ + redis_client = get_client() + trace_cache_key = f"otel_merge:trace:{trace_id}:{span_id}" + logs_cache_key = f"otel_merge:logs:{trace_id}:{span_id}" + + try: + if is_trace: + # Trace arriving - check if logs are already cached + logs_json = redis_client.get(logs_cache_key) + + if logs_json: + # Logs already cached - merge and send + logs_properties = json.loads(logs_json) + merged = {**logs_properties, **properties} # Trace props override + + # Clean up logs cache + redis_client.delete(logs_cache_key) + + logger.info("event_merger_success: Merged trace+logs", extra={"trace_id": trace_id, "span_id": span_id}) + + return merged + else: + # No logs yet - cache trace + redis_client.setex(trace_cache_key, _CACHE_TTL, json.dumps(properties)) + + logger.info( + "event_merger_cache: Cached trace properties", extra={"trace_id": trace_id, "span_id": span_id} + ) + + return None + else: + # Log arriving - accumulate with other logs first + logs_json = redis_client.get(logs_cache_key) + + if logs_json: + # Another log already cached - accumulate + existing_logs = json.loads(logs_json) + merged_logs = {**existing_logs, **properties} # Later log props override + + # Re-cache accumulated logs + redis_client.setex(logs_cache_key, _CACHE_TTL, json.dumps(merged_logs)) + + logger.info( + "event_merger_accumulate: Accumulated log properties", + extra={"trace_id": trace_id, "span_id": span_id}, + ) + + # Check if trace is ready + trace_json = redis_client.get(trace_cache_key) + if trace_json: + # Trace is ready - merge and send + trace_properties = json.loads(trace_json) + final_merged = {**merged_logs, **trace_properties} # Trace props override + + # Clean up both caches + redis_client.delete(logs_cache_key) + redis_client.delete(trace_cache_key) + + logger.info( + "event_merger_success: Merged accumulated logs+trace", + extra={"trace_id": trace_id, "span_id": span_id}, + ) + + return final_merged + + # Trace not ready yet - wait for it + return None + else: + # First log - check if trace is already cached + trace_json = redis_client.get(trace_cache_key) + + if trace_json: + # Trace already cached - merge and send + trace_properties = json.loads(trace_json) + merged = {**properties, **trace_properties} # Trace props override + + # Clean up trace cache + redis_client.delete(trace_cache_key) + + logger.info( + "event_merger_success: Merged logs+trace", extra={"trace_id": trace_id, "span_id": span_id} + ) + + return merged + else: + # No trace yet - cache this log + redis_client.setex(logs_cache_key, _CACHE_TTL, json.dumps(properties)) + + logger.info( + "event_merger_cache: Cached log properties", extra={"trace_id": trace_id, "span_id": span_id} + ) + + return None + + except Exception as e: + # Redis error - log and fall back to sending immediately + logger.exception( + f"event_merger_error: Redis error during merge, sending immediately", + extra={ + "trace_id": trace_id, + "span_id": span_id, + "error": str(e), + "is_trace": is_trace, + }, + ) + # Fallback: send immediately without merging + return properties diff --git a/products/llm_analytics/backend/api/otel/ingestion.py b/products/llm_analytics/backend/api/otel/ingestion.py index 0dc474358942..d2749481a563 100644 --- a/products/llm_analytics/backend/api/otel/ingestion.py +++ b/products/llm_analytics/backend/api/otel/ingestion.py @@ -393,6 +393,9 @@ def transform_spans_to_ai_events(parsed_request: dict[str, Any], baggage: dict[s Uses waterfall pattern for attribute extraction: 1. PostHog native (posthog.ai.*) 2. GenAI semantic conventions (gen_ai.*) + + Note: Returns only events ready to send. Events that are first arrivals + (cached, waiting for logs) are filtered out. """ spans = parsed_request.get("spans", []) resource = parsed_request.get("resource", {}) @@ -401,7 +404,8 @@ def transform_spans_to_ai_events(parsed_request: dict[str, Any], baggage: dict[s events = [] for span in spans: event = transform_span_to_ai_event(span, resource, scope, baggage) - events.append(event) + if event is not None: # Filter out first arrivals (cached, waiting for logs) + events.append(event) return events @@ -562,7 +566,7 @@ def otel_logs_endpoint(request: HttpRequest, project_id: int) -> Response: status=status.HTTP_400_BAD_REQUEST, ) - # Transform logs to AI events + # Transform logs to AI events (also caches properties for merging with traces) events = transform_logs_to_ai_events(parsed_request) logger.info( @@ -571,7 +575,12 @@ def otel_logs_endpoint(request: HttpRequest, project_id: int) -> Response: events_created=len(events), ) - # Route to capture pipeline + # True bidirectional merge with Redis: + # - First arrivals (logs before traces): Cached in Redis, no events to send + # - Second arrivals (logs after traces): Merged and sent to capture + # The transform function filters out first arrivals, so events only contains second arrivals + + # Route merged events to capture pipeline capture_events(events, team) logger.info( @@ -678,6 +687,9 @@ def validate_otlp_logs_request(parsed_request: dict[str, Any]) -> list[dict[str, def transform_logs_to_ai_events(parsed_request: dict[str, Any]) -> list[dict[str, Any]]: """ Transform OTel log records to PostHog AI events. + + Note: Returns only events ready to send. Events that are first arrivals + (cached, waiting for traces) are filtered out. """ logs = parsed_request.get("logs", []) resource = parsed_request.get("resource", {}) @@ -686,6 +698,7 @@ def transform_logs_to_ai_events(parsed_request: dict[str, Any]) -> list[dict[str events = [] for log_record in logs: event = transform_log_to_ai_event(log_record, resource, scope) - events.append(event) + if event is not None: # Filter out first arrivals (cached, waiting for trace) + events.append(event) return events diff --git a/products/llm_analytics/backend/api/otel/logs_transformer.py b/products/llm_analytics/backend/api/otel/logs_transformer.py index 95a04e0747ea..5a3f02c74b88 100644 --- a/products/llm_analytics/backend/api/otel/logs_transformer.py +++ b/products/llm_analytics/backend/api/otel/logs_transformer.py @@ -10,6 +10,8 @@ from datetime import UTC, datetime from typing import Any +from .event_merger import cache_and_merge_properties + OTEL_TRANSFORMER_VERSION = "1.0.0" @@ -17,7 +19,7 @@ def transform_log_to_ai_event( log_record: dict[str, Any], resource: dict[str, Any], scope: dict[str, Any], -) -> dict[str, Any]: +) -> dict[str, Any] | None: """ Transform a single OTel log record to PostHog AI event. @@ -27,17 +29,31 @@ def transform_log_to_ai_event( scope: Instrumentation scope info Returns: - PostHog AI event dict with: + PostHog AI event dict OR None if this is first arrival (cached, waiting for trace): - event: Event type ($ai_generation, $ai_span, etc.) - distinct_id: User identifier - timestamp: ISO 8601 timestamp - properties: AI event properties + - uuid: Event UUID (for deduplication with trace events) """ attributes = log_record.get("attributes", {}) # Build AI event properties properties = build_event_properties(log_record, attributes, resource, scope) + # True bidirectional merge with Redis (no blocking) + # First arrival caches, second arrival merges and sends + trace_id = log_record.get("trace_id", "") + span_id = log_record.get("span_id", "") + if trace_id and span_id: + merged = cache_and_merge_properties(trace_id, span_id, properties, is_trace=False) + if merged is None: + # This is first arrival - log cached, waiting for trace + # Don't send this event yet + return None + # Second arrival - trace already cached, merged contains complete event + properties = merged + # Determine event type event_type = determine_event_type(log_record, attributes) @@ -47,13 +63,28 @@ def transform_log_to_ai_event( # Get distinct_id distinct_id = extract_distinct_id(resource, attributes) - return { + # Generate consistent UUID from trace_id + span_id for deduplication + # This allows log events and trace events for the same span to merge + import uuid + + event_uuid = None + if trace_id and span_id: + # Create deterministic UUID from trace_id + span_id + namespace = uuid.UUID("00000000-0000-0000-0000-000000000000") + event_uuid = str(uuid.uuid5(namespace, f"{trace_id}:{span_id}")) + + result = { "event": event_type, "distinct_id": distinct_id, "timestamp": timestamp, "properties": properties, } + if event_uuid: + result["uuid"] = event_uuid + + return result + def build_event_properties( log_record: dict[str, Any], @@ -103,46 +134,78 @@ def build_event_properties( if attributes.get("gen_ai.usage.output_tokens") is not None: properties["$ai_output_tokens"] = attributes["gen_ai.usage.output_tokens"] - # Message content - handle v2 structured log events + # Handle v2 structured log events (message content from body) # v2 instrumentation sends logs with event names like: # - gen_ai.user.message (body: {"content": "..."}) + # - gen_ai.system.message (body: {"content": "..."}) # - gen_ai.assistant.message (body: {"content": "..."} or {"tool_calls": [...]}) # - gen_ai.choice (body: {"index": 0, "finish_reason": "stop", "message": {...}}) + event_name = attributes.get("event.name", "").lower() + + import logging + + logger = logging.getLogger(__name__) + logger.info( + "v2_log_event_debug", + extra={"event_name": event_name, "body_keys": list(body.keys()) if isinstance(body, dict) else None}, + ) - # Handle v2 message events - # v2 uses LogRecord.event_name (in attributes) and structured body if isinstance(body, dict): - # v2 user/system messages: {"content": "..."} - if "content" in body and isinstance(body["content"], str): - content_text = body["content"] - # Determine if input or output based on context - # For now, store as generic message - the span will have role info - properties["$ai_message_content"] = content_text - - # v2 assistant messages with tool calls: {"tool_calls": [...]} - if "tool_calls" in body and isinstance(body["tool_calls"], list): - properties["$ai_tool_calls"] = body["tool_calls"] - - # v2 choice events: {"index": 0, "finish_reason": "stop", "message": {...}} - if "message" in body and isinstance(body["message"], dict): + # User/system messages: {"content": "..."} + if "gen_ai.user.message" in event_name or "gen_ai.system.message" in event_name: + if "content" in body: + role = "system" if "system" in event_name else "user" + properties["$ai_input"] = [{"role": role, "content": body["content"]}] + + # Assistant messages: {"content": "..."} or {"tool_calls": [...]} + elif "gen_ai.assistant.message" in event_name: + message = {"role": "assistant"} + if "content" in body: + message["content"] = body["content"] + if "tool_calls" in body: + message["tool_calls"] = body["tool_calls"] + properties["$ai_output_choices"] = [message] + + # Choice events: {"index": 0, "finish_reason": "stop", "message": {...}} + elif "gen_ai.choice" in event_name and "message" in body: message_obj = body["message"] + choice = {"role": message_obj.get("role", "assistant")} if "content" in message_obj: - properties["$ai_output_content"] = message_obj["content"] + choice["content"] = message_obj["content"] if "tool_calls" in message_obj: - properties["$ai_tool_calls"] = message_obj["tool_calls"] - if "role" in message_obj: - properties["$ai_message_role"] = message_obj["role"] + choice["tool_calls"] = message_obj["tool_calls"] if "finish_reason" in body: - properties["$ai_finish_reason"] = body["finish_reason"] - - # Fallback: legacy v1-style attributes - if attributes.get("gen_ai.prompt"): - properties["$ai_input"] = stringify_content(attributes["gen_ai.prompt"]) - elif attributes.get("message.content"): - properties["$ai_input"] = stringify_content(attributes["message.content"]) - - if attributes.get("gen_ai.completion"): - properties["$ai_output_choices"] = stringify_content(attributes["gen_ai.completion"]) + choice["finish_reason"] = body["finish_reason"] + properties["$ai_output_choices"] = [choice] + + # Fallback: Handle gen_ai.prompt/completion from span attributes (v1-style) + if "$ai_input" not in properties and attributes.get("gen_ai.prompt"): + prompt = attributes["gen_ai.prompt"] + if isinstance(prompt, str): + try: + parsed = json.loads(prompt) + properties["$ai_input"] = parsed if isinstance(parsed, list) else [{"role": "user", "content": prompt}] + except (json.JSONDecodeError, TypeError): + properties["$ai_input"] = [{"role": "user", "content": prompt}] + elif isinstance(prompt, list): + properties["$ai_input"] = prompt + elif isinstance(prompt, dict): + properties["$ai_input"] = [prompt] + + if "$ai_output_choices" not in properties and attributes.get("gen_ai.completion"): + completion = attributes["gen_ai.completion"] + if isinstance(completion, str): + try: + parsed = json.loads(completion) + properties["$ai_output_choices"] = ( + parsed if isinstance(parsed, list) else [{"role": "assistant", "content": completion}] + ) + except (json.JSONDecodeError, TypeError): + properties["$ai_output_choices"] = [{"role": "assistant", "content": completion}] + elif isinstance(completion, list): + properties["$ai_output_choices"] = completion + elif isinstance(completion, dict): + properties["$ai_output_choices"] = [completion] # Severity if log_record.get("severity_number"): @@ -193,6 +256,11 @@ def determine_event_type(log_record: dict[str, Any], attributes: dict[str, Any]) """Determine AI event type from log record.""" event_name = attributes.get("event.name", "").lower() + # v2 instrumentation events (gen_ai.user.message, gen_ai.assistant.message, gen_ai.choice) + # These should be $ai_generation events to merge with trace span data + if "gen_ai." in event_name and ("message" in event_name or "choice" in event_name): + return "$ai_generation" + # Check event name for GenAI events if "prompt" in event_name or "input" in event_name: return "$ai_generation" diff --git a/products/llm_analytics/backend/api/otel/test_event_merger.py b/products/llm_analytics/backend/api/otel/test_event_merger.py new file mode 100644 index 000000000000..abee465b34e8 --- /dev/null +++ b/products/llm_analytics/backend/api/otel/test_event_merger.py @@ -0,0 +1,110 @@ +""" +Unit test for event_merger bidirectional merging logic. +""" + +import sys +from pathlib import Path + +# Add parent directory to path for imports +sys.path.insert(0, str(Path(__file__).parent)) + +from event_merger import _event_cache, cache_and_merge_properties + + +def test_logs_first_then_traces(): + """Test scenario: logs arrive before traces (rare but possible).""" + _event_cache.clear() + + # 1. Log arrives first + log_props = {"$ai_message_content": "Hello from user", "$ai_trace_id": "trace123", "$ai_span_id": "span456"} + result = cache_and_merge_properties("trace123", "span456", log_props, send_immediately=False) + + assert result is None, "Logs should not return properties (they don't send)" + assert ("trace123", "span456") in _event_cache, "Log should be cached" + + # 2. Trace arrives second + trace_props = {"$ai_model": "gpt-4", "$ai_input_tokens": 10, "$ai_trace_id": "trace123", "$ai_span_id": "span456"} + result = cache_and_merge_properties("trace123", "span456", trace_props, send_immediately=True) + + assert result is not None, "Trace should return merged properties" + assert "$ai_message_content" in result, "Should include log message content" + assert "$ai_model" in result, "Should include trace model" + assert result["$ai_input_tokens"] == 10, "Should include trace tokens" + assert ("trace123", "span456") not in _event_cache, "Cache should be cleaned up after merge" + + +def test_traces_first_then_logs(): + """Test scenario: traces arrive before logs (now waits for logs!).""" + import threading + + _event_cache.clear() + + # Simulate logs arriving 50ms after trace starts processing + def cache_logs_with_delay(): + import time + + time.sleep(0.05) # 50ms delay + log_props = {"$ai_message_content": "Hello from user", "$ai_trace_id": "trace789", "$ai_span_id": "span012"} + cache_and_merge_properties("trace789", "span012", log_props, send_immediately=False) + + # Start background thread to cache logs + log_thread = threading.Thread(target=cache_logs_with_delay) + log_thread.start() + + # 1. Trace arrives first and waits for logs + trace_props = {"$ai_model": "gpt-4", "$ai_input_tokens": 10, "$ai_trace_id": "trace789", "$ai_span_id": "span012"} + result = cache_and_merge_properties("trace789", "span012", trace_props, send_immediately=True) + + # Wait for background thread + log_thread.join() + + assert result is not None, "Trace should return merged properties" + assert "$ai_message_content" in result, "Should HAVE message content (waited for logs)" + assert "$ai_model" in result, "Should include trace model" + assert result["$ai_input_tokens"] == 10, "Should include trace tokens" + assert ("trace789", "span012") not in _event_cache, "Cache should be cleaned up after merge" + + +def test_property_override_order(): + """Test that trace properties override log properties on conflicts.""" + _event_cache.clear() + + # 1. Log with model info + log_props = {"$ai_model": "gpt-3.5", "$ai_message_content": "Hello"} + cache_and_merge_properties("trace111", "span222", log_props, send_immediately=False) + + # 2. Trace with different model (should override) + trace_props = {"$ai_model": "gpt-4", "$ai_input_tokens": 10} + result = cache_and_merge_properties("trace111", "span222", trace_props, send_immediately=True) + + assert result["$ai_model"] == "gpt-4", "Trace properties should override log properties" + assert result["$ai_message_content"] == "Hello", "Should keep non-conflicting log properties" + assert result["$ai_input_tokens"] == 10, "Should include trace-only properties" + + +def test_trace_timeout_no_logs(): + """Test scenario: trace waits but logs never arrive (timeout).""" + _event_cache.clear() + + import time + + start = time.time() + + # Trace arrives, waits 1500ms, but logs never arrive + trace_props = {"$ai_model": "gpt-4", "$ai_input_tokens": 10, "$ai_trace_id": "trace999", "$ai_span_id": "span999"} + result = cache_and_merge_properties("trace999", "span999", trace_props, send_immediately=True) + + elapsed = time.time() - start + + assert result is not None, "Trace should return properties after timeout" + assert result == trace_props, "Should return trace properties unchanged (no logs arrived)" + assert "$ai_message_content" not in result, "Should NOT have message content (logs never arrived)" + assert elapsed >= 1.5, f"Should have waited ~1500ms, waited {elapsed*1000:.1f}ms" + assert elapsed < 1.6, f"Should not wait too long, waited {elapsed*1000:.1f}ms" + + +if __name__ == "__main__": + test_logs_first_then_traces() + test_traces_first_then_logs() + test_property_override_order() + test_trace_timeout_no_logs() diff --git a/products/llm_analytics/backend/api/otel/transformer.py b/products/llm_analytics/backend/api/otel/transformer.py index c2c919b892fe..3f444b5c0e9a 100644 --- a/products/llm_analytics/backend/api/otel/transformer.py +++ b/products/llm_analytics/backend/api/otel/transformer.py @@ -14,6 +14,7 @@ from .conventions.genai import extract_genai_attributes, has_genai_attributes from .conventions.posthog_native import extract_posthog_native_attributes, has_posthog_attributes +from .event_merger import cache_and_merge_properties OTEL_TRANSFORMER_VERSION = "1.0.0" @@ -28,7 +29,7 @@ def transform_span_to_ai_event( resource: dict[str, Any], scope: dict[str, Any], baggage: dict[str, str] | None = None, -) -> dict[str, Any]: +) -> dict[str, Any] | None: """ Transform a single OTel span to PostHog AI event. @@ -39,11 +40,12 @@ def transform_span_to_ai_event( baggage: Baggage context (session_id, etc.) Returns: - PostHog AI event dict with: + PostHog AI event dict OR None if this is first arrival (cached, waiting for logs): - event: Event type ($ai_generation, $ai_span, etc.) - distinct_id: User identifier - timestamp: ISO 8601 timestamp - properties: AI event properties + - uuid: Event UUID (for deduplication with log events) """ baggage = baggage or {} @@ -57,6 +59,19 @@ def transform_span_to_ai_event( # Build AI event properties properties = build_event_properties(span, merged_attrs, resource, scope, baggage) + # True bidirectional merge with Redis (no blocking) + # First arrival caches, second arrival merges and sends + trace_id = span.get("trace_id", "") + span_id = span.get("span_id", "") + if trace_id and span_id: + merged = cache_and_merge_properties(trace_id, span_id, properties, is_trace=True) + if merged is None: + # This is first arrival - trace cached, waiting for logs + # Don't send this event yet + return None + # Second arrival - logs already cached, merged contains complete event + properties = merged + # Determine event type event_type = determine_event_type(span, merged_attrs) @@ -66,13 +81,30 @@ def transform_span_to_ai_event( # Get distinct_id distinct_id = extract_distinct_id(resource, baggage) - return { + # Generate consistent UUID from trace_id + span_id for deduplication + # This allows log events and trace events for the same span to merge + import uuid + + trace_id = span.get("trace_id", "") + span_id = span.get("span_id", "") + event_uuid = None + if trace_id and span_id: + # Create deterministic UUID from trace_id + span_id + namespace = uuid.UUID("00000000-0000-0000-0000-000000000000") + event_uuid = str(uuid.uuid5(namespace, f"{trace_id}:{span_id}")) + + result = { "event": event_type, "distinct_id": distinct_id, "timestamp": timestamp, "properties": properties, } + if event_uuid: + result["uuid"] = event_uuid + + return result + def build_event_properties( span: dict[str, Any], @@ -131,9 +163,9 @@ def build_event_properties( if attrs.get("output_tokens") is not None: properties["$ai_output_tokens"] = attrs["output_tokens"] if attrs.get("cache_read_tokens") is not None: - properties["$ai_cache_read_tokens"] = attrs["cache_read_tokens"] + properties["$ai_cache_read_input_tokens"] = attrs["cache_read_tokens"] if attrs.get("cache_write_tokens") is not None: - properties["$ai_cache_write_tokens"] = attrs["cache_write_tokens"] + properties["$ai_cache_creation_input_tokens"] = attrs["cache_write_tokens"] # Cost if attrs.get("input_cost_usd") is not None: @@ -151,7 +183,7 @@ def build_event_properties( if is_error: properties["$ai_is_error"] = is_error if error_message: - properties["$ai_error_message"] = error_message + properties["$ai_error"] = error_message # Model parameters if attrs.get("temperature") is not None: @@ -162,13 +194,41 @@ def build_event_properties( properties["$ai_stream"] = attrs["stream"] # Content (handle both direct input/output and prompt/completion) + # Ensure $ai_input is array of messages content_input = attrs.get("input") or attrs.get("prompt") if content_input: - properties["$ai_input"] = stringify_content(content_input) - + if isinstance(content_input, list): + properties["$ai_input"] = content_input + elif isinstance(content_input, dict): + properties["$ai_input"] = [content_input] + elif isinstance(content_input, str): + try: + parsed = json.loads(content_input) + properties["$ai_input"] = ( + parsed if isinstance(parsed, list) else [{"role": "user", "content": content_input}] + ) + except (json.JSONDecodeError, TypeError): + properties["$ai_input"] = [{"role": "user", "content": content_input}] + else: + properties["$ai_input"] = [{"role": "user", "content": str(content_input)}] + + # Ensure $ai_output_choices is array of choices content_output = attrs.get("output") or attrs.get("completion") if content_output: - properties["$ai_output_choices"] = stringify_content(content_output) + if isinstance(content_output, list): + properties["$ai_output_choices"] = content_output + elif isinstance(content_output, dict): + properties["$ai_output_choices"] = [content_output] + elif isinstance(content_output, str): + try: + parsed = json.loads(content_output) + properties["$ai_output_choices"] = ( + parsed if isinstance(parsed, list) else [{"role": "assistant", "content": content_output}] + ) + except (json.JSONDecodeError, TypeError): + properties["$ai_output_choices"] = [{"role": "assistant", "content": content_output}] + else: + properties["$ai_output_choices"] = [{"role": "assistant", "content": str(content_output)}] # Span name if span.get("name"): From c3211d15ffcc5e3e8a90fa8d8935042494857606 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Fri, 21 Nov 2025 00:43:34 +0000 Subject: [PATCH 14/34] fix(otel): accumulate all v2 logs before merging to prevent race condition MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Problem: v2 instrumentation sends multiple log events (user + assistant messages) in the same HTTP request. Processing them sequentially caused a race condition where: 1. Log 1 (user message) caches 2. Trace arrives (parallel thread), merges with Log 1, sends event 3. Log 2 (assistant message) arrives, finds no trace (already consumed), caches forever Result: Events missing $ai_output_choices from assistant messages Solution: Group all logs by (trace_id, span_id) and accumulate their properties atomically BEFORE calling the event merger. This ensures all log content is merged together in a single operation, preventing the trace from consuming partial logs. Testing: Verified with real OpenAI API calls that v2 events now correctly contain both $ai_input (from user message logs) and $ai_output_choices (from assistant logs). 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .../backend/api/otel/ingestion.py | 72 +++++++++++++++++-- 1 file changed, 67 insertions(+), 5 deletions(-) diff --git a/products/llm_analytics/backend/api/otel/ingestion.py b/products/llm_analytics/backend/api/otel/ingestion.py index d2749481a563..a5ae26c3fdc1 100644 --- a/products/llm_analytics/backend/api/otel/ingestion.py +++ b/products/llm_analytics/backend/api/otel/ingestion.py @@ -35,7 +35,13 @@ from posthog.models import Team from .logs_parser import parse_otlp_logs_request -from .logs_transformer import transform_log_to_ai_event +from .logs_transformer import ( + build_event_properties, + calculate_timestamp, + determine_event_type, + extract_distinct_id, + transform_log_to_ai_event, +) from .parser import parse_baggage_header, parse_otlp_request from .transformer import transform_span_to_ai_event @@ -688,6 +694,10 @@ def transform_logs_to_ai_events(parsed_request: dict[str, Any]) -> list[dict[str """ Transform OTel log records to PostHog AI events. + CRITICAL: In v2 instrumentation, multiple log events (user message, assistant message, etc.) + arrive in the SAME HTTP request. We must accumulate ALL logs for the same span BEFORE + calling the event merger to avoid race conditions where the trace consumes partial logs. + Note: Returns only events ready to send. Events that are first arrivals (cached, waiting for traces) are filtered out. """ @@ -695,10 +705,62 @@ def transform_logs_to_ai_events(parsed_request: dict[str, Any]) -> list[dict[str resource = parsed_request.get("resource", {}) scope = parsed_request.get("scope", {}) - events = [] + # Group logs by (trace_id, span_id) to accumulate them before merging + from collections import defaultdict + + logs_by_span = defaultdict(list) + for log_record in logs: - event = transform_log_to_ai_event(log_record, resource, scope) - if event is not None: # Filter out first arrivals (cached, waiting for trace) - events.append(event) + trace_id = log_record.get("trace_id", "") + span_id = log_record.get("span_id", "") + if trace_id and span_id: + logs_by_span[(trace_id, span_id)].append(log_record) + else: + # No trace/span ID - process individually + logs_by_span[(None, None)].append(log_record) + + events = [] + + # Process each span's logs together + for (trace_id, span_id), span_logs in logs_by_span.items(): + if trace_id and span_id: + # Accumulate properties from all logs for this span + accumulated_props = {} + for log_record in span_logs: + props = build_event_properties(log_record, log_record.get("attributes", {}), resource, scope) + # Merge properties (later logs override earlier ones) + accumulated_props = {**accumulated_props, **props} + + # Now call event merger once with all accumulated properties + from .event_merger import cache_and_merge_properties + + merged = cache_and_merge_properties(trace_id, span_id, accumulated_props, is_trace=False) + + if merged is not None: + # Ready to send - create event + event_type = determine_event_type(span_logs[0], span_logs[0].get("attributes", {})) + timestamp = calculate_timestamp(span_logs[0]) + distinct_id = extract_distinct_id(resource, span_logs[0].get("attributes", {})) + + # Generate consistent UUID from trace_id + span_id + import uuid + + namespace = uuid.UUID("00000000-0000-0000-0000-000000000000") + event_uuid = str(uuid.uuid5(namespace, f"{trace_id}:{span_id}")) + + event = { + "event": event_type, + "distinct_id": distinct_id, + "timestamp": timestamp, + "properties": merged, + "uuid": event_uuid, + } + events.append(event) + else: + # No trace/span ID - process logs individually (shouldn't happen in normal v2) + for log_record in span_logs: + event = transform_log_to_ai_event(log_record, resource, scope) + if event is not None: + events.append(event) return events From b3d58f3458642f2bd237205ceeedac2487b42e67 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Fri, 21 Nov 2025 00:51:56 +0000 Subject: [PATCH 15/34] fix(llm-analytics): Skip event merger for v1 OTEL spans v1 OpenTelemetry instrumentation puts complete data (prompts, completions, and metadata) in span attributes (gen_ai.prompt, gen_ai.completion), unlike v2 which separates metadata (traces) and content (logs). Previously, v1 spans were cached in Redis waiting for logs that never arrive, causing events to expire without being sent. This change detects v1 spans by checking for gen_ai.prompt/completion attributes and skips the event merger, sending v1 events immediately. v2 spans continue to use the event merger for bidirectional merging with logs. --- .../backend/api/otel/transformer.py | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/products/llm_analytics/backend/api/otel/transformer.py b/products/llm_analytics/backend/api/otel/transformer.py index 3f444b5c0e9a..f44d0b68abaa 100644 --- a/products/llm_analytics/backend/api/otel/transformer.py +++ b/products/llm_analytics/backend/api/otel/transformer.py @@ -59,18 +59,24 @@ def transform_span_to_ai_event( # Build AI event properties properties = build_event_properties(span, merged_attrs, resource, scope, baggage) - # True bidirectional merge with Redis (no blocking) - # First arrival caches, second arrival merges and sends - trace_id = span.get("trace_id", "") - span_id = span.get("span_id", "") - if trace_id and span_id: - merged = cache_and_merge_properties(trace_id, span_id, properties, is_trace=True) - if merged is None: - # This is first arrival - trace cached, waiting for logs - # Don't send this event yet - return None - # Second arrival - logs already cached, merged contains complete event - properties = merged + # Detect v1 vs v2 instrumentation: + # v1: Everything in span attributes (gen_ai.prompt, gen_ai.completion) - send immediately + # v2: Metadata in span, content in logs - use event merger + is_v1_span = bool(merged_attrs.get("gen_ai.prompt") or merged_attrs.get("gen_ai.completion")) + + if not is_v1_span: + # v2 instrumentation - use event merger for bidirectional merge with logs + trace_id = span.get("trace_id", "") + span_id = span.get("span_id", "") + if trace_id and span_id: + merged = cache_and_merge_properties(trace_id, span_id, properties, is_trace=True) + if merged is None: + # This is first arrival - trace cached, waiting for logs + # Don't send this event yet + return None + # Second arrival - logs already cached, merged contains complete event + properties = merged + # else: v1 span has everything - send immediately without merging # Determine event type event_type = determine_event_type(span, merged_attrs) From 28e12d36b2727125ff832f63f1fc92bba7d4497a Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Fri, 21 Nov 2025 01:02:02 +0000 Subject: [PATCH 16/34] fix(llm-analytics): Fix v1 OTEL span detection The previous v1 detection was checking for 'gen_ai.prompt' and 'gen_ai.completion' in merged_attrs, but v1 actually sends indexed attributes like 'gen_ai.prompt.0.content' which get extracted as 'prompt' and 'completion' (without the 'gen_ai.' prefix). Updated detection to check for 'prompt' or 'completion' in merged_attrs instead, which correctly identifies v1 spans and skips the event merger. Verified with test: v1 events now appear in database with complete $ai_input and $ai_output_choices arrays. --- products/llm_analytics/backend/api/otel/transformer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/products/llm_analytics/backend/api/otel/transformer.py b/products/llm_analytics/backend/api/otel/transformer.py index f44d0b68abaa..09f9d079a946 100644 --- a/products/llm_analytics/backend/api/otel/transformer.py +++ b/products/llm_analytics/backend/api/otel/transformer.py @@ -60,9 +60,9 @@ def transform_span_to_ai_event( properties = build_event_properties(span, merged_attrs, resource, scope, baggage) # Detect v1 vs v2 instrumentation: - # v1: Everything in span attributes (gen_ai.prompt, gen_ai.completion) - send immediately + # v1: Everything in span attributes (extracted as "prompt", "completion") - send immediately # v2: Metadata in span, content in logs - use event merger - is_v1_span = bool(merged_attrs.get("gen_ai.prompt") or merged_attrs.get("gen_ai.completion")) + is_v1_span = bool(merged_attrs.get("prompt") or merged_attrs.get("completion")) if not is_v1_span: # v2 instrumentation - use event merger for bidirectional merge with logs From 3093228daef18c8876a2396c08a275f2a8fec402 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Fri, 21 Nov 2025 08:36:29 +0000 Subject: [PATCH 17/34] docs(llm-analytics): Add comprehensive README for OTEL ingestion Added detailed documentation covering: - Architecture and data flow diagrams - v1 vs v2 instrumentation differences - Component descriptions and responsibilities - Event merger Redis cache mechanics - PostHog AI event schema specification - Testing and troubleshooting guides - Recent bug fixes and solutions This README serves as the primary reference for developers working on or debugging the OTEL ingestion pipeline. --- .../llm_analytics/backend/api/otel/README.md | 329 ++++++++++++++++++ 1 file changed, 329 insertions(+) create mode 100644 products/llm_analytics/backend/api/otel/README.md diff --git a/products/llm_analytics/backend/api/otel/README.md b/products/llm_analytics/backend/api/otel/README.md new file mode 100644 index 000000000000..353c3ba64ba7 --- /dev/null +++ b/products/llm_analytics/backend/api/otel/README.md @@ -0,0 +1,329 @@ +# OpenTelemetry AI Event Ingestion + +This module handles ingestion of OpenTelemetry traces and logs for LLM/AI observability, transforming them into PostHog AI events. + +## Overview + +The OTEL ingestion system receives OpenTelemetry Protocol (OTLP) data via HTTP and transforms it into PostHog AI events that comply with the PostHog LLM Analytics schema. It supports two instrumentation versions (v1 and v2) with different data delivery patterns. + +## Architecture + +```text +OTLP HTTP Request + | + v + ingestion.py (parse protobuf) + | + +---> parser.py (decode spans) + | | + | v + | transformer.py (detect v1/v2, transform spans) + | | + | +---> conventions/ (extract attributes) + | | | + | | +---> posthog_native.py + | | +---> genai.py + | | + | +---> event_merger.py (Redis cache for v2) + | + +---> logs_parser.py (decode logs) + | + v + logs_transformer.py (transform logs) + | + v + event_merger.py (merge with traces for v2) +``` + +## v1 vs v2 Instrumentation + +### v1 (Single Span - Deprecated) + +**Data Pattern**: Everything in span attributes + +- Prompts: `gen_ai.prompt.0.role`, `gen_ai.prompt.0.content` +- Completions: `gen_ai.completion.0.role`, `gen_ai.completion.0.content` +- Metadata: `gen_ai.request.model`, `gen_ai.usage.input_tokens`, etc. + +**Ingestion Flow**: + +1. Span arrives with all data +2. Transformer detects v1 (presence of `prompt` or `completion` in extracted attributes) +3. Skip event merger - send immediately +4. Result: Single complete event + +**Package**: `opentelemetry-instrumentation-openai` (v1) + +### v2 (Traces + Logs - Current) + +**Data Pattern**: Separated metadata and content + +- Traces: Model, tokens, timing, structure +- Logs: Message content (user prompts, assistant responses) + +**Ingestion Flow**: + +1. Trace arrives first (usually) - cached in Redis +2. Logs arrive (multiple per span) - accumulated atomically +3. Event merger combines trace + logs +4. Result: Single complete event with all data + +**Package**: `opentelemetry-instrumentation-openai-v2` + +**Critical Implementation Detail**: v2 sends multiple log events (user message, assistant message, tool calls, etc.) in a single HTTP request. The ingestion system MUST accumulate all logs for the same (trace_id, span_id) before calling the event merger to prevent race conditions. + +## Components + +### ingestion.py + +Main entry point for OTLP HTTP requests. Handles: + +- Protobuf parsing +- Routing to trace or log transformers +- **v2 Log Accumulation**: Groups logs by (trace_id, span_id) before merging + +### transformer.py + +Transforms OTel spans to PostHog AI events. Key features: + +- Waterfall attribute extraction (PostHog native > GenAI conventions) +- **v1/v2 Detection**: Checks for `prompt`/`completion` in extracted attributes +- Event type determination ($ai_generation, $ai_embedding, $ai_trace, $ai_span) +- Timestamp and latency calculation + +### logs_transformer.py + +Transforms OTel log records to AI event properties. Extracts: + +- Message content from log body +- Event metadata from log attributes +- Works with event_merger for v2 ingestion + +### event_merger.py + +Redis-based non-blocking cache for v2 trace/log merging. Features: + +- **Bidirectional merge**: Either traces or logs can arrive first +- **Atomic operations**: Thread-safe using Redis transactions +- **TTL-based cleanup**: 60-second expiration prevents orphaned entries +- **First arrival caches, second arrival merges and returns complete event** + +### parser.py + +Decodes OTLP protobuf spans into Python dictionaries. Handles: + +- Trace/span ID conversion (bytes to hex) +- Attribute type mapping +- Timestamp conversion + +### logs_parser.py + +Decodes OTLP protobuf log records. Similar to parser.py but for logs. + +### conventions/ + +Attribute extraction modules following semantic conventions: + +#### posthog_native.py + +Extracts PostHog-native attributes (highest priority): + +- `posthog.ai.model`, `posthog.ai.provider` +- `posthog.ai.input_tokens`, `posthog.ai.output_tokens` +- `posthog.ai.input`, `posthog.ai.output` +- Cost attributes, session IDs, etc. + +#### genai.py + +Extracts GenAI semantic convention attributes (fallback): + +- `gen_ai.request.model`, `gen_ai.system` +- `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens` +- **Indexed messages**: `gen_ai.prompt.0.role`, `gen_ai.prompt.0.content` + +## PostHog AI Event Schema + +All transformed events comply with the PostHog LLM Analytics schema: + +**Required Properties**: + +- `$ai_input`: Array of message objects with `role` and `content` +- `$ai_output_choices`: Array of choice objects with `role` and `content` +- `$ai_model`: Model identifier (e.g., "gpt-4o-mini") +- `$ai_provider`: Provider name (e.g., "openai") +- `$ai_input_tokens`: Integer token count +- `$ai_output_tokens`: Integer token count + +**Optional Properties**: + +- `$ai_trace_id`, `$ai_span_id`, `$ai_parent_id` +- `$ai_session_id`, `$ai_generation_id` +- `$ai_latency`: Duration in seconds +- `$ai_total_cost_usd`, `$ai_input_cost_usd`, `$ai_output_cost_usd` +- `$ai_temperature`, `$ai_max_tokens`, `$ai_stream` +- `$ai_tools`: Array of tool definitions +- Error tracking: `$ai_is_error`, `$ai_error` + +## Event Types + +The transformer determines event types based on span characteristics: + +- **$ai_generation**: LLM chat/completion requests (has model + tokens + input) +- **$ai_embedding**: Embedding generation requests (operation_name = "embedding") +- **$ai_trace**: Root spans (no parent_span_id) +- **$ai_span**: Generic spans (default) + +## Testing + +### Unit Tests + +```bash +pytest products/llm_analytics/backend/api/otel/test/ +``` + +### Integration Tests + +Test scripts available in `/tmp/`: + +- `test_posthog_sdk.py`: PostHog Python SDK with AI wrapper +- `test_otel_v1.py`: v1 instrumentation test +- `test_otel_v2.py`: v2 instrumentation test +- `compare_events.py`: Database comparison of all three methods + +Run with llm-analytics-apps venv: + +```bash +source /Users/andrewmaguire/Documents/GitHub/llm-analytics-apps/.env +source /Users/andrewmaguire/Documents/GitHub/llm-analytics-apps/python/venv/bin/activate +python /tmp/test_otel_v2.py +``` + +Check results: + +```bash +python /tmp/compare_events.py +``` + +## Troubleshooting + +### v1 Events Not Appearing + +**Symptom**: v1 spans cached in Redis but never sent + +**Cause**: Detection logic not identifying spans as v1 + +**Debug**: + +```python +import redis +r = redis.Redis(host='localhost', port=6379, db=0, decode_responses=True) +keys = r.keys('otel_merge:*') +# Check cached data for prompt/completion attributes +``` + +**Fix**: Ensure transformer checks for extracted `prompt`/`completion` attributes, not raw `gen_ai.prompt` + +### v2 Missing Output + +**Symptom**: Events have `$ai_input` but no `$ai_output_choices` + +**Cause**: Multiple logs processed sequentially, race condition + +**Solution**: Already fixed - logs are accumulated by (trace_id, span_id) before merging + +### Events Stuck in Redis + +**Symptom**: Redis cache grows over time + +**Cause**: Traces waiting for logs that never arrive (or vice versa) + +**Debug**: + +```python +# Check cache contents +for key in r.keys('otel_merge:*'): + ttl = r.ttl(key) + print(f"{key}: TTL={ttl}s") +``` + +**Fix**: TTL is 60 seconds - entries auto-expire. If persistent, check instrumentation configuration. + +### Schema Violations + +**Symptom**: Events have unexpected property names + +**Cause**: Attribute extraction waterfall not working correctly + +**Debug**: Check conventions/ extractors and transformer.py property mapping + +## Redis Cache Management + +The event merger uses Redis keys with pattern: `otel_merge:{type}:{trace_id}:{span_id}` + +**Key Types**: + +- `otel_merge:trace:{trace_id}:{span_id}`: Cached trace waiting for logs +- `otel_merge:logs:{trace_id}:{span_id}`: Cached logs waiting for trace + +**TTL**: 60 seconds (auto-cleanup) + +**Manual Cleanup**: + +```python +import redis +r = redis.Redis(host='localhost', port=6379, db=0, decode_responses=True) +r.delete(*r.keys('otel_merge:*')) +``` + +## API Endpoints + +**Traces**: `POST /api/projects/{project_id}/ai/otel/v1/traces` + +- Accepts: `application/x-protobuf` +- Authorization: `Bearer {project_api_key}` + +**Logs**: `POST /api/projects/{project_id}/ai/otel/v1/logs` + +- Accepts: `application/x-protobuf` +- Authorization: `Bearer {project_api_key}` + +## Performance Considerations + +- **Redis Operations**: Atomic using transactions (WATCH/MULTI/EXEC) +- **Non-blocking**: Returns None on first arrival (caches), complete event on second +- **TTL Cleanup**: Automatic expiration prevents memory growth +- **Log Accumulation**: All logs for same span processed atomically (prevents N+1 Redis calls) + +## Recent Bug Fixes + +### v2 Log Accumulation (2025-11-21) + +**Problem**: Race condition when multiple logs arrive in same HTTP request + +**Solution**: Group and accumulate all logs by (trace_id, span_id) before calling event merger + +**Commit**: `cd4d4e500c` + +### v1 Detection (2025-11-21) + +**Problem**: v1 spans incorrectly treated as v2, cached waiting for logs + +**Solution**: Check for extracted `prompt`/`completion` instead of raw `gen_ai.prompt`/`gen_ai.completion` + +**Commits**: `618b4f06ab`, `2e088f790e` + +## Contributing + +When modifying this system: + +1. **Preserve v1/v2 compatibility**: Both versions must work correctly +2. **Test both paths**: Run integration tests for v1 and v2 +3. **Check Redis cache**: Ensure no orphaned entries after changes +4. **Validate schema**: Events must comply with PostHog AI schema +5. **Document changes**: Update this README for significant changes + +## References + +- [OpenTelemetry GenAI Semantic Conventions](https://opentelemetry.io/docs/specs/semconv/gen-ai/) +- [PostHog LLM Analytics Documentation](https://posthog.com/docs/ai-engineering) +- [OTLP Specification](https://opentelemetry.io/docs/specs/otlp/) From 1d6759b812b8579abef22dad573f913b8590a24c Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Fri, 21 Nov 2025 08:39:57 +0000 Subject: [PATCH 18/34] docs(llm-analytics): Refactor OTEL README to be standalone Rewrote README to focus on current implementation and design decisions rather than development-specific context: - Removed references to specific bugs encountered during development - Removed troubleshooting for transient development issues - Removed references to temporary test scripts - Added Design Decisions section explaining architectural choices - Added Extending the System section with examples - Reorganized content to flow from overview to architecture to usage - Focused on what the system does and why, not historical problems The README now stands alone as documentation for the current codebase. --- .../llm_analytics/backend/api/otel/README.md | 317 +++++++----------- 1 file changed, 130 insertions(+), 187 deletions(-) diff --git a/products/llm_analytics/backend/api/otel/README.md b/products/llm_analytics/backend/api/otel/README.md index 353c3ba64ba7..43719181e6f4 100644 --- a/products/llm_analytics/backend/api/otel/README.md +++ b/products/llm_analytics/backend/api/otel/README.md @@ -1,10 +1,10 @@ # OpenTelemetry AI Event Ingestion -This module handles ingestion of OpenTelemetry traces and logs for LLM/AI observability, transforming them into PostHog AI events. +This module transforms OpenTelemetry traces and logs into PostHog AI events for LLM observability. ## Overview -The OTEL ingestion system receives OpenTelemetry Protocol (OTLP) data via HTTP and transforms it into PostHog AI events that comply with the PostHog LLM Analytics schema. It supports two instrumentation versions (v1 and v2) with different data delivery patterns. +The OTEL ingestion pipeline accepts OpenTelemetry Protocol (OTLP) data via HTTP endpoints and converts it into PostHog AI events that comply with the PostHog LLM Analytics schema. The system supports two instrumentation patterns (v1 and v2) that differ in how they deliver trace and log data. ## Architecture @@ -35,295 +35,238 @@ OTLP HTTP Request event_merger.py (merge with traces for v2) ``` -## v1 vs v2 Instrumentation +## Instrumentation Patterns -### v1 (Single Span - Deprecated) +### v1 Instrumentation -**Data Pattern**: Everything in span attributes +**Data Model**: All data in span attributes + +v1 instrumentation sends complete LLM call data within span attributes using indexed fields: - Prompts: `gen_ai.prompt.0.role`, `gen_ai.prompt.0.content` - Completions: `gen_ai.completion.0.role`, `gen_ai.completion.0.content` - Metadata: `gen_ai.request.model`, `gen_ai.usage.input_tokens`, etc. -**Ingestion Flow**: - -1. Span arrives with all data -2. Transformer detects v1 (presence of `prompt` or `completion` in extracted attributes) -3. Skip event merger - send immediately -4. Result: Single complete event +**Processing**: When a span contains `prompt` or `completion` attributes (after extraction), the transformer recognizes it as v1 and sends the event immediately without caching. This works because v1 spans are self-contained. -**Package**: `opentelemetry-instrumentation-openai` (v1) +**Package**: `opentelemetry-instrumentation-openai` -### v2 (Traces + Logs - Current) +### v2 Instrumentation -**Data Pattern**: Separated metadata and content +**Data Model**: Separated metadata and content -- Traces: Model, tokens, timing, structure -- Logs: Message content (user prompts, assistant responses) +v2 instrumentation splits LLM call data across two channels: -**Ingestion Flow**: +- Traces: Model name, token counts, timing, span structure +- Logs: Message content (user prompts, assistant completions, tool calls) -1. Trace arrives first (usually) - cached in Redis -2. Logs arrive (multiple per span) - accumulated atomically -3. Event merger combines trace + logs -4. Result: Single complete event with all data +**Processing**: The event merger provides bidirectional merging - either traces or logs can arrive first. The first arrival caches data in Redis, the second arrival merges and returns a complete event. Multiple log events for the same span are accumulated atomically before merging to ensure completeness. **Package**: `opentelemetry-instrumentation-openai-v2` -**Critical Implementation Detail**: v2 sends multiple log events (user message, assistant message, tool calls, etc.) in a single HTTP request. The ingestion system MUST accumulate all logs for the same (trace_id, span_id) before calling the event merger to prevent race conditions. +**Design Rationale**: Separating traces and logs allows v2 to stream content while maintaining trace context, but requires a merge layer to recombine data into complete events. ## Components ### ingestion.py -Main entry point for OTLP HTTP requests. Handles: - -- Protobuf parsing -- Routing to trace or log transformers -- **v2 Log Accumulation**: Groups logs by (trace_id, span_id) before merging +Main entry point for OTLP HTTP requests. Parses protobuf payloads and routes to appropriate transformers. For v2 logs, groups all log records by (trace_id, span_id) before processing to ensure atomic accumulation of multi-log spans. ### transformer.py -Transforms OTel spans to PostHog AI events. Key features: - -- Waterfall attribute extraction (PostHog native > GenAI conventions) -- **v1/v2 Detection**: Checks for `prompt`/`completion` in extracted attributes -- Event type determination ($ai_generation, $ai_embedding, $ai_trace, $ai_span) -- Timestamp and latency calculation +Converts OTel spans to PostHog AI events using a waterfall attribute extraction pattern: -### logs_transformer.py - -Transforms OTel log records to AI event properties. Extracts: +1. Extract PostHog-native attributes (highest priority) +2. Extract GenAI semantic convention attributes (fallback) +3. Merge with PostHog attributes taking precedence -- Message content from log body -- Event metadata from log attributes -- Works with event_merger for v2 ingestion - -### event_merger.py +Determines event type based on span characteristics: -Redis-based non-blocking cache for v2 trace/log merging. Features: +- `$ai_generation`: LLM completion requests (has model, tokens, and input) +- `$ai_embedding`: Embedding requests (operation_name matches embedding patterns) +- `$ai_trace`: Root spans (no parent) +- `$ai_span`: All other spans -- **Bidirectional merge**: Either traces or logs can arrive first -- **Atomic operations**: Thread-safe using Redis transactions -- **TTL-based cleanup**: 60-second expiration prevents orphaned entries -- **First arrival caches, second arrival merges and returns complete event** +Detects v1 vs v2 by checking for `prompt` or `completion` in extracted attributes. v1 spans bypass the event merger. -### parser.py +### logs_transformer.py -Decodes OTLP protobuf spans into Python dictionaries. Handles: +Converts OTel log records to AI event properties. Extracts message content from log body and metadata from log attributes. Designed to work with event_merger for v2 ingestion. -- Trace/span ID conversion (bytes to hex) -- Attribute type mapping -- Timestamp conversion +### event_merger.py -### logs_parser.py +Redis-based non-blocking cache for v2 trace/log coordination. Uses atomic operations (WATCH/MULTI/EXEC) to safely merge data from concurrent arrivals. Keys expire after 60 seconds to prevent orphaned entries. -Decodes OTLP protobuf log records. Similar to parser.py but for logs. +**Merge Logic**: -### conventions/ +- First arrival: Cache data, return None (event not ready) +- Second arrival: Retrieve cached data, merge properties, delete cache, return complete event -Attribute extraction modules following semantic conventions: +**Key Pattern**: `otel_merge:{type}:{trace_id}:{span_id}` -#### posthog_native.py +### parser.py / logs_parser.py -Extracts PostHog-native attributes (highest priority): +Decode OTLP protobuf messages into Python dictionaries. Handle type conversions (bytes to hex for IDs, nanoseconds to seconds for timestamps) and attribute flattening. -- `posthog.ai.model`, `posthog.ai.provider` -- `posthog.ai.input_tokens`, `posthog.ai.output_tokens` -- `posthog.ai.input`, `posthog.ai.output` -- Cost attributes, session IDs, etc. +### conventions/ -#### genai.py +Attribute extraction modules implementing semantic conventions: -Extracts GenAI semantic convention attributes (fallback): +**posthog_native.py**: Extracts PostHog-specific attributes prefixed with `posthog.ai.*`. These take precedence in the waterfall. -- `gen_ai.request.model`, `gen_ai.system` -- `gen_ai.usage.input_tokens`, `gen_ai.usage.output_tokens` -- **Indexed messages**: `gen_ai.prompt.0.role`, `gen_ai.prompt.0.content` +**genai.py**: Extracts OpenTelemetry GenAI semantic convention attributes (`gen_ai.*`). Handles indexed message fields by collecting attributes like `gen_ai.prompt.0.role` into structured message arrays. -## PostHog AI Event Schema +## Event Schema -All transformed events comply with the PostHog LLM Analytics schema: +All events conform to the PostHog LLM Analytics schema: -**Required Properties**: +**Core Properties**: -- `$ai_input`: Array of message objects with `role` and `content` -- `$ai_output_choices`: Array of choice objects with `role` and `content` +- `$ai_input`: Array of message objects `[{role: str, content: str}]` +- `$ai_output_choices`: Array of completion objects `[{role: str, content: str}]` - `$ai_model`: Model identifier (e.g., "gpt-4o-mini") - `$ai_provider`: Provider name (e.g., "openai") -- `$ai_input_tokens`: Integer token count -- `$ai_output_tokens`: Integer token count +- `$ai_input_tokens`: Input token count +- `$ai_output_tokens`: Output token count -**Optional Properties**: +**Trace Context**: - `$ai_trace_id`, `$ai_span_id`, `$ai_parent_id` - `$ai_session_id`, `$ai_generation_id` + +**Metrics**: + - `$ai_latency`: Duration in seconds - `$ai_total_cost_usd`, `$ai_input_cost_usd`, `$ai_output_cost_usd` -- `$ai_temperature`, `$ai_max_tokens`, `$ai_stream` -- `$ai_tools`: Array of tool definitions -- Error tracking: `$ai_is_error`, `$ai_error` - -## Event Types -The transformer determines event types based on span characteristics: +**Configuration**: -- **$ai_generation**: LLM chat/completion requests (has model + tokens + input) -- **$ai_embedding**: Embedding generation requests (operation_name = "embedding") -- **$ai_trace**: Root spans (no parent_span_id) -- **$ai_span**: Generic spans (default) +- `$ai_temperature`, `$ai_max_tokens`, `$ai_stream` +- `$ai_tools`: Array of tool definitions -## Testing +**Error Tracking**: -### Unit Tests +- `$ai_is_error`: Boolean +- `$ai_error`: Error message string -```bash -pytest products/llm_analytics/backend/api/otel/test/ -``` +## API Endpoints -### Integration Tests +**Traces**: `POST /api/projects/{project_id}/ai/otel/v1/traces` -Test scripts available in `/tmp/`: +- Content-Type: `application/x-protobuf` +- Authorization: `Bearer {project_api_key}` +- Accepts OTLP trace payloads -- `test_posthog_sdk.py`: PostHog Python SDK with AI wrapper -- `test_otel_v1.py`: v1 instrumentation test -- `test_otel_v2.py`: v2 instrumentation test -- `compare_events.py`: Database comparison of all three methods +**Logs**: `POST /api/projects/{project_id}/ai/otel/v1/logs` -Run with llm-analytics-apps venv: +- Content-Type: `application/x-protobuf` +- Authorization: `Bearer {project_api_key}` +- Accepts OTLP log payloads -```bash -source /Users/andrewmaguire/Documents/GitHub/llm-analytics-apps/.env -source /Users/andrewmaguire/Documents/GitHub/llm-analytics-apps/python/venv/bin/activate -python /tmp/test_otel_v2.py -``` +## Testing -Check results: +Run unit tests: ```bash -python /tmp/compare_events.py +pytest products/llm_analytics/backend/api/otel/test/ ``` -## Troubleshooting - -### v1 Events Not Appearing - -**Symptom**: v1 spans cached in Redis but never sent +Integration testing requires: -**Cause**: Detection logic not identifying spans as v1 +1. Running PostHog instance with OTEL endpoints enabled +2. OpenTelemetry SDK configured to send to local endpoints +3. Redis instance for event merger cache -**Debug**: +Example v2 test configuration: ```python -import redis -r = redis.Redis(host='localhost', port=6379, db=0, decode_responses=True) -keys = r.keys('otel_merge:*') -# Check cached data for prompt/completion attributes +from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter +from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter + +trace_exporter = OTLPSpanExporter( + endpoint=f"{posthog_host}/api/projects/{project_id}/ai/otel/v1/traces", + headers={"Authorization": f"Bearer {api_key}"} +) + +log_exporter = OTLPLogExporter( + endpoint=f"{posthog_host}/api/projects/{project_id}/ai/otel/v1/logs", + headers={"Authorization": f"Bearer {api_key}"} +) ``` -**Fix**: Ensure transformer checks for extracted `prompt`/`completion` attributes, not raw `gen_ai.prompt` +## Design Decisions -### v2 Missing Output +### Waterfall Attribute Extraction -**Symptom**: Events have `$ai_input` but no `$ai_output_choices` +PostHog-native attributes take precedence over GenAI conventions, allowing instrumentation to override standard attributes when needed. This provides flexibility for custom instrumentation while maintaining compatibility with standard OTEL instrumentation. -**Cause**: Multiple logs processed sequentially, race condition +### Non-Blocking Event Merger -**Solution**: Already fixed - logs are accumulated by (trace_id, span_id) before merging +The merger returns None on first arrival rather than blocking. This prevents the ingestion pipeline from waiting on Redis and keeps request processing fast. The tradeoff is that v2 requires two round-trips (trace + logs) before emitting events. -### Events Stuck in Redis +### Atomic Log Accumulation -**Symptom**: Redis cache grows over time +v2 can send multiple log events in a single HTTP request. The ingestion layer groups these by (trace_id, span_id) and accumulates their properties before calling the merger. This prevents race conditions where partial log data gets merged before all logs arrive. -**Cause**: Traces waiting for logs that never arrive (or vice versa) - -**Debug**: - -```python -# Check cache contents -for key in r.keys('otel_merge:*'): - ttl = r.ttl(key) - print(f"{key}: TTL={ttl}s") -``` +### v1/v2 Detection -**Fix**: TTL is 60 seconds - entries auto-expire. If persistent, check instrumentation configuration. +Rather than requiring explicit configuration, the transformer auto-detects instrumentation version by checking for `prompt` or `completion` attributes. This allows both patterns to coexist without configuration. -### Schema Violations +### TTL-Based Cleanup -**Symptom**: Events have unexpected property names +The event merger uses 60-second TTL on cache entries. This automatically cleans up orphaned data from incomplete traces (e.g., lost log packets) without requiring background jobs or manual cleanup. -**Cause**: Attribute extraction waterfall not working correctly +## Extending the System -**Debug**: Check conventions/ extractors and transformer.py property mapping +### Adding New Semantic Conventions -## Redis Cache Management +Create a new extractor in `conventions/`: -The event merger uses Redis keys with pattern: `otel_merge:{type}:{trace_id}:{span_id}` - -**Key Types**: +```python +def extract_custom_attributes(span: dict[str, Any]) -> dict[str, Any]: + attributes = span.get("attributes", {}) + result = {} -- `otel_merge:trace:{trace_id}:{span_id}`: Cached trace waiting for logs -- `otel_merge:logs:{trace_id}:{span_id}`: Cached logs waiting for trace + # Extract custom attributes + if custom_attr := attributes.get("custom.attribute"): + result["custom_field"] = custom_attr -**TTL**: 60 seconds (auto-cleanup) + return result +``` -**Manual Cleanup**: +Add to waterfall in `transformer.py`: ```python -import redis -r = redis.Redis(host='localhost', port=6379, db=0, decode_responses=True) -r.delete(*r.keys('otel_merge:*')) +custom_attrs = extract_custom_attributes(span) +merged_attrs = {**genai_attrs, **posthog_attrs, **custom_attrs} ``` -## API Endpoints +### Supporting New Event Types -**Traces**: `POST /api/projects/{project_id}/ai/otel/v1/traces` - -- Accepts: `application/x-protobuf` -- Authorization: `Bearer {project_api_key}` +Add logic to `determine_event_type()` in `transformer.py`: -**Logs**: `POST /api/projects/{project_id}/ai/otel/v1/logs` - -- Accepts: `application/x-protobuf` -- Authorization: `Bearer {project_api_key}` - -## Performance Considerations - -- **Redis Operations**: Atomic using transactions (WATCH/MULTI/EXEC) -- **Non-blocking**: Returns None on first arrival (caches), complete event on second -- **TTL Cleanup**: Automatic expiration prevents memory growth -- **Log Accumulation**: All logs for same span processed atomically (prevents N+1 Redis calls) - -## Recent Bug Fixes - -### v2 Log Accumulation (2025-11-21) - -**Problem**: Race condition when multiple logs arrive in same HTTP request - -**Solution**: Group and accumulate all logs by (trace_id, span_id) before calling event merger - -**Commit**: `cd4d4e500c` - -### v1 Detection (2025-11-21) - -**Problem**: v1 spans incorrectly treated as v2, cached waiting for logs +```python +def determine_event_type(span: dict[str, Any], attrs: dict[str, Any]) -> str: + op_name = attrs.get("operation_name", "").lower() -**Solution**: Check for extracted `prompt`/`completion` instead of raw `gen_ai.prompt`/`gen_ai.completion` + if op_name == "new_operation": + return "$ai_new_event_type" + # ... existing logic +``` -**Commits**: `618b4f06ab`, `2e088f790e` +### Custom Property Mapping -## Contributing +Extend `build_event_properties()` in `transformer.py` to map additional attributes to event properties. -When modifying this system: +## Performance Characteristics -1. **Preserve v1/v2 compatibility**: Both versions must work correctly -2. **Test both paths**: Run integration tests for v1 and v2 -3. **Check Redis cache**: Ensure no orphaned entries after changes -4. **Validate schema**: Events must comply with PostHog AI schema -5. **Document changes**: Update this README for significant changes +- **Throughput**: Limited by Redis round-trip time for v2 merging +- **Latency**: v1 has single-pass latency, v2 has cache lookup latency +- **Memory**: Redis cache bounded by TTL (60s max retention) +- **Concurrency**: Redis transactions provide safe concurrent merging ## References - [OpenTelemetry GenAI Semantic Conventions](https://opentelemetry.io/docs/specs/semconv/gen-ai/) -- [PostHog LLM Analytics Documentation](https://posthog.com/docs/ai-engineering) - [OTLP Specification](https://opentelemetry.io/docs/specs/otlp/) +- [PostHog AI Engineering Documentation](https://posthog.com/docs/ai-engineering) From 239b1468b58e99643dbcf18ed1895950e41a9e18 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Fri, 21 Nov 2025 11:08:37 +0000 Subject: [PATCH 19/34] fix(llm-analytics): Rewrite event_merger tests for Redis-backed implementation Completely rewrote test suite to match current Redis-backed event merger API: - Removed references to _event_cache (no longer exists) - Changed send_immediately parameter to is_trace boolean - Added mock_redis fixture using unittest.mock - Removed blocking/timeout tests (obsolete with non-blocking design) - Added test for multiple log accumulation (critical v2 feature) - Added test for Redis error fallback - Added test for TTL configuration - Fixed test expectations to match actual merge behavior All 7 tests now pass successfully. --- .../backend/api/otel/test_event_merger.py | 192 ++++++++++++------ 1 file changed, 125 insertions(+), 67 deletions(-) diff --git a/products/llm_analytics/backend/api/otel/test_event_merger.py b/products/llm_analytics/backend/api/otel/test_event_merger.py index abee465b34e8..8120af795c72 100644 --- a/products/llm_analytics/backend/api/otel/test_event_merger.py +++ b/products/llm_analytics/backend/api/otel/test_event_merger.py @@ -1,110 +1,168 @@ """ -Unit test for event_merger bidirectional merging logic. +Unit tests for Redis-backed event_merger bidirectional merging logic. """ -import sys -from pathlib import Path +import json -# Add parent directory to path for imports -sys.path.insert(0, str(Path(__file__).parent)) +import pytest +from unittest.mock import MagicMock, patch -from event_merger import _event_cache, cache_and_merge_properties +from products.llm_analytics.backend.api.otel.event_merger import cache_and_merge_properties -def test_logs_first_then_traces(): - """Test scenario: logs arrive before traces (rare but possible).""" - _event_cache.clear() +@pytest.fixture +def mock_redis(): + """Mock Redis client for testing.""" + redis_mock = MagicMock() + redis_mock.cache = {} # Internal dict to simulate Redis storage + def mock_get(key): + return redis_mock.cache.get(key) + + def mock_setex(key, ttl, value): + redis_mock.cache[key] = value + return True + + def mock_delete(*keys): + for key in keys: + redis_mock.cache.pop(key, None) + return len(keys) + + redis_mock.get.side_effect = mock_get + redis_mock.setex.side_effect = mock_setex + redis_mock.delete.side_effect = mock_delete + + with patch("products.llm_analytics.backend.api.otel.event_merger.get_client", return_value=redis_mock): + yield redis_mock + + +def test_logs_first_then_trace(mock_redis): + """Test scenario: logs arrive before trace.""" # 1. Log arrives first - log_props = {"$ai_message_content": "Hello from user", "$ai_trace_id": "trace123", "$ai_span_id": "span456"} - result = cache_and_merge_properties("trace123", "span456", log_props, send_immediately=False) + log_props = {"$ai_input": [{"role": "user", "content": "Hello"}], "$ai_trace_id": "trace123"} + result = cache_and_merge_properties("trace123", "span456", log_props, is_trace=False) - assert result is None, "Logs should not return properties (they don't send)" - assert ("trace123", "span456") in _event_cache, "Log should be cached" + assert result is None, "First arrival (log) should cache and return None" + assert "otel_merge:logs:trace123:span456" in mock_redis.cache, "Log should be cached" # 2. Trace arrives second - trace_props = {"$ai_model": "gpt-4", "$ai_input_tokens": 10, "$ai_trace_id": "trace123", "$ai_span_id": "span456"} - result = cache_and_merge_properties("trace123", "span456", trace_props, send_immediately=True) + trace_props = {"$ai_model": "gpt-4", "$ai_input_tokens": 10, "$ai_output_tokens": 20} + result = cache_and_merge_properties("trace123", "span456", trace_props, is_trace=True) - assert result is not None, "Trace should return merged properties" - assert "$ai_message_content" in result, "Should include log message content" - assert "$ai_model" in result, "Should include trace model" + assert result is not None, "Second arrival (trace) should return merged properties" + assert "$ai_input" in result, "Should include log content" + assert "$ai_model" in result, "Should include trace metadata" assert result["$ai_input_tokens"] == 10, "Should include trace tokens" - assert ("trace123", "span456") not in _event_cache, "Cache should be cleaned up after merge" + assert "otel_merge:logs:trace123:span456" not in mock_redis.cache, "Cache should be cleaned up" -def test_traces_first_then_logs(): - """Test scenario: traces arrive before logs (now waits for logs!).""" - import threading +def test_trace_first_then_logs(mock_redis): + """Test scenario: trace arrives before logs.""" + # 1. Trace arrives first + trace_props = {"$ai_model": "gpt-4", "$ai_input_tokens": 10, "$ai_output_tokens": 20} + result = cache_and_merge_properties("trace789", "span012", trace_props, is_trace=True) - _event_cache.clear() + assert result is None, "First arrival (trace) should cache and return None" + assert "otel_merge:trace:trace789:span012" in mock_redis.cache, "Trace should be cached" - # Simulate logs arriving 50ms after trace starts processing - def cache_logs_with_delay(): - import time + # 2. Log arrives second + log_props = {"$ai_input": [{"role": "user", "content": "Hello"}]} + result = cache_and_merge_properties("trace789", "span012", log_props, is_trace=False) - time.sleep(0.05) # 50ms delay - log_props = {"$ai_message_content": "Hello from user", "$ai_trace_id": "trace789", "$ai_span_id": "span012"} - cache_and_merge_properties("trace789", "span012", log_props, send_immediately=False) + assert result is not None, "Second arrival (log) should return merged properties" + assert "$ai_input" in result, "Should include log content" + assert "$ai_model" in result, "Should include trace metadata" + assert result["$ai_input_tokens"] == 10, "Should include trace tokens" + assert "otel_merge:trace:trace789:span012" not in mock_redis.cache, "Cache should be cleaned up" - # Start background thread to cache logs - log_thread = threading.Thread(target=cache_logs_with_delay) - log_thread.start() - # 1. Trace arrives first and waits for logs - trace_props = {"$ai_model": "gpt-4", "$ai_input_tokens": 10, "$ai_trace_id": "trace789", "$ai_span_id": "span012"} - result = cache_and_merge_properties("trace789", "span012", trace_props, send_immediately=True) +def test_multiple_logs_accumulation(mock_redis): + """Test that multiple logs accumulate before merging with trace.""" + # 1. First log arrives + log1_props = {"$ai_input": [{"role": "user", "content": "Hello"}]} + result = cache_and_merge_properties("trace111", "span222", log1_props, is_trace=False) - # Wait for background thread - log_thread.join() + assert result is None, "First log should cache and return None" + cached_logs = json.loads(mock_redis.cache["otel_merge:logs:trace111:span222"]) + assert "$ai_input" in cached_logs - assert result is not None, "Trace should return merged properties" - assert "$ai_message_content" in result, "Should HAVE message content (waited for logs)" - assert "$ai_model" in result, "Should include trace model" - assert result["$ai_input_tokens"] == 10, "Should include trace tokens" - assert ("trace789", "span012") not in _event_cache, "Cache should be cleaned up after merge" + # 2. Second log arrives (assistant response) + log2_props = {"$ai_output_choices": [{"role": "assistant", "content": "Hi there"}]} + result = cache_and_merge_properties("trace111", "span222", log2_props, is_trace=False) + assert result is None, "Second log should accumulate and return None (no trace yet)" + cached_logs = json.loads(mock_redis.cache["otel_merge:logs:trace111:span222"]) + assert "$ai_input" in cached_logs, "Should have first log content" + assert "$ai_output_choices" in cached_logs, "Should have second log content" -def test_property_override_order(): - """Test that trace properties override log properties on conflicts.""" - _event_cache.clear() + # 3. Trace arrives - merges with accumulated logs + trace_props = {"$ai_model": "gpt-4", "$ai_input_tokens": 10, "$ai_output_tokens": 20} + result = cache_and_merge_properties("trace111", "span222", trace_props, is_trace=True) + assert result is not None, "Trace should return merged properties with accumulated logs" + assert "$ai_input" in result, "Should include first log" + assert "$ai_output_choices" in result, "Should include second log" + assert "$ai_model" in result, "Should include trace metadata" + assert "otel_merge:logs:trace111:span222" not in mock_redis.cache, "Logs cache cleaned up" + assert "otel_merge:trace:trace111:span222" not in mock_redis.cache, "Trace cache cleaned up" + + +def test_property_precedence(mock_redis): + """Test that trace properties override log properties on conflicts.""" # 1. Log with model info - log_props = {"$ai_model": "gpt-3.5", "$ai_message_content": "Hello"} - cache_and_merge_properties("trace111", "span222", log_props, send_immediately=False) + log_props = {"$ai_model": "gpt-3.5", "$ai_input": [{"role": "user", "content": "Hello"}]} + cache_and_merge_properties("trace333", "span444", log_props, is_trace=False) # 2. Trace with different model (should override) trace_props = {"$ai_model": "gpt-4", "$ai_input_tokens": 10} - result = cache_and_merge_properties("trace111", "span222", trace_props, send_immediately=True) + result = cache_and_merge_properties("trace333", "span444", trace_props, is_trace=True) assert result["$ai_model"] == "gpt-4", "Trace properties should override log properties" - assert result["$ai_message_content"] == "Hello", "Should keep non-conflicting log properties" + assert "$ai_input" in result, "Should keep non-conflicting log properties" assert result["$ai_input_tokens"] == 10, "Should include trace-only properties" -def test_trace_timeout_no_logs(): - """Test scenario: trace waits but logs never arrive (timeout).""" - _event_cache.clear() +def test_redis_error_fallback(mock_redis): + """Test that Redis errors fall back to immediate send.""" + # Simulate Redis error + mock_redis.get.side_effect = Exception("Redis connection failed") + + trace_props = {"$ai_model": "gpt-4", "$ai_input_tokens": 10} + result = cache_and_merge_properties("trace555", "span666", trace_props, is_trace=True) + + # Should return properties immediately on error (no merging) + assert result is not None, "Should fallback to immediate send on Redis error" + assert result == trace_props, "Should return original properties unchanged" + + +def test_cache_keys_are_separate(mock_redis): + """Test that trace and log caches use separate key namespaces.""" + # Cache a trace for span1 + trace_props = {"$ai_model": "gpt-4"} + result = cache_and_merge_properties("trace777", "span888", trace_props, is_trace=True) + + assert result is None, "First arrival should cache" + assert "otel_merge:trace:trace777:span888" in mock_redis.cache - import time + # Cache a log for different span (span2) - should not merge with span1 + log_props = {"$ai_input": [{"role": "user", "content": "Hello"}]} + result = cache_and_merge_properties("trace777", "span999", log_props, is_trace=False) - start = time.time() + assert result is None, "Log for different span should cache separately" + assert "otel_merge:logs:trace777:span999" in mock_redis.cache + assert "otel_merge:trace:trace777:span888" in mock_redis.cache, "Original trace still cached" - # Trace arrives, waits 1500ms, but logs never arrive - trace_props = {"$ai_model": "gpt-4", "$ai_input_tokens": 10, "$ai_trace_id": "trace999", "$ai_span_id": "span999"} - result = cache_and_merge_properties("trace999", "span999", trace_props, send_immediately=True) - elapsed = time.time() - start +def test_ttl_is_set(mock_redis): + """Test that cached entries have TTL set.""" + trace_props = {"$ai_model": "gpt-4"} + cache_and_merge_properties("trace999", "span000", trace_props, is_trace=True) - assert result is not None, "Trace should return properties after timeout" - assert result == trace_props, "Should return trace properties unchanged (no logs arrived)" - assert "$ai_message_content" not in result, "Should NOT have message content (logs never arrived)" - assert elapsed >= 1.5, f"Should have waited ~1500ms, waited {elapsed*1000:.1f}ms" - assert elapsed < 1.6, f"Should not wait too long, waited {elapsed*1000:.1f}ms" + # Verify setex was called with TTL (60 seconds) + mock_redis.setex.assert_called() + call_args = mock_redis.setex.call_args + assert call_args[0][1] == 60, "TTL should be 60 seconds" if __name__ == "__main__": - test_logs_first_then_traces() - test_traces_first_then_logs() - test_property_override_order() - test_trace_timeout_no_logs() + pytest.main([__file__, "-v"]) From ce69feb31a4b9ebeedb6bc893e3a07b494fd83c5 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Fri, 21 Nov 2025 11:16:59 +0000 Subject: [PATCH 20/34] chore(llm-analytics): Remove unused TypeScript OTEL transformer The TypeScript OTEL transformer in plugin-server was never used for ingestion. All OTEL transformation happens in the Python backend via the Django API endpoints. Plugin-server only does post-processing (cost calculation, normalization) after events are already transformed and sent through Kafka. This removes dead code that was diverging from the actual Python implementation. --- .../llm-analytics/otel/conventions/genai.ts | 222 ------------- .../otel/conventions/posthog-native.ts | 147 --------- plugin-server/src/llm-analytics/otel/index.ts | 23 -- .../src/llm-analytics/otel/transformer.ts | 305 ------------------ plugin-server/src/llm-analytics/otel/types.ts | 220 ------------- .../src/llm-analytics/otel/validation.ts | 154 --------- 6 files changed, 1071 deletions(-) delete mode 100644 plugin-server/src/llm-analytics/otel/conventions/genai.ts delete mode 100644 plugin-server/src/llm-analytics/otel/conventions/posthog-native.ts delete mode 100644 plugin-server/src/llm-analytics/otel/index.ts delete mode 100644 plugin-server/src/llm-analytics/otel/transformer.ts delete mode 100644 plugin-server/src/llm-analytics/otel/types.ts delete mode 100644 plugin-server/src/llm-analytics/otel/validation.ts diff --git a/plugin-server/src/llm-analytics/otel/conventions/genai.ts b/plugin-server/src/llm-analytics/otel/conventions/genai.ts deleted file mode 100644 index 302e8db53911..000000000000 --- a/plugin-server/src/llm-analytics/otel/conventions/genai.ts +++ /dev/null @@ -1,222 +0,0 @@ -/** - * OpenTelemetry GenAI semantic conventions. - * - * Based on: https://opentelemetry.io/docs/specs/semconv/gen-ai/ - * - * These are standard attributes defined by the OpenTelemetry community - * for generative AI workloads. - * - * Example usage in OTel SDK: - * ```python - * from opentelemetry.semconv.ai import SpanAttributes - * - * span.set_attribute(SpanAttributes.GEN_AI_SYSTEM, "openai") - * span.set_attribute(SpanAttributes.GEN_AI_REQUEST_MODEL, "gpt-4") - * span.set_attribute(SpanAttributes.GEN_AI_OPERATION_NAME, "chat") - * ``` - */ -import type { AttributeValue, ExtractedAttributes, OTelSpan } from '../types' - -/** - * Get attribute value - */ -function getAttr(span: OTelSpan, key: string): AttributeValue | undefined { - return span.attributes[key] -} - -/** - * Get string attribute - */ -function getString(span: OTelSpan, key: string): string | undefined { - const value = getAttr(span, key) - return typeof value === 'string' ? value : undefined -} - -/** - * Get number attribute - */ -function getNumber(span: OTelSpan, key: string): number | undefined { - const value = getAttr(span, key) - return typeof value === 'number' ? value : undefined -} - -/** - * Parse JSON attribute if it's a string - */ -function parseJSON(value: AttributeValue | undefined): any { - if (typeof value === 'string') { - try { - return JSON.parse(value) - } catch { - return value - } - } - return value -} - -/** - * Extract model name from either request or response - */ -function extractModel(span: OTelSpan): string | undefined { - return getString(span, 'gen_ai.request.model') || getString(span, 'gen_ai.response.model') -} - -/** - * Extract provider/system - */ -function extractProvider(span: OTelSpan): string | undefined { - return getString(span, 'gen_ai.system') -} - -/** - * Extract token usage - */ -function extractTokenUsage(span: OTelSpan): { - input_tokens?: number - output_tokens?: number - cache_read_tokens?: number - cache_write_tokens?: number -} { - return { - input_tokens: getNumber(span, 'gen_ai.usage.input_tokens'), - output_tokens: getNumber(span, 'gen_ai.usage.output_tokens'), - // Cache tokens might not be standardized yet, but support if present - cache_read_tokens: getNumber(span, 'gen_ai.usage.cache_read_input_tokens'), - cache_write_tokens: getNumber(span, 'gen_ai.usage.cache_creation_input_tokens'), - } -} - -/** - * Extract cost - */ -function extractCost(span: OTelSpan): { - input_cost_usd?: number - output_cost_usd?: number - total_cost_usd?: number -} { - const totalCost = getNumber(span, 'gen_ai.usage.cost') - - return { - input_cost_usd: getNumber(span, 'gen_ai.usage.input_cost'), - output_cost_usd: getNumber(span, 'gen_ai.usage.output_cost'), - total_cost_usd: totalCost, - } -} - -/** - * Extract model parameters - */ -function extractModelParams(span: OTelSpan): { - temperature?: number - max_tokens?: number -} { - return { - temperature: getNumber(span, 'gen_ai.request.temperature'), - max_tokens: - getNumber(span, 'gen_ai.request.max_tokens') || getNumber(span, 'gen_ai.request.max_completion_tokens'), - } -} - -/** - * Extract content (prompt/completion) - * GenAI conventions support multiple formats: - * 1. JSON string: gen_ai.prompt = '[{"role": "user", "content": "..."}]' - * 2. Flattened: gen_ai.prompt.0.role, gen_ai.prompt.0.content - * 3. Simple string: gen_ai.prompt = "text" - */ -function extractContent(span: OTelSpan): { - prompt?: string | object - completion?: string | object - input?: string - output?: string -} { - const prompt = getAttr(span, 'gen_ai.prompt') - const completion = getAttr(span, 'gen_ai.completion') - - return { - prompt: parseJSON(prompt), - completion: parseJSON(completion), - // Some implementations use these simpler names - input: getString(span, 'gen_ai.input'), - output: getString(span, 'gen_ai.output'), - } -} - -/** - * Extract GenAI semantic convention attributes from span. - * Returns undefined for missing attributes. - */ -export function extractGenAIAttributes(span: OTelSpan): ExtractedAttributes { - const tokens = extractTokenUsage(span) - const cost = extractCost(span) - const params = extractModelParams(span) - const content = extractContent(span) - - return { - // Core - model: extractModel(span), - provider: extractProvider(span), - operation_name: getString(span, 'gen_ai.operation.name'), - - // Tokens - ...tokens, - - // Cost - ...cost, - - // Parameters - ...params, - - // Content - ...content, - } -} - -/** - * Check if span has any GenAI semantic convention attributes - */ -export function hasGenAIAttributes(span: OTelSpan): boolean { - return Object.keys(span.attributes).some((key) => key.startsWith('gen_ai.')) -} - -/** - * List of supported GenAI semantic convention attributes - */ -export const SUPPORTED_GENAI_ATTRIBUTES = [ - // Core - 'gen_ai.system', - 'gen_ai.request.model', - 'gen_ai.response.model', - 'gen_ai.operation.name', - - // Usage - 'gen_ai.usage.input_tokens', - 'gen_ai.usage.output_tokens', - 'gen_ai.usage.cache_read_input_tokens', - 'gen_ai.usage.cache_creation_input_tokens', - - // Cost - 'gen_ai.usage.cost', - 'gen_ai.usage.input_cost', - 'gen_ai.usage.output_cost', - - // Request params - 'gen_ai.request.temperature', - 'gen_ai.request.max_tokens', - 'gen_ai.request.max_completion_tokens', - 'gen_ai.request.top_p', - 'gen_ai.request.top_k', - 'gen_ai.request.frequency_penalty', - 'gen_ai.request.presence_penalty', - 'gen_ai.request.stop_sequences', - - // Content - 'gen_ai.prompt', - 'gen_ai.completion', - 'gen_ai.input', - 'gen_ai.output', - - // Response - 'gen_ai.response.id', - 'gen_ai.response.finish_reasons', -] as const diff --git a/plugin-server/src/llm-analytics/otel/conventions/posthog-native.ts b/plugin-server/src/llm-analytics/otel/conventions/posthog-native.ts deleted file mode 100644 index eb8ce638b003..000000000000 --- a/plugin-server/src/llm-analytics/otel/conventions/posthog-native.ts +++ /dev/null @@ -1,147 +0,0 @@ -/** - * PostHog-native OpenTelemetry attribute conventions. - * - * These attributes have the highest priority and follow the pattern: - * posthog.ai.* - * - * Example usage in OTel SDK: - * ```python - * span.set_attribute("posthog.ai.model", "gpt-4") - * span.set_attribute("posthog.ai.provider", "openai") - * span.set_attribute("posthog.ai.session_id", "sess_123") - * ``` - */ -import type { AttributeValue, ExtractedAttributes, OTelSpan } from '../types' - -/** - * PostHog-native attribute namespace - */ -const POSTHOG_PREFIX = 'posthog.ai.' - -/** - * Get attribute with PostHog prefix - */ -function getPostHogAttr(span: OTelSpan, key: string): AttributeValue | undefined { - return span.attributes[`${POSTHOG_PREFIX}${key}`] -} - -/** - * Get string attribute - */ -function getString(span: OTelSpan, key: string): string | undefined { - const value = getPostHogAttr(span, key) - return typeof value === 'string' ? value : undefined -} - -/** - * Get number attribute - */ -function getNumber(span: OTelSpan, key: string): number | undefined { - const value = getPostHogAttr(span, key) - return typeof value === 'number' ? value : undefined -} - -/** - * Get boolean attribute - */ -function getBoolean(span: OTelSpan, key: string): boolean | undefined { - const value = getPostHogAttr(span, key) - return typeof value === 'boolean' ? value : undefined -} - -/** - * Extract PostHog-native attributes from span. - * Returns undefined for missing attributes (not null). - */ -export function extractPostHogNativeAttributes(span: OTelSpan): ExtractedAttributes { - return { - // Core identifiers - model: getString(span, 'model'), - provider: getString(span, 'provider'), - trace_id: getString(span, 'trace_id'), - span_id: getString(span, 'span_id'), - parent_id: getString(span, 'parent_id'), - session_id: getString(span, 'session_id'), - - // Token usage - input_tokens: getNumber(span, 'input_tokens'), - output_tokens: getNumber(span, 'output_tokens'), - cache_read_tokens: getNumber(span, 'cache_read_tokens'), - cache_write_tokens: getNumber(span, 'cache_write_tokens'), - - // Cost - input_cost_usd: getNumber(span, 'input_cost_usd'), - output_cost_usd: getNumber(span, 'output_cost_usd'), - request_cost_usd: getNumber(span, 'request_cost_usd'), - total_cost_usd: getNumber(span, 'total_cost_usd'), - - // Operation - operation_name: getString(span, 'operation_name'), - - // Content - input: getString(span, 'input'), - output: getString(span, 'output'), - prompt: getPostHogAttr(span, 'prompt'), - completion: getPostHogAttr(span, 'completion'), - - // Model parameters - temperature: getNumber(span, 'temperature'), - max_tokens: getNumber(span, 'max_tokens'), - stream: getBoolean(span, 'stream'), - - // Error tracking - is_error: getBoolean(span, 'is_error'), - error_message: getString(span, 'error_message'), - } -} - -/** - * Check if span has any PostHog-native attributes - */ -export function hasPostHogAttributes(span: OTelSpan): boolean { - return Object.keys(span.attributes).some((key) => key.startsWith(POSTHOG_PREFIX)) -} - -/** - * List of supported PostHog-native attributes for documentation - */ -export const SUPPORTED_POSTHOG_ATTRIBUTES = [ - // Core - 'posthog.ai.model', - 'posthog.ai.provider', - 'posthog.ai.trace_id', - 'posthog.ai.span_id', - 'posthog.ai.parent_id', - 'posthog.ai.session_id', - 'posthog.ai.generation_id', - - // Tokens - 'posthog.ai.input_tokens', - 'posthog.ai.output_tokens', - 'posthog.ai.cache_read_tokens', - 'posthog.ai.cache_write_tokens', - - // Cost - 'posthog.ai.input_cost_usd', - 'posthog.ai.output_cost_usd', - 'posthog.ai.request_cost_usd', - 'posthog.ai.total_cost_usd', - - // Operation - 'posthog.ai.operation_name', - - // Content - 'posthog.ai.input', - 'posthog.ai.output', - 'posthog.ai.prompt', - 'posthog.ai.completion', - - // Parameters - 'posthog.ai.temperature', - 'posthog.ai.max_tokens', - 'posthog.ai.stream', - - // Error - 'posthog.ai.is_error', - 'posthog.ai.error_message', -] as const diff --git a/plugin-server/src/llm-analytics/otel/index.ts b/plugin-server/src/llm-analytics/otel/index.ts deleted file mode 100644 index 09ade8e4d7bd..000000000000 --- a/plugin-server/src/llm-analytics/otel/index.ts +++ /dev/null @@ -1,23 +0,0 @@ -/** - * OpenTelemetry traces ingestion for PostHog LLM Analytics. - * - * This module provides transformation of OpenTelemetry spans to PostHog AI events. - * It supports both PostHog-native and GenAI semantic conventions. - * - * Architecture: - * 1. Python API endpoint receives OTLP protobuf HTTP requests - * 2. Python parses protobuf and creates PostHog events - * 3. Events go through standard ingestion pipeline (Kafka) - * 4. Plugin-server processes AI events (cost calculation, normalization) - * - * This TypeScript code can be used for: - * - Documentation of the transformation logic - * - Testing transformation behavior - * - Future: Direct TypeScript-based ingestion if needed - */ - -export { transformSpanToAIEvent, spanUsesKnownConventions, OTEL_TRANSFORMER_VERSION } from './transformer' -export { extractPostHogNativeAttributes, hasPostHogAttributes } from './conventions/posthog-native' -export { extractGenAIAttributes, hasGenAIAttributes } from './conventions/genai' -export { OTEL_LIMITS } from './validation' -export type * from './types' diff --git a/plugin-server/src/llm-analytics/otel/transformer.ts b/plugin-server/src/llm-analytics/otel/transformer.ts deleted file mode 100644 index 2d558b02cf57..000000000000 --- a/plugin-server/src/llm-analytics/otel/transformer.ts +++ /dev/null @@ -1,305 +0,0 @@ -/** - * Core OTel span to PostHog AI event transformer. - * - * Transforms OpenTelemetry spans into PostHog AI events using a waterfall - * pattern for attribute extraction: - * 1. PostHog native attributes (highest priority) - * 2. GenAI semantic conventions (fallback) - * 3. OTel span built-ins (trace_id, span_id, etc.) - */ -import { extractGenAIAttributes, hasGenAIAttributes } from './conventions/genai' -import { extractPostHogNativeAttributes, hasPostHogAttributes } from './conventions/posthog-native' -import type { - AIEvent, - AIEventProperties, - AIEventType, - Baggage, - ExtractedAttributes, - InstrumentationScope, - OTEL_TRANSFORMER_VERSION, - OTelSpan, - Resource, - SpanStatusCode, -} from './types' -import { - type ValidationError, - validateAttributeCount, - validateAttributeValue, - validateEventCount, - validateLinkCount, - validateSpanName, -} from './validation' - -export { OTEL_TRANSFORMER_VERSION } from './types' - -/** - * Transform a single OTel span to PostHog AI event. - */ -export function transformSpanToAIEvent( - span: OTelSpan, - resource: Resource, - scope: InstrumentationScope, - baggage?: Baggage -): { event: AIEvent; errors: ValidationError[] } { - const errors: ValidationError[] = [] - - // Validate span - const validation = validateSpan(span) - errors.push(...validation) - - // Extract attributes using waterfall pattern - const posthogAttrs = extractPostHogNativeAttributes(span) - const genaiAttrs = extractGenAIAttributes(span) - - // Merge with precedence: PostHog > GenAI - const mergedAttrs: ExtractedAttributes = { - ...genaiAttrs, - ...posthogAttrs, // PostHog overrides GenAI - } - - // Build AI event properties - const properties = buildEventProperties(span, mergedAttrs, resource, scope, baggage) - - // Determine event type - const eventType = determineEventType(span, mergedAttrs) - - // Calculate timestamp and latency - const timestamp = calculateTimestamp(span) - - // Get distinct_id (from resource attributes or default) - const distinct_id = extractDistinctId(resource, baggage) - - const event: AIEvent = { - event: eventType, - distinct_id, - timestamp, - properties, - } - - return { event, errors } -} - -/** - * Validate span against limits - */ -function validateSpan(span: OTelSpan): ValidationError[] { - const errors: ValidationError[] = [] - - // Validate span name - const nameError = validateSpanName(span.name) - if (nameError) errors.push(nameError) - - // Validate attribute count - const attrCountError = validateAttributeCount(Object.keys(span.attributes).length) - if (attrCountError) errors.push(attrCountError) - - // Validate event count - const eventCountError = validateEventCount(span.events?.length || 0) - if (eventCountError) errors.push(eventCountError) - - // Validate link count - const linkCountError = validateLinkCount(span.links?.length || 0) - if (linkCountError) errors.push(linkCountError) - - // Validate attribute values - for (const [key, value] of Object.entries(span.attributes)) { - const valueError = validateAttributeValue(key, value) - if (valueError) errors.push(valueError) - } - - return errors -} - -/** - * Build PostHog AI event properties from extracted attributes - */ -function buildEventProperties( - span: OTelSpan, - attrs: ExtractedAttributes, - resource: Resource, - scope: InstrumentationScope, - baggage?: Baggage -): AIEventProperties { - // Core identifiers (prefer extracted, fallback to span built-ins) - const trace_id = attrs.trace_id || span.trace_id - const span_id = attrs.span_id || span.span_id - const parent_id = attrs.parent_id || span.parent_span_id - - // Session ID (prefer extracted, fallback to baggage) - const session_id = attrs.session_id || baggage?.session_id || baggage?.['posthog.session_id'] - - // Calculate latency - const latency = calculateLatency(span) - - // Detect error from span status - const is_error = attrs.is_error !== undefined ? attrs.is_error : span.status.code === SpanStatusCode.ERROR - const error_message = attrs.error_message || (is_error ? span.status.message : undefined) - - // Build base properties - const properties: AIEventProperties = { - // Core IDs - $ai_trace_id: trace_id, - $ai_span_id: span_id, - ...(parent_id && { $ai_parent_id: parent_id }), - ...(session_id && { $ai_session_id: session_id }), - - // Model info - ...(attrs.model && { $ai_model: attrs.model }), - ...(attrs.provider && { $ai_provider: attrs.provider }), - - // Tokens - ...(attrs.input_tokens !== undefined && { $ai_input_tokens: attrs.input_tokens }), - ...(attrs.output_tokens !== undefined && { $ai_output_tokens: attrs.output_tokens }), - ...(attrs.cache_read_tokens !== undefined && { $ai_cache_read_tokens: attrs.cache_read_tokens }), - ...(attrs.cache_write_tokens !== undefined && { $ai_cache_write_tokens: attrs.cache_write_tokens }), - - // Cost - ...(attrs.input_cost_usd !== undefined && { $ai_input_cost_usd: attrs.input_cost_usd }), - ...(attrs.output_cost_usd !== undefined && { $ai_output_cost_usd: attrs.output_cost_usd }), - ...(attrs.total_cost_usd !== undefined && { $ai_total_cost_usd: attrs.total_cost_usd }), - - // Timing - ...(latency !== undefined && { $ai_latency: latency }), - - // Error - ...(is_error && { $ai_is_error: is_error }), - ...(error_message && { $ai_error_message: error_message }), - - // Model parameters - ...(attrs.temperature !== undefined && { $ai_temperature: attrs.temperature }), - ...(attrs.max_tokens !== undefined && { $ai_max_tokens: attrs.max_tokens }), - ...(attrs.stream !== undefined && { $ai_stream: attrs.stream }), - - // Content (if present and not too large) - ...(attrs.input && { $ai_input: stringifyContent(attrs.input) }), - ...(attrs.output && { $ai_output_choices: stringifyContent(attrs.output) }), - ...(attrs.prompt && { $ai_input: stringifyContent(attrs.prompt) }), - ...(attrs.completion && { $ai_output_choices: stringifyContent(attrs.completion) }), - - // Metadata - $ai_otel_transformer_version: '1.0.0', - $ai_otel_span_kind: span.kind.toString(), - $ai_otel_status_code: span.status.code.toString(), - - // Resource attributes (service name, etc.) - ...(resource.attributes['service.name'] && { - $ai_service_name: resource.attributes['service.name'] as string, - }), - - // Instrumentation scope - $ai_instrumentation_scope_name: scope.name, - ...(scope.version && { $ai_instrumentation_scope_version: scope.version }), - } - - // Add remaining span attributes (not already mapped) - const mappedKeys = new Set([ - 'posthog.ai.model', - 'posthog.ai.provider', - 'gen_ai.system', - 'gen_ai.request.model', - 'gen_ai.response.model', - 'gen_ai.operation.name', - 'gen_ai.usage.input_tokens', - 'gen_ai.usage.output_tokens', - 'gen_ai.prompt', - 'gen_ai.completion', - 'service.name', - ]) - - for (const [key, value] of Object.entries(span.attributes)) { - if (!mappedKeys.has(key) && !key.startsWith('posthog.ai.') && !key.startsWith('gen_ai.')) { - // Add unmapped attributes with prefix - properties[`otel.${key}`] = value - } - } - - return properties -} - -/** - * Determine AI event type from span - */ -function determineEventType(span: OTelSpan, attrs: ExtractedAttributes): AIEventType { - const opName = attrs.operation_name?.toLowerCase() - - // Check operation name - if (opName === 'chat' || opName === 'completion') { - return '$ai_generation' - } else if (opName === 'embedding' || opName === 'embeddings') { - return '$ai_embedding' - } - - // Check if span is root (no parent) - if (!span.parent_span_id) { - return '$ai_trace' - } - - // Default to generic span - return '$ai_span' -} - -/** - * Calculate timestamp from span start time - */ -function calculateTimestamp(span: OTelSpan): string { - const nanos = BigInt(span.start_time_unix_nano) - const millis = Number(nanos / BigInt(1_000_000)) - return new Date(millis).toISOString() -} - -/** - * Calculate latency in seconds from span start/end time - */ -function calculateLatency(span: OTelSpan): number | undefined { - if (!span.end_time_unix_nano) return undefined - - const startNanos = BigInt(span.start_time_unix_nano) - const endNanos = BigInt(span.end_time_unix_nano) - const durationNanos = endNanos - startNanos - - // Convert to seconds - return Number(durationNanos) / 1_000_000_000 -} - -/** - * Extract distinct_id from resource or baggage - */ -function extractDistinctId(resource: Resource, baggage?: Baggage): string { - // Try resource attributes - const userId = - resource.attributes['user.id'] || - resource.attributes['enduser.id'] || - resource.attributes['posthog.distinct_id'] - - if (typeof userId === 'string') { - return userId - } - - // Try baggage - if (baggage?.user_id) { - return baggage.user_id - } - if (baggage?.distinct_id) { - return baggage.distinct_id - } - - // Default to anonymous - return 'anonymous' -} - -/** - * Stringify content (handles objects and strings) - */ -function stringifyContent(content: any): string { - if (typeof content === 'string') { - return content - } - return JSON.stringify(content) -} - -/** - * Check if span uses PostHog or GenAI conventions - */ -export function spanUsesKnownConventions(span: OTelSpan): boolean { - return hasPostHogAttributes(span) || hasGenAIAttributes(span) -} diff --git a/plugin-server/src/llm-analytics/otel/types.ts b/plugin-server/src/llm-analytics/otel/types.ts deleted file mode 100644 index 2d164d168afc..000000000000 --- a/plugin-server/src/llm-analytics/otel/types.ts +++ /dev/null @@ -1,220 +0,0 @@ -/** - * TypeScript types for OpenTelemetry trace ingestion. - * - * These types represent the subset of OTLP trace data we care about - * for converting to PostHog AI events. - */ - -/** - * Simplified OTel Span representation. - * Maps to opentelemetry.proto.trace.v1.Span - */ -export interface OTelSpan { - trace_id: string // hex-encoded 16-byte trace ID - span_id: string // hex-encoded 8-byte span ID - parent_span_id?: string // hex-encoded 8-byte parent span ID - name: string - kind: SpanKind - start_time_unix_nano: string // nanoseconds since epoch - end_time_unix_nano: string // nanoseconds since epoch - attributes: Record - events: SpanEvent[] - links: SpanLink[] - status: SpanStatus -} - -/** - * Span kind enum - */ -export enum SpanKind { - UNSPECIFIED = 0, - INTERNAL = 1, - SERVER = 2, - CLIENT = 3, - PRODUCER = 4, - CONSUMER = 5, -} - -/** - * Attribute value types - */ -export type AttributeValue = string | number | boolean | string[] | number[] - -/** - * Span event - */ -export interface SpanEvent { - time_unix_nano: string - name: string - attributes: Record -} - -/** - * Span link - */ -export interface SpanLink { - trace_id: string - span_id: string - attributes: Record -} - -/** - * Span status - */ -export interface SpanStatus { - code: SpanStatusCode - message?: string -} - -export enum SpanStatusCode { - UNSET = 0, - OK = 1, - ERROR = 2, -} - -/** - * Resource represents the entity producing telemetry. - */ -export interface Resource { - attributes: Record -} - -/** - * Instrumentation scope (library/tracer info) - */ -export interface InstrumentationScope { - name: string - version?: string - attributes?: Record -} - -/** - * Baggage context propagation - */ -export interface Baggage { - [key: string]: string -} - -/** - * Parsed OTLP trace request - */ -export interface ParsedOTLPRequest { - spans: OTelSpan[] - resource: Resource - scope: InstrumentationScope - baggage?: Baggage -} - -/** - * PostHog AI event properties (subset relevant for OTel mapping) - */ -export interface AIEventProperties { - // Core identifiers - $ai_trace_id: string - $ai_span_id: string - $ai_parent_id?: string - $ai_session_id?: string - $ai_generation_id?: string - - // Model info - $ai_model?: string - $ai_provider?: string - - // Token usage - $ai_input_tokens?: number - $ai_output_tokens?: number - $ai_cache_read_tokens?: number - $ai_cache_write_tokens?: number - - // Cost - $ai_input_cost_usd?: number - $ai_output_cost_usd?: number - $ai_total_cost_usd?: number - - // Timing - $ai_latency?: number // seconds - - // Error tracking - $ai_is_error?: boolean - $ai_error_message?: string - - // Model parameters - $ai_temperature?: number - $ai_max_tokens?: number - $ai_stream?: boolean - - // Content (may be URLs to blob storage) - $ai_input?: string - $ai_output_choices?: string - - // Metadata - $ai_otel_transformer_version: string - $ai_otel_span_kind?: string - $ai_otel_status_code?: string - - // Additional properties - [key: string]: AttributeValue | undefined -} - -/** - * PostHog AI event - */ -export interface AIEvent { - event: AIEventType - distinct_id: string - timestamp: string // ISO 8601 - properties: AIEventProperties -} - -/** - * AI event types - */ -export type AIEventType = '$ai_generation' | '$ai_embedding' | '$ai_span' | '$ai_trace' | '$ai_metric' | '$ai_feedback' - -/** - * Transformer version - */ -export const OTEL_TRANSFORMER_VERSION = '1.0.0' - -/** - * Extracted attributes from conventions - */ -export interface ExtractedAttributes { - // Core - model?: string - provider?: string - trace_id?: string - span_id?: string - parent_id?: string - session_id?: string - - // Tokens - input_tokens?: number - output_tokens?: number - cache_read_tokens?: number - cache_write_tokens?: number - - // Cost - input_cost_usd?: number - output_cost_usd?: number - request_cost_usd?: number - total_cost_usd?: number - - // Operation - operation_name?: string // chat, completion, embedding - - // Content - input?: string - output?: string - prompt?: string | object - completion?: string | object - - // Parameters - temperature?: number - max_tokens?: number - stream?: boolean - - // Error - is_error?: boolean - error_message?: string -} diff --git a/plugin-server/src/llm-analytics/otel/validation.ts b/plugin-server/src/llm-analytics/otel/validation.ts deleted file mode 100644 index d13d52c1ed7b..000000000000 --- a/plugin-server/src/llm-analytics/otel/validation.ts +++ /dev/null @@ -1,154 +0,0 @@ -/** - * OpenTelemetry ingestion validation constants and limits. - * - * These limits align with: - * - OpenTelemetry SDK defaults - * - Industry standard observability platforms (Jaeger, New Relic, Datadog) - * - PostHog infrastructure constraints - */ - -export const OTEL_LIMITS = { - /** - * Maximum number of spans per OTLP export request. - * Most OTel SDKs batch export around 512 spans by default. - */ - MAX_SPANS_PER_REQUEST: 1000, - - /** - * Maximum number of attributes per span. - * Aligns with OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT default. - */ - MAX_ATTRIBUTES_PER_SPAN: 128, - - /** - * Maximum number of events per span. - * Aligns with OTEL_SPAN_EVENT_COUNT_LIMIT default. - */ - MAX_EVENTS_PER_SPAN: 128, - - /** - * Maximum number of links per span. - * Aligns with OTEL_SPAN_LINK_COUNT_LIMIT default. - */ - MAX_LINKS_PER_SPAN: 128, - - /** - * Maximum length of an attribute value in bytes. - * Set generously to 100KB to accommodate LLM prompts/completions. - * Can be extended to blob storage in future if needed. - */ - MAX_ATTRIBUTE_VALUE_LENGTH: 100_000, - - /** - * Maximum length of span name. - * Aligns with OTel recommendations. - */ - MAX_SPAN_NAME_LENGTH: 1024, - - /** - * Maximum length of resource attributes. - */ - MAX_RESOURCE_ATTRIBUTES: 64, -} as const - -export interface ValidationError { - field: string - value: number - limit: number - message: string -} - -export interface ValidationResult { - valid: boolean - errors: ValidationError[] -} - -/** - * Validate attribute value length. - */ -export function validateAttributeValue(key: string, value: unknown): ValidationError | null { - if (typeof value === 'string' && value.length > OTEL_LIMITS.MAX_ATTRIBUTE_VALUE_LENGTH) { - return { - field: `attribute.${key}`, - value: value.length, - limit: OTEL_LIMITS.MAX_ATTRIBUTE_VALUE_LENGTH, - message: `Attribute '${key}' exceeds ${OTEL_LIMITS.MAX_ATTRIBUTE_VALUE_LENGTH} bytes (${value.length} bytes). Consider reducing payload size.`, - } - } - return null -} - -/** - * Validate span name length. - */ -export function validateSpanName(name: string): ValidationError | null { - if (name.length > OTEL_LIMITS.MAX_SPAN_NAME_LENGTH) { - return { - field: 'span.name', - value: name.length, - limit: OTEL_LIMITS.MAX_SPAN_NAME_LENGTH, - message: `Span name exceeds ${OTEL_LIMITS.MAX_SPAN_NAME_LENGTH} characters (${name.length} characters).`, - } - } - return null -} - -/** - * Validate number of attributes in span. - */ -export function validateAttributeCount(attributeCount: number): ValidationError | null { - if (attributeCount > OTEL_LIMITS.MAX_ATTRIBUTES_PER_SPAN) { - return { - field: 'span.attributes', - value: attributeCount, - limit: OTEL_LIMITS.MAX_ATTRIBUTES_PER_SPAN, - message: `Span has ${attributeCount} attributes, maximum is ${OTEL_LIMITS.MAX_ATTRIBUTES_PER_SPAN}.`, - } - } - return null -} - -/** - * Validate number of events in span. - */ -export function validateEventCount(eventCount: number): ValidationError | null { - if (eventCount > OTEL_LIMITS.MAX_EVENTS_PER_SPAN) { - return { - field: 'span.events', - value: eventCount, - limit: OTEL_LIMITS.MAX_EVENTS_PER_SPAN, - message: `Span has ${eventCount} events, maximum is ${OTEL_LIMITS.MAX_EVENTS_PER_SPAN}.`, - } - } - return null -} - -/** - * Validate number of links in span. - */ -export function validateLinkCount(linkCount: number): ValidationError | null { - if (linkCount > OTEL_LIMITS.MAX_LINKS_PER_SPAN) { - return { - field: 'span.links', - value: linkCount, - limit: OTEL_LIMITS.MAX_LINKS_PER_SPAN, - message: `Span has ${linkCount} links, maximum is ${OTEL_LIMITS.MAX_LINKS_PER_SPAN}.`, - } - } - return null -} - -/** - * Validate total span count in request. - */ -export function validateSpanCount(spanCount: number): ValidationError | null { - if (spanCount > OTEL_LIMITS.MAX_SPANS_PER_REQUEST) { - return { - field: 'request.spans', - value: spanCount, - limit: OTEL_LIMITS.MAX_SPANS_PER_REQUEST, - message: `Request contains ${spanCount} spans, maximum is ${OTEL_LIMITS.MAX_SPANS_PER_REQUEST}. Configure batch size in your OTel SDK (e.g., OTEL_BSP_MAX_EXPORT_BATCH_SIZE).`, - } - } - return null -} From d8374fa5564fbc893b67d90da82b186d8f3c5868 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Fri, 21 Nov 2025 11:19:31 +0000 Subject: [PATCH 21/34] docs(llm-analytics): Remove OTEL_QUICKSTART.md The detailed README in backend/api/otel/ is sufficient documentation. --- products/llm_analytics/OTEL_QUICKSTART.md | 374 ---------------------- 1 file changed, 374 deletions(-) delete mode 100644 products/llm_analytics/OTEL_QUICKSTART.md diff --git a/products/llm_analytics/OTEL_QUICKSTART.md b/products/llm_analytics/OTEL_QUICKSTART.md deleted file mode 100644 index 97897274f4d3..000000000000 --- a/products/llm_analytics/OTEL_QUICKSTART.md +++ /dev/null @@ -1,374 +0,0 @@ -# OpenTelemetry Ingestion for PostHog LLM Analytics - Quickstart - -This guide shows how to configure OpenTelemetry SDKs to send LLM traces and logs to PostHog. - -## Overview - -PostHog LLM Analytics supports OpenTelemetry Protocol (OTLP) ingestion for: - -- **Traces**: Spans from LLM operations (chat, completions, embeddings) -- **Logs**: Log records containing message content (prompts/completions) - -## Endpoints - -### Base URL Pattern - -```text -{posthog_host}/api/projects/{project_id}/ai/otel/v1/ -``` - -### Specific Endpoints - -- **Traces**: `POST /api/projects/{project_id}/ai/otel/v1/traces` -- **Logs**: `POST /api/projects/{project_id}/ai/otel/v1/logs` - -### Authentication - -Use Personal API Key as Bearer token: - -```text -Authorization: Bearer {personal_api_key} -``` - -## Quick Start - -### 1. Get Credentials - -1. Find your Project ID in PostHog UI (Settings → Project) -2. Create a Personal API Key (Settings → Personal API Keys) - -### 2. Configure Environment - -```bash -export POSTHOG_PROJECT_ID=123 -export POSTHOG_PERSONAL_API_KEY=phc_... -export POSTHOG_HOST=https://app.posthog.com # or http://localhost:8000 for local -``` - -### 3. Configure OpenTelemetry SDK - -#### Python (OpenTelemetry SDK) - -```python -from opentelemetry import trace -from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter -from opentelemetry.sdk.resources import Resource -from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.sdk.trace.export import BatchSpanProcessor - -# Create resource -resource = Resource.create({ - "service.name": "my-llm-app", - "service.version": "1.0.0", -}) - -# Create tracer provider -tracer_provider = TracerProvider(resource=resource) - -# Configure OTLP exporter -traces_endpoint = f"{posthog_host}/api/projects/{project_id}/ai/otel/v1/traces" -trace_exporter = OTLPSpanExporter( - endpoint=traces_endpoint, - headers={"Authorization": f"Bearer {personal_api_key}"}, -) - -# Add batch processor -tracer_provider.add_span_processor(BatchSpanProcessor(trace_exporter)) - -# Set as global tracer -trace.set_tracer_provider(tracer_provider) -``` - -#### JavaScript/TypeScript (OpenTelemetry SDK) - -```typescript -import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http'; -import { Resource } from '@opentelemetry/resources'; -import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-base'; -import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node'; - -const provider = new NodeTracerProvider({ - resource: new Resource({ - 'service.name': 'my-llm-app', - 'service.version': '1.0.0', - }), -}); - -const exporter = new OTLPTraceExporter({ - url: `${posthogHost}/api/projects/${projectId}/ai/otel/v1/traces`, - headers: { - 'Authorization': `Bearer ${personalApiKey}`, - }, -}); - -provider.addSpanProcessor(new BatchSpanProcessor(exporter)); -provider.register(); -``` - -### 4. Instrument Your LLM SDK - -#### OpenAI Python - -```python -from opentelemetry.instrumentation.openai import OpenAIInstrumentor - -# Instrument OpenAI SDK -OpenAIInstrumentor().instrument() - -# Now use OpenAI as normal -import openai -response = openai.chat.completions.create( - model="gpt-4o-mini", - messages=[{"role": "user", "content": "Hello!"}] -) -``` - -#### Anthropic (with LangChain) - -```python -from opentelemetry.instrumentation.langchain import LangChainInstrumentor - -# Instrument LangChain -LangChainInstrumentor().instrument() - -# Now use LangChain/Anthropic as normal -from langchain_anthropic import ChatAnthropic -llm = ChatAnthropic(model="claude-3-5-sonnet-20241022") -response = llm.invoke("Hello!") -``` - -## Supported Conventions - -### PostHog Native Attributes (Highest Priority) - -Use these for direct control over AI event properties: - -```python -span.set_attribute("posthog.ai.model", "gpt-4o-mini") -span.set_attribute("posthog.ai.provider", "openai") -span.set_attribute("posthog.ai.input_tokens", 100) -span.set_attribute("posthog.ai.output_tokens", 50) -span.set_attribute("posthog.ai.total_cost_usd", 0.0042) -span.set_attribute("posthog.ai.input", "What is 2+2?") -span.set_attribute("posthog.ai.output", "4") -``` - -### GenAI Semantic Conventions (Fallback) - -Standard OpenTelemetry GenAI conventions: - -```python -span.set_attribute("gen_ai.system", "openai") -span.set_attribute("gen_ai.request.model", "gpt-4o-mini") -span.set_attribute("gen_ai.operation.name", "chat") -span.set_attribute("gen_ai.usage.input_tokens", 100) -span.set_attribute("gen_ai.usage.output_tokens", 50) -span.set_attribute("gen_ai.prompt", "What is 2+2?") -span.set_attribute("gen_ai.completion", "4") -``` - -## Generated AI Events - -PostHog automatically creates these event types: - -- `$ai_generation`: Chat/completion operations -- `$ai_embedding`: Embedding operations -- `$ai_span`: Generic LLM spans -- `$ai_trace`: Root spans (traces) - -### Event Properties - -Events include these properties (when available): - -```python -{ - # Core IDs - "$ai_trace_id": "hex-string", - "$ai_span_id": "hex-string", - "$ai_parent_id": "hex-string", - "$ai_session_id": "session-123", - - # Model info - "$ai_model": "gpt-4o-mini", - "$ai_provider": "openai", - - # Tokens & Cost - "$ai_input_tokens": 100, - "$ai_output_tokens": 50, - "$ai_total_cost_usd": 0.0042, - - # Timing & Error - "$ai_latency": 1.234, - "$ai_is_error": false, - - # Content - "$ai_input": "...", - "$ai_output_choices": "...", - - # Metadata - "$ai_service_name": "my-app", - "$ai_otel_transformer_version": "1.0.0" -} -``` - -## Framework Examples - -### Pydantic AI - -```python -from pydantic_ai import Agent -from pydantic_ai.models.openai import OpenAIModel - -# Setup OpenTelemetry (see above) - -# Instrument OpenAI -from opentelemetry.instrumentation.openai import OpenAIInstrumentor -OpenAIInstrumentor().instrument() - -# Use Pydantic AI as normal -agent = Agent(OpenAIModel("gpt-4o-mini")) -result = await agent.run("Hello!") -``` - -### LangChain - -```python -from langchain_openai import ChatOpenAI - -# Setup OpenTelemetry (see above) - -# Instrument LangChain -from opentelemetry.instrumentation.langchain import LangChainInstrumentor -LangChainInstrumentor().instrument() - -# Use LangChain as normal -llm = ChatOpenAI(model="gpt-4o-mini") -response = llm.invoke("Hello!") -``` - -### LlamaIndex - -```python -from llama_index.core import VectorStoreIndex, SimpleDirectoryReader - -# Setup OpenTelemetry (see above) - -# Instrument OpenAI (used by LlamaIndex) -from opentelemetry.instrumentation.openai import OpenAIInstrumentor -OpenAIInstrumentor().instrument() - -# Use LlamaIndex as normal -documents = SimpleDirectoryReader("data").load_data() -index = VectorStoreIndex.from_documents(documents) -response = index.as_query_engine().query("What is...?") -``` - -## Validation Limits - -The ingestion endpoint enforces these limits: - -- **Traces:** - - Max 1000 spans per request - - Max 128 attributes per span - - Max 128 events per span - - Max 128 links per span - - Max 100KB per attribute value - - Max 1024 characters for span names - -- **Logs:** - - Max 1000 log records per request - - Max 128 attributes per log - - Max 100KB for log body - - Max 100KB per attribute value - -Configure your SDK's batch settings if you hit these limits: - -```python -# Python -processor = BatchSpanProcessor( - exporter, - max_export_batch_size=500, # Lower than 1000 -) -``` - -## Troubleshooting - -### No events appearing in PostHog - -1. **Check authentication:** - - Verify Personal API Key is valid - - Check for 401/403 responses in network logs - -2. **Check endpoint URL:** - - Ensure project ID is correct - - Verify host URL (no trailing slash) - -3. **Check OTLP exporter:** - - Verify protobuf content type is being sent - - Check for connection errors in SDK logs - -### Events missing properties - -1. **Check instrumentation:** - - Ensure SDK instrumentation is installed - - Verify instrumentation is called before SDK usage - -2. **Check conventions:** - - Use PostHog native (`posthog.ai.*`) or GenAI (`gen_ai.*`) attributes - - Verify attribute names are correct - -### High latency or timeouts - -1. **Check batch settings:** - - Reduce batch size if hitting limits - - Increase batch timeout for better throughput - -2. **Check network:** - - Verify PostHog host is reachable - - Check for proxy/firewall issues - -## Development & Testing - -### Local Development - -1. Start PostHog locally: - - ```bash - cd /path/to/posthog - ./bin/start - ``` - -2. Use local endpoint: - - ```text - http://localhost:8000/api/projects/{project_id}/ai/otel/v1/ - ``` - -### Testing with Console Exporter - -Add console exporter for debugging: - -```python -from opentelemetry.sdk.trace.export import ConsoleSpanExporter - -# Add console exporter alongside PostHog exporter -console_exporter = ConsoleSpanExporter() -tracer_provider.add_span_processor(BatchSpanProcessor(console_exporter)) -``` - -This will print spans to stdout for verification. - -## Next Steps - -1. **Verify ingestion:** Check PostHog LLM Analytics UI for events -2. **Add session tracking:** Set `$ai_session_id` for grouping traces -3. **Add custom attributes:** Enrich spans with app-specific data -4. **Monitor costs:** Track `$ai_total_cost_usd` per model/provider -5. **Set up alerts:** Create insights for errors, latency, costs - -## References - -- [OpenTelemetry Python SDK](https://opentelemetry.io/docs/instrumentation/python/) -- [OpenTelemetry JavaScript SDK](https://opentelemetry.io/docs/instrumentation/js/) -- [GenAI Semantic Conventions](https://opentelemetry.io/docs/specs/semconv/gen-ai/) -- [OTLP Specification](https://opentelemetry.io/docs/specs/otlp/) From d16f5a2d491c11f3c325ebde80e96eae18911d18 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Fri, 21 Nov 2025 11:23:14 +0000 Subject: [PATCH 22/34] docs(llm-analytics): Fix README inaccuracies - Remove incorrect claim about Redis transactions (WATCH/MULTI/EXEC) The event_merger uses simple get/setex/delete operations - Fix test path from test/ subdirectory to correct location - Update performance characteristics to reflect actual implementation --- products/llm_analytics/backend/api/otel/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/products/llm_analytics/backend/api/otel/README.md b/products/llm_analytics/backend/api/otel/README.md index 43719181e6f4..eaba937e4613 100644 --- a/products/llm_analytics/backend/api/otel/README.md +++ b/products/llm_analytics/backend/api/otel/README.md @@ -95,7 +95,7 @@ Converts OTel log records to AI event properties. Extracts message content from ### event_merger.py -Redis-based non-blocking cache for v2 trace/log coordination. Uses atomic operations (WATCH/MULTI/EXEC) to safely merge data from concurrent arrivals. Keys expire after 60 seconds to prevent orphaned entries. +Redis-based non-blocking cache for v2 trace/log coordination. Uses simple Redis operations (get/setex/delete) for fast caching and retrieval. Keys expire after 60 seconds to prevent orphaned entries. **Merge Logic**: @@ -168,7 +168,7 @@ All events conform to the PostHog LLM Analytics schema: Run unit tests: ```bash -pytest products/llm_analytics/backend/api/otel/test/ +pytest products/llm_analytics/backend/api/otel/ ``` Integration testing requires: @@ -263,7 +263,7 @@ Extend `build_event_properties()` in `transformer.py` to map additional attribut - **Throughput**: Limited by Redis round-trip time for v2 merging - **Latency**: v1 has single-pass latency, v2 has cache lookup latency - **Memory**: Redis cache bounded by TTL (60s max retention) -- **Concurrency**: Redis transactions provide safe concurrent merging +- **Concurrency**: Simple Redis operations enable fast merging with minimal race condition risk ## References From 4a692441348c47bfe856cd7d137656adad54a8e3 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Fri, 21 Nov 2025 11:27:03 +0000 Subject: [PATCH 23/34] docs(llm-analytics): Expand architecture diagram to show full event flow Show complete flow from OTLP ingestion through to ClickHouse: - OTEL transformation (Python) - PostHog capture API - Kafka (events_plugin_ingestion topic) - Plugin-server post-processing (cost calculation, normalization) - ClickHouse storage This clarifies that OTEL transformation happens in Python, while plugin-server only handles post-processing after events are already in the standard PostHog ingestion pipeline. --- .../llm_analytics/backend/api/otel/README.md | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/products/llm_analytics/backend/api/otel/README.md b/products/llm_analytics/backend/api/otel/README.md index eaba937e4613..eaf2c0d0e15e 100644 --- a/products/llm_analytics/backend/api/otel/README.md +++ b/products/llm_analytics/backend/api/otel/README.md @@ -33,8 +33,35 @@ OTLP HTTP Request | v event_merger.py (merge with traces for v2) + | + v + PostHog AI Events ($ai_generation, $ai_span, etc.) + | + v + capture_batch_internal (PostHog capture API) + | + v + Kafka (events_plugin_ingestion topic) + | + v + Plugin-server (ingestion-consumer) + | + +---> process-ai-event.ts (cost calculation, normalization) + | + v + ClickHouse (sharded_events table) ``` +**Post-transformation flow**: + +1. **capture_batch_internal**: OTEL-transformed events enter PostHog's standard event ingestion pipeline +2. **Kafka**: Events are published to the `events_plugin_ingestion` topic for async processing +3. **Plugin-server**: The ingestion consumer processes events through `process-ai-event.ts`: + - Calculates costs based on token counts and model pricing (`$ai_input_cost_usd`, `$ai_output_cost_usd`, `$ai_total_cost_usd`) + - Normalizes trace IDs to strings + - Extracts model parameters (temperature, max_tokens, stream) from `$ai_model_parameters` +4. **ClickHouse**: Events are written to the sharded_events table for querying in PostHog UI + ## Instrumentation Patterns ### v1 Instrumentation From 05380d05732c1f240ff52801542b87be9979ecfd Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Fri, 21 Nov 2025 12:42:37 +0000 Subject: [PATCH 24/34] fix(otel): Add gen_ai.tool.message handler for OTEL v2 ingestion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes bug where tool messages were missing from OTEL v2 conversation traces. Changes: - Add handler for gen_ai.tool.message events in logs_transformer.py - Fix assistant messages to go to $ai_input (conversation history) instead of $ai_output_choices - Implement message array accumulation in ingestion.py for multiple log events per span - Add unit tests for message accumulation and tool message handling Before: OTEL v2 traces had incorrect message counts (2→4→5→7→8→10) with missing tool messages After: Correct message counts (2→4→6→8→10→12) with all tool messages properly captured Tool messages now include: - role: "tool" - content: tool execution result - tool_call_id: links response to tool call 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .../backend/api/otel/ingestion.py | 13 +- .../backend/api/otel/logs_transformer.py | 14 +- .../api/otel/test_logs_accumulation.py | 213 ++++++++++++++++++ 3 files changed, 237 insertions(+), 3 deletions(-) create mode 100644 products/llm_analytics/backend/api/otel/test_logs_accumulation.py diff --git a/products/llm_analytics/backend/api/otel/ingestion.py b/products/llm_analytics/backend/api/otel/ingestion.py index a5ae26c3fdc1..781a303909cc 100644 --- a/products/llm_analytics/backend/api/otel/ingestion.py +++ b/products/llm_analytics/backend/api/otel/ingestion.py @@ -728,8 +728,17 @@ def transform_logs_to_ai_events(parsed_request: dict[str, Any]) -> list[dict[str accumulated_props = {} for log_record in span_logs: props = build_event_properties(log_record, log_record.get("attributes", {}), resource, scope) - # Merge properties (later logs override earlier ones) - accumulated_props = {**accumulated_props, **props} + # Merge properties with special handling for arrays + for key, value in props.items(): + if key in ("$ai_input", "$ai_output_choices"): + # Concatenate message arrays instead of overwriting + if key in accumulated_props and isinstance(accumulated_props[key], list): + accumulated_props[key] = accumulated_props[key] + value + else: + accumulated_props[key] = value + else: + # For non-array fields, later values override earlier ones + accumulated_props[key] = value # Now call event merger once with all accumulated properties from .event_merger import cache_and_merge_properties diff --git a/products/llm_analytics/backend/api/otel/logs_transformer.py b/products/llm_analytics/backend/api/otel/logs_transformer.py index 5a3f02c74b88..f3c53bf38cf1 100644 --- a/products/llm_analytics/backend/api/otel/logs_transformer.py +++ b/products/llm_analytics/backend/api/otel/logs_transformer.py @@ -158,15 +158,27 @@ def build_event_properties( properties["$ai_input"] = [{"role": role, "content": body["content"]}] # Assistant messages: {"content": "..."} or {"tool_calls": [...]} + # These are previous messages in conversation history, so they go into $ai_input elif "gen_ai.assistant.message" in event_name: message = {"role": "assistant"} if "content" in body: message["content"] = body["content"] if "tool_calls" in body: message["tool_calls"] = body["tool_calls"] - properties["$ai_output_choices"] = [message] + properties["$ai_input"] = [message] + + # Tool messages: {"content": "...", "id": "tool_call_id"} + # These are tool execution results in conversation history, so they go into $ai_input + elif "gen_ai.tool.message" in event_name: + message = {"role": "tool"} + if "content" in body: + message["content"] = body["content"] + if "id" in body: + message["tool_call_id"] = body["id"] + properties["$ai_input"] = [message] # Choice events: {"index": 0, "finish_reason": "stop", "message": {...}} + # This is the CURRENT response, so it goes into $ai_output_choices elif "gen_ai.choice" in event_name and "message" in body: message_obj = body["message"] choice = {"role": message_obj.get("role", "assistant")} diff --git a/products/llm_analytics/backend/api/otel/test_logs_accumulation.py b/products/llm_analytics/backend/api/otel/test_logs_accumulation.py new file mode 100644 index 000000000000..04f5fa24fea4 --- /dev/null +++ b/products/llm_analytics/backend/api/otel/test_logs_accumulation.py @@ -0,0 +1,213 @@ +""" +Unit tests for v2 logs accumulation in ingestion.py. + +Tests that multiple log events for the same span correctly accumulate +message arrays instead of overwriting them. +""" + +import pytest +from unittest.mock import patch + +from products.llm_analytics.backend.api.otel.ingestion import transform_logs_to_ai_events + + +def test_multiple_user_messages_accumulate(): + """Test that multiple user messages accumulate into $ai_input array.""" + # Simulate multiple log records for same span with different user messages + parsed_request = { + "logs": [ + { + "trace_id": "trace123", + "span_id": "span456", + "attributes": {"event.name": "gen_ai.user.message"}, + "body": {"content": "hi there"}, + "time_unix_nano": 1000000000, + }, + { + "trace_id": "trace123", + "span_id": "span456", + "attributes": {"event.name": "gen_ai.user.message"}, + "body": {"content": "k bye"}, + "time_unix_nano": 2000000000, + }, + ], + "resource": {"service.name": "test-service"}, + "scope": {"name": "test-scope"}, + } + + # Mock event merger to return merged properties on second call + with patch("products.llm_analytics.backend.api.otel.event_merger.cache_and_merge_properties") as mock_merger: + # First log: cache (return None) + # Second log: return merged (should have both messages) + def merger_side_effect(trace_id, span_id, props, is_trace): + # Check that props contains both messages on second call + if "$ai_input" in props and len(props["$ai_input"]) == 2: + return props # Return accumulated properties + return None # First arrival, cache + + mock_merger.side_effect = merger_side_effect + + _events = transform_logs_to_ai_events(parsed_request) + + # Verify accumulation happened before calling merger + # The merger should have been called once with both messages accumulated + assert mock_merger.call_count == 1 + call_args = mock_merger.call_args + props = call_args[0][2] # Third argument is properties + assert "$ai_input" in props + assert len(props["$ai_input"]) == 2 + assert props["$ai_input"][0]["content"] == "hi there" + assert props["$ai_input"][1]["content"] == "k bye" + + +def test_user_and_assistant_messages_accumulate(): + """Test that conversation history (including assistant messages) goes into $ai_input.""" + parsed_request = { + "logs": [ + { + "trace_id": "trace789", + "span_id": "span012", + "attributes": {"event.name": "gen_ai.user.message"}, + "body": {"content": "hello"}, + "time_unix_nano": 1000000000, + }, + { + "trace_id": "trace789", + "span_id": "span012", + "attributes": {"event.name": "gen_ai.assistant.message"}, + "body": {"content": "hi there!"}, + "time_unix_nano": 2000000000, + }, + { + "trace_id": "trace789", + "span_id": "span012", + "attributes": {"event.name": "gen_ai.user.message"}, + "body": {"content": "tell me a joke"}, + "time_unix_nano": 3000000000, + }, + { + "trace_id": "trace789", + "span_id": "span012", + "attributes": {"event.name": "gen_ai.choice"}, + "body": { + "message": {"role": "assistant", "content": "Why did the chicken cross the road?"}, + "finish_reason": "stop", + }, + "time_unix_nano": 4000000000, + }, + ], + "resource": {"service.name": "test-service"}, + "scope": {"name": "test-scope"}, + } + + with patch("products.llm_analytics.backend.api.otel.event_merger.cache_and_merge_properties") as mock_merger: + + def merger_side_effect(trace_id, span_id, props, is_trace): + # Return accumulated properties if we have both input and output + if "$ai_input" in props and "$ai_output_choices" in props: + return props + return None + + mock_merger.side_effect = merger_side_effect + + _events = transform_logs_to_ai_events(parsed_request) + + # Verify conversation history (user + assistant) accumulated in $ai_input + # and only current response in $ai_output_choices + assert mock_merger.call_count == 1 + call_args = mock_merger.call_args + props = call_args[0][2] + assert "$ai_input" in props + assert "$ai_output_choices" in props + # $ai_input should have: user1, assistant1, user2 (conversation context) + assert len(props["$ai_input"]) == 3 + assert props["$ai_input"][0]["content"] == "hello" + assert props["$ai_input"][1]["content"] == "hi there!" + assert props["$ai_input"][2]["content"] == "tell me a joke" + # $ai_output_choices should have only current response + assert len(props["$ai_output_choices"]) == 1 + assert props["$ai_output_choices"][0]["content"] == "Why did the chicken cross the road?" + + +def test_non_array_properties_are_overwritten(): + """Test that non-array properties use last-value-wins behavior.""" + parsed_request = { + "logs": [ + { + "trace_id": "trace111", + "span_id": "span222", + "attributes": { + "event.name": "gen_ai.user.message", + "gen_ai.request.model": "gpt-3.5", + "gen_ai.usage.input_tokens": 10, + }, + "body": {"content": "hello"}, + "time_unix_nano": 1000000000, + }, + { + "trace_id": "trace111", + "span_id": "span222", + "attributes": { + "event.name": "gen_ai.choice", + "gen_ai.response.model": "gpt-4", # Different model in response + "gen_ai.usage.output_tokens": 20, + }, + "body": {"message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}, + "time_unix_nano": 2000000000, + }, + ], + "resource": {"service.name": "test-service"}, + "scope": {"name": "test-scope"}, + } + + with patch("products.llm_analytics.backend.api.otel.event_merger.cache_and_merge_properties") as mock_merger: + + def merger_side_effect(trace_id, span_id, props, is_trace): + if "$ai_input" in props and "$ai_output_choices" in props: + return props + return None + + mock_merger.side_effect = merger_side_effect + + _events = transform_logs_to_ai_events(parsed_request) + + # Verify non-array properties were overwritten (last value wins) + call_args = mock_merger.call_args + props = call_args[0][2] + assert props["$ai_model"] == "gpt-4" # Second log's model wins + assert props["$ai_input_tokens"] == 10 # From first log + assert props["$ai_output_tokens"] == 20 # From second log + + +def test_single_log_event_works(): + """Test that single log events still work (no accumulation needed).""" + parsed_request = { + "logs": [ + { + "trace_id": "trace333", + "span_id": "span444", + "attributes": {"event.name": "gen_ai.user.message"}, + "body": {"content": "single message"}, + "time_unix_nano": 1000000000, + } + ], + "resource": {"service.name": "test-service"}, + "scope": {"name": "test-scope"}, + } + + with patch("products.llm_analytics.backend.api.otel.event_merger.cache_and_merge_properties") as mock_merger: + mock_merger.return_value = None # First arrival, cache + + _events = transform_logs_to_ai_events(parsed_request) + + # Verify single message was processed + assert mock_merger.call_count == 1 + call_args = mock_merger.call_args + props = call_args[0][2] + assert "$ai_input" in props + assert len(props["$ai_input"]) == 1 + assert props["$ai_input"][0]["content"] == "single message" + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) From fbf810b045c1be42fe0051d03c7cb41cb0201bb5 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Fri, 21 Nov 2025 12:44:37 +0000 Subject: [PATCH 25/34] test(otel): Add test coverage for tool message handling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add test_tool_messages_accumulate to verify: - gen_ai.tool.message events are properly parsed - Tool messages include role, content, and tool_call_id - Tool messages accumulate in $ai_input with conversation history - Message order is preserved (user → assistant with tool_calls → tool → final response) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .../api/otel/test_logs_accumulation.py | 90 +++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/products/llm_analytics/backend/api/otel/test_logs_accumulation.py b/products/llm_analytics/backend/api/otel/test_logs_accumulation.py index 04f5fa24fea4..a2d2fbc9ddf5 100644 --- a/products/llm_analytics/backend/api/otel/test_logs_accumulation.py +++ b/products/llm_analytics/backend/api/otel/test_logs_accumulation.py @@ -129,6 +129,96 @@ def merger_side_effect(trace_id, span_id, props, is_trace): assert props["$ai_output_choices"][0]["content"] == "Why did the chicken cross the road?" +def test_tool_messages_accumulate(): + """Test that tool messages are properly handled and accumulate in conversation history.""" + parsed_request = { + "logs": [ + { + "trace_id": "trace999", + "span_id": "span888", + "attributes": {"event.name": "gen_ai.user.message"}, + "body": {"content": "What's the weather in Paris?"}, + "time_unix_nano": 1000000000, + }, + { + "trace_id": "trace999", + "span_id": "span888", + "attributes": {"event.name": "gen_ai.assistant.message"}, + "body": { + "content": None, + "tool_calls": [ + { + "id": "call_123", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"location":"Paris"}'}, + } + ], + }, + "time_unix_nano": 2000000000, + }, + { + "trace_id": "trace999", + "span_id": "span888", + "attributes": {"event.name": "gen_ai.tool.message"}, + "body": {"content": "Sunny, 18°C", "id": "call_123"}, + "time_unix_nano": 3000000000, + }, + { + "trace_id": "trace999", + "span_id": "span888", + "attributes": {"event.name": "gen_ai.choice"}, + "body": { + "message": {"role": "assistant", "content": "The weather in Paris is sunny with 18°C."}, + "finish_reason": "stop", + }, + "time_unix_nano": 4000000000, + }, + ], + "resource": {"service.name": "test-service"}, + "scope": {"name": "test-scope"}, + } + + with patch("products.llm_analytics.backend.api.otel.event_merger.cache_and_merge_properties") as mock_merger: + + def merger_side_effect(trace_id, span_id, props, is_trace): + # Return accumulated properties once we have all messages + if "$ai_input" in props and "$ai_output_choices" in props and len(props["$ai_input"]) >= 3: + return props + return None + + mock_merger.side_effect = merger_side_effect + + _events = transform_logs_to_ai_events(parsed_request) + + # Verify tool message was properly accumulated + assert mock_merger.call_count == 1 + call_args = mock_merger.call_args + props = call_args[0][2] + + # $ai_input should have: user, assistant (with tool_calls), tool + assert "$ai_input" in props + assert len(props["$ai_input"]) == 3 + + # Verify user message + assert props["$ai_input"][0]["role"] == "user" + assert props["$ai_input"][0]["content"] == "What's the weather in Paris?" + + # Verify assistant message with tool_calls + assert props["$ai_input"][1]["role"] == "assistant" + assert "tool_calls" in props["$ai_input"][1] + assert props["$ai_input"][1]["tool_calls"][0]["id"] == "call_123" + + # Verify tool message with tool_call_id + assert props["$ai_input"][2]["role"] == "tool" + assert props["$ai_input"][2]["content"] == "Sunny, 18°C" + assert props["$ai_input"][2]["tool_call_id"] == "call_123" + + # Verify final response in output + assert "$ai_output_choices" in props + assert len(props["$ai_output_choices"]) == 1 + assert props["$ai_output_choices"][0]["content"] == "The weather in Paris is sunny with 18°C." + + def test_non_array_properties_are_overwritten(): """Test that non-array properties use last-value-wins behavior.""" parsed_request = { From 36cf7480819ad74ea306600ddf24b541d1ca7df1 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Fri, 21 Nov 2025 16:49:58 +0000 Subject: [PATCH 26/34] fix(otel): Add Mastra OTEL ingestion support with provider transformers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Detect Mastra by instrumentation scope name (@mastra/otel) - Treat Mastra as v1 framework (all attributes in spans, no log merging) - Mark v1 framework root spans as $ai_span instead of $ai_trace to fix tree hierarchy - Add provider transformer pattern for framework-specific data transformations - Filter out raw input/output attributes to prevent duplicate otel.input/otel.output Fixes tree display issue where Mastra generations weren't appearing as children under the trace. Root spans from v1 frameworks must be $ai_span (not $ai_trace) since TraceQueryRunner filters out $ai_trace events from the events array. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .../backend/api/otel/conventions/genai.py | 87 ++++++++- .../otel/conventions/providers/__init__.py | 22 +++ .../api/otel/conventions/providers/base.py | 67 +++++++ .../api/otel/conventions/providers/mastra.py | 119 +++++++++++++ .../otel/conventions/providers/test_mastra.py | 166 ++++++++++++++++++ .../backend/api/otel/transformer.py | 26 ++- 6 files changed, 479 insertions(+), 8 deletions(-) create mode 100644 products/llm_analytics/backend/api/otel/conventions/providers/__init__.py create mode 100644 products/llm_analytics/backend/api/otel/conventions/providers/base.py create mode 100644 products/llm_analytics/backend/api/otel/conventions/providers/mastra.py create mode 100644 products/llm_analytics/backend/api/otel/conventions/providers/test_mastra.py diff --git a/products/llm_analytics/backend/api/otel/conventions/genai.py b/products/llm_analytics/backend/api/otel/conventions/genai.py index 62248d8f799f..55627b6cc8c7 100644 --- a/products/llm_analytics/backend/api/otel/conventions/genai.py +++ b/products/llm_analytics/backend/api/otel/conventions/genai.py @@ -4,12 +4,17 @@ Implements the GenAI semantic conventions (gen_ai.*) as fallback when PostHog-native attributes are not present. +Supports provider-specific transformations for frameworks like Mastra +that use custom OTEL formats. + Reference: https://opentelemetry.io/docs/specs/semconv/gen-ai/ """ from collections import defaultdict from typing import Any +from .providers import PROVIDER_TRANSFORMERS + def has_genai_attributes(span: dict[str, Any]) -> bool: """Check if span uses GenAI semantic conventions.""" @@ -60,16 +65,44 @@ def _extract_indexed_messages(attributes: dict[str, Any], prefix: str) -> list[d return messages if messages else None -def extract_genai_attributes(span: dict[str, Any]) -> dict[str, Any]: +def extract_genai_attributes(span: dict[str, Any], scope: dict[str, Any] | None = None) -> dict[str, Any]: """ Extract GenAI semantic convention attributes from span. GenAI conventions use `gen_ai.*` prefix and are fallback when PostHog-native attributes are not present. + + Supports provider-specific transformations for frameworks that use + custom OTEL formats (e.g., Mastra). + + Args: + span: Parsed OTEL span + scope: Instrumentation scope info (for provider detection) + + Returns: + Extracted attributes dict """ + import structlog + + logger = structlog.get_logger(__name__) attributes = span.get("attributes", {}) + scope = scope or {} result: dict[str, Any] = {} + # Detect provider-specific transformer + provider_transformer = None + for transformer_class in PROVIDER_TRANSFORMERS: + transformer = transformer_class() + if transformer.can_handle(span, scope): + provider_transformer = transformer + logger.info( + "provider_transformer_detected", + provider=transformer.get_provider_name(), + scope_name=scope.get("name"), + span_name=span.get("name"), + ) + break + # Model (prefer request, fallback to response, then system) model = ( attributes.get("gen_ai.request.model") @@ -100,14 +133,62 @@ def extract_genai_attributes(span: dict[str, Any]) -> dict[str, Any]: result["prompt"] = prompts # Fallback to direct gen_ai.prompt attribute elif (prompt := attributes.get("gen_ai.prompt")) is not None: - result["prompt"] = prompt + # Try provider-specific transformation + if provider_transformer: + logger.info( + "provider_transform_prompt_attempt", + provider=provider_transformer.get_provider_name(), + prompt_type=type(prompt).__name__, + prompt_length=len(str(prompt)) if prompt else 0, + ) + transformed = provider_transformer.transform_prompt(prompt) + if transformed is not None: + logger.info( + "provider_transform_prompt_success", + provider=provider_transformer.get_provider_name(), + result_type=type(transformed).__name__, + result_length=len(transformed) if isinstance(transformed, list) else 0, + ) + result["prompt"] = transformed + else: + logger.info( + "provider_transform_prompt_none", + provider=provider_transformer.get_provider_name(), + ) + result["prompt"] = prompt + else: + result["prompt"] = prompt completions = _extract_indexed_messages(attributes, "gen_ai.completion") if completions: result["completion"] = completions # Fallback to direct gen_ai.completion attribute elif (completion := attributes.get("gen_ai.completion")) is not None: - result["completion"] = completion + # Try provider-specific transformation + if provider_transformer: + logger.info( + "provider_transform_completion_attempt", + provider=provider_transformer.get_provider_name(), + completion_type=type(completion).__name__, + completion_length=len(str(completion)) if completion else 0, + ) + transformed = provider_transformer.transform_completion(completion) + if transformed is not None: + logger.info( + "provider_transform_completion_success", + provider=provider_transformer.get_provider_name(), + result_type=type(transformed).__name__, + result_length=len(transformed) if isinstance(transformed, list) else 0, + ) + result["completion"] = transformed + else: + logger.info( + "provider_transform_completion_none", + provider=provider_transformer.get_provider_name(), + ) + result["completion"] = completion + else: + result["completion"] = completion # Model parameters if (temperature := attributes.get("gen_ai.request.temperature")) is not None: diff --git a/products/llm_analytics/backend/api/otel/conventions/providers/__init__.py b/products/llm_analytics/backend/api/otel/conventions/providers/__init__.py new file mode 100644 index 000000000000..9f05237b0aa4 --- /dev/null +++ b/products/llm_analytics/backend/api/otel/conventions/providers/__init__.py @@ -0,0 +1,22 @@ +""" +Provider-specific OTEL transformers. + +Each provider (Mastra, Langchain, LlamaIndex, etc.) handles their +specific OTEL format quirks and normalizes to PostHog format. +""" + +from .base import ProviderTransformer +from .mastra import MastraTransformer + +# Registry of all available provider transformers +# Add new providers here as they're implemented +PROVIDER_TRANSFORMERS: list[type[ProviderTransformer]] = [ + MastraTransformer, + # Future: LangchainTransformer, LlamaIndexTransformer, etc. +] + +__all__ = [ + "ProviderTransformer", + "MastraTransformer", + "PROVIDER_TRANSFORMERS", +] diff --git a/products/llm_analytics/backend/api/otel/conventions/providers/base.py b/products/llm_analytics/backend/api/otel/conventions/providers/base.py new file mode 100644 index 000000000000..417be1ef1fac --- /dev/null +++ b/products/llm_analytics/backend/api/otel/conventions/providers/base.py @@ -0,0 +1,67 @@ +""" +Base provider transformer interface. + +Provider transformers handle framework/library-specific OTEL formats +and normalize them to PostHog's standard format. +""" + +from abc import ABC, abstractmethod +from typing import Any + + +class ProviderTransformer(ABC): + """ + Base class for provider-specific OTEL transformers. + + Each provider (Mastra, Langchain, LlamaIndex, etc.) can implement + a transformer to handle their specific OTEL format quirks. + """ + + @abstractmethod + def can_handle(self, span: dict[str, Any], scope: dict[str, Any]) -> bool: + """ + Detect if this transformer can handle the given span. + + Args: + span: Parsed OTEL span + scope: Instrumentation scope info + + Returns: + True if this transformer recognizes and can handle this span + """ + pass + + @abstractmethod + def transform_prompt(self, prompt: Any) -> Any: + """ + Transform provider-specific prompt format to standard format. + + Args: + prompt: Raw prompt value from gen_ai.prompt attribute + + Returns: + Normalized prompt (list of message dicts, string, or None if no transformation needed) + """ + pass + + @abstractmethod + def transform_completion(self, completion: Any) -> Any: + """ + Transform provider-specific completion format to standard format. + + Args: + completion: Raw completion value from gen_ai.completion attribute + + Returns: + Normalized completion (list of message dicts, string, or None if no transformation needed) + """ + pass + + def get_provider_name(self) -> str: + """ + Get the provider name for logging/debugging. + + Returns: + Human-readable provider name + """ + return self.__class__.__name__.replace("Transformer", "") diff --git a/products/llm_analytics/backend/api/otel/conventions/providers/mastra.py b/products/llm_analytics/backend/api/otel/conventions/providers/mastra.py new file mode 100644 index 000000000000..6bb09a4f6f14 --- /dev/null +++ b/products/llm_analytics/backend/api/otel/conventions/providers/mastra.py @@ -0,0 +1,119 @@ +""" +Mastra provider transformer. + +Handles Mastra's OTEL format which wraps messages in custom structures: +- Input: {"messages": [{"role": "user", "content": [...]}]} +- Output: {"files": [], "text": "...", "warnings": [], ...} +""" + +import json +from typing import Any + +from .base import ProviderTransformer + + +class MastraTransformer(ProviderTransformer): + """ + Transform Mastra's OTEL format to PostHog standard format. + + Mastra uses @mastra/otel instrumentation scope and wraps messages + in custom structures that need unwrapping. + """ + + def can_handle(self, span: dict[str, Any], scope: dict[str, Any]) -> bool: + """ + Detect Mastra by instrumentation scope name. + + Mastra sets scope.name to "@mastra/otel" in its span converter. + """ + scope_name = scope.get("name", "") + + # Primary detection: instrumentation scope + if scope_name == "@mastra/otel": + return True + + # Fallback: check for mastra-prefixed attributes + attributes = span.get("attributes", {}) + return any(key.startswith("mastra.") for key in attributes.keys()) + + def transform_prompt(self, prompt: Any) -> Any: + """ + Transform Mastra's wrapped input format. + + Mastra wraps messages as: {"messages": [{"role": "user", "content": [...]}]} + where content can be an array of objects like [{"type": "text", "text": "..."}] + """ + import structlog + + logger = structlog.get_logger(__name__) + + if not isinstance(prompt, str): + logger.info("mastra_transform_prompt_skip_not_string", prompt_type=type(prompt).__name__) + return None # No transformation needed + + try: + parsed = json.loads(prompt) + logger.info( + "mastra_transform_prompt_parsed", + has_messages=("messages" in parsed) if isinstance(parsed, dict) else False, + parsed_type=type(parsed).__name__, + ) + + # Check for Mastra input format: {"messages": [...]} + if not isinstance(parsed, dict) or "messages" not in parsed: + return None # Not Mastra format + + messages = parsed["messages"] + if not isinstance(messages, list): + return None + + # Transform Mastra messages to standard format + result = [] + for msg in messages: + if not isinstance(msg, dict) or "role" not in msg: + continue + + # Handle Mastra's content array format: [{"type": "text", "text": "..."}] + if "content" in msg and isinstance(msg["content"], list): + text_parts = [] + for content_item in msg["content"]: + if isinstance(content_item, dict): + if content_item.get("type") == "text" and "text" in content_item: + text_parts.append(content_item["text"]) + + if text_parts: + result.append({"role": msg["role"], "content": " ".join(text_parts)}) + else: + # Keep as-is if we can't extract text + result.append(msg) + else: + # Standard format message + result.append(msg) + + return result if result else None + + except (json.JSONDecodeError, TypeError, KeyError): + return None + + def transform_completion(self, completion: Any) -> Any: + """ + Transform Mastra's wrapped output format. + + Mastra wraps output as: {"files": [], "text": "...", "warnings": [], ...} + Extract just the text content. + """ + if not isinstance(completion, str): + return None # No transformation needed + + try: + parsed = json.loads(completion) + + # Check for Mastra output format: {"text": "...", ...} + if not isinstance(parsed, dict) or "text" not in parsed: + return None # Not Mastra format + + # Extract text content as assistant message + return [{"role": "assistant", "content": parsed["text"]}] + + except (json.JSONDecodeError, TypeError, KeyError): + return None diff --git a/products/llm_analytics/backend/api/otel/conventions/providers/test_mastra.py b/products/llm_analytics/backend/api/otel/conventions/providers/test_mastra.py new file mode 100644 index 000000000000..b6da88cb4be2 --- /dev/null +++ b/products/llm_analytics/backend/api/otel/conventions/providers/test_mastra.py @@ -0,0 +1,166 @@ +""" +Tests for Mastra provider transformer. +""" + +import json + +import pytest + +from products.llm_analytics.backend.api.otel.conventions.providers.mastra import MastraTransformer + + +class TestMastraTransformer: + """Test Mastra-specific format transformations.""" + + def setup_method(self): + """Setup test instance.""" + self.transformer = MastraTransformer() + + def test_can_handle_mastra_scope(self): + """Test detection by @mastra/otel scope name.""" + span = {"attributes": {}} + scope = {"name": "@mastra/otel", "version": "1.0.0"} + + assert self.transformer.can_handle(span, scope) is True + + def test_can_handle_mastra_attributes(self): + """Test detection by mastra.* attributes.""" + span = { + "attributes": { + "mastra.trace_id": "abc123", + "mastra.span_id": "def456", + } + } + scope = {"name": "other"} + + assert self.transformer.can_handle(span, scope) is True + + def test_cannot_handle_non_mastra(self): + """Test that non-Mastra spans are not handled.""" + span = {"attributes": {"gen_ai.system": "openai"}} + scope = {"name": "opentelemetry-instrumentation-openai"} + + assert self.transformer.can_handle(span, scope) is False + + def test_transform_prompt_simple_messages(self): + """Test transforming Mastra's wrapped messages format.""" + mastra_input = json.dumps( + { + "messages": [ + {"role": "system", "content": "You are a helpful assistant"}, + {"role": "user", "content": "Hello"}, + ] + } + ) + + result = self.transformer.transform_prompt(mastra_input) + + assert result is not None + assert len(result) == 2 + assert result[0] == {"role": "system", "content": "You are a helpful assistant"} + assert result[1] == {"role": "user", "content": "Hello"} + + def test_transform_prompt_with_content_array(self): + """Test transforming Mastra's content array format.""" + mastra_input = json.dumps( + { + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What's the weather"}, + {"type": "text", "text": " in Paris?"}, + ], + } + ] + } + ) + + result = self.transformer.transform_prompt(mastra_input) + + assert result is not None + assert len(result) == 1 + assert result[0]["role"] == "user" + assert result[0]["content"] == "What's the weather in Paris?" + + def test_transform_prompt_non_mastra_format(self): + """Test that non-Mastra formats return None.""" + # Standard GenAI format (already correct) + standard_input = json.dumps([{"role": "user", "content": "Hello"}]) + + result = self.transformer.transform_prompt(standard_input) + + assert result is None # No transformation needed + + def test_transform_prompt_invalid_json(self): + """Test handling of invalid JSON.""" + result = self.transformer.transform_prompt("not valid json") + + assert result is None + + def test_transform_prompt_non_string(self): + """Test handling of non-string input.""" + result = self.transformer.transform_prompt(["already", "a", "list"]) + + assert result is None + + def test_transform_completion_text_format(self): + """Test transforming Mastra's output format.""" + mastra_output = json.dumps( + {"files": [], "text": "The weather in Paris is sunny.", "warnings": [], "reasoning": [], "sources": []} + ) + + result = self.transformer.transform_completion(mastra_output) + + assert result is not None + assert len(result) == 1 + assert result[0] == {"role": "assistant", "content": "The weather in Paris is sunny."} + + def test_transform_completion_non_mastra_format(self): + """Test that non-Mastra output formats return None.""" + standard_output = json.dumps([{"role": "assistant", "content": "Hello"}]) + + result = self.transformer.transform_completion(standard_output) + + assert result is None # No transformation needed + + def test_transform_completion_invalid_json(self): + """Test handling of invalid JSON in completion.""" + result = self.transformer.transform_completion("not valid json") + + assert result is None + + def test_transform_completion_non_string(self): + """Test handling of non-string completion.""" + result = self.transformer.transform_completion({"already": "dict"}) + + assert result is None + + def test_end_to_end_conversation(self): + """Test a full conversation flow with Mastra format.""" + # Simulate what Mastra sends + prompt = json.dumps( + { + "messages": [ + {"role": "system", "content": "You are helpful"}, + {"role": "user", "content": [{"type": "text", "text": "Hi there!"}]}, + ] + } + ) + + completion = json.dumps({"text": "Hello! How can I help you?", "files": [], "warnings": []}) + + # Transform + prompt_result = self.transformer.transform_prompt(prompt) + completion_result = self.transformer.transform_completion(completion) + + # Verify + assert prompt_result == [ + {"role": "system", "content": "You are helpful"}, + {"role": "user", "content": "Hi there!"}, + ] + assert completion_result == [{"role": "assistant", "content": "Hello! How can I help you?"}] + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/products/llm_analytics/backend/api/otel/transformer.py b/products/llm_analytics/backend/api/otel/transformer.py index 09f9d079a946..9d98e54d7f65 100644 --- a/products/llm_analytics/backend/api/otel/transformer.py +++ b/products/llm_analytics/backend/api/otel/transformer.py @@ -51,7 +51,7 @@ def transform_span_to_ai_event( # Extract attributes using waterfall pattern posthog_attrs = extract_posthog_native_attributes(span) - genai_attrs = extract_genai_attributes(span) + genai_attrs = extract_genai_attributes(span, scope) # Merge with precedence: PostHog > GenAI merged_attrs = {**genai_attrs, **posthog_attrs} @@ -62,7 +62,11 @@ def transform_span_to_ai_event( # Detect v1 vs v2 instrumentation: # v1: Everything in span attributes (extracted as "prompt", "completion") - send immediately # v2: Metadata in span, content in logs - use event merger - is_v1_span = bool(merged_attrs.get("prompt") or merged_attrs.get("completion")) + # Some frameworks (e.g., Mastra) are v1 but may have spans without prompt/completion (parent spans) + # Detect these by instrumentation scope name + scope_name = scope.get("name", "") + is_v1_framework = scope_name == "@mastra/otel" # Mastra uses v1 (attributes in spans) + is_v1_span = bool(merged_attrs.get("prompt") or merged_attrs.get("completion")) or is_v1_framework if not is_v1_span: # v2 instrumentation - use event merger for bidirectional merge with logs @@ -79,7 +83,7 @@ def transform_span_to_ai_event( # else: v1 span has everything - send immediately without merging # Determine event type - event_type = determine_event_type(span, merged_attrs) + event_type = determine_event_type(span, merged_attrs, scope) # Calculate timestamp timestamp = calculate_timestamp(span) @@ -272,6 +276,8 @@ def build_event_properties( "gen_ai.prompt", "gen_ai.completion", "service.name", + "input", # Generic input (e.g., Mastra for Laminar compatibility) + "output", # Generic output (e.g., Mastra for Laminar compatibility) } for key, value in attributes.items(): @@ -333,8 +339,9 @@ def _extract_tools_from_attributes(attributes: dict[str, Any]) -> list[dict[str, return [tools_by_index[i] for i in sorted(tools_by_index.keys())] -def determine_event_type(span: dict[str, Any], attrs: dict[str, Any]) -> str: +def determine_event_type(span: dict[str, Any], attrs: dict[str, Any], scope: dict[str, Any] | None = None) -> str: """Determine AI event type from span.""" + scope = scope or {} op_name = attrs.get("operation_name", "").lower() # Check operation name first (highest priority) @@ -352,8 +359,17 @@ def determine_event_type(span: dict[str, Any], attrs: dict[str, Any]) -> str: return "$ai_generation" # Check if span is root (no parent) + # For v1 frameworks (like Mastra), root spans should be $ai_span, not $ai_trace + # $ai_trace events get filtered out by TraceQueryRunner, breaking tree hierarchy if not span.get("parent_span_id"): - return "$ai_trace" + scope_name = scope.get("name", "") + is_v1_framework = scope_name == "@mastra/otel" + if is_v1_framework: + # v1 frameworks: root span should be $ai_span (will be included in tree) + return "$ai_span" + else: + # v2 frameworks: root is $ai_trace (separate from event tree) + return "$ai_trace" # Default to generic span return "$ai_span" From e9f3cffccdb600a847e17f3795c982fa90274b1b Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Fri, 21 Nov 2025 16:52:57 +0000 Subject: [PATCH 27/34] docs(otel): Update README with provider transformers and Mastra support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Document provider transformers pattern for framework-specific data transformations - Add Mastra to architecture diagram and v1 instrumentation section - Explain v1 framework detection via instrumentation scope name - Document event type determination for v1 frameworks ($ai_span for root spans) - Add section on adding new provider transformers - Update design decisions with provider transformers and v1 event type logic 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .../llm_analytics/backend/api/otel/README.md | 93 +++++++++++++++++-- 1 file changed, 85 insertions(+), 8 deletions(-) diff --git a/products/llm_analytics/backend/api/otel/README.md b/products/llm_analytics/backend/api/otel/README.md index eaf2c0d0e15e..db62015409ca 100644 --- a/products/llm_analytics/backend/api/otel/README.md +++ b/products/llm_analytics/backend/api/otel/README.md @@ -23,6 +23,11 @@ OTLP HTTP Request | | | | | +---> posthog_native.py | | +---> genai.py + | | | | + | | | +---> providers/ (framework-specific transformations) + | | | | + | | | +---> mastra.py + | | | +---> base.py | | | +---> event_merger.py (Redis cache for v2) | @@ -74,9 +79,14 @@ v1 instrumentation sends complete LLM call data within span attributes using ind - Completions: `gen_ai.completion.0.role`, `gen_ai.completion.0.content` - Metadata: `gen_ai.request.model`, `gen_ai.usage.input_tokens`, etc. -**Processing**: When a span contains `prompt` or `completion` attributes (after extraction), the transformer recognizes it as v1 and sends the event immediately without caching. This works because v1 spans are self-contained. +**Processing**: The transformer recognizes v1 in two ways: -**Package**: `opentelemetry-instrumentation-openai` +1. Span contains `prompt` or `completion` attributes (after extraction) +2. Framework detection via instrumentation scope name (e.g., `@mastra/otel` for Mastra) + +When detected, events are sent immediately without caching since v1 spans are self-contained. + +**Packages**: `opentelemetry-instrumentation-openai`, Mastra framework (`@mastra/otel-exporter`) ### v2 Instrumentation @@ -104,17 +114,19 @@ Main entry point for OTLP HTTP requests. Parses protobuf payloads and routes to Converts OTel spans to PostHog AI events using a waterfall attribute extraction pattern: 1. Extract PostHog-native attributes (highest priority) -2. Extract GenAI semantic convention attributes (fallback) +2. Extract GenAI semantic convention attributes (fallback, with provider transformers) 3. Merge with PostHog attributes taking precedence Determines event type based on span characteristics: - `$ai_generation`: LLM completion requests (has model, tokens, and input) - `$ai_embedding`: Embedding requests (operation_name matches embedding patterns) -- `$ai_trace`: Root spans (no parent) -- `$ai_span`: All other spans +- `$ai_trace`: Root spans (no parent) for v2 frameworks +- `$ai_span`: All other spans, including root spans from v1 frameworks + +**v1 Detection**: Checks for `prompt` or `completion` attributes OR framework scope name (e.g., `@mastra/otel`). v1 spans bypass the event merger. -Detects v1 vs v2 by checking for `prompt` or `completion` in extracted attributes. v1 spans bypass the event merger. +**Event Type Logic**: For v1 frameworks like Mastra, root spans are marked as `$ai_span` (not `$ai_trace`) to ensure they appear in the tree hierarchy. This is necessary because `TraceQueryRunner` filters out `$ai_trace` events from the events array. ### logs_transformer.py @@ -141,7 +153,12 @@ Attribute extraction modules implementing semantic conventions: **posthog_native.py**: Extracts PostHog-specific attributes prefixed with `posthog.ai.*`. These take precedence in the waterfall. -**genai.py**: Extracts OpenTelemetry GenAI semantic convention attributes (`gen_ai.*`). Handles indexed message fields by collecting attributes like `gen_ai.prompt.0.role` into structured message arrays. +**genai.py**: Extracts OpenTelemetry GenAI semantic convention attributes (`gen_ai.*`). Handles indexed message fields by collecting attributes like `gen_ai.prompt.0.role` into structured message arrays. Supports provider-specific transformations for frameworks that use custom OTEL formats. + +**providers/**: Framework-specific transformers for handling custom OTEL formats: + +- **base.py**: Abstract base class defining the provider transformer interface (`can_handle()`, `transform_prompt()`, `transform_completion()`) +- **mastra.py**: Transforms Mastra's wrapped message format (e.g., `{"messages": [...]}` for input, `{"text": "...", "files": [], ...}` for output) into standard PostHog format. Detected by instrumentation scope name `@mastra/otel`. ## Event Schema @@ -237,7 +254,22 @@ v2 can send multiple log events in a single HTTP request. The ingestion layer gr ### v1/v2 Detection -Rather than requiring explicit configuration, the transformer auto-detects instrumentation version by checking for `prompt` or `completion` attributes. This allows both patterns to coexist without configuration. +Rather than requiring explicit configuration, the transformer auto-detects instrumentation version by: + +1. Checking for `prompt` or `completion` attributes (after extraction) +2. Detecting framework via instrumentation scope name (e.g., `@mastra/otel`) + +This allows both patterns to coexist without configuration, and supports frameworks that don't follow standard attribute conventions. + +### Provider Transformers + +Some frameworks (like Mastra) wrap OTEL data in custom structures that don't match standard GenAI conventions. Provider transformers detect these frameworks (via instrumentation scope or attribute prefixes) and unwrap their data into standard format. This keeps framework-specific logic isolated while maintaining compatibility with the core transformer pipeline. + +**Example**: Mastra wraps prompts as `{"messages": [{"role": "user", "content": [...]}]}` where content is an array of `{"type": "text", "text": "..."}` objects. The Mastra transformer unwraps this into standard `[{"role": "user", "content": "..."}]` format. + +### Event Type Determination for v1 Frameworks + +v1 frameworks create root spans that should appear in the tree hierarchy alongside their children. These root spans are marked as `$ai_span` (not `$ai_trace`) because `TraceQueryRunner` filters out `$ai_trace` events from the events array. This ensures v1 framework traces display correctly with proper parent-child relationships in the UI. ### TTL-Based Cleanup @@ -245,6 +277,51 @@ The event merger uses 60-second TTL on cache entries. This automatically cleans ## Extending the System +### Adding New Provider Transformers + +Create a new transformer in `conventions/providers/`: + +```python +from .base import ProviderTransformer +from typing import Any + +class CustomFrameworkTransformer(ProviderTransformer): + """Transform CustomFramework's OTEL format.""" + + def can_handle(self, span: dict[str, Any], scope: dict[str, Any]) -> bool: + """Detect CustomFramework by scope name or attributes.""" + scope_name = scope.get("name", "") + return scope_name == "custom-framework-scope" + + def transform_prompt(self, prompt: Any) -> Any: + """Transform wrapped prompt format to standard.""" + if not isinstance(prompt, str): + return None + + try: + parsed = json.loads(prompt) + # Transform custom format to standard + return [{"role": "user", "content": parsed["text"]}] + except (json.JSONDecodeError, KeyError): + return None + + def transform_completion(self, completion: Any) -> Any: + """Transform wrapped completion format to standard.""" + # Similar transformation logic + pass +``` + +Register in `conventions/providers/__init__.py`: + +```python +from .custom_framework import CustomFrameworkTransformer + +PROVIDER_TRANSFORMERS = [ + CustomFrameworkTransformer, + MastraTransformer, +] +``` + ### Adding New Semantic Conventions Create a new extractor in `conventions/`: From 1e0264dc9fad742991831090131888e378acb7eb Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Thu, 27 Nov 2025 11:43:52 +0000 Subject: [PATCH 28/34] refactor(otel): Add OtelInstrumentationPattern enum for provider pattern detection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add OtelInstrumentationPattern enum (V1_ATTRIBUTES, V2_TRACES_AND_LOGS) - Providers declare pattern via get_instrumentation_pattern() method - Add detect_provider() for centralized provider detection - Remove hardcoded @mastra/otel checks from transformer.py - Update README with Provider Reference section and pattern documentation - Add test_ingestion_parity.py for v1/v2 parity testing - Document Mastra behavior (no conversation history accumulation) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- docker-compose.base.yml | 2 - posthog/urls.py | 4 +- .../llm_analytics/backend/api/otel/README.md | 113 +++- .../backend/api/otel/conventions/genai.py | 46 +- .../otel/conventions/providers/__init__.py | 3 +- .../api/otel/conventions/providers/base.py | 39 ++ .../api/otel/conventions/providers/mastra.py | 22 +- .../backend/api/otel/ingestion.py | 19 +- .../backend/api/otel/test_ingestion_parity.py | 550 ++++++++++++++++++ .../backend/api/otel/transformer.py | 63 +- 10 files changed, 799 insertions(+), 62 deletions(-) create mode 100644 products/llm_analytics/backend/api/otel/test_ingestion_parity.py diff --git a/docker-compose.base.yml b/docker-compose.base.yml index 465bc08153e6..5882def3ebfa 100644 --- a/docker-compose.base.yml +++ b/docker-compose.base.yml @@ -264,8 +264,6 @@ services: KAFKA_HOSTS: kafka:9092 JWT_SECRET: '' KAFKA_TOPIC: logs_ingestion - ports: - - '4318:4318' networks: - otel_network - default diff --git a/posthog/urls.py b/posthog/urls.py index bab1a59761f1..00772ec0dac9 100644 --- a/posthog/urls.py +++ b/posthog/urls.py @@ -212,9 +212,9 @@ def proxy_logs_to_capture_service(request: HttpRequest) -> HttpResponse: *ee_urlpatterns, # api # OpenTelemetry traces ingestion for LLM Analytics - path("api/projects//ai/otel/v1/traces", csrf_exempt(otel_traces_endpoint)), + path("api/projects//ai/otel/traces", csrf_exempt(otel_traces_endpoint)), # OpenTelemetry logs ingestion for LLM Analytics - path("api/projects//ai/otel/v1/logs", csrf_exempt(otel_logs_endpoint)), + path("api/projects//ai/otel/logs", csrf_exempt(otel_logs_endpoint)), # OpenTelemetry logs proxy to capture-logs service (legacy) path("i/v1/logs", proxy_logs_to_capture_service), path("api/environments//progress/", progress), diff --git a/products/llm_analytics/backend/api/otel/README.md b/products/llm_analytics/backend/api/otel/README.md index db62015409ca..4c5c119d0476 100644 --- a/products/llm_analytics/backend/api/otel/README.md +++ b/products/llm_analytics/backend/api/otel/README.md @@ -124,9 +124,15 @@ Determines event type based on span characteristics: - `$ai_trace`: Root spans (no parent) for v2 frameworks - `$ai_span`: All other spans, including root spans from v1 frameworks -**v1 Detection**: Checks for `prompt` or `completion` attributes OR framework scope name (e.g., `@mastra/otel`). v1 spans bypass the event merger. +**Pattern Detection**: Uses `OtelInstrumentationPattern` enum to determine routing: -**Event Type Logic**: For v1 frameworks like Mastra, root spans are marked as `$ai_span` (not `$ai_trace`) to ensure they appear in the tree hierarchy. This is necessary because `TraceQueryRunner` filters out `$ai_trace` events from the events array. +1. Provider declares pattern via `get_instrumentation_pattern()` (most reliable) +2. Span has `prompt` or `completion` attributes (indicates V1 data present) +3. Default to V2 (safer - waits for logs rather than sending incomplete) + +V1 spans bypass the event merger and are sent immediately. + +**Event Type Logic**: For V1 frameworks, root spans are marked as `$ai_span` (not `$ai_trace`) to ensure they appear in the tree hierarchy. This is necessary because `TraceQueryRunner` filters out `$ai_trace` events from the events array. ### logs_transformer.py @@ -153,12 +159,16 @@ Attribute extraction modules implementing semantic conventions: **posthog_native.py**: Extracts PostHog-specific attributes prefixed with `posthog.ai.*`. These take precedence in the waterfall. -**genai.py**: Extracts OpenTelemetry GenAI semantic convention attributes (`gen_ai.*`). Handles indexed message fields by collecting attributes like `gen_ai.prompt.0.role` into structured message arrays. Supports provider-specific transformations for frameworks that use custom OTEL formats. +**genai.py**: Extracts OpenTelemetry GenAI semantic convention attributes (`gen_ai.*`). Handles indexed message fields by collecting attributes like `gen_ai.prompt.0.role` into structured message arrays. Provides `detect_provider()` function for centralized provider detection. Supports provider-specific transformations for frameworks that use custom OTEL formats. **providers/**: Framework-specific transformers for handling custom OTEL formats: -- **base.py**: Abstract base class defining the provider transformer interface (`can_handle()`, `transform_prompt()`, `transform_completion()`) -- **mastra.py**: Transforms Mastra's wrapped message format (e.g., `{"messages": [...]}` for input, `{"text": "...", "files": [], ...}` for output) into standard PostHog format. Detected by instrumentation scope name `@mastra/otel`. +- **base.py**: Abstract base class defining the provider transformer interface: + - `can_handle()`: Detect if transformer handles this span + - `transform_prompt()`: Transform provider-specific prompt format + - `transform_completion()`: Transform provider-specific completion format + - `get_instrumentation_pattern()`: Declare V1 or V2 pattern (returns `OtelInstrumentationPattern` enum) +- **mastra.py**: Transforms Mastra's wrapped message format (e.g., `{"messages": [...]}` for input, `{"text": "...", "files": [], ...}` for output) into standard PostHog format. Detected by instrumentation scope name `@mastra/otel`. Declares `V1_ATTRIBUTES` pattern. ## Event Schema @@ -195,13 +205,13 @@ All events conform to the PostHog LLM Analytics schema: ## API Endpoints -**Traces**: `POST /api/projects/{project_id}/ai/otel/v1/traces` +**Traces**: `POST /api/projects/{project_id}/ai/otel/traces` - Content-Type: `application/x-protobuf` - Authorization: `Bearer {project_api_key}` - Accepts OTLP trace payloads -**Logs**: `POST /api/projects/{project_id}/ai/otel/v1/logs` +**Logs**: `POST /api/projects/{project_id}/ai/otel/logs` - Content-Type: `application/x-protobuf` - Authorization: `Bearer {project_api_key}` @@ -228,12 +238,12 @@ from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExport from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter trace_exporter = OTLPSpanExporter( - endpoint=f"{posthog_host}/api/projects/{project_id}/ai/otel/v1/traces", + endpoint=f"{posthog_host}/api/projects/{project_id}/ai/otel/traces", headers={"Authorization": f"Bearer {api_key}"} ) log_exporter = OTLPLogExporter( - endpoint=f"{posthog_host}/api/projects/{project_id}/ai/otel/v1/logs", + endpoint=f"{posthog_host}/api/projects/{project_id}/ai/otel/logs", headers={"Authorization": f"Bearer {api_key}"} ) ``` @@ -252,14 +262,15 @@ The merger returns None on first arrival rather than blocking. This prevents the v2 can send multiple log events in a single HTTP request. The ingestion layer groups these by (trace_id, span_id) and accumulates their properties before calling the merger. This prevents race conditions where partial log data gets merged before all logs arrive. -### v1/v2 Detection +### Pattern Detection via Provider Transformers -Rather than requiring explicit configuration, the transformer auto-detects instrumentation version by: +Rather than hardcoding framework names, the transformer uses a layered detection approach: -1. Checking for `prompt` or `completion` attributes (after extraction) -2. Detecting framework via instrumentation scope name (e.g., `@mastra/otel`) +1. **Provider declaration** (most reliable): Providers implement `get_instrumentation_pattern()` returning `OtelInstrumentationPattern.V1_ATTRIBUTES` or `V2_TRACES_AND_LOGS` +2. **Content detection** (fallback): Span has `prompt` or `completion` attributes after extraction +3. **Safe default**: Unknown providers default to V2 (waits for logs rather than sending incomplete events) -This allows both patterns to coexist without configuration, and supports frameworks that don't follow standard attribute conventions. +This allows both patterns to coexist without configuration, and new providers only need to declare their pattern in one place. ### Provider Transformers @@ -267,9 +278,9 @@ Some frameworks (like Mastra) wrap OTEL data in custom structures that don't mat **Example**: Mastra wraps prompts as `{"messages": [{"role": "user", "content": [...]}]}` where content is an array of `{"type": "text", "text": "..."}` objects. The Mastra transformer unwraps this into standard `[{"role": "user", "content": "..."}]` format. -### Event Type Determination for v1 Frameworks +### Event Type Determination for V1 Frameworks -v1 frameworks create root spans that should appear in the tree hierarchy alongside their children. These root spans are marked as `$ai_span` (not `$ai_trace`) because `TraceQueryRunner` filters out `$ai_trace` events from the events array. This ensures v1 framework traces display correctly with proper parent-child relationships in the UI. +V1 frameworks create root spans that should appear in the tree hierarchy alongside their children. The `determine_event_type()` function checks `provider.get_instrumentation_pattern()` and marks V1 root spans as `$ai_span` (not `$ai_trace`) because `TraceQueryRunner` filters out `$ai_trace` events from the events array. This ensures V1 framework traces display correctly with proper parent-child relationships in the UI. ### TTL-Based Cleanup @@ -282,8 +293,9 @@ The event merger uses 60-second TTL on cache entries. This automatically cleans Create a new transformer in `conventions/providers/`: ```python -from .base import ProviderTransformer +from .base import OtelInstrumentationPattern, ProviderTransformer from typing import Any +import json class CustomFrameworkTransformer(ProviderTransformer): """Transform CustomFramework's OTEL format.""" @@ -293,6 +305,12 @@ class CustomFrameworkTransformer(ProviderTransformer): scope_name = scope.get("name", "") return scope_name == "custom-framework-scope" + def get_instrumentation_pattern(self) -> OtelInstrumentationPattern: + """Declare V1 or V2 pattern - determines event routing.""" + # V1: All data in span attributes, send immediately + # V2: Metadata in spans, content in logs, requires merge + return OtelInstrumentationPattern.V1_ATTRIBUTES + def transform_prompt(self, prompt: Any) -> Any: """Transform wrapped prompt format to standard.""" if not isinstance(prompt, str): @@ -369,6 +387,67 @@ Extend `build_event_properties()` in `transformer.py` to map additional attribut - **Memory**: Redis cache bounded by TTL (60s max retention) - **Concurrency**: Simple Redis operations enable fast merging with minimal race condition risk +## Provider Reference + +Different LLM frameworks implement OTEL instrumentation with their own nuances. This section documents known provider behaviors to help understand what to expect from each. + +### Mastra (`@mastra/otel`) + +**Detection**: Instrumentation scope name `@mastra/otel` or `mastra.*` attribute prefix + +**OTEL Pattern**: `V1_ATTRIBUTES` (all data in span attributes) + +**Key Behaviors**: + +- **No conversation history accumulation**: Each `agent.generate()` call creates a separate, independent trace. The `gen_ai.prompt` only contains that specific call's input (typically system message + current user message), not the accumulated conversation history from previous turns. +- **Wrapped message format**: Prompts are JSON-wrapped as `{"messages": [{"role": "user", "content": [{"type": "text", "text": "..."}]}]}` where content is an array of typed objects. +- **Wrapped completion format**: Completions are JSON-wrapped as `{"text": "...", "files": [], "warnings": [], ...}`. +- **Multi-turn traces**: In a multi-turn conversation, you'll see multiple separate traces (one per `agent.generate()` call), each showing only that turn's input/output. + +**Implications for PostHog**: + +- Each turn appears as a separate trace in LLM Analytics +- To see full conversation context, users need to look at the sequence of traces +- The Mastra transformer unwraps the custom JSON format into standard PostHog message arrays + +**Example**: A 4-turn conversation produces 4 traces, where turn 4's input only shows "Thanks, bye!" (not the previous greeting, weather query, and joke request). + +### OpenTelemetry Instrumentation OpenAI v1 (`opentelemetry-instrumentation-openai`) + +**Detection**: Span attributes with indexed prompt/completion fields (no custom provider transformer needed - uses standard GenAI conventions) + +**OTEL Pattern**: `V1_ATTRIBUTES` (all data in span attributes) + +**Key Behaviors**: + +- **Full conversation in each call**: The `gen_ai.prompt.*` attributes contain all messages passed to the API call +- **Indexed attributes**: Messages use `gen_ai.prompt.0.role`, `gen_ai.prompt.0.content`, etc. +- **Direct attribute format**: No JSON wrapping, values are stored directly as span attributes + +**Implications for PostHog**: + +- If the application maintains conversation state, later turns show full history +- Each trace is self-contained with complete context + +### OpenTelemetry Instrumentation OpenAI v2 (`opentelemetry-instrumentation-openai-v2`) + +**Detection**: Spans without prompt/completion attributes, accompanied by OTEL log events (no custom provider transformer needed - detected by absence of V1 content) + +**OTEL Pattern**: `V2_TRACES_AND_LOGS` (traces + logs separated) + +**Key Behaviors**: + +- **Split data model**: Traces contain metadata (model, tokens, timing), logs contain message content +- **Log events**: Uses `gen_ai.user.message`, `gen_ai.assistant.message`, `gen_ai.tool.message`, etc. +- **Full conversation in each call**: Like v1, if the app maintains state, messages accumulate +- **Requires merge**: PostHog's event merger combines traces and logs into complete events + +**Implications for PostHog**: + +- Slightly higher latency due to merge process +- Supports streaming better than v1 +- Both traces and logs endpoints must be configured + ## References - [OpenTelemetry GenAI Semantic Conventions](https://opentelemetry.io/docs/specs/semconv/gen-ai/) diff --git a/products/llm_analytics/backend/api/otel/conventions/genai.py b/products/llm_analytics/backend/api/otel/conventions/genai.py index 55627b6cc8c7..f24e600deaf3 100644 --- a/products/llm_analytics/backend/api/otel/conventions/genai.py +++ b/products/llm_analytics/backend/api/otel/conventions/genai.py @@ -11,9 +11,10 @@ """ from collections import defaultdict -from typing import Any +from typing import TYPE_CHECKING, Any -from .providers import PROVIDER_TRANSFORMERS +if TYPE_CHECKING: + from .providers.base import ProviderTransformer def has_genai_attributes(span: dict[str, Any]) -> bool: @@ -65,6 +66,27 @@ def _extract_indexed_messages(attributes: dict[str, Any], prefix: str) -> list[d return messages if messages else None +def detect_provider(span: dict[str, Any], scope: dict[str, Any] | None = None) -> "ProviderTransformer | None": + """ + Detect which provider transformer handles this span. + + Args: + span: Parsed OTEL span + scope: Instrumentation scope info + + Returns: + Matching ProviderTransformer instance, or None if no provider matches + """ + from .providers import PROVIDER_TRANSFORMERS + + scope = scope or {} + for transformer_class in PROVIDER_TRANSFORMERS: + transformer = transformer_class() + if transformer.can_handle(span, scope): + return transformer + return None + + def extract_genai_attributes(span: dict[str, Any], scope: dict[str, Any] | None = None) -> dict[str, Any]: """ Extract GenAI semantic convention attributes from span. @@ -90,18 +112,14 @@ def extract_genai_attributes(span: dict[str, Any], scope: dict[str, Any] | None result: dict[str, Any] = {} # Detect provider-specific transformer - provider_transformer = None - for transformer_class in PROVIDER_TRANSFORMERS: - transformer = transformer_class() - if transformer.can_handle(span, scope): - provider_transformer = transformer - logger.info( - "provider_transformer_detected", - provider=transformer.get_provider_name(), - scope_name=scope.get("name"), - span_name=span.get("name"), - ) - break + provider_transformer = detect_provider(span, scope) + if provider_transformer: + logger.info( + "provider_transformer_detected", + provider=provider_transformer.get_provider_name(), + scope_name=scope.get("name"), + span_name=span.get("name"), + ) # Model (prefer request, fallback to response, then system) model = ( diff --git a/products/llm_analytics/backend/api/otel/conventions/providers/__init__.py b/products/llm_analytics/backend/api/otel/conventions/providers/__init__.py index 9f05237b0aa4..24eacf7aad3a 100644 --- a/products/llm_analytics/backend/api/otel/conventions/providers/__init__.py +++ b/products/llm_analytics/backend/api/otel/conventions/providers/__init__.py @@ -5,7 +5,7 @@ specific OTEL format quirks and normalizes to PostHog format. """ -from .base import ProviderTransformer +from .base import OtelInstrumentationPattern, ProviderTransformer from .mastra import MastraTransformer # Registry of all available provider transformers @@ -16,6 +16,7 @@ ] __all__ = [ + "OtelInstrumentationPattern", "ProviderTransformer", "MastraTransformer", "PROVIDER_TRANSFORMERS", diff --git a/products/llm_analytics/backend/api/otel/conventions/providers/base.py b/products/llm_analytics/backend/api/otel/conventions/providers/base.py index 417be1ef1fac..982718495822 100644 --- a/products/llm_analytics/backend/api/otel/conventions/providers/base.py +++ b/products/llm_analytics/backend/api/otel/conventions/providers/base.py @@ -3,12 +3,39 @@ Provider transformers handle framework/library-specific OTEL formats and normalize them to PostHog's standard format. + +When adding a new provider transformer, document these aspects: +1. Detection method (scope name, attribute prefix, etc.) +2. OTEL pattern (v1 attributes-only vs v2 traces+logs) +3. Message format quirks (JSON wrapping, content arrays, etc.) +4. Conversation history behavior (accumulated vs per-call) +5. Any other notable behaviors + +See mastra.py for an example of well-documented provider behavior. """ from abc import ABC, abstractmethod +from enum import Enum from typing import Any +class OtelInstrumentationPattern(Enum): + """ + OTEL instrumentation patterns for LLM frameworks. + + V1_ATTRIBUTES: All data (metadata + content) in span attributes + - Send events immediately, no waiting for logs + - Example: opentelemetry-instrumentation-openai, Mastra + + V2_TRACES_AND_LOGS: Metadata in spans, content in separate log events + - Requires event merger to combine traces + logs + - Example: opentelemetry-instrumentation-openai-v2 + """ + + V1_ATTRIBUTES = "v1_attributes" + V2_TRACES_AND_LOGS = "v2_traces_and_logs" + + class ProviderTransformer(ABC): """ Base class for provider-specific OTEL transformers. @@ -65,3 +92,15 @@ def get_provider_name(self) -> str: Human-readable provider name """ return self.__class__.__name__.replace("Transformer", "") + + def get_instrumentation_pattern(self) -> OtelInstrumentationPattern: + """ + Get the OTEL instrumentation pattern this provider uses. + + Override in subclass to declare the pattern. Default is V2_TRACES_AND_LOGS + for safety - better to wait for logs than to send incomplete events. + + Returns: + The instrumentation pattern enum value + """ + return OtelInstrumentationPattern.V2_TRACES_AND_LOGS diff --git a/products/llm_analytics/backend/api/otel/conventions/providers/mastra.py b/products/llm_analytics/backend/api/otel/conventions/providers/mastra.py index 6bb09a4f6f14..fec6f6fa28d8 100644 --- a/products/llm_analytics/backend/api/otel/conventions/providers/mastra.py +++ b/products/llm_analytics/backend/api/otel/conventions/providers/mastra.py @@ -4,12 +4,28 @@ Handles Mastra's OTEL format which wraps messages in custom structures: - Input: {"messages": [{"role": "user", "content": [...]}]} - Output: {"files": [], "text": "...", "warnings": [], ...} + +Provider Behavior Notes: +------------------------ +Mastra uses the @mastra/otel instrumentation scope and sends OTEL data in v1 pattern +(all data in span attributes, no separate log events). + +Key characteristic: Mastra does NOT accumulate conversation history across calls. +Each `agent.generate()` call creates a separate, independent trace containing only +that turn's input (system message + current user message) and output. This means: + +- A 4-turn conversation produces 4 separate traces +- Turn 4's trace only shows "Thanks, bye!" as input, not previous turns +- To see full conversation context, users must look at the sequence of traces + +This is expected Mastra behavior, not a limitation of our ingestion. The framework +treats each generate() call as an independent operation. """ import json from typing import Any -from .base import ProviderTransformer +from .base import OtelInstrumentationPattern, ProviderTransformer class MastraTransformer(ProviderTransformer): @@ -36,6 +52,10 @@ def can_handle(self, span: dict[str, Any], scope: dict[str, Any]) -> bool: attributes = span.get("attributes", {}) return any(key.startswith("mastra.") for key in attributes.keys()) + def get_instrumentation_pattern(self) -> OtelInstrumentationPattern: + """Mastra uses v1 pattern - all data in span attributes.""" + return OtelInstrumentationPattern.V1_ATTRIBUTES + def transform_prompt(self, prompt: Any) -> Any: """ Transform Mastra's wrapped input format. diff --git a/products/llm_analytics/backend/api/otel/ingestion.py b/products/llm_analytics/backend/api/otel/ingestion.py index 781a303909cc..b55f05e66136 100644 --- a/products/llm_analytics/backend/api/otel/ingestion.py +++ b/products/llm_analytics/backend/api/otel/ingestion.py @@ -8,8 +8,8 @@ - v2 (opentelemetry-instrumentation-openai-v2): Sends metadata as spans, message content as logs Endpoints: -- POST /api/projects/:project_id/ai/otel/v1/traces - Required for all instrumentation -- POST /api/projects/:project_id/ai/otel/v1/logs - Required for v2 instrumentation with message content +- POST /api/projects/:project_id/ai/otel/traces - Required for all instrumentation +- POST /api/projects/:project_id/ai/otel/logs - Required for v2 instrumentation with message content Content-Type: application/x-protobuf Authorization: Bearer @@ -136,7 +136,7 @@ def authenticate_header(cls, request) -> str: from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter exporter = OTLPSpanExporter( - endpoint="https://app.posthog.com/api/projects/{project_id}/ai/otel/v1/traces", + endpoint="https://app.posthog.com/api/projects/{project_id}/ai/otel/traces", headers={"Authorization": "Bearer phc_your_project_token"} ) ``` @@ -470,7 +470,7 @@ def capture_events(events: list[dict[str, Any]], team: Team) -> None: from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter exporter = OTLPLogExporter( - endpoint="https://app.posthog.com/api/projects/{project_id}/ai/otel/v1/logs", + endpoint="https://app.posthog.com/api/projects/{project_id}/ai/otel/logs", headers={"Authorization": "Bearer phc_your_project_token"} ) ``` @@ -713,6 +713,17 @@ def transform_logs_to_ai_events(parsed_request: dict[str, Any]) -> list[dict[str for log_record in logs: trace_id = log_record.get("trace_id", "") span_id = log_record.get("span_id", "") + + # Debug logging for ingestion parity validation + logger.debug( + "otel_log_received", + trace_id=trace_id, + span_id=span_id, + event_name=log_record.get("attributes", {}).get("event.name"), + body_keys=list(log_record.get("body", {}).keys()) if isinstance(log_record.get("body"), dict) else None, + body_content_preview=str(log_record.get("body", {}))[:200] if log_record.get("body") else None, + ) + if trace_id and span_id: logs_by_span[(trace_id, span_id)].append(log_record) else: diff --git a/products/llm_analytics/backend/api/otel/test_ingestion_parity.py b/products/llm_analytics/backend/api/otel/test_ingestion_parity.py new file mode 100644 index 000000000000..71765819f228 --- /dev/null +++ b/products/llm_analytics/backend/api/otel/test_ingestion_parity.py @@ -0,0 +1,550 @@ +""" +Integration tests for ingestion parity between SDK, OTEL v1, and OTEL v2. + +Validates that all three ingestion methods produce structurally equivalent +AI events with the same core properties. The goal is to ensure that +regardless of how data enters PostHog (SDK, OTEL v1, OTEL v2), the resulting +events are consistent and comparable. + +Expected differences (acceptable): +- trace_id format: SDK uses UUIDs, OTEL uses hex strings +- tool_calls structure: SDK and v2 use nested arrays, v1 uses flattened keys +- output structure: SDK may wrap content differently + +Required parity (must match): +- $ai_input contains full conversation history +- $ai_output_choices contains assistant response +- $ai_model, $ai_provider, $ai_input_tokens, $ai_output_tokens present +- All messages from multi-turn conversation preserved +""" + +import pytest +from unittest.mock import patch + +from parameterized import parameterized + +from products.llm_analytics.backend.api.otel.ingestion import transform_logs_to_ai_events +from products.llm_analytics.backend.api.otel.transformer import transform_span_to_ai_event + + +def create_v1_span_with_conversation( + trace_id: str = "a6b23ecb43aa99ff43ff70948b0a377f", + span_id: str = "fee4e58c7137b7ef", + model: str = "gpt-4o-mini", + input_tokens: int = 264, + output_tokens: int = 25, +) -> tuple[dict, dict, dict]: + """ + Create a v1 OTEL span with multi-turn conversation in indexed attributes. + + Based on actual opentelemetry.instrumentation.openai.v1 output: + - Messages in gen_ai.prompt.{i}.role/content + - Tool calls in gen_ai.prompt.{i}.tool_calls.{j}.id/name/arguments + - Tool responses have tool_call_id + - Functions in llm.request.functions.{i}.name/description/parameters + - Completions in gen_ai.completion.{i}.role/content/finish_reason + """ + # Build attributes matching real v1 instrumentation format + attributes = { + # Request metadata + "llm.request.type": "chat", + "gen_ai.system": "openai", + "gen_ai.request.model": model, + "gen_ai.request.max_tokens": 100, + "llm.headers": "None", + "llm.is_streaming": False, + "gen_ai.openai.api_base": "https://api.openai.com/v1/", + # Conversation messages (indexed) + "gen_ai.prompt.0.role": "system", + "gen_ai.prompt.0.content": "You are a helpful assistant.", + "gen_ai.prompt.1.role": "user", + "gen_ai.prompt.1.content": "Hi there!", + "gen_ai.prompt.2.role": "assistant", + "gen_ai.prompt.2.content": "Hello! How can I help?", + "gen_ai.prompt.3.role": "user", + "gen_ai.prompt.3.content": "What's the weather?", + # Assistant tool call (no content, has tool_calls) + "gen_ai.prompt.4.role": "assistant", + "gen_ai.prompt.4.tool_calls.0.id": "call_abc123", + "gen_ai.prompt.4.tool_calls.0.name": "get_weather", + "gen_ai.prompt.4.tool_calls.0.arguments": '{"location":"Paris"}', + # Tool response + "gen_ai.prompt.5.role": "tool", + "gen_ai.prompt.5.content": "Sunny, 18°C", + "gen_ai.prompt.5.tool_call_id": "call_abc123", + # Continued conversation + "gen_ai.prompt.6.role": "assistant", + "gen_ai.prompt.6.content": "The weather is sunny at 18°C.", + "gen_ai.prompt.7.role": "user", + "gen_ai.prompt.7.content": "Thanks, bye!", + # Tool definitions + "llm.request.functions.0.name": "get_weather", + "llm.request.functions.0.description": "Get weather for a location", + "llm.request.functions.0.parameters": '{"type": "object", "properties": {"location": {"type": "string"}}, "required": ["location"]}', + # Response metadata + "gen_ai.response.model": "gpt-4o-mini-2024-07-18", + "gen_ai.response.id": "chatcmpl-test123", + "llm.usage.total_tokens": input_tokens + output_tokens, + "gen_ai.usage.input_tokens": input_tokens, + "gen_ai.usage.output_tokens": output_tokens, + # Completion + "gen_ai.completion.0.finish_reason": "stop", + "gen_ai.completion.0.role": "assistant", + "gen_ai.completion.0.content": "You're welcome! Goodbye!", + } + + span = { + "trace_id": trace_id, + "span_id": span_id, + "parent_span_id": None, + "name": "openai.chat", + "kind": 3, + "start_time_unix_nano": 1700000000000000000, + "end_time_unix_nano": 1700000001000000000, + "attributes": attributes, + "status": {"code": 1}, + } + + resource = {"service.name": "test-service"} + # Match actual scope name from real instrumentation + scope = {"name": "opentelemetry.instrumentation.openai.v1", "version": "0.40.0"} + + return span, resource, scope + + +def create_v2_logs_with_conversation( + trace_id: str = "af4e25c0d86a2f7bebd2e0c84f072499", + span_id: str = "a19561fe0a9d2d73", +) -> dict: + """ + Create v2 OTEL logs request with multi-turn conversation. + + Based on actual opentelemetry.instrumentation.openai_v2 output: + - Each log has attributes with gen_ai.system and event.name + - Body contains content directly (not nested in message for input) + - gen_ai.choice logs have message wrapper with role/content + finish_reason + - Event names: gen_ai.system.message, gen_ai.user.message, gen_ai.choice + """ + base_time = 1700000000000000000 + + logs = [ + # System message + { + "trace_id": trace_id, + "span_id": span_id, + "attributes": { + "gen_ai.system": "openai", + "event.name": "gen_ai.system.message", + }, + "body": {"content": "You are a helpful assistant."}, + "time_unix_nano": base_time, + }, + # User message 1 + { + "trace_id": trace_id, + "span_id": span_id, + "attributes": { + "gen_ai.system": "openai", + "event.name": "gen_ai.user.message", + }, + "body": {"content": "Hi there!"}, + "time_unix_nano": base_time + 1000000, + }, + # Assistant message (from previous turn, now in context) + { + "trace_id": trace_id, + "span_id": span_id, + "attributes": { + "gen_ai.system": "openai", + "event.name": "gen_ai.assistant.message", + }, + "body": {"content": "Hello! How can I help?"}, + "time_unix_nano": base_time + 2000000, + }, + # User message 2 + { + "trace_id": trace_id, + "span_id": span_id, + "attributes": { + "gen_ai.system": "openai", + "event.name": "gen_ai.user.message", + }, + "body": {"content": "What's the weather?"}, + "time_unix_nano": base_time + 3000000, + }, + # Assistant tool call + { + "trace_id": trace_id, + "span_id": span_id, + "attributes": { + "gen_ai.system": "openai", + "event.name": "gen_ai.assistant.message", + }, + "body": { + "tool_calls": [ + { + "id": "call_123", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"location":"Paris"}'}, + } + ], + }, + "time_unix_nano": base_time + 4000000, + }, + # Tool response + { + "trace_id": trace_id, + "span_id": span_id, + "attributes": { + "gen_ai.system": "openai", + "event.name": "gen_ai.tool.message", + }, + "body": {"content": "Sunny, 18°C", "id": "call_123"}, + "time_unix_nano": base_time + 5000000, + }, + # Assistant message after tool + { + "trace_id": trace_id, + "span_id": span_id, + "attributes": { + "gen_ai.system": "openai", + "event.name": "gen_ai.assistant.message", + }, + "body": {"content": "The weather is sunny at 18°C."}, + "time_unix_nano": base_time + 6000000, + }, + # User message 3 + { + "trace_id": trace_id, + "span_id": span_id, + "attributes": { + "gen_ai.system": "openai", + "event.name": "gen_ai.user.message", + }, + "body": {"content": "Thanks, bye!"}, + "time_unix_nano": base_time + 7000000, + }, + # Final choice/completion - note the different structure + { + "trace_id": trace_id, + "span_id": span_id, + "attributes": { + "gen_ai.system": "openai", + "event.name": "gen_ai.choice", + }, + "body": { + "index": 0, + "finish_reason": "stop", + "message": {"role": "assistant", "content": "You're welcome! Goodbye!"}, + }, + "time_unix_nano": base_time + 8000000, + }, + ] + + return { + "logs": logs, + "resource": {"service.name": "test-service"}, + # Match actual scope name from real instrumentation + "scope": {"name": "opentelemetry.instrumentation.openai_v2", "version": "2.0.0"}, + } + + +def create_v2_span_metadata( + trace_id: str = "af4e25c0d86a2f7bebd2e0c84f072499", + span_id: str = "a19561fe0a9d2d73", + model: str = "gpt-4o-mini", + input_tokens: int = 234, + output_tokens: int = 18, +) -> tuple[dict, dict, dict]: + """ + Create v2 OTEL span (metadata only, no content). + + In v2 instrumentation, spans contain only metadata (model, tokens, etc.) + while message content is sent via separate log records. + """ + span = { + "trace_id": trace_id, + "span_id": span_id, + "parent_span_id": None, + "name": "chat gpt-4o-mini", + "kind": 3, + "start_time_unix_nano": 1700000000000000000, + "end_time_unix_nano": 1700000001000000000, + "attributes": { + "gen_ai.system": "openai", + "gen_ai.request.model": model, + "gen_ai.operation.name": "chat", + "gen_ai.usage.input_tokens": input_tokens, + "gen_ai.usage.output_tokens": output_tokens, + }, + "status": {"code": 1}, + } + + resource = {"service.name": "test-service"} + # Match actual scope name from real v2 instrumentation + scope = {"name": "opentelemetry.instrumentation.openai_v2", "version": "2.0.0"} + + return span, resource, scope + + +class TestV1SpanTransformation: + """Tests for OTEL v1 span transformation.""" + + def test_v1_span_produces_ai_generation_event(self): + """v1 span with conversation should produce $ai_generation event.""" + span, resource, scope = create_v1_span_with_conversation() + + event = transform_span_to_ai_event(span, resource, scope) + + assert event is not None + assert event["event"] == "$ai_generation" + + def test_v1_span_contains_full_conversation_history(self): + """v1 span $ai_input should contain all conversation messages.""" + span, resource, scope = create_v1_span_with_conversation() + + event = transform_span_to_ai_event(span, resource, scope) + + assert event is not None + props = event["properties"] + assert "$ai_input" in props + assert len(props["$ai_input"]) == 8 + + def test_v1_span_contains_output_choices(self): + """v1 span $ai_output_choices should contain assistant response.""" + span, resource, scope = create_v1_span_with_conversation() + + event = transform_span_to_ai_event(span, resource, scope) + + assert event is not None + props = event["properties"] + assert "$ai_output_choices" in props + assert len(props["$ai_output_choices"]) == 1 + assert props["$ai_output_choices"][0]["content"] == "You're welcome! Goodbye!" + + def test_v1_span_contains_model_metadata(self): + """v1 span should contain model, provider, and token counts.""" + span, resource, scope = create_v1_span_with_conversation() + + event = transform_span_to_ai_event(span, resource, scope) + + assert event is not None + props = event["properties"] + assert props["$ai_model"] == "gpt-4o-mini" + assert props["$ai_provider"] == "openai" + assert props["$ai_input_tokens"] == 264 + assert props["$ai_output_tokens"] == 25 + + def test_v1_span_preserves_trace_context(self): + """v1 span should preserve trace_id and span_id.""" + span, resource, scope = create_v1_span_with_conversation() + + event = transform_span_to_ai_event(span, resource, scope) + + assert event is not None + props = event["properties"] + assert props["$ai_trace_id"] == "a6b23ecb43aa99ff43ff70948b0a377f" + assert props["$ai_span_id"] == "fee4e58c7137b7ef" + + +class TestV2LogsAccumulation: + """Tests for OTEL v2 logs accumulation.""" + + def test_v2_logs_accumulate_conversation_history(self): + """v2 logs should accumulate full conversation in $ai_input.""" + parsed_request = create_v2_logs_with_conversation() + + with patch("products.llm_analytics.backend.api.otel.event_merger.cache_and_merge_properties") as mock_merger: + + def return_accumulated(trace_id, span_id, props, is_trace): + if "$ai_input" in props and len(props["$ai_input"]) >= 7: + return props + return None + + mock_merger.side_effect = return_accumulated + + _events = transform_logs_to_ai_events(parsed_request) + + assert mock_merger.call_count == 1 + call_args = mock_merger.call_args + props = call_args[0][2] + + assert "$ai_input" in props + assert len(props["$ai_input"]) >= 7 + + def test_v2_logs_preserve_message_order(self): + """v2 logs should preserve chronological message order.""" + parsed_request = create_v2_logs_with_conversation() + + with patch("products.llm_analytics.backend.api.otel.event_merger.cache_and_merge_properties") as mock_merger: + + def return_accumulated(trace_id, span_id, props, is_trace): + if "$ai_input" in props: + return props + return None + + mock_merger.side_effect = return_accumulated + + _events = transform_logs_to_ai_events(parsed_request) + + props = mock_merger.call_args[0][2] + + assert props["$ai_input"][0]["role"] == "system" + assert props["$ai_input"][1]["role"] == "user" + assert props["$ai_input"][1]["content"] == "Hi there!" + + def test_v2_logs_contain_output_choices(self): + """v2 logs should contain final response in $ai_output_choices.""" + parsed_request = create_v2_logs_with_conversation() + + with patch("products.llm_analytics.backend.api.otel.event_merger.cache_and_merge_properties") as mock_merger: + + def return_accumulated(trace_id, span_id, props, is_trace): + if "$ai_output_choices" in props: + return props + return None + + mock_merger.side_effect = return_accumulated + + _events = transform_logs_to_ai_events(parsed_request) + + props = mock_merger.call_args[0][2] + + assert "$ai_output_choices" in props + assert len(props["$ai_output_choices"]) == 1 + assert props["$ai_output_choices"][0]["content"] == "You're welcome! Goodbye!" + + +class TestIngestionParity: + """Tests for parity between SDK-style, OTEL v1, and OTEL v2 events.""" + + @parameterized.expand( + [ + ("model", "$ai_model"), + ("provider", "$ai_provider"), + ("input_tokens", "$ai_input_tokens"), + ("output_tokens", "$ai_output_tokens"), + ] + ) + def test_v1_contains_required_property(self, name: str, property_key: str): + """v1 events must contain core properties.""" + span, resource, scope = create_v1_span_with_conversation() + event = transform_span_to_ai_event(span, resource, scope) + + assert event is not None + assert property_key in event["properties"], f"Missing {property_key}" + + def test_v1_and_v2_produce_same_message_count(self): + """v1 and v2 should produce same number of input messages.""" + v1_span, v1_resource, v1_scope = create_v1_span_with_conversation() + v1_event = transform_span_to_ai_event(v1_span, v1_resource, v1_scope) + + v2_logs = create_v2_logs_with_conversation() + + with patch("products.llm_analytics.backend.api.otel.event_merger.cache_and_merge_properties") as mock_merger: + + def return_accumulated(trace_id, span_id, props, is_trace): + if "$ai_input" in props: + return props + return None + + mock_merger.side_effect = return_accumulated + _events = transform_logs_to_ai_events(v2_logs) + v2_props = mock_merger.call_args[0][2] + + v1_input = v1_event["properties"]["$ai_input"] + v2_input = v2_props["$ai_input"] + + assert len(v1_input) == len(v2_input), f"Message count mismatch: v1={len(v1_input)}, v2={len(v2_input)}" + + def test_v1_and_v2_have_same_output_structure(self): + """v1 and v2 should both have $ai_output_choices as array.""" + v1_span, v1_resource, v1_scope = create_v1_span_with_conversation() + v1_event = transform_span_to_ai_event(v1_span, v1_resource, v1_scope) + + v2_logs = create_v2_logs_with_conversation() + + with patch("products.llm_analytics.backend.api.otel.event_merger.cache_and_merge_properties") as mock_merger: + + def return_accumulated(trace_id, span_id, props, is_trace): + if "$ai_output_choices" in props: + return props + return None + + mock_merger.side_effect = return_accumulated + _events = transform_logs_to_ai_events(v2_logs) + v2_props = mock_merger.call_args[0][2] + + v1_output = v1_event["properties"]["$ai_output_choices"] + v2_output = v2_props["$ai_output_choices"] + + assert isinstance(v1_output, list), "v1 output should be list" + assert isinstance(v2_output, list), "v2 output should be list" + assert len(v1_output) == len(v2_output) == 1 + + +class TestToolCallParity: + """Tests for tool call handling parity.""" + + def test_v1_preserves_tool_messages(self): + """v1 should preserve tool messages in conversation.""" + span, resource, scope = create_v1_span_with_conversation() + event = transform_span_to_ai_event(span, resource, scope) + + props = event["properties"] + roles = [msg.get("role") for msg in props["$ai_input"]] + + assert "tool" in roles, "v1 should preserve tool messages" + + def test_v2_preserves_tool_messages(self): + """v2 should preserve tool messages in conversation.""" + parsed_request = create_v2_logs_with_conversation() + + with patch("products.llm_analytics.backend.api.otel.event_merger.cache_and_merge_properties") as mock_merger: + + def return_accumulated(trace_id, span_id, props, is_trace): + if "$ai_input" in props: + return props + return None + + mock_merger.side_effect = return_accumulated + _events = transform_logs_to_ai_events(parsed_request) + props = mock_merger.call_args[0][2] + + roles = [msg.get("role") for msg in props["$ai_input"]] + assert "tool" in roles, "v2 should preserve tool messages" + + +class TestEventTypeConsistency: + """Tests for event type determination consistency.""" + + def test_v1_generation_span_is_ai_generation(self): + """v1 span with LLM attrs should be $ai_generation.""" + span, resource, scope = create_v1_span_with_conversation() + event = transform_span_to_ai_event(span, resource, scope) + + assert event["event"] == "$ai_generation" + + def test_v2_merged_event_can_be_ai_generation(self): + """v2 merged event with LLM attrs should be $ai_generation.""" + span, resource, scope = create_v2_span_metadata() + + mock_merged_props = { + "$ai_model": "gpt-4o-mini", + "$ai_provider": "openai", + "$ai_input_tokens": 234, + "$ai_output_tokens": 18, + "$ai_input": [{"role": "user", "content": "Hello"}], + "$ai_output_choices": [{"role": "assistant", "content": "Hi!"}], + } + + with patch("products.llm_analytics.backend.api.otel.transformer.cache_and_merge_properties") as mock_merger: + mock_merger.return_value = mock_merged_props + event = transform_span_to_ai_event(span, resource, scope) + + assert event is not None + assert event["event"] == "$ai_generation" + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/products/llm_analytics/backend/api/otel/transformer.py b/products/llm_analytics/backend/api/otel/transformer.py index 9d98e54d7f65..25cf01fe2c4d 100644 --- a/products/llm_analytics/backend/api/otel/transformer.py +++ b/products/llm_analytics/backend/api/otel/transformer.py @@ -12,8 +12,9 @@ from datetime import UTC, datetime from typing import Any -from .conventions.genai import extract_genai_attributes, has_genai_attributes +from .conventions.genai import detect_provider, extract_genai_attributes, has_genai_attributes from .conventions.posthog_native import extract_posthog_native_attributes, has_posthog_attributes +from .conventions.providers import OtelInstrumentationPattern from .event_merger import cache_and_merge_properties OTEL_TRANSFORMER_VERSION = "1.0.0" @@ -47,8 +48,23 @@ def transform_span_to_ai_event( - properties: AI event properties - uuid: Event UUID (for deduplication with log events) """ + import structlog + + logger = structlog.get_logger(__name__) baggage = baggage or {} + # Debug logging for ingestion parity validation + logger.debug( + "otel_span_received", + trace_id=span.get("trace_id"), + span_id=span.get("span_id"), + span_name=span.get("name"), + scope_name=scope.get("name"), + attributes_keys=list(span.get("attributes", {}).keys()), + has_prompt_attrs=any(k.startswith("gen_ai.prompt.") for k in span.get("attributes", {}).keys()), + has_completion_attrs=any(k.startswith("gen_ai.completion.") for k in span.get("attributes", {}).keys()), + ) + # Extract attributes using waterfall pattern posthog_attrs = extract_posthog_native_attributes(span) genai_attrs = extract_genai_attributes(span, scope) @@ -59,17 +75,22 @@ def transform_span_to_ai_event( # Build AI event properties properties = build_event_properties(span, merged_attrs, resource, scope, baggage) - # Detect v1 vs v2 instrumentation: - # v1: Everything in span attributes (extracted as "prompt", "completion") - send immediately - # v2: Metadata in span, content in logs - use event merger - # Some frameworks (e.g., Mastra) are v1 but may have spans without prompt/completion (parent spans) - # Detect these by instrumentation scope name - scope_name = scope.get("name", "") - is_v1_framework = scope_name == "@mastra/otel" # Mastra uses v1 (attributes in spans) - is_v1_span = bool(merged_attrs.get("prompt") or merged_attrs.get("completion")) or is_v1_framework - - if not is_v1_span: - # v2 instrumentation - use event merger for bidirectional merge with logs + # Detect instrumentation pattern to determine event routing: + # + # V1_ATTRIBUTES: Everything in span attributes - send immediately + # V2_TRACES_AND_LOGS: Metadata in span, content in logs - use event merger + # + # Detection priority: + # 1. Provider declares pattern via get_instrumentation_pattern() - most reliable + # 2. Span has prompt/completion attributes - indicates V1 data present + # 3. Default to V2 (safer - waits for logs rather than sending incomplete) + provider = detect_provider(span, scope) + provider_pattern = provider.get_instrumentation_pattern() if provider else None + has_v1_content = bool(merged_attrs.get("prompt") or merged_attrs.get("completion")) + uses_v1_pattern = provider_pattern == OtelInstrumentationPattern.V1_ATTRIBUTES or has_v1_content + + if not uses_v1_pattern: + # V2 instrumentation - use event merger for bidirectional merge with logs trace_id = span.get("trace_id", "") span_id = span.get("span_id", "") if trace_id and span_id: @@ -359,17 +380,17 @@ def determine_event_type(span: dict[str, Any], attrs: dict[str, Any], scope: dic return "$ai_generation" # Check if span is root (no parent) - # For v1 frameworks (like Mastra), root spans should be $ai_span, not $ai_trace + # For V1 frameworks, root spans should be $ai_span, not $ai_trace # $ai_trace events get filtered out by TraceQueryRunner, breaking tree hierarchy if not span.get("parent_span_id"): - scope_name = scope.get("name", "") - is_v1_framework = scope_name == "@mastra/otel" - if is_v1_framework: - # v1 frameworks: root span should be $ai_span (will be included in tree) - return "$ai_span" - else: - # v2 frameworks: root is $ai_trace (separate from event tree) - return "$ai_trace" + provider = detect_provider(span, scope) + if provider: + pattern = provider.get_instrumentation_pattern() + if pattern == OtelInstrumentationPattern.V1_ATTRIBUTES: + # V1 frameworks: root span should be $ai_span (will be included in tree) + return "$ai_span" + # V2 frameworks or unknown: root is $ai_trace (separate from event tree) + return "$ai_trace" # Default to generic span return "$ai_span" From dc7eb6b72db958e2d5307403cf337c52a7a4f703 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Thu, 27 Nov 2025 12:17:53 +0000 Subject: [PATCH 29/34] chore(otel): Remove legacy proxy_logs_to_capture_service MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove unused proxy function and /i/v1/logs route - Port 4318 was already removed from docker-compose, making this dead code - LLM Analytics OTEL ingestion now uses /api/projects/:id/ai/otel/logs 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- posthog/urls.py | 44 -------------------------------------------- 1 file changed, 44 deletions(-) diff --git a/posthog/urls.py b/posthog/urls.py index 00772ec0dac9..1be3e1e17a0d 100644 --- a/posthog/urls.py +++ b/posthog/urls.py @@ -146,48 +146,6 @@ def authorize_and_redirect(request: HttpRequest) -> HttpResponse: ) -@csrf_exempt -def proxy_logs_to_capture_service(request: HttpRequest) -> HttpResponse: - """ - Proxy OTLP logs to the capture-logs Rust service. - - This allows OpenTelemetry SDKs to send logs to /i/v1/logs in local dev, - which then forwards to the capture-logs service. - - Note: Using Docker container IP directly since Django runs on host. - """ - import requests - - # Forward to capture-logs service (exposed on host port 4318) - capture_logs_url = "http://localhost:4318/v1/logs" - - try: - # Forward the request with all headers and body - response = requests.post( - capture_logs_url, - data=request.body, - headers={ - "Content-Type": request.META.get("CONTENT_TYPE", ""), - "Authorization": request.META.get("HTTP_AUTHORIZATION", ""), - }, - timeout=10, - ) - - # Return the response from capture-logs - return HttpResponse( - response.content, - status=response.status_code, - content_type=response.headers.get("Content-Type", "application/json"), - ) - except Exception: - logger.exception("Error proxying logs to capture service") - return HttpResponse( - b'{"error": "Failed to forward logs"}', - status=500, - content_type="application/json", - ) - - urlpatterns = [ path("api/schema/", SpectacularAPIView.as_view(), name="schema"), # Optional UI: @@ -215,8 +173,6 @@ def proxy_logs_to_capture_service(request: HttpRequest) -> HttpResponse: path("api/projects//ai/otel/traces", csrf_exempt(otel_traces_endpoint)), # OpenTelemetry logs ingestion for LLM Analytics path("api/projects//ai/otel/logs", csrf_exempt(otel_logs_endpoint)), - # OpenTelemetry logs proxy to capture-logs service (legacy) - path("i/v1/logs", proxy_logs_to_capture_service), path("api/environments//progress/", progress), path("api/environments//query//progress/", progress), path("api/environments//query//progress", progress), From 3b12436221143516c65f7cd6b58991da4c820ae5 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Thu, 27 Nov 2025 12:29:44 +0000 Subject: [PATCH 30/34] chore(otel): Remove dead code and reduce verbose logging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove unused stringify_content() functions from transformer.py and logs_transformer.py - Remove unused span_uses_known_conventions() function from transformer.py - Remove debug logging statement from logs_transformer.py - Change verbose logger.info() calls to logger.debug() for per-request operational logs - Remove verbose transformation logging from genai.py and mastra.py 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .../backend/api/otel/conventions/genai.py | 34 +------------------ .../api/otel/conventions/providers/mastra.py | 10 ------ .../backend/api/otel/event_merger.py | 14 ++++---- .../backend/api/otel/ingestion.py | 16 ++++----- .../backend/api/otel/logs_transformer.py | 15 -------- .../backend/api/otel/transformer.py | 28 ++------------- 6 files changed, 19 insertions(+), 98 deletions(-) diff --git a/products/llm_analytics/backend/api/otel/conventions/genai.py b/products/llm_analytics/backend/api/otel/conventions/genai.py index f24e600deaf3..a9ea79235b6a 100644 --- a/products/llm_analytics/backend/api/otel/conventions/genai.py +++ b/products/llm_analytics/backend/api/otel/conventions/genai.py @@ -114,7 +114,7 @@ def extract_genai_attributes(span: dict[str, Any], scope: dict[str, Any] | None # Detect provider-specific transformer provider_transformer = detect_provider(span, scope) if provider_transformer: - logger.info( + logger.debug( "provider_transformer_detected", provider=provider_transformer.get_provider_name(), scope_name=scope.get("name"), @@ -153,26 +153,10 @@ def extract_genai_attributes(span: dict[str, Any], scope: dict[str, Any] | None elif (prompt := attributes.get("gen_ai.prompt")) is not None: # Try provider-specific transformation if provider_transformer: - logger.info( - "provider_transform_prompt_attempt", - provider=provider_transformer.get_provider_name(), - prompt_type=type(prompt).__name__, - prompt_length=len(str(prompt)) if prompt else 0, - ) transformed = provider_transformer.transform_prompt(prompt) if transformed is not None: - logger.info( - "provider_transform_prompt_success", - provider=provider_transformer.get_provider_name(), - result_type=type(transformed).__name__, - result_length=len(transformed) if isinstance(transformed, list) else 0, - ) result["prompt"] = transformed else: - logger.info( - "provider_transform_prompt_none", - provider=provider_transformer.get_provider_name(), - ) result["prompt"] = prompt else: result["prompt"] = prompt @@ -184,26 +168,10 @@ def extract_genai_attributes(span: dict[str, Any], scope: dict[str, Any] | None elif (completion := attributes.get("gen_ai.completion")) is not None: # Try provider-specific transformation if provider_transformer: - logger.info( - "provider_transform_completion_attempt", - provider=provider_transformer.get_provider_name(), - completion_type=type(completion).__name__, - completion_length=len(str(completion)) if completion else 0, - ) transformed = provider_transformer.transform_completion(completion) if transformed is not None: - logger.info( - "provider_transform_completion_success", - provider=provider_transformer.get_provider_name(), - result_type=type(transformed).__name__, - result_length=len(transformed) if isinstance(transformed, list) else 0, - ) result["completion"] = transformed else: - logger.info( - "provider_transform_completion_none", - provider=provider_transformer.get_provider_name(), - ) result["completion"] = completion else: result["completion"] = completion diff --git a/products/llm_analytics/backend/api/otel/conventions/providers/mastra.py b/products/llm_analytics/backend/api/otel/conventions/providers/mastra.py index fec6f6fa28d8..f5436ccb121d 100644 --- a/products/llm_analytics/backend/api/otel/conventions/providers/mastra.py +++ b/products/llm_analytics/backend/api/otel/conventions/providers/mastra.py @@ -63,21 +63,11 @@ def transform_prompt(self, prompt: Any) -> Any: Mastra wraps messages as: {"messages": [{"role": "user", "content": [...]}]} where content can be an array of objects like [{"type": "text", "text": "..."}] """ - import structlog - - logger = structlog.get_logger(__name__) - if not isinstance(prompt, str): - logger.info("mastra_transform_prompt_skip_not_string", prompt_type=type(prompt).__name__) return None # No transformation needed try: parsed = json.loads(prompt) - logger.info( - "mastra_transform_prompt_parsed", - has_messages=("messages" in parsed) if isinstance(parsed, dict) else False, - parsed_type=type(parsed).__name__, - ) # Check for Mastra input format: {"messages": [...]} if not isinstance(parsed, dict) or "messages" not in parsed: diff --git a/products/llm_analytics/backend/api/otel/event_merger.py b/products/llm_analytics/backend/api/otel/event_merger.py index 1e06a96b108c..b7246509efa4 100644 --- a/products/llm_analytics/backend/api/otel/event_merger.py +++ b/products/llm_analytics/backend/api/otel/event_merger.py @@ -64,14 +64,16 @@ def cache_and_merge_properties( # Clean up logs cache redis_client.delete(logs_cache_key) - logger.info("event_merger_success: Merged trace+logs", extra={"trace_id": trace_id, "span_id": span_id}) + logger.debug( + "event_merger_success: Merged trace+logs", extra={"trace_id": trace_id, "span_id": span_id} + ) return merged else: # No logs yet - cache trace redis_client.setex(trace_cache_key, _CACHE_TTL, json.dumps(properties)) - logger.info( + logger.debug( "event_merger_cache: Cached trace properties", extra={"trace_id": trace_id, "span_id": span_id} ) @@ -88,7 +90,7 @@ def cache_and_merge_properties( # Re-cache accumulated logs redis_client.setex(logs_cache_key, _CACHE_TTL, json.dumps(merged_logs)) - logger.info( + logger.debug( "event_merger_accumulate: Accumulated log properties", extra={"trace_id": trace_id, "span_id": span_id}, ) @@ -104,7 +106,7 @@ def cache_and_merge_properties( redis_client.delete(logs_cache_key) redis_client.delete(trace_cache_key) - logger.info( + logger.debug( "event_merger_success: Merged accumulated logs+trace", extra={"trace_id": trace_id, "span_id": span_id}, ) @@ -125,7 +127,7 @@ def cache_and_merge_properties( # Clean up trace cache redis_client.delete(trace_cache_key) - logger.info( + logger.debug( "event_merger_success: Merged logs+trace", extra={"trace_id": trace_id, "span_id": span_id} ) @@ -134,7 +136,7 @@ def cache_and_merge_properties( # No trace yet - cache this log redis_client.setex(logs_cache_key, _CACHE_TTL, json.dumps(properties)) - logger.info( + logger.debug( "event_merger_cache: Cached log properties", extra={"trace_id": trace_id, "span_id": span_id} ) diff --git a/products/llm_analytics/backend/api/otel/ingestion.py b/products/llm_analytics/backend/api/otel/ingestion.py index b55f05e66136..863c568a91aa 100644 --- a/products/llm_analytics/backend/api/otel/ingestion.py +++ b/products/llm_analytics/backend/api/otel/ingestion.py @@ -208,7 +208,7 @@ def otel_traces_endpoint(request: HttpRequest, project_id: int) -> Response: status=status.HTTP_400_BAD_REQUEST, ) - logger.info( + logger.debug( "otel_traces_received", team_id=team.id, content_length=len(protobuf_data), @@ -223,7 +223,7 @@ def otel_traces_endpoint(request: HttpRequest, project_id: int) -> Response: # Parse OTLP protobuf parsed_request = parse_otlp_trace_request(protobuf_data) - logger.info( + logger.debug( "otel_traces_parsed", team_id=team.id, spans_count=len(parsed_request["spans"]), @@ -246,7 +246,7 @@ def otel_traces_endpoint(request: HttpRequest, project_id: int) -> Response: # Transform spans to AI events events = transform_spans_to_ai_events(parsed_request, baggage) - logger.info( + logger.debug( "otel_traces_transformed", team_id=team.id, events_created=len(events), @@ -255,7 +255,7 @@ def otel_traces_endpoint(request: HttpRequest, project_id: int) -> Response: # Route to capture pipeline capture_events(events, team) - logger.info( + logger.debug( "otel_traces_captured", team_id=team.id, events_captured=len(events), @@ -542,7 +542,7 @@ def otel_logs_endpoint(request: HttpRequest, project_id: int) -> Response: status=status.HTTP_400_BAD_REQUEST, ) - logger.info( + logger.debug( "otel_logs_received", team_id=team.id, content_length=len(protobuf_data), @@ -553,7 +553,7 @@ def otel_logs_endpoint(request: HttpRequest, project_id: int) -> Response: # Parse OTLP protobuf parsed_request = parse_otlp_logs_request(protobuf_data) - logger.info( + logger.debug( "otel_logs_parsed", team_id=team.id, logs_count=len(parsed_request["logs"]), @@ -575,7 +575,7 @@ def otel_logs_endpoint(request: HttpRequest, project_id: int) -> Response: # Transform logs to AI events (also caches properties for merging with traces) events = transform_logs_to_ai_events(parsed_request) - logger.info( + logger.debug( "otel_logs_transformed", team_id=team.id, events_created=len(events), @@ -589,7 +589,7 @@ def otel_logs_endpoint(request: HttpRequest, project_id: int) -> Response: # Route merged events to capture pipeline capture_events(events, team) - logger.info( + logger.debug( "otel_logs_captured", team_id=team.id, events_captured=len(events), diff --git a/products/llm_analytics/backend/api/otel/logs_transformer.py b/products/llm_analytics/backend/api/otel/logs_transformer.py index f3c53bf38cf1..fd44bf542e3a 100644 --- a/products/llm_analytics/backend/api/otel/logs_transformer.py +++ b/products/llm_analytics/backend/api/otel/logs_transformer.py @@ -142,14 +142,6 @@ def build_event_properties( # - gen_ai.choice (body: {"index": 0, "finish_reason": "stop", "message": {...}}) event_name = attributes.get("event.name", "").lower() - import logging - - logger = logging.getLogger(__name__) - logger.info( - "v2_log_event_debug", - extra={"event_name": event_name, "body_keys": list(body.keys()) if isinstance(body, dict) else None}, - ) - if isinstance(body, dict): # User/system messages: {"content": "..."} if "gen_ai.user.message" in event_name or "gen_ai.system.message" in event_name: @@ -319,10 +311,3 @@ def extract_distinct_id(resource: dict[str, Any], attributes: dict[str, Any]) -> # Default to anonymous return "anonymous" - - -def stringify_content(content: Any) -> str: - """Stringify content (handles objects and strings).""" - if isinstance(content, str): - return content - return json.dumps(content) diff --git a/products/llm_analytics/backend/api/otel/transformer.py b/products/llm_analytics/backend/api/otel/transformer.py index 25cf01fe2c4d..b3737630a768 100644 --- a/products/llm_analytics/backend/api/otel/transformer.py +++ b/products/llm_analytics/backend/api/otel/transformer.py @@ -12,8 +12,8 @@ from datetime import UTC, datetime from typing import Any -from .conventions.genai import detect_provider, extract_genai_attributes, has_genai_attributes -from .conventions.posthog_native import extract_posthog_native_attributes, has_posthog_attributes +from .conventions.genai import detect_provider, extract_genai_attributes +from .conventions.posthog_native import extract_posthog_native_attributes from .conventions.providers import OtelInstrumentationPattern from .event_merger import cache_and_merge_properties @@ -433,27 +433,3 @@ def extract_distinct_id(resource: dict[str, Any], baggage: dict[str, str]) -> st # Default to anonymous return "anonymous" - - -def stringify_content(content: Any) -> Any: - """ - Return content in appropriate format for PostHog properties. - - Keep structured data (lists/dicts) as-is for better UI rendering. - Only convert to JSON string if it's already a string (rare case). - """ - if isinstance(content, list | dict): - return content - if isinstance(content, str): - # If it's already a JSON string, parse it to get structured data - try: - parsed = json.loads(content) - return parsed - except (json.JSONDecodeError, TypeError): - return content - return content - - -def span_uses_known_conventions(span: dict[str, Any]) -> bool: - """Check if span uses PostHog or GenAI conventions.""" - return has_posthog_attributes(span) or has_genai_attributes(span) From e69422882823dad874bbf381af43599aa96d9757 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Thu, 27 Nov 2025 12:39:54 +0000 Subject: [PATCH 31/34] refactor(otel): Remove verbose debug logging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove debug logs to minimize PR scope. Keep only warning, error, and exception logs which are appropriate for production monitoring. Removed 130 lines of debug logging code: - event_merger.py: Remove all debug logs, keep exception logging - ingestion.py: Remove debug logs for received/parsed/transformed/captured - transformer.py: Remove structlog import and debug log - conventions/genai.py: Remove structlog import and provider detection debug log 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .../backend/api/otel/conventions/genai.py | 10 --- .../backend/api/otel/event_merger.py | 39 ----------- .../backend/api/otel/ingestion.py | 66 ------------------- .../backend/api/otel/transformer.py | 15 ----- 4 files changed, 130 deletions(-) diff --git a/products/llm_analytics/backend/api/otel/conventions/genai.py b/products/llm_analytics/backend/api/otel/conventions/genai.py index a9ea79235b6a..83e7f74c08e9 100644 --- a/products/llm_analytics/backend/api/otel/conventions/genai.py +++ b/products/llm_analytics/backend/api/otel/conventions/genai.py @@ -104,22 +104,12 @@ def extract_genai_attributes(span: dict[str, Any], scope: dict[str, Any] | None Returns: Extracted attributes dict """ - import structlog - - logger = structlog.get_logger(__name__) attributes = span.get("attributes", {}) scope = scope or {} result: dict[str, Any] = {} # Detect provider-specific transformer provider_transformer = detect_provider(span, scope) - if provider_transformer: - logger.debug( - "provider_transformer_detected", - provider=provider_transformer.get_provider_name(), - scope_name=scope.get("name"), - span_name=span.get("name"), - ) # Model (prefer request, fallback to response, then system) model = ( diff --git a/products/llm_analytics/backend/api/otel/event_merger.py b/products/llm_analytics/backend/api/otel/event_merger.py index b7246509efa4..b83b74a5e0c6 100644 --- a/products/llm_analytics/backend/api/otel/event_merger.py +++ b/products/llm_analytics/backend/api/otel/event_merger.py @@ -60,23 +60,11 @@ def cache_and_merge_properties( # Logs already cached - merge and send logs_properties = json.loads(logs_json) merged = {**logs_properties, **properties} # Trace props override - - # Clean up logs cache redis_client.delete(logs_cache_key) - - logger.debug( - "event_merger_success: Merged trace+logs", extra={"trace_id": trace_id, "span_id": span_id} - ) - return merged else: # No logs yet - cache trace redis_client.setex(trace_cache_key, _CACHE_TTL, json.dumps(properties)) - - logger.debug( - "event_merger_cache: Cached trace properties", extra={"trace_id": trace_id, "span_id": span_id} - ) - return None else: # Log arriving - accumulate with other logs first @@ -86,31 +74,16 @@ def cache_and_merge_properties( # Another log already cached - accumulate existing_logs = json.loads(logs_json) merged_logs = {**existing_logs, **properties} # Later log props override - - # Re-cache accumulated logs redis_client.setex(logs_cache_key, _CACHE_TTL, json.dumps(merged_logs)) - logger.debug( - "event_merger_accumulate: Accumulated log properties", - extra={"trace_id": trace_id, "span_id": span_id}, - ) - # Check if trace is ready trace_json = redis_client.get(trace_cache_key) if trace_json: # Trace is ready - merge and send trace_properties = json.loads(trace_json) final_merged = {**merged_logs, **trace_properties} # Trace props override - - # Clean up both caches redis_client.delete(logs_cache_key) redis_client.delete(trace_cache_key) - - logger.debug( - "event_merger_success: Merged accumulated logs+trace", - extra={"trace_id": trace_id, "span_id": span_id}, - ) - return final_merged # Trace not ready yet - wait for it @@ -123,23 +96,11 @@ def cache_and_merge_properties( # Trace already cached - merge and send trace_properties = json.loads(trace_json) merged = {**properties, **trace_properties} # Trace props override - - # Clean up trace cache redis_client.delete(trace_cache_key) - - logger.debug( - "event_merger_success: Merged logs+trace", extra={"trace_id": trace_id, "span_id": span_id} - ) - return merged else: # No trace yet - cache this log redis_client.setex(logs_cache_key, _CACHE_TTL, json.dumps(properties)) - - logger.debug( - "event_merger_cache: Cached log properties", extra={"trace_id": trace_id, "span_id": span_id} - ) - return None except Exception as e: diff --git a/products/llm_analytics/backend/api/otel/ingestion.py b/products/llm_analytics/backend/api/otel/ingestion.py index 863c568a91aa..fab816405129 100644 --- a/products/llm_analytics/backend/api/otel/ingestion.py +++ b/products/llm_analytics/backend/api/otel/ingestion.py @@ -208,13 +208,6 @@ def otel_traces_endpoint(request: HttpRequest, project_id: int) -> Response: status=status.HTTP_400_BAD_REQUEST, ) - logger.debug( - "otel_traces_received", - team_id=team.id, - content_length=len(protobuf_data), - content_type=content_type, - ) - try: # Parse baggage from headers (for session context) baggage_header = request.headers.get("baggage") @@ -223,13 +216,6 @@ def otel_traces_endpoint(request: HttpRequest, project_id: int) -> Response: # Parse OTLP protobuf parsed_request = parse_otlp_trace_request(protobuf_data) - logger.debug( - "otel_traces_parsed", - team_id=team.id, - spans_count=len(parsed_request["spans"]), - has_baggage=bool(baggage), - ) - # Validate request validation_errors = validate_otlp_request(parsed_request) if validation_errors: @@ -246,21 +232,9 @@ def otel_traces_endpoint(request: HttpRequest, project_id: int) -> Response: # Transform spans to AI events events = transform_spans_to_ai_events(parsed_request, baggage) - logger.debug( - "otel_traces_transformed", - team_id=team.id, - events_created=len(events), - ) - # Route to capture pipeline capture_events(events, team) - logger.debug( - "otel_traces_captured", - team_id=team.id, - events_captured=len(events), - ) - return Response( { "status": "success", @@ -542,23 +516,10 @@ def otel_logs_endpoint(request: HttpRequest, project_id: int) -> Response: status=status.HTTP_400_BAD_REQUEST, ) - logger.debug( - "otel_logs_received", - team_id=team.id, - content_length=len(protobuf_data), - content_type=content_type, - ) - try: # Parse OTLP protobuf parsed_request = parse_otlp_logs_request(protobuf_data) - logger.debug( - "otel_logs_parsed", - team_id=team.id, - logs_count=len(parsed_request["logs"]), - ) - # Validate request validation_errors = validate_otlp_logs_request(parsed_request) if validation_errors: @@ -575,26 +536,9 @@ def otel_logs_endpoint(request: HttpRequest, project_id: int) -> Response: # Transform logs to AI events (also caches properties for merging with traces) events = transform_logs_to_ai_events(parsed_request) - logger.debug( - "otel_logs_transformed", - team_id=team.id, - events_created=len(events), - ) - - # True bidirectional merge with Redis: - # - First arrivals (logs before traces): Cached in Redis, no events to send - # - Second arrivals (logs after traces): Merged and sent to capture - # The transform function filters out first arrivals, so events only contains second arrivals - # Route merged events to capture pipeline capture_events(events, team) - logger.debug( - "otel_logs_captured", - team_id=team.id, - events_captured=len(events), - ) - return Response( { "status": "success", @@ -714,16 +658,6 @@ def transform_logs_to_ai_events(parsed_request: dict[str, Any]) -> list[dict[str trace_id = log_record.get("trace_id", "") span_id = log_record.get("span_id", "") - # Debug logging for ingestion parity validation - logger.debug( - "otel_log_received", - trace_id=trace_id, - span_id=span_id, - event_name=log_record.get("attributes", {}).get("event.name"), - body_keys=list(log_record.get("body", {}).keys()) if isinstance(log_record.get("body"), dict) else None, - body_content_preview=str(log_record.get("body", {}))[:200] if log_record.get("body") else None, - ) - if trace_id and span_id: logs_by_span[(trace_id, span_id)].append(log_record) else: diff --git a/products/llm_analytics/backend/api/otel/transformer.py b/products/llm_analytics/backend/api/otel/transformer.py index b3737630a768..242717ef6a39 100644 --- a/products/llm_analytics/backend/api/otel/transformer.py +++ b/products/llm_analytics/backend/api/otel/transformer.py @@ -48,23 +48,8 @@ def transform_span_to_ai_event( - properties: AI event properties - uuid: Event UUID (for deduplication with log events) """ - import structlog - - logger = structlog.get_logger(__name__) baggage = baggage or {} - # Debug logging for ingestion parity validation - logger.debug( - "otel_span_received", - trace_id=span.get("trace_id"), - span_id=span.get("span_id"), - span_name=span.get("name"), - scope_name=scope.get("name"), - attributes_keys=list(span.get("attributes", {}).keys()), - has_prompt_attrs=any(k.startswith("gen_ai.prompt.") for k in span.get("attributes", {}).keys()), - has_completion_attrs=any(k.startswith("gen_ai.completion.") for k in span.get("attributes", {}).keys()), - ) - # Extract attributes using waterfall pattern posthog_attrs = extract_posthog_native_attributes(span) genai_attrs = extract_genai_attributes(span, scope) From 6f0bf261ce43fa398145b76bc937d8f744807e38 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Thu, 27 Nov 2025 12:48:03 +0000 Subject: [PATCH 32/34] fix(otel): Send embedding spans immediately without waiting for logs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Embedding operations don't have associated log events (no prompt/completion), so they were getting cached in Redis waiting for logs that never arrive. Now detect embedding spans by operation_name and send them immediately, preventing $ai_embedding events from being silently lost. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- products/llm_analytics/backend/api/otel/transformer.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/products/llm_analytics/backend/api/otel/transformer.py b/products/llm_analytics/backend/api/otel/transformer.py index 242717ef6a39..4824b3651feb 100644 --- a/products/llm_analytics/backend/api/otel/transformer.py +++ b/products/llm_analytics/backend/api/otel/transformer.py @@ -68,11 +68,17 @@ def transform_span_to_ai_event( # Detection priority: # 1. Provider declares pattern via get_instrumentation_pattern() - most reliable # 2. Span has prompt/completion attributes - indicates V1 data present - # 3. Default to V2 (safer - waits for logs rather than sending incomplete) + # 3. Span is embedding operation - embeddings don't have associated logs + # 4. Default to V2 (safer - waits for logs rather than sending incomplete) provider = detect_provider(span, scope) provider_pattern = provider.get_instrumentation_pattern() if provider else None has_v1_content = bool(merged_attrs.get("prompt") or merged_attrs.get("completion")) - uses_v1_pattern = provider_pattern == OtelInstrumentationPattern.V1_ATTRIBUTES or has_v1_content + + # Embedding spans are self-contained - they don't have prompt/completion logs + op_name = merged_attrs.get("operation_name", "").lower() + is_embedding = op_name in ("embedding", "embeddings") + + uses_v1_pattern = provider_pattern == OtelInstrumentationPattern.V1_ATTRIBUTES or has_v1_content or is_embedding if not uses_v1_pattern: # V2 instrumentation - use event merger for bidirectional merge with logs From 3c7822cf00bffe1213c8b3149f6e72033b632943 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Thu, 27 Nov 2025 13:17:55 +0000 Subject: [PATCH 33/34] fix(otel): Preserve per-span resource/scope instead of flattening to last-seen MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously, parse_otlp_request and parse_otlp_logs_request would overwrite resource_attrs/scope_info inside loops, causing all spans/logs in a request with multiple resource_spans/resource_logs to emit with only the final resource/scope attached. This fix changes the parsers to return per-item context: - parse_otlp_request returns list of {span, resource, scope} - parse_otlp_logs_request returns list of {log, resource, scope} Updated transform_spans_to_ai_events and transform_logs_to_ai_events to extract resource/scope per item, ensuring correct service.name, distinct_id, and other resource-derived attributes. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- .../backend/api/otel/ingestion.py | 101 +++--- .../backend/api/otel/logs_parser.py | 35 ++- .../llm_analytics/backend/api/otel/parser.py | 37 ++- .../backend/api/otel/test_ingestion_parity.py | 18 +- .../api/otel/test_logs_accumulation.py | 288 +++++++++--------- 5 files changed, 256 insertions(+), 223 deletions(-) diff --git a/products/llm_analytics/backend/api/otel/ingestion.py b/products/llm_analytics/backend/api/otel/ingestion.py index fab816405129..be648c0ad9c8 100644 --- a/products/llm_analytics/backend/api/otel/ingestion.py +++ b/products/llm_analytics/backend/api/otel/ingestion.py @@ -239,7 +239,7 @@ def otel_traces_endpoint(request: HttpRequest, project_id: int) -> Response: { "status": "success", "message": "Traces ingested successfully", - "spans_received": len(parsed_request["spans"]), + "spans_received": len(parsed_request), "events_created": len(events), }, status=status.HTTP_200_OK, @@ -269,40 +269,41 @@ def otel_traces_endpoint(request: HttpRequest, project_id: int) -> Response: ) -def parse_otlp_trace_request(protobuf_data: bytes) -> dict[str, Any]: +def parse_otlp_trace_request(protobuf_data: bytes) -> list[dict[str, Any]]: """ Parse OTLP ExportTraceServiceRequest from protobuf bytes. - Returns dict with: - - spans: list of parsed span dicts - - resource: dict of resource attributes - - scope: dict of instrumentation scope info + Returns list of dicts, each containing: + - span: parsed span dict + - resource: dict of resource attributes for this span + - scope: dict of instrumentation scope info for this span """ return parse_otlp_request(protobuf_data) -def validate_otlp_request(parsed_request: dict[str, Any]) -> list[dict[str, Any]]: +def validate_otlp_request(parsed_items: list[dict[str, Any]]) -> list[dict[str, Any]]: """ Validate OTLP request against limits. Returns list of validation errors (empty if valid). """ errors = [] - spans = parsed_request.get("spans", []) # Check span count - if len(spans) > OTEL_LIMITS["MAX_SPANS_PER_REQUEST"]: + if len(parsed_items) > OTEL_LIMITS["MAX_SPANS_PER_REQUEST"]: errors.append( { "field": "request.spans", - "value": len(spans), + "value": len(parsed_items), "limit": OTEL_LIMITS["MAX_SPANS_PER_REQUEST"], - "message": f"Request contains {len(spans)} spans, maximum is {OTEL_LIMITS['MAX_SPANS_PER_REQUEST']}. Configure batch size in your OTel SDK (e.g., OTEL_BSP_MAX_EXPORT_BATCH_SIZE).", + "message": f"Request contains {len(parsed_items)} spans, maximum is {OTEL_LIMITS['MAX_SPANS_PER_REQUEST']}. Configure batch size in your OTel SDK (e.g., OTEL_BSP_MAX_EXPORT_BATCH_SIZE).", } ) # Validate each span - for i, span in enumerate(spans): + for i, item in enumerate(parsed_items): + span = item.get("span", {}) + # Check span name length span_name = span.get("name", "") if len(span_name) > OTEL_LIMITS["MAX_SPAN_NAME_LENGTH"]: @@ -366,7 +367,7 @@ def validate_otlp_request(parsed_request: dict[str, Any]) -> list[dict[str, Any] return errors -def transform_spans_to_ai_events(parsed_request: dict[str, Any], baggage: dict[str, str]) -> list[dict[str, Any]]: +def transform_spans_to_ai_events(parsed_items: list[dict[str, Any]], baggage: dict[str, str]) -> list[dict[str, Any]]: """ Transform OTel spans to PostHog AI events. @@ -374,15 +375,18 @@ def transform_spans_to_ai_events(parsed_request: dict[str, Any], baggage: dict[s 1. PostHog native (posthog.ai.*) 2. GenAI semantic conventions (gen_ai.*) + Each item contains its own span, resource, and scope context to correctly + handle requests with multiple resource_spans/scope_spans. + Note: Returns only events ready to send. Events that are first arrivals (cached, waiting for logs) are filtered out. """ - spans = parsed_request.get("spans", []) - resource = parsed_request.get("resource", {}) - scope = parsed_request.get("scope", {}) - events = [] - for span in spans: + for item in parsed_items: + span = item.get("span", {}) + resource = item.get("resource", {}) + scope = item.get("scope", {}) + event = transform_span_to_ai_event(span, resource, scope, baggage) if event is not None: # Filter out first arrivals (cached, waiting for logs) events.append(event) @@ -543,7 +547,7 @@ def otel_logs_endpoint(request: HttpRequest, project_id: int) -> Response: { "status": "success", "message": "Logs ingested successfully", - "logs_received": len(parsed_request["logs"]), + "logs_received": len(parsed_request), "events_created": len(events), }, status=status.HTTP_200_OK, @@ -573,28 +577,29 @@ def otel_logs_endpoint(request: HttpRequest, project_id: int) -> Response: ) -def validate_otlp_logs_request(parsed_request: dict[str, Any]) -> list[dict[str, Any]]: +def validate_otlp_logs_request(parsed_items: list[dict[str, Any]]) -> list[dict[str, Any]]: """ Validate OTLP logs request against limits. Returns list of validation errors (empty if valid). """ errors = [] - logs = parsed_request.get("logs", []) # Check log count - if len(logs) > OTEL_LIMITS["MAX_LOGS_PER_REQUEST"]: + if len(parsed_items) > OTEL_LIMITS["MAX_LOGS_PER_REQUEST"]: errors.append( { "field": "request.logs", - "value": len(logs), + "value": len(parsed_items), "limit": OTEL_LIMITS["MAX_LOGS_PER_REQUEST"], - "message": f"Request contains {len(logs)} logs, maximum is {OTEL_LIMITS['MAX_LOGS_PER_REQUEST']}. Configure batch size in your OTel SDK.", + "message": f"Request contains {len(parsed_items)} logs, maximum is {OTEL_LIMITS['MAX_LOGS_PER_REQUEST']}. Configure batch size in your OTel SDK.", } ) # Validate each log record - for i, log_record in enumerate(logs): + for i, item in enumerate(parsed_items): + log_record = item.get("log", {}) + # Check attribute count attributes = log_record.get("attributes", {}) if len(attributes) > OTEL_LIMITS["MAX_ATTRIBUTES_PER_LOG"]: @@ -634,7 +639,7 @@ def validate_otlp_logs_request(parsed_request: dict[str, Any]) -> list[dict[str, return errors -def transform_logs_to_ai_events(parsed_request: dict[str, Any]) -> list[dict[str, Any]]: +def transform_logs_to_ai_events(parsed_items: list[dict[str, Any]]) -> list[dict[str, Any]]: """ Transform OTel log records to PostHog AI events. @@ -642,37 +647,47 @@ def transform_logs_to_ai_events(parsed_request: dict[str, Any]) -> list[dict[str arrive in the SAME HTTP request. We must accumulate ALL logs for the same span BEFORE calling the event merger to avoid race conditions where the trace consumes partial logs. + Each item contains its own log, resource, and scope context to correctly + handle requests with multiple resource_logs/scope_logs. + Note: Returns only events ready to send. Events that are first arrivals (cached, waiting for traces) are filtered out. """ - logs = parsed_request.get("logs", []) - resource = parsed_request.get("resource", {}) - scope = parsed_request.get("scope", {}) - # Group logs by (trace_id, span_id) to accumulate them before merging + # Store full items (with resource/scope) not just log records from collections import defaultdict - logs_by_span = defaultdict(list) + logs_by_span: dict[tuple[str | None, str | None], list[dict[str, Any]]] = defaultdict(list) - for log_record in logs: + for item in parsed_items: + log_record = item.get("log", {}) trace_id = log_record.get("trace_id", "") span_id = log_record.get("span_id", "") if trace_id and span_id: - logs_by_span[(trace_id, span_id)].append(log_record) + logs_by_span[(trace_id, span_id)].append(item) else: # No trace/span ID - process individually - logs_by_span[(None, None)].append(log_record) + logs_by_span[(None, None)].append(item) events = [] # Process each span's logs together - for (trace_id, span_id), span_logs in logs_by_span.items(): + for (trace_id, span_id), span_items in logs_by_span.items(): if trace_id and span_id: # Accumulate properties from all logs for this span + # Use the first item's resource/scope (logs from same span should have same context) + first_item = span_items[0] + resource = first_item.get("resource", {}) + scope = first_item.get("scope", {}) + accumulated_props = {} - for log_record in span_logs: - props = build_event_properties(log_record, log_record.get("attributes", {}), resource, scope) + for item in span_items: + log_record = item.get("log", {}) + # Use each log's own resource/scope for property building + item_resource = item.get("resource", {}) + item_scope = item.get("scope", {}) + props = build_event_properties(log_record, log_record.get("attributes", {}), item_resource, item_scope) # Merge properties with special handling for arrays for key, value in props.items(): if key in ("$ai_input", "$ai_output_choices"): @@ -692,9 +707,10 @@ def transform_logs_to_ai_events(parsed_request: dict[str, Any]) -> list[dict[str if merged is not None: # Ready to send - create event - event_type = determine_event_type(span_logs[0], span_logs[0].get("attributes", {})) - timestamp = calculate_timestamp(span_logs[0]) - distinct_id = extract_distinct_id(resource, span_logs[0].get("attributes", {})) + first_log = first_item.get("log", {}) + event_type = determine_event_type(first_log, first_log.get("attributes", {})) + timestamp = calculate_timestamp(first_log) + distinct_id = extract_distinct_id(resource, first_log.get("attributes", {})) # Generate consistent UUID from trace_id + span_id import uuid @@ -712,7 +728,10 @@ def transform_logs_to_ai_events(parsed_request: dict[str, Any]) -> list[dict[str events.append(event) else: # No trace/span ID - process logs individually (shouldn't happen in normal v2) - for log_record in span_logs: + for item in span_items: + log_record = item.get("log", {}) + resource = item.get("resource", {}) + scope = item.get("scope", {}) event = transform_log_to_ai_event(log_record, resource, scope) if event is not None: events.append(event) diff --git a/products/llm_analytics/backend/api/otel/logs_parser.py b/products/llm_analytics/backend/api/otel/logs_parser.py index 31a1d2a1d323..d7aa7f29085d 100644 --- a/products/llm_analytics/backend/api/otel/logs_parser.py +++ b/products/llm_analytics/backend/api/otel/logs_parser.py @@ -13,31 +13,34 @@ from .parser import parse_any_value, parse_attributes -def parse_otlp_logs_request(protobuf_data: bytes) -> dict[str, Any]: +def parse_otlp_logs_request(protobuf_data: bytes) -> list[dict[str, Any]]: """ Parse OTLP ExportLogsServiceRequest from protobuf bytes. - Returns a dict with: - - logs: list of parsed log record dicts - - resource: dict of resource attributes - - scope: dict of instrumentation scope info + Returns a list of dicts, each containing: + - log: parsed log record dict + - resource: dict of resource attributes for this log + - scope: dict of instrumentation scope info for this log + + Each log carries its own resource/scope context to handle requests + containing multiple resource_logs/scope_logs correctly. """ request = ExportLogsServiceRequest() request.ParseFromString(protobuf_data) - parsed_logs = [] - resource_attrs = {} - scope_info = {} + results = [] # OTLP structure: resource_logs -> scope_logs -> log_records for resource_logs in request.resource_logs: # Extract resource attributes (service.name, etc.) + resource_attrs = {} if resource_logs.HasField("resource"): resource_attrs = parse_attributes(resource_logs.resource.attributes) # Iterate through scope logs for scope_logs in resource_logs.scope_logs: # Extract instrumentation scope + scope_info = {} if scope_logs.HasField("scope"): scope_info = { "name": scope_logs.scope.name, @@ -45,16 +48,18 @@ def parse_otlp_logs_request(protobuf_data: bytes) -> dict[str, Any]: "attributes": parse_attributes(scope_logs.scope.attributes) if scope_logs.scope.attributes else {}, } - # Parse each log record + # Parse each log record with its resource/scope context for log_record in scope_logs.log_records: parsed_log = parse_log_record(log_record) - parsed_logs.append(parsed_log) + results.append( + { + "log": parsed_log, + "resource": resource_attrs, + "scope": scope_info, + } + ) - return { - "logs": parsed_logs, - "resource": resource_attrs, - "scope": scope_info, - } + return results def parse_log_record(log_record: LogRecord) -> dict[str, Any]: diff --git a/products/llm_analytics/backend/api/otel/parser.py b/products/llm_analytics/backend/api/otel/parser.py index cb3281b542a4..817751b3b620 100644 --- a/products/llm_analytics/backend/api/otel/parser.py +++ b/products/llm_analytics/backend/api/otel/parser.py @@ -12,31 +12,34 @@ from opentelemetry.proto.trace.v1.trace_pb2 import Span -def parse_otlp_request(protobuf_data: bytes) -> dict[str, Any]: +def parse_otlp_request(protobuf_data: bytes) -> list[dict[str, Any]]: """ Parse OTLP ExportTraceServiceRequest from protobuf bytes. - Returns a dict with: - - spans: list of parsed span dicts - - resource: dict of resource attributes - - scope: dict of instrumentation scope info + Returns a list of dicts, each containing: + - span: parsed span dict + - resource: dict of resource attributes for this span + - scope: dict of instrumentation scope info for this span + + Each span carries its own resource/scope context to handle requests + containing multiple resource_spans/scope_spans correctly. """ request = ExportTraceServiceRequest() request.ParseFromString(protobuf_data) - parsed_spans = [] - resource_attrs = {} - scope_info = {} + results = [] # OTLP structure: resource_spans -> scope_spans -> spans for resource_spans in request.resource_spans: # Extract resource attributes (service.name, etc.) + resource_attrs = {} if resource_spans.HasField("resource"): resource_attrs = parse_attributes(resource_spans.resource.attributes) # Iterate through scope spans for scope_spans in resource_spans.scope_spans: # Extract instrumentation scope + scope_info = {} if scope_spans.HasField("scope"): scope_info = { "name": scope_spans.scope.name, @@ -46,16 +49,18 @@ def parse_otlp_request(protobuf_data: bytes) -> dict[str, Any]: else {}, } - # Parse each span + # Parse each span with its resource/scope context for span in scope_spans.spans: parsed_span = parse_span(span) - parsed_spans.append(parsed_span) - - return { - "spans": parsed_spans, - "resource": resource_attrs, - "scope": scope_info, - } + results.append( + { + "span": parsed_span, + "resource": resource_attrs, + "scope": scope_info, + } + ) + + return results def parse_span(span: Span) -> dict[str, Any]: diff --git a/products/llm_analytics/backend/api/otel/test_ingestion_parity.py b/products/llm_analytics/backend/api/otel/test_ingestion_parity.py index 71765819f228..df4544b881d8 100644 --- a/products/llm_analytics/backend/api/otel/test_ingestion_parity.py +++ b/products/llm_analytics/backend/api/otel/test_ingestion_parity.py @@ -115,7 +115,7 @@ def create_v1_span_with_conversation( def create_v2_logs_with_conversation( trace_id: str = "af4e25c0d86a2f7bebd2e0c84f072499", span_id: str = "a19561fe0a9d2d73", -) -> dict: +) -> list[dict]: """ Create v2 OTEL logs request with multi-turn conversation. @@ -124,8 +124,16 @@ def create_v2_logs_with_conversation( - Body contains content directly (not nested in message for input) - gen_ai.choice logs have message wrapper with role/content + finish_reason - Event names: gen_ai.system.message, gen_ai.user.message, gen_ai.choice + + Returns a list of dicts, each containing: + - log: parsed log record dict + - resource: dict of resource attributes for this log + - scope: dict of instrumentation scope info for this log """ base_time = 1700000000000000000 + resource = {"service.name": "test-service"} + # Match actual scope name from real instrumentation + scope = {"name": "opentelemetry.instrumentation.openai_v2", "version": "2.0.0"} logs = [ # System message @@ -241,12 +249,8 @@ def create_v2_logs_with_conversation( }, ] - return { - "logs": logs, - "resource": {"service.name": "test-service"}, - # Match actual scope name from real instrumentation - "scope": {"name": "opentelemetry.instrumentation.openai_v2", "version": "2.0.0"}, - } + # Return in per-item format with resource/scope context + return [{"log": log, "resource": resource, "scope": scope} for log in logs] def create_v2_span_metadata( diff --git a/products/llm_analytics/backend/api/otel/test_logs_accumulation.py b/products/llm_analytics/backend/api/otel/test_logs_accumulation.py index a2d2fbc9ddf5..017809fa28bb 100644 --- a/products/llm_analytics/backend/api/otel/test_logs_accumulation.py +++ b/products/llm_analytics/backend/api/otel/test_logs_accumulation.py @@ -11,29 +11,33 @@ from products.llm_analytics.backend.api.otel.ingestion import transform_logs_to_ai_events +def _make_parsed_items(logs: list[dict], resource: dict, scope: dict) -> list[dict]: + """Helper to convert old format to new per-item format.""" + return [{"log": log, "resource": resource, "scope": scope} for log in logs] + + def test_multiple_user_messages_accumulate(): """Test that multiple user messages accumulate into $ai_input array.""" # Simulate multiple log records for same span with different user messages - parsed_request = { - "logs": [ - { - "trace_id": "trace123", - "span_id": "span456", - "attributes": {"event.name": "gen_ai.user.message"}, - "body": {"content": "hi there"}, - "time_unix_nano": 1000000000, - }, - { - "trace_id": "trace123", - "span_id": "span456", - "attributes": {"event.name": "gen_ai.user.message"}, - "body": {"content": "k bye"}, - "time_unix_nano": 2000000000, - }, - ], - "resource": {"service.name": "test-service"}, - "scope": {"name": "test-scope"}, - } + logs = [ + { + "trace_id": "trace123", + "span_id": "span456", + "attributes": {"event.name": "gen_ai.user.message"}, + "body": {"content": "hi there"}, + "time_unix_nano": 1000000000, + }, + { + "trace_id": "trace123", + "span_id": "span456", + "attributes": {"event.name": "gen_ai.user.message"}, + "body": {"content": "k bye"}, + "time_unix_nano": 2000000000, + }, + ] + resource = {"service.name": "test-service"} + scope = {"name": "test-scope"} + parsed_items = _make_parsed_items(logs, resource, scope) # Mock event merger to return merged properties on second call with patch("products.llm_analytics.backend.api.otel.event_merger.cache_and_merge_properties") as mock_merger: @@ -47,7 +51,7 @@ def merger_side_effect(trace_id, span_id, props, is_trace): mock_merger.side_effect = merger_side_effect - _events = transform_logs_to_ai_events(parsed_request) + _events = transform_logs_to_ai_events(parsed_items) # Verify accumulation happened before calling merger # The merger should have been called once with both messages accumulated @@ -62,43 +66,42 @@ def merger_side_effect(trace_id, span_id, props, is_trace): def test_user_and_assistant_messages_accumulate(): """Test that conversation history (including assistant messages) goes into $ai_input.""" - parsed_request = { - "logs": [ - { - "trace_id": "trace789", - "span_id": "span012", - "attributes": {"event.name": "gen_ai.user.message"}, - "body": {"content": "hello"}, - "time_unix_nano": 1000000000, - }, - { - "trace_id": "trace789", - "span_id": "span012", - "attributes": {"event.name": "gen_ai.assistant.message"}, - "body": {"content": "hi there!"}, - "time_unix_nano": 2000000000, + logs = [ + { + "trace_id": "trace789", + "span_id": "span012", + "attributes": {"event.name": "gen_ai.user.message"}, + "body": {"content": "hello"}, + "time_unix_nano": 1000000000, + }, + { + "trace_id": "trace789", + "span_id": "span012", + "attributes": {"event.name": "gen_ai.assistant.message"}, + "body": {"content": "hi there!"}, + "time_unix_nano": 2000000000, + }, + { + "trace_id": "trace789", + "span_id": "span012", + "attributes": {"event.name": "gen_ai.user.message"}, + "body": {"content": "tell me a joke"}, + "time_unix_nano": 3000000000, + }, + { + "trace_id": "trace789", + "span_id": "span012", + "attributes": {"event.name": "gen_ai.choice"}, + "body": { + "message": {"role": "assistant", "content": "Why did the chicken cross the road?"}, + "finish_reason": "stop", }, - { - "trace_id": "trace789", - "span_id": "span012", - "attributes": {"event.name": "gen_ai.user.message"}, - "body": {"content": "tell me a joke"}, - "time_unix_nano": 3000000000, - }, - { - "trace_id": "trace789", - "span_id": "span012", - "attributes": {"event.name": "gen_ai.choice"}, - "body": { - "message": {"role": "assistant", "content": "Why did the chicken cross the road?"}, - "finish_reason": "stop", - }, - "time_unix_nano": 4000000000, - }, - ], - "resource": {"service.name": "test-service"}, - "scope": {"name": "test-scope"}, - } + "time_unix_nano": 4000000000, + }, + ] + resource = {"service.name": "test-service"} + scope = {"name": "test-scope"} + parsed_items = _make_parsed_items(logs, resource, scope) with patch("products.llm_analytics.backend.api.otel.event_merger.cache_and_merge_properties") as mock_merger: @@ -110,7 +113,7 @@ def merger_side_effect(trace_id, span_id, props, is_trace): mock_merger.side_effect = merger_side_effect - _events = transform_logs_to_ai_events(parsed_request) + _events = transform_logs_to_ai_events(parsed_items) # Verify conversation history (user + assistant) accumulated in $ai_input # and only current response in $ai_output_choices @@ -131,52 +134,51 @@ def merger_side_effect(trace_id, span_id, props, is_trace): def test_tool_messages_accumulate(): """Test that tool messages are properly handled and accumulate in conversation history.""" - parsed_request = { - "logs": [ - { - "trace_id": "trace999", - "span_id": "span888", - "attributes": {"event.name": "gen_ai.user.message"}, - "body": {"content": "What's the weather in Paris?"}, - "time_unix_nano": 1000000000, - }, - { - "trace_id": "trace999", - "span_id": "span888", - "attributes": {"event.name": "gen_ai.assistant.message"}, - "body": { - "content": None, - "tool_calls": [ - { - "id": "call_123", - "type": "function", - "function": {"name": "get_weather", "arguments": '{"location":"Paris"}'}, - } - ], - }, - "time_unix_nano": 2000000000, - }, - { - "trace_id": "trace999", - "span_id": "span888", - "attributes": {"event.name": "gen_ai.tool.message"}, - "body": {"content": "Sunny, 18°C", "id": "call_123"}, - "time_unix_nano": 3000000000, + logs = [ + { + "trace_id": "trace999", + "span_id": "span888", + "attributes": {"event.name": "gen_ai.user.message"}, + "body": {"content": "What's the weather in Paris?"}, + "time_unix_nano": 1000000000, + }, + { + "trace_id": "trace999", + "span_id": "span888", + "attributes": {"event.name": "gen_ai.assistant.message"}, + "body": { + "content": None, + "tool_calls": [ + { + "id": "call_123", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"location":"Paris"}'}, + } + ], }, - { - "trace_id": "trace999", - "span_id": "span888", - "attributes": {"event.name": "gen_ai.choice"}, - "body": { - "message": {"role": "assistant", "content": "The weather in Paris is sunny with 18°C."}, - "finish_reason": "stop", - }, - "time_unix_nano": 4000000000, + "time_unix_nano": 2000000000, + }, + { + "trace_id": "trace999", + "span_id": "span888", + "attributes": {"event.name": "gen_ai.tool.message"}, + "body": {"content": "Sunny, 18°C", "id": "call_123"}, + "time_unix_nano": 3000000000, + }, + { + "trace_id": "trace999", + "span_id": "span888", + "attributes": {"event.name": "gen_ai.choice"}, + "body": { + "message": {"role": "assistant", "content": "The weather in Paris is sunny with 18°C."}, + "finish_reason": "stop", }, - ], - "resource": {"service.name": "test-service"}, - "scope": {"name": "test-scope"}, - } + "time_unix_nano": 4000000000, + }, + ] + resource = {"service.name": "test-service"} + scope = {"name": "test-scope"} + parsed_items = _make_parsed_items(logs, resource, scope) with patch("products.llm_analytics.backend.api.otel.event_merger.cache_and_merge_properties") as mock_merger: @@ -188,7 +190,7 @@ def merger_side_effect(trace_id, span_id, props, is_trace): mock_merger.side_effect = merger_side_effect - _events = transform_logs_to_ai_events(parsed_request) + _events = transform_logs_to_ai_events(parsed_items) # Verify tool message was properly accumulated assert mock_merger.call_count == 1 @@ -221,34 +223,33 @@ def merger_side_effect(trace_id, span_id, props, is_trace): def test_non_array_properties_are_overwritten(): """Test that non-array properties use last-value-wins behavior.""" - parsed_request = { - "logs": [ - { - "trace_id": "trace111", - "span_id": "span222", - "attributes": { - "event.name": "gen_ai.user.message", - "gen_ai.request.model": "gpt-3.5", - "gen_ai.usage.input_tokens": 10, - }, - "body": {"content": "hello"}, - "time_unix_nano": 1000000000, + logs = [ + { + "trace_id": "trace111", + "span_id": "span222", + "attributes": { + "event.name": "gen_ai.user.message", + "gen_ai.request.model": "gpt-3.5", + "gen_ai.usage.input_tokens": 10, }, - { - "trace_id": "trace111", - "span_id": "span222", - "attributes": { - "event.name": "gen_ai.choice", - "gen_ai.response.model": "gpt-4", # Different model in response - "gen_ai.usage.output_tokens": 20, - }, - "body": {"message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}, - "time_unix_nano": 2000000000, + "body": {"content": "hello"}, + "time_unix_nano": 1000000000, + }, + { + "trace_id": "trace111", + "span_id": "span222", + "attributes": { + "event.name": "gen_ai.choice", + "gen_ai.response.model": "gpt-4", # Different model in response + "gen_ai.usage.output_tokens": 20, }, - ], - "resource": {"service.name": "test-service"}, - "scope": {"name": "test-scope"}, - } + "body": {"message": {"role": "assistant", "content": "hi"}, "finish_reason": "stop"}, + "time_unix_nano": 2000000000, + }, + ] + resource = {"service.name": "test-service"} + scope = {"name": "test-scope"} + parsed_items = _make_parsed_items(logs, resource, scope) with patch("products.llm_analytics.backend.api.otel.event_merger.cache_and_merge_properties") as mock_merger: @@ -259,7 +260,7 @@ def merger_side_effect(trace_id, span_id, props, is_trace): mock_merger.side_effect = merger_side_effect - _events = transform_logs_to_ai_events(parsed_request) + _events = transform_logs_to_ai_events(parsed_items) # Verify non-array properties were overwritten (last value wins) call_args = mock_merger.call_args @@ -271,24 +272,23 @@ def merger_side_effect(trace_id, span_id, props, is_trace): def test_single_log_event_works(): """Test that single log events still work (no accumulation needed).""" - parsed_request = { - "logs": [ - { - "trace_id": "trace333", - "span_id": "span444", - "attributes": {"event.name": "gen_ai.user.message"}, - "body": {"content": "single message"}, - "time_unix_nano": 1000000000, - } - ], - "resource": {"service.name": "test-service"}, - "scope": {"name": "test-scope"}, - } + logs = [ + { + "trace_id": "trace333", + "span_id": "span444", + "attributes": {"event.name": "gen_ai.user.message"}, + "body": {"content": "single message"}, + "time_unix_nano": 1000000000, + } + ] + resource = {"service.name": "test-service"} + scope = {"name": "test-scope"} + parsed_items = _make_parsed_items(logs, resource, scope) with patch("products.llm_analytics.backend.api.otel.event_merger.cache_and_merge_properties") as mock_merger: mock_merger.return_value = None # First arrival, cache - _events = transform_logs_to_ai_events(parsed_request) + _events = transform_logs_to_ai_events(parsed_items) # Verify single message was processed assert mock_merger.call_count == 1 From 80f1a80c215a1697de994da6bc2c20d1b46f30f1 Mon Sep 17 00:00:00 2001 From: Andrew Maguire Date: Thu, 27 Nov 2025 13:41:22 +0000 Subject: [PATCH 34/34] docs(otel): Update README with embedding span handling and per-item parser format MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Added embedding operation detection to pattern detection list - Documented per-item return format for parser.py and logs_parser.py - Clarified that each item carries its own resource/scope context 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude --- products/llm_analytics/backend/api/otel/README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/products/llm_analytics/backend/api/otel/README.md b/products/llm_analytics/backend/api/otel/README.md index 4c5c119d0476..4bc04dcde726 100644 --- a/products/llm_analytics/backend/api/otel/README.md +++ b/products/llm_analytics/backend/api/otel/README.md @@ -128,7 +128,8 @@ Determines event type based on span characteristics: 1. Provider declares pattern via `get_instrumentation_pattern()` (most reliable) 2. Span has `prompt` or `completion` attributes (indicates V1 data present) -3. Default to V2 (safer - waits for logs rather than sending incomplete) +3. Span is an embedding operation (embeddings don't have associated logs) +4. Default to V2 (safer - waits for logs rather than sending incomplete) V1 spans bypass the event merger and are sent immediately. @@ -153,6 +154,13 @@ Redis-based non-blocking cache for v2 trace/log coordination. Uses simple Redis Decode OTLP protobuf messages into Python dictionaries. Handle type conversions (bytes to hex for IDs, nanoseconds to seconds for timestamps) and attribute flattening. +**Return Format**: Both parsers return a list of items where each item contains its own resource and scope context: + +- `parse_otlp_request()`: Returns `[{"span": {...}, "resource": {...}, "scope": {...}}, ...]` +- `parse_otlp_logs_request()`: Returns `[{"log": {...}, "resource": {...}, "scope": {...}}, ...]` + +This per-item format ensures correct resource/scope attribution when a single OTLP request contains multiple `resource_spans`/`resource_logs` (e.g., from different services or scopes). + ### conventions/ Attribute extraction modules implementing semantic conventions: