From dad213ac62e4fa01ec1b843e1b92048c90d10442 Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Fri, 21 Nov 2025 10:37:40 -0500 Subject: [PATCH 01/12] feat: implement remaining interceptors with OTEL --- .../src/client/index.ts | 164 +++++++++++++++++- .../src/instrumentation.ts | 57 +++++- .../src/worker/index.ts | 27 ++- .../src/workflow/definitions.ts | 38 ++++ .../src/workflow/index.ts | 119 ++++++++++++- packages/workflow/src/flags.ts | 10 ++ 6 files changed, 397 insertions(+), 18 deletions(-) diff --git a/packages/interceptors-opentelemetry/src/client/index.ts b/packages/interceptors-opentelemetry/src/client/index.ts index b427236bb..6198ac57d 100644 --- a/packages/interceptors-opentelemetry/src/client/index.ts +++ b/packages/interceptors-opentelemetry/src/client/index.ts @@ -1,6 +1,30 @@ import * as otel from '@opentelemetry/api'; -import type { Next, WorkflowSignalInput, WorkflowStartInput, WorkflowClientInterceptor } from '@temporalio/client'; -import { instrument, headersWithContext, RUN_ID_ATTR_KEY } from '../instrumentation'; +import type { + Next, + WorkflowSignalInput, + WorkflowSignalWithStartInput, + WorkflowStartInput, + WorkflowStartOutput, + WorkflowStartUpdateInput, + WorkflowStartUpdateOutput, + WorkflowStartUpdateWithStartInput, + WorkflowStartUpdateWithStartOutput, + WorkflowQueryInput, + WorkflowTerminateInput, + WorkflowCancelInput, + WorkflowDescribeInput, + WorkflowClientInterceptor, + TerminateWorkflowExecutionResponse, + RequestCancelWorkflowExecutionResponse, + DescribeWorkflowExecutionResponse, +} from '@temporalio/client'; +import { + instrument, + headersWithContext, + RUN_ID_ATTR_KEY, + WORKFLOW_ID_ATTR_KEY, + TERMINATE_REASON_ATTR_KEY, +} from '../instrumentation'; import { SpanName, SPAN_DELIMITER } from '../workflow'; export interface InterceptorOptions { @@ -12,7 +36,7 @@ export interface InterceptorOptions { * * Wraps the operation in an opentelemetry Span and passes it to the Workflow via headers. */ -export class OpenTelemetryWorkflowClientInterceptor implements WorkflowClientInterceptor { +export class OpenTelemetryWorkflowClientInterceptor implements Required { protected readonly tracer: otel.Tracer; constructor(options?: InterceptorOptions) { @@ -42,4 +66,138 @@ export class OpenTelemetryWorkflowClientInterceptor implements WorkflowClientInt }, }); } + + async startWithDetails( + input: WorkflowStartInput, + next: Next + ): Promise { + return await instrument({ + tracer: this.tracer, + spanName: `${SpanName.WORKFLOW_START}${SPAN_DELIMITER}${input.workflowType}`, + fn: async (span) => { + const headers = headersWithContext(input.headers); + const output = await next({ ...input, headers }); + span.setAttribute(RUN_ID_ATTR_KEY, output.runId); + return output; + }, + }); + } + + async startUpdate( + input: WorkflowStartUpdateInput, + next: Next + ): Promise { + return await instrument({ + tracer: this.tracer, + spanName: `${SpanName.WORKFLOW_UPDATE}${SPAN_DELIMITER}${input.updateName}`, + fn: async (span) => { + const headers = headersWithContext(input.headers); + const output = await next({ ...input, headers }); + span.setAttribute(RUN_ID_ATTR_KEY, output.workflowRunId); + return output; + }, + }); + } + + async startUpdateWithStart( + input: WorkflowStartUpdateWithStartInput, + next: Next + ): Promise { + return await instrument({ + tracer: this.tracer, + spanName: `${SpanName.WORKFLOW_UPDATE_WITH_START}${SPAN_DELIMITER}${input.workflowType}${SPAN_DELIMITER}${input.updateName}`, + fn: async (span) => { + const workflowStartHeaders = headersWithContext(input.workflowStartHeaders); + const updateHeaders = headersWithContext(input.updateHeaders); + const output = await next({ ...input, workflowStartHeaders, updateHeaders }); + span.setAttribute(RUN_ID_ATTR_KEY, output.workflowExecution.runId ?? ''); + return output; + }, + }); + } + + async signalWithStart( + input: WorkflowSignalWithStartInput, + next: Next + ): Promise { + return await instrument({ + tracer: this.tracer, + spanName: `${SpanName.WORKFLOW_SIGNAL_WITH_START}${SPAN_DELIMITER}${input.workflowType}${SPAN_DELIMITER}${input.signalName}`, + fn: async (span) => { + const headers = headersWithContext(input.headers); + const runId = await next({ ...input, headers }); + span.setAttribute(RUN_ID_ATTR_KEY, runId); + return runId; + }, + }); + } + + async query(input: WorkflowQueryInput, next: Next): Promise { + return await instrument({ + tracer: this.tracer, + spanName: `${SpanName.WORKFLOW_QUERY}${SPAN_DELIMITER}${input.queryType}`, + fn: async (span) => { + const headers = headersWithContext(input.headers); + span.setAttribute(WORKFLOW_ID_ATTR_KEY, input.workflowExecution.workflowId); + if (input.workflowExecution.runId) { + span.setAttribute(RUN_ID_ATTR_KEY, input.workflowExecution.runId); + } + return await next({ ...input, headers }); + }, + }); + } + + async terminate( + input: WorkflowTerminateInput, + next: Next + ): Promise { + return await instrument({ + tracer: this.tracer, + spanName: SpanName.WORKFLOW_TERMINATE, + fn: async (span) => { + span.setAttribute(WORKFLOW_ID_ATTR_KEY, input.workflowExecution.workflowId); + if (input.workflowExecution.runId) { + span.setAttribute(RUN_ID_ATTR_KEY, input.workflowExecution.runId); + } + if (input.reason) { + span.setAttribute(TERMINATE_REASON_ATTR_KEY, input.reason); + } + return await next(input); + }, + }); + } + + async cancel( + input: WorkflowCancelInput, + next: Next + ): Promise { + return await instrument({ + tracer: this.tracer, + spanName: SpanName.WORKFLOW_CANCEL, + fn: async (span) => { + span.setAttribute(WORKFLOW_ID_ATTR_KEY, input.workflowExecution.workflowId); + if (input.workflowExecution.runId) { + span.setAttribute(RUN_ID_ATTR_KEY, input.workflowExecution.runId); + } + return await next(input); + }, + }); + } + + async describe( + input: WorkflowDescribeInput, + next: Next + ): Promise { + return await instrument({ + tracer: this.tracer, + spanName: SpanName.WORKFLOW_DESCRIBE, + fn: async (span) => { + span.setAttribute(WORKFLOW_ID_ATTR_KEY, input.workflowExecution.workflowId); + if (input.workflowExecution.runId) { + span.setAttribute(RUN_ID_ATTR_KEY, input.workflowExecution.runId); + } + return await next(input); + }, + }); + } } diff --git a/packages/interceptors-opentelemetry/src/instrumentation.ts b/packages/interceptors-opentelemetry/src/instrumentation.ts index 610e05723..0dcb4cd74 100644 --- a/packages/interceptors-opentelemetry/src/instrumentation.ts +++ b/packages/interceptors-opentelemetry/src/instrumentation.ts @@ -12,8 +12,17 @@ import { /** Default trace header for opentelemetry interceptors */ export const TRACE_HEADER = '_tracer-data'; + +// Span attribute keys /** As in workflow run id */ export const RUN_ID_ATTR_KEY = 'run_id'; +export const WORKFLOW_ID_ATTR_KEY = 'workflow_id'; +export const EAGER_START_ATTR_KEY = 'eager_start'; +export const TERMINATE_REASON_ATTR_KEY = 'terminate_reason'; +export const TIMER_DURATION_MS_ATTR_KEY = 'timer_duration_ms'; +export const NEXUS_SERVICE_ATTR_KEY = 'nexus_service'; +export const NEXUS_OPERATION_ATTR_KEY = 'nexus_operation'; +export const NEXUS_ENDPOINT_ATTR_KEY = 'nexus_endpoint'; const payloadConverter = defaultPayloadConverter; @@ -48,20 +57,41 @@ async function wrapWithSpan( span.setStatus({ code: otel.SpanStatusCode.OK }); return ret; } catch (err: any) { - const isBenignErr = err instanceof ApplicationFailure && err.category === ApplicationFailureCategory.BENIGN; - if (acceptableErrors === undefined || !acceptableErrors(err)) { - const statusCode = isBenignErr ? otel.SpanStatusCode.UNSET : otel.SpanStatusCode.ERROR; - span.setStatus({ code: statusCode, message: (err as Error).message ?? String(err) }); - span.recordException(err); - } else { - span.setStatus({ code: otel.SpanStatusCode.OK }); - } + handleError(err, span, acceptableErrors); throw err; } finally { span.end(); } } +function wrapWithSpanSync( + span: otel.Span, + fn: (span: otel.Span) => T, + acceptableErrors?: (err: unknown) => boolean +): T { + try { + const ret = fn(span); + span.setStatus({ code: otel.SpanStatusCode.OK }); + return ret; + } catch (err: any) { + handleError(err, span, acceptableErrors); + throw err; + } finally { + span.end(); + } +} + +function handleError(err: any, span: otel.Span, acceptableErrors?: (err: unknown) => boolean): void { + const isBenignErr = err instanceof ApplicationFailure && err.category === ApplicationFailureCategory.BENIGN; + if (acceptableErrors === undefined || !acceptableErrors(err)) { + const statusCode = isBenignErr ? otel.SpanStatusCode.UNSET : otel.SpanStatusCode.ERROR; + span.setStatus({ code: statusCode, message: (err as Error).message ?? String(err) }); + span.recordException(err); + } else { + span.setStatus({ code: otel.SpanStatusCode.OK }); + } +} + export interface InstrumentOptions { tracer: otel.Tracer; spanName: string; @@ -70,6 +100,8 @@ export interface InstrumentOptions { acceptableErrors?: (err: unknown) => boolean; } +export type InstrumentOptionsSync = Omit, 'fn'> & { fn: (span: otel.Span) => T }; + /** * Wraps `fn` in a span which ends when function returns or throws */ @@ -87,3 +119,12 @@ export async function instrument({ } return await tracer.startActiveSpan(spanName, async (span) => await wrapWithSpan(span, fn, acceptableErrors)); } + +export function instrumentSync({ tracer, spanName, fn, context, acceptableErrors }: InstrumentOptionsSync): T { + if (context) { + return otel.context.with(context, () => { + return tracer.startActiveSpan(spanName, (span) => wrapWithSpanSync(span, fn, acceptableErrors)); + }); + } + return tracer.startActiveSpan(spanName, (span) => wrapWithSpanSync(span, fn, acceptableErrors)); +} diff --git a/packages/interceptors-opentelemetry/src/worker/index.ts b/packages/interceptors-opentelemetry/src/worker/index.ts index 506200bdb..24a889068 100644 --- a/packages/interceptors-opentelemetry/src/worker/index.ts +++ b/packages/interceptors-opentelemetry/src/worker/index.ts @@ -8,6 +8,7 @@ import type { ActivityOutboundCallsInterceptor, InjectedSink, GetLogAttributesInput, + GetMetricTagsInput, ActivityExecuteInput, } from '@temporalio/worker'; import { instrument, extractContextFromHeaders } from '../instrumentation'; @@ -23,7 +24,7 @@ export interface InterceptorOptions { * Wraps the operation in an opentelemetry Span and links it to a parent Span context if one is * provided in the Activity input headers. */ -export class OpenTelemetryActivityInboundInterceptor implements ActivityInboundCallsInterceptor { +export class OpenTelemetryActivityInboundInterceptor implements Required { protected readonly tracer: otel.Tracer; constructor( @@ -41,11 +42,11 @@ export class OpenTelemetryActivityInboundInterceptor implements ActivityInboundC } /** - * Intercepts calls to emit logs from an Activity. + * Intercepts calls to emit logs and metrics from an Activity. * - * Attach OpenTelemetry context tracing attributes to emitted log messages, if appropriate. + * Attach OpenTelemetry context tracing attributes to emitted log messages and metrics, if appropriate. */ -export class OpenTelemetryActivityOutboundInterceptor implements ActivityOutboundCallsInterceptor { +export class OpenTelemetryActivityOutboundInterceptor implements Required { constructor(protected readonly ctx: ActivityContext) {} public getLogAttributes( @@ -65,6 +66,24 @@ export class OpenTelemetryActivityOutboundInterceptor implements ActivityOutboun return next(input); } } + + public getMetricTags( + input: GetMetricTagsInput, + next: Next + ): GetMetricTagsInput { + const span = otel.trace.getSpan(otel.context.active()); + const spanContext = span?.spanContext(); + if (spanContext && otel.isSpanContextValid(spanContext)) { + return next({ + trace_id: spanContext.traceId, + span_id: spanContext.spanId, + trace_flags: `0${spanContext.traceFlags.toString(16)}`, + ...input, + }); + } else { + return next(input); + } + } } /** diff --git a/packages/interceptors-opentelemetry/src/workflow/definitions.ts b/packages/interceptors-opentelemetry/src/workflow/definitions.ts index 6c70c0264..651751c83 100644 --- a/packages/interceptors-opentelemetry/src/workflow/definitions.ts +++ b/packages/interceptors-opentelemetry/src/workflow/definitions.ts @@ -54,6 +54,36 @@ export enum SpanName { */ WORKFLOW_SIGNAL_WITH_START = 'SignalWithStartWorkflow', + /** + * Workflow is queried + */ + WORKFLOW_QUERY = 'QueryWorkflow', + + /** + * Workflow is updated + */ + WORKFLOW_UPDATE = 'UpdateWorkflow', + + /** + * Workflow is started with an update + */ + WORKFLOW_UPDATE_WITH_START = 'UpdateWithStartWorkflow', + + /** + * Workflow is terminated + */ + WORKFLOW_TERMINATE = 'TerminateWorkflow', + + /** + * Workflow is cancelled + */ + WORKFLOW_CANCEL = 'CancelWorkflow', + + /** + * Workflow is described + */ + WORKFLOW_DESCRIBE = 'DescribeWorkflow', + /** * Workflow run is executing */ @@ -74,6 +104,14 @@ export enum SpanName { * Workflow is continuing as new */ CONTINUE_AS_NEW = 'ContinueAsNew', + /** + * Workflow timer is started + */ + WORKFLOW_TIMER = 'StartTimer', + /** + * Nexus operation is started + */ + NEXUS_OPERATION_START = 'StartNexusOperation', } export const SPAN_DELIMITER = ':'; diff --git a/packages/interceptors-opentelemetry/src/workflow/index.ts b/packages/interceptors-opentelemetry/src/workflow/index.ts index b47f58545..efc9a1549 100644 --- a/packages/interceptors-opentelemetry/src/workflow/index.ts +++ b/packages/interceptors-opentelemetry/src/workflow/index.ts @@ -8,17 +8,32 @@ import type { ContinueAsNewInput, DisposeInput, GetLogAttributesInput, + GetMetricTagsInput, LocalActivityInput, Next, + QueryInput, SignalInput, SignalWorkflowInput, StartChildWorkflowExecutionInput, + UpdateInput, WorkflowExecuteInput, WorkflowInboundCallsInterceptor, WorkflowInternalsInterceptor, WorkflowOutboundCallsInterceptor, + TimerInput, + StartNexusOperationInput, + StartNexusOperationOutput, } from '@temporalio/workflow'; -import { instrument, extractContextFromHeaders, headersWithContext } from '../instrumentation'; +import { + instrument, + instrumentSync, + extractContextFromHeaders, + headersWithContext, + TIMER_DURATION_MS_ATTR_KEY, + NEXUS_SERVICE_ATTR_KEY, + NEXUS_OPERATION_ATTR_KEY, + NEXUS_ENDPOINT_ATTR_KEY, +} from '../instrumentation'; import { ContextManager } from './context-manager'; import { SpanName, SPAN_DELIMITER } from './definitions'; import { SpanExporter } from './span-exporter'; @@ -50,7 +65,7 @@ function getTracer(): otel.Tracer { * * `@temporalio/workflow` must be provided by host package in order to function. */ -export class OpenTelemetryInboundInterceptor implements WorkflowInboundCallsInterceptor { +export class OpenTelemetryInboundInterceptor implements Required { protected readonly tracer = getTracer(); public constructor() { @@ -92,6 +107,50 @@ export class OpenTelemetryInboundInterceptor implements WorkflowInboundCallsInte context, }); } + + public async handleUpdate( + input: UpdateInput, + next: Next + ): Promise { + if (!hasSdkFlag('OpenTelemetryInterceptorsInstrumentsAllMethods')) return next(input); + + const context = extractContextFromHeaders(input.headers); + + return await instrument({ + tracer: this.tracer, + spanName: `${SpanName.WORKFLOW_UPDATE}${SPAN_DELIMITER}${input.name}`, + fn: () => next(input), + context, + }); + } + + public validateUpdate(input: UpdateInput, next: Next): void { + if (!hasSdkFlag('OpenTelemetryInterceptorsInstrumentsAllMethods')) return next(input); + + const context = extractContextFromHeaders(input.headers); + instrumentSync({ + tracer: this.tracer, + spanName: `${SpanName.WORKFLOW_UPDATE}${SPAN_DELIMITER}${input.name}:validate`, + fn: () => next(input), + context, + }); + } + + public async handleQuery( + input: QueryInput, + next: Next + ): Promise { + if (!hasSdkFlag('OpenTelemetryInterceptorsInstrumentsAllMethods')) return next(input); + + const context = extractContextFromHeaders(input.headers); + + return await instrument({ + tracer: this.tracer, + spanName: `${SpanName.WORKFLOW_QUERY}${SPAN_DELIMITER}${input.queryName}`, + fn: () => next(input), + context, + }); + } } /** @@ -101,13 +160,31 @@ export class OpenTelemetryInboundInterceptor implements WorkflowInboundCallsInte * * `@temporalio/workflow` must be provided by host package in order to function. */ -export class OpenTelemetryOutboundInterceptor implements WorkflowOutboundCallsInterceptor { +export class OpenTelemetryOutboundInterceptor implements Required { protected readonly tracer = getTracer(); public constructor() { ensureWorkflowModuleLoaded(); } + public async startTimer( + input: TimerInput, + next: Next + ): Promise { + if (!hasSdkFlag('OpenTelemetryInterceptorsInstrumentsAllMethods')) return next(input); + + return await instrument({ + tracer: this.tracer, + spanName: input.options?.summary + ? `${SpanName.WORKFLOW_TIMER}${SPAN_DELIMITER}${input.options.summary}` + : SpanName.WORKFLOW_TIMER, + fn: async (span) => { + span.setAttribute(TIMER_DURATION_MS_ATTR_KEY, input.durationMs); + return await next(input); + }, + }); + } + public async scheduleActivity( input: ActivityInput, next: Next @@ -149,6 +226,24 @@ export class OpenTelemetryOutboundInterceptor implements WorkflowOutboundCallsIn }); } + public async startNexusOperation( + input: StartNexusOperationInput, + next: Next + ): Promise { + if (!hasSdkFlag('OpenTelemetryInterceptorsInstrumentsAllMethods')) return next(input); + + return await instrument({ + tracer: this.tracer, + spanName: `${SpanName.NEXUS_OPERATION_START}${SPAN_DELIMITER}${input.service}${SPAN_DELIMITER}${input.operation}`, + fn: async (span) => { + span.setAttribute(NEXUS_SERVICE_ATTR_KEY, input.service); + span.setAttribute(NEXUS_OPERATION_ATTR_KEY, input.operation); + span.setAttribute(NEXUS_ENDPOINT_ATTR_KEY, input.endpoint); + return await next(input); + }, + }); + } + public async startChildWorkflowExecution( input: StartChildWorkflowExecutionInput, next: Next @@ -225,6 +320,24 @@ export class OpenTelemetryOutboundInterceptor implements WorkflowOutboundCallsIn return next(input); } } + + public getMetricTags( + input: GetMetricTagsInput, + next: Next + ): GetMetricTagsInput { + const span = otel.trace.getSpan(otel.context.active()); + const spanContext = span?.spanContext(); + if (spanContext && otel.isSpanContextValid(spanContext)) { + return next({ + trace_id: spanContext.traceId, + span_id: spanContext.spanId, + trace_flags: `0${spanContext.traceFlags.toString(16)}`, + ...input, + }); + } else { + return next(input); + } + } } export class OpenTelemetryInternalsInterceptor implements WorkflowInternalsInterceptor { diff --git a/packages/workflow/src/flags.ts b/packages/workflow/src/flags.ts index 40c994c8b..051eb991c 100644 --- a/packages/workflow/src/flags.ts +++ b/packages/workflow/src/flags.ts @@ -76,6 +76,16 @@ export const SdkFlags = { * @since Introduced in 1.13.2 */ OpenTelemetryInterceporsAvoidsExtraYields: defineFlag(5, true, [isAtLeast({ major: 1, minor: 13, patch: 2 })]), + + /** + * In 1.13.3, all remaining interceptor methods were implemented in @temporalio/interceptors-opentelemetry + * including `handleUpdate`, `validateUpdate`, `handleQuery`, `startTimer`, and `startNexusOperation`. + * These add instrumentation and yield points that were not present in earlier versions. + * This flag gates these new interceptor methods to prevent NDE on replay of workflows from earlier versions. + * + * @since Introduced in 1.13.3 + */ + OpenTelemetryInterceptorsInstrumentsAllMethods: defineFlag(6, true, [isAtLeast({ major: 1, minor: 13, patch: 3 })]), } as const; function defineFlag(id: number, def: boolean, alternativeConditions?: AltConditionFn[]): SdkFlag { From 3544bfe621e87c4f85350b43bee6634ea59790fc Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Fri, 21 Nov 2025 11:29:54 -0500 Subject: [PATCH 02/12] add test to verify update with start behavior --- packages/test/src/test-otel.ts | 57 ++++++++++++++++++- packages/test/src/workflows/index.ts | 1 + .../test/src/workflows/update-start-otel.ts | 12 ++++ 3 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 packages/test/src/workflows/update-start-otel.ts diff --git a/packages/test/src/test-otel.ts b/packages/test/src/test-otel.ts index 93dcb2e9d..348f42655 100644 --- a/packages/test/src/test-otel.ts +++ b/packages/test/src/test-otel.ts @@ -12,7 +12,7 @@ import { BasicTracerProvider, InMemorySpanExporter, SimpleSpanProcessor } from ' import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions'; import test from 'ava'; import { v4 as uuid4 } from 'uuid'; -import { WorkflowClient } from '@temporalio/client'; +import { WorkflowClient, WithStartWorkflowOperation } from '@temporalio/client'; import { OpenTelemetryWorkflowClientInterceptor } from '@temporalio/interceptors-opentelemetry/lib/client'; import { OpenTelemetryWorkflowClientCallsInterceptor } from '@temporalio/interceptors-opentelemetry'; import { instrument } from '@temporalio/interceptors-opentelemetry/lib/instrumentation'; @@ -510,6 +510,61 @@ if (RUN_INTEGRATION_TESTS) { t.is(spans[1].status.message, 'benign'); t.is(spans[2].status.code, SpanStatusCode.OK); }); + + test('executeUpdateWithStart works correctly with OTEL interceptors', async (t) => { + const staticResource = new opentelemetry.resources.Resource({ + [SemanticResourceAttributes.SERVICE_NAME]: 'ts-test-otel-worker', + }); + const traceExporter: opentelemetry.tracing.SpanExporter = { + export(_spans, resultCallback) { + resultCallback({ code: ExportResultCode.SUCCESS }); + }, + async shutdown() {}, + }; + + const sinks: InjectedSinks = { + exporter: makeWorkflowExporter(traceExporter, staticResource), + }; + + const worker = await Worker.create({ + workflowBundle: await createTestWorkflowBundle({ + workflowsPath: require.resolve('./workflows'), + workflowInterceptorModules: [require.resolve('./workflows/otel-interceptors')], + }), + activities, + taskQueue: 'test-otel-update-start', + interceptors: { + client: { + workflow: [new OpenTelemetryWorkflowClientCallsInterceptor()], + }, + workflowModules: [require.resolve('./workflows/otel-interceptors')], + }, + sinks, + }); + + const client = new WorkflowClient(); + + const startWorkflowOperation = new WithStartWorkflowOperation(workflows.updateStartOtel, { + workflowId: uuid4(), + taskQueue: 'test-otel-update-start', + workflowIdConflictPolicy: 'FAIL', + }); + + const { updateResult, workflowResult } = await worker.runUntil(async () => { + const updateResult = await client.executeUpdateWithStart(workflows.otelUpdate, { + args: [true], + startWorkflowOperation, + }); + + const handle = await startWorkflowOperation.workflowHandle(); + const workflowResult = await handle.result(); + + return { updateResult, workflowResult }; + }); + + t.is(updateResult, true); + t.is(workflowResult, true); + }); } test('Can replay otel history from 1.11.3', async (t) => { diff --git a/packages/test/src/workflows/index.ts b/packages/test/src/workflows/index.ts index 0d8ef95ac..1ac8c4021 100644 --- a/packages/test/src/workflows/index.ts +++ b/packages/test/src/workflows/index.ts @@ -93,5 +93,6 @@ export * from './upsert-and-read-search-attributes'; export * from './wait-on-user'; export * from './workflow-cancellation-scenarios'; export * from './upsert-and-read-memo'; +export * from './update-start-otel'; export * from './updates-ordering'; export * from './wait-on-signal-then-activity'; diff --git a/packages/test/src/workflows/update-start-otel.ts b/packages/test/src/workflows/update-start-otel.ts new file mode 100644 index 000000000..db50f438f --- /dev/null +++ b/packages/test/src/workflows/update-start-otel.ts @@ -0,0 +1,12 @@ +import * as workflow from '@temporalio/workflow'; + +export const otelUpdate = workflow.defineUpdate('otelUpdate'); + +export async function updateStartOtel(): Promise { + let updateResult = false; + workflow.setHandler(otelUpdate, (value: boolean): boolean => { + updateResult = value; + return true; + }); + return updateResult; +} From a3745bab84a5e21cf728e1a3d9f6920b7c73de65 Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Fri, 21 Nov 2025 11:53:11 -0500 Subject: [PATCH 03/12] chore: update otel test to check for new spans --- packages/test/src/test-otel.ts | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/packages/test/src/test-otel.ts b/packages/test/src/test-otel.ts index 348f42655..bf6fa1295 100644 --- a/packages/test/src/test-otel.ts +++ b/packages/test/src/test-otel.ts @@ -251,7 +251,7 @@ if (RUN_INTEGRATION_TESTS) { resource: staticResource, traceExporter, }); - await otel.start(); + otel.start(); const sinks: InjectedSinks = { exporter: makeWorkflowExporter(traceExporter, staticResource), @@ -379,6 +379,19 @@ if (RUN_INTEGRATION_TESTS) { ); t.true(activityStartedSignalSpan !== undefined); + const timerSpan = spans.find( + ({ name, parentSpanId }) => + name === SpanName.WORKFLOW_TIMER && parentSpanId === parentExecuteSpan?.spanContext().spanId + ); + t.true(timerSpan !== undefined); + + const querySpan = spans.find( + ({ name, parentSpanId }) => + name === `${SpanName.WORKFLOW_QUERY}${SPAN_DELIMITER}step` && + parentSpanId === secondActivityExecuteSpan?.spanContext().spanId + ); + t.true(querySpan !== undefined); + t.deepEqual(new Set(spans.map((span) => span.spanContext().traceId)).size, 1); } finally { // Cleanup the runtime so that it doesn't interfere with other tests From b9e320c8e6659087c7334ea29e670f2a2079ec3b Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Fri, 21 Nov 2025 14:07:12 -0500 Subject: [PATCH 04/12] chore: add 1.13.2 replay test --- .../otel_smorgasbord_1_13_2.json | 786 ++++++++++++++++++ packages/test/src/test-otel.ts | 24 + 2 files changed, 810 insertions(+) create mode 100644 packages/test/history_files/otel_smorgasbord_1_13_2.json diff --git a/packages/test/history_files/otel_smorgasbord_1_13_2.json b/packages/test/history_files/otel_smorgasbord_1_13_2.json new file mode 100644 index 000000000..70334119d --- /dev/null +++ b/packages/test/history_files/otel_smorgasbord_1_13_2.json @@ -0,0 +1,786 @@ +{ + "events": [ + { + "eventId": "1", + "eventTime": "2025-11-21T19:00:01.803145Z", + "eventType": "EVENT_TYPE_WORKFLOW_EXECUTION_STARTED", + "taskId": "1081192", + "workflowExecutionStartedEventAttributes": { + "workflowType": { + "name": "smorgasbord" + }, + "taskQueue": { + "name": "test-otel", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "input": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "Mg==" + } + ] + }, + "workflowRunTimeout": "0s", + "workflowTaskTimeout": "10s", + "continuedExecutionRunId": "3d668831-7352-4f7c-ba76-8e7c7de380b5", + "initiator": "CONTINUE_AS_NEW_INITIATOR_WORKFLOW", + "originalExecutionRunId": "080bf2cb-bf94-483d-b2a7-e1251e3098af", + "firstExecutionRunId": "019aa7c9-9e5b-775a-a997-fec1585dd910", + "attempt": 1, + "prevAutoResetPoints": { + "points": [ + { + "runId": "019aa7c9-9e5b-775a-a997-fec1585dd910", + "firstWorkflowTaskCompletedId": "4", + "createTime": "2025-11-21T18:59:59.753854Z", + "expireTime": "2025-11-22T18:59:59.754061Z", + "resettable": true, + "buildId": "@temporalio/worker@1.13.2+ac7b57aeab0ae668597ad4cbec3fab06bdd17b2f350444b91b30378957c819ad" + } + ] + }, + "header": { + "fields": { + "_tracer-data": { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJ0cmFjZXBhcmVudCI6IjAwLTRhNWYzN2NiNjdlMjE1NWUyOGYxZjYzNTQ4NWEwMWQ2LWVlYzliMWQ0OGM5OWMwZGEtMDEifQ==" + } + } + }, + "workflowId": "3710ab85-dcbf-46f7-93b0-b7ce7dac0b1c" + } + }, + { + "eventId": "2", + "eventTime": "2025-11-21T19:00:01.803221Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "taskId": "1081193", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "test-otel", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "startToCloseTimeout": "10s", + "attempt": 1 + } + }, + { + "eventId": "3", + "eventTime": "2025-11-21T19:00:01.808162Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "taskId": "1081200", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "2", + "identity": "32020@mac.lan", + "requestId": "94944c1a-dbb6-4473-afca-222d59a3950e", + "historySizeBytes": "590", + "workerVersion": { + "buildId": "@temporalio/worker@1.13.2+ac7b57aeab0ae668597ad4cbec3fab06bdd17b2f350444b91b30378957c819ad" + } + } + }, + { + "eventId": "4", + "eventTime": "2025-11-21T19:00:01.825464Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "taskId": "1081204", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "2", + "startedEventId": "3", + "identity": "32020@mac.lan", + "workerVersion": { + "buildId": "@temporalio/worker@1.13.2+ac7b57aeab0ae668597ad4cbec3fab06bdd17b2f350444b91b30378957c819ad" + }, + "sdkMetadata": { + "coreUsedFlags": [1, 3, 2], + "langUsedFlags": [5, 4], + "sdkName": "temporal-typescript", + "sdkVersion": "1.13.2" + }, + "meteringMetadata": {} + } + }, + { + "eventId": "5", + "eventTime": "2025-11-21T19:00:01.825503Z", + "eventType": "EVENT_TYPE_ACTIVITY_TASK_SCHEDULED", + "taskId": "1081205", + "activityTaskScheduledEventAttributes": { + "activityId": "1", + "activityType": { + "name": "fakeProgress" + }, + "taskQueue": { + "name": "test-otel", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "header": { + "fields": { + "_tracer-data": { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJ0cmFjZXBhcmVudCI6IjAwLTRhNWYzN2NiNjdlMjE1NWUyOGYxZjYzNTQ4NWEwMWQ2LWJkMzNiZTU1ZDdiZjNjZjgtMDEifQ==" + } + } + }, + "input": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "MTAw" + }, + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "MTA=" + } + ] + }, + "scheduleToCloseTimeout": "0s", + "scheduleToStartTimeout": "0s", + "startToCloseTimeout": "60s", + "heartbeatTimeout": "0s", + "workflowTaskCompletedEventId": "4", + "retryPolicy": { + "initialInterval": "1s", + "backoffCoefficient": 2, + "maximumInterval": "100s" + }, + "useWorkflowBuildId": true + } + }, + { + "eventId": "6", + "eventTime": "2025-11-21T19:00:01.825521Z", + "eventType": "EVENT_TYPE_ACTIVITY_TASK_SCHEDULED", + "taskId": "1081206", + "activityTaskScheduledEventAttributes": { + "activityId": "2", + "activityType": { + "name": "queryOwnWf" + }, + "taskQueue": { + "name": "test-otel", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "header": { + "fields": { + "_tracer-data": { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJ0cmFjZXBhcmVudCI6IjAwLTRhNWYzN2NiNjdlMjE1NWUyOGYxZjYzNTQ4NWEwMWQ2LTA3YTIwNzJlYTdmODljMTItMDEifQ==" + } + } + }, + "input": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJ0eXBlIjoicXVlcnkiLCJuYW1lIjoic3RlcCJ9" + } + ] + }, + "scheduleToCloseTimeout": "0s", + "scheduleToStartTimeout": "0s", + "startToCloseTimeout": "60s", + "heartbeatTimeout": "0s", + "workflowTaskCompletedEventId": "4", + "retryPolicy": { + "initialInterval": "1s", + "backoffCoefficient": 2, + "maximumInterval": "100s" + }, + "useWorkflowBuildId": true + } + }, + { + "eventId": "7", + "eventTime": "2025-11-21T19:00:01.825528Z", + "eventType": "EVENT_TYPE_TIMER_STARTED", + "taskId": "1081207", + "timerStartedEventAttributes": { + "timerId": "1", + "startToFireTimeout": "1s", + "workflowTaskCompletedEventId": "4" + } + }, + { + "eventId": "8", + "eventTime": "2025-11-21T19:00:01.825710Z", + "eventType": "EVENT_TYPE_START_CHILD_WORKFLOW_EXECUTION_INITIATED", + "taskId": "1081208", + "startChildWorkflowExecutionInitiatedEventAttributes": { + "namespace": "default", + "workflowId": "3e74e356-25a4-4e0f-b477-eb83679117a1", + "workflowType": { + "name": "signalTarget" + }, + "taskQueue": { + "name": "test-otel", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "workflowRunTimeout": "0s", + "workflowTaskTimeout": "10s", + "parentClosePolicy": "PARENT_CLOSE_POLICY_TERMINATE", + "workflowTaskCompletedEventId": "4", + "workflowIdReusePolicy": "WORKFLOW_ID_REUSE_POLICY_ALLOW_DUPLICATE", + "header": { + "fields": { + "_tracer-data": { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJ0cmFjZXBhcmVudCI6IjAwLTRhNWYzN2NiNjdlMjE1NWUyOGYxZjYzNTQ4NWEwMWQ2LTFjYTY2MzQ0ZGJhYThhYTctMDEifQ==" + } + } + }, + "memo": { + "fields": {} + }, + "searchAttributes": { + "indexedFields": {} + }, + "namespaceId": "019a935a-2ac1-7c29-b43c-a2876bf2993f", + "inheritBuildId": true + } + }, + { + "eventId": "9", + "eventTime": "2025-11-21T19:00:01.825724Z", + "eventType": "EVENT_TYPE_MARKER_RECORDED", + "taskId": "1081209", + "markerRecordedEventAttributes": { + "details": { + "result": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "ImxvY2FsLWFjdGl2aXR5Ig==" + } + ] + }, + "data": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJzZXEiOjMsImF0dGVtcHQiOjEsImFjdGl2aXR5X2lkIjoiMyIsImFjdGl2aXR5X3R5cGUiOiJlY2hvIiwiY29tcGxldGVfdGltZSI6eyJzZWNvbmRzIjoxNzYzNzUxNjAxLCJuYW5vcyI6ODA5MDAzNzkyfSwiYmFja29mZiI6bnVsbCwib3JpZ2luYWxfc2NoZWR1bGVfdGltZSI6eyJzZWNvbmRzIjoxNzYzNzUxNjAxLCJuYW5vcyI6ODIxOTM4MDAwfX0=" + } + ] + } + }, + "markerName": "core_local_activity", + "workflowTaskCompletedEventId": "4" + } + }, + { + "eventId": "10", + "eventTime": "2025-11-21T19:00:01.827165Z", + "eventType": "EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_STARTED", + "taskId": "1081220", + "childWorkflowExecutionStartedEventAttributes": { + "namespace": "default", + "initiatedEventId": "8", + "workflowExecution": { + "workflowId": "3e74e356-25a4-4e0f-b477-eb83679117a1", + "runId": "019aa7c9-a6a2-772f-90b3-d51e43a30ea9" + }, + "workflowType": { + "name": "signalTarget" + }, + "header": { + "fields": { + "_tracer-data": { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJ0cmFjZXBhcmVudCI6IjAwLTRhNWYzN2NiNjdlMjE1NWUyOGYxZjYzNTQ4NWEwMWQ2LTFjYTY2MzQ0ZGJhYThhYTctMDEifQ==" + } + } + }, + "namespaceId": "019a935a-2ac1-7c29-b43c-a2876bf2993f" + } + }, + { + "eventId": "11", + "eventTime": "2025-11-21T19:00:01.827169Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "taskId": "1081221", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "32020@mac.lan-98c600bbf919422e9e73c902149dc629", + "kind": "TASK_QUEUE_KIND_STICKY", + "normalName": "test-otel" + }, + "startToCloseTimeout": "10s", + "attempt": 1 + } + }, + { + "eventId": "12", + "eventTime": "2025-11-21T19:00:01.827660Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "taskId": "1081229", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "11", + "identity": "32020@mac.lan", + "requestId": "29c691ee-a563-4b88-b078-9f63696dd791", + "historySizeBytes": "2560", + "workerVersion": { + "buildId": "@temporalio/worker@1.13.2+ac7b57aeab0ae668597ad4cbec3fab06bdd17b2f350444b91b30378957c819ad" + } + } + }, + { + "eventId": "13", + "eventTime": "2025-11-21T19:00:01.837926Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "taskId": "1081243", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "11", + "startedEventId": "12", + "identity": "32020@mac.lan", + "workerVersion": { + "buildId": "@temporalio/worker@1.13.2+ac7b57aeab0ae668597ad4cbec3fab06bdd17b2f350444b91b30378957c819ad" + }, + "sdkMetadata": {}, + "meteringMetadata": {} + } + }, + { + "eventId": "14", + "eventTime": "2025-11-21T19:00:01.837954Z", + "eventType": "EVENT_TYPE_SIGNAL_EXTERNAL_WORKFLOW_EXECUTION_INITIATED", + "taskId": "1081244", + "signalExternalWorkflowExecutionInitiatedEventAttributes": { + "workflowTaskCompletedEventId": "13", + "namespace": "default", + "workflowExecution": { + "workflowId": "3e74e356-25a4-4e0f-b477-eb83679117a1" + }, + "signalName": "unblock", + "childWorkflowOnly": true, + "header": { + "fields": { + "_tracer-data": { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJ0cmFjZXBhcmVudCI6IjAwLTRhNWYzN2NiNjdlMjE1NWUyOGYxZjYzNTQ4NWEwMWQ2LTcyMTYyMmVkMDI3ZjVkM2UtMDEifQ==" + } + } + }, + "namespaceId": "019a935a-2ac1-7c29-b43c-a2876bf2993f" + } + }, + { + "eventId": "15", + "eventTime": "2025-11-21T19:00:01.831344Z", + "eventType": "EVENT_TYPE_WORKFLOW_EXECUTION_SIGNALED", + "taskId": "1081245", + "workflowExecutionSignaledEventAttributes": { + "signalName": "activityStarted", + "input": {}, + "identity": "32020@mac.lan", + "header": { + "fields": { + "_tracer-data": { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "e30=" + } + } + } + } + }, + { + "eventId": "16", + "eventTime": "2025-11-21T19:00:01.837968Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "taskId": "1081246", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "32020@mac.lan-98c600bbf919422e9e73c902149dc629", + "kind": "TASK_QUEUE_KIND_STICKY", + "normalName": "test-otel" + }, + "startToCloseTimeout": "10s", + "attempt": 1 + } + }, + { + "eventId": "17", + "eventTime": "2025-11-21T19:00:01.837969Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "taskId": "1081247", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "16", + "identity": "32020@mac.lan", + "requestId": "request-from-RespondWorkflowTaskCompleted", + "historySizeBytes": "2740", + "workerVersion": { + "buildId": "@temporalio/worker@1.13.2+ac7b57aeab0ae668597ad4cbec3fab06bdd17b2f350444b91b30378957c819ad" + } + } + }, + { + "eventId": "18", + "eventTime": "2025-11-21T19:00:01.845193Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "taskId": "1081266", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "16", + "startedEventId": "17", + "identity": "32020@mac.lan", + "workerVersion": { + "buildId": "@temporalio/worker@1.13.2+ac7b57aeab0ae668597ad4cbec3fab06bdd17b2f350444b91b30378957c819ad" + }, + "sdkMetadata": { + "langUsedFlags": [3, 2] + }, + "meteringMetadata": {} + } + }, + { + "eventId": "19", + "eventTime": "2025-11-21T19:00:01.839843Z", + "eventType": "EVENT_TYPE_EXTERNAL_WORKFLOW_EXECUTION_SIGNALED", + "taskId": "1081267", + "externalWorkflowExecutionSignaledEventAttributes": { + "initiatedEventId": "14", + "namespace": "default", + "workflowExecution": { + "workflowId": "3e74e356-25a4-4e0f-b477-eb83679117a1" + }, + "namespaceId": "019a935a-2ac1-7c29-b43c-a2876bf2993f" + } + }, + { + "eventId": "20", + "eventTime": "2025-11-21T19:00:01.844480Z", + "eventType": "EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_COMPLETED", + "taskId": "1081268", + "childWorkflowExecutionCompletedEventAttributes": { + "result": { + "payloads": [ + { + "metadata": { + "encoding": "YmluYXJ5L251bGw=" + } + } + ] + }, + "namespace": "default", + "workflowExecution": { + "workflowId": "3e74e356-25a4-4e0f-b477-eb83679117a1", + "runId": "019aa7c9-a6a2-772f-90b3-d51e43a30ea9" + }, + "workflowType": { + "name": "signalTarget" + }, + "initiatedEventId": "8", + "startedEventId": "10", + "namespaceId": "019a935a-2ac1-7c29-b43c-a2876bf2993f" + } + }, + { + "eventId": "21", + "eventTime": "2025-11-21T19:00:01.845205Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "taskId": "1081269", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "32020@mac.lan-98c600bbf919422e9e73c902149dc629", + "kind": "TASK_QUEUE_KIND_STICKY", + "normalName": "test-otel" + }, + "startToCloseTimeout": "10s", + "attempt": 1 + } + }, + { + "eventId": "22", + "eventTime": "2025-11-21T19:00:01.845207Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "taskId": "1081270", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "21", + "identity": "32020@mac.lan", + "requestId": "request-from-RespondWorkflowTaskCompleted", + "historySizeBytes": "3521", + "workerVersion": { + "buildId": "@temporalio/worker@1.13.2+ac7b57aeab0ae668597ad4cbec3fab06bdd17b2f350444b91b30378957c819ad" + } + } + }, + { + "eventId": "23", + "eventTime": "2025-11-21T19:00:01.847849Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "taskId": "1081274", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "21", + "startedEventId": "22", + "identity": "32020@mac.lan", + "workerVersion": { + "buildId": "@temporalio/worker@1.13.2+ac7b57aeab0ae668597ad4cbec3fab06bdd17b2f350444b91b30378957c819ad" + }, + "sdkMetadata": {}, + "meteringMetadata": {} + } + }, + { + "eventId": "24", + "eventTime": "2025-11-21T19:00:01.828311Z", + "eventType": "EVENT_TYPE_ACTIVITY_TASK_STARTED", + "taskId": "1081275", + "activityTaskStartedEventAttributes": { + "scheduledEventId": "6", + "identity": "32020@mac.lan", + "requestId": "66e731c3-2834-47be-8f7e-ba139a9dfe56", + "attempt": 1, + "workerVersion": { + "buildId": "@temporalio/worker@1.13.2+ac7b57aeab0ae668597ad4cbec3fab06bdd17b2f350444b91b30378957c819ad" + } + } + }, + { + "eventId": "25", + "eventTime": "2025-11-21T19:00:01.846596Z", + "eventType": "EVENT_TYPE_ACTIVITY_TASK_COMPLETED", + "taskId": "1081276", + "activityTaskCompletedEventAttributes": { + "result": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "Mg==" + } + ] + }, + "scheduledEventId": "6", + "startedEventId": "24", + "identity": "32020@mac.lan" + } + }, + { + "eventId": "26", + "eventTime": "2025-11-21T19:00:01.847862Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "taskId": "1081277", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "32020@mac.lan-98c600bbf919422e9e73c902149dc629", + "kind": "TASK_QUEUE_KIND_STICKY", + "normalName": "test-otel" + }, + "startToCloseTimeout": "10s", + "attempt": 1 + } + }, + { + "eventId": "27", + "eventTime": "2025-11-21T19:00:01.847863Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "taskId": "1081278", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "26", + "identity": "32020@mac.lan", + "requestId": "request-from-RespondWorkflowTaskCompleted", + "historySizeBytes": "4268", + "workerVersion": { + "buildId": "@temporalio/worker@1.13.2+ac7b57aeab0ae668597ad4cbec3fab06bdd17b2f350444b91b30378957c819ad" + } + } + }, + { + "eventId": "28", + "eventTime": "2025-11-21T19:00:01.849788Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "taskId": "1081281", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "26", + "startedEventId": "27", + "identity": "32020@mac.lan", + "workerVersion": { + "buildId": "@temporalio/worker@1.13.2+ac7b57aeab0ae668597ad4cbec3fab06bdd17b2f350444b91b30378957c819ad" + }, + "sdkMetadata": {}, + "meteringMetadata": {} + } + }, + { + "eventId": "29", + "eventTime": "2025-11-21T19:00:02.827865Z", + "eventType": "EVENT_TYPE_TIMER_FIRED", + "taskId": "1081283", + "timerFiredEventAttributes": { + "timerId": "1", + "startedEventId": "7" + } + }, + { + "eventId": "30", + "eventTime": "2025-11-21T19:00:02.827881Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "taskId": "1081284", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "32020@mac.lan-98c600bbf919422e9e73c902149dc629", + "kind": "TASK_QUEUE_KIND_STICKY", + "normalName": "test-otel" + }, + "startToCloseTimeout": "10s", + "attempt": 1 + } + }, + { + "eventId": "31", + "eventTime": "2025-11-21T19:00:02.831703Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "taskId": "1081288", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "30", + "identity": "32020@mac.lan", + "requestId": "a52a6af0-fbd2-4d5c-b435-2d32ef3888ac", + "historySizeBytes": "5220", + "workerVersion": { + "buildId": "@temporalio/worker@1.13.2+ac7b57aeab0ae668597ad4cbec3fab06bdd17b2f350444b91b30378957c819ad" + } + } + }, + { + "eventId": "32", + "eventTime": "2025-11-21T19:00:02.844934Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "taskId": "1081292", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "30", + "startedEventId": "31", + "identity": "32020@mac.lan", + "workerVersion": { + "buildId": "@temporalio/worker@1.13.2+ac7b57aeab0ae668597ad4cbec3fab06bdd17b2f350444b91b30378957c819ad" + }, + "sdkMetadata": {}, + "meteringMetadata": {} + } + }, + { + "eventId": "33", + "eventTime": "2025-11-21T19:00:01.828550Z", + "eventType": "EVENT_TYPE_ACTIVITY_TASK_STARTED", + "taskId": "1081294", + "activityTaskStartedEventAttributes": { + "scheduledEventId": "5", + "identity": "32020@mac.lan", + "requestId": "c1d854bb-7445-4340-8289-4964108bcec2", + "attempt": 1, + "workerVersion": { + "buildId": "@temporalio/worker@1.13.2+ac7b57aeab0ae668597ad4cbec3fab06bdd17b2f350444b91b30378957c819ad" + } + } + }, + { + "eventId": "34", + "eventTime": "2025-11-21T19:00:02.847108Z", + "eventType": "EVENT_TYPE_ACTIVITY_TASK_COMPLETED", + "taskId": "1081295", + "activityTaskCompletedEventAttributes": { + "result": { + "payloads": [ + { + "metadata": { + "encoding": "YmluYXJ5L251bGw=" + } + } + ] + }, + "scheduledEventId": "5", + "startedEventId": "33", + "identity": "32020@mac.lan" + } + }, + { + "eventId": "35", + "eventTime": "2025-11-21T19:00:02.847118Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "taskId": "1081296", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "32020@mac.lan-98c600bbf919422e9e73c902149dc629", + "kind": "TASK_QUEUE_KIND_STICKY", + "normalName": "test-otel" + }, + "startToCloseTimeout": "10s", + "attempt": 1 + } + }, + { + "eventId": "36", + "eventTime": "2025-11-21T19:00:02.849311Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "taskId": "1081300", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "35", + "identity": "32020@mac.lan", + "requestId": "391bd0e5-41e7-466c-a054-f8b87b38651b", + "historySizeBytes": "5894", + "workerVersion": { + "buildId": "@temporalio/worker@1.13.2+ac7b57aeab0ae668597ad4cbec3fab06bdd17b2f350444b91b30378957c819ad" + } + } + }, + { + "eventId": "37", + "eventTime": "2025-11-21T19:00:02.860057Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "taskId": "1081304", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "35", + "startedEventId": "36", + "identity": "32020@mac.lan", + "workerVersion": { + "buildId": "@temporalio/worker@1.13.2+ac7b57aeab0ae668597ad4cbec3fab06bdd17b2f350444b91b30378957c819ad" + }, + "sdkMetadata": {}, + "meteringMetadata": {} + } + }, + { + "eventId": "38", + "eventTime": "2025-11-21T19:00:02.860101Z", + "eventType": "EVENT_TYPE_WORKFLOW_EXECUTION_COMPLETED", + "taskId": "1081305", + "workflowExecutionCompletedEventAttributes": { + "result": { + "payloads": [ + { + "metadata": { + "encoding": "YmluYXJ5L251bGw=" + } + } + ] + }, + "workflowTaskCompletedEventId": "37" + } + } + ] +} diff --git a/packages/test/src/test-otel.ts b/packages/test/src/test-otel.ts index bf6fa1295..02c072b21 100644 --- a/packages/test/src/test-otel.ts +++ b/packages/test/src/test-otel.ts @@ -672,3 +672,27 @@ test('Can replay signal workflow from 1.13.1', async (t) => { ); }); }); + +test('Can replay smorgasbord from 1.13.2', async (t) => { + const hist = await loadHistory('otel_smorgasbord_1_13_2.json'); + await t.notThrowsAsync(async () => { + await Worker.runReplayHistory( + { + workflowBundle: await createTestWorkflowBundle({ + workflowsPath: require.resolve('./workflows'), + workflowInterceptorModules: [require.resolve('./workflows/otel-interceptors')], + }), + interceptors: { + workflowModules: [require.resolve('./workflows/otel-interceptors')], + activity: [ + (ctx) => ({ + inbound: new OpenTelemetryActivityInboundInterceptor(ctx), + outbound: new OpenTelemetryActivityOutboundInterceptor(ctx), + }), + ], + }, + }, + hist + ); + }); +}); From 6ed1f58f723f472e7308d9fa5148762b94d38b16 Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Fri, 21 Nov 2025 15:41:07 -0500 Subject: [PATCH 05/12] PR cleanup, additional span test --- packages/interceptors-opentelemetry/src/client/index.ts | 4 +++- .../interceptors-opentelemetry/src/instrumentation.ts | 9 +++++++-- packages/test/src/test-otel.ts | 3 ++- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/packages/interceptors-opentelemetry/src/client/index.ts b/packages/interceptors-opentelemetry/src/client/index.ts index 6198ac57d..9017a9ca4 100644 --- a/packages/interceptors-opentelemetry/src/client/index.ts +++ b/packages/interceptors-opentelemetry/src/client/index.ts @@ -110,7 +110,9 @@ export class OpenTelemetryWorkflowClientInterceptor implements Required From b51d25ef31171aa20db3b3c18dab686ed3bbf36c Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Fri, 21 Nov 2025 16:28:50 -0500 Subject: [PATCH 06/12] remove unused attr --- packages/interceptors-opentelemetry/src/instrumentation.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/interceptors-opentelemetry/src/instrumentation.ts b/packages/interceptors-opentelemetry/src/instrumentation.ts index 7700d622f..35c85e8e2 100644 --- a/packages/interceptors-opentelemetry/src/instrumentation.ts +++ b/packages/interceptors-opentelemetry/src/instrumentation.ts @@ -16,8 +16,6 @@ export const TRACE_HEADER = '_tracer-data'; export const RUN_ID_ATTR_KEY = 'run_id'; /** As in workflow id */ export const WORKFLOW_ID_ATTR_KEY = 'workflow_id'; -/** As in if workflow was eagererly started */ -export const EAGER_START_ATTR_KEY = 'eager_start'; /** As in termination reason */ export const TERMINATE_REASON_ATTR_KEY = 'terminate_reason'; /** As in timer duration */ From 16add4745347725e1b622eb9ad571ee37d36af7d Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Wed, 26 Nov 2025 17:09:22 -0600 Subject: [PATCH 07/12] match ruby otel --- .../src/client/index.ts | 25 ++++++++++++++----- .../src/instrumentation.ts | 8 ++++-- .../src/worker/index.ts | 20 +++++++++++++-- .../src/workflow/definitions.ts | 24 ++++++++++++++++-- .../src/workflow/index.ts | 19 +++++++++----- 5 files changed, 78 insertions(+), 18 deletions(-) diff --git a/packages/interceptors-opentelemetry/src/client/index.ts b/packages/interceptors-opentelemetry/src/client/index.ts index 9017a9ca4..f53659869 100644 --- a/packages/interceptors-opentelemetry/src/client/index.ts +++ b/packages/interceptors-opentelemetry/src/client/index.ts @@ -23,6 +23,7 @@ import { headersWithContext, RUN_ID_ATTR_KEY, WORKFLOW_ID_ATTR_KEY, + UPDATE_ID_ATTR_KEY, TERMINATE_REASON_ATTR_KEY, } from '../instrumentation'; import { SpanName, SPAN_DELIMITER } from '../workflow'; @@ -48,7 +49,8 @@ export class OpenTelemetryWorkflowClientInterceptor implements Required { - const headers = await headersWithContext(input.headers); + const headers = headersWithContext(input.headers); + span.setAttribute(WORKFLOW_ID_ATTR_KEY, input.options.workflowId); const runId = await next({ ...input, headers }); span.setAttribute(RUN_ID_ATTR_KEY, runId); return runId; @@ -60,8 +62,9 @@ export class OpenTelemetryWorkflowClientInterceptor implements Required { - const headers = await headersWithContext(input.headers); + fn: async (span) => { + span.setAttribute(WORKFLOW_ID_ATTR_KEY, input.workflowExecution.workflowId); + const headers = headersWithContext(input.headers); await next({ ...input, headers }); }, }); @@ -76,6 +79,7 @@ export class OpenTelemetryWorkflowClientInterceptor implements Required { const headers = headersWithContext(input.headers); + span.setAttribute(WORKFLOW_ID_ATTR_KEY, input.options.workflowId); const output = await next({ ...input, headers }); span.setAttribute(RUN_ID_ATTR_KEY, output.runId); return output; @@ -89,8 +93,12 @@ export class OpenTelemetryWorkflowClientInterceptor implements Required { return await instrument({ tracer: this.tracer, - spanName: `${SpanName.WORKFLOW_UPDATE}${SPAN_DELIMITER}${input.updateName}`, + spanName: `${SpanName.WORKFLOW_START_UPDATE}${SPAN_DELIMITER}${input.updateName}`, fn: async (span) => { + span.setAttribute(WORKFLOW_ID_ATTR_KEY, input.workflowExecution.workflowId); + if (input.options.updateId) { + span.setAttribute(UPDATE_ID_ATTR_KEY, input.options.updateId); + } const headers = headersWithContext(input.headers); const output = await next({ ...input, headers }); span.setAttribute(RUN_ID_ATTR_KEY, output.workflowRunId); @@ -105,8 +113,12 @@ export class OpenTelemetryWorkflowClientInterceptor implements Required { return await instrument({ tracer: this.tracer, - spanName: `${SpanName.WORKFLOW_UPDATE_WITH_START}${SPAN_DELIMITER}${input.workflowType}${SPAN_DELIMITER}${input.updateName}`, + spanName: `${SpanName.WORKFLOW_UPDATE_WITH_START}${SPAN_DELIMITER}${input.updateName}`, fn: async (span) => { + span.setAttribute(WORKFLOW_ID_ATTR_KEY, input.workflowStartOptions.workflowId); + if (input.updateOptions.updateId) { + span.setAttribute(UPDATE_ID_ATTR_KEY, input.updateOptions.updateId); + } const workflowStartHeaders = headersWithContext(input.workflowStartHeaders); const updateHeaders = headersWithContext(input.updateHeaders); const output = await next({ ...input, workflowStartHeaders, updateHeaders }); @@ -124,8 +136,9 @@ export class OpenTelemetryWorkflowClientInterceptor implements Required { return await instrument({ tracer: this.tracer, - spanName: `${SpanName.WORKFLOW_SIGNAL_WITH_START}${SPAN_DELIMITER}${input.workflowType}${SPAN_DELIMITER}${input.signalName}`, + spanName: `${SpanName.WORKFLOW_SIGNAL_WITH_START}${SPAN_DELIMITER}${input.workflowType}`, fn: async (span) => { + span.setAttribute(WORKFLOW_ID_ATTR_KEY, input.options.workflowId); const headers = headersWithContext(input.headers); const runId = await next({ ...input, headers }); span.setAttribute(RUN_ID_ATTR_KEY, runId); diff --git a/packages/interceptors-opentelemetry/src/instrumentation.ts b/packages/interceptors-opentelemetry/src/instrumentation.ts index 35c85e8e2..a46b31a9e 100644 --- a/packages/interceptors-opentelemetry/src/instrumentation.ts +++ b/packages/interceptors-opentelemetry/src/instrumentation.ts @@ -13,9 +13,13 @@ import { /** Default trace header for opentelemetry interceptors */ export const TRACE_HEADER = '_tracer-data'; /** As in workflow run id */ -export const RUN_ID_ATTR_KEY = 'run_id'; +export const RUN_ID_ATTR_KEY = 'temporalRunID'; /** As in workflow id */ -export const WORKFLOW_ID_ATTR_KEY = 'workflow_id'; +export const WORKFLOW_ID_ATTR_KEY = 'temporalWorkflowID'; +/** As in activity id */ +export const ACTIVITY_ID_ATTR_KEY = 'temporalActivityID'; +/** As in update id */ +export const UPDATE_ID_ATTR_KEY = 'temporalUpdateID'; /** As in termination reason */ export const TERMINATE_REASON_ATTR_KEY = 'terminate_reason'; /** As in timer duration */ diff --git a/packages/interceptors-opentelemetry/src/worker/index.ts b/packages/interceptors-opentelemetry/src/worker/index.ts index 24a889068..2ad0a5a3b 100644 --- a/packages/interceptors-opentelemetry/src/worker/index.ts +++ b/packages/interceptors-opentelemetry/src/worker/index.ts @@ -11,7 +11,13 @@ import type { GetMetricTagsInput, ActivityExecuteInput, } from '@temporalio/worker'; -import { instrument, extractContextFromHeaders } from '../instrumentation'; +import { + instrument, + extractContextFromHeaders, + WORKFLOW_ID_ATTR_KEY, + RUN_ID_ATTR_KEY, + ACTIVITY_ID_ATTR_KEY, +} from '../instrumentation'; import { type OpenTelemetryWorkflowExporter, type SerializableSpan, SpanName, SPAN_DELIMITER } from '../workflow'; export interface InterceptorOptions { @@ -37,7 +43,17 @@ export class OpenTelemetryActivityInboundInterceptor implements Required): Promise { const context = extractContextFromHeaders(input.headers); const spanName = `${SpanName.ACTIVITY_EXECUTE}${SPAN_DELIMITER}${this.ctx.info.activityType}`; - return await instrument({ tracer: this.tracer, spanName, fn: () => next(input), context }); + return await instrument({ + tracer: this.tracer, + spanName, + fn: (span) => { + span.setAttribute(WORKFLOW_ID_ATTR_KEY, this.ctx.info.workflowExecution.workflowId); + span.setAttribute(RUN_ID_ATTR_KEY, this.ctx.info.workflowExecution.runId); + span.setAttribute(ACTIVITY_ID_ATTR_KEY, this.ctx.info.activityId); + return next(input); + }, + context, + }); } } diff --git a/packages/interceptors-opentelemetry/src/workflow/definitions.ts b/packages/interceptors-opentelemetry/src/workflow/definitions.ts index 651751c83..4e7083c91 100644 --- a/packages/interceptors-opentelemetry/src/workflow/definitions.ts +++ b/packages/interceptors-opentelemetry/src/workflow/definitions.ts @@ -60,15 +60,35 @@ export enum SpanName { WORKFLOW_QUERY = 'QueryWorkflow', /** - * Workflow is updated + * Workflow update is started by client */ - WORKFLOW_UPDATE = 'UpdateWorkflow', + WORKFLOW_START_UPDATE = 'StartWorkflowUpdate', /** * Workflow is started with an update */ WORKFLOW_UPDATE_WITH_START = 'UpdateWithStartWorkflow', + /** + * Workflow handles an incoming signal + */ + WORKFLOW_HANDLE_SIGNAL = 'HandleSignal', + + /** + * Workflow handles an incoming query + */ + WORKFLOW_HANDLE_QUERY = 'HandleQuery', + + /** + * Workflow handles an incoming update + */ + WORKFLOW_HANDLE_UPDATE = 'HandleUpdate', + + /** + * Workflow validates an incoming update + */ + WORKFLOW_VALIDATE_UPDATE = 'ValidateUpdate', + /** * Workflow is terminated */ diff --git a/packages/interceptors-opentelemetry/src/workflow/index.ts b/packages/interceptors-opentelemetry/src/workflow/index.ts index efc9a1549..676e39166 100644 --- a/packages/interceptors-opentelemetry/src/workflow/index.ts +++ b/packages/interceptors-opentelemetry/src/workflow/index.ts @@ -30,6 +30,7 @@ import { extractContextFromHeaders, headersWithContext, TIMER_DURATION_MS_ATTR_KEY, + UPDATE_ID_ATTR_KEY, NEXUS_SERVICE_ATTR_KEY, NEXUS_OPERATION_ATTR_KEY, NEXUS_ENDPOINT_ATTR_KEY, @@ -102,7 +103,7 @@ export class OpenTelemetryInboundInterceptor implements Required next(input), context, }); @@ -118,8 +119,11 @@ export class OpenTelemetryInboundInterceptor implements Required next(input), + spanName: `${SpanName.WORKFLOW_HANDLE_UPDATE}${SPAN_DELIMITER}${input.name}`, + fn: (span) => { + span.setAttribute(UPDATE_ID_ATTR_KEY, input.updateId); + return next(input); + }, context, }); } @@ -130,8 +134,11 @@ export class OpenTelemetryInboundInterceptor implements Required next(input), + spanName: `${SpanName.WORKFLOW_VALIDATE_UPDATE}${SPAN_DELIMITER}${input.name}`, + fn: (span) => { + span.setAttribute(UPDATE_ID_ATTR_KEY, input.updateId); + return next(input); + }, context, }); } @@ -146,7 +153,7 @@ export class OpenTelemetryInboundInterceptor implements Required next(input), context, }); From 1b86c7fb1f37f00d37739d290046eda8f00c8a48 Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Mon, 1 Dec 2025 08:58:38 -0500 Subject: [PATCH 08/12] use consistent casing for attributes --- .../interceptors-opentelemetry/src/instrumentation.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/interceptors-opentelemetry/src/instrumentation.ts b/packages/interceptors-opentelemetry/src/instrumentation.ts index a46b31a9e..818aad34d 100644 --- a/packages/interceptors-opentelemetry/src/instrumentation.ts +++ b/packages/interceptors-opentelemetry/src/instrumentation.ts @@ -13,13 +13,13 @@ import { /** Default trace header for opentelemetry interceptors */ export const TRACE_HEADER = '_tracer-data'; /** As in workflow run id */ -export const RUN_ID_ATTR_KEY = 'temporalRunID'; +export const RUN_ID_ATTR_KEY = 'run_id'; /** As in workflow id */ -export const WORKFLOW_ID_ATTR_KEY = 'temporalWorkflowID'; +export const WORKFLOW_ID_ATTR_KEY = 'temporal_workflow_id'; /** As in activity id */ -export const ACTIVITY_ID_ATTR_KEY = 'temporalActivityID'; +export const ACTIVITY_ID_ATTR_KEY = 'temporal_activity_id'; /** As in update id */ -export const UPDATE_ID_ATTR_KEY = 'temporalUpdateID'; +export const UPDATE_ID_ATTR_KEY = 'temporal_update_id'; /** As in termination reason */ export const TERMINATE_REASON_ATTR_KEY = 'terminate_reason'; /** As in timer duration */ From dc415c2f92ed3bf70d28fe369e9fc348bb97187e Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Tue, 2 Dec 2025 11:26:42 -0500 Subject: [PATCH 09/12] remove alt condition --- packages/workflow/src/flags.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/workflow/src/flags.ts b/packages/workflow/src/flags.ts index 051eb991c..117b987f1 100644 --- a/packages/workflow/src/flags.ts +++ b/packages/workflow/src/flags.ts @@ -85,7 +85,7 @@ export const SdkFlags = { * * @since Introduced in 1.13.3 */ - OpenTelemetryInterceptorsInstrumentsAllMethods: defineFlag(6, true, [isAtLeast({ major: 1, minor: 13, patch: 3 })]), + OpenTelemetryInterceptorsInstrumentsAllMethods: defineFlag(6, true), } as const; function defineFlag(id: number, def: boolean, alternativeConditions?: AltConditionFn[]): SdkFlag { From 788f63775e4fbfd86dceb60bd798f549211b4d5e Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Tue, 2 Dec 2025 15:17:44 -0500 Subject: [PATCH 10/12] remove Required from impl instead using satisfies --- .../src/client/index.ts | 2 +- .../src/instrumentation.ts | 2 - .../src/worker/index.ts | 4 +- .../src/workflow/definitions.ts | 4 -- .../src/workflow/index.ts | 24 +--------- packages/test/src/test-otel.ts | 44 ++++++++++++++----- 6 files changed, 38 insertions(+), 42 deletions(-) diff --git a/packages/interceptors-opentelemetry/src/client/index.ts b/packages/interceptors-opentelemetry/src/client/index.ts index f53659869..87a9c77a4 100644 --- a/packages/interceptors-opentelemetry/src/client/index.ts +++ b/packages/interceptors-opentelemetry/src/client/index.ts @@ -37,7 +37,7 @@ export interface InterceptorOptions { * * Wraps the operation in an opentelemetry Span and passes it to the Workflow via headers. */ -export class OpenTelemetryWorkflowClientInterceptor implements Required { +export class OpenTelemetryWorkflowClientInterceptor implements WorkflowClientInterceptor { protected readonly tracer: otel.Tracer; constructor(options?: InterceptorOptions) { diff --git a/packages/interceptors-opentelemetry/src/instrumentation.ts b/packages/interceptors-opentelemetry/src/instrumentation.ts index 818aad34d..b5bb60bee 100644 --- a/packages/interceptors-opentelemetry/src/instrumentation.ts +++ b/packages/interceptors-opentelemetry/src/instrumentation.ts @@ -22,8 +22,6 @@ export const ACTIVITY_ID_ATTR_KEY = 'temporal_activity_id'; export const UPDATE_ID_ATTR_KEY = 'temporal_update_id'; /** As in termination reason */ export const TERMINATE_REASON_ATTR_KEY = 'terminate_reason'; -/** As in timer duration */ -export const TIMER_DURATION_MS_ATTR_KEY = 'timer_duration_ms'; /** As in Nexus service */ export const NEXUS_SERVICE_ATTR_KEY = 'nexus_service'; /** As in Nexus operation */ diff --git a/packages/interceptors-opentelemetry/src/worker/index.ts b/packages/interceptors-opentelemetry/src/worker/index.ts index 2ad0a5a3b..f865f5008 100644 --- a/packages/interceptors-opentelemetry/src/worker/index.ts +++ b/packages/interceptors-opentelemetry/src/worker/index.ts @@ -30,7 +30,7 @@ export interface InterceptorOptions { * Wraps the operation in an opentelemetry Span and links it to a parent Span context if one is * provided in the Activity input headers. */ -export class OpenTelemetryActivityInboundInterceptor implements Required { +export class OpenTelemetryActivityInboundInterceptor implements ActivityInboundCallsInterceptor { protected readonly tracer: otel.Tracer; constructor( @@ -62,7 +62,7 @@ export class OpenTelemetryActivityInboundInterceptor implements Required { +export class OpenTelemetryActivityOutboundInterceptor implements ActivityOutboundCallsInterceptor { constructor(protected readonly ctx: ActivityContext) {} public getLogAttributes( diff --git a/packages/interceptors-opentelemetry/src/workflow/definitions.ts b/packages/interceptors-opentelemetry/src/workflow/definitions.ts index 4e7083c91..8b6dc7ba7 100644 --- a/packages/interceptors-opentelemetry/src/workflow/definitions.ts +++ b/packages/interceptors-opentelemetry/src/workflow/definitions.ts @@ -124,10 +124,6 @@ export enum SpanName { * Workflow is continuing as new */ CONTINUE_AS_NEW = 'ContinueAsNew', - /** - * Workflow timer is started - */ - WORKFLOW_TIMER = 'StartTimer', /** * Nexus operation is started */ diff --git a/packages/interceptors-opentelemetry/src/workflow/index.ts b/packages/interceptors-opentelemetry/src/workflow/index.ts index 676e39166..113d08d12 100644 --- a/packages/interceptors-opentelemetry/src/workflow/index.ts +++ b/packages/interceptors-opentelemetry/src/workflow/index.ts @@ -20,7 +20,6 @@ import type { WorkflowInboundCallsInterceptor, WorkflowInternalsInterceptor, WorkflowOutboundCallsInterceptor, - TimerInput, StartNexusOperationInput, StartNexusOperationOutput, } from '@temporalio/workflow'; @@ -29,7 +28,6 @@ import { instrumentSync, extractContextFromHeaders, headersWithContext, - TIMER_DURATION_MS_ATTR_KEY, UPDATE_ID_ATTR_KEY, NEXUS_SERVICE_ATTR_KEY, NEXUS_OPERATION_ATTR_KEY, @@ -66,7 +64,7 @@ function getTracer(): otel.Tracer { * * `@temporalio/workflow` must be provided by host package in order to function. */ -export class OpenTelemetryInboundInterceptor implements Required { +export class OpenTelemetryInboundInterceptor implements WorkflowInboundCallsInterceptor { protected readonly tracer = getTracer(); public constructor() { @@ -167,31 +165,13 @@ export class OpenTelemetryInboundInterceptor implements Required { +export class OpenTelemetryOutboundInterceptor implements WorkflowOutboundCallsInterceptor { protected readonly tracer = getTracer(); public constructor() { ensureWorkflowModuleLoaded(); } - public async startTimer( - input: TimerInput, - next: Next - ): Promise { - if (!hasSdkFlag('OpenTelemetryInterceptorsInstrumentsAllMethods')) return next(input); - - return await instrument({ - tracer: this.tracer, - spanName: input.options?.summary - ? `${SpanName.WORKFLOW_TIMER}${SPAN_DELIMITER}${input.options.summary}` - : SpanName.WORKFLOW_TIMER, - fn: async (span) => { - span.setAttribute(TIMER_DURATION_MS_ATTR_KEY, input.durationMs); - return await next(input); - }, - }); - } - public async scheduleActivity( input: ActivityInput, next: Next diff --git a/packages/test/src/test-otel.ts b/packages/test/src/test-otel.ts index a6b88bc44..97b29e074 100644 --- a/packages/test/src/test-otel.ts +++ b/packages/test/src/test-otel.ts @@ -12,17 +12,30 @@ import { BasicTracerProvider, InMemorySpanExporter, SimpleSpanProcessor } from ' import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions'; import test from 'ava'; import { v4 as uuid4 } from 'uuid'; -import { WorkflowClient, WithStartWorkflowOperation } from '@temporalio/client'; +import { WorkflowClient, WithStartWorkflowOperation, WorkflowClientInterceptor } from '@temporalio/client'; import { OpenTelemetryWorkflowClientInterceptor } from '@temporalio/interceptors-opentelemetry/lib/client'; import { OpenTelemetryWorkflowClientCallsInterceptor } from '@temporalio/interceptors-opentelemetry'; -import { instrument, TIMER_DURATION_MS_ATTR_KEY } from '@temporalio/interceptors-opentelemetry/lib/instrumentation'; +import { instrument } from '@temporalio/interceptors-opentelemetry/lib/instrumentation'; import { makeWorkflowExporter, OpenTelemetryActivityInboundInterceptor, OpenTelemetryActivityOutboundInterceptor, } from '@temporalio/interceptors-opentelemetry/lib/worker'; -import { OpenTelemetrySinks, SpanName, SPAN_DELIMITER } from '@temporalio/interceptors-opentelemetry/lib/workflow'; -import { DefaultLogger, InjectedSinks, Runtime } from '@temporalio/worker'; +import { + OpenTelemetrySinks, + SpanName, + SPAN_DELIMITER, + OpenTelemetryOutboundInterceptor, + OpenTelemetryInboundInterceptor, +} from '@temporalio/interceptors-opentelemetry/lib/workflow'; +import { + ActivityInboundCallsInterceptor, + ActivityOutboundCallsInterceptor, + DefaultLogger, + InjectedSinks, + Runtime, +} from '@temporalio/worker'; +import { WorkflowInboundCallsInterceptor, WorkflowOutboundCallsInterceptor } from '@temporalio/workflow'; import * as activities from './activities'; import { loadHistory, RUN_INTEGRATION_TESTS, TestWorkflowEnvironment, Worker } from './helpers'; import * as workflows from './workflows'; @@ -379,13 +392,6 @@ if (RUN_INTEGRATION_TESTS) { ); t.true(activityStartedSignalSpan !== undefined); - const timerSpan = spans.find( - ({ name, parentSpanId }) => - name === SpanName.WORKFLOW_TIMER && parentSpanId === parentExecuteSpan?.spanContext().spanId - ); - t.true(timerSpan !== undefined); - t.deepEqual(timerSpan!.attributes[TIMER_DURATION_MS_ATTR_KEY], 1000); - const querySpan = spans.find( ({ name, parentSpanId }) => name === `${SpanName.WORKFLOW_QUERY}${SPAN_DELIMITER}step` && @@ -697,3 +703,19 @@ test('Can replay smorgasbord from 1.13.2', async (t) => { ); }); }); + +// Skipped as we only care that it compiles +test.skip('otel interceptors are complete', async (t) => { + // We only use this to verify that we trace all spans via typechecking + // Doing this instead of directly changing the `implements` to avoid leaking this in the docs + const _wfl_inbound = {} as OpenTelemetryInboundInterceptor satisfies Required; + const _wfl_outbound = {} as OpenTelemetryOutboundInterceptor satisfies Required< + Omit + >; + const _act_inbound = + {} as OpenTelemetryActivityInboundInterceptor satisfies Required; + const _act_outbound = + {} as OpenTelemetryActivityOutboundInterceptor satisfies Required; + const _client = {} as OpenTelemetryWorkflowClientInterceptor satisfies Required; + t.pass(); +}); From af26de100102f94b07cc157ff00ad81dedf4ad7a Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Tue, 2 Dec 2025 15:28:33 -0500 Subject: [PATCH 11/12] match other SDK otel tags --- .../src/instrumentation.ts | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/interceptors-opentelemetry/src/instrumentation.ts b/packages/interceptors-opentelemetry/src/instrumentation.ts index b5bb60bee..d0c69b376 100644 --- a/packages/interceptors-opentelemetry/src/instrumentation.ts +++ b/packages/interceptors-opentelemetry/src/instrumentation.ts @@ -15,19 +15,19 @@ export const TRACE_HEADER = '_tracer-data'; /** As in workflow run id */ export const RUN_ID_ATTR_KEY = 'run_id'; /** As in workflow id */ -export const WORKFLOW_ID_ATTR_KEY = 'temporal_workflow_id'; +export const WORKFLOW_ID_ATTR_KEY = 'temporalWorkflowId'; /** As in activity id */ -export const ACTIVITY_ID_ATTR_KEY = 'temporal_activity_id'; +export const ACTIVITY_ID_ATTR_KEY = 'temporalActivityId'; /** As in update id */ -export const UPDATE_ID_ATTR_KEY = 'temporal_update_id'; +export const UPDATE_ID_ATTR_KEY = 'temporalUpdateId'; /** As in termination reason */ -export const TERMINATE_REASON_ATTR_KEY = 'terminate_reason'; +export const TERMINATE_REASON_ATTR_KEY = 'temporalTerminateReason'; /** As in Nexus service */ -export const NEXUS_SERVICE_ATTR_KEY = 'nexus_service'; +export const NEXUS_SERVICE_ATTR_KEY = 'temporalNexusService'; /** As in Nexus operation */ -export const NEXUS_OPERATION_ATTR_KEY = 'nexus_operation'; +export const NEXUS_OPERATION_ATTR_KEY = 'temporalNexusOperation'; /** As in Nexus endpoint */ -export const NEXUS_ENDPOINT_ATTR_KEY = 'nexus_endpoint'; +export const NEXUS_ENDPOINT_ATTR_KEY = 'temporalNexusEndpoint'; const payloadConverter = defaultPayloadConverter; From 677f95e090397c8393ca6bc0683a1142b645e2b9 Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Tue, 2 Dec 2025 15:37:14 -0500 Subject: [PATCH 12/12] rename handleError to more descriptive name --- packages/interceptors-opentelemetry/src/instrumentation.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/interceptors-opentelemetry/src/instrumentation.ts b/packages/interceptors-opentelemetry/src/instrumentation.ts index d0c69b376..f8d89a73f 100644 --- a/packages/interceptors-opentelemetry/src/instrumentation.ts +++ b/packages/interceptors-opentelemetry/src/instrumentation.ts @@ -62,7 +62,7 @@ async function wrapWithSpan( span.setStatus({ code: otel.SpanStatusCode.OK }); return ret; } catch (err: any) { - handleError(err, span, acceptableErrors); + maybeAddErrorToSpan(err, span, acceptableErrors); throw err; } finally { span.end(); @@ -79,14 +79,14 @@ function wrapWithSpanSync( span.setStatus({ code: otel.SpanStatusCode.OK }); return ret; } catch (err: any) { - handleError(err, span, acceptableErrors); + maybeAddErrorToSpan(err, span, acceptableErrors); throw err; } finally { span.end(); } } -function handleError(err: any, span: otel.Span, acceptableErrors?: (err: unknown) => boolean): void { +function maybeAddErrorToSpan(err: any, span: otel.Span, acceptableErrors?: (err: unknown) => boolean): void { const isBenignErr = err instanceof ApplicationFailure && err.category === ApplicationFailureCategory.BENIGN; if (acceptableErrors === undefined || !acceptableErrors(err)) { const statusCode = isBenignErr ? otel.SpanStatusCode.UNSET : otel.SpanStatusCode.ERROR;