diff --git a/packages/interceptors-opentelemetry/src/client/index.ts b/packages/interceptors-opentelemetry/src/client/index.ts index b427236bb..87a9c77a4 100644 --- a/packages/interceptors-opentelemetry/src/client/index.ts +++ b/packages/interceptors-opentelemetry/src/client/index.ts @@ -1,6 +1,31 @@ import * as otel from '@opentelemetry/api'; -import type { Next, WorkflowSignalInput, WorkflowStartInput, WorkflowClientInterceptor } from '@temporalio/client'; -import { instrument, headersWithContext, RUN_ID_ATTR_KEY } from '../instrumentation'; +import type { + Next, + WorkflowSignalInput, + WorkflowSignalWithStartInput, + WorkflowStartInput, + WorkflowStartOutput, + WorkflowStartUpdateInput, + WorkflowStartUpdateOutput, + WorkflowStartUpdateWithStartInput, + WorkflowStartUpdateWithStartOutput, + WorkflowQueryInput, + WorkflowTerminateInput, + WorkflowCancelInput, + WorkflowDescribeInput, + WorkflowClientInterceptor, + TerminateWorkflowExecutionResponse, + RequestCancelWorkflowExecutionResponse, + DescribeWorkflowExecutionResponse, +} from '@temporalio/client'; +import { + instrument, + headersWithContext, + RUN_ID_ATTR_KEY, + WORKFLOW_ID_ATTR_KEY, + UPDATE_ID_ATTR_KEY, + TERMINATE_REASON_ATTR_KEY, +} from '../instrumentation'; import { SpanName, SPAN_DELIMITER } from '../workflow'; export interface InterceptorOptions { @@ -24,7 +49,8 @@ export class OpenTelemetryWorkflowClientInterceptor implements WorkflowClientInt tracer: this.tracer, spanName: `${SpanName.WORKFLOW_START}${SPAN_DELIMITER}${input.workflowType}`, fn: async (span) => { - const headers = await headersWithContext(input.headers); + const headers = headersWithContext(input.headers); + span.setAttribute(WORKFLOW_ID_ATTR_KEY, input.options.workflowId); const runId = await next({ ...input, headers }); span.setAttribute(RUN_ID_ATTR_KEY, runId); return runId; @@ -36,10 +62,157 @@ export class OpenTelemetryWorkflowClientInterceptor implements WorkflowClientInt return await instrument({ tracer: this.tracer, spanName: `${SpanName.WORKFLOW_SIGNAL}${SPAN_DELIMITER}${input.signalName}`, - fn: async () => { - const headers = await headersWithContext(input.headers); + fn: async (span) => { + span.setAttribute(WORKFLOW_ID_ATTR_KEY, input.workflowExecution.workflowId); + const headers = headersWithContext(input.headers); await next({ ...input, headers }); }, }); } + + async startWithDetails( + input: WorkflowStartInput, + next: Next + ): Promise { + return await instrument({ + tracer: this.tracer, + spanName: `${SpanName.WORKFLOW_START}${SPAN_DELIMITER}${input.workflowType}`, + fn: async (span) => { + const headers = headersWithContext(input.headers); + span.setAttribute(WORKFLOW_ID_ATTR_KEY, input.options.workflowId); + const output = await next({ ...input, headers }); + span.setAttribute(RUN_ID_ATTR_KEY, output.runId); + return output; + }, + }); + } + + async startUpdate( + input: WorkflowStartUpdateInput, + next: Next + ): Promise { + return await instrument({ + tracer: this.tracer, + spanName: `${SpanName.WORKFLOW_START_UPDATE}${SPAN_DELIMITER}${input.updateName}`, + fn: async (span) => { + span.setAttribute(WORKFLOW_ID_ATTR_KEY, input.workflowExecution.workflowId); + if (input.options.updateId) { + span.setAttribute(UPDATE_ID_ATTR_KEY, input.options.updateId); + } + const headers = headersWithContext(input.headers); + const output = await next({ ...input, headers }); + span.setAttribute(RUN_ID_ATTR_KEY, output.workflowRunId); + return output; + }, + }); + } + + async startUpdateWithStart( + input: WorkflowStartUpdateWithStartInput, + next: Next + ): Promise { + return await instrument({ + tracer: this.tracer, + spanName: `${SpanName.WORKFLOW_UPDATE_WITH_START}${SPAN_DELIMITER}${input.updateName}`, + fn: async (span) => { + span.setAttribute(WORKFLOW_ID_ATTR_KEY, input.workflowStartOptions.workflowId); + if (input.updateOptions.updateId) { + span.setAttribute(UPDATE_ID_ATTR_KEY, input.updateOptions.updateId); + } + const workflowStartHeaders = headersWithContext(input.workflowStartHeaders); + const updateHeaders = headersWithContext(input.updateHeaders); + const output = await next({ ...input, workflowStartHeaders, updateHeaders }); + if (output.workflowExecution.runId) { + span.setAttribute(RUN_ID_ATTR_KEY, output.workflowExecution.runId); + } + return output; + }, + }); + } + + async signalWithStart( + input: WorkflowSignalWithStartInput, + next: Next + ): Promise { + return await instrument({ + tracer: this.tracer, + spanName: `${SpanName.WORKFLOW_SIGNAL_WITH_START}${SPAN_DELIMITER}${input.workflowType}`, + fn: async (span) => { + span.setAttribute(WORKFLOW_ID_ATTR_KEY, input.options.workflowId); + const headers = headersWithContext(input.headers); + const runId = await next({ ...input, headers }); + span.setAttribute(RUN_ID_ATTR_KEY, runId); + return runId; + }, + }); + } + + async query(input: WorkflowQueryInput, next: Next): Promise { + return await instrument({ + tracer: this.tracer, + spanName: `${SpanName.WORKFLOW_QUERY}${SPAN_DELIMITER}${input.queryType}`, + fn: async (span) => { + const headers = headersWithContext(input.headers); + span.setAttribute(WORKFLOW_ID_ATTR_KEY, input.workflowExecution.workflowId); + if (input.workflowExecution.runId) { + span.setAttribute(RUN_ID_ATTR_KEY, input.workflowExecution.runId); + } + return await next({ ...input, headers }); + }, + }); + } + + async terminate( + input: WorkflowTerminateInput, + next: Next + ): Promise { + return await instrument({ + tracer: this.tracer, + spanName: SpanName.WORKFLOW_TERMINATE, + fn: async (span) => { + span.setAttribute(WORKFLOW_ID_ATTR_KEY, input.workflowExecution.workflowId); + if (input.workflowExecution.runId) { + span.setAttribute(RUN_ID_ATTR_KEY, input.workflowExecution.runId); + } + if (input.reason) { + span.setAttribute(TERMINATE_REASON_ATTR_KEY, input.reason); + } + return await next(input); + }, + }); + } + + async cancel( + input: WorkflowCancelInput, + next: Next + ): Promise { + return await instrument({ + tracer: this.tracer, + spanName: SpanName.WORKFLOW_CANCEL, + fn: async (span) => { + span.setAttribute(WORKFLOW_ID_ATTR_KEY, input.workflowExecution.workflowId); + if (input.workflowExecution.runId) { + span.setAttribute(RUN_ID_ATTR_KEY, input.workflowExecution.runId); + } + return await next(input); + }, + }); + } + + async describe( + input: WorkflowDescribeInput, + next: Next + ): Promise { + return await instrument({ + tracer: this.tracer, + spanName: SpanName.WORKFLOW_DESCRIBE, + fn: async (span) => { + span.setAttribute(WORKFLOW_ID_ATTR_KEY, input.workflowExecution.workflowId); + if (input.workflowExecution.runId) { + span.setAttribute(RUN_ID_ATTR_KEY, input.workflowExecution.runId); + } + return await next(input); + }, + }); + } } diff --git a/packages/interceptors-opentelemetry/src/instrumentation.ts b/packages/interceptors-opentelemetry/src/instrumentation.ts index 610e05723..f8d89a73f 100644 --- a/packages/interceptors-opentelemetry/src/instrumentation.ts +++ b/packages/interceptors-opentelemetry/src/instrumentation.ts @@ -14,6 +14,20 @@ import { export const TRACE_HEADER = '_tracer-data'; /** As in workflow run id */ export const RUN_ID_ATTR_KEY = 'run_id'; +/** As in workflow id */ +export const WORKFLOW_ID_ATTR_KEY = 'temporalWorkflowId'; +/** As in activity id */ +export const ACTIVITY_ID_ATTR_KEY = 'temporalActivityId'; +/** As in update id */ +export const UPDATE_ID_ATTR_KEY = 'temporalUpdateId'; +/** As in termination reason */ +export const TERMINATE_REASON_ATTR_KEY = 'temporalTerminateReason'; +/** As in Nexus service */ +export const NEXUS_SERVICE_ATTR_KEY = 'temporalNexusService'; +/** As in Nexus operation */ +export const NEXUS_OPERATION_ATTR_KEY = 'temporalNexusOperation'; +/** As in Nexus endpoint */ +export const NEXUS_ENDPOINT_ATTR_KEY = 'temporalNexusEndpoint'; const payloadConverter = defaultPayloadConverter; @@ -48,20 +62,41 @@ async function wrapWithSpan( span.setStatus({ code: otel.SpanStatusCode.OK }); return ret; } catch (err: any) { - const isBenignErr = err instanceof ApplicationFailure && err.category === ApplicationFailureCategory.BENIGN; - if (acceptableErrors === undefined || !acceptableErrors(err)) { - const statusCode = isBenignErr ? otel.SpanStatusCode.UNSET : otel.SpanStatusCode.ERROR; - span.setStatus({ code: statusCode, message: (err as Error).message ?? String(err) }); - span.recordException(err); - } else { - span.setStatus({ code: otel.SpanStatusCode.OK }); - } + maybeAddErrorToSpan(err, span, acceptableErrors); throw err; } finally { span.end(); } } +function wrapWithSpanSync( + span: otel.Span, + fn: (span: otel.Span) => T, + acceptableErrors?: (err: unknown) => boolean +): T { + try { + const ret = fn(span); + span.setStatus({ code: otel.SpanStatusCode.OK }); + return ret; + } catch (err: any) { + maybeAddErrorToSpan(err, span, acceptableErrors); + throw err; + } finally { + span.end(); + } +} + +function maybeAddErrorToSpan(err: any, span: otel.Span, acceptableErrors?: (err: unknown) => boolean): void { + const isBenignErr = err instanceof ApplicationFailure && err.category === ApplicationFailureCategory.BENIGN; + if (acceptableErrors === undefined || !acceptableErrors(err)) { + const statusCode = isBenignErr ? otel.SpanStatusCode.UNSET : otel.SpanStatusCode.ERROR; + span.setStatus({ code: statusCode, message: (err as Error).message ?? String(err) }); + span.recordException(err); + } else { + span.setStatus({ code: otel.SpanStatusCode.OK }); + } +} + export interface InstrumentOptions { tracer: otel.Tracer; spanName: string; @@ -70,6 +105,8 @@ export interface InstrumentOptions { acceptableErrors?: (err: unknown) => boolean; } +export type InstrumentOptionsSync = Omit, 'fn'> & { fn: (span: otel.Span) => T }; + /** * Wraps `fn` in a span which ends when function returns or throws */ @@ -87,3 +124,12 @@ export async function instrument({ } return await tracer.startActiveSpan(spanName, async (span) => await wrapWithSpan(span, fn, acceptableErrors)); } + +export function instrumentSync({ tracer, spanName, fn, context, acceptableErrors }: InstrumentOptionsSync): T { + if (context) { + return otel.context.with(context, () => { + return tracer.startActiveSpan(spanName, (span) => wrapWithSpanSync(span, fn, acceptableErrors)); + }); + } + return tracer.startActiveSpan(spanName, (span) => wrapWithSpanSync(span, fn, acceptableErrors)); +} diff --git a/packages/interceptors-opentelemetry/src/worker/index.ts b/packages/interceptors-opentelemetry/src/worker/index.ts index 506200bdb..f865f5008 100644 --- a/packages/interceptors-opentelemetry/src/worker/index.ts +++ b/packages/interceptors-opentelemetry/src/worker/index.ts @@ -8,9 +8,16 @@ import type { ActivityOutboundCallsInterceptor, InjectedSink, GetLogAttributesInput, + GetMetricTagsInput, ActivityExecuteInput, } from '@temporalio/worker'; -import { instrument, extractContextFromHeaders } from '../instrumentation'; +import { + instrument, + extractContextFromHeaders, + WORKFLOW_ID_ATTR_KEY, + RUN_ID_ATTR_KEY, + ACTIVITY_ID_ATTR_KEY, +} from '../instrumentation'; import { type OpenTelemetryWorkflowExporter, type SerializableSpan, SpanName, SPAN_DELIMITER } from '../workflow'; export interface InterceptorOptions { @@ -36,14 +43,24 @@ export class OpenTelemetryActivityInboundInterceptor implements ActivityInboundC async execute(input: ActivityExecuteInput, next: Next): Promise { const context = extractContextFromHeaders(input.headers); const spanName = `${SpanName.ACTIVITY_EXECUTE}${SPAN_DELIMITER}${this.ctx.info.activityType}`; - return await instrument({ tracer: this.tracer, spanName, fn: () => next(input), context }); + return await instrument({ + tracer: this.tracer, + spanName, + fn: (span) => { + span.setAttribute(WORKFLOW_ID_ATTR_KEY, this.ctx.info.workflowExecution.workflowId); + span.setAttribute(RUN_ID_ATTR_KEY, this.ctx.info.workflowExecution.runId); + span.setAttribute(ACTIVITY_ID_ATTR_KEY, this.ctx.info.activityId); + return next(input); + }, + context, + }); } } /** - * Intercepts calls to emit logs from an Activity. + * Intercepts calls to emit logs and metrics from an Activity. * - * Attach OpenTelemetry context tracing attributes to emitted log messages, if appropriate. + * Attach OpenTelemetry context tracing attributes to emitted log messages and metrics, if appropriate. */ export class OpenTelemetryActivityOutboundInterceptor implements ActivityOutboundCallsInterceptor { constructor(protected readonly ctx: ActivityContext) {} @@ -65,6 +82,24 @@ export class OpenTelemetryActivityOutboundInterceptor implements ActivityOutboun return next(input); } } + + public getMetricTags( + input: GetMetricTagsInput, + next: Next + ): GetMetricTagsInput { + const span = otel.trace.getSpan(otel.context.active()); + const spanContext = span?.spanContext(); + if (spanContext && otel.isSpanContextValid(spanContext)) { + return next({ + trace_id: spanContext.traceId, + span_id: spanContext.spanId, + trace_flags: `0${spanContext.traceFlags.toString(16)}`, + ...input, + }); + } else { + return next(input); + } + } } /** diff --git a/packages/interceptors-opentelemetry/src/workflow/definitions.ts b/packages/interceptors-opentelemetry/src/workflow/definitions.ts index 6c70c0264..8b6dc7ba7 100644 --- a/packages/interceptors-opentelemetry/src/workflow/definitions.ts +++ b/packages/interceptors-opentelemetry/src/workflow/definitions.ts @@ -54,6 +54,56 @@ export enum SpanName { */ WORKFLOW_SIGNAL_WITH_START = 'SignalWithStartWorkflow', + /** + * Workflow is queried + */ + WORKFLOW_QUERY = 'QueryWorkflow', + + /** + * Workflow update is started by client + */ + WORKFLOW_START_UPDATE = 'StartWorkflowUpdate', + + /** + * Workflow is started with an update + */ + WORKFLOW_UPDATE_WITH_START = 'UpdateWithStartWorkflow', + + /** + * Workflow handles an incoming signal + */ + WORKFLOW_HANDLE_SIGNAL = 'HandleSignal', + + /** + * Workflow handles an incoming query + */ + WORKFLOW_HANDLE_QUERY = 'HandleQuery', + + /** + * Workflow handles an incoming update + */ + WORKFLOW_HANDLE_UPDATE = 'HandleUpdate', + + /** + * Workflow validates an incoming update + */ + WORKFLOW_VALIDATE_UPDATE = 'ValidateUpdate', + + /** + * Workflow is terminated + */ + WORKFLOW_TERMINATE = 'TerminateWorkflow', + + /** + * Workflow is cancelled + */ + WORKFLOW_CANCEL = 'CancelWorkflow', + + /** + * Workflow is described + */ + WORKFLOW_DESCRIBE = 'DescribeWorkflow', + /** * Workflow run is executing */ @@ -74,6 +124,10 @@ export enum SpanName { * Workflow is continuing as new */ CONTINUE_AS_NEW = 'ContinueAsNew', + /** + * Nexus operation is started + */ + NEXUS_OPERATION_START = 'StartNexusOperation', } export const SPAN_DELIMITER = ':'; diff --git a/packages/interceptors-opentelemetry/src/workflow/index.ts b/packages/interceptors-opentelemetry/src/workflow/index.ts index b47f58545..113d08d12 100644 --- a/packages/interceptors-opentelemetry/src/workflow/index.ts +++ b/packages/interceptors-opentelemetry/src/workflow/index.ts @@ -8,17 +8,31 @@ import type { ContinueAsNewInput, DisposeInput, GetLogAttributesInput, + GetMetricTagsInput, LocalActivityInput, Next, + QueryInput, SignalInput, SignalWorkflowInput, StartChildWorkflowExecutionInput, + UpdateInput, WorkflowExecuteInput, WorkflowInboundCallsInterceptor, WorkflowInternalsInterceptor, WorkflowOutboundCallsInterceptor, + StartNexusOperationInput, + StartNexusOperationOutput, } from '@temporalio/workflow'; -import { instrument, extractContextFromHeaders, headersWithContext } from '../instrumentation'; +import { + instrument, + instrumentSync, + extractContextFromHeaders, + headersWithContext, + UPDATE_ID_ATTR_KEY, + NEXUS_SERVICE_ATTR_KEY, + NEXUS_OPERATION_ATTR_KEY, + NEXUS_ENDPOINT_ATTR_KEY, +} from '../instrumentation'; import { ContextManager } from './context-manager'; import { SpanName, SPAN_DELIMITER } from './definitions'; import { SpanExporter } from './span-exporter'; @@ -87,7 +101,57 @@ export class OpenTelemetryInboundInterceptor implements WorkflowInboundCallsInte return await instrument({ tracer: this.tracer, - spanName: `${SpanName.WORKFLOW_SIGNAL}${SPAN_DELIMITER}${input.signalName}`, + spanName: `${SpanName.WORKFLOW_HANDLE_SIGNAL}${SPAN_DELIMITER}${input.signalName}`, + fn: () => next(input), + context, + }); + } + + public async handleUpdate( + input: UpdateInput, + next: Next + ): Promise { + if (!hasSdkFlag('OpenTelemetryInterceptorsInstrumentsAllMethods')) return next(input); + + const context = extractContextFromHeaders(input.headers); + + return await instrument({ + tracer: this.tracer, + spanName: `${SpanName.WORKFLOW_HANDLE_UPDATE}${SPAN_DELIMITER}${input.name}`, + fn: (span) => { + span.setAttribute(UPDATE_ID_ATTR_KEY, input.updateId); + return next(input); + }, + context, + }); + } + + public validateUpdate(input: UpdateInput, next: Next): void { + if (!hasSdkFlag('OpenTelemetryInterceptorsInstrumentsAllMethods')) return next(input); + + const context = extractContextFromHeaders(input.headers); + instrumentSync({ + tracer: this.tracer, + spanName: `${SpanName.WORKFLOW_VALIDATE_UPDATE}${SPAN_DELIMITER}${input.name}`, + fn: (span) => { + span.setAttribute(UPDATE_ID_ATTR_KEY, input.updateId); + return next(input); + }, + context, + }); + } + + public async handleQuery( + input: QueryInput, + next: Next + ): Promise { + if (!hasSdkFlag('OpenTelemetryInterceptorsInstrumentsAllMethods')) return next(input); + + const context = extractContextFromHeaders(input.headers); + + return await instrument({ + tracer: this.tracer, + spanName: `${SpanName.WORKFLOW_HANDLE_QUERY}${SPAN_DELIMITER}${input.queryName}`, fn: () => next(input), context, }); @@ -149,6 +213,24 @@ export class OpenTelemetryOutboundInterceptor implements WorkflowOutboundCallsIn }); } + public async startNexusOperation( + input: StartNexusOperationInput, + next: Next + ): Promise { + if (!hasSdkFlag('OpenTelemetryInterceptorsInstrumentsAllMethods')) return next(input); + + return await instrument({ + tracer: this.tracer, + spanName: `${SpanName.NEXUS_OPERATION_START}${SPAN_DELIMITER}${input.service}${SPAN_DELIMITER}${input.operation}`, + fn: async (span) => { + span.setAttribute(NEXUS_SERVICE_ATTR_KEY, input.service); + span.setAttribute(NEXUS_OPERATION_ATTR_KEY, input.operation); + span.setAttribute(NEXUS_ENDPOINT_ATTR_KEY, input.endpoint); + return await next(input); + }, + }); + } + public async startChildWorkflowExecution( input: StartChildWorkflowExecutionInput, next: Next @@ -225,6 +307,24 @@ export class OpenTelemetryOutboundInterceptor implements WorkflowOutboundCallsIn return next(input); } } + + public getMetricTags( + input: GetMetricTagsInput, + next: Next + ): GetMetricTagsInput { + const span = otel.trace.getSpan(otel.context.active()); + const spanContext = span?.spanContext(); + if (spanContext && otel.isSpanContextValid(spanContext)) { + return next({ + trace_id: spanContext.traceId, + span_id: spanContext.spanId, + trace_flags: `0${spanContext.traceFlags.toString(16)}`, + ...input, + }); + } else { + return next(input); + } + } } export class OpenTelemetryInternalsInterceptor implements WorkflowInternalsInterceptor { diff --git a/packages/test/history_files/otel_smorgasbord_1_13_2.json b/packages/test/history_files/otel_smorgasbord_1_13_2.json new file mode 100644 index 000000000..70334119d --- /dev/null +++ b/packages/test/history_files/otel_smorgasbord_1_13_2.json @@ -0,0 +1,786 @@ +{ + "events": [ + { + "eventId": "1", + "eventTime": "2025-11-21T19:00:01.803145Z", + "eventType": "EVENT_TYPE_WORKFLOW_EXECUTION_STARTED", + "taskId": "1081192", + "workflowExecutionStartedEventAttributes": { + "workflowType": { + "name": "smorgasbord" + }, + "taskQueue": { + "name": "test-otel", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "input": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "Mg==" + } + ] + }, + "workflowRunTimeout": "0s", + "workflowTaskTimeout": "10s", + "continuedExecutionRunId": "3d668831-7352-4f7c-ba76-8e7c7de380b5", + "initiator": "CONTINUE_AS_NEW_INITIATOR_WORKFLOW", + "originalExecutionRunId": "080bf2cb-bf94-483d-b2a7-e1251e3098af", + "firstExecutionRunId": "019aa7c9-9e5b-775a-a997-fec1585dd910", + "attempt": 1, + "prevAutoResetPoints": { + "points": [ + { + "runId": "019aa7c9-9e5b-775a-a997-fec1585dd910", + "firstWorkflowTaskCompletedId": "4", + "createTime": "2025-11-21T18:59:59.753854Z", + "expireTime": "2025-11-22T18:59:59.754061Z", + "resettable": true, + "buildId": "@temporalio/worker@1.13.2+ac7b57aeab0ae668597ad4cbec3fab06bdd17b2f350444b91b30378957c819ad" + } + ] + }, + "header": { + "fields": { + "_tracer-data": { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJ0cmFjZXBhcmVudCI6IjAwLTRhNWYzN2NiNjdlMjE1NWUyOGYxZjYzNTQ4NWEwMWQ2LWVlYzliMWQ0OGM5OWMwZGEtMDEifQ==" + } + } + }, + "workflowId": "3710ab85-dcbf-46f7-93b0-b7ce7dac0b1c" + } + }, + { + "eventId": "2", + "eventTime": "2025-11-21T19:00:01.803221Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "taskId": "1081193", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "test-otel", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "startToCloseTimeout": "10s", + "attempt": 1 + } + }, + { + "eventId": "3", + "eventTime": "2025-11-21T19:00:01.808162Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "taskId": "1081200", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "2", + "identity": "32020@mac.lan", + "requestId": "94944c1a-dbb6-4473-afca-222d59a3950e", + "historySizeBytes": "590", + "workerVersion": { + "buildId": "@temporalio/worker@1.13.2+ac7b57aeab0ae668597ad4cbec3fab06bdd17b2f350444b91b30378957c819ad" + } + } + }, + { + "eventId": "4", + "eventTime": "2025-11-21T19:00:01.825464Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "taskId": "1081204", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "2", + "startedEventId": "3", + "identity": "32020@mac.lan", + "workerVersion": { + "buildId": "@temporalio/worker@1.13.2+ac7b57aeab0ae668597ad4cbec3fab06bdd17b2f350444b91b30378957c819ad" + }, + "sdkMetadata": { + "coreUsedFlags": [1, 3, 2], + "langUsedFlags": [5, 4], + "sdkName": "temporal-typescript", + "sdkVersion": "1.13.2" + }, + "meteringMetadata": {} + } + }, + { + "eventId": "5", + "eventTime": "2025-11-21T19:00:01.825503Z", + "eventType": "EVENT_TYPE_ACTIVITY_TASK_SCHEDULED", + "taskId": "1081205", + "activityTaskScheduledEventAttributes": { + "activityId": "1", + "activityType": { + "name": "fakeProgress" + }, + "taskQueue": { + "name": "test-otel", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "header": { + "fields": { + "_tracer-data": { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJ0cmFjZXBhcmVudCI6IjAwLTRhNWYzN2NiNjdlMjE1NWUyOGYxZjYzNTQ4NWEwMWQ2LWJkMzNiZTU1ZDdiZjNjZjgtMDEifQ==" + } + } + }, + "input": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "MTAw" + }, + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "MTA=" + } + ] + }, + "scheduleToCloseTimeout": "0s", + "scheduleToStartTimeout": "0s", + "startToCloseTimeout": "60s", + "heartbeatTimeout": "0s", + "workflowTaskCompletedEventId": "4", + "retryPolicy": { + "initialInterval": "1s", + "backoffCoefficient": 2, + "maximumInterval": "100s" + }, + "useWorkflowBuildId": true + } + }, + { + "eventId": "6", + "eventTime": "2025-11-21T19:00:01.825521Z", + "eventType": "EVENT_TYPE_ACTIVITY_TASK_SCHEDULED", + "taskId": "1081206", + "activityTaskScheduledEventAttributes": { + "activityId": "2", + "activityType": { + "name": "queryOwnWf" + }, + "taskQueue": { + "name": "test-otel", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "header": { + "fields": { + "_tracer-data": { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJ0cmFjZXBhcmVudCI6IjAwLTRhNWYzN2NiNjdlMjE1NWUyOGYxZjYzNTQ4NWEwMWQ2LTA3YTIwNzJlYTdmODljMTItMDEifQ==" + } + } + }, + "input": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJ0eXBlIjoicXVlcnkiLCJuYW1lIjoic3RlcCJ9" + } + ] + }, + "scheduleToCloseTimeout": "0s", + "scheduleToStartTimeout": "0s", + "startToCloseTimeout": "60s", + "heartbeatTimeout": "0s", + "workflowTaskCompletedEventId": "4", + "retryPolicy": { + "initialInterval": "1s", + "backoffCoefficient": 2, + "maximumInterval": "100s" + }, + "useWorkflowBuildId": true + } + }, + { + "eventId": "7", + "eventTime": "2025-11-21T19:00:01.825528Z", + "eventType": "EVENT_TYPE_TIMER_STARTED", + "taskId": "1081207", + "timerStartedEventAttributes": { + "timerId": "1", + "startToFireTimeout": "1s", + "workflowTaskCompletedEventId": "4" + } + }, + { + "eventId": "8", + "eventTime": "2025-11-21T19:00:01.825710Z", + "eventType": "EVENT_TYPE_START_CHILD_WORKFLOW_EXECUTION_INITIATED", + "taskId": "1081208", + "startChildWorkflowExecutionInitiatedEventAttributes": { + "namespace": "default", + "workflowId": "3e74e356-25a4-4e0f-b477-eb83679117a1", + "workflowType": { + "name": "signalTarget" + }, + "taskQueue": { + "name": "test-otel", + "kind": "TASK_QUEUE_KIND_NORMAL" + }, + "workflowRunTimeout": "0s", + "workflowTaskTimeout": "10s", + "parentClosePolicy": "PARENT_CLOSE_POLICY_TERMINATE", + "workflowTaskCompletedEventId": "4", + "workflowIdReusePolicy": "WORKFLOW_ID_REUSE_POLICY_ALLOW_DUPLICATE", + "header": { + "fields": { + "_tracer-data": { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJ0cmFjZXBhcmVudCI6IjAwLTRhNWYzN2NiNjdlMjE1NWUyOGYxZjYzNTQ4NWEwMWQ2LTFjYTY2MzQ0ZGJhYThhYTctMDEifQ==" + } + } + }, + "memo": { + "fields": {} + }, + "searchAttributes": { + "indexedFields": {} + }, + "namespaceId": "019a935a-2ac1-7c29-b43c-a2876bf2993f", + "inheritBuildId": true + } + }, + { + "eventId": "9", + "eventTime": "2025-11-21T19:00:01.825724Z", + "eventType": "EVENT_TYPE_MARKER_RECORDED", + "taskId": "1081209", + "markerRecordedEventAttributes": { + "details": { + "result": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "ImxvY2FsLWFjdGl2aXR5Ig==" + } + ] + }, + "data": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJzZXEiOjMsImF0dGVtcHQiOjEsImFjdGl2aXR5X2lkIjoiMyIsImFjdGl2aXR5X3R5cGUiOiJlY2hvIiwiY29tcGxldGVfdGltZSI6eyJzZWNvbmRzIjoxNzYzNzUxNjAxLCJuYW5vcyI6ODA5MDAzNzkyfSwiYmFja29mZiI6bnVsbCwib3JpZ2luYWxfc2NoZWR1bGVfdGltZSI6eyJzZWNvbmRzIjoxNzYzNzUxNjAxLCJuYW5vcyI6ODIxOTM4MDAwfX0=" + } + ] + } + }, + "markerName": "core_local_activity", + "workflowTaskCompletedEventId": "4" + } + }, + { + "eventId": "10", + "eventTime": "2025-11-21T19:00:01.827165Z", + "eventType": "EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_STARTED", + "taskId": "1081220", + "childWorkflowExecutionStartedEventAttributes": { + "namespace": "default", + "initiatedEventId": "8", + "workflowExecution": { + "workflowId": "3e74e356-25a4-4e0f-b477-eb83679117a1", + "runId": "019aa7c9-a6a2-772f-90b3-d51e43a30ea9" + }, + "workflowType": { + "name": "signalTarget" + }, + "header": { + "fields": { + "_tracer-data": { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJ0cmFjZXBhcmVudCI6IjAwLTRhNWYzN2NiNjdlMjE1NWUyOGYxZjYzNTQ4NWEwMWQ2LTFjYTY2MzQ0ZGJhYThhYTctMDEifQ==" + } + } + }, + "namespaceId": "019a935a-2ac1-7c29-b43c-a2876bf2993f" + } + }, + { + "eventId": "11", + "eventTime": "2025-11-21T19:00:01.827169Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "taskId": "1081221", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "32020@mac.lan-98c600bbf919422e9e73c902149dc629", + "kind": "TASK_QUEUE_KIND_STICKY", + "normalName": "test-otel" + }, + "startToCloseTimeout": "10s", + "attempt": 1 + } + }, + { + "eventId": "12", + "eventTime": "2025-11-21T19:00:01.827660Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "taskId": "1081229", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "11", + "identity": "32020@mac.lan", + "requestId": "29c691ee-a563-4b88-b078-9f63696dd791", + "historySizeBytes": "2560", + "workerVersion": { + "buildId": "@temporalio/worker@1.13.2+ac7b57aeab0ae668597ad4cbec3fab06bdd17b2f350444b91b30378957c819ad" + } + } + }, + { + "eventId": "13", + "eventTime": "2025-11-21T19:00:01.837926Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "taskId": "1081243", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "11", + "startedEventId": "12", + "identity": "32020@mac.lan", + "workerVersion": { + "buildId": "@temporalio/worker@1.13.2+ac7b57aeab0ae668597ad4cbec3fab06bdd17b2f350444b91b30378957c819ad" + }, + "sdkMetadata": {}, + "meteringMetadata": {} + } + }, + { + "eventId": "14", + "eventTime": "2025-11-21T19:00:01.837954Z", + "eventType": "EVENT_TYPE_SIGNAL_EXTERNAL_WORKFLOW_EXECUTION_INITIATED", + "taskId": "1081244", + "signalExternalWorkflowExecutionInitiatedEventAttributes": { + "workflowTaskCompletedEventId": "13", + "namespace": "default", + "workflowExecution": { + "workflowId": "3e74e356-25a4-4e0f-b477-eb83679117a1" + }, + "signalName": "unblock", + "childWorkflowOnly": true, + "header": { + "fields": { + "_tracer-data": { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "eyJ0cmFjZXBhcmVudCI6IjAwLTRhNWYzN2NiNjdlMjE1NWUyOGYxZjYzNTQ4NWEwMWQ2LTcyMTYyMmVkMDI3ZjVkM2UtMDEifQ==" + } + } + }, + "namespaceId": "019a935a-2ac1-7c29-b43c-a2876bf2993f" + } + }, + { + "eventId": "15", + "eventTime": "2025-11-21T19:00:01.831344Z", + "eventType": "EVENT_TYPE_WORKFLOW_EXECUTION_SIGNALED", + "taskId": "1081245", + "workflowExecutionSignaledEventAttributes": { + "signalName": "activityStarted", + "input": {}, + "identity": "32020@mac.lan", + "header": { + "fields": { + "_tracer-data": { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "e30=" + } + } + } + } + }, + { + "eventId": "16", + "eventTime": "2025-11-21T19:00:01.837968Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "taskId": "1081246", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "32020@mac.lan-98c600bbf919422e9e73c902149dc629", + "kind": "TASK_QUEUE_KIND_STICKY", + "normalName": "test-otel" + }, + "startToCloseTimeout": "10s", + "attempt": 1 + } + }, + { + "eventId": "17", + "eventTime": "2025-11-21T19:00:01.837969Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "taskId": "1081247", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "16", + "identity": "32020@mac.lan", + "requestId": "request-from-RespondWorkflowTaskCompleted", + "historySizeBytes": "2740", + "workerVersion": { + "buildId": "@temporalio/worker@1.13.2+ac7b57aeab0ae668597ad4cbec3fab06bdd17b2f350444b91b30378957c819ad" + } + } + }, + { + "eventId": "18", + "eventTime": "2025-11-21T19:00:01.845193Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "taskId": "1081266", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "16", + "startedEventId": "17", + "identity": "32020@mac.lan", + "workerVersion": { + "buildId": "@temporalio/worker@1.13.2+ac7b57aeab0ae668597ad4cbec3fab06bdd17b2f350444b91b30378957c819ad" + }, + "sdkMetadata": { + "langUsedFlags": [3, 2] + }, + "meteringMetadata": {} + } + }, + { + "eventId": "19", + "eventTime": "2025-11-21T19:00:01.839843Z", + "eventType": "EVENT_TYPE_EXTERNAL_WORKFLOW_EXECUTION_SIGNALED", + "taskId": "1081267", + "externalWorkflowExecutionSignaledEventAttributes": { + "initiatedEventId": "14", + "namespace": "default", + "workflowExecution": { + "workflowId": "3e74e356-25a4-4e0f-b477-eb83679117a1" + }, + "namespaceId": "019a935a-2ac1-7c29-b43c-a2876bf2993f" + } + }, + { + "eventId": "20", + "eventTime": "2025-11-21T19:00:01.844480Z", + "eventType": "EVENT_TYPE_CHILD_WORKFLOW_EXECUTION_COMPLETED", + "taskId": "1081268", + "childWorkflowExecutionCompletedEventAttributes": { + "result": { + "payloads": [ + { + "metadata": { + "encoding": "YmluYXJ5L251bGw=" + } + } + ] + }, + "namespace": "default", + "workflowExecution": { + "workflowId": "3e74e356-25a4-4e0f-b477-eb83679117a1", + "runId": "019aa7c9-a6a2-772f-90b3-d51e43a30ea9" + }, + "workflowType": { + "name": "signalTarget" + }, + "initiatedEventId": "8", + "startedEventId": "10", + "namespaceId": "019a935a-2ac1-7c29-b43c-a2876bf2993f" + } + }, + { + "eventId": "21", + "eventTime": "2025-11-21T19:00:01.845205Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "taskId": "1081269", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "32020@mac.lan-98c600bbf919422e9e73c902149dc629", + "kind": "TASK_QUEUE_KIND_STICKY", + "normalName": "test-otel" + }, + "startToCloseTimeout": "10s", + "attempt": 1 + } + }, + { + "eventId": "22", + "eventTime": "2025-11-21T19:00:01.845207Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "taskId": "1081270", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "21", + "identity": "32020@mac.lan", + "requestId": "request-from-RespondWorkflowTaskCompleted", + "historySizeBytes": "3521", + "workerVersion": { + "buildId": "@temporalio/worker@1.13.2+ac7b57aeab0ae668597ad4cbec3fab06bdd17b2f350444b91b30378957c819ad" + } + } + }, + { + "eventId": "23", + "eventTime": "2025-11-21T19:00:01.847849Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "taskId": "1081274", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "21", + "startedEventId": "22", + "identity": "32020@mac.lan", + "workerVersion": { + "buildId": "@temporalio/worker@1.13.2+ac7b57aeab0ae668597ad4cbec3fab06bdd17b2f350444b91b30378957c819ad" + }, + "sdkMetadata": {}, + "meteringMetadata": {} + } + }, + { + "eventId": "24", + "eventTime": "2025-11-21T19:00:01.828311Z", + "eventType": "EVENT_TYPE_ACTIVITY_TASK_STARTED", + "taskId": "1081275", + "activityTaskStartedEventAttributes": { + "scheduledEventId": "6", + "identity": "32020@mac.lan", + "requestId": "66e731c3-2834-47be-8f7e-ba139a9dfe56", + "attempt": 1, + "workerVersion": { + "buildId": "@temporalio/worker@1.13.2+ac7b57aeab0ae668597ad4cbec3fab06bdd17b2f350444b91b30378957c819ad" + } + } + }, + { + "eventId": "25", + "eventTime": "2025-11-21T19:00:01.846596Z", + "eventType": "EVENT_TYPE_ACTIVITY_TASK_COMPLETED", + "taskId": "1081276", + "activityTaskCompletedEventAttributes": { + "result": { + "payloads": [ + { + "metadata": { + "encoding": "anNvbi9wbGFpbg==" + }, + "data": "Mg==" + } + ] + }, + "scheduledEventId": "6", + "startedEventId": "24", + "identity": "32020@mac.lan" + } + }, + { + "eventId": "26", + "eventTime": "2025-11-21T19:00:01.847862Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "taskId": "1081277", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "32020@mac.lan-98c600bbf919422e9e73c902149dc629", + "kind": "TASK_QUEUE_KIND_STICKY", + "normalName": "test-otel" + }, + "startToCloseTimeout": "10s", + "attempt": 1 + } + }, + { + "eventId": "27", + "eventTime": "2025-11-21T19:00:01.847863Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "taskId": "1081278", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "26", + "identity": "32020@mac.lan", + "requestId": "request-from-RespondWorkflowTaskCompleted", + "historySizeBytes": "4268", + "workerVersion": { + "buildId": "@temporalio/worker@1.13.2+ac7b57aeab0ae668597ad4cbec3fab06bdd17b2f350444b91b30378957c819ad" + } + } + }, + { + "eventId": "28", + "eventTime": "2025-11-21T19:00:01.849788Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "taskId": "1081281", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "26", + "startedEventId": "27", + "identity": "32020@mac.lan", + "workerVersion": { + "buildId": "@temporalio/worker@1.13.2+ac7b57aeab0ae668597ad4cbec3fab06bdd17b2f350444b91b30378957c819ad" + }, + "sdkMetadata": {}, + "meteringMetadata": {} + } + }, + { + "eventId": "29", + "eventTime": "2025-11-21T19:00:02.827865Z", + "eventType": "EVENT_TYPE_TIMER_FIRED", + "taskId": "1081283", + "timerFiredEventAttributes": { + "timerId": "1", + "startedEventId": "7" + } + }, + { + "eventId": "30", + "eventTime": "2025-11-21T19:00:02.827881Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "taskId": "1081284", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "32020@mac.lan-98c600bbf919422e9e73c902149dc629", + "kind": "TASK_QUEUE_KIND_STICKY", + "normalName": "test-otel" + }, + "startToCloseTimeout": "10s", + "attempt": 1 + } + }, + { + "eventId": "31", + "eventTime": "2025-11-21T19:00:02.831703Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "taskId": "1081288", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "30", + "identity": "32020@mac.lan", + "requestId": "a52a6af0-fbd2-4d5c-b435-2d32ef3888ac", + "historySizeBytes": "5220", + "workerVersion": { + "buildId": "@temporalio/worker@1.13.2+ac7b57aeab0ae668597ad4cbec3fab06bdd17b2f350444b91b30378957c819ad" + } + } + }, + { + "eventId": "32", + "eventTime": "2025-11-21T19:00:02.844934Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "taskId": "1081292", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "30", + "startedEventId": "31", + "identity": "32020@mac.lan", + "workerVersion": { + "buildId": "@temporalio/worker@1.13.2+ac7b57aeab0ae668597ad4cbec3fab06bdd17b2f350444b91b30378957c819ad" + }, + "sdkMetadata": {}, + "meteringMetadata": {} + } + }, + { + "eventId": "33", + "eventTime": "2025-11-21T19:00:01.828550Z", + "eventType": "EVENT_TYPE_ACTIVITY_TASK_STARTED", + "taskId": "1081294", + "activityTaskStartedEventAttributes": { + "scheduledEventId": "5", + "identity": "32020@mac.lan", + "requestId": "c1d854bb-7445-4340-8289-4964108bcec2", + "attempt": 1, + "workerVersion": { + "buildId": "@temporalio/worker@1.13.2+ac7b57aeab0ae668597ad4cbec3fab06bdd17b2f350444b91b30378957c819ad" + } + } + }, + { + "eventId": "34", + "eventTime": "2025-11-21T19:00:02.847108Z", + "eventType": "EVENT_TYPE_ACTIVITY_TASK_COMPLETED", + "taskId": "1081295", + "activityTaskCompletedEventAttributes": { + "result": { + "payloads": [ + { + "metadata": { + "encoding": "YmluYXJ5L251bGw=" + } + } + ] + }, + "scheduledEventId": "5", + "startedEventId": "33", + "identity": "32020@mac.lan" + } + }, + { + "eventId": "35", + "eventTime": "2025-11-21T19:00:02.847118Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_SCHEDULED", + "taskId": "1081296", + "workflowTaskScheduledEventAttributes": { + "taskQueue": { + "name": "32020@mac.lan-98c600bbf919422e9e73c902149dc629", + "kind": "TASK_QUEUE_KIND_STICKY", + "normalName": "test-otel" + }, + "startToCloseTimeout": "10s", + "attempt": 1 + } + }, + { + "eventId": "36", + "eventTime": "2025-11-21T19:00:02.849311Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_STARTED", + "taskId": "1081300", + "workflowTaskStartedEventAttributes": { + "scheduledEventId": "35", + "identity": "32020@mac.lan", + "requestId": "391bd0e5-41e7-466c-a054-f8b87b38651b", + "historySizeBytes": "5894", + "workerVersion": { + "buildId": "@temporalio/worker@1.13.2+ac7b57aeab0ae668597ad4cbec3fab06bdd17b2f350444b91b30378957c819ad" + } + } + }, + { + "eventId": "37", + "eventTime": "2025-11-21T19:00:02.860057Z", + "eventType": "EVENT_TYPE_WORKFLOW_TASK_COMPLETED", + "taskId": "1081304", + "workflowTaskCompletedEventAttributes": { + "scheduledEventId": "35", + "startedEventId": "36", + "identity": "32020@mac.lan", + "workerVersion": { + "buildId": "@temporalio/worker@1.13.2+ac7b57aeab0ae668597ad4cbec3fab06bdd17b2f350444b91b30378957c819ad" + }, + "sdkMetadata": {}, + "meteringMetadata": {} + } + }, + { + "eventId": "38", + "eventTime": "2025-11-21T19:00:02.860101Z", + "eventType": "EVENT_TYPE_WORKFLOW_EXECUTION_COMPLETED", + "taskId": "1081305", + "workflowExecutionCompletedEventAttributes": { + "result": { + "payloads": [ + { + "metadata": { + "encoding": "YmluYXJ5L251bGw=" + } + } + ] + }, + "workflowTaskCompletedEventId": "37" + } + } + ] +} diff --git a/packages/test/src/test-otel.ts b/packages/test/src/test-otel.ts index 93dcb2e9d..97b29e074 100644 --- a/packages/test/src/test-otel.ts +++ b/packages/test/src/test-otel.ts @@ -12,7 +12,7 @@ import { BasicTracerProvider, InMemorySpanExporter, SimpleSpanProcessor } from ' import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions'; import test from 'ava'; import { v4 as uuid4 } from 'uuid'; -import { WorkflowClient } from '@temporalio/client'; +import { WorkflowClient, WithStartWorkflowOperation, WorkflowClientInterceptor } from '@temporalio/client'; import { OpenTelemetryWorkflowClientInterceptor } from '@temporalio/interceptors-opentelemetry/lib/client'; import { OpenTelemetryWorkflowClientCallsInterceptor } from '@temporalio/interceptors-opentelemetry'; import { instrument } from '@temporalio/interceptors-opentelemetry/lib/instrumentation'; @@ -21,8 +21,21 @@ import { OpenTelemetryActivityInboundInterceptor, OpenTelemetryActivityOutboundInterceptor, } from '@temporalio/interceptors-opentelemetry/lib/worker'; -import { OpenTelemetrySinks, SpanName, SPAN_DELIMITER } from '@temporalio/interceptors-opentelemetry/lib/workflow'; -import { DefaultLogger, InjectedSinks, Runtime } from '@temporalio/worker'; +import { + OpenTelemetrySinks, + SpanName, + SPAN_DELIMITER, + OpenTelemetryOutboundInterceptor, + OpenTelemetryInboundInterceptor, +} from '@temporalio/interceptors-opentelemetry/lib/workflow'; +import { + ActivityInboundCallsInterceptor, + ActivityOutboundCallsInterceptor, + DefaultLogger, + InjectedSinks, + Runtime, +} from '@temporalio/worker'; +import { WorkflowInboundCallsInterceptor, WorkflowOutboundCallsInterceptor } from '@temporalio/workflow'; import * as activities from './activities'; import { loadHistory, RUN_INTEGRATION_TESTS, TestWorkflowEnvironment, Worker } from './helpers'; import * as workflows from './workflows'; @@ -251,7 +264,7 @@ if (RUN_INTEGRATION_TESTS) { resource: staticResource, traceExporter, }); - await otel.start(); + otel.start(); const sinks: InjectedSinks = { exporter: makeWorkflowExporter(traceExporter, staticResource), @@ -379,6 +392,13 @@ if (RUN_INTEGRATION_TESTS) { ); t.true(activityStartedSignalSpan !== undefined); + const querySpan = spans.find( + ({ name, parentSpanId }) => + name === `${SpanName.WORKFLOW_QUERY}${SPAN_DELIMITER}step` && + parentSpanId === secondActivityExecuteSpan?.spanContext().spanId + ); + t.true(querySpan !== undefined); + t.deepEqual(new Set(spans.map((span) => span.spanContext().traceId)).size, 1); } finally { // Cleanup the runtime so that it doesn't interfere with other tests @@ -510,6 +530,61 @@ if (RUN_INTEGRATION_TESTS) { t.is(spans[1].status.message, 'benign'); t.is(spans[2].status.code, SpanStatusCode.OK); }); + + test('executeUpdateWithStart works correctly with OTEL interceptors', async (t) => { + const staticResource = new opentelemetry.resources.Resource({ + [SemanticResourceAttributes.SERVICE_NAME]: 'ts-test-otel-worker', + }); + const traceExporter: opentelemetry.tracing.SpanExporter = { + export(_spans, resultCallback) { + resultCallback({ code: ExportResultCode.SUCCESS }); + }, + async shutdown() {}, + }; + + const sinks: InjectedSinks = { + exporter: makeWorkflowExporter(traceExporter, staticResource), + }; + + const worker = await Worker.create({ + workflowBundle: await createTestWorkflowBundle({ + workflowsPath: require.resolve('./workflows'), + workflowInterceptorModules: [require.resolve('./workflows/otel-interceptors')], + }), + activities, + taskQueue: 'test-otel-update-start', + interceptors: { + client: { + workflow: [new OpenTelemetryWorkflowClientCallsInterceptor()], + }, + workflowModules: [require.resolve('./workflows/otel-interceptors')], + }, + sinks, + }); + + const client = new WorkflowClient(); + + const startWorkflowOperation = new WithStartWorkflowOperation(workflows.updateStartOtel, { + workflowId: uuid4(), + taskQueue: 'test-otel-update-start', + workflowIdConflictPolicy: 'FAIL', + }); + + const { updateResult, workflowResult } = await worker.runUntil(async () => { + const updateResult = await client.executeUpdateWithStart(workflows.otelUpdate, { + args: [true], + startWorkflowOperation, + }); + + const handle = await startWorkflowOperation.workflowHandle(); + const workflowResult = await handle.result(); + + return { updateResult, workflowResult }; + }); + + t.is(updateResult, true); + t.is(workflowResult, true); + }); } test('Can replay otel history from 1.11.3', async (t) => { @@ -604,3 +679,43 @@ test('Can replay signal workflow from 1.13.1', async (t) => { ); }); }); + +test('Can replay smorgasbord from 1.13.2', async (t) => { + const hist = await loadHistory('otel_smorgasbord_1_13_2.json'); + await t.notThrowsAsync(async () => { + await Worker.runReplayHistory( + { + workflowBundle: await createTestWorkflowBundle({ + workflowsPath: require.resolve('./workflows'), + workflowInterceptorModules: [require.resolve('./workflows/otel-interceptors')], + }), + interceptors: { + workflowModules: [require.resolve('./workflows/otel-interceptors')], + activity: [ + (ctx) => ({ + inbound: new OpenTelemetryActivityInboundInterceptor(ctx), + outbound: new OpenTelemetryActivityOutboundInterceptor(ctx), + }), + ], + }, + }, + hist + ); + }); +}); + +// Skipped as we only care that it compiles +test.skip('otel interceptors are complete', async (t) => { + // We only use this to verify that we trace all spans via typechecking + // Doing this instead of directly changing the `implements` to avoid leaking this in the docs + const _wfl_inbound = {} as OpenTelemetryInboundInterceptor satisfies Required; + const _wfl_outbound = {} as OpenTelemetryOutboundInterceptor satisfies Required< + Omit + >; + const _act_inbound = + {} as OpenTelemetryActivityInboundInterceptor satisfies Required; + const _act_outbound = + {} as OpenTelemetryActivityOutboundInterceptor satisfies Required; + const _client = {} as OpenTelemetryWorkflowClientInterceptor satisfies Required; + t.pass(); +}); diff --git a/packages/test/src/workflows/index.ts b/packages/test/src/workflows/index.ts index 0d8ef95ac..1ac8c4021 100644 --- a/packages/test/src/workflows/index.ts +++ b/packages/test/src/workflows/index.ts @@ -93,5 +93,6 @@ export * from './upsert-and-read-search-attributes'; export * from './wait-on-user'; export * from './workflow-cancellation-scenarios'; export * from './upsert-and-read-memo'; +export * from './update-start-otel'; export * from './updates-ordering'; export * from './wait-on-signal-then-activity'; diff --git a/packages/test/src/workflows/update-start-otel.ts b/packages/test/src/workflows/update-start-otel.ts new file mode 100644 index 000000000..db50f438f --- /dev/null +++ b/packages/test/src/workflows/update-start-otel.ts @@ -0,0 +1,12 @@ +import * as workflow from '@temporalio/workflow'; + +export const otelUpdate = workflow.defineUpdate('otelUpdate'); + +export async function updateStartOtel(): Promise { + let updateResult = false; + workflow.setHandler(otelUpdate, (value: boolean): boolean => { + updateResult = value; + return true; + }); + return updateResult; +} diff --git a/packages/workflow/src/flags.ts b/packages/workflow/src/flags.ts index 40c994c8b..117b987f1 100644 --- a/packages/workflow/src/flags.ts +++ b/packages/workflow/src/flags.ts @@ -76,6 +76,16 @@ export const SdkFlags = { * @since Introduced in 1.13.2 */ OpenTelemetryInterceporsAvoidsExtraYields: defineFlag(5, true, [isAtLeast({ major: 1, minor: 13, patch: 2 })]), + + /** + * In 1.13.3, all remaining interceptor methods were implemented in @temporalio/interceptors-opentelemetry + * including `handleUpdate`, `validateUpdate`, `handleQuery`, `startTimer`, and `startNexusOperation`. + * These add instrumentation and yield points that were not present in earlier versions. + * This flag gates these new interceptor methods to prevent NDE on replay of workflows from earlier versions. + * + * @since Introduced in 1.13.3 + */ + OpenTelemetryInterceptorsInstrumentsAllMethods: defineFlag(6, true), } as const; function defineFlag(id: number, def: boolean, alternativeConditions?: AltConditionFn[]): SdkFlag {