From db67bc0e509e7e9939b346410d47976c6bded301 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 13 Nov 2025 19:17:16 +0000 Subject: [PATCH] feat: Add support for OpenAI Responses API in tracer Co-authored-by: vinicius --- CHANGELOG.md | 11 + docs/openai-integration.md | 344 +++++++++++++++++++++++++ examples/openai-responses-tracing.ts | 120 +++++++++ package.json | 2 +- src/lib/integrations/openAiTracer.ts | 149 ++++++++++- tests/openai-tracer.test.ts | 368 +++++++++++++++++++++++++++ yarn.lock | 15 -- 7 files changed, 989 insertions(+), 20 deletions(-) create mode 100644 docs/openai-integration.md create mode 100644 examples/openai-responses-tracing.ts create mode 100644 tests/openai-tracer.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index a0dd2533..0bbce3f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## Unreleased + +### Features + +* **integrations:** add support for OpenAI Responses API in tracer + - Added automatic tracing for the new Responses API (`client.responses.create()`) + - Maintained backward compatibility with Chat Completions API + - Support for streaming, function/tool calling, and multi-turn conversations + - Token usage tracking with proper mapping (input_tokens, output_tokens) + - Comprehensive examples and documentation + ## 0.19.0 (2025-10-22) Full Changelog: [v0.18.0...v0.19.0](https://github.com/openlayer-ai/openlayer-ts/compare/v0.18.0...v0.19.0) diff --git a/docs/openai-integration.md b/docs/openai-integration.md new file mode 100644 index 00000000..92dc65a6 --- /dev/null +++ b/docs/openai-integration.md @@ -0,0 +1,344 @@ +# OpenAI Integration Guide + +Openlayer provides seamless integration with OpenAI's APIs through automatic tracing. This guide covers how to use Openlayer with both the **Chat Completions API** (legacy) and the new **Responses API**. + +## Table of Contents + +- [Getting Started](#getting-started) +- [Chat Completions API (Legacy)](#chat-completions-api-legacy) +- [Responses API (New)](#responses-api-new) +- [Streaming Support](#streaming-support) +- [Function/Tool Calling](#functiontool-calling) +- [Migration Guide](#migration-guide) +- [API Reference](#api-reference) + +## Getting Started + +First, install the Openlayer SDK and OpenAI SDK: + +```bash +npm install openlayer openai +``` + +Set up your environment variables: + +```bash +export OPENAI_API_KEY="your-openai-api-key" +export OPENLAYER_API_KEY="your-openlayer-api-key" +export OPENLAYER_INFERENCE_PIPELINE_ID="your-inference-pipeline-id" +``` + +Wrap your OpenAI client with Openlayer's tracer: + +```typescript +import OpenAI from 'openai'; +import { traceOpenAI } from 'openlayer/lib/integrations/openAiTracer'; + +const client = traceOpenAI(new OpenAI()); +``` + +That's it! All requests to both `chat.completions` and `responses` endpoints will now be automatically traced and sent to your Openlayer inference pipeline. + +## Chat Completions API (Legacy) + +The Chat Completions API continues to work exactly as before, with full backward compatibility. + +### Basic Usage + +```typescript +const response = await client.chat.completions.create({ + model: 'gpt-4o', + messages: [{ role: 'user', content: 'Hello, how are you?' }], +}); + +console.log(response.choices[0].message.content); +``` + +### With Streaming + +```typescript +const stream = await client.chat.completions.create({ + model: 'gpt-4o', + messages: [{ role: 'user', content: 'Tell me a story' }], + stream: true, +}); + +for await (const chunk of stream) { + process.stdout.write(chunk.choices[0]?.delta?.content || ''); +} +``` + +### With Function Calling + +```typescript +const response = await client.chat.completions.create({ + model: 'gpt-4o', + messages: [{ role: 'user', content: "What's the weather in San Francisco?" }], + tools: [ + { + type: 'function', + function: { + name: 'get_weather', + description: 'Get the current weather for a location', + parameters: { + type: 'object', + properties: { + location: { type: 'string', description: 'City and state' }, + unit: { type: 'string', enum: ['celsius', 'fahrenheit'] }, + }, + required: ['location'], + }, + }, + }, + ], +}); +``` + +## Responses API (New) + +The Responses API is OpenAI's new unified interface that combines chat, text generation, tool use, and structured outputs under a single endpoint. + +### Basic Usage + +```typescript +const response = await client.responses.create({ + model: 'gpt-4o', + input: 'Hello, how are you?', +}); + +console.log(response.output_text); +``` + +### Key Differences from Chat Completions + +| Feature | Chat Completions API | Responses API | +| ------------------ | ---------------------------------- | --------------------------------- | +| Input parameter | `messages` (array) | `input` (string or object) | +| Output field | `choices[0].message.content` | `output_text` or `output` | +| Token usage | `usage.{prompt,completion}_tokens` | `usage.{input,output}_tokens` | +| Model parameter | `max_tokens` | `max_output_tokens` | +| Streaming endpoint | Same endpoint with `stream: true` | Same endpoint with `stream: true` | + +### With Streaming + +```typescript +const stream = await client.responses.create({ + model: 'gpt-4o', + input: 'Tell me a story', + stream: true, +}); + +for await (const event of stream) { + if (event.type === 'response.output_text.delta' && 'delta' in event) { + process.stdout.write(event.delta); + } +} +``` + +### Multi-turn Conversations + +The Responses API supports stateful conversations using `previous_response_id`: + +```typescript +// First turn +const response1 = await client.responses.create({ + model: 'gpt-4o', + input: 'My name is Alice.', +}); + +// Second turn - continues the conversation +const response2 = await client.responses.create({ + model: 'gpt-4o', + input: 'What is my name?', + previous_response_id: response1.id, +}); + +console.log(response2.output_text); // "Your name is Alice." +``` + +### With Tool Calling + +```typescript +const response = await client.responses.create({ + model: 'gpt-4o', + input: "What's the weather in San Francisco?", + tools: [ + { + type: 'function', + function: { + name: 'get_weather', + description: 'Get the current weather', + parameters: { + type: 'object', + properties: { + location: { type: 'string', description: 'City and state' }, + }, + required: ['location'], + }, + }, + }, + ], +}); + +// Check if the model called a function +const functionCall = response.output.find((item: any) => item.type === 'function_call'); +if (functionCall) { + console.log('Function:', functionCall.function.name); + console.log('Arguments:', functionCall.function.arguments); +} +``` + +## Streaming Support + +Both APIs support streaming with automatic trace collection: + +### Chat Completions Streaming + +Openlayer automatically: + +- Collects all streamed chunks +- Assembles the complete output +- Tracks time to first token +- Records token usage +- Sends the trace to your inference pipeline + +### Responses API Streaming + +Openlayer handles various event types: + +- `response.output_text.delta` - Text content chunks +- `response.function_call_arguments.delta` - Function argument chunks +- `response.completed` - Final response with usage statistics + +## Function/Tool Calling + +Both APIs support function calling with automatic tracing: + +### What Gets Traced + +- **Function name**: The name of the called function +- **Function arguments**: The arguments passed to the function +- **Model parameters**: Tool choice, parallel tool calls settings +- **Execution metadata**: Latency, token usage + +### Tool Call Output Format + +When a tool/function is called, Openlayer formats the output as a JSON string containing: + +- Function name +- Function arguments +- Execution status (if available) + +## Migration Guide + +### Migrating from Chat Completions to Responses API + +If you're using the Chat Completions API and want to migrate to the Responses API: + +1. **Update the endpoint**: Change `client.chat.completions.create()` to `client.responses.create()` + +2. **Update input parameter**: Convert `messages` array to `input` string or object + + ```typescript + // Before (Chat Completions) + const response = await client.chat.completions.create({ + model: 'gpt-4o', + messages: [{ role: 'user', content: 'Hello' }], + }); + + // After (Responses API) + const response = await client.responses.create({ + model: 'gpt-4o', + input: 'Hello', + }); + ``` + +3. **Update output access**: Change from `choices[0].message.content` to `output_text` + + ```typescript + // Before + const text = response.choices[0].message.content; + + // After + const text = response.output_text; + ``` + +4. **Update token field names**: Change `prompt_tokens`/`completion_tokens` to `input_tokens`/`output_tokens` + + ```typescript + // Before + const tokens = response.usage.prompt_tokens + response.usage.completion_tokens; + + // After + const tokens = response.usage.input_tokens + response.usage.output_tokens; + // or simply + const tokens = response.usage.total_tokens; + ``` + +### Why Migrate? + +The Responses API provides: + +- **Unified interface**: Single endpoint for all use cases +- **Better metadata**: Enhanced traceability and structured responses +- **Stateful conversations**: Native support via `previous_response_id` +- **Improved tool support**: Better function calling with detailed execution info +- **Future-proof**: OpenAI's recommended approach going forward + +## API Reference + +### `traceOpenAI(client: OpenAI): OpenAI` + +Wraps an OpenAI client instance to enable automatic tracing for both Chat Completions and Responses APIs. + +**Parameters:** + +- `client`: An initialized OpenAI client instance + +**Returns:** + +- The same OpenAI client with tracing enabled + +**Example:** + +```typescript +import OpenAI from 'openai'; +import { traceOpenAI } from 'openlayer/lib/integrations/openAiTracer'; + +const client = traceOpenAI(new OpenAI()); +``` + +### Traced Data + +For both APIs, Openlayer automatically captures: + +- **Inputs**: The prompt/messages sent to the model +- **Output**: The generated response +- **Model**: The model name used (e.g., `gpt-4o`) +- **Latency**: Total request duration in milliseconds +- **Token usage**: Input tokens, output tokens, and total tokens +- **Model parameters**: Temperature, top_p, max_tokens, etc. +- **Metadata**: Additional context like time to first token (streaming) +- **Provider**: Set to "OpenAI" + +### Environment Variables + +- `OPENAI_API_KEY`: Your OpenAI API key (required by OpenAI SDK) +- `OPENLAYER_API_KEY`: Your Openlayer API key (required) +- `OPENLAYER_INFERENCE_PIPELINE_ID`: Your inference pipeline ID (required) +- `OPENLAYER_DISABLE_PUBLISH`: Set to `"true"` to disable trace publishing (optional) + +## Examples + +See the `examples/` directory for complete working examples: + +- `examples/openai-tracing.ts` - Chat Completions API examples +- `examples/openai-responses-tracing.ts` - Responses API examples + +## Support + +For issues, questions, or feature requests: + +- GitHub: [openlayer-ai/openlayer-ts](https://github.com/openlayer-ai/openlayer-ts) +- Email: support@openlayer.com +- Documentation: [https://docs.openlayer.com](https://docs.openlayer.com) diff --git a/examples/openai-responses-tracing.ts b/examples/openai-responses-tracing.ts new file mode 100644 index 00000000..f071d365 --- /dev/null +++ b/examples/openai-responses-tracing.ts @@ -0,0 +1,120 @@ +import OpenAI from 'openai'; +import { traceOpenAI } from 'openlayer/lib/integrations/openAiTracer'; + +// First, make sure you export your: +// - OPENAI_API_KEY +// - OPENLAYER_API_KEY +// - OPENLAYER_INFERENCE_PIPELINE_ID +// as environment variables. + +// Then, wrap the OpenAI client with Openlayer's traceOpenAI +const client = traceOpenAI(new OpenAI() as any); + +// Example 1: Non-streaming response with the Responses API +async function basicResponseExample(input: string): Promise { + const response = await client.responses.create({ + model: 'gpt-4o', + input: input, + }); + return response?.output_text ?? ''; +} + +// Example 2: Streaming response with the Responses API +async function streamingResponseExample(input: string): Promise { + const stream = await client.responses.create({ + model: 'gpt-4o', + input: input, + stream: true, + }); + + for await (const chunk of stream) { + if (chunk.type === 'response.output_text.delta' && 'delta' in chunk) { + process.stdout.write(chunk.delta); + } + } + console.log('\n'); +} + +// Example 3: Response with tool calls (function calling) +async function toolCallExample(input: string): Promise { + const response = await client.responses.create({ + model: 'gpt-4o', + input: input, + tools: [ + { + type: 'function', + function: { + name: 'get_weather', + description: 'Get the current weather for a location', + parameters: { + type: 'object', + properties: { + location: { + type: 'string', + description: 'The city and state, e.g. San Francisco, CA', + }, + unit: { + type: 'string', + enum: ['celsius', 'fahrenheit'], + description: 'The temperature unit', + }, + }, + required: ['location'], + }, + }, + }, + ], + }); + + console.log('Response:', response.output_text); + console.log('Output items:', response.output); +} + +// Example 4: Multi-turn conversation using previous_response_id +async function multiTurnExample(): Promise { + // First turn + const response1 = await client.responses.create({ + model: 'gpt-4o', + input: 'My name is Alice. What is my name?', + }); + console.log('First response:', response1.output_text); + + // Second turn - continues the conversation + const response2 = await client.responses.create({ + model: 'gpt-4o', + input: 'What did I just tell you?', + previous_response_id: response1.id, + }); + console.log('Second response:', response2.output_text); +} + +// Example 5: Backwards compatibility - Chat Completions API still works +async function chatCompletionExample(input: string): Promise { + const response = await client.chat.completions.create({ + model: 'gpt-4o', + messages: [{ role: 'user', content: input }], + }); + return response?.choices[0]?.message?.content ?? null; +} + +// Run examples +async function main() { + console.log('=== Example 1: Basic Response ==='); + const result1 = await basicResponseExample('Hello, how are you?'); + console.log('Result:', result1); + + console.log('\n=== Example 2: Streaming Response ==='); + await streamingResponseExample('Tell me a short joke.'); + + console.log('\n=== Example 3: Tool Call Example ==='); + await toolCallExample("What's the weather like in San Francisco?"); + + console.log('\n=== Example 4: Multi-turn Conversation ==='); + await multiTurnExample(); + + console.log('\n=== Example 5: Chat Completions (Backwards Compatibility) ==='); + const result5 = await chatCompletionExample('Say hello!'); + console.log('Result:', result5); +} + +main().catch(console.error); diff --git a/package.json b/package.json index 912a727b..cb863b0c 100644 --- a/package.json +++ b/package.json @@ -78,4 +78,4 @@ "require": "./dist/*.js" } } -} \ No newline at end of file +} diff --git a/src/lib/integrations/openAiTracer.ts b/src/lib/integrations/openAiTracer.ts index 5666c0b0..5dd94897 100644 --- a/src/lib/integrations/openAiTracer.ts +++ b/src/lib/integrations/openAiTracer.ts @@ -2,13 +2,15 @@ import OpenAI from 'openai'; import { Stream } from 'openai/streaming'; import performanceNow from 'performance-now'; import { addChatCompletionStepToTrace } from '../tracing/tracer'; +import type { ResponseStreamEvent } from 'openai/resources/responses/responses'; export function traceOpenAI(openai: OpenAI): OpenAI { - const createFunction = openai.chat.completions.create; + // Wrap Chat Completions API + const chatCreateFunction = openai.chat.completions.create; openai.chat.completions.create = async function ( this: typeof openai.chat.completions, - ...args: Parameters + ...args: Parameters ): Promise | OpenAI.Chat.Completions.ChatCompletion> { const [params, options] = args; const stream = params?.stream ?? false; @@ -17,7 +19,7 @@ export function traceOpenAI(openai: OpenAI): OpenAI { const startTime = performanceNow(); // Call the original `create` function - let response = await createFunction.apply(this, args); + let response = await chatCreateFunction.apply(this, args); if (stream) { // Handle streaming responses @@ -114,7 +116,135 @@ export function traceOpenAI(openai: OpenAI): OpenAI { } // Ensure a return statement is present return undefined as any; - } as typeof createFunction; + } as typeof chatCreateFunction; + + // Wrap Responses API + const responsesCreateFunction = openai.responses.create; + + openai.responses.create = async function ( + this: typeof openai.responses, + ...args: Parameters + ): Promise | OpenAI.Responses.Response> { + const [params, options] = args; + const stream = params?.stream ?? false; + + try { + const startTime = performanceNow(); + + // Call the original `create` function + let response = await responsesCreateFunction.apply(this, args); + + if (stream) { + // Handle streaming responses + const chunks: ResponseStreamEvent[] = []; + let collectedOutputData: any[] = []; + let firstTokenTime: number | undefined; + let completionTokens: number = 0; + if (isAsyncIterable(response)) { + async function* tracedOutputGenerator(): AsyncGenerator { + for await (const rawChunk of response as AsyncIterable) { + if (chunks.length === 0) { + firstTokenTime = performanceNow(); + } + chunks.push(rawChunk); + + // Handle different event types + if (rawChunk.type === 'response.output_text.delta') { + if ('delta' in rawChunk && rawChunk.delta) { + collectedOutputData.push(rawChunk.delta); + } + } else if (rawChunk.type === 'response.function_call_arguments.delta') { + if ('delta' in rawChunk && rawChunk.delta) { + collectedOutputData.push(rawChunk.delta); + } + } else if (rawChunk.type === 'response.completed') { + // Extract final response data + if ('response' in rawChunk && rawChunk.response) { + const finalResponse = rawChunk.response; + completionTokens = finalResponse.usage?.output_tokens ?? 0; + } + } + + yield rawChunk; + } + const endTime = performanceNow(); + + // Find the final response from chunks + const doneChunk = chunks.find((c) => c.type === 'response.completed'); + const finalResponse = doneChunk && 'response' in doneChunk ? (doneChunk as any).response : null; + + const traceData = { + name: 'OpenAI Response', + inputs: { prompt: params.input }, + output: collectedOutputData.join(''), + latency: endTime - startTime, + model: finalResponse?.model ?? params.model, + modelParameters: getResponsesModelParameters(args), + metadata: { timeToFistToken: firstTokenTime ? firstTokenTime - startTime : null }, + provider: 'OpenAI', + completionTokens: finalResponse?.usage?.output_tokens ?? completionTokens, + promptTokens: finalResponse?.usage?.input_tokens ?? 0, + tokens: finalResponse?.usage?.total_tokens ?? completionTokens, + startTime: startTime, + endTime: endTime, + }; + addChatCompletionStepToTrace(traceData); + } + return tracedOutputGenerator() as unknown as Stream; + } + } else { + // Handle non-streaming responses + response = response as OpenAI.Responses.Response; + const endTime = performanceNow(); + + // Extract output from the response + let output: string = response.output_text ?? ''; + + // If output_text is empty, try to extract from output array + if (!output && response.output && response.output.length > 0) { + const firstOutput = response.output[0]; + if (firstOutput && 'type' in firstOutput) { + if (firstOutput.type === 'message' && 'content' in firstOutput) { + // Extract text from message content + const content = firstOutput.content; + if (Array.isArray(content)) { + output = content + .filter((c: any) => c.type === 'output_text') + .map((c: any) => c.text) + .join(''); + } + } else if (firstOutput.type === 'function_call' && 'function' in firstOutput) { + // Extract function call details + output = JSON.stringify(firstOutput.function, null, 2); + } + } + } + + const traceData = { + name: 'OpenAI Response', + inputs: { prompt: params.input }, + output: output, + latency: endTime - startTime, + tokens: response.usage?.total_tokens ?? null, + promptTokens: response.usage?.input_tokens ?? null, + completionTokens: response.usage?.output_tokens ?? null, + model: response.model, + modelParameters: getResponsesModelParameters(args), + metadata: {}, + provider: 'OpenAI', + startTime: startTime, + endTime: endTime, + }; + addChatCompletionStepToTrace(traceData); + return response; + } + } catch (error) { + console.error('Failed to trace the create response request with Openlayer', error); + throw error; + } + // Ensure a return statement is present + return undefined as any; + } as typeof responsesCreateFunction; return openai; } @@ -136,5 +266,16 @@ function getModelParameters(args: any): Record { }; } +function getResponsesModelParameters(args: any): Record { + const params = args[0]; + return { + max_output_tokens: params?.max_output_tokens ?? null, + temperature: params?.temperature ?? null, + top_p: params?.top_p ?? null, + parallel_tool_calls: params?.parallel_tool_calls ?? null, + tool_choice: params?.tool_choice ?? null, + }; +} + const isAsyncIterable = (x: any) => x != null && typeof x === 'object' && typeof x[Symbol.asyncIterator] === 'function'; diff --git a/tests/openai-tracer.test.ts b/tests/openai-tracer.test.ts new file mode 100644 index 00000000..493759e8 --- /dev/null +++ b/tests/openai-tracer.test.ts @@ -0,0 +1,368 @@ +import OpenAI from 'openai'; +import { traceOpenAI } from '../src/lib/integrations/openAiTracer'; +import { addChatCompletionStepToTrace } from '../src/lib/tracing/tracer'; + +// Mock the tracer module +jest.mock('../src/lib/tracing/tracer', () => ({ + addChatCompletionStepToTrace: jest.fn(), +})); + +describe('OpenAI Tracer', () => { + let mockOpenAI: any; + let addChatCompletionStepToTraceMock: jest.Mock; + + beforeEach(() => { + jest.clearAllMocks(); + addChatCompletionStepToTraceMock = addChatCompletionStepToTrace as jest.Mock; + + // Create a mock OpenAI client + mockOpenAI = { + chat: { + completions: { + create: jest.fn(), + }, + }, + responses: { + create: jest.fn(), + }, + }; + }); + + describe('Chat Completions API (Backwards Compatibility)', () => { + it('should trace non-streaming chat completion requests', async () => { + const mockResponse: OpenAI.Chat.Completions.ChatCompletion = { + id: 'chatcmpl-123', + object: 'chat.completion', + created: 1677652288, + model: 'gpt-4o', + choices: [ + { + index: 0, + message: { + role: 'assistant', + content: 'Hello! How can I assist you today?', + refusal: null, + }, + logprobs: null, + finish_reason: 'stop', + }, + ], + usage: { + prompt_tokens: 10, + completion_tokens: 20, + total_tokens: 30, + prompt_tokens_details: { cached_tokens: 0 }, + completion_tokens_details: { reasoning_tokens: 0 }, + }, + }; + + mockOpenAI.chat.completions.create.mockResolvedValue(mockResponse); + + const tracedClient = traceOpenAI(mockOpenAI); + const result = await tracedClient.chat.completions.create({ + model: 'gpt-4o', + messages: [{ role: 'user', content: 'Hello' }], + }); + + expect(result).toEqual(mockResponse); + expect(addChatCompletionStepToTraceMock).toHaveBeenCalledWith( + expect.objectContaining({ + name: 'OpenAI Chat Completion', + output: 'Hello! How can I assist you today?', + model: 'gpt-4o', + provider: 'OpenAI', + tokens: 30, + promptTokens: 10, + completionTokens: 20, + }), + ); + }); + + it('should trace chat completion with tool calls', async () => { + const mockResponse: OpenAI.Chat.Completions.ChatCompletion = { + id: 'chatcmpl-123', + object: 'chat.completion', + created: 1677652288, + model: 'gpt-4o', + choices: [ + { + index: 0, + message: { + role: 'assistant', + content: null, + refusal: null, + tool_calls: [ + { + id: 'call_123', + type: 'function', + function: { + name: 'get_weather', + arguments: '{"location": "San Francisco"}', + }, + }, + ], + }, + logprobs: null, + finish_reason: 'tool_calls', + }, + ], + usage: { + prompt_tokens: 15, + completion_tokens: 25, + total_tokens: 40, + prompt_tokens_details: { cached_tokens: 0 }, + completion_tokens_details: { reasoning_tokens: 0 }, + }, + }; + + mockOpenAI.chat.completions.create.mockResolvedValue(mockResponse); + + const tracedClient = traceOpenAI(mockOpenAI); + await tracedClient.chat.completions.create({ + model: 'gpt-4o', + messages: [{ role: 'user', content: "What's the weather in SF?" }], + tools: [ + { + type: 'function', + function: { + name: 'get_weather', + description: 'Get weather', + parameters: { + type: 'object', + properties: { location: { type: 'string' } }, + required: ['location'], + }, + }, + }, + ], + }); + + expect(addChatCompletionStepToTraceMock).toHaveBeenCalledWith( + expect.objectContaining({ + name: 'OpenAI Chat Completion', + provider: 'OpenAI', + output: expect.stringContaining('get_weather'), + }), + ); + }); + }); + + describe('Responses API', () => { + it('should trace non-streaming response requests', async () => { + const mockResponse: OpenAI.Responses.Response = { + id: 'resp-123', + object: 'response', + created_at: 1677652288, + model: 'gpt-4o', + output_text: 'Hello! How can I help you today?', + output: [ + { + type: 'message', + role: 'assistant', + status: 'completed', + content: [ + { + type: 'output_text', + text: 'Hello! How can I help you today?', + }, + ], + } as any, + ], + usage: { + input_tokens: 10, + output_tokens: 20, + total_tokens: 30, + input_tokens_details: { cached_tokens: 0 }, + output_tokens_details: { reasoning_tokens: 0 }, + }, + instructions: null, + metadata: null, + temperature: 1, + top_p: 1, + parallel_tool_calls: false, + tool_choice: 'auto', + tools: [], + error: null, + incomplete_details: null, + } as any; + + mockOpenAI.responses.create.mockResolvedValue(mockResponse); + + const tracedClient = traceOpenAI(mockOpenAI); + const result = await tracedClient.responses.create({ + model: 'gpt-4o', + input: 'Hello', + }); + + expect(result).toEqual(mockResponse); + expect(addChatCompletionStepToTraceMock).toHaveBeenCalledWith( + expect.objectContaining({ + name: 'OpenAI Response', + output: 'Hello! How can I help you today?', + model: 'gpt-4o', + provider: 'OpenAI', + tokens: 30, + promptTokens: 10, + completionTokens: 20, + }), + ); + }); + + it('should handle streaming response requests', async () => { + const mockStreamEvents = [ + { type: 'response.created', sequence_number: 0 }, + { + type: 'response.output_text.delta', + delta: 'Hello', + sequence_number: 1, + }, + { + type: 'response.output_text.delta', + delta: ' there!', + sequence_number: 2, + }, + { + type: 'response.completed', + response: { + id: 'resp-123', + model: 'gpt-4o', + usage: { + input_tokens: 5, + output_tokens: 10, + total_tokens: 15, + }, + }, + sequence_number: 3, + }, + ]; + + async function* mockStreamGenerator() { + for (const event of mockStreamEvents) { + yield event; + } + } + + mockOpenAI.responses.create.mockResolvedValue(mockStreamGenerator()); + + const tracedClient = traceOpenAI(mockOpenAI); + const stream = await tracedClient.responses.create({ + model: 'gpt-4o', + input: 'Hello', + stream: true, + }); + + const chunks: any[] = []; + for await (const chunk of stream) { + chunks.push(chunk); + } + + expect(chunks.length).toBe(4); + expect(addChatCompletionStepToTraceMock).toHaveBeenCalledWith( + expect.objectContaining({ + name: 'OpenAI Response', + output: 'Hello there!', + provider: 'OpenAI', + model: 'gpt-4o', + tokens: 15, + }), + ); + }); + + it('should handle response with function calls', async () => { + const mockResponse: OpenAI.Responses.Response = { + id: 'resp-123', + object: 'response', + created_at: 1677652288, + model: 'gpt-4o', + output_text: '', + output: [ + { + type: 'function_call', + id: 'call_123', + status: 'completed', + function: { + name: 'get_weather', + arguments: '{"location": "San Francisco"}', + }, + } as any, + ], + usage: { + input_tokens: 15, + output_tokens: 25, + total_tokens: 40, + input_tokens_details: { cached_tokens: 0 }, + output_tokens_details: { reasoning_tokens: 0 }, + }, + instructions: null, + metadata: null, + temperature: 1, + top_p: 1, + parallel_tool_calls: false, + tool_choice: 'auto', + tools: [], + error: null, + incomplete_details: null, + } as any; + + mockOpenAI.responses.create.mockResolvedValue(mockResponse); + + const tracedClient = traceOpenAI(mockOpenAI); + await tracedClient.responses.create({ + model: 'gpt-4o', + input: "What's the weather in San Francisco?", + tools: [ + { + type: 'function', + function: { + name: 'get_weather', + description: 'Get weather', + parameters: { + type: 'object', + properties: { location: { type: 'string' } }, + required: ['location'], + }, + }, + } as any, + ], + }); + + expect(addChatCompletionStepToTraceMock).toHaveBeenCalledWith( + expect.objectContaining({ + name: 'OpenAI Response', + provider: 'OpenAI', + output: expect.stringContaining('get_weather'), + }), + ); + }); + }); + + describe('Error Handling', () => { + it('should handle errors in chat completions gracefully', async () => { + const error = new Error('API Error'); + mockOpenAI.chat.completions.create.mockRejectedValue(error); + + const tracedClient = traceOpenAI(mockOpenAI); + + await expect( + tracedClient.chat.completions.create({ + model: 'gpt-4o', + messages: [{ role: 'user', content: 'Hello' }], + }), + ).rejects.toThrow('API Error'); + }); + + it('should handle errors in responses gracefully', async () => { + const error = new Error('API Error'); + mockOpenAI.responses.create.mockRejectedValue(error); + + const tracedClient = traceOpenAI(mockOpenAI); + + await expect( + tracedClient.responses.create({ + model: 'gpt-4o', + input: 'Hello', + }), + ).rejects.toThrow('API Error'); + }); + }); +}); diff --git a/yarn.lock b/yarn.lock index 2905b8f6..8f6e41e8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1935,11 +1935,6 @@ "@smithy/util-buffer-from" "^4.2.0" tslib "^2.6.2" -"@swc/core-darwin-arm64@1.13.3": - version "1.13.3" - resolved "https://registry.npmjs.org/@swc/core-darwin-arm64/-/core-darwin-arm64-1.13.3.tgz" - integrity sha512-ux0Ws4pSpBTqbDS9GlVP354MekB1DwYlbxXU3VhnDr4GBcCOimpocx62x7cFJkSpEBF8bmX8+/TTCGKh4PbyXw== - "@swc/core@*", "@swc/core@^1.3.102", "@swc/core@>=1.2.50": version "1.13.3" resolved "https://registry.npmjs.org/@swc/core/-/core-1.13.3.tgz" @@ -3187,16 +3182,6 @@ fs.realpath@^1.0.0: resolved "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz" integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw== -fsevents@^2.3.2: - version "2.3.3" - resolved "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz" - integrity sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw== - -fsevents@2.3.2: - version "2.3.2" - resolved "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz" - integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA== - function-bind@^1.1.2: version "1.1.2" resolved "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz"