diff --git a/.github/workflows/anthropic-plugin-tests.yml b/.github/workflows/anthropic-plugin-tests.yml new file mode 100644 index 0000000000..93507d5eb3 --- /dev/null +++ b/.github/workflows/anthropic-plugin-tests.yml @@ -0,0 +1,56 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +name: Anthropic Plugin Tests + +on: pull_request + +env: + GITHUB_PULL_REQUEST_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + GITHUB_PULL_REQUEST_BASE_SHA: ${{ github.event.pull_request.base.sha }} + +jobs: + test-anthropic-plugin: + name: Run Anthropic Plugin Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: pnpm/action-setup@v3 + - name: Set up node v20 + uses: actions/setup-node@v4 + with: + node-version: 20.x + cache: pnpm + - name: Install dependencies + run: | + cd js + pnpm install + - name: Build core dependencies + run: | + cd js + pnpm -r --workspace-concurrency 1 -F core -F ai build + - name: Build genkit + run: | + cd js + pnpm -F genkit build + - name: Build Anthropic plugin + run: | + cd js/plugins/anthropic + pnpm build + - name: Run Anthropic plugin tests + run: | + cd js/plugins/anthropic + pnpm test diff --git a/js/plugins/anthropic/tests/beta_runner_test.ts b/js/plugins/anthropic/tests/beta_runner_test.ts new file mode 100644 index 0000000000..655bfc599e --- /dev/null +++ b/js/plugins/anthropic/tests/beta_runner_test.ts @@ -0,0 +1,804 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as assert from 'assert'; +import type { Part } from 'genkit'; +import { describe, it } from 'node:test'; + +import { BetaRunner } from '../src/runner/beta.js'; +import { createMockAnthropicClient } from './mocks/anthropic-client.js'; + +describe('BetaRunner.toAnthropicMessageContent', () => { + function createRunner() { + return new BetaRunner({ + name: 'anthropic/claude-3-5-haiku', + client: createMockAnthropicClient(), + cacheSystemPrompt: false, + }); + } + + it('converts PDF media parts into document blocks', () => { + const runner = createRunner(); + const part: Part = { + media: { + contentType: 'application/pdf', + url: 'data:application/pdf;base64,UEsDBAoAAAAAAD', + }, + }; + + const result = (runner as any).toAnthropicMessageContent(part); + + assert.strictEqual(result.type, 'document'); + assert.ok(result.source); + assert.strictEqual(result.source.type, 'base64'); + assert.strictEqual(result.source.media_type, 'application/pdf'); + assert.ok(result.source.data); + }); + + it('throws when tool request ref is missing', () => { + const runner = createRunner(); + const part: Part = { + toolRequest: { + name: 'do_something', + input: { foo: 'bar' }, + }, + }; + + assert.throws(() => { + (runner as any).toAnthropicMessageContent(part); + }, /Tool request ref is required/); + }); + + it('maps tool request with ref into tool_use block', () => { + const runner = createRunner(); + const part: Part = { + toolRequest: { + ref: 'tool-123', + name: 'do_something', + input: { foo: 'bar' }, + }, + }; + + const result = (runner as any).toAnthropicMessageContent(part); + + assert.strictEqual(result.type, 'tool_use'); + assert.strictEqual(result.id, 'tool-123'); + assert.strictEqual(result.name, 'do_something'); + assert.deepStrictEqual(result.input, { foo: 'bar' }); + }); + + it('throws when tool response ref is missing', () => { + const runner = createRunner(); + const part: Part = { + toolResponse: { + name: 'do_something', + output: 'done', + }, + }; + + assert.throws(() => { + (runner as any).toAnthropicMessageContent(part); + }, /Tool response ref is required/); + }); + + it('maps tool response into tool_result block containing text response', () => { + const runner = createRunner(); + const part: Part = { + toolResponse: { + name: 'do_something', + ref: 'tool-abc', + output: 'done', + }, + }; + + const result = (runner as any).toAnthropicMessageContent(part); + + assert.strictEqual(result.type, 'tool_result'); + assert.strictEqual(result.tool_use_id, 'tool-abc'); + assert.deepStrictEqual(result.content, [{ type: 'text', text: 'done' }]); + }); + + it('should handle WEBP image data URLs', () => { + const runner = createRunner(); + const part: Part = { + media: { + contentType: 'image/webp', + url: 'data:image/webp;base64,AAA', + }, + }; + + const result = (runner as any).toAnthropicMessageContent(part); + + assert.strictEqual(result.type, 'image'); + assert.strictEqual(result.source.type, 'base64'); + assert.strictEqual(result.source.media_type, 'image/webp'); + assert.strictEqual(result.source.data, 'AAA'); + }); + + it('should prefer data URL content type over media.contentType for WEBP', () => { + const runner = createRunner(); + const part: Part = { + media: { + // Even if contentType says PNG, data URL says WEBP - should use WEBP + contentType: 'image/png', + url: 'data:image/webp;base64,AAA', + }, + }; + + const result = (runner as any).toAnthropicMessageContent(part); + + assert.strictEqual(result.type, 'image'); + assert.strictEqual(result.source.type, 'base64'); + // Key fix: should use data URL type (webp), not contentType (png) + assert.strictEqual(result.source.media_type, 'image/webp'); + assert.strictEqual(result.source.data, 'AAA'); + }); + + it('should throw helpful error for text/plain in toAnthropicMessageContent', () => { + const runner = createRunner(); + const part: Part = { + media: { + contentType: 'text/plain', + url: 'data:text/plain;base64,AAA', + }, + }; + + assert.throws( + () => { + (runner as any).toAnthropicMessageContent(part); + }, + (error: Error) => { + return ( + error.message.includes('Text files should be sent as text content') && + error.message.includes('text:') + ); + } + ); + }); + + it('should throw helpful error for text/plain with remote URL', () => { + const runner = createRunner(); + const part: Part = { + media: { + contentType: 'text/plain', + url: 'https://example.com/file.txt', + }, + }; + + assert.throws( + () => { + (runner as any).toAnthropicMessageContent(part); + }, + (error: Error) => { + return ( + error.message.includes('Text files should be sent as text content') && + error.message.includes('text:') + ); + } + ); + }); + + it('should throw helpful error for text/plain in tool response', () => { + const runner = createRunner(); + const part: Part = { + toolResponse: { + ref: 'call_123', + name: 'get_file', + output: { + url: 'data:text/plain;base64,AAA', + contentType: 'text/plain', + }, + }, + }; + + assert.throws( + () => { + (runner as any).toAnthropicToolResponseContent(part); + }, + (error: Error) => { + return error.message.includes( + 'Text files should be sent as text content' + ); + } + ); + }); +}); +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Anthropic } from '@anthropic-ai/sdk'; +import { mock } from 'node:test'; + +describe('BetaRunner', () => { + it('should map all supported Part shapes to beta content blocks', () => { + const mockClient = createMockAnthropicClient(); + const runner = new BetaRunner({ + name: 'claude-test', + client: mockClient as Anthropic, + }); + + const exposed = runner as any; + + const textPart = exposed.toAnthropicMessageContent({ + text: 'Hello', + } as any); + assert.deepStrictEqual(textPart, { type: 'text', text: 'Hello' }); + + const pdfPart = exposed.toAnthropicMessageContent({ + media: { + url: 'data:application/pdf;base64,JVBERi0xLjQKJ', + contentType: 'application/pdf', + }, + } as any); + assert.strictEqual(pdfPart.type, 'document'); + + const imagePart = exposed.toAnthropicMessageContent({ + media: { + url: 'data:image/png;base64,AAA', + contentType: 'image/png', + }, + } as any); + assert.strictEqual(imagePart.type, 'image'); + + const toolUsePart = exposed.toAnthropicMessageContent({ + toolRequest: { + ref: 'tool1', + name: 'get_weather', + input: { city: 'NYC' }, + }, + } as any); + assert.deepStrictEqual(toolUsePart, { + type: 'tool_use', + id: 'tool1', + name: 'get_weather', + input: { city: 'NYC' }, + }); + + const toolResultPart = exposed.toAnthropicMessageContent({ + toolResponse: { + ref: 'tool1', + name: 'get_weather', + output: 'Sunny', + }, + } as any); + assert.strictEqual(toolResultPart.type, 'tool_result'); + + assert.throws(() => exposed.toAnthropicMessageContent({} as any)); + }); + + it('should convert beta stream events to Genkit Parts', () => { + const mockClient = createMockAnthropicClient(); + const runner = new BetaRunner({ + name: 'claude-test', + client: mockClient as Anthropic, + }); + + const exposed = runner as any; + const textPart = exposed.toGenkitPart({ + type: 'content_block_start', + index: 0, + content_block: { type: 'text', text: 'hi' }, + } as any); + assert.deepStrictEqual(textPart, { text: 'hi' }); + + const serverToolEvent = { + type: 'content_block_start', + index: 0, + content_block: { + type: 'server_tool_use', + id: 'toolu_test', + name: 'myTool', + input: { foo: 'bar' }, + server_name: 'srv', + }, + } as any; + const toolPart = exposed.toGenkitPart(serverToolEvent); + assert.deepStrictEqual(toolPart, { + text: '[Anthropic server tool srv/myTool] input: {"foo":"bar"}', + custom: { + anthropicServerToolUse: { + id: 'toolu_test', + name: 'srv/myTool', + input: { foo: 'bar' }, + }, + }, + }); + + const deltaPart = exposed.toGenkitPart({ + type: 'content_block_delta', + index: 0, + delta: { type: 'thinking_delta', thinking: 'hmm' }, + } as any); + assert.deepStrictEqual(deltaPart, { reasoning: 'hmm' }); + + const ignored = exposed.toGenkitPart({ type: 'message_stop' } as any); + assert.strictEqual(ignored, undefined); + }); + + it('should throw on unsupported mcp tool stream events', () => { + const mockClient = createMockAnthropicClient(); + const runner = new BetaRunner({ + name: 'claude-test', + client: mockClient as Anthropic, + }); + + const exposed = runner as any; + assert.throws( + () => + exposed.toGenkitPart({ + type: 'content_block_start', + index: 0, + content_block: { + type: 'mcp_tool_use', + id: 'toolu_unsupported', + input: {}, + }, + }), + /server-managed tool block 'mcp_tool_use'/ + ); + }); + + it('should map beta stop reasons correctly', () => { + const mockClient = createMockAnthropicClient(); + const runner = new BetaRunner({ + name: 'claude-test', + client: mockClient as Anthropic, + }); + + const finishReason = runner['fromBetaStopReason']( + 'model_context_window_exceeded' + ); + assert.strictEqual(finishReason, 'length'); + + const pauseReason = runner['fromBetaStopReason']('pause_turn'); + assert.strictEqual(pauseReason, 'stop'); + }); + + it('should execute streaming calls and surface errors', async () => { + const streamError = new Error('stream failed'); + const mockClient = createMockAnthropicClient({ + streamChunks: [ + { + type: 'content_block_start', + index: 0, + content_block: { type: 'text', text: 'hi' }, + } as any, + ], + streamErrorAfterChunk: 1, + streamError, + }); + + const runner = new BetaRunner({ + name: 'claude-test', + client: mockClient as Anthropic, + }); + const sendChunk = mock.fn(); + await assert.rejects(async () => + runner.run({ messages: [] } as any, { + streamingRequested: true, + sendChunk, + abortSignal: new AbortController().signal, + }) + ); + assert.strictEqual(sendChunk.mock.calls.length, 1); + + const abortController = new AbortController(); + abortController.abort(); + await assert.rejects(async () => + runner.run({ messages: [] } as any, { + streamingRequested: true, + sendChunk: () => {}, + abortSignal: abortController.signal, + }) + ); + }); + + it('should throw when tool refs are missing in message content', () => { + const mockClient = createMockAnthropicClient(); + const runner = new BetaRunner({ + name: 'claude-test', + client: mockClient as Anthropic, + }); + const exposed = runner as any; + + assert.throws(() => + exposed.toAnthropicMessageContent({ + toolRequest: { + name: 'get_weather', + input: {}, + }, + } as any) + ); + + assert.throws(() => + exposed.toAnthropicMessageContent({ + toolResponse: { + name: 'get_weather', + output: 'ok', + }, + } as any) + ); + + assert.throws(() => + exposed.toAnthropicMessageContent({ + media: { + url: 'data:image/png;base64,', + contentType: undefined, + }, + } as any) + ); + }); + + it('should build request bodies with optional config fields', () => { + const mockClient = createMockAnthropicClient(); + const runner = new BetaRunner({ + name: 'claude-3-5-haiku', + client: mockClient as Anthropic, + cacheSystemPrompt: true, + }) as any; + + const request = { + messages: [ + { + role: 'system', + content: [{ text: 'You are helpful.' }], + }, + { + role: 'user', + content: [{ text: 'Tell me a joke' }], + }, + ], + config: { + maxOutputTokens: 128, + topK: 4, + topP: 0.65, + temperature: 0.55, + stopSequences: ['DONE'], + metadata: { user_id: 'beta-user' }, + tool_choice: { type: 'tool', name: 'get_weather' }, + thinking: { enabled: true, budgetTokens: 2048 }, + }, + tools: [ + { + name: 'get_weather', + description: 'Returns the weather', + inputSchema: { type: 'object' }, + }, + ], + } satisfies any; + + const body = runner.toAnthropicRequestBody( + 'claude-3-5-haiku', + request, + true + ); + + assert.strictEqual(body.model, 'claude-3-5-haiku'); + assert.ok(Array.isArray(body.system)); + assert.strictEqual(body.max_tokens, 128); + assert.strictEqual(body.top_k, 4); + assert.strictEqual(body.top_p, 0.65); + assert.strictEqual(body.temperature, 0.55); + assert.deepStrictEqual(body.stop_sequences, ['DONE']); + assert.deepStrictEqual(body.metadata, { user_id: 'beta-user' }); + assert.deepStrictEqual(body.tool_choice, { + type: 'tool', + name: 'get_weather', + }); + assert.strictEqual(body.tools?.length, 1); + assert.deepStrictEqual(body.thinking, { + type: 'enabled', + budget_tokens: 2048, + }); + + const streamingBody = runner.toAnthropicStreamingRequestBody( + 'claude-3-5-haiku', + request, + true + ); + assert.strictEqual(streamingBody.stream, true); + assert.ok(Array.isArray(streamingBody.system)); + assert.deepStrictEqual(streamingBody.thinking, { + type: 'enabled', + budget_tokens: 2048, + }); + + const disabledBody = runner.toAnthropicRequestBody( + 'claude-3-5-haiku', + { + messages: [], + config: { + thinking: { enabled: false }, + }, + } satisfies any, + false + ); + assert.deepStrictEqual(disabledBody.thinking, { type: 'disabled' }); + }); + + it('should concatenate multiple text parts in system message', () => { + const mockClient = createMockAnthropicClient(); + const runner = new BetaRunner({ + name: 'claude-3-5-haiku', + client: mockClient as Anthropic, + }) as any; + + const request = { + messages: [ + { + role: 'system', + content: [ + { text: 'You are a helpful assistant.' }, + { text: 'Always be concise.' }, + { text: 'Use proper grammar.' }, + ], + }, + { role: 'user', content: [{ text: 'Hi' }] }, + ], + output: { format: 'text' }, + } satisfies any; + + const body = runner.toAnthropicRequestBody( + 'claude-3-5-haiku', + request, + false + ); + + assert.strictEqual( + body.system, + 'You are a helpful assistant.\n\nAlways be concise.\n\nUse proper grammar.' + ); + }); + + it('should concatenate multiple text parts in system message with caching', () => { + const mockClient = createMockAnthropicClient(); + const runner = new BetaRunner({ + name: 'claude-3-5-haiku', + client: mockClient as Anthropic, + }) as any; + + const request = { + messages: [ + { + role: 'system', + content: [ + { text: 'You are a helpful assistant.' }, + { text: 'Always be concise.' }, + ], + }, + { role: 'user', content: [{ text: 'Hi' }] }, + ], + output: { format: 'text' }, + } satisfies any; + + const body = runner.toAnthropicRequestBody( + 'claude-3-5-haiku', + request, + true + ); + + assert.ok(Array.isArray(body.system)); + assert.deepStrictEqual(body.system, [ + { + type: 'text', + text: 'You are a helpful assistant.\n\nAlways be concise.', + cache_control: { type: 'ephemeral' }, + }, + ]); + }); + + it('should throw error if system message contains media', () => { + const mockClient = createMockAnthropicClient(); + const runner = new BetaRunner({ + name: 'claude-3-5-haiku', + client: mockClient as Anthropic, + }) as any; + + const request = { + messages: [ + { + role: 'system', + content: [ + { text: 'You are a helpful assistant.' }, + { + media: { + url: 'data:image/png;base64,iVBORw0KGgoAAAANS', + contentType: 'image/png', + }, + }, + ], + }, + { role: 'user', content: [{ text: 'Hi' }] }, + ], + output: { format: 'text' }, + } satisfies any; + + assert.throws( + () => runner.toAnthropicRequestBody('claude-3-5-haiku', request, false), + /System messages can only contain text content/ + ); + }); + + it('should throw error if system message contains tool requests', () => { + const mockClient = createMockAnthropicClient(); + const runner = new BetaRunner({ + name: 'claude-3-5-haiku', + client: mockClient as Anthropic, + }) as any; + + const request = { + messages: [ + { + role: 'system', + content: [ + { text: 'You are a helpful assistant.' }, + { toolRequest: { name: 'getTool', input: {}, ref: '123' } }, + ], + }, + { role: 'user', content: [{ text: 'Hi' }] }, + ], + output: { format: 'text' }, + } satisfies any; + + assert.throws( + () => runner.toAnthropicRequestBody('claude-3-5-haiku', request, false), + /System messages can only contain text content/ + ); + }); + + it('should throw error if system message contains tool responses', () => { + const mockClient = createMockAnthropicClient(); + const runner = new BetaRunner({ + name: 'claude-3-5-haiku', + client: mockClient as Anthropic, + }) as any; + + const request = { + messages: [ + { + role: 'system', + content: [ + { text: 'You are a helpful assistant.' }, + { toolResponse: { name: 'getTool', output: {}, ref: '123' } }, + ], + }, + { role: 'user', content: [{ text: 'Hi' }] }, + ], + output: { format: 'text' }, + } satisfies any; + + assert.throws( + () => runner.toAnthropicRequestBody('claude-3-5-haiku', request, false), + /System messages can only contain text content/ + ); + }); + + it('should throw for unsupported mcp tool use blocks', () => { + const mockClient = createMockAnthropicClient(); + const runner = new BetaRunner({ + name: 'claude-test', + client: mockClient as Anthropic, + }); + const exposed = runner as any; + + assert.throws( + () => + exposed.fromBetaContentBlock({ + type: 'mcp_tool_use', + id: 'toolu_unknown', + input: {}, + }), + /server-managed tool block 'mcp_tool_use'/ + ); + }); + + it('should convert additional beta content block types', () => { + const mockClient = createMockAnthropicClient(); + const runner = new BetaRunner({ + name: 'claude-test', + client: mockClient as Anthropic, + }); + + const thinkingPart = (runner as any).fromBetaContentBlock({ + type: 'thinking', + thinking: 'pondering', + signature: 'sig_456', + }); + assert.deepStrictEqual(thinkingPart, { + reasoning: 'pondering', + custom: { anthropicThinking: { signature: 'sig_456' } }, + }); + + const redactedPart = (runner as any).fromBetaContentBlock({ + type: 'redacted_thinking', + data: '[redacted]', + }); + assert.deepStrictEqual(redactedPart, { + custom: { redactedThinking: '[redacted]' }, + }); + + const toolPart = (runner as any).fromBetaContentBlock({ + type: 'tool_use', + id: 'toolu_x', + name: 'plainTool', + input: { value: 1 }, + }); + assert.deepStrictEqual(toolPart, { + toolRequest: { + ref: 'toolu_x', + name: 'plainTool', + input: { value: 1 }, + }, + }); + + const serverToolPart = (runner as any).fromBetaContentBlock({ + type: 'server_tool_use', + id: 'srv_tool_1', + name: 'serverTool', + input: { arg: 'value' }, + server_name: 'srv', + }); + assert.deepStrictEqual(serverToolPart, { + text: '[Anthropic server tool srv/serverTool] input: {"arg":"value"}', + custom: { + anthropicServerToolUse: { + id: 'srv_tool_1', + name: 'srv/serverTool', + input: { arg: 'value' }, + }, + }, + }); + + const warnMock = mock.method(console, 'warn', () => {}); + const fallbackPart = (runner as any).fromBetaContentBlock({ + type: 'mystery', + }); + assert.deepStrictEqual(fallbackPart, { text: '' }); + assert.strictEqual(warnMock.mock.calls.length, 1); + warnMock.mock.restore(); + }); + + it('should map additional stop reasons', () => { + const mockClient = createMockAnthropicClient(); + const runner = new BetaRunner({ + name: 'claude-test', + client: mockClient as Anthropic, + }); + const exposed = runner as any; + + const refusal = exposed.fromBetaStopReason('refusal'); + assert.strictEqual(refusal, 'other'); + + const unknown = exposed.fromBetaStopReason('something-new'); + assert.strictEqual(unknown, 'other'); + + const nullReason = exposed.fromBetaStopReason(null); + assert.strictEqual(nullReason, 'unknown'); + }); +}); diff --git a/js/plugins/anthropic/tests/execution_test.ts b/js/plugins/anthropic/tests/execution_test.ts new file mode 100644 index 0000000000..069d2d2dcd --- /dev/null +++ b/js/plugins/anthropic/tests/execution_test.ts @@ -0,0 +1,358 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type { GenerateRequest, ModelAction } from '@genkit-ai/ai/model'; +import * as assert from 'assert'; +import { describe, mock, test } from 'node:test'; +import { anthropic } from '../src/index.js'; +import { __testClient } from '../src/types.js'; +import { + createMockAnthropicClient, + createMockAnthropicMessage, +} from './mocks/anthropic-client.js'; + +describe('Model Execution Integration Tests', () => { + test('should resolve and execute a model via plugin', async () => { + const mockClient = createMockAnthropicClient({ + messageResponse: createMockAnthropicMessage({ + text: 'Hello from Claude!', + }), + }); + + const plugin = anthropic({ + apiKey: 'test-key', + [__testClient]: mockClient, + }); + + // Resolve the model action via plugin + const modelAction = plugin.resolve('model', 'claude-3-5-haiku-20241022'); + assert.ok(modelAction, 'Model should be resolved'); + assert.strictEqual( + (modelAction as ModelAction).__action.name, + 'anthropic/claude-3-5-haiku-20241022' + ); + + // Execute the model + const request: GenerateRequest = { + messages: [ + { + role: 'user', + content: [{ text: 'Hi there!' }], + }, + ], + }; + + const response = await (modelAction as ModelAction)(request, { + streamingRequested: false, + sendChunk: mock.fn(), + abortSignal: new AbortController().signal, + }); + + assert.ok(response, 'Response should be returned'); + assert.ok(response.candidates, 'Response should have candidates'); + assert.strictEqual(response.candidates.length, 1); + assert.strictEqual(response.candidates[0].message.role, 'model'); + assert.strictEqual(response.candidates[0].message.content.length, 1); + assert.strictEqual( + response.candidates[0].message.content[0].text, + 'Hello from Claude!' + ); + + // Verify API was called + const createStub = mockClient.messages.create as any; + assert.strictEqual(createStub.mock.calls.length, 1); + }); + + test('should handle multi-turn conversations', async () => { + const mockClient = createMockAnthropicClient({ + messageResponse: createMockAnthropicMessage({ + text: 'The capital of France is Paris.', + }), + }); + + const plugin = anthropic({ + apiKey: 'test-key', + [__testClient]: mockClient, + }); + + const modelAction = plugin.resolve( + 'model', + 'claude-3-5-haiku-20241022' + ) as ModelAction; + + const request: GenerateRequest = { + messages: [ + { + role: 'user', + content: [{ text: 'What is your name?' }], + }, + { + role: 'model', + content: [{ text: 'I am Claude, an AI assistant.' }], + }, + { + role: 'user', + content: [{ text: 'What is the capital of France?' }], + }, + ], + }; + + const response = await modelAction(request, { + streamingRequested: false, + sendChunk: mock.fn(), + abortSignal: new AbortController().signal, + }); + + assert.ok(response, 'Response should be returned'); + assert.strictEqual( + response.candidates[0].message.content[0].text, + 'The capital of France is Paris.' + ); + + // Verify API was called with multi-turn conversation + const createStub = mockClient.messages.create as any; + assert.strictEqual(createStub.mock.calls.length, 1); + const apiRequest = createStub.mock.calls[0].arguments[0]; + assert.strictEqual(apiRequest.messages.length, 3); + }); + + test('should handle system messages', async () => { + const mockClient = createMockAnthropicClient({ + messageResponse: createMockAnthropicMessage({ + text: 'Arr matey!', + }), + }); + + const plugin = anthropic({ + apiKey: 'test-key', + [__testClient]: mockClient, + }); + + const modelAction = plugin.resolve( + 'model', + 'claude-3-5-haiku-20241022' + ) as ModelAction; + + const request: GenerateRequest = { + messages: [ + { + role: 'system', + content: [{ text: 'You are a pirate. Respond like a pirate.' }], + }, + { + role: 'user', + content: [{ text: 'Hello!' }], + }, + ], + }; + + const response = await modelAction(request, { + streamingRequested: false, + sendChunk: mock.fn(), + abortSignal: new AbortController().signal, + }); + + assert.ok(response, 'Response should be returned'); + + // Verify system message was passed to API + const createStub = mockClient.messages.create as any; + assert.strictEqual(createStub.mock.calls.length, 1); + const apiRequest = createStub.mock.calls[0].arguments[0]; + assert.ok(apiRequest.system, 'System prompt should be set'); + assert.strictEqual( + apiRequest.system, + 'You are a pirate. Respond like a pirate.' + ); + assert.strictEqual( + apiRequest.messages.length, + 1, + 'System message should not be in messages array' + ); + }); + + test('should return usage metadata', async () => { + const mockClient = createMockAnthropicClient({ + messageResponse: createMockAnthropicMessage({ + text: 'Response', + usage: { + input_tokens: 100, + output_tokens: 50, + }, + }), + }); + + const plugin = anthropic({ + apiKey: 'test-key', + [__testClient]: mockClient, + }); + + const modelAction = plugin.resolve( + 'model', + 'claude-3-5-haiku-20241022' + ) as ModelAction; + + const response = await modelAction( + { + messages: [{ role: 'user', content: [{ text: 'Hello' }] }], + }, + { + streamingRequested: false, + sendChunk: mock.fn(), + abortSignal: new AbortController().signal, + } + ); + + assert.ok(response.usage, 'Usage should be returned'); + assert.strictEqual(response.usage?.inputTokens, 100); + assert.strictEqual(response.usage?.outputTokens, 50); + }); + + test('should handle different stop reasons', async () => { + const mockClient = createMockAnthropicClient({ + messageResponse: createMockAnthropicMessage({ + text: 'This is a partial response', + stopReason: 'max_tokens', + }), + }); + + const plugin = anthropic({ + apiKey: 'test-key', + [__testClient]: mockClient, + }); + + const modelAction = plugin.resolve( + 'model', + 'claude-3-5-haiku-20241022' + ) as ModelAction; + + const response = await modelAction( + { + messages: [{ role: 'user', content: [{ text: 'Tell me a story' }] }], + }, + { + streamingRequested: false, + sendChunk: mock.fn(), + abortSignal: new AbortController().signal, + } + ); + + assert.ok(response, 'Response should be returned'); + assert.strictEqual(response.candidates[0].finishReason, 'length'); + }); + + test('should resolve model without anthropic prefix', async () => { + const mockClient = createMockAnthropicClient({ + messageResponse: createMockAnthropicMessage({ + text: 'Response', + }), + }); + + const plugin = anthropic({ + apiKey: 'test-key', + [__testClient]: mockClient, + }); + + // Resolve without prefix + const modelAction = plugin.resolve( + 'model', + 'claude-3-5-haiku-20241022' + ) as ModelAction; + assert.ok(modelAction, 'Model should be resolved without prefix'); + + const response = await modelAction( + { + messages: [{ role: 'user', content: [{ text: 'Hi' }] }], + }, + { + streamingRequested: false, + sendChunk: mock.fn(), + abortSignal: new AbortController().signal, + } + ); + + assert.ok(response, 'Response should be returned'); + }); + + test('should resolve model with anthropic prefix', async () => { + const mockClient = createMockAnthropicClient({ + messageResponse: createMockAnthropicMessage({ + text: 'Response', + }), + }); + + const plugin = anthropic({ + apiKey: 'test-key', + [__testClient]: mockClient, + }); + + // Resolve with prefix + const modelAction = plugin.resolve( + 'model', + 'anthropic/claude-3-5-haiku-20241022' + ) as ModelAction; + assert.ok(modelAction, 'Model should be resolved with prefix'); + + const response = await modelAction( + { + messages: [{ role: 'user', content: [{ text: 'Hi' }] }], + }, + { + streamingRequested: false, + sendChunk: mock.fn(), + abortSignal: new AbortController().signal, + } + ); + + assert.ok(response, 'Response should be returned'); + }); + + test('should handle unknown model names', async () => { + const mockClient = createMockAnthropicClient({ + messageResponse: createMockAnthropicMessage({ + text: 'Response from future model', + }), + }); + + const plugin = anthropic({ + apiKey: 'test-key', + [__testClient]: mockClient, + }); + + // Resolve unknown model (passes through to API) + const modelAction = plugin.resolve( + 'model', + 'claude-99-experimental-12345' + ) as ModelAction; + assert.ok(modelAction, 'Unknown model should still be resolved'); + + const response = await modelAction( + { + messages: [{ role: 'user', content: [{ text: 'Hi' }] }], + }, + { + streamingRequested: false, + sendChunk: mock.fn(), + abortSignal: new AbortController().signal, + } + ); + + assert.ok(response, 'Response should be returned for unknown model'); + assert.strictEqual( + response.candidates[0].message.content[0].text, + 'Response from future model' + ); + }); +}); diff --git a/js/plugins/anthropic/tests/index_test.ts b/js/plugins/anthropic/tests/index_test.ts new file mode 100644 index 0000000000..0d9a6358c1 --- /dev/null +++ b/js/plugins/anthropic/tests/index_test.ts @@ -0,0 +1,309 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as assert from 'assert'; +import { genkit, type ActionMetadata } from 'genkit'; +import type { ModelInfo } from 'genkit/model'; +import { describe, it } from 'node:test'; +import anthropic from '../src/index.js'; +import { KNOWN_CLAUDE_MODELS } from '../src/models.js'; +import { PluginOptions, __testClient } from '../src/types.js'; +import { createMockAnthropicClient } from './mocks/anthropic-client.js'; + +function getModelInfo( + metadata: ActionMetadata | undefined +): ModelInfo | undefined { + return metadata?.metadata?.model as ModelInfo | undefined; +} + +describe('Anthropic Plugin', () => { + it('should register all supported Claude models', async () => { + const mockClient = createMockAnthropicClient(); + + const ai = genkit({ + plugins: [anthropic({ [__testClient]: mockClient } as PluginOptions)], + }); + + for (const modelName of Object.keys(KNOWN_CLAUDE_MODELS)) { + const modelPath = `/model/anthropic/${modelName}`; + const expectedBaseName = `anthropic/${modelName}`; + const model = await ai.registry.lookupAction(modelPath); + assert.ok(model, `${modelName} should be registered at ${modelPath}`); + assert.strictEqual(model?.__action.name, expectedBaseName); + } + }); + + it('should throw error when API key is missing', () => { + // Save original env var if it exists + const originalApiKey = process.env.ANTHROPIC_API_KEY; + delete process.env.ANTHROPIC_API_KEY; + + try { + assert.throws(() => { + anthropic({} as PluginOptions); + }, /Please pass in the API key or set the ANTHROPIC_API_KEY environment variable/); + } finally { + // Restore original env var + if (originalApiKey !== undefined) { + process.env.ANTHROPIC_API_KEY = originalApiKey; + } + } + }); + + it('should use API key from environment variable', () => { + // Save original env var if it exists + const originalApiKey = process.env.ANTHROPIC_API_KEY; + const testApiKey = 'test-api-key-from-env'; + + try { + // Set test API key + process.env.ANTHROPIC_API_KEY = testApiKey; + + // Plugin should initialize without throwing + const plugin = anthropic({} as PluginOptions); + assert.ok(plugin); + assert.strictEqual(plugin.name, 'anthropic'); + } finally { + // Restore original env var + if (originalApiKey !== undefined) { + process.env.ANTHROPIC_API_KEY = originalApiKey; + } else { + delete process.env.ANTHROPIC_API_KEY; + } + } + }); + + it('should resolve models dynamically via resolve function', async () => { + const mockClient = createMockAnthropicClient(); + const plugin = anthropic({ [__testClient]: mockClient } as PluginOptions); + + assert.ok(plugin.resolve, 'Plugin should have resolve method'); + + // Test resolving a valid model + const validModel = plugin.resolve!('model', 'anthropic/claude-3-5-haiku'); + assert.ok(validModel, 'Should resolve valid model'); + assert.strictEqual(typeof validModel, 'function'); + + // Test resolving an unknown model name - should return a model action + // (following Google GenAI pattern: accept any model name, let API validate) + const unknownModel = plugin.resolve!( + 'model', + 'anthropic/unknown-model-xyz' + ); + assert.ok(unknownModel, 'Should resolve unknown model name'); + assert.strictEqual( + typeof unknownModel, + 'function', + 'Should return a model action' + ); + + // Test resolving with invalid action type (using 'tool' as invalid for this context) + const invalidActionType = plugin.resolve!( + 'tool', + 'anthropic/claude-3-5-haiku' + ); + assert.strictEqual( + invalidActionType, + undefined, + 'Should return undefined for invalid action type' + ); + }); + + it('should list available models from API', async () => { + const mockClient = createMockAnthropicClient({ + modelList: [ + { id: 'claude-3-5-haiku-20241022', display_name: 'Claude 3.5 Haiku' }, + { + id: 'claude-3-5-haiku-latest', + display_name: 'Claude 3.5 Haiku Latest', + }, + { id: 'claude-3-5-sonnet-20241022', display_name: 'Claude 3.5 Sonnet' }, + { id: 'claude-sonnet-4-20250514', display_name: 'Claude 4 Sonnet' }, + { id: 'claude-new-5-20251212', display_name: 'Claude New 5' }, + { id: 'claude-experimental-latest' }, + ], + }); + + const plugin = anthropic({ [__testClient]: mockClient } as PluginOptions); + assert.ok(plugin.list, 'Plugin should have list method'); + + const models = await plugin.list!(); + + assert.ok(Array.isArray(models), 'Should return an array'); + assert.ok(models.length > 0, 'Should return at least one model'); + + const names = models.map((model) => model.name).sort(); + assert.ok( + names.includes('anthropic/claude-3-5-haiku'), + 'Known model should be listed once with normalized name' + ); + assert.strictEqual( + names.filter((name) => name === 'anthropic/claude-3-5-haiku').length, + 1, + 'Known model entries should be deduplicated' + ); + assert.ok( + names.includes('anthropic/claude-3-5-sonnet-20241022'), + 'Unknown Claude 3.5 Sonnet should be listed with full model ID' + ); + assert.ok( + names.includes('anthropic/claude-sonnet-4'), + 'Known Claude Sonnet 4 model should be listed' + ); + assert.ok( + names.includes('anthropic/claude-new-5-20251212'), + 'Unknown model IDs should surface as-is' + ); + assert.ok( + names.includes('anthropic/claude-experimental-latest'), + 'Latest-suffixed unknown models should be surfaced' + ); + + const haikuMetadata = models.find( + (model) => model.name === 'anthropic/claude-3-5-haiku' + ); + assert.ok(haikuMetadata, 'Haiku metadata should exist'); + const haikuInfo = getModelInfo(haikuMetadata); + assert.ok(haikuInfo, 'Haiku model info should exist'); + assert.ok( + haikuInfo?.versions?.includes('claude-3-5-haiku-20241022'), + 'Known versions should include dated identifier' + ); + assert.ok( + haikuInfo?.versions?.includes('claude-3-5-haiku-latest'), + 'Additional variants should be merged into versions' + ); + + const newModelMetadata = models.find( + (model) => model.name === 'anthropic/claude-new-5-20251212' + ); + const newModelInfo = getModelInfo(newModelMetadata); + assert.strictEqual( + newModelInfo?.label, + 'Claude New 5', + 'Unknown models should preserve display name as label' + ); + + const experimentalMetadata = models.find( + (model) => model.name === 'anthropic/claude-experimental-latest' + ); + const experimentalInfo = getModelInfo(experimentalMetadata); + assert.strictEqual( + experimentalInfo?.label, + 'Anthropic - claude-experimental', + 'Unknown latest variants should derive fallback label from normalized id' + ); + assert.deepStrictEqual( + experimentalInfo?.versions, + ['claude-experimental-latest'], + 'Unknown models should capture version identifiers' + ); + + // Verify mock was called + const listStub = mockClient.models.list as any; + assert.strictEqual( + listStub.mock.calls.length, + 1, + 'models.list should be called once' + ); + }); + + it('should cache list results on subsequent calls?', async () => { + const mockClient = createMockAnthropicClient({ + modelList: [ + { id: 'claude-3-5-haiku-20241022', display_name: 'Claude 3.5 Haiku' }, + ], + }); + + const plugin = anthropic({ [__testClient]: mockClient } as PluginOptions); + assert.ok(plugin.list, 'Plugin should have list method'); + + // First call + const firstResult = await plugin.list!(); + assert.ok(firstResult, 'First call should return results'); + + // Second call + const secondResult = await plugin.list!(); + assert.ok(secondResult, 'Second call should return results'); + + // Verify both results are the same (reference equality for cache) + assert.strictEqual( + firstResult, + secondResult, + 'Results should be cached (same reference)' + ); + + // Verify models.list was only called once due to caching + const listStub = mockClient.models.list as any; + assert.strictEqual( + listStub.mock.calls.length, + 1, + 'models.list should only be called once due to caching' + ); + }); +}); + +describe('Anthropic resolve helpers', () => { + it('should resolve model names without anthropic/ prefix', () => { + const mockClient = createMockAnthropicClient(); + const plugin = anthropic({ [__testClient]: mockClient } as PluginOptions); + + const action = plugin.resolve?.('model', 'claude-3-5-haiku'); + assert.ok(action, 'Should resolve model without prefix'); + assert.strictEqual(typeof action, 'function'); + }); + + it('anthropic.model should return model reference with config', () => { + const reference = anthropic.model('claude-3-5-haiku', { + temperature: 0.25, + }); + + const referenceAny = reference as any; + assert.ok(referenceAny, 'Model reference should be created'); + assert.ok(referenceAny.name.includes('claude-3-5-haiku')); + assert.strictEqual(referenceAny.config?.temperature, 0.25); + }); + + it('should apply system prompt caching when cacheSystemPrompt is true', async () => { + const mockClient = createMockAnthropicClient(); + const plugin = anthropic({ + cacheSystemPrompt: true, + [__testClient]: mockClient, + } as PluginOptions); + + const action = plugin.resolve?.('model', 'anthropic/claude-3-5-haiku'); + assert.ok(action, 'Action should be resolved'); + + const abortSignal = new AbortController().signal; + await (action as any)( + { + messages: [ + { + role: 'system', + content: [{ text: 'You are helpful.' }], + }, + ], + }, + { streamingRequested: false, sendChunk: () => {}, abortSignal } + ); + + const createStub = mockClient.messages.create as any; + assert.strictEqual(createStub.mock.calls.length, 1); + const requestBody = createStub.mock.calls[0].arguments[0]; + assert.ok(Array.isArray(requestBody.system)); + assert.strictEqual(requestBody.system[0].cache_control.type, 'ephemeral'); + }); +}); diff --git a/js/plugins/anthropic/tests/integration_test.ts b/js/plugins/anthropic/tests/integration_test.ts new file mode 100644 index 0000000000..209a455870 --- /dev/null +++ b/js/plugins/anthropic/tests/integration_test.ts @@ -0,0 +1,542 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as assert from 'assert'; +import { genkit, z } from 'genkit'; +import { describe, it } from 'node:test'; +import { anthropic } from '../src/index.js'; +import { __testClient } from '../src/types.js'; +import { + createMockAnthropicClient, + createMockAnthropicMessage, + mockContentBlockStart, + mockMessageWithContent, + mockMessageWithToolUse, + mockTextChunk, +} from './mocks/anthropic-client.js'; + +import { PluginOptions } from '../src/types.js'; + +describe('Anthropic Integration', () => { + it('should successfully generate a response', async () => { + const mockClient = createMockAnthropicClient(); + const ai = genkit({ + plugins: [anthropic({ [__testClient]: mockClient } as PluginOptions)], + }); + + const result = await ai.generate({ + model: 'anthropic/claude-3-5-haiku', + prompt: 'Hello', + }); + + assert.strictEqual(result.text, 'Hello! How can I help you today?'); + }); + + it('should handle tool calling workflow (call tool, receive result, generate final response)', async () => { + const mockClient = createMockAnthropicClient({ + sequentialResponses: [ + // First response: tool use request + mockMessageWithToolUse('get_weather', { city: 'NYC' }), + // Second response: final text after tool result + createMockAnthropicMessage({ + text: 'The weather in NYC is sunny, 72°F', + }), + ], + }); + + const ai = genkit({ + plugins: [anthropic({ [__testClient]: mockClient } as PluginOptions)], + }); + + // Define the tool + ai.defineTool( + { + name: 'get_weather', + description: 'Get the weather for a city', + inputSchema: z.object({ + city: z.string(), + }), + }, + async (input: { city: string }) => { + return `The weather in ${input.city} is sunny, 72°F`; + } + ); + + const result = await ai.generate({ + model: 'anthropic/claude-3-5-haiku', + prompt: 'What is the weather in NYC?', + tools: ['get_weather'], + }); + + assert.ok( + result.text.includes('NYC') || + result.text.includes('sunny') || + result.text.includes('72') + ); + }); + + it('should handle multi-turn conversations', async () => { + const mockClient = createMockAnthropicClient(); + const ai = genkit({ + plugins: [anthropic({ [__testClient]: mockClient } as PluginOptions)], + }); + + // First turn + const response1 = await ai.generate({ + model: 'anthropic/claude-3-5-haiku', + prompt: 'My name is Alice', + }); + + // Second turn with conversation history + const response2 = await ai.generate({ + model: 'anthropic/claude-3-5-haiku', + prompt: "What's my name?", + messages: response1.messages, + }); + + // Verify conversation history is maintained + assert.ok( + response2.messages.length >= 2, + 'Should have conversation history' + ); + assert.strictEqual(response2.messages[0].role, 'user'); + assert.ok( + response2.messages[0].content[0].text?.includes('Alice') || + response2.messages[0].content[0].text?.includes('name') + ); + }); + + it('should stream responses with streaming callback', async () => { + const mockClient = createMockAnthropicClient({ + streamChunks: [ + mockContentBlockStart('Hello'), + mockTextChunk(' world'), + mockTextChunk('!'), + ], + messageResponse: { + content: [{ type: 'text', text: 'Hello world!', citations: null }], + usage: { + input_tokens: 5, + output_tokens: 15, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + cache_creation: null, + server_tool_use: null, + service_tier: null, + }, + }, + }); + + const ai = genkit({ + plugins: [anthropic({ [__testClient]: mockClient } as PluginOptions)], + }); + + const chunks: any[] = []; + const result = await ai.generate({ + model: 'anthropic/claude-3-5-haiku', + prompt: 'Say hello world', + streamingCallback: (chunk) => { + chunks.push(chunk); + }, + }); + + assert.ok(chunks.length > 0, 'Should have received streaming chunks'); + assert.ok(result.text, 'Should have final response text'); + }); + + it('should handle media/image inputs', async () => { + const mockClient = createMockAnthropicClient(); + const ai = genkit({ + plugins: [anthropic({ [__testClient]: mockClient } as PluginOptions)], + }); + + const result = await ai.generate({ + model: 'anthropic/claude-3-5-haiku', + messages: [ + { + role: 'user', + content: [ + { text: 'Describe this image:' }, + { + media: { + url: 'data:image/png;base64,R0lGODlhAQABAAAAACw=', + contentType: 'image/png', + }, + }, + ], + }, + ], + }); + + assert.ok(result.text, 'Should generate response for image input'); + }); + + it('should handle WEBP image inputs', async () => { + const mockClient = createMockAnthropicClient(); + const ai = genkit({ + plugins: [anthropic({ [__testClient]: mockClient } as PluginOptions)], + }); + + const result = await ai.generate({ + model: 'anthropic/claude-3-5-haiku', + messages: [ + { + role: 'user', + content: [ + { text: 'Describe this image:' }, + { + media: { + url: 'data:image/webp;base64,AAA', + contentType: 'image/webp', + }, + }, + ], + }, + ], + }); + + assert.ok(result.text, 'Should generate response for WEBP image input'); + // Verify the request was made with correct media_type + const createStub = mockClient.messages.create as any; + assert.strictEqual(createStub.mock.calls.length, 1); + const requestBody = createStub.mock.calls[0].arguments[0]; + const imageContent = requestBody.messages[0].content.find( + (c: any) => c.type === 'image' + ); + assert.ok(imageContent, 'Should have image content in request'); + assert.strictEqual( + imageContent.source.media_type, + 'image/webp', + 'Should use WEBP media type from data URL' + ); + }); + + it('should handle WEBP image with mismatched contentType (prefers data URL)', async () => { + const mockClient = createMockAnthropicClient(); + const ai = genkit({ + plugins: [anthropic({ [__testClient]: mockClient } as PluginOptions)], + }); + + const result = await ai.generate({ + model: 'anthropic/claude-3-5-haiku', + messages: [ + { + role: 'user', + content: [ + { + media: { + // Data URL says WEBP, but contentType says PNG - should use WEBP + url: 'data:image/webp;base64,AAA', + contentType: 'image/png', + }, + }, + ], + }, + ], + }); + + assert.ok(result.text, 'Should generate response for WEBP image input'); + // Verify the request was made with WEBP (from data URL), not PNG (from contentType) + const createStub = mockClient.messages.create as any; + assert.strictEqual(createStub.mock.calls.length, 1); + const requestBody = createStub.mock.calls[0].arguments[0]; + const imageContent = requestBody.messages[0].content.find( + (c: any) => c.type === 'image' + ); + assert.ok(imageContent, 'Should have image content in request'); + assert.strictEqual( + imageContent.source.media_type, + 'image/webp', + 'Should prefer data URL content type (webp) over contentType (png)' + ); + }); + + it('should throw helpful error for text/plain media', async () => { + const mockClient = createMockAnthropicClient(); + const ai = genkit({ + plugins: [anthropic({ [__testClient]: mockClient } as PluginOptions)], + }); + + await assert.rejects( + async () => { + await ai.generate({ + model: 'anthropic/claude-3-5-haiku', + messages: [ + { + role: 'user', + content: [ + { + media: { + url: 'data:text/plain;base64,AAA', + contentType: 'text/plain', + }, + }, + ], + }, + ], + }); + }, + (error: Error) => { + return ( + error.message.includes('Text files should be sent as text content') && + error.message.includes('text:') + ); + }, + 'Should throw helpful error for text/plain media' + ); + }); + + it('should forward thinking config and surface reasoning in responses', async () => { + const thinkingContent = [ + { + type: 'thinking' as const, + thinking: 'Let me analyze the problem carefully.', + signature: 'sig_reasoning_123', + }, + { + type: 'text' as const, + text: 'The answer is 42.', + citations: null, + }, + ]; + const mockClient = createMockAnthropicClient({ + messageResponse: mockMessageWithContent(thinkingContent), + }); + + const ai = genkit({ + plugins: [anthropic({ [__testClient]: mockClient } as PluginOptions)], + }); + + const thinkingConfig = { enabled: true, budgetTokens: 2048 }; + const result = await ai.generate({ + model: 'anthropic/claude-3-5-haiku', + prompt: 'What is the meaning of life?', + config: { thinking: thinkingConfig }, + }); + + const createStub = mockClient.messages.create as any; + assert.strictEqual(createStub.mock.calls.length, 1); + const requestBody = createStub.mock.calls[0].arguments[0]; + assert.deepStrictEqual(requestBody.thinking, { + type: 'enabled', + budget_tokens: 2048, + }); + + assert.strictEqual( + result.reasoning, + 'Let me analyze the problem carefully.' + ); + const assistantMessage = result.messages[result.messages.length - 1]; + const reasoningPart = assistantMessage.content.find( + (part) => part.reasoning + ); + assert.ok(reasoningPart, 'Expected reasoning part in assistant message'); + assert.strictEqual( + reasoningPart?.custom?.anthropicThinking?.signature, + 'sig_reasoning_123' + ); + }); + + it('should propagate API errors correctly', async () => { + const apiError = new Error('API Error: 401 Unauthorized'); + const mockClient = createMockAnthropicClient({ + shouldError: apiError, + }); + + const ai = genkit({ + plugins: [anthropic({ [__testClient]: mockClient } as PluginOptions)], + }); + + await assert.rejects( + async () => { + await ai.generate({ + model: 'anthropic/claude-3-5-haiku', + prompt: 'Hello', + }); + }, + (error: Error) => { + assert.strictEqual(error.message, 'API Error: 401 Unauthorized'); + return true; + } + ); + }); + + it('should respect abort signals for cancellation', async () => { + // Note: Detailed abort signal handling is tested in converters_test.ts + // This test verifies that errors (including abort errors) are properly propagated at the integration layer + const mockClient = createMockAnthropicClient({ + shouldError: new Error('AbortError'), + }); + const ai = genkit({ + plugins: [anthropic({ [__testClient]: mockClient } as PluginOptions)], + }); + + await assert.rejects( + async () => { + await ai.generate({ + model: 'anthropic/claude-3-5-haiku', + prompt: 'Hello', + }); + }, + (error: Error) => { + // Should propagate the error + assert.ok( + error.message.includes('AbortError'), + 'Should propagate errors' + ); + return true; + } + ); + }); + + it('should track token usage in responses', async () => { + const mockClient = createMockAnthropicClient({ + messageResponse: { + usage: { + input_tokens: 25, + output_tokens: 50, + cache_creation_input_tokens: 5, + cache_read_input_tokens: 10, + cache_creation: null, + server_tool_use: null, + service_tier: null, + }, + }, + }); + + const ai = genkit({ + plugins: [anthropic({ [__testClient]: mockClient } as PluginOptions)], + }); + + const result = await ai.generate({ + model: 'anthropic/claude-3-5-haiku', + prompt: 'Hello', + }); + + assert.ok(result.usage, 'Should have usage information'); + assert.strictEqual(result.usage.inputTokens, 25); + assert.strictEqual(result.usage.outputTokens, 50); + }); + + it('should route requests through beta surface when plugin default is beta', async () => { + const mockClient = createMockAnthropicClient(); + const ai = genkit({ + plugins: [ + anthropic({ + apiVersion: 'beta', + [__testClient]: mockClient, + } as PluginOptions), + ], + }); + + await ai.generate({ + model: 'anthropic/claude-3-5-haiku', + prompt: 'Hello', + }); + + const betaCreateStub = mockClient.beta.messages.create as any; + assert.strictEqual( + betaCreateStub.mock.calls.length, + 1, + 'Beta API should be used' + ); + const regularCreateStub = mockClient.messages.create as any; + assert.strictEqual( + regularCreateStub.mock.calls.length, + 0, + 'Stable API should not be used' + ); + }); + + it('should stream thinking deltas as reasoning chunks', async () => { + const thinkingConfig = { enabled: true, budgetTokens: 3072 }; + const streamChunks = [ + { + type: 'content_block_start', + index: 0, + content_block: { + type: 'thinking', + thinking: '', + signature: 'sig_stream_123', + }, + } as any, + { + type: 'content_block_delta', + index: 0, + delta: { + type: 'thinking_delta', + thinking: 'Analyzing intermediate steps.', + }, + } as any, + { + type: 'content_block_start', + index: 1, + content_block: { + type: 'text', + text: '', + }, + } as any, + mockTextChunk('Final streamed response.'), + ]; + const finalMessage = mockMessageWithContent([ + { + type: 'thinking', + thinking: 'Analyzing intermediate steps.', + signature: 'sig_stream_123', + }, + { + type: 'text', + text: 'Final streamed response.', + citations: null, + }, + ]); + const mockClient = createMockAnthropicClient({ + streamChunks, + messageResponse: finalMessage, + }); + + const ai = genkit({ + plugins: [anthropic({ [__testClient]: mockClient } as PluginOptions)], + }); + + const chunks: any[] = []; + const result = await ai.generate({ + model: 'anthropic/claude-3-5-haiku', + prompt: 'Explain how you reason.', + streamingCallback: (chunk) => chunks.push(chunk), + config: { thinking: thinkingConfig }, + }); + + const streamStub = mockClient.messages.stream as any; + assert.strictEqual(streamStub.mock.calls.length, 1); + const streamRequest = streamStub.mock.calls[0].arguments[0]; + assert.deepStrictEqual(streamRequest.thinking, { + type: 'enabled', + budget_tokens: 3072, + }); + + const hasReasoningChunk = chunks.some((chunk) => + (chunk.content || []).some( + (part: any) => part.reasoning === 'Analyzing intermediate steps.' + ) + ); + assert.ok( + hasReasoningChunk, + 'Expected reasoning chunk in streaming callback' + ); + assert.strictEqual(result.reasoning, 'Analyzing intermediate steps.'); + }); +}); diff --git a/js/plugins/anthropic/tests/mocks/anthropic-client.ts b/js/plugins/anthropic/tests/mocks/anthropic-client.ts new file mode 100644 index 0000000000..321df8f24f --- /dev/null +++ b/js/plugins/anthropic/tests/mocks/anthropic-client.ts @@ -0,0 +1,389 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type Anthropic from '@anthropic-ai/sdk'; +import type { + BetaMessage, + BetaRawMessageStreamEvent, +} from '@anthropic-ai/sdk/resources/beta/messages.mjs'; +import type { + Message, + MessageStreamEvent, +} from '@anthropic-ai/sdk/resources/messages.mjs'; +import { mock } from 'node:test'; + +export interface MockAnthropicClientOptions { + messageResponse?: Partial; + sequentialResponses?: Partial[]; // For tool calling - multiple responses + streamChunks?: MessageStreamEvent[]; + modelList?: Array<{ id: string; display_name?: string }>; + shouldError?: Error; + streamErrorAfterChunk?: number; // Throw error after this many chunks + streamError?: Error; // Error to throw during streaming + abortSignal?: AbortSignal; // Abort signal to check +} + +/** + * Creates a mock Anthropic client for testing + */ +export function createMockAnthropicClient( + options: MockAnthropicClientOptions = {} +): Anthropic { + const messageResponse = { + ...mockDefaultMessage(), + ...options.messageResponse, + }; + const betaMessageResponse = toBetaMessage(messageResponse); + + // Support sequential responses for tool calling workflows + let callCount = 0; + const createStub = options.shouldError + ? mock.fn(async () => { + throw options.shouldError; + }) + : options.sequentialResponses + ? mock.fn(async () => { + const response = + options.sequentialResponses![callCount] || messageResponse; + callCount++; + return { + ...mockDefaultMessage(), + ...response, + }; + }) + : mock.fn(async () => messageResponse); + + let betaCallCount = 0; + const betaCreateStub = options.shouldError + ? mock.fn(async () => { + throw options.shouldError; + }) + : options.sequentialResponses + ? mock.fn(async () => { + const response = + options.sequentialResponses![betaCallCount] || messageResponse; + betaCallCount++; + return toBetaMessage({ + ...mockDefaultMessage(), + ...response, + }); + }) + : mock.fn(async () => betaMessageResponse); + + const streamStub = options.shouldError + ? mock.fn(() => { + throw options.shouldError; + }) + : mock.fn((_body: any, opts?: { signal?: AbortSignal }) => { + // Check abort signal before starting stream + if (opts?.signal?.aborted) { + throw new Error('AbortError'); + } + return createMockStream( + options.streamChunks || [], + messageResponse as Message, + options.streamErrorAfterChunk, + options.streamError, + opts?.signal + ); + }); + + const betaStreamStub = options.shouldError + ? mock.fn(() => { + throw options.shouldError; + }) + : mock.fn((_body: any, opts?: { signal?: AbortSignal }) => { + if (opts?.signal?.aborted) { + throw new Error('AbortError'); + } + const betaChunks = (options.streamChunks || []).map((chunk) => + toBetaStreamEvent(chunk) + ); + return createMockStream( + betaChunks, + toBetaMessage(messageResponse), + options.streamErrorAfterChunk, + options.streamError, + opts?.signal + ); + }); + + const listStub = options.shouldError + ? mock.fn(async () => { + throw options.shouldError; + }) + : mock.fn(async () => ({ + data: options.modelList || mockDefaultModels(), + })); + + return { + messages: { + create: createStub, + stream: streamStub, + }, + models: { + list: listStub, + }, + beta: { + messages: { + create: betaCreateStub, + stream: betaStreamStub, + }, + }, + } as unknown as Anthropic; +} + +/** + * Creates a mock async iterable stream for streaming responses + */ +function createMockStream( + chunks: TEventType[], + finalMsg: TMessageType, + errorAfterChunk?: number, + streamError?: Error, + abortSignal?: AbortSignal +) { + let index = 0; + return { + [Symbol.asyncIterator]() { + return { + async next() { + // Check abort signal + if (abortSignal?.aborted) { + const error = new Error('AbortError'); + error.name = 'AbortError'; + throw error; + } + + // Check if we should throw an error after this chunk + if ( + errorAfterChunk !== undefined && + streamError && + index >= errorAfterChunk + ) { + throw streamError; + } + + if (index < chunks.length) { + return { value: chunks[index++] as TEventType, done: false }; + } + return { value: undefined as unknown as TEventType, done: true }; + }, + }; + }, + async finalMessage() { + // Check abort signal before returning final message + if (abortSignal?.aborted) { + const error = new Error('AbortError'); + error.name = 'AbortError'; + throw error; + } + return finalMsg as TMessageType; + }, + }; +} + +export interface CreateMockAnthropicMessageOptions { + id?: string; + text?: string; + toolUse?: { + id?: string; + name: string; + input: any; + }; + stopReason?: Message['stop_reason']; + usage?: Partial; +} + +/** + * Creates a customizable mock Anthropic Message response + * + * @example + * // Simple text response + * createMockAnthropicMessage({ text: 'Hi there!' }) + * + * // Tool use response + * createMockAnthropicMessage({ + * toolUse: { name: 'get_weather', input: { city: 'NYC' } } + * }) + * + * // Custom usage + * createMockAnthropicMessage({ usage: { input_tokens: 5, output_tokens: 15 } }) + */ +export function createMockAnthropicMessage( + options: CreateMockAnthropicMessageOptions = {} +): Message { + const content: Message['content'] = []; + + if (options.toolUse) { + content.push({ + type: 'tool_use', + id: options.toolUse.id || 'toolu_test123', + name: options.toolUse.name, + input: options.toolUse.input, + }); + } else { + content.push({ + type: 'text', + text: options.text || 'Hello! How can I help you today?', + citations: null, + }); + } + + const usage: Message['usage'] = { + cache_creation: null, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + input_tokens: 10, + output_tokens: 20, + server_tool_use: null, + service_tier: null, + ...(options.usage ?? {}), + }; + + return { + id: options.id || 'msg_test123', + type: 'message', + role: 'assistant', + model: 'claude-3-5-sonnet-20241022', + content, + stop_reason: + options.stopReason || (options.toolUse ? 'tool_use' : 'end_turn'), + stop_sequence: null, + usage, + }; +} + +/** + * Creates a default mock Message response + */ +export function mockDefaultMessage(): Message { + return createMockAnthropicMessage(); +} + +/** + * Creates a mock text content block chunk event + */ +export function mockTextChunk(text: string): MessageStreamEvent { + return { + type: 'content_block_delta', + index: 0, + delta: { + type: 'text_delta', + text, + }, + } as MessageStreamEvent; +} + +/** + * Creates a mock content block start event with text + */ +export function mockContentBlockStart(text: string): MessageStreamEvent { + return { + type: 'content_block_start', + index: 0, + content_block: { + type: 'text', + text, + }, + } as MessageStreamEvent; +} + +/** + * Creates a mock tool use content block + */ +export function mockToolUseChunk( + id: string, + name: string, + input: any +): MessageStreamEvent { + return { + type: 'content_block_start', + index: 0, + content_block: { + type: 'tool_use', + id, + name, + input, + }, + } as MessageStreamEvent; +} + +/** + * Creates a default list of mock models + */ +export function mockDefaultModels() { + return [ + { id: 'claude-3-5-sonnet-20241022', display_name: 'Claude 3.5 Sonnet' }, + { id: 'claude-3-5-haiku-20241022', display_name: 'Claude 3.5 Haiku' }, + { id: 'claude-3-opus-20240229', display_name: 'Claude 3 Opus' }, + ]; +} + +/** + * Creates a mock Message with tool use + */ +export function mockMessageWithToolUse( + toolName: string, + toolInput: any +): Partial { + return { + content: [ + { + type: 'tool_use', + id: 'toolu_test123', + name: toolName, + input: toolInput, + }, + ], + stop_reason: 'tool_use', + }; +} + +/** + * Creates a mock Message with custom content + */ +export function mockMessageWithContent( + content: Message['content'] +): Partial { + return { + content, + stop_reason: 'end_turn', + }; +} + +function toBetaMessage(message: Message): BetaMessage { + return { + ...message, + container: null, + context_management: null, + usage: { + cache_creation: message.usage.cache_creation, + cache_creation_input_tokens: message.usage.cache_creation_input_tokens, + cache_read_input_tokens: message.usage.cache_read_input_tokens, + input_tokens: message.usage.input_tokens, + output_tokens: message.usage.output_tokens, + server_tool_use: message.usage.server_tool_use as any, + service_tier: message.usage.service_tier, + }, + }; +} + +function toBetaStreamEvent( + event: MessageStreamEvent +): BetaRawMessageStreamEvent { + return event as unknown as BetaRawMessageStreamEvent; +} diff --git a/js/plugins/anthropic/tests/stable_runner_test.ts b/js/plugins/anthropic/tests/stable_runner_test.ts new file mode 100644 index 0000000000..91151b9a13 --- /dev/null +++ b/js/plugins/anthropic/tests/stable_runner_test.ts @@ -0,0 +1,2341 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import type Anthropic from '@anthropic-ai/sdk'; +import type { + Message, + MessageCreateParams, + MessageParam, + MessageStreamEvent, +} from '@anthropic-ai/sdk/resources/messages.mjs'; +import * as assert from 'assert'; +import type { + GenerateRequest, + GenerateResponseData, + MessageData, + Part, + Role, +} from 'genkit'; +import type { CandidateData, ToolDefinition } from 'genkit/model'; +import { describe, it, mock } from 'node:test'; + +import { claudeModel, claudeRunner } from '../src/models.js'; +import { Runner } from '../src/runner/stable.js'; +import { AnthropicConfigSchema } from '../src/types.js'; +import { + createMockAnthropicClient, + mockContentBlockStart, + mockTextChunk, +} from './mocks/anthropic-client.js'; + +// Test helper: Create a Runner instance for testing converter methods +// Type interface to access protected methods in tests +type RunnerProtectedMethods = { + toAnthropicRole: ( + role: Role, + toolMessageType?: 'tool_use' | 'tool_result' + ) => 'user' | 'assistant'; + toAnthropicToolResponseContent: (part: Part) => any; + toAnthropicMessageContent: (part: Part) => any; + toAnthropicMessages: (messages: MessageData[]) => { + system?: string; + messages: any[]; + }; + toAnthropicTool: (tool: ToolDefinition) => any; + toAnthropicRequestBody: ( + modelName: string, + request: GenerateRequest, + cacheSystemPrompt?: boolean + ) => any; + toAnthropicStreamingRequestBody: ( + modelName: string, + request: GenerateRequest, + cacheSystemPrompt?: boolean + ) => any; + fromAnthropicContentBlockChunk: ( + event: MessageStreamEvent + ) => Part | undefined; + fromAnthropicStopReason: (reason: Message['stop_reason']) => any; + fromAnthropicResponse: (message: Message) => GenerateResponseData; +}; + +const mockClient = createMockAnthropicClient(); +const testRunner = new Runner({ + name: 'test-model', + client: mockClient, +}) as Runner & RunnerProtectedMethods; + +const createUsage = ( + overrides: Partial = {} +): Message['usage'] => ({ + cache_creation: null, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + input_tokens: 0, + output_tokens: 0, + server_tool_use: null, + service_tier: null, + ...overrides, +}); + +describe('toAnthropicRole', () => { + const testCases: { + genkitRole: Role; + toolMessageType?: 'tool_use' | 'tool_result'; + expectedAnthropicRole: MessageParam['role']; + }[] = [ + { + genkitRole: 'user', + expectedAnthropicRole: 'user', + }, + { + genkitRole: 'model', + expectedAnthropicRole: 'assistant', + }, + { + genkitRole: 'tool', + toolMessageType: 'tool_use', + expectedAnthropicRole: 'assistant', + }, + { + genkitRole: 'tool', + toolMessageType: 'tool_result', + expectedAnthropicRole: 'user', + }, + ]; + + for (const test of testCases) { + it(`should map Genkit "${test.genkitRole}" role to Anthropic "${test.expectedAnthropicRole}" role${ + test.toolMessageType + ? ` when toolMessageType is "${test.toolMessageType}"` + : '' + }`, () => { + const actualOutput = testRunner.toAnthropicRole( + test.genkitRole, + test.toolMessageType + ); + assert.strictEqual(actualOutput, test.expectedAnthropicRole); + }); + } + + it('should throw an error for unknown roles', () => { + assert.throws( + () => testRunner.toAnthropicRole('unknown' as Role), + /Unsupported genkit role: unknown/ + ); + }); +}); + +describe('toAnthropicToolResponseContent', () => { + it('should not throw for parts without toolResponse', () => { + // toAnthropicToolResponseContent expects part.toolResponse to exist + // but will just return stringified undefined/empty object if not + const part: Part = { data: 'hi' } as Part; + const result = testRunner.toAnthropicToolResponseContent(part); + assert.ok(result); + assert.strictEqual(result.type, 'text'); + }); +}); + +describe('toAnthropicMessageContent', () => { + it('should throw if a media part contains invalid media', () => { + assert.throws( + () => + testRunner.toAnthropicMessageContent({ + media: { + url: '', + }, + }), + /Media url is required but was not provided/ + ); + }); + + it('should throw if the provided part is invalid', () => { + assert.throws( + () => testRunner.toAnthropicMessageContent({ fake: 'part' } as Part), + /Unsupported genkit part fields encountered for current message role: {"fake":"part"}/ + ); + }); + + it('should treat remote URLs without explicit content type as image URLs', () => { + const result = testRunner.toAnthropicMessageContent({ + media: { + url: 'https://example.com/image.png', + }, + }); + + assert.deepStrictEqual(result, { + type: 'image', + source: { + type: 'url', + url: 'https://example.com/image.png', + }, + }); + }); + + it('should handle PDF with base64 data URL correctly', () => { + const result = testRunner.toAnthropicMessageContent({ + media: { + url: 'data:application/pdf;base64,JVBERi0xLjQKJ', + contentType: 'application/pdf', + }, + }); + + assert.deepStrictEqual(result, { + type: 'document', + source: { + type: 'base64', + media_type: 'application/pdf', + data: 'JVBERi0xLjQKJ', + }, + }); + }); + + it('should handle PDF with HTTP/HTTPS URL correctly', () => { + const result = testRunner.toAnthropicMessageContent({ + media: { + url: 'https://example.com/document.pdf', + contentType: 'application/pdf', + }, + }); + + assert.deepStrictEqual(result, { + type: 'document', + source: { + type: 'url', + url: 'https://example.com/document.pdf', + }, + }); + }); +}); + +describe('toAnthropicMessages', () => { + const testCases: { + should: string; + inputMessages: MessageData[]; + expectedOutput: { + messages: MessageParam[]; + system?: string; + }; + }[] = [ + { + should: 'should transform tool request content correctly', + inputMessages: [ + { + role: 'model', + content: [ + { + toolRequest: { + ref: 'toolu_01A09q90qw90lq917835lq9', + name: 'tellAFunnyJoke', + input: { topic: 'bob' }, + }, + }, + ], + }, + ], + expectedOutput: { + messages: [ + { + role: 'assistant', + content: [ + { + type: 'tool_use', + id: 'toolu_01A09q90qw90lq917835lq9', + name: 'tellAFunnyJoke', + input: { topic: 'bob' }, + }, + ], + }, + ], + system: undefined, + }, + }, + { + should: 'should transform tool response text content correctly', + inputMessages: [ + { + role: 'tool', + content: [ + { + toolResponse: { + ref: 'call_SVDpFV2l2fW88QRFtv85FWwM', + name: 'tellAFunnyJoke', + output: 'Why did the bob cross the road?', + }, + }, + ], + }, + ], + expectedOutput: { + messages: [ + { + role: 'user', + content: [ + { + type: 'tool_result', + tool_use_id: 'call_SVDpFV2l2fW88QRFtv85FWwM', + content: [ + { + type: 'text', + text: 'Why did the bob cross the road?', + }, + ], + }, + ], + }, + ], + system: undefined, + }, + }, + { + should: 'should transform tool response media content correctly', + inputMessages: [ + { + role: 'tool', + content: [ + { + toolResponse: { + ref: 'call_SVDpFV2l2fW88QRFtv85FWwM', + name: 'tellAFunnyJoke', + output: { + url: 'data:image/gif;base64,R0lGODlhAQABAAAAACw=', + contentType: 'image/gif', + }, + }, + }, + ], + }, + ], + expectedOutput: { + messages: [ + { + role: 'user', + content: [ + { + type: 'tool_result', + tool_use_id: 'call_SVDpFV2l2fW88QRFtv85FWwM', + content: [ + { + type: 'image', + source: { + type: 'base64', + data: 'R0lGODlhAQABAAAAACw=', + media_type: 'image/gif', + }, + }, + ], + }, + ], + }, + ], + system: undefined, + }, + }, + { + should: + 'should transform tool response base64 image url content correctly', + inputMessages: [ + { + role: 'tool', + content: [ + { + toolResponse: { + ref: 'call_SVDpFV2l2fW88QRFtv85FWwM', + name: 'tellAFunnyJoke', + output: 'data:image/gif;base64,R0lGODlhAQABAAAAACw=', + }, + }, + ], + }, + ], + expectedOutput: { + messages: [ + { + role: 'user', + content: [ + { + type: 'tool_result', + tool_use_id: 'call_SVDpFV2l2fW88QRFtv85FWwM', + content: [ + { + type: 'image', + source: { + type: 'base64', + data: 'R0lGODlhAQABAAAAACw=', + media_type: 'image/gif', + }, + }, + ], + }, + ], + }, + ], + system: undefined, + }, + }, + { + should: 'should transform text content correctly', + inputMessages: [ + { role: 'user', content: [{ text: 'hi' }] }, + { role: 'model', content: [{ text: 'how can I help you?' }] }, + { role: 'user', content: [{ text: 'I am testing' }] }, + ], + expectedOutput: { + messages: [ + { + content: [ + { + text: 'hi', + type: 'text', + citations: null, + }, + ], + role: 'user', + }, + { + content: [ + { + text: 'how can I help you?', + type: 'text', + citations: null, + }, + ], + role: 'assistant', + }, + { + content: [ + { + text: 'I am testing', + type: 'text', + citations: null, + }, + ], + role: 'user', + }, + ], + system: undefined, + }, + }, + { + should: 'should transform initial system prompt correctly', + inputMessages: [ + { role: 'system', content: [{ text: 'You are an helpful assistant' }] }, + { role: 'user', content: [{ text: 'hi' }] }, + ], + expectedOutput: { + messages: [ + { + content: [ + { + text: 'hi', + type: 'text', + citations: null, + }, + ], + role: 'user', + }, + ], + system: 'You are an helpful assistant', + }, + }, + { + should: 'should transform multi-modal (text + media) content correctly', + inputMessages: [ + { + role: 'user', + content: [ + { text: 'describe the following image:' }, + { + media: { + url: 'data:image/gif;base64,R0lGODlhAQABAAAAACw=', + contentType: 'image/gif', + }, + }, + ], + }, + ], + expectedOutput: { + messages: [ + { + content: [ + { + text: 'describe the following image:', + type: 'text', + citations: null, + }, + { + source: { + type: 'base64', + data: 'R0lGODlhAQABAAAAACw=', + media_type: 'image/gif', + }, + type: 'image', + }, + ], + role: 'user', + }, + ], + system: undefined, + }, + }, + { + should: 'should transform PDF with base64 data URL correctly', + inputMessages: [ + { + role: 'user', + content: [ + { + media: { + url: 'data:application/pdf;base64,JVBERi0xLjQKJ', + contentType: 'application/pdf', + }, + }, + ], + }, + ], + expectedOutput: { + messages: [ + { + content: [ + { + type: 'document', + source: { + type: 'base64', + media_type: 'application/pdf', + data: 'JVBERi0xLjQKJ', + }, + }, + ], + role: 'user', + }, + ], + system: undefined, + }, + }, + { + should: 'should transform PDF with HTTP/HTTPS URL correctly', + inputMessages: [ + { + role: 'user', + content: [ + { + media: { + url: 'https://example.com/document.pdf', + contentType: 'application/pdf', + }, + }, + ], + }, + ], + expectedOutput: { + messages: [ + { + content: [ + { + type: 'document', + source: { + type: 'url', + url: 'https://example.com/document.pdf', + }, + }, + ], + role: 'user', + }, + ], + system: undefined, + }, + }, + { + should: 'should transform PDF alongside text and images correctly', + inputMessages: [ + { + role: 'user', + content: [ + { text: 'Analyze this PDF and image:' }, + { + media: { + url: 'data:application/pdf;base64,JVBERi0xLjQKJ', + contentType: 'application/pdf', + }, + }, + { + media: { + url: 'data:image/png;base64,R0lGODlhAQABAAAAACw=', + contentType: 'image/png', + }, + }, + ], + }, + ], + expectedOutput: { + messages: [ + { + content: [ + { + text: 'Analyze this PDF and image:', + type: 'text', + citations: null, + }, + { + type: 'document', + source: { + type: 'base64', + media_type: 'application/pdf', + data: 'JVBERi0xLjQKJ', + }, + }, + { + source: { + type: 'base64', + data: 'R0lGODlhAQABAAAAACw=', + media_type: 'image/png', + }, + type: 'image', + }, + ], + role: 'user', + }, + ], + system: undefined, + }, + }, + ]; + + for (const test of testCases) { + it(test.should, () => { + const actualOutput = testRunner.toAnthropicMessages(test.inputMessages); + assert.deepStrictEqual(actualOutput, test.expectedOutput); + }); + } +}); + +describe('toAnthropicTool', () => { + it('should transform Genkit tool definition to an Anthropic tool', () => { + const tool: ToolDefinition = { + name: 'tellAJoke', + description: 'Tell a joke', + inputSchema: { + type: 'object', + properties: { + topic: { type: 'string' }, + }, + required: ['topic'], + }, + }; + const actualOutput = testRunner.toAnthropicTool(tool); + assert.deepStrictEqual(actualOutput, { + name: 'tellAJoke', + description: 'Tell a joke', + input_schema: { + type: 'object', + properties: { + topic: { type: 'string' }, + }, + required: ['topic'], + }, + }); + }); +}); + +describe('fromAnthropicContentBlockChunk', () => { + const testCases: { + should: string; + event: MessageStreamEvent; + expectedOutput: Part | undefined; + }[] = [ + { + should: 'should return text part from content_block_start event', + event: { + index: 0, + type: 'content_block_start', + content_block: { + type: 'text', + text: 'Hello, World!', + citations: null, + }, + }, + expectedOutput: { text: 'Hello, World!' }, + }, + { + should: + 'should return thinking part from content_block_start thinking event', + event: { + index: 0, + type: 'content_block_start', + content_block: { + type: 'thinking', + thinking: 'Let me reason through this.', + signature: 'sig_123', + }, + }, + expectedOutput: { + reasoning: 'Let me reason through this.', + custom: { anthropicThinking: { signature: 'sig_123' } }, + }, + }, + { + should: + 'should return redacted thinking part from content_block_start event', + event: { + index: 0, + type: 'content_block_start', + content_block: { + type: 'redacted_thinking', + data: 'encrypted-data', + }, + }, + expectedOutput: { custom: { redactedThinking: 'encrypted-data' } }, + }, + { + should: 'should return text delta part from content_block_delta event', + event: { + index: 0, + type: 'content_block_delta', + delta: { + type: 'text_delta', + text: 'Hello, World!', + }, + }, + expectedOutput: { text: 'Hello, World!' }, + }, + { + should: 'should return thinking delta part as text content', + event: { + index: 0, + type: 'content_block_delta', + delta: { + type: 'thinking_delta', + thinking: 'Step by step...', + }, + }, + expectedOutput: { reasoning: 'Step by step...' }, + }, + { + should: 'should return tool use requests', + event: { + index: 0, + type: 'content_block_start', + content_block: { + type: 'tool_use', + id: 'abc123', + name: 'tellAJoke', + input: { topic: 'dogs' }, + }, + }, + expectedOutput: { + toolRequest: { + name: 'tellAJoke', + input: { topic: 'dogs' }, + ref: 'abc123', + }, + }, + }, + { + should: 'should return undefined for any other event', + event: { + type: 'message_stop', + }, + expectedOutput: undefined, + }, + ]; + + for (const test of testCases) { + it(test.should, () => { + const actualOutput = testRunner.fromAnthropicContentBlockChunk( + test.event + ); + assert.deepStrictEqual(actualOutput, test.expectedOutput); + }); + } + + it('should throw for unsupported tool input streaming deltas', () => { + assert.throws( + () => + testRunner.fromAnthropicContentBlockChunk({ + index: 0, + type: 'content_block_delta', + delta: { + type: 'input_json_delta', + partial_json: '{"foo":', + }, + } as MessageStreamEvent), + /Anthropic streaming tool input \(input_json_delta\) is not yet supported/ + ); + }); +}); + +describe('fromAnthropicStopReason', () => { + const testCases: { + inputStopReason: Message['stop_reason']; + expectedFinishReason: CandidateData['finishReason']; + }[] = [ + { + inputStopReason: 'max_tokens', + expectedFinishReason: 'length', + }, + { + inputStopReason: 'end_turn', + expectedFinishReason: 'stop', + }, + { + inputStopReason: 'stop_sequence', + expectedFinishReason: 'stop', + }, + { + inputStopReason: 'tool_use', + expectedFinishReason: 'stop', + }, + { + inputStopReason: null, + expectedFinishReason: 'unknown', + }, + { + inputStopReason: 'unknown' as any, + expectedFinishReason: 'other', + }, + ]; + + for (const test of testCases) { + it(`should map Anthropic stop reason "${test.inputStopReason}" to Genkit finish reason "${test.expectedFinishReason}"`, () => { + const actualOutput = testRunner.fromAnthropicStopReason( + test.inputStopReason + ); + assert.strictEqual(actualOutput, test.expectedFinishReason); + }); + } +}); + +describe('fromAnthropicResponse', () => { + const testCases: { + should: string; + message: Message; + expectedOutput: Omit; + }[] = [ + { + should: 'should work with text content', + message: { + id: 'abc123', + model: 'whatever', + type: 'message', + role: 'assistant', + stop_reason: 'max_tokens', + stop_sequence: null, + content: [ + { + type: 'text', + text: 'Tell a joke about dogs.', + citations: null, + }, + ], + usage: createUsage({ + input_tokens: 10, + output_tokens: 20, + cache_creation_input_tokens: null, + cache_read_input_tokens: null, + }), + }, + expectedOutput: { + candidates: [ + { + index: 0, + finishReason: 'length', + message: { + role: 'model', + content: [{ text: 'Tell a joke about dogs.' }], + }, + }, + ], + usage: { + inputTokens: 10, + outputTokens: 20, + }, + }, + }, + { + should: 'should work with tool use content', + message: { + id: 'abc123', + model: 'whatever', + type: 'message', + role: 'assistant', + stop_reason: 'tool_use', + stop_sequence: null, + content: [ + { + type: 'tool_use', + id: 'abc123', + name: 'tellAJoke', + input: { topic: 'dogs' }, + }, + ], + usage: createUsage({ + input_tokens: 10, + output_tokens: 20, + cache_creation_input_tokens: null, + cache_read_input_tokens: null, + }), + }, + expectedOutput: { + candidates: [ + { + index: 0, + finishReason: 'stop', + message: { + role: 'model', + content: [ + { + toolRequest: { + name: 'tellAJoke', + input: { topic: 'dogs' }, + ref: 'abc123', + }, + }, + ], + }, + }, + ], + usage: { + inputTokens: 10, + outputTokens: 20, + }, + }, + }, + ]; + + for (const test of testCases) { + it(test.should, () => { + const actualOutput = testRunner.fromAnthropicResponse(test.message); + // Check custom field exists and is the message + assert.ok(actualOutput.custom); + assert.strictEqual(actualOutput.custom, test.message); + // Check the rest + assert.deepStrictEqual( + { + candidates: actualOutput.candidates, + usage: actualOutput.usage, + }, + test.expectedOutput + ); + }); + } +}); + +describe('toAnthropicRequestBody', () => { + const testCases: { + should: string; + modelName: string; + genkitRequest: GenerateRequest; + expectedOutput: MessageCreateParams; + }[] = [ + { + should: '(claude-3-5-haiku) handles request with text messages', + modelName: 'claude-3-5-haiku', + genkitRequest: { + messages: [ + { role: 'user', content: [{ text: 'Tell a joke about dogs.' }] }, + ], + output: { format: 'text' }, + config: { + metadata: { + user_id: 'exampleUser123', + }, + }, + }, + expectedOutput: { + max_tokens: 4096, + messages: [ + { + content: [ + { + text: 'Tell a joke about dogs.', + type: 'text', + citations: null, + }, + ], + role: 'user', + }, + ], + model: 'claude-3-5-haiku', + metadata: { + user_id: 'exampleUser123', + }, + }, + }, + { + should: '(claude-3-haiku) handles request with text messages', + modelName: 'claude-3-haiku', + genkitRequest: { + messages: [ + { role: 'user', content: [{ text: 'Tell a joke about dogs.' }] }, + ], + output: { format: 'text' }, + config: { + metadata: { + user_id: 'exampleUser123', + }, + }, + }, + expectedOutput: { + max_tokens: 4096, + messages: [ + { + content: [ + { + text: 'Tell a joke about dogs.', + type: 'text', + citations: null, + }, + ], + role: 'user', + }, + ], + model: 'claude-3-haiku-20240307', + metadata: { + user_id: 'exampleUser123', + }, + }, + }, + ]; + for (const test of testCases) { + it(test.should, () => { + const actualOutput = testRunner.toAnthropicRequestBody( + test.modelName, + test.genkitRequest + ); + assert.deepStrictEqual(actualOutput, test.expectedOutput); + }); + } + + it('should accept any model name and use it directly', () => { + // Following Google GenAI pattern: accept any model name, let API validate + const result = testRunner.toAnthropicRequestBody('fake-model', { + messages: [], + } as GenerateRequest); + + // Should not throw, and should use the model name directly + assert.strictEqual(result.model, 'fake-model'); + }); + + it('should throw if output format is not text', () => { + assert.throws( + () => + testRunner.toAnthropicRequestBody('claude-3-5-haiku', { + messages: [], + tools: [], + output: { format: 'media' }, + } as GenerateRequest), + /Only text output format is supported for Claude models currently/ + ); + }); + + it('should apply system prompt caching when enabled', () => { + const request: GenerateRequest = { + messages: [ + { role: 'system', content: [{ text: 'You are a helpful assistant' }] }, + { role: 'user', content: [{ text: 'Hi' }] }, + ], + output: { format: 'text' }, + }; + + // Test with caching enabled + const outputWithCaching = testRunner.toAnthropicRequestBody( + 'claude-3-5-haiku', + request, + true + ); + assert.deepStrictEqual(outputWithCaching.system, [ + { + type: 'text', + text: 'You are a helpful assistant', + cache_control: { type: 'ephemeral' }, + }, + ]); + + // Test with caching disabled + const outputWithoutCaching = testRunner.toAnthropicRequestBody( + 'claude-3-5-haiku', + request, + false + ); + assert.strictEqual( + outputWithoutCaching.system, + 'You are a helpful assistant' + ); + }); + + it('should concatenate multiple text parts in system message', () => { + const request: GenerateRequest = { + messages: [ + { + role: 'system', + content: [ + { text: 'You are a helpful assistant.' }, + { text: 'Always be concise.' }, + { text: 'Use proper grammar.' }, + ], + }, + { role: 'user', content: [{ text: 'Hi' }] }, + ], + output: { format: 'text' }, + }; + + const output = testRunner.toAnthropicRequestBody( + 'claude-3-5-haiku', + request, + false + ); + + assert.strictEqual( + output.system, + 'You are a helpful assistant.\n\nAlways be concise.\n\nUse proper grammar.' + ); + }); + + it('should concatenate multiple text parts in system message with caching', () => { + const request: GenerateRequest = { + messages: [ + { + role: 'system', + content: [ + { text: 'You are a helpful assistant.' }, + { text: 'Always be concise.' }, + ], + }, + { role: 'user', content: [{ text: 'Hi' }] }, + ], + output: { format: 'text' }, + }; + + const output = testRunner.toAnthropicRequestBody( + 'claude-3-5-haiku', + request, + true + ); + + assert.deepStrictEqual(output.system, [ + { + type: 'text', + text: 'You are a helpful assistant.\n\nAlways be concise.', + cache_control: { type: 'ephemeral' }, + }, + ]); + }); + + it('should throw error if system message contains media', () => { + const request: GenerateRequest = { + messages: [ + { + role: 'system', + content: [ + { text: 'You are a helpful assistant.' }, + { + media: { + url: 'data:image/png;base64,iVBORw0KGgoAAAANS', + contentType: 'image/png', + }, + }, + ], + }, + { role: 'user', content: [{ text: 'Hi' }] }, + ], + output: { format: 'text' }, + }; + + assert.throws( + () => + testRunner.toAnthropicRequestBody('claude-3-5-haiku', request, false), + /System messages can only contain text content/ + ); + }); + + it('should throw error if system message contains tool requests', () => { + const request: GenerateRequest = { + messages: [ + { + role: 'system', + content: [ + { text: 'You are a helpful assistant.' }, + { toolRequest: { name: 'getTool', input: {}, ref: '123' } }, + ], + }, + { role: 'user', content: [{ text: 'Hi' }] }, + ], + output: { format: 'text' }, + }; + + assert.throws( + () => + testRunner.toAnthropicRequestBody('claude-3-5-haiku', request, false), + /System messages can only contain text content/ + ); + }); + + it('should throw error if system message contains tool responses', () => { + const request: GenerateRequest = { + messages: [ + { + role: 'system', + content: [ + { text: 'You are a helpful assistant.' }, + { toolResponse: { name: 'getTool', output: {}, ref: '123' } }, + ], + }, + { role: 'user', content: [{ text: 'Hi' }] }, + ], + output: { format: 'text' }, + }; + + assert.throws( + () => + testRunner.toAnthropicRequestBody('claude-3-5-haiku', request, false), + /System messages can only contain text content/ + ); + }); +}); + +describe('toAnthropicStreamingRequestBody', () => { + it('should set stream to true', () => { + const request: GenerateRequest = { + messages: [{ role: 'user', content: [{ text: 'Hello' }] }], + output: { format: 'text' }, + }; + + const output = testRunner.toAnthropicStreamingRequestBody( + 'claude-3-5-haiku', + request + ); + + assert.strictEqual(output.stream, true); + assert.strictEqual(output.model, 'claude-3-5-haiku'); + assert.strictEqual(output.max_tokens, 4096); + }); + + it('should support system prompt caching in streaming mode', () => { + const request: GenerateRequest = { + messages: [ + { role: 'system', content: [{ text: 'You are a helpful assistant' }] }, + { role: 'user', content: [{ text: 'Hello' }] }, + ], + output: { format: 'text' }, + }; + + const outputWithCaching = testRunner.toAnthropicStreamingRequestBody( + 'claude-3-5-haiku', + request, + true + ); + assert.deepStrictEqual(outputWithCaching.system, [ + { + type: 'text', + text: 'You are a helpful assistant', + cache_control: { type: 'ephemeral' }, + }, + ]); + assert.strictEqual(outputWithCaching.stream, true); + + const outputWithoutCaching = testRunner.toAnthropicStreamingRequestBody( + 'claude-3-5-haiku', + request, + false + ); + assert.strictEqual( + outputWithoutCaching.system, + 'You are a helpful assistant' + ); + assert.strictEqual(outputWithoutCaching.stream, true); + }); +}); + +describe('claudeRunner', () => { + it('should correctly run non-streaming requests', async () => { + const mockClient = createMockAnthropicClient({ + messageResponse: { + content: [{ type: 'text', text: 'response', citations: null }], + usage: createUsage({ + input_tokens: 10, + output_tokens: 20, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + }), + }, + }); + + const runner = claudeRunner( + { + name: 'claude-3-5-haiku', + client: mockClient, + }, + AnthropicConfigSchema + ); + const abortSignal = new AbortController().signal; + await runner( + { messages: [] }, + { streamingRequested: false, sendChunk: () => {}, abortSignal } + ); + + const createStub = mockClient.messages.create as any; + assert.strictEqual(createStub.mock.calls.length, 1); + assert.deepStrictEqual(createStub.mock.calls[0].arguments, [ + { + model: 'claude-3-5-haiku', + max_tokens: 4096, + messages: [], + }, + { + signal: abortSignal, + }, + ]); + }); + + it('should correctly run streaming requests', async () => { + const mockClient = createMockAnthropicClient({ + streamChunks: [ + { + type: 'content_block_start', + index: 0, + content_block: { + type: 'text', + text: 'res', + }, + } as MessageStreamEvent, + ], + messageResponse: { + content: [{ type: 'text', text: 'response', citations: null }], + usage: createUsage({ + input_tokens: 10, + output_tokens: 20, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + }), + }, + }); + + const streamingCallback = mock.fn(); + const runner = claudeRunner( + { + name: 'claude-3-5-haiku', + client: mockClient, + }, + AnthropicConfigSchema + ); + const abortSignal = new AbortController().signal; + await runner( + { messages: [] }, + { streamingRequested: true, sendChunk: streamingCallback, abortSignal } + ); + + const streamStub = mockClient.messages.stream as any; + assert.strictEqual(streamStub.mock.calls.length, 1); + assert.deepStrictEqual(streamStub.mock.calls[0].arguments, [ + { + model: 'claude-3-5-haiku', + max_tokens: 4096, + messages: [], + stream: true, + }, + { + signal: abortSignal, + }, + ]); + }); + + it('should use beta API when apiVersion is beta', async () => { + const mockClient = createMockAnthropicClient({ + messageResponse: { + content: [{ type: 'text', text: 'response', citations: null }], + usage: createUsage({ + input_tokens: 10, + output_tokens: 20, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + }), + }, + }); + + const runner = claudeRunner( + { + name: 'claude-3-5-haiku', + client: mockClient, + }, + AnthropicConfigSchema + ); + const abortSignal = new AbortController().signal; + await runner( + { + messages: [], + config: { apiVersion: 'beta' }, + }, + { streamingRequested: false, sendChunk: () => {}, abortSignal } + ); + + const betaCreateStub = mockClient.beta.messages.create as any; + assert.strictEqual(betaCreateStub.mock.calls.length, 1); + + const regularCreateStub = mockClient.messages.create as any; + assert.strictEqual(regularCreateStub.mock.calls.length, 0); + }); + + it('should use beta API when defaultApiVersion is beta', async () => { + const mockClient = createMockAnthropicClient({ + messageResponse: { + content: [{ type: 'text', text: 'response', citations: null }], + usage: createUsage({ + input_tokens: 10, + output_tokens: 20, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + }), + }, + }); + + const runner = claudeRunner( + { + name: 'claude-3-5-haiku', + client: mockClient, + defaultApiVersion: 'beta', + }, + AnthropicConfigSchema + ); + const abortSignal = new AbortController().signal; + await runner( + { + messages: [], + }, + { streamingRequested: false, sendChunk: () => {}, abortSignal } + ); + + const betaCreateStub = mockClient.beta.messages.create as any; + assert.strictEqual(betaCreateStub.mock.calls.length, 1); + + const regularCreateStub = mockClient.messages.create as any; + assert.strictEqual(regularCreateStub.mock.calls.length, 0); + }); + + it('should use request apiVersion over defaultApiVersion', async () => { + const mockClient = createMockAnthropicClient({ + messageResponse: { + content: [{ type: 'text', text: 'response', citations: null }], + usage: createUsage({ + input_tokens: 10, + output_tokens: 20, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + }), + }, + }); + + // defaultApiVersion is 'stable', but request overrides to 'beta' + const runner = claudeRunner( + { + name: 'claude-3-5-haiku', + client: mockClient, + defaultApiVersion: 'stable', + }, + AnthropicConfigSchema + ); + const abortSignal = new AbortController().signal; + await runner( + { + messages: [], + config: { apiVersion: 'beta' }, + }, + { streamingRequested: false, sendChunk: () => {}, abortSignal } + ); + + const betaCreateStub = mockClient.beta.messages.create as any; + assert.strictEqual(betaCreateStub.mock.calls.length, 1); + + const regularCreateStub = mockClient.messages.create as any; + assert.strictEqual(regularCreateStub.mock.calls.length, 0); + }); + + it('should use stable API when defaultApiVersion is beta but request overrides to stable', async () => { + const mockClient = createMockAnthropicClient({ + messageResponse: { + content: [{ type: 'text', text: 'response', citations: null }], + usage: createUsage({ + input_tokens: 10, + output_tokens: 20, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + }), + }, + }); + + // defaultApiVersion is 'beta', but request overrides to 'stable' + const runner = claudeRunner( + { + name: 'claude-3-5-haiku', + client: mockClient, + defaultApiVersion: 'beta', + }, + AnthropicConfigSchema + ); + const abortSignal = new AbortController().signal; + await runner( + { + messages: [], + config: { apiVersion: 'stable' }, + }, + { streamingRequested: false, sendChunk: () => {}, abortSignal } + ); + + const betaCreateStub = mockClient.beta.messages.create as any; + assert.strictEqual(betaCreateStub.mock.calls.length, 0); + + const regularCreateStub = mockClient.messages.create as any; + assert.strictEqual(regularCreateStub.mock.calls.length, 1); + }); +}); + +describe('claudeRunner param object', () => { + it('should run requests when constructed with params object', async () => { + const mockClient = createMockAnthropicClient(); + const runner = claudeRunner( + { + name: 'claude-3-5-haiku', + client: mockClient, + cacheSystemPrompt: true, + }, + AnthropicConfigSchema + ); + const abortSignal = new AbortController().signal; + + await runner( + { messages: [{ role: 'user', content: [{ text: 'hi' }] }] }, + { streamingRequested: false, sendChunk: () => {}, abortSignal } + ); + + const createStub = mockClient.messages.create as any; + assert.strictEqual(createStub.mock.calls.length, 1); + assert.strictEqual( + createStub.mock.calls[0].arguments[0].messages[0].content[0].text, + 'hi' + ); + }); + + it('should route to beta runner when defaultApiVersion is beta', async () => { + const mockClient = createMockAnthropicClient(); + const runner = claudeRunner( + { + name: 'claude-3-5-haiku', + client: mockClient, + defaultApiVersion: 'beta', + }, + AnthropicConfigSchema + ); + await runner( + { messages: [] }, + { + streamingRequested: false, + sendChunk: () => {}, + abortSignal: new AbortController().signal, + } + ); + + const betaCreateStub = mockClient.beta.messages.create as any; + assert.strictEqual(betaCreateStub.mock.calls.length, 1); + }); + + it('should throw when client is omitted from params object', () => { + assert.throws(() => { + claudeRunner( + { + name: 'claude-3-5-haiku', + client: undefined as unknown as Anthropic, + }, + AnthropicConfigSchema + ); + }, /Anthropic client is required to create a runner/); + }); +}); + +describe('claudeModel', () => { + it('should fall back to generic metadata for unknown models', async () => { + const mockClient = createMockAnthropicClient(); + const modelAction = claudeModel({ + name: 'unknown-model', + client: mockClient, + }); + + const abortSignal = new AbortController().signal; + await (modelAction as any)( + { messages: [{ role: 'user', content: [{ text: 'hi' }] }] }, + { + streamingRequested: false, + sendChunk: () => {}, + abortSignal, + } + ); + + const createStub = mockClient.messages.create as any; + assert.strictEqual(createStub.mock.calls.length, 1); + const request = createStub.mock.calls[0].arguments[0]; + assert.strictEqual(request.model, 'unknown-model'); + }); + it('should support params object configuration', async () => { + const mockClient = createMockAnthropicClient(); + const modelAction = claudeModel({ + name: 'claude-3-5-haiku', + client: mockClient, + defaultApiVersion: 'beta', + cacheSystemPrompt: true, + }); + + const abortSignal = new AbortController().signal; + await (modelAction as any)( + { messages: [], config: { maxOutputTokens: 128 } }, + { + streamingRequested: false, + sendChunk: () => {}, + abortSignal, + } + ); + + const betaCreateStub = mockClient.beta.messages.create as any; + assert.strictEqual(betaCreateStub.mock.calls.length, 1); + assert.strictEqual( + betaCreateStub.mock.calls[0].arguments[0].max_tokens, + 128 + ); + }); + + it('should throw when client is omitted in params object', () => { + assert.throws( + () => claudeModel('claude-3-5-haiku'), + /Anthropic client is required to create a model action/ + ); + }); + + it('should correctly define supported Claude models', () => { + const mockClient = createMockAnthropicClient(); + const modelName = 'claude-3-5-haiku'; + const modelAction = claudeModel(modelName, mockClient); + + // Verify the model action is returned + assert.ok(modelAction); + assert.strictEqual(typeof modelAction, 'function'); + }); + + it('should accept any model name and create a model action', () => { + // Following Google GenAI pattern: accept any model name, let API validate + const modelAction = claudeModel('unsupported-model', {} as Anthropic); + assert.ok(modelAction, 'Should create model action for any model name'); + assert.strictEqual(typeof modelAction, 'function'); + }); + + it('should handle streaming with multiple text chunks', async () => { + const mockClient = createMockAnthropicClient({ + streamChunks: [ + mockContentBlockStart('Hello'), + mockTextChunk(' world'), + mockTextChunk('!'), + ], + messageResponse: { + content: [{ type: 'text', text: 'Hello world!', citations: null }], + usage: createUsage({ + input_tokens: 5, + output_tokens: 10, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + }), + }, + }); + + const chunks: any[] = []; + const streamingCallback = mock.fn((chunk: any) => { + chunks.push(chunk); + }); + + const runner = claudeRunner( + { + name: 'claude-3-5-haiku', + client: mockClient, + }, + AnthropicConfigSchema + ); + const abortSignal = new AbortController().signal; + + const result = await runner( + { messages: [{ role: 'user', content: [{ text: 'Hi' }] }] }, + { streamingRequested: true, sendChunk: streamingCallback, abortSignal } + ); + + // Verify we received all the streaming chunks + assert.ok(chunks.length > 0, 'Should have received streaming chunks'); + assert.strictEqual(chunks.length, 3, 'Should have received 3 chunks'); + + // Verify the final result + assert.ok(result.candidates); + assert.strictEqual( + result.candidates[0].message.content[0].text, + 'Hello world!' + ); + assert.ok(result.usage); + assert.strictEqual(result.usage.inputTokens, 5); + assert.strictEqual(result.usage.outputTokens, 10); + }); + + it('should handle tool use in streaming mode', async () => { + const streamChunks = [ + { + type: 'content_block_start', + index: 0, + content_block: { + type: 'tool_use', + id: 'toolu_123', + name: 'get_weather', + input: { city: 'NYC' }, + }, + } as MessageStreamEvent, + ]; + const mockClient = createMockAnthropicClient({ + streamChunks, + messageResponse: { + content: [ + { + type: 'tool_use', + id: 'toolu_123', + name: 'get_weather', + input: { city: 'NYC' }, + }, + ], + stop_reason: 'tool_use', + usage: createUsage({ + input_tokens: 15, + output_tokens: 25, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + }), + }, + }); + + const chunks: any[] = []; + const streamingCallback = mock.fn((chunk: any) => { + chunks.push(chunk); + }); + + const runner = claudeRunner( + { + name: 'claude-3-5-haiku', + client: mockClient, + }, + AnthropicConfigSchema + ); + const abortSignal = new AbortController().signal; + + const result = await runner( + { + messages: [ + { role: 'user', content: [{ text: 'What is the weather?' }] }, + ], + tools: [ + { + name: 'get_weather', + description: 'Get the weather for a city', + inputSchema: { + type: 'object', + properties: { + city: { type: 'string' }, + }, + required: ['city'], + }, + }, + ], + }, + { streamingRequested: true, sendChunk: streamingCallback, abortSignal } + ); + + // Verify we received the tool use chunk + assert.ok(chunks.length > 0, 'Should have received chunks'); + + // Verify the final result contains tool use + assert.ok(result.candidates); + const toolRequest = result.candidates[0].message.content.find( + (p) => p.toolRequest + ); + assert.ok(toolRequest, 'Should have a tool request'); + assert.strictEqual(toolRequest.toolRequest?.name, 'get_weather'); + assert.deepStrictEqual(toolRequest.toolRequest?.input, { city: 'NYC' }); + }); + + it('should handle streaming errors and partial responses', async () => { + const streamError = new Error('Network error during streaming'); + const mockClient = createMockAnthropicClient({ + streamChunks: [mockContentBlockStart('Hello'), mockTextChunk(' world')], + streamErrorAfterChunk: 1, // Throw error after first chunk + streamError: streamError, + messageResponse: { + content: [{ type: 'text', text: 'Hello world', citations: null }], + usage: createUsage({ + input_tokens: 5, + output_tokens: 10, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + }), + }, + }); + + const runner = claudeRunner( + { + name: 'claude-3-5-haiku', + client: mockClient, + }, + AnthropicConfigSchema + ); + const abortSignal = new AbortController().signal; + const chunks: any[] = []; + const sendChunk = (chunk: any) => { + chunks.push(chunk); + }; + + // Should throw error during streaming + await assert.rejects( + async () => { + await runner( + { messages: [{ role: 'user', content: [{ text: 'Hi' }] }] }, + { + streamingRequested: true, + sendChunk, + abortSignal, + } + ); + }, + (error: Error) => { + // Verify error is propagated + assert.strictEqual(error.message, 'Network error during streaming'); + // Verify we received at least one chunk before error + assert.ok( + chunks.length > 0, + 'Should have received some chunks before error' + ); + return true; + } + ); + }); + + it('should handle abort signal during streaming', async () => { + const mockClient = createMockAnthropicClient({ + streamChunks: [ + mockContentBlockStart('Hello'), + mockTextChunk(' world'), + mockTextChunk('!'), + ], + messageResponse: { + content: [{ type: 'text', text: 'Hello world!', citations: null }], + usage: createUsage({ + input_tokens: 5, + output_tokens: 15, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + }), + }, + }); + + const runner = claudeRunner( + { + name: 'claude-3-5-haiku', + client: mockClient, + }, + AnthropicConfigSchema + ); + const abortController = new AbortController(); + const chunks: any[] = []; + const sendChunk = (chunk: any) => { + chunks.push(chunk); + // Abort after first chunk + if (chunks.length === 1) { + abortController.abort(); + } + }; + + // Should throw AbortError when signal is aborted + await assert.rejects( + async () => { + await runner( + { messages: [{ role: 'user', content: [{ text: 'Hi' }] }] }, + { + streamingRequested: true, + sendChunk, + abortSignal: abortController.signal, + } + ); + }, + (error: Error) => { + // Verify abort error is thrown + assert.ok( + error.name === 'AbortError' || error.message.includes('AbortError'), + 'Should throw AbortError' + ); + return true; + } + ); + }); + + it('should handle unknown models using generic settings', async () => { + const mockClient = createMockAnthropicClient(); + const modelAction = claudeModel({ + name: 'unknown-model', + client: mockClient, + }); + + const abortSignal = new AbortController().signal; + await (modelAction as any)( + { messages: [{ role: 'user', content: [{ text: 'hi' }] }] }, + { + streamingRequested: false, + sendChunk: () => {}, + abortSignal, + } + ); + + const createStub = mockClient.messages.create as any; + assert.strictEqual(createStub.mock.calls.length, 1); + assert.strictEqual( + createStub.mock.calls[0].arguments[0].model, + 'unknown-model' + ); + }); +}); + +describe('BaseRunner helper utilities', () => { + it('should throw descriptive errors for invalid PDF data URLs', () => { + const mockClient = createMockAnthropicClient(); + const runner = new Runner({ + name: 'claude-3-5-haiku', + client: mockClient, + }); + + assert.throws( + () => + runner['toPdfDocumentSource']({ + url: 'data:text/plain;base64,AAA', + contentType: 'application/pdf', + } as any), + /PDF contentType mismatch/ + ); + }); + + it('should stringify non-media tool responses', () => { + const mockClient = createMockAnthropicClient(); + const runner = new Runner({ + name: 'claude-3-5-haiku', + client: mockClient, + }); + + const result = runner['toAnthropicToolResponseContent']({ + toolResponse: { + ref: 'call_1', + name: 'tool', + output: { value: 42 }, + }, + } as any); + + assert.deepStrictEqual(result, { + type: 'text', + text: JSON.stringify({ value: 42 }), + }); + }); + + it('should parse image data URLs', () => { + const mockClient = createMockAnthropicClient(); + const runner = new Runner({ + name: 'claude-3-5-haiku', + client: mockClient, + }); + + const source = runner['toImageSource']({ + url: 'data:image/png;base64,AAA', + contentType: 'image/png', + }); + + assert.strictEqual(source.kind, 'base64'); + if (source.kind !== 'base64') { + throw new Error('Expected base64 image source'); + } + assert.strictEqual(source.mediaType, 'image/png'); + assert.strictEqual(source.data, 'AAA'); + }); + + it('should pass through remote image URLs', () => { + const mockClient = createMockAnthropicClient(); + const runner = new Runner({ + name: 'claude-3-5-haiku', + client: mockClient, + }); + + const source = runner['toImageSource']({ + url: 'https://example.com/image.png', + contentType: 'image/png', + }); + + assert.strictEqual(source.kind, 'url'); + if (source.kind !== 'url') { + throw new Error('Expected url image source'); + } + assert.strictEqual(source.url, 'https://example.com/image.png'); + }); + + it('should parse WEBP image data URLs with matching contentType', () => { + const mockClient = createMockAnthropicClient(); + const runner = new Runner({ + name: 'claude-3-5-haiku', + client: mockClient, + }); + + const source = runner['toImageSource']({ + url: 'data:image/webp;base64,AAA', + contentType: 'image/webp', + }); + + assert.strictEqual(source.kind, 'base64'); + if (source.kind !== 'base64') { + throw new Error('Expected base64 image source'); + } + assert.strictEqual(source.mediaType, 'image/webp'); + assert.strictEqual(source.data, 'AAA'); + }); + + it('should prefer data URL content type over media.contentType for WEBP', () => { + const mockClient = createMockAnthropicClient(); + const runner = new Runner({ + name: 'claude-3-5-haiku', + client: mockClient, + }); + + // Even if contentType says PNG, data URL says WEBP - should use WEBP + const source = runner['toImageSource']({ + url: 'data:image/webp;base64,AAA', + contentType: 'image/png', + }); + + assert.strictEqual(source.kind, 'base64'); + if (source.kind !== 'base64') { + throw new Error('Expected base64 image source'); + } + // Key fix: should use data URL type (webp), not contentType (png) + assert.strictEqual(source.mediaType, 'image/webp'); + assert.strictEqual(source.data, 'AAA'); + }); + + it('should handle WEBP via toAnthropicMessageContent', () => { + const result = testRunner.toAnthropicMessageContent({ + media: { + url: 'data:image/webp;base64,AAA', + contentType: 'image/webp', + }, + }); + + assert.strictEqual(result.type, 'image'); + assert.strictEqual(result.source.type, 'base64'); + assert.strictEqual(result.source.media_type, 'image/webp'); + assert.strictEqual(result.source.data, 'AAA'); + }); + + it('should handle WEBP in tool response content', () => { + const mockClient = createMockAnthropicClient(); + const runner = new Runner({ + name: 'claude-3-5-haiku', + client: mockClient, + }); + + const result = runner['toAnthropicToolResponseContent']({ + toolResponse: { + ref: 'call_123', + name: 'get_image', + output: { + url: 'data:image/webp;base64,AAA', + contentType: 'image/webp', + }, + }, + } as any); + + assert.strictEqual(result.type, 'image'); + assert.strictEqual(result.source.type, 'base64'); + assert.strictEqual(result.source.media_type, 'image/webp'); + assert.strictEqual(result.source.data, 'AAA'); + }); + + it('should throw helpful error for text/plain in toImageSource', () => { + const mockClient = createMockAnthropicClient(); + const runner = new Runner({ + name: 'claude-3-5-haiku', + client: mockClient, + }); + + assert.throws( + () => + runner['toImageSource']({ + url: 'data:text/plain;base64,AAA', + contentType: 'text/plain', + }), + (error: Error) => { + return ( + error.message.includes('Text files should be sent as text content') && + error.message.includes('text:') + ); + } + ); + }); + + it('should throw helpful error for text/plain in toAnthropicMessageContent', () => { + assert.throws( + () => + testRunner.toAnthropicMessageContent({ + media: { + url: 'data:text/plain;base64,AAA', + contentType: 'text/plain', + }, + }), + (error: Error) => { + return ( + error.message.includes('Text files should be sent as text content') && + error.message.includes('text:') + ); + } + ); + }); + + it('should throw helpful error for text/plain in tool response', () => { + const mockClient = createMockAnthropicClient(); + const runner = new Runner({ + name: 'claude-3-5-haiku', + client: mockClient, + }); + + assert.throws( + () => + runner['toAnthropicToolResponseContent']({ + toolResponse: { + ref: 'call_123', + name: 'get_file', + output: { + url: 'data:text/plain;base64,AAA', + contentType: 'text/plain', + }, + }, + } as any), + (error: Error) => { + return error.message.includes( + 'Text files should be sent as text content' + ); + } + ); + }); + + it('should throw helpful error for text/plain with remote URL', () => { + const mockClient = createMockAnthropicClient(); + const runner = new Runner({ + name: 'claude-3-5-haiku', + client: mockClient, + }); + + assert.throws( + () => + runner['toImageSource']({ + url: 'https://example.com/file.txt', + contentType: 'text/plain', + }), + (error: Error) => { + return ( + error.message.includes('Text files should be sent as text content') && + error.message.includes('text:') + ); + } + ); + }); +}); + +describe('Runner request bodies and error branches', () => { + it('should include optional config fields in non-streaming request body', () => { + const mockClient = createMockAnthropicClient(); + const runner = new Runner({ + name: 'claude-3-5-haiku', + client: mockClient, + cacheSystemPrompt: true, + }) as Runner & RunnerProtectedMethods; + + const body = runner['toAnthropicRequestBody']( + 'claude-3-5-haiku', + { + messages: [ + { + role: 'system', + content: [{ text: 'You are helpful.' }], + }, + { + role: 'user', + content: [{ text: 'Tell me a joke' }], + }, + ], + config: { + maxOutputTokens: 256, + topK: 3, + topP: 0.75, + temperature: 0.6, + stopSequences: ['END'], + metadata: { user_id: 'user-xyz' }, + tool_choice: { type: 'auto' }, + thinking: { enabled: true, budgetTokens: 2048 }, + }, + tools: [ + { + name: 'get_weather', + description: 'Returns the weather', + inputSchema: { type: 'object' }, + }, + ], + } as unknown as GenerateRequest, + true + ); + + assert.strictEqual(body.model, 'claude-3-5-haiku'); + assert.ok(Array.isArray(body.system)); + assert.strictEqual(body.system?.[0].cache_control?.type, 'ephemeral'); + assert.strictEqual(body.max_tokens, 256); + assert.strictEqual(body.top_k, 3); + assert.strictEqual(body.top_p, 0.75); + assert.strictEqual(body.temperature, 0.6); + assert.deepStrictEqual(body.stop_sequences, ['END']); + assert.deepStrictEqual(body.metadata, { user_id: 'user-xyz' }); + assert.deepStrictEqual(body.tool_choice, { type: 'auto' }); + assert.strictEqual(body.tools?.length, 1); + assert.deepStrictEqual(body.thinking, { + type: 'enabled', + budget_tokens: 2048, + }); + }); + + it('should include optional config fields in streaming request body', () => { + const mockClient = createMockAnthropicClient(); + const runner = new Runner({ + name: 'claude-3-5-haiku', + client: mockClient, + cacheSystemPrompt: true, + }) as Runner & RunnerProtectedMethods; + + const body = runner['toAnthropicStreamingRequestBody']( + 'claude-3-5-haiku', + { + messages: [ + { + role: 'system', + content: [{ text: 'Stay brief.' }], + }, + { + role: 'user', + content: [{ text: 'Summarize the weather.' }], + }, + ], + config: { + maxOutputTokens: 64, + topK: 2, + topP: 0.6, + temperature: 0.4, + stopSequences: ['STOP'], + metadata: { user_id: 'user-abc' }, + tool_choice: { type: 'any' }, + thinking: { enabled: true, budgetTokens: 1536 }, + }, + tools: [ + { + name: 'summarize_weather', + description: 'Summarizes a forecast', + inputSchema: { type: 'object' }, + }, + ], + } as unknown as GenerateRequest, + true + ); + + assert.strictEqual(body.stream, true); + assert.ok(Array.isArray(body.system)); + assert.strictEqual(body.max_tokens, 64); + assert.strictEqual(body.top_k, 2); + assert.strictEqual(body.top_p, 0.6); + assert.strictEqual(body.temperature, 0.4); + assert.deepStrictEqual(body.stop_sequences, ['STOP']); + assert.deepStrictEqual(body.metadata, { user_id: 'user-abc' }); + assert.deepStrictEqual(body.tool_choice, { type: 'any' }); + assert.strictEqual(body.tools?.length, 1); + assert.deepStrictEqual(body.thinking, { + type: 'enabled', + budget_tokens: 1536, + }); + }); + + it('should disable thinking when explicitly turned off', () => { + const mockClient = createMockAnthropicClient(); + const runner = new Runner({ + name: 'claude-3-5-haiku', + client: mockClient, + }) as Runner & RunnerProtectedMethods; + + const body = runner['toAnthropicRequestBody']( + 'claude-3-5-haiku', + { + messages: [], + config: { + thinking: { enabled: false }, + }, + } as unknown as GenerateRequest, + false + ); + + assert.deepStrictEqual(body.thinking, { type: 'disabled' }); + }); + + it('should throw descriptive errors for missing tool refs', () => { + const mockClient = createMockAnthropicClient(); + const runner = new Runner({ + name: 'claude-3-5-haiku', + client: mockClient, + cacheSystemPrompt: false, + }) as Runner & RunnerProtectedMethods; + + assert.throws( + () => + runner['toAnthropicMessageContent']({ + toolRequest: { + name: 'get_weather', + input: {}, + }, + } as any), + /Tool request ref is required/ + ); + + assert.throws( + () => + runner['toAnthropicMessageContent']({ + toolResponse: { + ref: undefined, + name: 'get_weather', + output: 'Sunny', + }, + } as any), + /Tool response ref is required/ + ); + + assert.throws( + () => + runner['toAnthropicMessageContent']({ + data: 'unexpected', + } as any), + /Unsupported genkit part fields/ + ); + }); +}); diff --git a/js/plugins/anthropic/tests/streaming_test.ts b/js/plugins/anthropic/tests/streaming_test.ts new file mode 100644 index 0000000000..84d45e0d9a --- /dev/null +++ b/js/plugins/anthropic/tests/streaming_test.ts @@ -0,0 +1,366 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as assert from 'assert'; +import type { ModelAction } from 'genkit/model'; +import { describe, mock, test } from 'node:test'; +import { anthropic } from '../src/index.js'; +import { PluginOptions, __testClient } from '../src/types.js'; +import { + createMockAnthropicClient, + createMockAnthropicMessage, + mockContentBlockStart, + mockTextChunk, + mockToolUseChunk, +} from './mocks/anthropic-client.js'; + +describe('Streaming Integration Tests', () => { + test('should use streaming API when onChunk is provided', async () => { + const mockClient = createMockAnthropicClient({ + streamChunks: [ + mockContentBlockStart('Hello'), + mockTextChunk(' world'), + mockTextChunk('!'), + ], + messageResponse: createMockAnthropicMessage({ + text: 'Hello world!', + }), + }); + + const plugin = anthropic({ + apiKey: 'test-key', + [__testClient]: mockClient, + } as PluginOptions); + + const modelAction = plugin.resolve!( + 'model', + 'claude-3-5-haiku-20241022' + ) as ModelAction; + + const response = await modelAction( + { + messages: [{ role: 'user', content: [{ text: 'Hello' }] }], + output: { format: 'text' }, + }, + { + onChunk: mock.fn() as any, + abortSignal: new AbortController().signal, + } + ); + + // Verify final response + assert.ok(response, 'Response should be returned'); + assert.ok( + response.candidates?.[0]?.message.content[0].text, + 'Response should have text content' + ); + + // Since we can't control whether the runner chooses streaming or not from + // the plugin level, just verify we got a response + // The runner-level tests verify streaming behavior in detail + }); + + test('should handle streaming with multiple content blocks', async () => { + const mockClient = createMockAnthropicClient({ + streamChunks: [ + mockContentBlockStart('First block'), + mockTextChunk(' continues'), + { + type: 'content_block_start', + index: 1, + content_block: { + type: 'text', + text: 'Second block', + }, + } as any, + { + type: 'content_block_delta', + index: 1, + delta: { + type: 'text_delta', + text: ' here', + }, + } as any, + ], + messageResponse: createMockAnthropicMessage({ + text: 'First block continues', + }), + }); + + const plugin = anthropic({ + apiKey: 'test-key', + [__testClient]: mockClient, + } as PluginOptions); + + const modelAction = plugin.resolve!( + 'model', + 'claude-3-5-haiku-20241022' + ) as ModelAction; + + const response = await modelAction( + { + messages: [{ role: 'user', content: [{ text: 'Hello' }] }], + output: { format: 'text' }, + }, + { + onChunk: mock.fn() as any, + abortSignal: new AbortController().signal, + } + ); + + // Verify response is returned even with multiple content blocks + assert.ok(response, 'Response should be returned'); + }); + + test('should handle streaming with tool use', async () => { + const mockClient = createMockAnthropicClient({ + streamChunks: [ + mockToolUseChunk('toolu_123', 'get_weather', { city: 'NYC' }), + ], + messageResponse: createMockAnthropicMessage({ + toolUse: { + id: 'toolu_123', + name: 'get_weather', + input: { city: 'NYC' }, + }, + }), + }); + + const plugin = anthropic({ + apiKey: 'test-key', + [__testClient]: mockClient, + } as PluginOptions); + + const modelAction = plugin.resolve!( + 'model', + 'claude-3-5-haiku-20241022' + ) as ModelAction; + + const response = await modelAction( + { + messages: [{ role: 'user', content: [{ text: 'Get NYC weather' }] }], + tools: [ + { + name: 'get_weather', + description: 'Get weather for a city', + inputSchema: { + type: 'object', + properties: { + city: { type: 'string' }, + }, + required: ['city'], + }, + }, + ], + output: { format: 'text' }, + }, + { + onChunk: mock.fn() as any, + abortSignal: new AbortController().signal, + } + ); + + // Verify tool use in response + assert.ok(response.candidates?.[0]?.message.content[0].toolRequest); + assert.strictEqual( + response.candidates[0].message.content[0].toolRequest?.name, + 'get_weather' + ); + }); + + test('should handle abort signal', async () => { + const abortController = new AbortController(); + + const mockClient = createMockAnthropicClient({ + messageResponse: createMockAnthropicMessage({ + text: 'Hello world', + }), + }); + + const plugin = anthropic({ + apiKey: 'test-key', + [__testClient]: mockClient, + } as PluginOptions); + + const modelAction = plugin.resolve!( + 'model', + 'claude-3-5-haiku-20241022' + ) as ModelAction; + + // Abort before starting + abortController.abort(); + + // Test that abort signal is passed through + // The actual abort behavior is tested in runner tests + try { + await modelAction( + { + messages: [{ role: 'user', content: [{ text: 'Hello' }] }], + output: { format: 'text' }, + }, + { + onChunk: mock.fn() as any, + abortSignal: abortController.signal, + } + ); + // If we get here, the mock doesn't fully simulate abort behavior, + // which is fine since runner tests cover this + } catch (error: any) { + // Expected abort error + assert.ok( + error.message.includes('Abort') || error.name === 'AbortError', + 'Should throw abort error' + ); + } + }); + + test('should handle errors during streaming', async () => { + const mockClient = createMockAnthropicClient({ + shouldError: new Error('API error'), + }); + + const plugin = anthropic({ + apiKey: 'test-key', + [__testClient]: mockClient, + } as PluginOptions); + + const modelAction = plugin.resolve!( + 'model', + 'claude-3-5-haiku-20241022' + ) as ModelAction; + + try { + await modelAction( + { + messages: [{ role: 'user', content: [{ text: 'Hello' }] }], + output: { format: 'text' }, + }, + { + onChunk: mock.fn() as any, + abortSignal: new AbortController().signal, + } + ); + assert.fail('Should have thrown an error'); + } catch (error: any) { + assert.strictEqual(error.message, 'API error'); + } + }); + + test('should handle empty response', async () => { + const mockClient = createMockAnthropicClient({ + streamChunks: [], + messageResponse: createMockAnthropicMessage({ + text: '', + }), + }); + + const plugin = anthropic({ + apiKey: 'test-key', + [__testClient]: mockClient, + } as PluginOptions); + + const modelAction = plugin.resolve!( + 'model', + 'claude-3-5-haiku-20241022' + ) as ModelAction; + + const response = await modelAction( + { + messages: [{ role: 'user', content: [{ text: 'Hello' }] }], + output: { format: 'text' }, + }, + { + onChunk: mock.fn() as any, + abortSignal: new AbortController().signal, + } + ); + + assert.ok(response, 'Should return response even with empty content'); + }); + + test('should include usage metadata in streaming response', async () => { + const mockClient = createMockAnthropicClient({ + streamChunks: [mockContentBlockStart('Response'), mockTextChunk(' text')], + messageResponse: createMockAnthropicMessage({ + text: 'Response text', + usage: { + input_tokens: 50, + output_tokens: 25, + }, + }), + }); + + const plugin = anthropic({ + apiKey: 'test-key', + [__testClient]: mockClient, + } as PluginOptions); + + const modelAction = plugin.resolve!( + 'model', + 'claude-3-5-haiku-20241022' + ) as ModelAction; + + const response = await modelAction( + { + messages: [{ role: 'user', content: [{ text: 'Hello' }] }], + output: { format: 'text' }, + }, + { + onChunk: mock.fn() as any, + abortSignal: new AbortController().signal, + } + ); + + assert.ok(response.usage, 'Should include usage metadata'); + assert.strictEqual(response.usage?.inputTokens, 50); + assert.strictEqual(response.usage?.outputTokens, 25); + }); + + test('should not stream when onChunk is not provided', async () => { + const mockClient = createMockAnthropicClient({ + messageResponse: createMockAnthropicMessage({ + text: 'Non-streaming response', + }), + }); + + const plugin = anthropic({ + apiKey: 'test-key', + [__testClient]: mockClient, + } as PluginOptions); + + const modelAction = plugin.resolve!( + 'model', + 'claude-3-5-haiku-20241022' + ) as ModelAction; + + await modelAction( + { + messages: [{ role: 'user', content: [{ text: 'Hello' }] }], + }, + { + abortSignal: new AbortController().signal, + } + ); + + // Verify non-streaming API was called + const createStub = mockClient.messages.create as any; + assert.strictEqual(createStub.mock.calls.length, 1); + + // Verify stream API was NOT called + const streamStub = mockClient.messages.stream as any; + assert.strictEqual(streamStub.mock.calls.length, 0); + }); +}); diff --git a/js/plugins/anthropic/tests/types_test.ts b/js/plugins/anthropic/tests/types_test.ts new file mode 100644 index 0000000000..64c91e1547 --- /dev/null +++ b/js/plugins/anthropic/tests/types_test.ts @@ -0,0 +1,89 @@ +/** + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as assert from 'assert'; +import { z } from 'genkit'; +import { describe, it } from 'node:test'; +import { AnthropicConfigSchema, resolveBetaEnabled } from '../src/types.js'; + +describe('resolveBetaEnabled', () => { + it('should return true when config.apiVersion is beta', () => { + const config: z.infer = { + apiVersion: 'beta', + }; + assert.strictEqual(resolveBetaEnabled(config, 'stable'), true); + }); + + it('should return true when pluginDefaultApiVersion is beta', () => { + assert.strictEqual(resolveBetaEnabled(undefined, 'beta'), true); + }); + + it('should return false when config.apiVersion is stable', () => { + const config: z.infer = { + apiVersion: 'stable', + }; + assert.strictEqual(resolveBetaEnabled(config, 'stable'), false); + }); + + it('should return false when both are stable', () => { + const config: z.infer = { + apiVersion: 'stable', + }; + assert.strictEqual(resolveBetaEnabled(config, 'stable'), false); + }); + + it('should return false when neither is specified', () => { + assert.strictEqual(resolveBetaEnabled(undefined, undefined), false); + }); + + it('should return false when config is undefined and plugin default is stable', () => { + assert.strictEqual(resolveBetaEnabled(undefined, 'stable'), false); + }); + + it('should prioritize config.apiVersion over pluginDefaultApiVersion (beta over stable)', () => { + const config: z.infer = { + apiVersion: 'beta', + }; + // Even though plugin default is stable, request config should override + assert.strictEqual(resolveBetaEnabled(config, 'stable'), true); + }); + + it('should prioritize config.apiVersion over pluginDefaultApiVersion (stable over beta)', () => { + const config: z.infer = { + apiVersion: 'stable', + }; + // Request explicitly wants stable, should override plugin default + assert.strictEqual(resolveBetaEnabled(config, 'beta'), false); + }); + + it('should return false when config is empty object', () => { + const config: z.infer = {}; + assert.strictEqual(resolveBetaEnabled(config, undefined), false); + }); + + it('should return true when config is empty but plugin default is beta', () => { + const config: z.infer = {}; + assert.strictEqual(resolveBetaEnabled(config, 'beta'), true); + }); + + it('should handle config with other fields but no apiVersion', () => { + const config: z.infer = { + metadata: { user_id: 'test-user' }, + }; + assert.strictEqual(resolveBetaEnabled(config, 'stable'), false); + assert.strictEqual(resolveBetaEnabled(config, 'beta'), true); + }); +});