From d5cbd8f9a8d4c88372a841426498363c2fcfc4a5 Mon Sep 17 00:00:00 2001 From: Markus Ecker Date: Tue, 16 Sep 2025 14:38:21 +0200 Subject: [PATCH 01/25] ensure each agent is cloneable --- .../integrations/mastra/src/mastra.ts | 9 ++- .../integrations/vercel-ai-sdk/src/index.ts | 7 +- .../packages/client/src/agent/agent.ts | 78 ++++++++++--------- 3 files changed, 56 insertions(+), 38 deletions(-) diff --git a/typescript-sdk/integrations/mastra/src/mastra.ts b/typescript-sdk/integrations/mastra/src/mastra.ts index f8310767b..9de9e70dc 100644 --- a/typescript-sdk/integrations/mastra/src/mastra.ts +++ b/typescript-sdk/integrations/mastra/src/mastra.ts @@ -52,13 +52,18 @@ export class MastraAgent extends AbstractAgent { resourceId?: string; runtimeContext?: RuntimeContext; - constructor({ agent, resourceId, runtimeContext, ...rest }: MastraAgentConfig) { + constructor(private config: MastraAgentConfig) { + const { agent, resourceId, runtimeContext, ...rest } = config; super(rest); this.agent = agent; this.resourceId = resourceId; this.runtimeContext = runtimeContext ?? new RuntimeContext(); } + public clone() { + return new MastraAgent(this.config); + } + protected run(input: RunAgentInput): Observable { let messageId = randomUUID(); @@ -250,7 +255,7 @@ export class MastraAgent extends AbstractAgent { ); const resourceId = this.resourceId ?? threadId; const convertedMessages = convertAGUIMessagesToMastra(messages); - this.runtimeContext?.set('ag-ui', { context: inputContext }); + this.runtimeContext?.set("ag-ui", { context: inputContext }); const runtimeContext = this.runtimeContext; if (this.isLocalMastraAgent(this.agent)) { diff --git a/typescript-sdk/integrations/vercel-ai-sdk/src/index.ts b/typescript-sdk/integrations/vercel-ai-sdk/src/index.ts index b06664f34..602e45e14 100644 --- a/typescript-sdk/integrations/vercel-ai-sdk/src/index.ts +++ b/typescript-sdk/integrations/vercel-ai-sdk/src/index.ts @@ -48,13 +48,18 @@ export class VercelAISDKAgent extends AbstractAgent { model: LanguageModelV1; maxSteps: number; toolChoice: ToolChoice>; - constructor({ model, maxSteps, toolChoice, ...rest }: VercelAISDKAgentConfig) { + constructor(private config: VercelAISDKAgentConfig) { + const { model, maxSteps, toolChoice, ...rest } = config; super({ ...rest }); this.model = model; this.maxSteps = maxSteps ?? 1; this.toolChoice = toolChoice ?? "auto"; } + public clone() { + return new VercelAISDKAgent(this.config); + } + protected run(input: RunAgentInput): Observable { const finalMessages: Message[] = input.messages; diff --git a/typescript-sdk/packages/client/src/agent/agent.ts b/typescript-sdk/packages/client/src/agent/agent.ts index f596a625c..6fbbf989c 100644 --- a/typescript-sdk/packages/client/src/agent/agent.ts +++ b/typescript-sdk/packages/client/src/agent/agent.ts @@ -27,6 +27,7 @@ export abstract class AbstractAgent { public state: State; public debug: boolean = false; public subscribers: AgentSubscriber[] = []; + public isRunning: boolean = false; constructor({ agentId, @@ -59,43 +60,50 @@ export abstract class AbstractAgent { parameters?: RunAgentParameters, subscriber?: AgentSubscriber, ): Promise { - this.agentId = this.agentId ?? uuidv4(); - const input = this.prepareRunAgentInput(parameters); - let result: any = undefined; - const currentMessageIds = new Set(this.messages.map((message) => message.id)); - - const subscribers: AgentSubscriber[] = [ - { - onRunFinishedEvent: (params) => { - result = params.result; + try { + this.isRunning = true; + this.agentId = this.agentId ?? uuidv4(); + const input = this.prepareRunAgentInput(parameters); + let result: any = undefined; + const currentMessageIds = new Set(this.messages.map((message) => message.id)); + + const subscribers: AgentSubscriber[] = [ + { + onRunFinishedEvent: (params) => { + result = params.result; + }, }, - }, - ...this.subscribers, - subscriber ?? {}, - ]; - - await this.onInitialize(input, subscribers); - - const pipeline = pipe( - () => this.run(input), - transformChunks(this.debug), - verifyEvents(this.debug), - (source$) => this.apply(input, source$, subscribers), - (source$) => this.processApplyEvents(input, source$, subscribers), - catchError((error) => { - return this.onError(input, error, subscribers); - }), - finalize(() => { - void this.onFinalize(input, subscribers); - }), - ); - - return lastValueFrom(pipeline(of(null))).then(() => { - const newMessages = structuredClone_(this.messages).filter( - (message: Message) => !currentMessageIds.has(message.id), + ...this.subscribers, + subscriber ?? {}, + ]; + + await this.onInitialize(input, subscribers); + + const pipeline = pipe( + () => this.run(input), + transformChunks(this.debug), + verifyEvents(this.debug), + (source$) => this.apply(input, source$, subscribers), + (source$) => this.processApplyEvents(input, source$, subscribers), + catchError((error) => { + this.isRunning = false; + return this.onError(input, error, subscribers); + }), + finalize(() => { + this.isRunning = false; + void this.onFinalize(input, subscribers); + }), ); - return { result, newMessages }; - }); + + return lastValueFrom(pipeline(of(null))).then(() => { + const newMessages = structuredClone_(this.messages).filter( + (message: Message) => !currentMessageIds.has(message.id), + ); + return { result, newMessages }; + }); + } finally { + this.isRunning = false; + } } public abortRun() {} From 1d0896aeb5252db1d866740046159a9e74b78e4b Mon Sep 17 00:00:00 2001 From: Markus Ecker Date: Tue, 16 Sep 2025 15:17:18 +0200 Subject: [PATCH 02/25] add observable events$ property --- .../src/agent/__tests__/agent-result.test.ts | 71 +++++++++++++++++++ .../packages/client/src/agent/agent.ts | 10 ++- 2 files changed, 80 insertions(+), 1 deletion(-) diff --git a/typescript-sdk/packages/client/src/agent/__tests__/agent-result.test.ts b/typescript-sdk/packages/client/src/agent/__tests__/agent-result.test.ts index 38a51c0a9..b25b8ded4 100644 --- a/typescript-sdk/packages/client/src/agent/__tests__/agent-result.test.ts +++ b/typescript-sdk/packages/client/src/agent/__tests__/agent-result.test.ts @@ -309,6 +309,77 @@ describe("Agent Result", () => { }); }); + describe("events$ stream", () => { + it("collects events from multiple runs", async () => { + const receivedEvents: BaseEvent[] = []; + const subscription = agent.events$.subscribe((event) => { + receivedEvents.push(event); + }); + + const firstRunEvents: BaseEvent[] = [ + { + type: EventType.RUN_STARTED, + threadId: "test-thread", + runId: "run-1", + } as RunStartedEvent, + { + type: EventType.RUN_FINISHED, + threadId: "test-thread", + runId: "run-1", + } as RunFinishedEvent, + ]; + + agent.setEventsToEmit(firstRunEvents); + await agent.runAgent(); + + const secondRunEvents: BaseEvent[] = [ + { + type: EventType.RUN_STARTED, + threadId: "test-thread", + runId: "run-2", + } as RunStartedEvent, + { + type: EventType.RUN_FINISHED, + threadId: "test-thread", + runId: "run-2", + } as RunFinishedEvent, + ]; + + agent.setEventsToEmit(secondRunEvents); + await agent.runAgent(); + + expect(receivedEvents).toEqual([...firstRunEvents, ...secondRunEvents]); + + subscription.unsubscribe(); + }); + + it("replays historical events to late subscribers", async () => { + const eventsToEmit: BaseEvent[] = [ + { + type: EventType.RUN_STARTED, + threadId: "test-thread", + runId: "run-1", + } as RunStartedEvent, + { + type: EventType.RUN_FINISHED, + threadId: "test-thread", + runId: "run-1", + } as RunFinishedEvent, + ]; + + agent.setEventsToEmit(eventsToEmit); + await agent.runAgent(); + + const lateSubscriberEvents: BaseEvent[] = []; + const subscription = agent.events$.subscribe((event) => { + lateSubscriberEvents.push(event); + }); + subscription.unsubscribe(); + + expect(lateSubscriberEvents).toEqual(eventsToEmit); + }); + }); + describe("combined result and newMessages", () => { it("should return both result and newMessages correctly", async () => { const newMessages: Message[] = [ diff --git a/typescript-sdk/packages/client/src/agent/agent.ts b/typescript-sdk/packages/client/src/agent/agent.ts index 6fbbf989c..86e58107c 100644 --- a/typescript-sdk/packages/client/src/agent/agent.ts +++ b/typescript-sdk/packages/client/src/agent/agent.ts @@ -6,7 +6,7 @@ import { v4 as uuidv4 } from "uuid"; import { structuredClone_ } from "@/utils"; import { catchError, map, tap } from "rxjs/operators"; import { finalize } from "rxjs/operators"; -import { pipe, Observable, from, of } from "rxjs"; +import { pipe, Observable, ReplaySubject, from, of } from "rxjs"; import { verifyEvents } from "@/verify"; import { convertToLegacyEvents } from "@/legacy/convert"; import { LegacyRuntimeProtocolEvent } from "@/legacy/types"; @@ -28,6 +28,8 @@ export abstract class AbstractAgent { public debug: boolean = false; public subscribers: AgentSubscriber[] = []; public isRunning: boolean = false; + private readonly eventsSubject = new ReplaySubject(); + public readonly events$ = this.eventsSubject.asObservable(); constructor({ agentId, @@ -83,6 +85,9 @@ export abstract class AbstractAgent { () => this.run(input), transformChunks(this.debug), verifyEvents(this.debug), + tap((event) => { + this.eventsSubject.next(event); + }), (source$) => this.apply(input, source$, subscribers), (source$) => this.processApplyEvents(input, source$, subscribers), catchError((error) => { @@ -427,6 +432,9 @@ export abstract class AbstractAgent { return this.run(input).pipe( transformChunks(this.debug), verifyEvents(this.debug), + tap((event) => { + this.eventsSubject.next(event); + }), convertToLegacyEvents(this.threadId, input.runId, this.agentId), (events$: Observable) => { return events$.pipe( From 8c1a810603201d2e6bf2b76b6a996b69e0f0b388 Mon Sep 17 00:00:00 2001 From: Markus Ecker Date: Tue, 16 Sep 2025 16:20:20 +0200 Subject: [PATCH 03/25] v0.0.38-alpha.0 --- typescript-sdk/packages/cli/package.json | 2 +- typescript-sdk/packages/client/package.json | 2 +- typescript-sdk/packages/core/package.json | 2 +- typescript-sdk/packages/encoder/package.json | 2 +- typescript-sdk/packages/proto/package.json | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/typescript-sdk/packages/cli/package.json b/typescript-sdk/packages/cli/package.json index b8e37cb32..546356ed7 100644 --- a/typescript-sdk/packages/cli/package.json +++ b/typescript-sdk/packages/cli/package.json @@ -1,7 +1,7 @@ { "name": "create-ag-ui-app", "author": "Markus Ecker ", - "version": "0.0.39", + "version": "0.0.40-alpha.0", "private": false, "publishConfig": { "access": "public" diff --git a/typescript-sdk/packages/client/package.json b/typescript-sdk/packages/client/package.json index c503b0268..2ef5560c2 100644 --- a/typescript-sdk/packages/client/package.json +++ b/typescript-sdk/packages/client/package.json @@ -1,7 +1,7 @@ { "name": "@ag-ui/client", "author": "Markus Ecker ", - "version": "0.0.37", + "version": "0.0.38-alpha.0", "private": false, "publishConfig": { "access": "public" diff --git a/typescript-sdk/packages/core/package.json b/typescript-sdk/packages/core/package.json index 67bca0bbc..6ad585b1c 100644 --- a/typescript-sdk/packages/core/package.json +++ b/typescript-sdk/packages/core/package.json @@ -1,7 +1,7 @@ { "name": "@ag-ui/core", "author": "Markus Ecker ", - "version": "0.0.37", + "version": "0.0.38-alpha.0", "private": false, "publishConfig": { "access": "public" diff --git a/typescript-sdk/packages/encoder/package.json b/typescript-sdk/packages/encoder/package.json index 221b7afe4..7108ca48a 100644 --- a/typescript-sdk/packages/encoder/package.json +++ b/typescript-sdk/packages/encoder/package.json @@ -1,7 +1,7 @@ { "name": "@ag-ui/encoder", "author": "Markus Ecker ", - "version": "0.0.37", + "version": "0.0.38-alpha.0", "private": false, "publishConfig": { "access": "public" diff --git a/typescript-sdk/packages/proto/package.json b/typescript-sdk/packages/proto/package.json index 787edfabe..c069f26b6 100644 --- a/typescript-sdk/packages/proto/package.json +++ b/typescript-sdk/packages/proto/package.json @@ -1,7 +1,7 @@ { "name": "@ag-ui/proto", "author": "Markus Ecker ", - "version": "0.0.37", + "version": "0.0.38-alpha.0", "private": false, "publishConfig": { "access": "public" From 32dc8a1996dc022ba219b39fbc36fa5e8051fd81 Mon Sep 17 00:00:00 2001 From: Markus Ecker Date: Wed, 17 Sep 2025 11:18:38 +0200 Subject: [PATCH 04/25] Implement connect --- .../packages/client/src/agent/agent.ts | 62 ++++++++++++++++++- typescript-sdk/packages/core/src/types.ts | 6 ++ 2 files changed, 67 insertions(+), 1 deletion(-) diff --git a/typescript-sdk/packages/client/src/agent/agent.ts b/typescript-sdk/packages/client/src/agent/agent.ts index 86e58107c..8eac12b68 100644 --- a/typescript-sdk/packages/client/src/agent/agent.ts +++ b/typescript-sdk/packages/client/src/agent/agent.ts @@ -6,13 +6,14 @@ import { v4 as uuidv4 } from "uuid"; import { structuredClone_ } from "@/utils"; import { catchError, map, tap } from "rxjs/operators"; import { finalize } from "rxjs/operators"; -import { pipe, Observable, ReplaySubject, from, of } from "rxjs"; +import { pipe, Observable, ReplaySubject, from, of, EMPTY } from "rxjs"; import { verifyEvents } from "@/verify"; import { convertToLegacyEvents } from "@/legacy/convert"; import { LegacyRuntimeProtocolEvent } from "@/legacy/types"; import { lastValueFrom } from "rxjs"; import { transformChunks } from "@/chunks"; import { AgentStateMutation, AgentSubscriber, runSubscribersWithMutation } from "./subscriber"; +import { AGUIConnectNotImplementedError } from "@ag-ui/core"; export interface RunAgentResult { result: any; @@ -111,6 +112,65 @@ export abstract class AbstractAgent { } } + protected connect(input: RunAgentInput): Observable { + throw new AGUIConnectNotImplementedError(); + } + public async connectAgent( + parameters?: RunAgentParameters, + subscriber?: AgentSubscriber, + ): Promise { + try { + this.isRunning = true; + this.agentId = this.agentId ?? uuidv4(); + const input = this.prepareRunAgentInput(parameters); + let result: any = undefined; + const currentMessageIds = new Set(this.messages.map((message) => message.id)); + + const subscribers: AgentSubscriber[] = [ + { + onRunFinishedEvent: (params) => { + result = params.result; + }, + }, + ...this.subscribers, + subscriber ?? {}, + ]; + + await this.onInitialize(input, subscribers); + + const pipeline = pipe( + () => this.connect(input), + transformChunks(this.debug), + verifyEvents(this.debug), + tap((event) => { + this.eventsSubject.next(event); + }), + (source$) => this.apply(input, source$, subscribers), + (source$) => this.processApplyEvents(input, source$, subscribers), + catchError((error) => { + this.isRunning = false; + if (!(error instanceof AGUIConnectNotImplementedError)) { + return this.onError(input, error, subscribers); + } + return EMPTY; + }), + finalize(() => { + this.isRunning = false; + void this.onFinalize(input, subscribers); + }), + ); + + return lastValueFrom(pipeline(of(null))).then(() => { + const newMessages = structuredClone_(this.messages).filter( + (message: Message) => !currentMessageIds.has(message.id), + ); + return { result, newMessages }; + }); + } finally { + this.isRunning = false; + } + } + public abortRun() {} protected apply( diff --git a/typescript-sdk/packages/core/src/types.ts b/typescript-sdk/packages/core/src/types.ts index 1abb31a0b..0b5b17224 100644 --- a/typescript-sdk/packages/core/src/types.ts +++ b/typescript-sdk/packages/core/src/types.ts @@ -105,3 +105,9 @@ export class AGUIError extends Error { super(message); } } + +export class AGUIConnectNotImplementedError extends AGUIError { + constructor() { + super("Connect not implemented. This method is not supported by the current agent."); + } +} From cc833dc567fcc241db511e3e4ad58452b3dfce52 Mon Sep 17 00:00:00 2001 From: Markus Ecker Date: Wed, 17 Sep 2025 11:18:52 +0200 Subject: [PATCH 05/25] v0.0.40-alpha.1 --- typescript-sdk/packages/cli/package.json | 2 +- typescript-sdk/packages/client/package.json | 2 +- typescript-sdk/packages/core/package.json | 2 +- typescript-sdk/packages/encoder/package.json | 2 +- typescript-sdk/packages/proto/package.json | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/typescript-sdk/packages/cli/package.json b/typescript-sdk/packages/cli/package.json index 546356ed7..86a5d9a80 100644 --- a/typescript-sdk/packages/cli/package.json +++ b/typescript-sdk/packages/cli/package.json @@ -1,7 +1,7 @@ { "name": "create-ag-ui-app", "author": "Markus Ecker ", - "version": "0.0.40-alpha.0", + "version": "0.0.40-alpha.1", "private": false, "publishConfig": { "access": "public" diff --git a/typescript-sdk/packages/client/package.json b/typescript-sdk/packages/client/package.json index 2ef5560c2..e1835b796 100644 --- a/typescript-sdk/packages/client/package.json +++ b/typescript-sdk/packages/client/package.json @@ -1,7 +1,7 @@ { "name": "@ag-ui/client", "author": "Markus Ecker ", - "version": "0.0.38-alpha.0", + "version": "0.0.38-alpha.1", "private": false, "publishConfig": { "access": "public" diff --git a/typescript-sdk/packages/core/package.json b/typescript-sdk/packages/core/package.json index 6ad585b1c..5f1cd241c 100644 --- a/typescript-sdk/packages/core/package.json +++ b/typescript-sdk/packages/core/package.json @@ -1,7 +1,7 @@ { "name": "@ag-ui/core", "author": "Markus Ecker ", - "version": "0.0.38-alpha.0", + "version": "0.0.38-alpha.1", "private": false, "publishConfig": { "access": "public" diff --git a/typescript-sdk/packages/encoder/package.json b/typescript-sdk/packages/encoder/package.json index 7108ca48a..4ab8e3e7c 100644 --- a/typescript-sdk/packages/encoder/package.json +++ b/typescript-sdk/packages/encoder/package.json @@ -1,7 +1,7 @@ { "name": "@ag-ui/encoder", "author": "Markus Ecker ", - "version": "0.0.38-alpha.0", + "version": "0.0.38-alpha.1", "private": false, "publishConfig": { "access": "public" diff --git a/typescript-sdk/packages/proto/package.json b/typescript-sdk/packages/proto/package.json index c069f26b6..ccb76de6c 100644 --- a/typescript-sdk/packages/proto/package.json +++ b/typescript-sdk/packages/proto/package.json @@ -1,7 +1,7 @@ { "name": "@ag-ui/proto", "author": "Markus Ecker ", - "version": "0.0.38-alpha.0", + "version": "0.0.38-alpha.1", "private": false, "publishConfig": { "access": "public" From 954cfdccafe2b9589774358d37fa1de82a3e1ffd Mon Sep 17 00:00:00 2001 From: Markus Ecker Date: Thu, 18 Sep 2025 16:05:24 +0200 Subject: [PATCH 06/25] v0.0.38-alpha.2 --- typescript-sdk/packages/cli/package.json | 2 +- typescript-sdk/packages/client/package.json | 2 +- typescript-sdk/packages/client/src/agent/index.ts | 1 + typescript-sdk/packages/core/package.json | 2 +- typescript-sdk/packages/encoder/package.json | 2 +- typescript-sdk/packages/proto/package.json | 2 +- 6 files changed, 6 insertions(+), 5 deletions(-) diff --git a/typescript-sdk/packages/cli/package.json b/typescript-sdk/packages/cli/package.json index 86a5d9a80..629b098c4 100644 --- a/typescript-sdk/packages/cli/package.json +++ b/typescript-sdk/packages/cli/package.json @@ -1,7 +1,7 @@ { "name": "create-ag-ui-app", "author": "Markus Ecker ", - "version": "0.0.40-alpha.1", + "version": "0.0.40-alpha.2", "private": false, "publishConfig": { "access": "public" diff --git a/typescript-sdk/packages/client/package.json b/typescript-sdk/packages/client/package.json index e1835b796..0ce92a719 100644 --- a/typescript-sdk/packages/client/package.json +++ b/typescript-sdk/packages/client/package.json @@ -1,7 +1,7 @@ { "name": "@ag-ui/client", "author": "Markus Ecker ", - "version": "0.0.38-alpha.1", + "version": "0.0.38-alpha.2", "private": false, "publishConfig": { "access": "public" diff --git a/typescript-sdk/packages/client/src/agent/index.ts b/typescript-sdk/packages/client/src/agent/index.ts index 945724cd0..046bfa90b 100644 --- a/typescript-sdk/packages/client/src/agent/index.ts +++ b/typescript-sdk/packages/client/src/agent/index.ts @@ -2,3 +2,4 @@ export { AbstractAgent } from "./agent"; export type { RunAgentResult } from "./agent"; export { HttpAgent } from "./http"; export type { AgentConfig, HttpAgentConfig, RunAgentParameters } from "./types"; +export type { AgentSubscriber, AgentStateMutation, AgentSubscriberParams } from "./subscriber"; diff --git a/typescript-sdk/packages/core/package.json b/typescript-sdk/packages/core/package.json index 5f1cd241c..be3cfeee0 100644 --- a/typescript-sdk/packages/core/package.json +++ b/typescript-sdk/packages/core/package.json @@ -1,7 +1,7 @@ { "name": "@ag-ui/core", "author": "Markus Ecker ", - "version": "0.0.38-alpha.1", + "version": "0.0.38-alpha.2", "private": false, "publishConfig": { "access": "public" diff --git a/typescript-sdk/packages/encoder/package.json b/typescript-sdk/packages/encoder/package.json index 4ab8e3e7c..15ce98b57 100644 --- a/typescript-sdk/packages/encoder/package.json +++ b/typescript-sdk/packages/encoder/package.json @@ -1,7 +1,7 @@ { "name": "@ag-ui/encoder", "author": "Markus Ecker ", - "version": "0.0.38-alpha.1", + "version": "0.0.38-alpha.2", "private": false, "publishConfig": { "access": "public" diff --git a/typescript-sdk/packages/proto/package.json b/typescript-sdk/packages/proto/package.json index ccb76de6c..8e980061b 100644 --- a/typescript-sdk/packages/proto/package.json +++ b/typescript-sdk/packages/proto/package.json @@ -1,7 +1,7 @@ { "name": "@ag-ui/proto", "author": "Markus Ecker ", - "version": "0.0.38-alpha.1", + "version": "0.0.38-alpha.2", "private": false, "publishConfig": { "access": "public" From 49f95e163e66e17a81a0b27b6a05896bb2d9d72f Mon Sep 17 00:00:00 2001 From: Markus Ecker Date: Wed, 24 Sep 2025 21:56:14 +0200 Subject: [PATCH 07/25] Add docs --- docs/sdk/js/client/abstract-agent.mdx | 40 +++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/docs/sdk/js/client/abstract-agent.mdx b/docs/sdk/js/client/abstract-agent.mdx index 3738b0aae..cf322d214 100644 --- a/docs/sdk/js/client/abstract-agent.mdx +++ b/docs/sdk/js/client/abstract-agent.mdx @@ -111,6 +111,33 @@ Creates a deep copy of the agent instance. clone(): AbstractAgent ``` +### connectAgent() + +Establishes a persistent connection with an agent that implements the +`connect()` method. + +```typescript +connectAgent(parameters?: RunAgentParameters, subscriber?: AgentSubscriber): Promise +``` + +Similar to `runAgent()` but uses the `connect()` method internally. The agent +must implement `connect()` or this functionality must be provided by a framework +like [CopilotKit](https://copilotkit.ai). + +## Observable Properties + +### events$ + +An observable stream of all events emitted during agent execution. + +```typescript +events$: Observable +``` + +This property provides direct access to the agent's event stream. Events are +stored using a `ReplaySubject`, allowing late subscribers will receive all +historical events. + ## Properties - `agentId`: Unique identifier for the agent instance @@ -118,6 +145,8 @@ clone(): AbstractAgent - `threadId`: Conversation thread identifier - `messages`: Array of conversation messages - `state`: Current agent state object +- `events$`: Observable stream of all `BaseEvent` objects emitted during agent + execution (replayed for late subscribers) ## Protected Methods @@ -131,6 +160,17 @@ Executes the agent and returns an observable event stream. protected abstract run(input: RunAgentInput): RunAgent ``` +### connect() + +Establishes a persistent connection and returns an observable event stream. + +```typescript +protected connect(input: RunAgentInput): RunAgent +``` + +Override this method to implement persistent connections. Default implementation +throws `ConnectNotImplementedError`. + ### apply() Processes events from the run and updates the agent state. From 18be93f28e5390055a00f2a284f19a6546c03393 Mon Sep 17 00:00:00 2001 From: Markus Ecker Date: Wed, 1 Oct 2025 13:15:15 +0200 Subject: [PATCH 08/25] remove events$ --- .../src/agent/__tests__/agent-result.test.ts | 71 ------------------- .../packages/client/src/agent/agent.ts | 13 +--- 2 files changed, 1 insertion(+), 83 deletions(-) diff --git a/typescript-sdk/packages/client/src/agent/__tests__/agent-result.test.ts b/typescript-sdk/packages/client/src/agent/__tests__/agent-result.test.ts index b25b8ded4..38a51c0a9 100644 --- a/typescript-sdk/packages/client/src/agent/__tests__/agent-result.test.ts +++ b/typescript-sdk/packages/client/src/agent/__tests__/agent-result.test.ts @@ -309,77 +309,6 @@ describe("Agent Result", () => { }); }); - describe("events$ stream", () => { - it("collects events from multiple runs", async () => { - const receivedEvents: BaseEvent[] = []; - const subscription = agent.events$.subscribe((event) => { - receivedEvents.push(event); - }); - - const firstRunEvents: BaseEvent[] = [ - { - type: EventType.RUN_STARTED, - threadId: "test-thread", - runId: "run-1", - } as RunStartedEvent, - { - type: EventType.RUN_FINISHED, - threadId: "test-thread", - runId: "run-1", - } as RunFinishedEvent, - ]; - - agent.setEventsToEmit(firstRunEvents); - await agent.runAgent(); - - const secondRunEvents: BaseEvent[] = [ - { - type: EventType.RUN_STARTED, - threadId: "test-thread", - runId: "run-2", - } as RunStartedEvent, - { - type: EventType.RUN_FINISHED, - threadId: "test-thread", - runId: "run-2", - } as RunFinishedEvent, - ]; - - agent.setEventsToEmit(secondRunEvents); - await agent.runAgent(); - - expect(receivedEvents).toEqual([...firstRunEvents, ...secondRunEvents]); - - subscription.unsubscribe(); - }); - - it("replays historical events to late subscribers", async () => { - const eventsToEmit: BaseEvent[] = [ - { - type: EventType.RUN_STARTED, - threadId: "test-thread", - runId: "run-1", - } as RunStartedEvent, - { - type: EventType.RUN_FINISHED, - threadId: "test-thread", - runId: "run-1", - } as RunFinishedEvent, - ]; - - agent.setEventsToEmit(eventsToEmit); - await agent.runAgent(); - - const lateSubscriberEvents: BaseEvent[] = []; - const subscription = agent.events$.subscribe((event) => { - lateSubscriberEvents.push(event); - }); - subscription.unsubscribe(); - - expect(lateSubscriberEvents).toEqual(eventsToEmit); - }); - }); - describe("combined result and newMessages", () => { it("should return both result and newMessages correctly", async () => { const newMessages: Message[] = [ diff --git a/typescript-sdk/packages/client/src/agent/agent.ts b/typescript-sdk/packages/client/src/agent/agent.ts index 8eac12b68..b81c3a140 100644 --- a/typescript-sdk/packages/client/src/agent/agent.ts +++ b/typescript-sdk/packages/client/src/agent/agent.ts @@ -6,7 +6,7 @@ import { v4 as uuidv4 } from "uuid"; import { structuredClone_ } from "@/utils"; import { catchError, map, tap } from "rxjs/operators"; import { finalize } from "rxjs/operators"; -import { pipe, Observable, ReplaySubject, from, of, EMPTY } from "rxjs"; +import { pipe, Observable, from, of, EMPTY } from "rxjs"; import { verifyEvents } from "@/verify"; import { convertToLegacyEvents } from "@/legacy/convert"; import { LegacyRuntimeProtocolEvent } from "@/legacy/types"; @@ -29,8 +29,6 @@ export abstract class AbstractAgent { public debug: boolean = false; public subscribers: AgentSubscriber[] = []; public isRunning: boolean = false; - private readonly eventsSubject = new ReplaySubject(); - public readonly events$ = this.eventsSubject.asObservable(); constructor({ agentId, @@ -86,9 +84,6 @@ export abstract class AbstractAgent { () => this.run(input), transformChunks(this.debug), verifyEvents(this.debug), - tap((event) => { - this.eventsSubject.next(event); - }), (source$) => this.apply(input, source$, subscribers), (source$) => this.processApplyEvents(input, source$, subscribers), catchError((error) => { @@ -142,9 +137,6 @@ export abstract class AbstractAgent { () => this.connect(input), transformChunks(this.debug), verifyEvents(this.debug), - tap((event) => { - this.eventsSubject.next(event); - }), (source$) => this.apply(input, source$, subscribers), (source$) => this.processApplyEvents(input, source$, subscribers), catchError((error) => { @@ -492,9 +484,6 @@ export abstract class AbstractAgent { return this.run(input).pipe( transformChunks(this.debug), verifyEvents(this.debug), - tap((event) => { - this.eventsSubject.next(event); - }), convertToLegacyEvents(this.threadId, input.runId, this.agentId), (events$: Observable) => { return events$.pipe( From 11067c0e44f84eab9e51517db76483a2d3cd3688 Mon Sep 17 00:00:00 2001 From: Markus Ecker Date: Wed, 1 Oct 2025 13:19:35 +0200 Subject: [PATCH 09/25] 0.0.40-alpha.2 --- typescript-sdk/packages/client/package.json | 2 +- typescript-sdk/packages/core/package.json | 2 +- typescript-sdk/packages/encoder/package.json | 2 +- typescript-sdk/packages/proto/package.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/typescript-sdk/packages/client/package.json b/typescript-sdk/packages/client/package.json index a5aef3881..a80cf385f 100644 --- a/typescript-sdk/packages/client/package.json +++ b/typescript-sdk/packages/client/package.json @@ -1,7 +1,7 @@ { "name": "@ag-ui/client", "author": "Markus Ecker ", - "version": "0.0.38", + "version": "0.0.40-alpha.2", "private": false, "publishConfig": { "access": "public" diff --git a/typescript-sdk/packages/core/package.json b/typescript-sdk/packages/core/package.json index e6eebbddb..9662886fc 100644 --- a/typescript-sdk/packages/core/package.json +++ b/typescript-sdk/packages/core/package.json @@ -1,7 +1,7 @@ { "name": "@ag-ui/core", "author": "Markus Ecker ", - "version": "0.0.38", + "version": "0.0.40-alpha.2", "private": false, "publishConfig": { "access": "public" diff --git a/typescript-sdk/packages/encoder/package.json b/typescript-sdk/packages/encoder/package.json index c1694462f..d5b9e20d0 100644 --- a/typescript-sdk/packages/encoder/package.json +++ b/typescript-sdk/packages/encoder/package.json @@ -1,7 +1,7 @@ { "name": "@ag-ui/encoder", "author": "Markus Ecker ", - "version": "0.0.38", + "version": "0.0.40-alpha.2", "private": false, "publishConfig": { "access": "public" diff --git a/typescript-sdk/packages/proto/package.json b/typescript-sdk/packages/proto/package.json index 87f5db877..8e8afbef0 100644 --- a/typescript-sdk/packages/proto/package.json +++ b/typescript-sdk/packages/proto/package.json @@ -1,7 +1,7 @@ { "name": "@ag-ui/proto", "author": "Markus Ecker ", - "version": "0.0.38", + "version": "0.0.40-alpha.2", "private": false, "publishConfig": { "access": "public" From b8e19d2d26cfd13c026e3105b1d4d91e4190f346 Mon Sep 17 00:00:00 2001 From: Markus Ecker Date: Wed, 1 Oct 2025 13:54:22 +0200 Subject: [PATCH 10/25] adapt cloning logic --- .../src/agent/__tests__/agent-clone.test.ts | 81 +++++++++++++++++++ .../packages/client/src/agent/agent.ts | 14 ++-- .../packages/client/src/agent/http.ts | 15 ++++ 3 files changed, 104 insertions(+), 6 deletions(-) create mode 100644 typescript-sdk/packages/client/src/agent/__tests__/agent-clone.test.ts diff --git a/typescript-sdk/packages/client/src/agent/__tests__/agent-clone.test.ts b/typescript-sdk/packages/client/src/agent/__tests__/agent-clone.test.ts new file mode 100644 index 000000000..3fb00e8a0 --- /dev/null +++ b/typescript-sdk/packages/client/src/agent/__tests__/agent-clone.test.ts @@ -0,0 +1,81 @@ +import { AbstractAgent } from "../agent"; +import { HttpAgent } from "../http"; +import { BaseEvent, Message, RunAgentInput } from "@ag-ui/core"; +import { EMPTY, Observable } from "rxjs"; + +class CloneableTestAgent extends AbstractAgent { + constructor() { + super({ + agentId: "test-agent", + description: "Cloneable test agent", + threadId: "thread-test", + initialMessages: [ + { + id: "msg-1", + role: "user", + content: "Hello world", + toolCalls: [], + } as Message, + ], + initialState: { stage: "initial" }, + }); + } + + protected run(_: RunAgentInput): Observable { + return EMPTY as Observable; + } +} + +describe("AbstractAgent cloning", () => { + it("clones subclass instances with independent state", () => { + const agent = new CloneableTestAgent(); + + const cloned = agent.clone() as CloneableTestAgent; + + expect(cloned).toBeInstanceOf(CloneableTestAgent); + expect(cloned).not.toBe(agent); + expect(cloned.agentId).toBe(agent.agentId); + expect(cloned.threadId).toBe(agent.threadId); + expect(cloned.messages).toEqual(agent.messages); + expect(cloned.messages).not.toBe(agent.messages); + expect(cloned.state).toEqual(agent.state); + expect(cloned.state).not.toBe(agent.state); + }); +}); + +describe("HttpAgent cloning", () => { + it("produces a new HttpAgent with cloned configuration and abort controller", () => { + const httpAgent = new HttpAgent({ + url: "https://example.com/agent", + headers: { Authorization: "Bearer token" }, + threadId: "thread-http", + initialMessages: [ + { + id: "msg-http", + role: "assistant", + content: "response", + toolCalls: [], + } as Message, + ], + initialState: { status: "ready" }, + }); + + httpAgent.abortController.abort("cancelled"); + + const cloned = httpAgent.clone() as HttpAgent; + + expect(cloned).toBeInstanceOf(HttpAgent); + expect(cloned).not.toBe(httpAgent); + expect(cloned.url).toBe(httpAgent.url); + expect(cloned.headers).toEqual(httpAgent.headers); + expect(cloned.headers).not.toBe(httpAgent.headers); + expect(cloned.messages).toEqual(httpAgent.messages); + expect(cloned.messages).not.toBe(httpAgent.messages); + expect(cloned.state).toEqual(httpAgent.state); + expect(cloned.state).not.toBe(httpAgent.state); + expect(cloned.abortController).not.toBe(httpAgent.abortController); + expect(cloned.abortController).toBeInstanceOf(AbortController); + expect(cloned.abortController.signal.aborted).toBe(true); + expect(cloned.abortController.signal.reason).toBe("cancelled"); + }); +}); diff --git a/typescript-sdk/packages/client/src/agent/agent.ts b/typescript-sdk/packages/client/src/agent/agent.ts index b81c3a140..081cd1934 100644 --- a/typescript-sdk/packages/client/src/agent/agent.ts +++ b/typescript-sdk/packages/client/src/agent/agent.ts @@ -346,12 +346,14 @@ export abstract class AbstractAgent { public clone() { const cloned = Object.create(Object.getPrototypeOf(this)); - for (const key of Object.getOwnPropertyNames(this)) { - const value = (this as any)[key]; - if (typeof value !== "function") { - cloned[key] = structuredClone_(value); - } - } + cloned.agentId = this.agentId; + cloned.description = this.description; + cloned.threadId = this.threadId; + cloned.messages = structuredClone_(this.messages); + cloned.state = structuredClone_(this.state); + cloned.debug = this.debug; + cloned.isRunning = this.isRunning; + cloned.subscribers = [...this.subscribers]; return cloned; } diff --git a/typescript-sdk/packages/client/src/agent/http.ts b/typescript-sdk/packages/client/src/agent/http.ts index 49fae2173..f9d9c3002 100644 --- a/typescript-sdk/packages/client/src/agent/http.ts +++ b/typescript-sdk/packages/client/src/agent/http.ts @@ -58,4 +58,19 @@ export class HttpAgent extends AbstractAgent { const httpEvents = runHttpRequest(this.url, this.requestInit(input)); return transformHttpEventStream(httpEvents); } + + public clone(): HttpAgent { + const cloned = super.clone() as HttpAgent; + cloned.url = this.url; + cloned.headers = structuredClone_(this.headers ?? {}); + + const newController = new AbortController(); + const originalSignal = this.abortController.signal as AbortSignal & { reason?: unknown }; + if (originalSignal.aborted) { + newController.abort(originalSignal.reason); + } + cloned.abortController = newController; + + return cloned; + } } From 48ffd44f58ed826f0a9fffdd1f20674498444867 Mon Sep 17 00:00:00 2001 From: Markus Ecker Date: Wed, 1 Oct 2025 13:54:41 +0200 Subject: [PATCH 11/25] v0.0.40-alpha.3 --- typescript-sdk/packages/cli/package.json | 2 +- typescript-sdk/packages/client/package.json | 2 +- typescript-sdk/packages/core/package.json | 2 +- typescript-sdk/packages/encoder/package.json | 2 +- typescript-sdk/packages/proto/package.json | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/typescript-sdk/packages/cli/package.json b/typescript-sdk/packages/cli/package.json index 629b098c4..f0b0a012c 100644 --- a/typescript-sdk/packages/cli/package.json +++ b/typescript-sdk/packages/cli/package.json @@ -1,7 +1,7 @@ { "name": "create-ag-ui-app", "author": "Markus Ecker ", - "version": "0.0.40-alpha.2", + "version": "0.0.40-alpha.3", "private": false, "publishConfig": { "access": "public" diff --git a/typescript-sdk/packages/client/package.json b/typescript-sdk/packages/client/package.json index a80cf385f..acf9e509c 100644 --- a/typescript-sdk/packages/client/package.json +++ b/typescript-sdk/packages/client/package.json @@ -1,7 +1,7 @@ { "name": "@ag-ui/client", "author": "Markus Ecker ", - "version": "0.0.40-alpha.2", + "version": "0.0.40-alpha.3", "private": false, "publishConfig": { "access": "public" diff --git a/typescript-sdk/packages/core/package.json b/typescript-sdk/packages/core/package.json index 9662886fc..77c8c5154 100644 --- a/typescript-sdk/packages/core/package.json +++ b/typescript-sdk/packages/core/package.json @@ -1,7 +1,7 @@ { "name": "@ag-ui/core", "author": "Markus Ecker ", - "version": "0.0.40-alpha.2", + "version": "0.0.40-alpha.3", "private": false, "publishConfig": { "access": "public" diff --git a/typescript-sdk/packages/encoder/package.json b/typescript-sdk/packages/encoder/package.json index d5b9e20d0..e2d93960e 100644 --- a/typescript-sdk/packages/encoder/package.json +++ b/typescript-sdk/packages/encoder/package.json @@ -1,7 +1,7 @@ { "name": "@ag-ui/encoder", "author": "Markus Ecker ", - "version": "0.0.40-alpha.2", + "version": "0.0.40-alpha.3", "private": false, "publishConfig": { "access": "public" diff --git a/typescript-sdk/packages/proto/package.json b/typescript-sdk/packages/proto/package.json index 8e8afbef0..feffba233 100644 --- a/typescript-sdk/packages/proto/package.json +++ b/typescript-sdk/packages/proto/package.json @@ -1,7 +1,7 @@ { "name": "@ag-ui/proto", "author": "Markus Ecker ", - "version": "0.0.40-alpha.2", + "version": "0.0.40-alpha.3", "private": false, "publishConfig": { "access": "public" From 4d1773cbfbb20c97a5439c5d84b960d3f75e25de Mon Sep 17 00:00:00 2001 From: Markus Ecker Date: Mon, 13 Oct 2025 14:35:54 +0200 Subject: [PATCH 12/25] move compact logic to AG-UI package --- .../src/compact/__tests__/compact.test.ts | 287 ++++++++++++++++++ .../packages/client/src/compact/compact.ts | 252 +++++++++++++++ .../packages/client/src/compact/index.ts | 0 3 files changed, 539 insertions(+) create mode 100644 typescript-sdk/packages/client/src/compact/__tests__/compact.test.ts create mode 100644 typescript-sdk/packages/client/src/compact/compact.ts create mode 100644 typescript-sdk/packages/client/src/compact/index.ts diff --git a/typescript-sdk/packages/client/src/compact/__tests__/compact.test.ts b/typescript-sdk/packages/client/src/compact/__tests__/compact.test.ts new file mode 100644 index 000000000..9ece346e9 --- /dev/null +++ b/typescript-sdk/packages/client/src/compact/__tests__/compact.test.ts @@ -0,0 +1,287 @@ +import { compactEvents } from "../compact"; +import { BaseEvent, EventType } from "@ag-ui/core"; + +describe("Event Compaction", () => { + describe("Text Message Compaction", () => { + it("should compact multiple text message content events into one", () => { + const events: BaseEvent[] = [ + { type: EventType.TEXT_MESSAGE_START, messageId: "msg1", role: "user" }, + { type: EventType.TEXT_MESSAGE_CONTENT, messageId: "msg1", delta: "Hello" }, + { type: EventType.TEXT_MESSAGE_CONTENT, messageId: "msg1", delta: " " }, + { type: EventType.TEXT_MESSAGE_CONTENT, messageId: "msg1", delta: "world" }, + { type: EventType.TEXT_MESSAGE_END, messageId: "msg1" }, + ]; + + const compacted = compactEvents(events); + + expect(compacted).toHaveLength(3); + expect(compacted[0].type).toBe(EventType.TEXT_MESSAGE_START); + expect(compacted[1].type).toBe(EventType.TEXT_MESSAGE_CONTENT); + expect((compacted[1] as any).delta).toBe("Hello world"); + expect(compacted[2].type).toBe(EventType.TEXT_MESSAGE_END); + }); + + it("should move interleaved events to after text message events", () => { + const events: BaseEvent[] = [ + { type: EventType.TEXT_MESSAGE_START, messageId: "msg1", role: "assistant" }, + { type: EventType.TEXT_MESSAGE_CONTENT, messageId: "msg1", delta: "Processing" }, + { type: EventType.CUSTOM, id: "custom1", name: "thinking" }, + { type: EventType.TEXT_MESSAGE_CONTENT, messageId: "msg1", delta: "..." }, + { type: EventType.CUSTOM, id: "custom2", name: "done-thinking" }, + { type: EventType.TEXT_MESSAGE_END, messageId: "msg1" }, + ]; + + const compacted = compactEvents(events); + + expect(compacted).toHaveLength(5); + // Text message events should come first + expect(compacted[0].type).toBe(EventType.TEXT_MESSAGE_START); + expect(compacted[1].type).toBe(EventType.TEXT_MESSAGE_CONTENT); + expect((compacted[1] as any).delta).toBe("Processing..."); + expect(compacted[2].type).toBe(EventType.TEXT_MESSAGE_END); + // Other events should come after + expect(compacted[3].type).toBe(EventType.CUSTOM); + expect((compacted[3] as any).id).toBe("custom1"); + expect(compacted[4].type).toBe(EventType.CUSTOM); + expect((compacted[4] as any).id).toBe("custom2"); + }); + + it("should handle multiple messages independently", () => { + const events: BaseEvent[] = [ + { type: EventType.TEXT_MESSAGE_START, messageId: "msg1", role: "user" }, + { type: EventType.TEXT_MESSAGE_CONTENT, messageId: "msg1", delta: "Hi" }, + { type: EventType.TEXT_MESSAGE_END, messageId: "msg1" }, + { type: EventType.TEXT_MESSAGE_START, messageId: "msg2", role: "assistant" }, + { type: EventType.TEXT_MESSAGE_CONTENT, messageId: "msg2", delta: "Hello" }, + { type: EventType.TEXT_MESSAGE_CONTENT, messageId: "msg2", delta: " there" }, + { type: EventType.TEXT_MESSAGE_END, messageId: "msg2" }, + ]; + + const compacted = compactEvents(events); + + expect(compacted).toHaveLength(6); + // First message + expect(compacted[0].type).toBe(EventType.TEXT_MESSAGE_START); + expect((compacted[0] as any).messageId).toBe("msg1"); + expect(compacted[1].type).toBe(EventType.TEXT_MESSAGE_CONTENT); + expect((compacted[1] as any).delta).toBe("Hi"); + expect(compacted[2].type).toBe(EventType.TEXT_MESSAGE_END); + // Second message + expect(compacted[3].type).toBe(EventType.TEXT_MESSAGE_START); + expect((compacted[3] as any).messageId).toBe("msg2"); + expect(compacted[4].type).toBe(EventType.TEXT_MESSAGE_CONTENT); + expect((compacted[4] as any).delta).toBe("Hello there"); + expect(compacted[5].type).toBe(EventType.TEXT_MESSAGE_END); + }); + + it("should handle incomplete messages", () => { + const events: BaseEvent[] = [ + { type: EventType.TEXT_MESSAGE_START, messageId: "msg1", role: "user" }, + { type: EventType.TEXT_MESSAGE_CONTENT, messageId: "msg1", delta: "Incomplete" }, + // No END event + ]; + + const compacted = compactEvents(events); + + expect(compacted).toHaveLength(2); + expect(compacted[0].type).toBe(EventType.TEXT_MESSAGE_START); + expect(compacted[1].type).toBe(EventType.TEXT_MESSAGE_CONTENT); + expect((compacted[1] as any).delta).toBe("Incomplete"); + }); + + it("should pass through non-text-message events unchanged", () => { + const events: BaseEvent[] = [ + { type: EventType.CUSTOM, id: "custom1", name: "event1" }, + { type: EventType.TOOL_CALL_START, toolCallId: "tool1", toolCallName: "search" }, + { type: EventType.TOOL_CALL_END, toolCallId: "tool1" }, + ]; + + const compacted = compactEvents(events); + + expect(compacted).toEqual(events); + }); + + it("should handle empty content deltas", () => { + const events: BaseEvent[] = [ + { type: EventType.TEXT_MESSAGE_START, messageId: "msg1", role: "user" }, + { type: EventType.TEXT_MESSAGE_CONTENT, messageId: "msg1", delta: "" }, + { type: EventType.TEXT_MESSAGE_CONTENT, messageId: "msg1", delta: "Hello" }, + { type: EventType.TEXT_MESSAGE_CONTENT, messageId: "msg1", delta: "" }, + { type: EventType.TEXT_MESSAGE_END, messageId: "msg1" }, + ]; + + const compacted = compactEvents(events); + + expect(compacted).toHaveLength(3); + expect((compacted[1] as any).delta).toBe("Hello"); + }); + }); + + describe("Tool Call Compaction", () => { + it("should compact multiple tool call args events into one", () => { + const events: BaseEvent[] = [ + { + type: EventType.TOOL_CALL_START, + toolCallId: "tool1", + toolCallName: "search", + parentMessageId: "msg1", + }, + { type: EventType.TOOL_CALL_ARGS, toolCallId: "tool1", delta: '{"query": "' }, + { type: EventType.TOOL_CALL_ARGS, toolCallId: "tool1", delta: "weather" }, + { type: EventType.TOOL_CALL_ARGS, toolCallId: "tool1", delta: ' today"' }, + { type: EventType.TOOL_CALL_ARGS, toolCallId: "tool1", delta: "}" }, + { type: EventType.TOOL_CALL_END, toolCallId: "tool1" }, + ]; + + const compacted = compactEvents(events); + + expect(compacted).toHaveLength(3); + expect(compacted[0].type).toBe(EventType.TOOL_CALL_START); + expect(compacted[1].type).toBe(EventType.TOOL_CALL_ARGS); + expect((compacted[1] as any).delta).toBe('{"query": "weather today"}'); + expect(compacted[2].type).toBe(EventType.TOOL_CALL_END); + }); + + it("should move interleaved events to after tool call events", () => { + const events: BaseEvent[] = [ + { + type: EventType.TOOL_CALL_START, + toolCallId: "tool1", + toolCallName: "calculate", + parentMessageId: "msg1", + }, + { type: EventType.TOOL_CALL_ARGS, toolCallId: "tool1", delta: '{"a": ' }, + { type: EventType.CUSTOM, id: "custom1", name: "processing" }, + { type: EventType.TOOL_CALL_ARGS, toolCallId: "tool1", delta: '10, "b": 20}' }, + { type: EventType.CUSTOM, id: "custom2", name: "calculating" }, + { type: EventType.TOOL_CALL_END, toolCallId: "tool1" }, + ]; + + const compacted = compactEvents(events); + + expect(compacted).toHaveLength(5); + // Tool call events should come first + expect(compacted[0].type).toBe(EventType.TOOL_CALL_START); + expect(compacted[1].type).toBe(EventType.TOOL_CALL_ARGS); + expect((compacted[1] as any).delta).toBe('{"a": 10, "b": 20}'); + expect(compacted[2].type).toBe(EventType.TOOL_CALL_END); + // Other events should come after + expect(compacted[3].type).toBe(EventType.CUSTOM); + expect((compacted[3] as any).id).toBe("custom1"); + expect(compacted[4].type).toBe(EventType.CUSTOM); + expect((compacted[4] as any).id).toBe("custom2"); + }); + + it("should handle multiple tool calls independently", () => { + const events: BaseEvent[] = [ + { + type: EventType.TOOL_CALL_START, + toolCallId: "tool1", + toolCallName: "search", + parentMessageId: "msg1", + }, + { type: EventType.TOOL_CALL_ARGS, toolCallId: "tool1", delta: '{"query": "test"}' }, + { type: EventType.TOOL_CALL_END, toolCallId: "tool1" }, + { + type: EventType.TOOL_CALL_START, + toolCallId: "tool2", + toolCallName: "calculate", + parentMessageId: "msg1", + }, + { type: EventType.TOOL_CALL_ARGS, toolCallId: "tool2", delta: '{"a": ' }, + { type: EventType.TOOL_CALL_ARGS, toolCallId: "tool2", delta: "5}" }, + { type: EventType.TOOL_CALL_END, toolCallId: "tool2" }, + ]; + + const compacted = compactEvents(events); + + expect(compacted).toHaveLength(6); + // First tool call + expect(compacted[0].type).toBe(EventType.TOOL_CALL_START); + expect((compacted[0] as any).toolCallId).toBe("tool1"); + expect(compacted[1].type).toBe(EventType.TOOL_CALL_ARGS); + expect((compacted[1] as any).delta).toBe('{"query": "test"}'); + expect(compacted[2].type).toBe(EventType.TOOL_CALL_END); + // Second tool call + expect(compacted[3].type).toBe(EventType.TOOL_CALL_START); + expect((compacted[3] as any).toolCallId).toBe("tool2"); + expect(compacted[4].type).toBe(EventType.TOOL_CALL_ARGS); + expect((compacted[4] as any).delta).toBe('{"a": 5}'); + expect(compacted[5].type).toBe(EventType.TOOL_CALL_END); + }); + + it("should handle incomplete tool calls", () => { + const events: BaseEvent[] = [ + { + type: EventType.TOOL_CALL_START, + toolCallId: "tool1", + toolCallName: "search", + parentMessageId: "msg1", + }, + { type: EventType.TOOL_CALL_ARGS, toolCallId: "tool1", delta: '{"incomplete": ' }, + // No END event + ]; + + const compacted = compactEvents(events); + + expect(compacted).toHaveLength(2); + expect(compacted[0].type).toBe(EventType.TOOL_CALL_START); + expect(compacted[1].type).toBe(EventType.TOOL_CALL_ARGS); + expect((compacted[1] as any).delta).toBe('{"incomplete": '); + }); + + it("should handle empty args deltas", () => { + const events: BaseEvent[] = [ + { + type: EventType.TOOL_CALL_START, + toolCallId: "tool1", + toolCallName: "search", + parentMessageId: "msg1", + }, + { type: EventType.TOOL_CALL_ARGS, toolCallId: "tool1", delta: "" }, + { type: EventType.TOOL_CALL_ARGS, toolCallId: "tool1", delta: '{"test": true}' }, + { type: EventType.TOOL_CALL_ARGS, toolCallId: "tool1", delta: "" }, + { type: EventType.TOOL_CALL_END, toolCallId: "tool1" }, + ]; + + const compacted = compactEvents(events); + + expect(compacted).toHaveLength(3); + expect((compacted[1] as any).delta).toBe('{"test": true}'); + }); + }); + + describe("Mixed Compaction", () => { + it("should handle text messages and tool calls together", () => { + const events: BaseEvent[] = [ + { type: EventType.TEXT_MESSAGE_START, messageId: "msg1", role: "assistant" }, + { type: EventType.TEXT_MESSAGE_CONTENT, messageId: "msg1", delta: "Let me " }, + { type: EventType.TEXT_MESSAGE_CONTENT, messageId: "msg1", delta: "search for that" }, + { type: EventType.TEXT_MESSAGE_END, messageId: "msg1" }, + { + type: EventType.TOOL_CALL_START, + toolCallId: "tool1", + toolCallName: "search", + parentMessageId: "msg1", + }, + { type: EventType.TOOL_CALL_ARGS, toolCallId: "tool1", delta: '{"q": "' }, + { type: EventType.TOOL_CALL_ARGS, toolCallId: "tool1", delta: 'test"}' }, + { type: EventType.TOOL_CALL_END, toolCallId: "tool1" }, + ]; + + const compacted = compactEvents(events); + + expect(compacted).toHaveLength(6); + // Text message + expect(compacted[0].type).toBe(EventType.TEXT_MESSAGE_START); + expect(compacted[1].type).toBe(EventType.TEXT_MESSAGE_CONTENT); + expect((compacted[1] as any).delta).toBe("Let me search for that"); + expect(compacted[2].type).toBe(EventType.TEXT_MESSAGE_END); + // Tool call + expect(compacted[3].type).toBe(EventType.TOOL_CALL_START); + expect(compacted[4].type).toBe(EventType.TOOL_CALL_ARGS); + expect((compacted[4] as any).delta).toBe('{"q": "test"}'); + expect(compacted[5].type).toBe(EventType.TOOL_CALL_END); + }); + }); +}); diff --git a/typescript-sdk/packages/client/src/compact/compact.ts b/typescript-sdk/packages/client/src/compact/compact.ts new file mode 100644 index 000000000..b34c08e6a --- /dev/null +++ b/typescript-sdk/packages/client/src/compact/compact.ts @@ -0,0 +1,252 @@ +import { + BaseEvent, + EventType, + TextMessageStartEvent, + TextMessageContentEvent, + TextMessageEndEvent, + ToolCallStartEvent, + ToolCallArgsEvent, + ToolCallEndEvent, +} from "@ag-ui/core"; + +/** + * Compacts streaming events by consolidating multiple deltas into single events. + * For text messages: multiple content deltas become one concatenated delta. + * For tool calls: multiple args deltas become one concatenated delta. + * Events between related streaming events are reordered to keep streaming events together. + * + * @param events - Array of events to compact + * @returns Compacted array of events + */ +export function compactEvents(events: BaseEvent[]): BaseEvent[] { + const compacted: BaseEvent[] = []; + const pendingTextMessages = new Map< + string, + { + start?: TextMessageStartEvent; + contents: TextMessageContentEvent[]; + end?: TextMessageEndEvent; + otherEvents: BaseEvent[]; + } + >(); + const pendingToolCalls = new Map< + string, + { + start?: ToolCallStartEvent; + args: ToolCallArgsEvent[]; + end?: ToolCallEndEvent; + otherEvents: BaseEvent[]; + } + >(); + + for (const event of events) { + // Handle text message streaming events + if (event.type === EventType.TEXT_MESSAGE_START) { + const startEvent = event as TextMessageStartEvent; + const messageId = startEvent.messageId; + + if (!pendingTextMessages.has(messageId)) { + pendingTextMessages.set(messageId, { + contents: [], + otherEvents: [], + }); + } + + const pending = pendingTextMessages.get(messageId)!; + pending.start = startEvent; + } else if (event.type === EventType.TEXT_MESSAGE_CONTENT) { + const contentEvent = event as TextMessageContentEvent; + const messageId = contentEvent.messageId; + + if (!pendingTextMessages.has(messageId)) { + pendingTextMessages.set(messageId, { + contents: [], + otherEvents: [], + }); + } + + const pending = pendingTextMessages.get(messageId)!; + pending.contents.push(contentEvent); + } else if (event.type === EventType.TEXT_MESSAGE_END) { + const endEvent = event as TextMessageEndEvent; + const messageId = endEvent.messageId; + + if (!pendingTextMessages.has(messageId)) { + pendingTextMessages.set(messageId, { + contents: [], + otherEvents: [], + }); + } + + const pending = pendingTextMessages.get(messageId)!; + pending.end = endEvent; + + // Flush this message's events + flushTextMessage(messageId, pending, compacted); + pendingTextMessages.delete(messageId); + } else if (event.type === EventType.TOOL_CALL_START) { + const startEvent = event as ToolCallStartEvent; + const toolCallId = startEvent.toolCallId; + + if (!pendingToolCalls.has(toolCallId)) { + pendingToolCalls.set(toolCallId, { + args: [], + otherEvents: [], + }); + } + + const pending = pendingToolCalls.get(toolCallId)!; + pending.start = startEvent; + } else if (event.type === EventType.TOOL_CALL_ARGS) { + const argsEvent = event as ToolCallArgsEvent; + const toolCallId = argsEvent.toolCallId; + + if (!pendingToolCalls.has(toolCallId)) { + pendingToolCalls.set(toolCallId, { + args: [], + otherEvents: [], + }); + } + + const pending = pendingToolCalls.get(toolCallId)!; + pending.args.push(argsEvent); + } else if (event.type === EventType.TOOL_CALL_END) { + const endEvent = event as ToolCallEndEvent; + const toolCallId = endEvent.toolCallId; + + if (!pendingToolCalls.has(toolCallId)) { + pendingToolCalls.set(toolCallId, { + args: [], + otherEvents: [], + }); + } + + const pending = pendingToolCalls.get(toolCallId)!; + pending.end = endEvent; + + // Flush this tool call's events + flushToolCall(toolCallId, pending, compacted); + pendingToolCalls.delete(toolCallId); + } else { + // For non-streaming events, check if we're in the middle of any streaming sequences + let addedToBuffer = false; + + // Check text messages + for (const [messageId, pending] of pendingTextMessages) { + // If we have a start but no end yet, this event is "in between" + if (pending.start && !pending.end) { + pending.otherEvents.push(event); + addedToBuffer = true; + break; + } + } + + // Check tool calls if not already buffered + if (!addedToBuffer) { + for (const [toolCallId, pending] of pendingToolCalls) { + // If we have a start but no end yet, this event is "in between" + if (pending.start && !pending.end) { + pending.otherEvents.push(event); + addedToBuffer = true; + break; + } + } + } + + // If not in the middle of any streaming sequence, add directly to compacted + if (!addedToBuffer) { + compacted.push(event); + } + } + } + + // Flush any remaining incomplete messages + for (const [messageId, pending] of pendingTextMessages) { + flushTextMessage(messageId, pending, compacted); + } + + // Flush any remaining incomplete tool calls + for (const [toolCallId, pending] of pendingToolCalls) { + flushToolCall(toolCallId, pending, compacted); + } + + return compacted; +} + +function flushTextMessage( + messageId: string, + pending: { + start?: TextMessageStartEvent; + contents: TextMessageContentEvent[]; + end?: TextMessageEndEvent; + otherEvents: BaseEvent[]; + }, + compacted: BaseEvent[], +): void { + // Add start event if present + if (pending.start) { + compacted.push(pending.start); + } + + // Compact all content events into one + if (pending.contents.length > 0) { + const concatenatedDelta = pending.contents.map((c) => c.delta).join(""); + + const compactedContent: TextMessageContentEvent = { + type: EventType.TEXT_MESSAGE_CONTENT, + messageId: messageId, + delta: concatenatedDelta, + }; + + compacted.push(compactedContent); + } + + // Add end event if present + if (pending.end) { + compacted.push(pending.end); + } + + // Add any events that were in between + for (const otherEvent of pending.otherEvents) { + compacted.push(otherEvent); + } +} + +function flushToolCall( + toolCallId: string, + pending: { + start?: ToolCallStartEvent; + args: ToolCallArgsEvent[]; + end?: ToolCallEndEvent; + otherEvents: BaseEvent[]; + }, + compacted: BaseEvent[], +): void { + // Add start event if present + if (pending.start) { + compacted.push(pending.start); + } + + // Compact all args events into one + if (pending.args.length > 0) { + const concatenatedArgs = pending.args.map((a) => a.delta).join(""); + + const compactedArgs: ToolCallArgsEvent = { + type: EventType.TOOL_CALL_ARGS, + toolCallId: toolCallId, + delta: concatenatedArgs, + }; + + compacted.push(compactedArgs); + } + + // Add end event if present + if (pending.end) { + compacted.push(pending.end); + } + + // Add any events that were in between + for (const otherEvent of pending.otherEvents) { + compacted.push(otherEvent); + } +} diff --git a/typescript-sdk/packages/client/src/compact/index.ts b/typescript-sdk/packages/client/src/compact/index.ts new file mode 100644 index 000000000..e69de29bb From 98451824a51937a6f8da93929c6f5e45876994b9 Mon Sep 17 00:00:00 2001 From: Markus Ecker Date: Mon, 13 Oct 2025 14:40:46 +0200 Subject: [PATCH 13/25] fix linter errors --- .../src/compact/__tests__/compact.test.ts | 77 ++++++++++--------- 1 file changed, 42 insertions(+), 35 deletions(-) diff --git a/typescript-sdk/packages/client/src/compact/__tests__/compact.test.ts b/typescript-sdk/packages/client/src/compact/__tests__/compact.test.ts index 9ece346e9..17ccb017c 100644 --- a/typescript-sdk/packages/client/src/compact/__tests__/compact.test.ts +++ b/typescript-sdk/packages/client/src/compact/__tests__/compact.test.ts @@ -1,10 +1,17 @@ import { compactEvents } from "../compact"; -import { BaseEvent, EventType } from "@ag-ui/core"; +import { + EventType, + TextMessageStartEvent, + TextMessageContentEvent, + ToolCallStartEvent, + ToolCallArgsEvent, + CustomEvent, +} from "@ag-ui/core"; describe("Event Compaction", () => { describe("Text Message Compaction", () => { it("should compact multiple text message content events into one", () => { - const events: BaseEvent[] = [ + const events = [ { type: EventType.TEXT_MESSAGE_START, messageId: "msg1", role: "user" }, { type: EventType.TEXT_MESSAGE_CONTENT, messageId: "msg1", delta: "Hello" }, { type: EventType.TEXT_MESSAGE_CONTENT, messageId: "msg1", delta: " " }, @@ -17,12 +24,12 @@ describe("Event Compaction", () => { expect(compacted).toHaveLength(3); expect(compacted[0].type).toBe(EventType.TEXT_MESSAGE_START); expect(compacted[1].type).toBe(EventType.TEXT_MESSAGE_CONTENT); - expect((compacted[1] as any).delta).toBe("Hello world"); + expect((compacted[1] as TextMessageContentEvent).delta).toBe("Hello world"); expect(compacted[2].type).toBe(EventType.TEXT_MESSAGE_END); }); it("should move interleaved events to after text message events", () => { - const events: BaseEvent[] = [ + const events = [ { type: EventType.TEXT_MESSAGE_START, messageId: "msg1", role: "assistant" }, { type: EventType.TEXT_MESSAGE_CONTENT, messageId: "msg1", delta: "Processing" }, { type: EventType.CUSTOM, id: "custom1", name: "thinking" }, @@ -37,17 +44,17 @@ describe("Event Compaction", () => { // Text message events should come first expect(compacted[0].type).toBe(EventType.TEXT_MESSAGE_START); expect(compacted[1].type).toBe(EventType.TEXT_MESSAGE_CONTENT); - expect((compacted[1] as any).delta).toBe("Processing..."); + expect((compacted[1] as TextMessageContentEvent).delta).toBe("Processing..."); expect(compacted[2].type).toBe(EventType.TEXT_MESSAGE_END); // Other events should come after expect(compacted[3].type).toBe(EventType.CUSTOM); - expect((compacted[3] as any).id).toBe("custom1"); + expect((compacted[3] as CustomEvent & { id: string }).id).toBe("custom1"); expect(compacted[4].type).toBe(EventType.CUSTOM); - expect((compacted[4] as any).id).toBe("custom2"); + expect((compacted[4] as CustomEvent & { id: string }).id).toBe("custom2"); }); it("should handle multiple messages independently", () => { - const events: BaseEvent[] = [ + const events = [ { type: EventType.TEXT_MESSAGE_START, messageId: "msg1", role: "user" }, { type: EventType.TEXT_MESSAGE_CONTENT, messageId: "msg1", delta: "Hi" }, { type: EventType.TEXT_MESSAGE_END, messageId: "msg1" }, @@ -62,20 +69,20 @@ describe("Event Compaction", () => { expect(compacted).toHaveLength(6); // First message expect(compacted[0].type).toBe(EventType.TEXT_MESSAGE_START); - expect((compacted[0] as any).messageId).toBe("msg1"); + expect((compacted[0] as TextMessageStartEvent).messageId).toBe("msg1"); expect(compacted[1].type).toBe(EventType.TEXT_MESSAGE_CONTENT); - expect((compacted[1] as any).delta).toBe("Hi"); + expect((compacted[1] as TextMessageContentEvent).delta).toBe("Hi"); expect(compacted[2].type).toBe(EventType.TEXT_MESSAGE_END); // Second message expect(compacted[3].type).toBe(EventType.TEXT_MESSAGE_START); - expect((compacted[3] as any).messageId).toBe("msg2"); + expect((compacted[3] as TextMessageStartEvent).messageId).toBe("msg2"); expect(compacted[4].type).toBe(EventType.TEXT_MESSAGE_CONTENT); - expect((compacted[4] as any).delta).toBe("Hello there"); + expect((compacted[4] as TextMessageContentEvent).delta).toBe("Hello there"); expect(compacted[5].type).toBe(EventType.TEXT_MESSAGE_END); }); it("should handle incomplete messages", () => { - const events: BaseEvent[] = [ + const events = [ { type: EventType.TEXT_MESSAGE_START, messageId: "msg1", role: "user" }, { type: EventType.TEXT_MESSAGE_CONTENT, messageId: "msg1", delta: "Incomplete" }, // No END event @@ -86,11 +93,11 @@ describe("Event Compaction", () => { expect(compacted).toHaveLength(2); expect(compacted[0].type).toBe(EventType.TEXT_MESSAGE_START); expect(compacted[1].type).toBe(EventType.TEXT_MESSAGE_CONTENT); - expect((compacted[1] as any).delta).toBe("Incomplete"); + expect((compacted[1] as TextMessageContentEvent).delta).toBe("Incomplete"); }); it("should pass through non-text-message events unchanged", () => { - const events: BaseEvent[] = [ + const events = [ { type: EventType.CUSTOM, id: "custom1", name: "event1" }, { type: EventType.TOOL_CALL_START, toolCallId: "tool1", toolCallName: "search" }, { type: EventType.TOOL_CALL_END, toolCallId: "tool1" }, @@ -102,7 +109,7 @@ describe("Event Compaction", () => { }); it("should handle empty content deltas", () => { - const events: BaseEvent[] = [ + const events = [ { type: EventType.TEXT_MESSAGE_START, messageId: "msg1", role: "user" }, { type: EventType.TEXT_MESSAGE_CONTENT, messageId: "msg1", delta: "" }, { type: EventType.TEXT_MESSAGE_CONTENT, messageId: "msg1", delta: "Hello" }, @@ -113,13 +120,13 @@ describe("Event Compaction", () => { const compacted = compactEvents(events); expect(compacted).toHaveLength(3); - expect((compacted[1] as any).delta).toBe("Hello"); + expect((compacted[1] as TextMessageContentEvent).delta).toBe("Hello"); }); }); describe("Tool Call Compaction", () => { it("should compact multiple tool call args events into one", () => { - const events: BaseEvent[] = [ + const events = [ { type: EventType.TOOL_CALL_START, toolCallId: "tool1", @@ -138,12 +145,12 @@ describe("Event Compaction", () => { expect(compacted).toHaveLength(3); expect(compacted[0].type).toBe(EventType.TOOL_CALL_START); expect(compacted[1].type).toBe(EventType.TOOL_CALL_ARGS); - expect((compacted[1] as any).delta).toBe('{"query": "weather today"}'); + expect((compacted[1] as ToolCallArgsEvent).delta).toBe('{"query": "weather today"}'); expect(compacted[2].type).toBe(EventType.TOOL_CALL_END); }); it("should move interleaved events to after tool call events", () => { - const events: BaseEvent[] = [ + const events = [ { type: EventType.TOOL_CALL_START, toolCallId: "tool1", @@ -163,17 +170,17 @@ describe("Event Compaction", () => { // Tool call events should come first expect(compacted[0].type).toBe(EventType.TOOL_CALL_START); expect(compacted[1].type).toBe(EventType.TOOL_CALL_ARGS); - expect((compacted[1] as any).delta).toBe('{"a": 10, "b": 20}'); + expect((compacted[1] as ToolCallArgsEvent).delta).toBe('{"a": 10, "b": 20}'); expect(compacted[2].type).toBe(EventType.TOOL_CALL_END); // Other events should come after expect(compacted[3].type).toBe(EventType.CUSTOM); - expect((compacted[3] as any).id).toBe("custom1"); + expect((compacted[3] as CustomEvent & { id: string }).id).toBe("custom1"); expect(compacted[4].type).toBe(EventType.CUSTOM); - expect((compacted[4] as any).id).toBe("custom2"); + expect((compacted[4] as CustomEvent & { id: string }).id).toBe("custom2"); }); it("should handle multiple tool calls independently", () => { - const events: BaseEvent[] = [ + const events = [ { type: EventType.TOOL_CALL_START, toolCallId: "tool1", @@ -198,20 +205,20 @@ describe("Event Compaction", () => { expect(compacted).toHaveLength(6); // First tool call expect(compacted[0].type).toBe(EventType.TOOL_CALL_START); - expect((compacted[0] as any).toolCallId).toBe("tool1"); + expect((compacted[0] as ToolCallStartEvent).toolCallId).toBe("tool1"); expect(compacted[1].type).toBe(EventType.TOOL_CALL_ARGS); - expect((compacted[1] as any).delta).toBe('{"query": "test"}'); + expect((compacted[1] as ToolCallArgsEvent).delta).toBe('{"query": "test"}'); expect(compacted[2].type).toBe(EventType.TOOL_CALL_END); // Second tool call expect(compacted[3].type).toBe(EventType.TOOL_CALL_START); - expect((compacted[3] as any).toolCallId).toBe("tool2"); + expect((compacted[3] as ToolCallStartEvent).toolCallId).toBe("tool2"); expect(compacted[4].type).toBe(EventType.TOOL_CALL_ARGS); - expect((compacted[4] as any).delta).toBe('{"a": 5}'); + expect((compacted[4] as ToolCallArgsEvent).delta).toBe('{"a": 5}'); expect(compacted[5].type).toBe(EventType.TOOL_CALL_END); }); it("should handle incomplete tool calls", () => { - const events: BaseEvent[] = [ + const events = [ { type: EventType.TOOL_CALL_START, toolCallId: "tool1", @@ -227,11 +234,11 @@ describe("Event Compaction", () => { expect(compacted).toHaveLength(2); expect(compacted[0].type).toBe(EventType.TOOL_CALL_START); expect(compacted[1].type).toBe(EventType.TOOL_CALL_ARGS); - expect((compacted[1] as any).delta).toBe('{"incomplete": '); + expect((compacted[1] as ToolCallArgsEvent).delta).toBe('{"incomplete": '); }); it("should handle empty args deltas", () => { - const events: BaseEvent[] = [ + const events = [ { type: EventType.TOOL_CALL_START, toolCallId: "tool1", @@ -247,13 +254,13 @@ describe("Event Compaction", () => { const compacted = compactEvents(events); expect(compacted).toHaveLength(3); - expect((compacted[1] as any).delta).toBe('{"test": true}'); + expect((compacted[1] as ToolCallArgsEvent).delta).toBe('{"test": true}'); }); }); describe("Mixed Compaction", () => { it("should handle text messages and tool calls together", () => { - const events: BaseEvent[] = [ + const events = [ { type: EventType.TEXT_MESSAGE_START, messageId: "msg1", role: "assistant" }, { type: EventType.TEXT_MESSAGE_CONTENT, messageId: "msg1", delta: "Let me " }, { type: EventType.TEXT_MESSAGE_CONTENT, messageId: "msg1", delta: "search for that" }, @@ -275,12 +282,12 @@ describe("Event Compaction", () => { // Text message expect(compacted[0].type).toBe(EventType.TEXT_MESSAGE_START); expect(compacted[1].type).toBe(EventType.TEXT_MESSAGE_CONTENT); - expect((compacted[1] as any).delta).toBe("Let me search for that"); + expect((compacted[1] as TextMessageContentEvent).delta).toBe("Let me search for that"); expect(compacted[2].type).toBe(EventType.TEXT_MESSAGE_END); // Tool call expect(compacted[3].type).toBe(EventType.TOOL_CALL_START); expect(compacted[4].type).toBe(EventType.TOOL_CALL_ARGS); - expect((compacted[4] as any).delta).toBe('{"q": "test"}'); + expect((compacted[4] as ToolCallArgsEvent).delta).toBe('{"q": "test"}'); expect(compacted[5].type).toBe(EventType.TOOL_CALL_END); }); }); From cc46272e23a34d9e1f766982cded182318bbaf13 Mon Sep 17 00:00:00 2001 From: Markus Ecker Date: Mon, 13 Oct 2025 14:43:29 +0200 Subject: [PATCH 14/25] export compactEvents --- typescript-sdk/packages/client/src/compact/index.ts | 1 + typescript-sdk/packages/client/src/index.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/typescript-sdk/packages/client/src/compact/index.ts b/typescript-sdk/packages/client/src/compact/index.ts index e69de29bb..bb036168b 100644 --- a/typescript-sdk/packages/client/src/compact/index.ts +++ b/typescript-sdk/packages/client/src/compact/index.ts @@ -0,0 +1 @@ +export { compactEvents } from "./compact"; diff --git a/typescript-sdk/packages/client/src/index.ts b/typescript-sdk/packages/client/src/index.ts index dfad35750..52e47ba0f 100644 --- a/typescript-sdk/packages/client/src/index.ts +++ b/typescript-sdk/packages/client/src/index.ts @@ -4,4 +4,5 @@ export * from "./transform"; export * from "./run"; export * from "./legacy"; export * from "./agent"; +export * from "./compact"; export * from "@ag-ui/core"; From e08a21ceb3deab7591eb1a536de38d984070d71f Mon Sep 17 00:00:00 2001 From: Markus Ecker Date: Mon, 13 Oct 2025 15:35:58 +0200 Subject: [PATCH 15/25] ensure allowing extra fields for backwards compatibility --- python-sdk/ag_ui/core/types.py | 2 +- python-sdk/tests/test_types.py | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/python-sdk/ag_ui/core/types.py b/python-sdk/ag_ui/core/types.py index 47b7ae182..d22d4eab7 100644 --- a/python-sdk/ag_ui/core/types.py +++ b/python-sdk/ag_ui/core/types.py @@ -13,7 +13,7 @@ class ConfiguredBaseModel(BaseModel): A configurable base model. """ model_config = ConfigDict( - extra="forbid", + extra="allow", alias_generator=to_camel, populate_by_name=True, ) diff --git a/python-sdk/tests/test_types.py b/python-sdk/tests/test_types.py index e534aa5ab..27eedff0a 100644 --- a/python-sdk/tests/test_types.py +++ b/python-sdk/tests/test_types.py @@ -368,15 +368,17 @@ def test_validation_errors(self): with self.assertRaises(ValidationError): UserMessage.model_validate(missing_id_data) - # Test extra fields + # Test extra fields are now allowed for backwards compatibility extra_field_data = { "id": "msg_456", "role": "user", "content": "Hello", - "extra_field": "This shouldn't be here" # Extra field + "extra_field": "This is allowed for backwards compatibility" # Extra field } - with self.assertRaises(ValidationError): - UserMessage.model_validate(extra_field_data) + # Should not raise an error - extra fields are allowed + msg = UserMessage.model_validate(extra_field_data) + self.assertEqual(msg.id, "msg_456") + self.assertEqual(msg.content, "Hello") # Test invalid tool_call_id in ToolMessage invalid_tool_data = { From b2e387cbc769b399d531104b066f91ca823ef502 Mon Sep 17 00:00:00 2001 From: Markus Ecker Date: Mon, 13 Oct 2025 15:36:15 +0200 Subject: [PATCH 16/25] Add test for typescript backwards compat --- .../__tests__/backwards-compatibility.test.ts | 250 ++++++++++++++++++ 1 file changed, 250 insertions(+) create mode 100644 typescript-sdk/packages/core/src/__tests__/backwards-compatibility.test.ts diff --git a/typescript-sdk/packages/core/src/__tests__/backwards-compatibility.test.ts b/typescript-sdk/packages/core/src/__tests__/backwards-compatibility.test.ts new file mode 100644 index 000000000..2cdeb48be --- /dev/null +++ b/typescript-sdk/packages/core/src/__tests__/backwards-compatibility.test.ts @@ -0,0 +1,250 @@ +import { + UserMessageSchema, + AssistantMessageSchema, + RunAgentInputSchema, + TextMessageStartEventSchema, + RunStartedEventSchema, + ToolSchema, + ContextSchema, + EventType, +} from "../index"; + +describe("Backwards Compatibility", () => { + describe("Message Schemas", () => { + it("should accept UserMessage with extra fields from future versions", () => { + const messageWithExtraFields = { + id: "msg_1", + role: "user" as const, + content: "Hello", + futureField: "This is from a future version", + anotherNewProp: { nested: "data" }, + }; + + const result = UserMessageSchema.safeParse(messageWithExtraFields); + + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.id).toBe("msg_1"); + expect(result.data.role).toBe("user"); + expect(result.data.content).toBe("Hello"); + // Extra fields should be stripped (Zod default behavior) + expect('futureField' in result.data).toBe(false); + expect('anotherNewProp' in result.data).toBe(false); + } + }); + + it("should accept AssistantMessage with extra fields", () => { + const messageWithExtraFields = { + id: "msg_2", + role: "assistant" as const, + content: "Response", + newFeatureFlag: true, + experimentalData: [1, 2, 3], + }; + + const result = AssistantMessageSchema.safeParse(messageWithExtraFields); + + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.id).toBe("msg_2"); + expect(result.data.content).toBe("Response"); + } + }); + }); + + describe("RunAgentInput Schema", () => { + it("should accept RunAgentInput with extra fields at top level", () => { + const inputWithExtraFields = { + threadId: "thread_1", + runId: "run_1", + state: {}, + messages: [], + tools: [], + context: [], + forwardedProps: {}, + // Extra fields from future version + newFeatureFlag: true, + experimentalTimeout: 5000, + futureConfig: { option: "value" }, + }; + + const result = RunAgentInputSchema.safeParse(inputWithExtraFields); + + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.threadId).toBe("thread_1"); + expect(result.data.runId).toBe("run_1"); + } + }); + + it("should accept RunAgentInput with messages containing extra fields", () => { + const inputWithExtraFieldsInMessages = { + threadId: "thread_2", + runId: "run_2", + state: {}, + messages: [ + { + id: "m1", + role: "user" as const, + content: "Hi", + extraProp: "value", + metadata: { source: "web" }, + }, + ], + tools: [], + context: [], + forwardedProps: {}, + }; + + const result = RunAgentInputSchema.safeParse(inputWithExtraFieldsInMessages); + + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.messages.length).toBe(1); + expect(result.data.messages[0].content).toBe("Hi"); + } + }); + }); + + describe("Event Schemas", () => { + it("should accept TextMessageStartEvent with extra fields", () => { + const eventWithExtraFields = { + type: EventType.TEXT_MESSAGE_START, + messageId: "msg_1", + role: "assistant" as const, + // Extra fields from future version + metadata: { tokenCount: 10 }, + experimentalFeature: true, + }; + + const result = TextMessageStartEventSchema.safeParse(eventWithExtraFields); + + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.type).toBe(EventType.TEXT_MESSAGE_START); + expect(result.data.messageId).toBe("msg_1"); + } + }); + + it("should accept RunStartedEvent with extra fields", () => { + const eventWithExtraFields = { + type: EventType.RUN_STARTED, + threadId: "thread_1", + runId: "run_1", + // Extra fields from future version + startTime: Date.now(), + priority: "high", + }; + + const result = RunStartedEventSchema.safeParse(eventWithExtraFields); + + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.threadId).toBe("thread_1"); + expect(result.data.runId).toBe("run_1"); + } + }); + }); + + describe("Tool and Context Schemas", () => { + it("should accept Tool with extra fields", () => { + const toolWithExtraFields = { + name: "calculator", + description: "Performs calculations", + parameters: { type: "object" }, + // Extra fields from future version + version: "2.0", + deprecationWarning: null, + }; + + const result = ToolSchema.safeParse(toolWithExtraFields); + + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.name).toBe("calculator"); + expect(result.data.description).toBe("Performs calculations"); + } + }); + + it("should accept Context with extra fields", () => { + const contextWithExtraFields = { + description: "User preferences", + value: '{"theme":"dark"}', + // Extra fields from future version + priority: 1, + expiresAt: Date.now() + 3600000, + }; + + const result = ContextSchema.safeParse(contextWithExtraFields); + + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.description).toBe("User preferences"); + expect(result.data.value).toBe('{"theme":"dark"}'); + } + }); + }); + + describe("Complex nested structures", () => { + it("should handle deeply nested objects with extra fields at multiple levels", () => { + const complexInput = { + threadId: "thread_complex", + runId: "run_complex", + state: { currentStep: 1 }, + messages: [ + { + id: "m1", + role: "user" as const, + content: "Hello", + extraUserProp: "value1", + }, + { + id: "m2", + role: "assistant" as const, + content: "Hi there", + toolCalls: [ + { + id: "tc1", + type: "function" as const, + function: { + name: "search", + arguments: "{}", + extraFunctionProp: "value2", + }, + extraToolCallProp: "value3", + }, + ], + extraAssistantProp: "value4", + }, + ], + tools: [ + { + name: "search", + description: "Search tool", + parameters: {}, + extraToolProp: "value5", + }, + ], + context: [ + { + description: "ctx", + value: "val", + extraContextProp: "value6", + }, + ], + forwardedProps: { custom: true }, + extraTopLevelProp: "value7", + }; + + const result = RunAgentInputSchema.safeParse(complexInput); + + expect(result.success).toBe(true); + if (result.success) { + expect(result.data.messages.length).toBe(2); + expect(result.data.messages[1].toolCalls?.length).toBe(1); + expect(result.data.tools.length).toBe(1); + expect(result.data.context.length).toBe(1); + } + }); + }); +}); From e37b691655ae897ab1fb6fed3f075963d764c070 Mon Sep 17 00:00:00 2001 From: Markus Ecker Date: Tue, 14 Oct 2025 10:25:16 +0200 Subject: [PATCH 17/25] add docs --- docs/concepts/events.mdx | 10 ++++++---- docs/sdk/js/core/events.mdx | 12 ++++++++---- docs/sdk/python/core/events.mdx | 12 ++++++++---- python-sdk/ag_ui/core/events.py | 4 +++- typescript-sdk/packages/core/src/events.ts | 4 +++- 5 files changed, 28 insertions(+), 14 deletions(-) diff --git a/docs/concepts/events.mdx b/docs/concepts/events.mdx index 920b203b9..0f1c2b7ac 100644 --- a/docs/concepts/events.mdx +++ b/docs/concepts/events.mdx @@ -83,10 +83,12 @@ elements such as progress indicators or loading states. It also provides crucial identifiers that can be used to associate subsequent events with this specific run. -| Property | Description | -| ---------- | ----------------------------- | -| `threadId` | ID of the conversation thread | -| `runId` | ID of the agent run | +| Property | Description | +| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `threadId` | ID of the conversation thread | +| `runId` | ID of the agent run | +| `parentRunId` | (Optional) Lineage pointer for branching/time travel. If present, refers to a prior run within the same thread, creating a git-like append-only log | +| `input` | (Optional) The exact agent input payload that was sent to the agent for this run. May omit messages already present in history; compactEvents() will normalize | ### RunFinished diff --git a/docs/sdk/js/core/events.mdx b/docs/sdk/js/core/events.mdx index ea119090c..4e586f179 100644 --- a/docs/sdk/js/core/events.mdx +++ b/docs/sdk/js/core/events.mdx @@ -68,13 +68,17 @@ type RunStartedEvent = BaseEvent & { type: EventType.RUN_STARTED threadId: string runId: string + parentRunId?: string + input?: RunAgentInput } ``` -| Property | Type | Description | -| ---------- | -------- | ----------------------------- | -| `threadId` | `string` | ID of the conversation thread | -| `runId` | `string` | ID of the agent run | +| Property | Type | Description | +| -------------- | ----------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | +| `threadId` | `string` | ID of the conversation thread | +| `runId` | `string` | ID of the agent run | +| `parentRunId` | `string` (optional) | (Optional) Lineage pointer for branching/time travel. If present, refers to a prior run within the same thread | +| `input` | `RunAgentInput` (optional) | (Optional) The exact agent input payload sent to the agent for this run. May omit messages already in history | ### RunFinishedEvent diff --git a/docs/sdk/python/core/events.mdx b/docs/sdk/python/core/events.mdx index 6d5cdc934..1a9fe9c76 100644 --- a/docs/sdk/python/core/events.mdx +++ b/docs/sdk/python/core/events.mdx @@ -73,12 +73,16 @@ class RunStartedEvent(BaseEvent): type: Literal[EventType.RUN_STARTED] thread_id: str run_id: str + parent_run_id: Optional[str] = None + input: Optional[RunAgentInput] = None ``` -| Property | Type | Description | -| ----------- | ----- | ----------------------------- | -| `thread_id` | `str` | ID of the conversation thread | -| `run_id` | `str` | ID of the agent run | +| Property | Type | Description | +| ---------------- | -------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | +| `thread_id` | `str` | ID of the conversation thread | +| `run_id` | `str` | ID of the agent run | +| `parent_run_id` | `Optional[str]` | (Optional) Lineage pointer for branching/time travel. If present, refers to a prior run within the same thread | +| `input` | `Optional[RunAgentInput]` | (Optional) The exact agent input payload sent to the agent for this run. May omit messages already in history | ### RunFinishedEvent diff --git a/python-sdk/ag_ui/core/events.py b/python-sdk/ag_ui/core/events.py index 2a54a9c8e..411767e2b 100644 --- a/python-sdk/ag_ui/core/events.py +++ b/python-sdk/ag_ui/core/events.py @@ -7,7 +7,7 @@ from pydantic import Field -from .types import ConfiguredBaseModel, Message, State, Role +from .types import ConfiguredBaseModel, Message, State, Role, RunAgentInput # Text messages can have any role except "tool" TextMessageRole = Literal["developer", "system", "assistant", "user"] @@ -213,6 +213,8 @@ class RunStartedEvent(BaseEvent): type: Literal[EventType.RUN_STARTED] = EventType.RUN_STARTED # pyright: ignore[reportIncompatibleVariableOverride] thread_id: str run_id: str + parent_run_id: Optional[str] = None + input: Optional[RunAgentInput] = None class RunFinishedEvent(BaseEvent): diff --git a/typescript-sdk/packages/core/src/events.ts b/typescript-sdk/packages/core/src/events.ts index a95fc8e15..115182bea 100644 --- a/typescript-sdk/packages/core/src/events.ts +++ b/typescript-sdk/packages/core/src/events.ts @@ -1,5 +1,5 @@ import { z } from "zod"; -import { MessageSchema, StateSchema } from "./types"; +import { MessageSchema, StateSchema, RunAgentInputSchema } from "./types"; // Text messages can have any role except "tool" const TextMessageRoleSchema = z.union([ @@ -155,6 +155,8 @@ export const RunStartedEventSchema = BaseEventSchema.extend({ type: z.literal(EventType.RUN_STARTED), threadId: z.string(), runId: z.string(), + parentRunId: z.string().optional(), + input: RunAgentInputSchema.optional(), }); export const RunFinishedEventSchema = BaseEventSchema.extend({ From 00d47d07f5d373d707972de73c4982d1d0eb04dc Mon Sep 17 00:00:00 2001 From: Markus Ecker Date: Tue, 14 Oct 2025 11:29:08 +0200 Subject: [PATCH 18/25] Add new run started input messages --- .../apply/__tests__/run-started-input.test.ts | 416 ++++++++++++++++++ .../packages/client/src/apply/default.ts | 19 + 2 files changed, 435 insertions(+) create mode 100644 typescript-sdk/packages/client/src/apply/__tests__/run-started-input.test.ts diff --git a/typescript-sdk/packages/client/src/apply/__tests__/run-started-input.test.ts b/typescript-sdk/packages/client/src/apply/__tests__/run-started-input.test.ts new file mode 100644 index 000000000..9be1a0133 --- /dev/null +++ b/typescript-sdk/packages/client/src/apply/__tests__/run-started-input.test.ts @@ -0,0 +1,416 @@ +import { AbstractAgent } from "../../agent/agent"; +import { + BaseEvent, + EventType, + Message, + RunAgentInput, + RunStartedEvent, + RunFinishedEvent, + TextMessageStartEvent, + TextMessageContentEvent, + TextMessageEndEvent, +} from "@ag-ui/core"; +import { Observable, of } from "rxjs"; +import { AgentSubscriber } from "../../agent/subscriber"; + +describe("RunStartedEvent with input.messages", () => { + class TestAgent extends AbstractAgent { + private events: BaseEvent[] = []; + + setEvents(events: BaseEvent[]) { + this.events = events; + } + + protected run(input: RunAgentInput): Observable { + return of(...this.events); + } + } + + it("should add messages from RunStartedEvent.input that are not already present", async () => { + const agent = new TestAgent({ + threadId: "test-thread", + initialMessages: [], + }); + + const events: BaseEvent[] = [ + { + type: EventType.RUN_STARTED, + threadId: "test-thread", + runId: "run-1", + input: { + threadId: "test-thread", + runId: "run-1", + messages: [ + { + id: "msg-1", + role: "user", + content: "Hello", + }, + { + id: "msg-2", + role: "user", + content: "How are you?", + }, + ], + tools: [], + context: [], + state: {}, + forwardedProps: {}, + }, + } as RunStartedEvent, + { + type: EventType.RUN_FINISHED, + threadId: "test-thread", + runId: "run-1", + } as RunFinishedEvent, + ]; + + agent.setEvents(events); + const result = await agent.runAgent({ runId: "run-1" }); + + // Verify both messages were added + expect(agent.messages.length).toBe(2); + expect(agent.messages[0].id).toBe("msg-1"); + expect(agent.messages[0].content).toBe("Hello"); + expect(agent.messages[1].id).toBe("msg-2"); + expect(agent.messages[1].content).toBe("How are you?"); + + // Verify they appear in newMessages + expect(result.newMessages.length).toBe(2); + }); + + it("should not duplicate messages that already exist (by ID)", async () => { + const initialMessages: Message[] = [ + { + id: "msg-1", + role: "user", + content: "Existing message", + }, + ]; + + const agent = new TestAgent({ + threadId: "test-thread", + initialMessages, + }); + + const events: BaseEvent[] = [ + { + type: EventType.RUN_STARTED, + threadId: "test-thread", + runId: "run-1", + input: { + threadId: "test-thread", + runId: "run-1", + messages: [ + { + id: "msg-1", + role: "user", + content: "Duplicate message (should be ignored)", + }, + { + id: "msg-2", + role: "user", + content: "New message", + }, + ], + tools: [], + context: [], + state: {}, + forwardedProps: {}, + }, + } as RunStartedEvent, + { + type: EventType.RUN_FINISHED, + threadId: "test-thread", + runId: "run-1", + } as RunFinishedEvent, + ]; + + agent.setEvents(events); + const result = await agent.runAgent({ runId: "run-1" }); + + // Verify only the new message was added + expect(agent.messages.length).toBe(2); + expect(agent.messages[0].id).toBe("msg-1"); + expect(agent.messages[0].content).toBe("Existing message"); // Original content preserved + expect(agent.messages[1].id).toBe("msg-2"); + expect(agent.messages[1].content).toBe("New message"); + + // Verify only the new message appears in newMessages + expect(result.newMessages.length).toBe(1); + expect(result.newMessages[0].id).toBe("msg-2"); + }); + + it("should handle RunStartedEvent without input field", async () => { + const agent = new TestAgent({ + threadId: "test-thread", + initialMessages: [], + }); + + const events: BaseEvent[] = [ + { + type: EventType.RUN_STARTED, + threadId: "test-thread", + runId: "run-1", + // No input field + } as RunStartedEvent, + { + type: EventType.RUN_FINISHED, + threadId: "test-thread", + runId: "run-1", + } as RunFinishedEvent, + ]; + + agent.setEvents(events); + const result = await agent.runAgent({ runId: "run-1" }); + + // Verify no errors and messages remain empty + expect(agent.messages.length).toBe(0); + expect(result.newMessages.length).toBe(0); + }); + + it("should handle RunStartedEvent with input but no messages", async () => { + const agent = new TestAgent({ + threadId: "test-thread", + initialMessages: [], + }); + + const events: BaseEvent[] = [ + { + type: EventType.RUN_STARTED, + threadId: "test-thread", + runId: "run-1", + input: { + threadId: "test-thread", + runId: "run-1", + messages: [], // Empty messages array + tools: [], + context: [], + state: {}, + forwardedProps: {}, + }, + } as RunStartedEvent, + { + type: EventType.RUN_FINISHED, + threadId: "test-thread", + runId: "run-1", + } as RunFinishedEvent, + ]; + + agent.setEvents(events); + const result = await agent.runAgent({ runId: "run-1" }); + + // Verify no errors and messages remain empty + expect(agent.messages.length).toBe(0); + expect(result.newMessages.length).toBe(0); + }); + + it("should respect stopPropagation from subscribers", async () => { + const agent = new TestAgent({ + threadId: "test-thread", + initialMessages: [], + }); + + // Create a subscriber that stops propagation + const stopPropagationSubscriber: AgentSubscriber = { + onRunStartedEvent: () => { + return { stopPropagation: true }; + }, + }; + + const events: BaseEvent[] = [ + { + type: EventType.RUN_STARTED, + threadId: "test-thread", + runId: "run-1", + input: { + threadId: "test-thread", + runId: "run-1", + messages: [ + { + id: "msg-1", + role: "user", + content: "Should not be added", + }, + ], + tools: [], + context: [], + state: {}, + forwardedProps: {}, + }, + } as RunStartedEvent, + { + type: EventType.RUN_FINISHED, + threadId: "test-thread", + runId: "run-1", + } as RunFinishedEvent, + ]; + + agent.setEvents(events); + const result = await agent.runAgent({ runId: "run-1" }, stopPropagationSubscriber); + + // Verify messages were NOT added due to stopPropagation + expect(agent.messages.length).toBe(0); + expect(result.newMessages.length).toBe(0); + }); + + it("should add messages before other events in the same run", async () => { + const agent = new TestAgent({ + threadId: "test-thread", + initialMessages: [], + }); + + const events: BaseEvent[] = [ + { + type: EventType.RUN_STARTED, + threadId: "test-thread", + runId: "run-1", + input: { + threadId: "test-thread", + runId: "run-1", + messages: [ + { + id: "msg-from-input", + role: "user", + content: "From input", + }, + ], + tools: [], + context: [], + state: {}, + forwardedProps: {}, + }, + } as RunStartedEvent, + { + type: EventType.TEXT_MESSAGE_START, + messageId: "msg-streamed", + role: "assistant", + } as TextMessageStartEvent, + { + type: EventType.TEXT_MESSAGE_CONTENT, + messageId: "msg-streamed", + delta: "Streamed response", + } as TextMessageContentEvent, + { + type: EventType.TEXT_MESSAGE_END, + messageId: "msg-streamed", + } as TextMessageEndEvent, + { + type: EventType.RUN_FINISHED, + threadId: "test-thread", + runId: "run-1", + } as RunFinishedEvent, + ]; + + agent.setEvents(events); + const result = await agent.runAgent({ runId: "run-1" }); + + // Verify message order: input message first, then streamed message + expect(agent.messages.length).toBe(2); + expect(agent.messages[0].id).toBe("msg-from-input"); + expect(agent.messages[0].content).toBe("From input"); + expect(agent.messages[1].id).toBe("msg-streamed"); + expect(agent.messages[1].content).toBe("Streamed response"); + + expect(result.newMessages.length).toBe(2); + }); + + it("should handle multiple runs with input.messages", async () => { + const agent = new TestAgent({ + threadId: "test-thread", + initialMessages: [], + }); + + // First run with one message + const firstRunEvents: BaseEvent[] = [ + { + type: EventType.RUN_STARTED, + threadId: "test-thread", + runId: "run-1", + input: { + threadId: "test-thread", + runId: "run-1", + messages: [ + { + id: "msg-1", + role: "user", + content: "First message", + }, + ], + tools: [], + context: [], + state: {}, + forwardedProps: {}, + }, + } as RunStartedEvent, + { + type: EventType.RUN_FINISHED, + threadId: "test-thread", + runId: "run-1", + } as RunFinishedEvent, + ]; + + agent.setEvents(firstRunEvents); + const result1 = await agent.runAgent({ runId: "run-1" }); + + expect(agent.messages.length).toBe(1); + expect(agent.messages[0].id).toBe("msg-1"); + expect(result1.newMessages.length).toBe(1); + + // Second run with three messages (one duplicate, two new) + const secondRunEvents: BaseEvent[] = [ + { + type: EventType.RUN_STARTED, + threadId: "test-thread", + runId: "run-2", + input: { + threadId: "test-thread", + runId: "run-2", + messages: [ + { + id: "msg-1", + role: "user", + content: "First message (duplicate)", + }, + { + id: "msg-2", + role: "user", + content: "Second message", + }, + { + id: "msg-3", + role: "user", + content: "Third message", + }, + ], + tools: [], + context: [], + state: {}, + forwardedProps: {}, + }, + } as RunStartedEvent, + { + type: EventType.RUN_FINISHED, + threadId: "test-thread", + runId: "run-2", + } as RunFinishedEvent, + ]; + + agent.setEvents(secondRunEvents); + const result2 = await agent.runAgent({ runId: "run-2" }); + + // Verify only new messages were added + expect(agent.messages.length).toBe(3); + expect(agent.messages[0].id).toBe("msg-1"); + expect(agent.messages[0].content).toBe("First message"); // Original content preserved + expect(agent.messages[1].id).toBe("msg-2"); + expect(agent.messages[1].content).toBe("Second message"); + expect(agent.messages[2].id).toBe("msg-3"); + expect(agent.messages[2].content).toBe("Third message"); + + // Verify only the two new messages appear in newMessages for the second run + expect(result2.newMessages.length).toBe(2); + expect(result2.newMessages[0].id).toBe("msg-2"); + expect(result2.newMessages[1].id).toBe("msg-3"); + }); +}); diff --git a/typescript-sdk/packages/client/src/apply/default.ts b/typescript-sdk/packages/client/src/apply/default.ts index 8f720c7a0..a19ee3c6b 100644 --- a/typescript-sdk/packages/client/src/apply/default.ts +++ b/typescript-sdk/packages/client/src/apply/default.ts @@ -567,6 +567,25 @@ export const defaultApplyEvents = ( ); applyMutation(mutation); + // Handle input.messages if present and stopPropagation is not set + if (mutation.stopPropagation !== true) { + const runStartedEvent = event as RunStartedEvent; + + // Check if the event contains input with messages + if (runStartedEvent.input?.messages) { + // Add messages that aren't already present (checked by ID) + for (const message of runStartedEvent.input.messages) { + const existingMessage = messages.find((m) => m.id === message.id); + if (!existingMessage) { + messages.push(message); + } + } + + // Apply mutation to emit the updated messages + applyMutation({ messages }); + } + } + return emitUpdates(); } From 666cb1881a12641e05447351ec824de0fff3d0f0 Mon Sep 17 00:00:00 2001 From: Markus Ecker Date: Tue, 14 Oct 2025 11:29:57 +0200 Subject: [PATCH 19/25] v0.0.40-alpha.4 --- typescript-sdk/packages/cli/package.json | 2 +- typescript-sdk/packages/client/package.json | 2 +- typescript-sdk/packages/core/package.json | 2 +- typescript-sdk/packages/encoder/package.json | 2 +- typescript-sdk/packages/proto/package.json | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/typescript-sdk/packages/cli/package.json b/typescript-sdk/packages/cli/package.json index f0b0a012c..c3eeabd0c 100644 --- a/typescript-sdk/packages/cli/package.json +++ b/typescript-sdk/packages/cli/package.json @@ -1,7 +1,7 @@ { "name": "create-ag-ui-app", "author": "Markus Ecker ", - "version": "0.0.40-alpha.3", + "version": "0.0.40-alpha.4", "private": false, "publishConfig": { "access": "public" diff --git a/typescript-sdk/packages/client/package.json b/typescript-sdk/packages/client/package.json index acf9e509c..1fdf948af 100644 --- a/typescript-sdk/packages/client/package.json +++ b/typescript-sdk/packages/client/package.json @@ -1,7 +1,7 @@ { "name": "@ag-ui/client", "author": "Markus Ecker ", - "version": "0.0.40-alpha.3", + "version": "0.0.40-alpha.4", "private": false, "publishConfig": { "access": "public" diff --git a/typescript-sdk/packages/core/package.json b/typescript-sdk/packages/core/package.json index 77c8c5154..93c1fbf25 100644 --- a/typescript-sdk/packages/core/package.json +++ b/typescript-sdk/packages/core/package.json @@ -1,7 +1,7 @@ { "name": "@ag-ui/core", "author": "Markus Ecker ", - "version": "0.0.40-alpha.3", + "version": "0.0.40-alpha.4", "private": false, "publishConfig": { "access": "public" diff --git a/typescript-sdk/packages/encoder/package.json b/typescript-sdk/packages/encoder/package.json index e2d93960e..2b99fd694 100644 --- a/typescript-sdk/packages/encoder/package.json +++ b/typescript-sdk/packages/encoder/package.json @@ -1,7 +1,7 @@ { "name": "@ag-ui/encoder", "author": "Markus Ecker ", - "version": "0.0.40-alpha.3", + "version": "0.0.40-alpha.4", "private": false, "publishConfig": { "access": "public" diff --git a/typescript-sdk/packages/proto/package.json b/typescript-sdk/packages/proto/package.json index feffba233..456225162 100644 --- a/typescript-sdk/packages/proto/package.json +++ b/typescript-sdk/packages/proto/package.json @@ -1,7 +1,7 @@ { "name": "@ag-ui/proto", "author": "Markus Ecker ", - "version": "0.0.40-alpha.3", + "version": "0.0.40-alpha.4", "private": false, "publishConfig": { "access": "public" From d96c9a296141c215db94a937f002964a3536c364 Mon Sep 17 00:00:00 2001 From: Markus Ecker Date: Tue, 14 Oct 2025 12:49:32 +0200 Subject: [PATCH 20/25] add parentRunId --- docs/sdk/js/core/types.mdx | 2 ++ docs/sdk/python/core/types.mdx | 2 ++ python-sdk/ag_ui/core/types.py | 1 + python-sdk/tests/test_types.py | 2 ++ .../packages/core/src/__tests__/backwards-compatibility.test.ts | 2 ++ typescript-sdk/packages/core/src/types.ts | 1 + 6 files changed, 10 insertions(+) diff --git a/docs/sdk/js/core/types.mdx b/docs/sdk/js/core/types.mdx index afc011609..6e24c1952 100644 --- a/docs/sdk/js/core/types.mdx +++ b/docs/sdk/js/core/types.mdx @@ -20,6 +20,7 @@ Input parameters for running an agent. In the HTTP API, this is the body of the type RunAgentInput = { threadId: string runId: string + parentRunId?: string state: any messages: Message[] tools: Tool[] @@ -32,6 +33,7 @@ type RunAgentInput = { | ---------------- | ----------- | ---------------------------------------------- | | `threadId` | `string` | ID of the conversation thread | | `runId` | `string` | ID of the current run | +| `parentRunId` | `string (optional)` | ID of the run that spawned this run | | `state` | `any` | Current state of the agent | | `messages` | `Message[]` | Array of messages in the conversation | | `tools` | `Tool[]` | Array of tools available to the agent | diff --git a/docs/sdk/python/core/types.mdx b/docs/sdk/python/core/types.mdx index 4f9180455..1cca4073d 100644 --- a/docs/sdk/python/core/types.mdx +++ b/docs/sdk/python/core/types.mdx @@ -22,6 +22,7 @@ Input parameters for running an agent. In the HTTP API, this is the body of the class RunAgentInput(ConfiguredBaseModel): thread_id: str run_id: str + parent_run_id: Optional[str] = None state: Any messages: List[Message] tools: List[Tool] @@ -33,6 +34,7 @@ class RunAgentInput(ConfiguredBaseModel): | ----------------- | --------------- | --------------------------------------------- | | `thread_id` | `str` | ID of the conversation thread | | `run_id` | `str` | ID of the current run | +| `parent_run_id` | `Optional[str]` | (Optional) ID of the run that spawned this run| | `state` | `Any` | Current state of the agent | | `messages` | `List[Message]` | List of messages in the conversation | | `tools` | `List[Tool]` | List of tools available to the agent | diff --git a/python-sdk/ag_ui/core/types.py b/python-sdk/ag_ui/core/types.py index d22d4eab7..b7cb1520c 100644 --- a/python-sdk/ag_ui/core/types.py +++ b/python-sdk/ag_ui/core/types.py @@ -120,6 +120,7 @@ class RunAgentInput(ConfiguredBaseModel): """ thread_id: str run_id: str + parent_run_id: Optional[str] = None state: Any messages: List[Message] tools: List[Tool] diff --git a/python-sdk/tests/test_types.py b/python-sdk/tests/test_types.py index 27eedff0a..60182f231 100644 --- a/python-sdk/tests/test_types.py +++ b/python-sdk/tests/test_types.py @@ -209,6 +209,7 @@ def test_run_agent_input_deserialization(self): run_agent_input_data = { "threadId": "thread_12345", "runId": "run_67890", + "parentRunId": "run_parent_123", "state": {"conversation_state": "active", "custom_data": {"key": "value"}}, "messages": [ # System message @@ -307,6 +308,7 @@ def test_run_agent_input_deserialization(self): # Verify basic fields self.assertEqual(run_agent_input.thread_id, "thread_12345") self.assertEqual(run_agent_input.run_id, "run_67890") + self.assertEqual(run_agent_input.parent_run_id, "run_parent_123") self.assertEqual(run_agent_input.state["conversation_state"], "active") # Verify messages count and types diff --git a/typescript-sdk/packages/core/src/__tests__/backwards-compatibility.test.ts b/typescript-sdk/packages/core/src/__tests__/backwards-compatibility.test.ts index 2cdeb48be..34c47961f 100644 --- a/typescript-sdk/packages/core/src/__tests__/backwards-compatibility.test.ts +++ b/typescript-sdk/packages/core/src/__tests__/backwards-compatibility.test.ts @@ -57,6 +57,7 @@ describe("Backwards Compatibility", () => { const inputWithExtraFields = { threadId: "thread_1", runId: "run_1", + parentRunId: "parent_run_1", state: {}, messages: [], tools: [], @@ -74,6 +75,7 @@ describe("Backwards Compatibility", () => { if (result.success) { expect(result.data.threadId).toBe("thread_1"); expect(result.data.runId).toBe("run_1"); + expect(result.data.parentRunId).toBe("parent_run_1"); } }); diff --git a/typescript-sdk/packages/core/src/types.ts b/typescript-sdk/packages/core/src/types.ts index 0b5b17224..8b2352a13 100644 --- a/typescript-sdk/packages/core/src/types.ts +++ b/typescript-sdk/packages/core/src/types.ts @@ -77,6 +77,7 @@ export const ToolSchema = z.object({ export const RunAgentInputSchema = z.object({ threadId: z.string(), runId: z.string(), + parentRunId: z.string().optional(), state: z.any(), messages: z.array(MessageSchema), tools: z.array(ToolSchema), From 6a24af58f574843ea4afd75729cb0072a5bb2aff Mon Sep 17 00:00:00 2001 From: Markus Ecker Date: Tue, 14 Oct 2025 12:50:54 +0200 Subject: [PATCH 21/25] v0.0.40-alpha.5 --- typescript-sdk/packages/cli/package.json | 2 +- typescript-sdk/packages/client/package.json | 2 +- typescript-sdk/packages/core/package.json | 2 +- typescript-sdk/packages/encoder/package.json | 2 +- typescript-sdk/packages/proto/package.json | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/typescript-sdk/packages/cli/package.json b/typescript-sdk/packages/cli/package.json index c3eeabd0c..a5db98afa 100644 --- a/typescript-sdk/packages/cli/package.json +++ b/typescript-sdk/packages/cli/package.json @@ -1,7 +1,7 @@ { "name": "create-ag-ui-app", "author": "Markus Ecker ", - "version": "0.0.40-alpha.4", + "version": "0.0.40-alpha.5", "private": false, "publishConfig": { "access": "public" diff --git a/typescript-sdk/packages/client/package.json b/typescript-sdk/packages/client/package.json index 1fdf948af..f5e783401 100644 --- a/typescript-sdk/packages/client/package.json +++ b/typescript-sdk/packages/client/package.json @@ -1,7 +1,7 @@ { "name": "@ag-ui/client", "author": "Markus Ecker ", - "version": "0.0.40-alpha.4", + "version": "0.0.40-alpha.5", "private": false, "publishConfig": { "access": "public" diff --git a/typescript-sdk/packages/core/package.json b/typescript-sdk/packages/core/package.json index 93c1fbf25..905ca4961 100644 --- a/typescript-sdk/packages/core/package.json +++ b/typescript-sdk/packages/core/package.json @@ -1,7 +1,7 @@ { "name": "@ag-ui/core", "author": "Markus Ecker ", - "version": "0.0.40-alpha.4", + "version": "0.0.40-alpha.5", "private": false, "publishConfig": { "access": "public" diff --git a/typescript-sdk/packages/encoder/package.json b/typescript-sdk/packages/encoder/package.json index 2b99fd694..7e8e0d805 100644 --- a/typescript-sdk/packages/encoder/package.json +++ b/typescript-sdk/packages/encoder/package.json @@ -1,7 +1,7 @@ { "name": "@ag-ui/encoder", "author": "Markus Ecker ", - "version": "0.0.40-alpha.4", + "version": "0.0.40-alpha.5", "private": false, "publishConfig": { "access": "public" diff --git a/typescript-sdk/packages/proto/package.json b/typescript-sdk/packages/proto/package.json index 456225162..528e18034 100644 --- a/typescript-sdk/packages/proto/package.json +++ b/typescript-sdk/packages/proto/package.json @@ -1,7 +1,7 @@ { "name": "@ag-ui/proto", "author": "Markus Ecker ", - "version": "0.0.40-alpha.4", + "version": "0.0.40-alpha.5", "private": false, "publishConfig": { "access": "public" From b10e9f35c62be0c4a43f34629864a42c63f88090 Mon Sep 17 00:00:00 2001 From: Markus Ecker Date: Tue, 14 Oct 2025 15:29:28 +0200 Subject: [PATCH 22/25] Fix for early isRunning=false --- .../packages/client/src/agent/agent.ts | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/typescript-sdk/packages/client/src/agent/agent.ts b/typescript-sdk/packages/client/src/agent/agent.ts index 081cd1934..0560d049e 100644 --- a/typescript-sdk/packages/client/src/agent/agent.ts +++ b/typescript-sdk/packages/client/src/agent/agent.ts @@ -96,12 +96,11 @@ export abstract class AbstractAgent { }), ); - return lastValueFrom(pipeline(of(null))).then(() => { - const newMessages = structuredClone_(this.messages).filter( - (message: Message) => !currentMessageIds.has(message.id), - ); - return { result, newMessages }; - }); + await lastValueFrom(pipeline(of(null))); + const newMessages = structuredClone_(this.messages).filter( + (message: Message) => !currentMessageIds.has(message.id), + ); + return { result, newMessages }; } finally { this.isRunning = false; } @@ -152,12 +151,11 @@ export abstract class AbstractAgent { }), ); - return lastValueFrom(pipeline(of(null))).then(() => { - const newMessages = structuredClone_(this.messages).filter( - (message: Message) => !currentMessageIds.has(message.id), - ); - return { result, newMessages }; - }); + await lastValueFrom(pipeline(of(null))); // wait for stream completion before toggling isRunning + const newMessages = structuredClone_(this.messages).filter( + (message: Message) => !currentMessageIds.has(message.id), + ); + return { result, newMessages }; } finally { this.isRunning = false; } From 992b5ed19b9274f5d827c3d914f73d4e2cd3ea51 Mon Sep 17 00:00:00 2001 From: Markus Ecker Date: Tue, 14 Oct 2025 15:30:08 +0200 Subject: [PATCH 23/25] v0.0.40-alpha.6 --- typescript-sdk/packages/cli/package.json | 2 +- typescript-sdk/packages/client/package.json | 2 +- typescript-sdk/packages/core/package.json | 2 +- typescript-sdk/packages/encoder/package.json | 2 +- typescript-sdk/packages/proto/package.json | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/typescript-sdk/packages/cli/package.json b/typescript-sdk/packages/cli/package.json index a5db98afa..0a22e0736 100644 --- a/typescript-sdk/packages/cli/package.json +++ b/typescript-sdk/packages/cli/package.json @@ -1,7 +1,7 @@ { "name": "create-ag-ui-app", "author": "Markus Ecker ", - "version": "0.0.40-alpha.5", + "version": "0.0.40-alpha.6", "private": false, "publishConfig": { "access": "public" diff --git a/typescript-sdk/packages/client/package.json b/typescript-sdk/packages/client/package.json index f5e783401..b87185d1d 100644 --- a/typescript-sdk/packages/client/package.json +++ b/typescript-sdk/packages/client/package.json @@ -1,7 +1,7 @@ { "name": "@ag-ui/client", "author": "Markus Ecker ", - "version": "0.0.40-alpha.5", + "version": "0.0.40-alpha.6", "private": false, "publishConfig": { "access": "public" diff --git a/typescript-sdk/packages/core/package.json b/typescript-sdk/packages/core/package.json index 905ca4961..6aad73e41 100644 --- a/typescript-sdk/packages/core/package.json +++ b/typescript-sdk/packages/core/package.json @@ -1,7 +1,7 @@ { "name": "@ag-ui/core", "author": "Markus Ecker ", - "version": "0.0.40-alpha.5", + "version": "0.0.40-alpha.6", "private": false, "publishConfig": { "access": "public" diff --git a/typescript-sdk/packages/encoder/package.json b/typescript-sdk/packages/encoder/package.json index 7e8e0d805..e3145b022 100644 --- a/typescript-sdk/packages/encoder/package.json +++ b/typescript-sdk/packages/encoder/package.json @@ -1,7 +1,7 @@ { "name": "@ag-ui/encoder", "author": "Markus Ecker ", - "version": "0.0.40-alpha.5", + "version": "0.0.40-alpha.6", "private": false, "publishConfig": { "access": "public" diff --git a/typescript-sdk/packages/proto/package.json b/typescript-sdk/packages/proto/package.json index 528e18034..6d172cbe7 100644 --- a/typescript-sdk/packages/proto/package.json +++ b/typescript-sdk/packages/proto/package.json @@ -1,7 +1,7 @@ { "name": "@ag-ui/proto", "author": "Markus Ecker ", - "version": "0.0.40-alpha.5", + "version": "0.0.40-alpha.6", "private": false, "publishConfig": { "access": "public" From 2f3d135dfa368836beb794b460db9448a67b8e41 Mon Sep 17 00:00:00 2001 From: Markus Ecker Date: Thu, 16 Oct 2025 18:44:10 +0200 Subject: [PATCH 24/25] Multi-modal Messages --- docs/concepts/messages.mdx | 22 ++++++++- docs/drafts/multimodal-messages.mdx | 2 +- docs/sdk/js/core/types.mdx | 46 ++++++++++++++++--- docs/sdk/python/core/types.mdx | 54 ++++++++++++++++++++--- python-sdk/README.md | 17 +++++++ python-sdk/ag_ui/core/__init__.py | 10 ++++- python-sdk/ag_ui/core/types.py | 40 +++++++++++++++-- python-sdk/tests/test_types.py | 44 +++++++++++++++++- typescript-sdk/README.md | 18 ++++++++ typescript-sdk/packages/core/src/types.ts | 47 +++++++++++++++++++- 10 files changed, 275 insertions(+), 25 deletions(-) diff --git a/docs/concepts/messages.mdx b/docs/concepts/messages.mdx index 4777cbfe1..499e22730 100644 --- a/docs/concepts/messages.mdx +++ b/docs/concepts/messages.mdx @@ -41,11 +41,31 @@ Messages from the end user to the agent: interface UserMessage { id: string role: "user" - content: string // Text input from the user + content: string | InputContent[] // Text or multimodal input from the user name?: string // Optional user identifier } + +type InputContent = TextInputContent | BinaryInputContent + +interface TextInputContent { + type: "text" + text: string +} + +interface BinaryInputContent { + type: "binary" + mimeType: string + id?: string + url?: string + data?: string + filename?: string +} ``` +> For `BinaryInputContent`, provide at least one of `id`, `url`, or `data` to reference the payload. + +This structure keeps traditional plain-text inputs working while enabling richer payloads such as images, audio clips, or uploaded files in the same message. + ### Assistant Messages Messages from the AI assistant to the user: diff --git a/docs/drafts/multimodal-messages.mdx b/docs/drafts/multimodal-messages.mdx index 969032fc9..6b375ff89 100644 --- a/docs/drafts/multimodal-messages.mdx +++ b/docs/drafts/multimodal-messages.mdx @@ -21,7 +21,7 @@ apps. Inputs may include text, images, audio, and files. ## Status -- **Status**: Draft +- **Status**: Implemented — October 16, 2025 - **Author(s)**: Markus Ecker (mail@mme.xyz) ## Detailed Specification diff --git a/docs/sdk/js/core/types.mdx b/docs/sdk/js/core/types.mdx index 6e24c1952..528f46ee9 100644 --- a/docs/sdk/js/core/types.mdx +++ b/docs/sdk/js/core/types.mdx @@ -123,17 +123,49 @@ Represents a message from a user. type UserMessage = { id: string role: "user" - content: string + content: string | InputContent[] name?: string } ``` -| Property | Type | Description | -| --------- | -------- | ------------------------------------------- | -| `id` | `string` | Unique identifier for the message | -| `role` | `"user"` | Role of the message sender, fixed as "user" | -| `content` | `string` | Text content of the message (required) | -| `name` | `string` | Optional name of the sender | +| Property | Type | Description | +| --------- | --------------------------- | --------------------------------------------------------------------- | +| `id` | `string` | Unique identifier for the message | +| `role` | `"user"` | Role of the message sender, fixed as "user" | +| `content` | `string \| InputContent[]` | Either plain text or an ordered array of multimodal content fragments | +| `name` | `string` | Optional name of the sender | + +### InputContent + +Union of supported multimodal fragments. + +```typescript +type InputContent = TextInputContent | BinaryInputContent +``` + +### TextInputContent + +```typescript +type TextInputContent = { + type: "text" + text: string +} +``` + +### BinaryInputContent + +```typescript +type BinaryInputContent = { + type: "binary" + mimeType: string + id?: string + url?: string + data?: string + filename?: string +} +``` + +> At least one of `id`, `url`, or `data` must be provided. ### ToolMessage diff --git a/docs/sdk/python/core/types.mdx b/docs/sdk/python/core/types.mdx index 1cca4073d..5d864631a 100644 --- a/docs/sdk/python/core/types.mdx +++ b/docs/sdk/python/core/types.mdx @@ -124,15 +124,55 @@ Represents a message from a user. ```python class UserMessage(BaseMessage): role: Literal["user"] - content: str + content: Union[str, List["InputContent"]] +``` + +| Property | Type | Description | +| --------- | ---------------------------------- | --------------------------------------------------------------------- | +| `id` | `str` | Unique identifier for the message | +| `role` | `Literal["user"]` | Role of the message sender, fixed as "user" | +| `content` | `Union[str, List["InputContent"]]` | Either a plain text string or an ordered list of multimodal fragments | +| `name` | `Optional[str]` | Optional name of the sender | + +### TextInputContent + +Represents a text fragment inside a multimodal user message. + +```python +class TextInputContent(ConfiguredBaseModel): + type: Literal["text"] + text: str ``` -| Property | Type | Description | -| --------- | ----------------- | ------------------------------------------- | -| `id` | `str` | Unique identifier for the message | -| `role` | `Literal["user"]` | Role of the message sender, fixed as "user" | -| `content` | `str` | Text content of the message (required) | -| `name` | `Optional[str]` | Optional name of the sender | +| Property | Type | Description | +| -------- | --------------- | ---------------------------- | +| `type` | `Literal["text"]` | Identifies the fragment type | +| `text` | `str` | Text content | + +### BinaryInputContent + +Represents binary data such as images, audio, or files. + +```python +class BinaryInputContent(ConfiguredBaseModel): + type: Literal["binary"] + mime_type: str + id: Optional[str] = None + url: Optional[str] = None + data: Optional[str] = None + filename: Optional[str] = None +``` + +| Property | Type | Description | +| ---------- | ----------------- | ------------------------------------------------------------- | +| `type` | `Literal["binary"]` | Identifies the fragment type | +| `mime_type`| `str` | MIME type, for example `"image/png"` | +| `id` | `Optional[str]` | Reference to previously uploaded content | +| `url` | `Optional[str]` | Remote URL where the content can be retrieved | +| `data` | `Optional[str]` | Base64 encoded content | +| `filename` | `Optional[str]` | Optional filename hint | + +> **Validation:** At least one of `id`, `url`, or `data` must be provided. ### ToolMessage diff --git a/python-sdk/README.md b/python-sdk/README.md index 843d9d028..d9e48e26a 100644 --- a/python-sdk/README.md +++ b/python-sdk/README.md @@ -38,6 +38,23 @@ sse_data = encoder.encode(event) # Output: data: {"type":"TEXT_MESSAGE_CONTENT","messageId":"msg_123","delta":"Hello from Python!"}\n\n ``` +### Multimodal user message + +```python +from ag_ui.core import UserMessage, TextInputContent, BinaryInputContent + +message = UserMessage( + id="user-123", + content=[ + TextInputContent(text="Please describe this image"), + BinaryInputContent(mime_type="image/png", url="https://example.com/cat.png"), + ], +) + +payload = message.model_dump(by_alias=True) +# {"id": "user-123", "role": "user", "content": [...]} +``` + ## Packages - **`ag_ui.core`** – Types, events, and data models for AG-UI protocol diff --git a/python-sdk/ag_ui/core/__init__.py b/python-sdk/ag_ui/core/__init__.py index 7e909ad5b..67ae5ac6c 100644 --- a/python-sdk/ag_ui/core/__init__.py +++ b/python-sdk/ag_ui/core/__init__.py @@ -46,7 +46,10 @@ Context, Tool, RunAgentInput, - State + State, + TextInputContent, + BinaryInputContent, + InputContent, ) __all__ = [ @@ -92,5 +95,8 @@ "Context", "Tool", "RunAgentInput", - "State" + "State", + "TextInputContent", + "BinaryInputContent", + "InputContent", ] diff --git a/python-sdk/ag_ui/core/types.py b/python-sdk/ag_ui/core/types.py index b7cb1520c..06b253fe7 100644 --- a/python-sdk/ag_ui/core/types.py +++ b/python-sdk/ag_ui/core/types.py @@ -4,7 +4,7 @@ from typing import Annotated, Any, List, Literal, Optional, Union -from pydantic import BaseModel, ConfigDict, Field +from pydantic import BaseModel, ConfigDict, Field, model_validator from pydantic.alias_generators import to_camel @@ -70,12 +70,44 @@ class AssistantMessage(BaseMessage): tool_calls: Optional[List[ToolCall]] = None +class TextInputContent(ConfiguredBaseModel): + """A text fragment in a multimodal user message.""" + + type: Literal["text"] = "text" + text: str + + +class BinaryInputContent(ConfiguredBaseModel): + """A binary payload reference in a multimodal user message.""" + + type: Literal["binary"] = "binary" # pyright: ignore[reportIncompatibleVariableOverride] + mime_type: str + id: Optional[str] = None + url: Optional[str] = None + data: Optional[str] = None + filename: Optional[str] = None + + @model_validator(mode="after") + def validate_source(self) -> "BinaryInputContent": + """Ensure at least one binary payload source is provided.""" + if not any([self.id, self.url, self.data]): + raise ValueError("BinaryInputContent requires id, url, or data to be provided.") + return self + + +InputContent = Annotated[ + Union[TextInputContent, BinaryInputContent], + Field(discriminator="type"), +] + + class UserMessage(BaseMessage): """ - A user message. + A user message supporting text or multimodal content. """ - role: Literal["user"] = "user" # pyright: ignore[reportIncompatibleVariableOverride] - content: str + + role: Literal["user"] = "user" # pyright: ignore[reportIncompatibleVariableOverride] + content: Union[str, List[InputContent]] class ToolMessage(ConfiguredBaseModel): diff --git a/python-sdk/tests/test_types.py b/python-sdk/tests/test_types.py index 60182f231..94a5bdbc3 100644 --- a/python-sdk/tests/test_types.py +++ b/python-sdk/tests/test_types.py @@ -11,7 +11,9 @@ UserMessage, ToolMessage, Message, - RunAgentInput + RunAgentInput, + TextInputContent, + BinaryInputContent, ) @@ -143,6 +145,31 @@ def test_user_message(self): self.assertEqual(serialized["role"], "user") self.assertEqual(serialized["content"], "User query") + def test_user_message_multimodal_content(self): + """Test creating and serializing a multimodal user message""" + contents = [ + TextInputContent(text="Check this out"), + BinaryInputContent(mime_type="image/png", url="https://example.com/image.png"), + ] + msg = UserMessage( + id="user_multi", + content=contents, + ) + self.assertIsInstance(msg.content, list) + self.assertEqual(len(msg.content), 2) + serialized = msg.model_dump(by_alias=True) + self.assertIsInstance(serialized["content"], list) + self.assertEqual(serialized["content"][0]["type"], "text") + self.assertEqual(serialized["content"][0]["text"], "Check this out") + self.assertEqual(serialized["content"][1]["mimeType"], "image/png") + self.assertEqual(serialized["content"][1]["url"], "https://example.com/image.png") + + def test_binary_input_requires_payload_source(self): + """Binary content must specify at least one delivery channel""" + with self.assertRaises(ValidationError): + BinaryInputContent(mime_type="image/png") + + def test_message_union_deserialization(self): """Test that the Message union correctly deserializes to the appropriate type""" # Create type adapter for the union @@ -257,7 +284,14 @@ def test_run_agent_input_deserialization(self): { "id": "user_002", "role": "user", - "content": "Can you explain these results?" + "content": [ + {"type": "text", "text": "Can you explain these results?"}, + { + "type": "binary", + "mimeType": "image/png", + "url": "https://example.com/results-chart.png" + } + ] } ], "tools": [ @@ -323,6 +357,12 @@ def test_run_agent_input_deserialization(self): # Verify specific message content self.assertEqual(run_agent_input.messages[0].content, "You are a helpful assistant.") self.assertEqual(run_agent_input.messages[1].content, "Can you help me analyze this data?") + multimodal_content = run_agent_input.messages[5].content + self.assertIsInstance(multimodal_content, list) + self.assertEqual(multimodal_content[0].type, "text") + self.assertEqual(multimodal_content[0].text, "Can you explain these results?") + self.assertEqual(multimodal_content[1].mime_type, "image/png") + self.assertEqual(multimodal_content[1].url, "https://example.com/results-chart.png") # Verify assistant message with tool call assistant_msg = run_agent_input.messages[3] diff --git a/typescript-sdk/README.md b/typescript-sdk/README.md index 64b9a5f4a..1de4bd2f4 100644 --- a/typescript-sdk/README.md +++ b/typescript-sdk/README.md @@ -3,3 +3,21 @@ The TypeScript SDK for the [Agent User Interaction Protocol](https://ag-ui.com). For more information visit the [official documentation](https://docs.ag-ui.com/). + +## Multimodal user messages + +```ts +import { UserMessageSchema } from "@ag-ui/core"; + +const message = UserMessageSchema.parse({ + id: "user-123", + role: "user" as const, + content: [ + { type: "text", text: "Please describe this image" }, + { type: "binary", mimeType: "image/png", url: "https://example.com/cat.png" }, + ], +}); + +console.log(message); +// { id: "user-123", role: "user", content: [...] } +``` diff --git a/typescript-sdk/packages/core/src/types.ts b/typescript-sdk/packages/core/src/types.ts index 8b2352a13..3a73d4e0e 100644 --- a/typescript-sdk/packages/core/src/types.ts +++ b/typescript-sdk/packages/core/src/types.ts @@ -18,6 +18,48 @@ export const BaseMessageSchema = z.object({ name: z.string().optional(), }); +export const TextInputContentSchema = z.object({ + type: z.literal("text"), + text: z.string(), +}); + +const BinaryInputContentObjectSchema = z.object({ + type: z.literal("binary"), + mimeType: z.string(), + id: z.string().optional(), + url: z.string().optional(), + data: z.string().optional(), + filename: z.string().optional(), +}); + +const ensureBinaryPayload = ( + value: { id?: string; url?: string; data?: string }, + ctx: z.RefinementCtx, +) => { + if (!value.id && !value.url && !value.data) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: "BinaryInputContent requires at least one of id, url, or data.", + path: ["id"], + }); + } +}; + +export const BinaryInputContentSchema = BinaryInputContentObjectSchema.superRefine((value, ctx) => { + ensureBinaryPayload(value, ctx); +}); + +const InputContentBaseSchema = z.discriminatedUnion("type", [ + TextInputContentSchema, + BinaryInputContentObjectSchema, +]); + +export const InputContentSchema = InputContentBaseSchema.superRefine((value, ctx) => { + if (value.type === "binary") { + ensureBinaryPayload(value, ctx); + } +}); + export const DeveloperMessageSchema = BaseMessageSchema.extend({ role: z.literal("developer"), content: z.string(), @@ -36,7 +78,7 @@ export const AssistantMessageSchema = BaseMessageSchema.extend({ export const UserMessageSchema = BaseMessageSchema.extend({ role: z.literal("user"), - content: z.string(), + content: z.union([z.string(), z.array(InputContentSchema)]), }); export const ToolMessageSchema = z.object({ @@ -89,6 +131,9 @@ export const StateSchema = z.any(); export type ToolCall = z.infer; export type FunctionCall = z.infer; +export type TextInputContent = z.infer; +export type BinaryInputContent = z.infer; +export type InputContent = z.infer; export type DeveloperMessage = z.infer; export type SystemMessage = z.infer; export type AssistantMessage = z.infer; From c3624da1db32f4314dc020b86e5fbe71c7dcf225 Mon Sep 17 00:00:00 2001 From: Markus Ecker Date: Thu, 16 Oct 2025 18:44:22 +0200 Subject: [PATCH 25/25] Multi-modal Messages --- .../src/__tests__/multimodal-messages.test.ts | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 typescript-sdk/packages/core/src/__tests__/multimodal-messages.test.ts diff --git a/typescript-sdk/packages/core/src/__tests__/multimodal-messages.test.ts b/typescript-sdk/packages/core/src/__tests__/multimodal-messages.test.ts new file mode 100644 index 000000000..3bcca30e8 --- /dev/null +++ b/typescript-sdk/packages/core/src/__tests__/multimodal-messages.test.ts @@ -0,0 +1,52 @@ +import { + UserMessageSchema, + BinaryInputContentSchema, +} from "../types"; + +describe("Multimodal messages", () => { + it("parses user message with content array", () => { + const result = UserMessageSchema.parse({ + id: "user_multimodal", + role: "user" as const, + content: [ + { type: "text" as const, text: "Check this out" }, + { type: "binary" as const, mimeType: "image/png", url: "https://example.com/image.png" }, + ], + }); + + expect(Array.isArray(result.content)).toBe(true); + if (Array.isArray(result.content)) { + expect(result.content[0].type).toBe("text"); + expect(result.content[0].text).toBe("Check this out"); + expect(result.content[1].type).toBe("binary"); + expect(result.content[1].mimeType).toBe("image/png"); + expect(result.content[1].url).toBe("https://example.com/image.png"); + } + }); + + it("rejects binary content without payload source", () => { + const result = UserMessageSchema.safeParse({ + id: "user_invalid", + role: "user" as const, + content: [{ type: "binary" as const, mimeType: "image/png" }], + }); + + expect(result.success).toBe(false); + }); + + it("parses binary input with embedded data", () => { + const binary = BinaryInputContentSchema.parse({ + type: "binary" as const, + mimeType: "image/png", + data: "base64", + }); + + expect(binary.data).toBe("base64"); + }); + + it("requires binary payload source", () => { + expect(() => + BinaryInputContentSchema.parse({ type: "binary" as const, mimeType: "image/png" }), + ).toThrow(/id, url, or data/); + }); +});