From eaf812b2dd581d73acc5339f1dc3a8a5e8700392 Mon Sep 17 00:00:00 2001 From: Sahil Date: Fri, 6 Mar 2026 13:12:17 +0530 Subject: [PATCH 01/72] feat: add Resolvable pattern for dynamic config (v2.1.1) - Add Resolvable type for static or getter function values - Support dynamic headers, runtimeUrl, and body props - Values resolved fresh at request time - Backward compatible - static values still work Co-Authored-By: Claude --- packages/copilot-sdk/package.json | 2 +- .../copilot-sdk/src/chat/ChatWithTools.ts | 39 ++++++- .../src/chat/adapters/HttpTransport.ts | 73 ++++++++++++- .../src/chat/classes/AbstractChat.ts | 39 +++++++ .../src/chat/interfaces/ChatTransport.ts | 50 ++++++++- packages/copilot-sdk/src/chat/types/chat.ts | 28 ++++- packages/copilot-sdk/src/core/utils/index.ts | 1 + .../copilot-sdk/src/core/utils/resolvable.ts | 101 ++++++++++++++++++ .../src/react/provider/CopilotProvider.tsx | 56 +++++++++- 9 files changed, 368 insertions(+), 21 deletions(-) create mode 100644 packages/copilot-sdk/src/core/utils/resolvable.ts diff --git a/packages/copilot-sdk/package.json b/packages/copilot-sdk/package.json index 30d5cd8..ace9889 100644 --- a/packages/copilot-sdk/package.json +++ b/packages/copilot-sdk/package.json @@ -1,6 +1,6 @@ { "name": "@yourgpt/copilot-sdk", - "version": "2.1.0", + "version": "2.1.1", "description": "Copilot SDK for building Production-ready AI Copilots for any product. Connect any LLM, deploy on your infrastructure, own your data.", "type": "module", "types": "./dist/core/index.d.ts", diff --git a/packages/copilot-sdk/src/chat/ChatWithTools.ts b/packages/copilot-sdk/src/chat/ChatWithTools.ts index c42d705..766c107 100644 --- a/packages/copilot-sdk/src/chat/ChatWithTools.ts +++ b/packages/copilot-sdk/src/chat/ChatWithTools.ts @@ -16,6 +16,7 @@ import type { MessageAttachment, PermissionLevel, } from "../core"; +import type { Resolvable } from "../core/utils/resolvable"; import { AbstractChat } from "./classes/AbstractChat"; import { AbstractAgentLoop } from "./AbstractAgentLoop"; import type { ChatConfig, ChatCallbacks } from "./types"; @@ -26,18 +27,23 @@ import type { ChatTransport } from "./interfaces/ChatTransport"; /** * Configuration for ChatWithTools + * + * Supports both static values and getter functions for dynamic configuration. + * Getter functions are resolved at request time, ensuring fresh values. */ export interface ChatWithToolsConfig { - /** Runtime API endpoint */ - runtimeUrl: string; + /** Runtime API endpoint - can be static or getter function */ + runtimeUrl: Resolvable; /** LLM configuration */ llm?: ChatConfig["llm"]; /** System prompt */ systemPrompt?: string; /** Enable streaming (default: true) */ streaming?: boolean; - /** Request headers */ - headers?: Record; + /** Request headers - can be static or getter function */ + headers?: Resolvable>; + /** Additional body properties - can be static or getter function */ + body?: Resolvable>; /** Thread ID for conversation persistence */ threadId?: string; /** Debug mode */ @@ -125,6 +131,7 @@ export class ChatWithTools { systemPrompt: config.systemPrompt, streaming: config.streaming, headers: config.headers, + body: config.body, threadId: config.threadId, debug: config.debug, initialMessages: config.initialMessages, @@ -372,6 +379,30 @@ export class ChatWithTools { this.chat.setSystemPrompt(prompt); } + /** + * Set headers configuration + * Can be static headers or a getter function for dynamic resolution + */ + setHeaders(headers: ChatWithToolsConfig["headers"]): void { + this.chat.setHeaders(headers); + } + + /** + * Set URL configuration + * Can be static URL or a getter function for dynamic resolution + */ + setUrl(url: ChatWithToolsConfig["runtimeUrl"]): void { + this.chat.setUrl(url); + } + + /** + * Set body configuration + * Additional properties merged into every request body + */ + setBody(body: ChatWithToolsConfig["body"]): void { + this.chat.setBody(body); + } + // ============================================ // Tool Registration // ============================================ diff --git a/packages/copilot-sdk/src/chat/adapters/HttpTransport.ts b/packages/copilot-sdk/src/chat/adapters/HttpTransport.ts index c13ad36..07cb1de 100644 --- a/packages/copilot-sdk/src/chat/adapters/HttpTransport.ts +++ b/packages/copilot-sdk/src/chat/adapters/HttpTransport.ts @@ -2,6 +2,7 @@ * HttpTransport - HTTP/SSE implementation of ChatTransport * * Uses fetch with streaming for SSE responses. + * Supports dynamic configuration via getter functions. */ import type { @@ -12,15 +13,29 @@ import type { TransportConfig, } from "../interfaces"; import { parseSSELine } from "../functions"; +import { resolveValues } from "../../core/utils/resolvable"; /** * HTTP Transport for chat API * + * Supports both static and dynamic configuration. When using getter functions, + * values are resolved fresh on every request. + * * @example * ```typescript + * // Static config * const transport = new HttpTransport({ * url: '/api/chat', - * headers: { ... }, + * headers: { "x-api-key": "static" }, + * }); + * + * // Dynamic config (recommended for auth/runtime values) + * const transport = new HttpTransport({ + * url: () => getApiEndpoint(), + * headers: () => ({ + * Authorization: `Bearer ${getToken()}`, + * ...getCustomHeaders(), + * }), * }); * * const stream = await transport.send(request); @@ -44,6 +59,7 @@ export class HttpTransport implements ChatTransport { /** * Send a chat request + * Resolves dynamic config values (url, headers, body) fresh at request time */ async send( request: ChatRequest, @@ -53,11 +69,28 @@ export class HttpTransport implements ChatTransport { this.streaming = true; try { - const response = await fetch(this.config.url, { + // Resolve dynamic values at request time (not constructor time) + // This ensures fresh values on every request + // Optimized: skips async overhead if all values are static + console.log( + "[HttpTransport] Config headers type:", + typeof this.config.headers, + ); + console.log("[HttpTransport] Config headers:", this.config.headers); + + const resolved = await resolveValues({ + url: this.config.url, + headers: this.config.headers ?? {}, + configBody: this.config.body ?? {}, + }); + + console.log("[HttpTransport] Resolved headers:", resolved.headers); + + const response = await fetch(resolved.url as string, { method: "POST", headers: { "Content-Type": "application/json", - ...this.config.headers, + ...(resolved.headers as Record), }, body: JSON.stringify({ messages: request.messages, @@ -67,6 +100,7 @@ export class HttpTransport implements ChatTransport { tools: request.tools, actions: request.actions, streaming: this.config.streaming, + ...(resolved.configBody as Record), ...request.body, }), signal: this.abortController.signal, @@ -118,6 +152,39 @@ export class HttpTransport implements ChatTransport { return this.streaming; } + /** + * Update headers configuration + * Can be static headers or a getter function for dynamic resolution + * + * @example + * ```typescript + * // Static + * transport.setHeaders({ "x-api-key": "new-key" }); + * + * // Dynamic (resolved fresh on each request) + * transport.setHeaders(() => ({ Authorization: `Bearer ${getToken()}` })); + * ``` + */ + setHeaders(headers: TransportConfig["headers"]): void { + this.config.headers = headers; + } + + /** + * Update URL configuration + * Can be static URL or a getter function for dynamic resolution + */ + setUrl(url: TransportConfig["url"]): void { + this.config.url = url; + } + + /** + * Update body configuration + * Additional properties merged into every request body + */ + setBody(body: TransportConfig["body"]): void { + this.config.body = body; + } + /** * Create an async iterable from a ReadableStream */ diff --git a/packages/copilot-sdk/src/chat/classes/AbstractChat.ts b/packages/copilot-sdk/src/chat/classes/AbstractChat.ts index 2befd5c..caa4dca 100644 --- a/packages/copilot-sdk/src/chat/classes/AbstractChat.ts +++ b/packages/copilot-sdk/src/chat/classes/AbstractChat.ts @@ -182,6 +182,7 @@ export class AbstractChat { systemPrompt: init.systemPrompt, streaming: init.streaming ?? true, headers: init.headers, + body: init.body, threadId: init.threadId, debug: init.debug, }; @@ -192,11 +193,13 @@ export class AbstractChat { (new SimpleChatState() as ChatState); // Use provided transport or create default + // Pass Resolvable values - they are resolved at request time this.transport = init.transport ?? new HttpTransport({ url: init.runtimeUrl, headers: init.headers, + body: init.body, streaming: init.streaming ?? true, }); @@ -576,6 +579,42 @@ export class AbstractChat { this.debug("System prompt updated", { length: prompt.length }); } + /** + * Set headers configuration + * Can be static headers or a getter function for dynamic resolution + */ + setHeaders(headers: ChatConfig["headers"]): void { + this.config.headers = headers; + if (this.transport.setHeaders && headers !== undefined) { + this.transport.setHeaders(headers); + } + this.debug("Headers config updated"); + } + + /** + * Set URL configuration + * Can be static URL or a getter function for dynamic resolution + */ + setUrl(url: ChatConfig["runtimeUrl"]): void { + this.config.runtimeUrl = url; + if (this.transport.setUrl) { + this.transport.setUrl(url); + } + this.debug("URL config updated"); + } + + /** + * Set body configuration + * Additional properties merged into every request body + */ + setBody(body: ChatConfig["body"]): void { + this.config.body = body; + if (this.transport.setBody && body !== undefined) { + this.transport.setBody(body); + } + this.debug("Body config updated"); + } + /** * Build the request payload */ diff --git a/packages/copilot-sdk/src/chat/interfaces/ChatTransport.ts b/packages/copilot-sdk/src/chat/interfaces/ChatTransport.ts index 0b3ff20..ab7abe5 100644 --- a/packages/copilot-sdk/src/chat/interfaces/ChatTransport.ts +++ b/packages/copilot-sdk/src/chat/interfaces/ChatTransport.ts @@ -6,6 +6,7 @@ */ import type { UIMessage } from "../types"; +import type { Resolvable } from "../../core/utils/resolvable"; /** * Chat request to send @@ -150,16 +151,57 @@ export interface ChatTransport { * Check if currently streaming */ isStreaming(): boolean; + + /** + * Update headers configuration (optional) + * Can be static headers or a getter function for dynamic resolution + */ + setHeaders?(headers: Resolvable>): void; + + /** + * Update URL configuration (optional) + * Can be static URL or a getter function for dynamic resolution + */ + setUrl?(url: Resolvable): void; + + /** + * Update body configuration (optional) + * Additional properties merged into every request body + */ + setBody?(body: Resolvable>): void; } /** * Transport configuration + * + * Supports both static values and getter functions for dynamic configuration. + * Getter functions are resolved at request time, ensuring fresh values. + * + * @example + * ```typescript + * // Static config + * const config: TransportConfig = { + * url: "/api/chat", + * headers: { "x-api-key": "static-key" }, + * }; + * + * // Dynamic config (resolved fresh on each request) + * const config: TransportConfig = { + * url: () => getApiUrl(), + * headers: () => ({ + * Authorization: `Bearer ${getToken()}`, + * ...getCustomHeaders(), + * }), + * }; + * ``` */ export interface TransportConfig { - /** API endpoint URL */ - url: string; - /** Request headers */ - headers?: Record; + /** API endpoint URL - can be static or getter function */ + url: Resolvable; + /** Request headers - can be static or getter function */ + headers?: Resolvable>; + /** Additional body properties - can be static or getter function */ + body?: Resolvable>; /** Enable streaming (default: true) */ streaming?: boolean; /** Request timeout in ms */ diff --git a/packages/copilot-sdk/src/chat/types/chat.ts b/packages/copilot-sdk/src/chat/types/chat.ts index df413c4..f59c1d9 100644 --- a/packages/copilot-sdk/src/chat/types/chat.ts +++ b/packages/copilot-sdk/src/chat/types/chat.ts @@ -5,6 +5,7 @@ */ import type { LLMConfig, MessageAttachment, ToolDefinition } from "../../core"; +import type { Resolvable } from "../../core/utils/resolvable"; import type { UIMessage } from "./message"; /** @@ -14,18 +15,37 @@ export type ChatStatus = "ready" | "submitted" | "streaming" | "error"; /** * Chat configuration + * + * Supports both static values and getter functions for dynamic configuration. + * Using getter functions ensures fresh values on every request. + * + * @example + * ```typescript + * const config: ChatConfig = { + * // Static URL + * runtimeUrl: "/api/chat", + * + * // Dynamic headers - resolved fresh on every request + * headers: () => ({ + * Authorization: `Bearer ${getToken()}`, + * ...getCustomHeaders(), + * }), + * }; + * ``` */ export interface ChatConfig { - /** Runtime API endpoint */ - runtimeUrl: string; + /** Runtime API endpoint - can be static or getter function */ + runtimeUrl: Resolvable; /** LLM configuration */ llm?: Partial; /** System prompt */ systemPrompt?: string; /** Enable streaming (default: true) */ streaming?: boolean; - /** Request headers */ - headers?: Record; + /** Request headers - can be static or getter function */ + headers?: Resolvable>; + /** Additional body properties - can be static or getter function */ + body?: Resolvable>; /** Thread ID for conversation persistence */ threadId?: string; /** Debug mode */ diff --git a/packages/copilot-sdk/src/core/utils/index.ts b/packages/copilot-sdk/src/core/utils/index.ts index 57cb684..a780015 100644 --- a/packages/copilot-sdk/src/core/utils/index.ts +++ b/packages/copilot-sdk/src/core/utils/index.ts @@ -2,3 +2,4 @@ export * from "./stream"; export * from "./id"; export * from "./zod-to-json-schema"; export * from "./attachments"; +export * from "./resolvable"; diff --git a/packages/copilot-sdk/src/core/utils/resolvable.ts b/packages/copilot-sdk/src/core/utils/resolvable.ts new file mode 100644 index 0000000..596b6d5 --- /dev/null +++ b/packages/copilot-sdk/src/core/utils/resolvable.ts @@ -0,0 +1,101 @@ +/** + * Resolvable - Type utility for values that can be static or dynamic + * + * This pattern allows SDK consumers to pass either: + * - Static value: `headers: { "x-api-key": "abc123" }` + * - Getter function: `headers: () => ({ "x-api-key": getToken() })` + * + * Values are resolved at request time, ensuring fresh data on every API call. + * This is the modern pattern used by tanstack-query, tRPC, and axios interceptors. + * + * @example + * ```tsx + * // Static (for values that never change) + * + * + * // Dynamic (for values that change at runtime) + * ({ + * Authorization: `Bearer ${getToken()}`, + * ...getCustomHeaders(), + * })} + * /> + * ``` + */ + +/** + * A value that can be either static or a getter function + * Getter can be sync or async for flexibility + */ +export type Resolvable = T | (() => T) | (() => Promise); + +/** + * Check if a value is a getter function + */ +export function isGetter( + value: Resolvable, +): value is (() => T) | (() => Promise) { + return typeof value === "function"; +} + +/** + * Resolve a potentially dynamic value + * Handles: static value, sync getter, or async getter + * Optimized: skips async overhead for static values + */ +export async function resolveValue(value: Resolvable): Promise { + if (!isGetter(value)) { + return value; + } + try { + return await value(); + } catch (error) { + console.error("[Copilot SDK] Error resolving dynamic config value:", error); + throw error; + } +} + +/** + * Resolve multiple values in parallel + * Optimized: only uses Promise.all if there are actual getters + */ +export async function resolveValues< + T extends Record>, +>( + values: T, +): Promise<{ [K in keyof T]: T[K] extends Resolvable ? U : T[K] }> { + const entries = Object.entries(values); + const hasGetters = entries.some(([, v]) => isGetter(v)); + + if (!hasGetters) { + // Fast path: no getters, return as-is + return values as { + [K in keyof T]: T[K] extends Resolvable ? U : T[K]; + }; + } + + // Resolve all values in parallel + const resolved = await Promise.all( + entries.map(async ([key, val]) => [key, await resolveValue(val)]), + ); + + return Object.fromEntries(resolved) as { + [K in keyof T]: T[K] extends Resolvable ? U : T[K]; + }; +} + +/** + * Resolve a potentially dynamic value (sync only) + * Use when you know the getter is synchronous + */ +export function resolveValueSync(value: T | (() => T)): T { + if (typeof value === "function") { + return (value as () => T)(); + } + return value; +} + +/** + * Type to extract the resolved type from a Resolvable + */ +export type ResolvedType = T extends Resolvable ? U : T; diff --git a/packages/copilot-sdk/src/react/provider/CopilotProvider.tsx b/packages/copilot-sdk/src/react/provider/CopilotProvider.tsx index fca9380..531d697 100644 --- a/packages/copilot-sdk/src/react/provider/CopilotProvider.tsx +++ b/packages/copilot-sdk/src/react/provider/CopilotProvider.tsx @@ -28,6 +28,7 @@ import type { } from "../../core"; import type { MCPServerConfig } from "../../mcp/types"; +import type { Resolvable } from "../../core/utils/resolvable"; import type { UIMessage, ToolExecution } from "../../chat"; @@ -66,8 +67,11 @@ function MCPConnection({ config }: { config: MCPServerConfig }) { export interface CopilotProviderProps { children: React.ReactNode; - /** Runtime API endpoint URL */ - runtimeUrl: string; + /** + * Runtime API endpoint URL + * Can be static string or getter function for dynamic resolution. + */ + runtimeUrl: Resolvable; /** System prompt sent with each request */ systemPrompt?: string; /** @deprecated Use useTools() hook instead */ @@ -82,8 +86,16 @@ export interface CopilotProviderProps { onError?: (error: Error) => void; /** Enable/disable streaming (default: true) */ streaming?: boolean; - /** Custom headers to send with each request */ - headers?: Record; + /** + * Custom headers to send with each request + * Can be static object or getter function for dynamic resolution. + */ + headers?: Resolvable>; + /** + * Additional body properties to include in each request + * Can be static object or getter function for dynamic resolution. + */ + body?: Resolvable>; /** Enable debug logging */ debug?: boolean; /** Max tool execution iterations (default: 20) */ @@ -142,7 +154,11 @@ export interface CopilotContextValue { // Config threadId?: string; - runtimeUrl: string; + /** + * Runtime URL configuration. + * Can be a static string or getter function (matches what was passed to provider). + */ + runtimeUrl: Resolvable; toolsConfig?: ToolsConfig; } @@ -175,6 +191,7 @@ export function CopilotProvider({ onError, streaming, headers, + body, debug = false, maxIterations, maxIterationsMessage, @@ -240,6 +257,7 @@ export function CopilotProvider({ initialMessages: uiInitialMessages, streaming, headers, + body, debug, maxIterations, maxIterationsMessage, @@ -271,6 +289,34 @@ export function CopilotProvider({ } }, [systemPrompt, debugLog]); + // ============================================ + // Headers & Body Reactivity + // ============================================ + + // Watch for headers prop changes and update chat + useEffect(() => { + if (chatRef.current && headers !== undefined) { + chatRef.current.setHeaders(headers); + debugLog("Headers config updated from prop"); + } + }, [headers, debugLog]); + + // Watch for body prop changes + useEffect(() => { + if (chatRef.current && body !== undefined) { + chatRef.current.setBody(body); + debugLog("Body config updated from prop"); + } + }, [body, debugLog]); + + // Watch for runtimeUrl prop changes + useEffect(() => { + if (chatRef.current && runtimeUrl !== undefined) { + chatRef.current.setUrl(runtimeUrl); + debugLog("URL config updated from prop"); + } + }, [runtimeUrl, debugLog]); + // Subscribe to chat state with useSyncExternalStore const messages = useSyncExternalStore( chatRef.current.subscribe, From 51feaf5fefde4375dcc859bd37bf03c28fafb837 Mon Sep 17 00:00:00 2001 From: Sahil Date: Mon, 9 Mar 2026 11:46:42 +0530 Subject: [PATCH 02/72] feat: integrate Anthropic SDK and enhance tool execution tracking - Added '@anthropic-ai/sdk' dependency to the project. - Introduced a new test page for the Express demo showcasing custom tool renderers. - Enhanced server-side tool execution tracking with hidden flags to manage UI visibility. - Updated environment variables and package versions for better integration. - Improved knowledge base search functionality and tool execution management in the Copilot SDK. --- .../app/test-express-demo/page.tsx | 361 +++++++++++++++++ .../app/test-max-iterations/page.tsx | 3 - examples/express-demo/.env.example | 3 + examples/express-demo/package.json | 1 + examples/express-demo/src/index.ts | 372 ++++++++++++++---- packages/copilot-sdk/package.json | 2 +- .../copilot-sdk/src/chat/AbstractAgentLoop.ts | 67 +++- .../copilot-sdk/src/chat/ChatWithTools.ts | 60 +++ .../src/chat/adapters/HttpTransport.ts | 8 - .../src/chat/classes/AbstractChat.ts | 220 ++++++++++- .../src/chat/functions/stream/processChunk.ts | 1 + .../src/chat/interfaces/ChatTransport.ts | 22 +- packages/copilot-sdk/src/chat/types/chat.ts | 18 + packages/copilot-sdk/src/chat/types/index.ts | 1 + .../copilot-sdk/src/chat/types/message.ts | 2 + packages/copilot-sdk/src/chat/types/tool.ts | 5 + packages/copilot-sdk/src/core/index.ts | 1 + packages/copilot-sdk/src/core/types/tools.ts | 30 +- .../src/react/hooks/useToolExecutor.ts | 1 + packages/copilot-sdk/src/react/index.ts | 7 + .../composed/chat/default-message.tsx | 16 +- .../ui/components/composed/connected-chat.tsx | 41 +- .../composed/tools/tool-execution-list.tsx | 5 + packages/llm-sdk/package.json | 2 +- packages/llm-sdk/src/adapters/anthropic.ts | 50 ++- packages/llm-sdk/src/core/stream-events.ts | 11 + .../src/providers/anthropic/provider.ts | 72 ++-- packages/llm-sdk/src/server/agent-loop.ts | 3 +- packages/llm-sdk/src/server/runtime.ts | 7 + packages/llm-sdk/src/server/stream-result.ts | 23 ++ pnpm-lock.yaml | 3 + 31 files changed, 1255 insertions(+), 163 deletions(-) create mode 100644 examples/experimental/app/test-express-demo/page.tsx diff --git a/examples/experimental/app/test-express-demo/page.tsx b/examples/experimental/app/test-express-demo/page.tsx new file mode 100644 index 0000000..d971493 --- /dev/null +++ b/examples/experimental/app/test-express-demo/page.tsx @@ -0,0 +1,361 @@ +"use client"; + +import { useState } from "react"; +import { CopilotProvider, useTools, tool } from "@yourgpt/copilot-sdk/react"; +import { CopilotChat } from "@yourgpt/copilot-sdk/ui"; +import "@yourgpt/copilot-sdk/ui/styles.css"; +import "@yourgpt/copilot-sdk/ui/themes/claude.css"; + +// ============================================ +// CUSTOM TOOL RENDERERS +// ============================================ + +/** + * Custom renderer for get_current_time tool (server-side) + */ +function TimeCard({ + execution, +}: { + execution: { status: string; result?: unknown; error?: string }; +}) { + if (execution.status === "executing") { + return ( +
+ 🕐 + Getting time... +
+ ); + } + + if (execution.status === "error" || execution.status === "failed") { + return ( +
+ + + {execution.error || "Failed to get time"} + +
+ ); + } + + const result = execution.result as { + time?: string; + timezone?: string; + } | null; + + if (!result?.time) { + return null; + } + + const date = new Date(result.time); + const formattedTime = date.toLocaleTimeString("en-US", { + hour: "2-digit", + minute: "2-digit", + second: "2-digit", + hour12: true, + }); + const formattedDate = date.toLocaleDateString("en-US", { + weekday: "long", + year: "numeric", + month: "long", + day: "numeric", + }); + + return ( +
+
🕐
+
+ + {formattedTime} + + {formattedDate} + {result.timezone && ( + + 📍 {result.timezone} + + )} +
+
+ ); +} + +/** + * Custom renderer for calculate_expression tool (frontend) + */ +function CalculatorCard({ + execution, +}: { + execution: { + status: string; + result?: unknown; + error?: string; + args: Record; + }; +}) { + if (execution.status === "executing") { + return ( +
+ 🧮 + Calculating... +
+ ); + } + + if (execution.status === "error" || execution.status === "failed") { + return ( +
+ + + {execution.error || "Calculation failed"} + +
+ ); + } + + const result = execution.result as { + expression?: string; + result?: number; + } | null; + const expression = + (execution.args?.expression as string) || result?.expression || ""; + + return ( +
+
🧮
+
+ + {expression} + + + {result?.result} + +
+
+ ); +} + +/** + * Custom renderer for get_user_location tool (frontend) + */ +function LocationCard({ + execution, +}: { + execution: { status: string; result?: unknown; error?: string }; +}) { + if (execution.status === "executing") { + return ( +
+ 📍 + + Getting location... + +
+ ); + } + + if (execution.status === "error" || execution.status === "failed") { + return ( +
+ + + {execution.error || "Failed to get location"} + +
+ ); + } + + const result = execution.result as { + city?: string; + country?: string; + coordinates?: { lat: number; lng: number }; + } | null; + + if (!result) { + return null; + } + + return ( +
+
📍
+
+ {result.city} + {result.country} + {result.coordinates && ( + + {result.coordinates.lat.toFixed(4)},{" "} + {result.coordinates.lng.toFixed(4)} + + )} +
+
+ ); +} + +// ============================================ +// FRONTEND TOOLS REGISTRATION +// ============================================ + +/** + * Component that registers frontend tools + */ +function FrontendToolsRegistration() { + useTools({ + calculate_expression: tool({ + description: + "Calculate a mathematical expression. Use this when the user asks to compute math.", + location: "client", + inputSchema: { + type: "object", + properties: { + expression: { + type: "string", + description: + "The math expression to evaluate (e.g., '2 + 2', '10 * 5')", + }, + }, + required: ["expression"], + }, + handler: async (params) => { + const { expression } = params as { expression: string }; + try { + // Safe math evaluation using Function constructor (safer than eval) + const sanitized = expression.replace(/[^0-9+\-*/().%\s]/g, ""); + // eslint-disable-next-line @typescript-eslint/no-implied-eval + const result = new Function(`return ${sanitized}`)(); + return { + success: true, + expression, + result: Number(result), + }; + } catch (error) { + return { + success: false, + error: `Invalid expression: ${error instanceof Error ? error.message : "Unknown error"}`, + }; + } + }, + }), + get_user_location: tool({ + description: + "Get the user's current location (simulated). Use this when user asks about their location.", + location: "client", + inputSchema: { + type: "object", + properties: {}, + }, + handler: async () => { + // Simulate async location lookup + await new Promise((resolve) => setTimeout(resolve, 500)); + + // Simulated location data + return { + success: true, + city: "San Francisco", + country: "United States", + coordinates: { + lat: 37.7749, + lng: -122.4194, + }, + }; + }, + }), + }); + + return null; +} + +// ============================================ +// MAIN PAGE COMPONENT +// ============================================ + +/** + * Test page for Express Demo with server-side + frontend tools + * + * Make sure express-demo is running on port 3001: + * cd examples/express-demo && pnpm dev + * + * Then run experimental: + * cd examples/experimental && pnpm dev + * + * Visit: http://localhost:3000/test-express-demo + */ +export default function TestExpressDemoPage() { + const [isStreaming, setIsStreaming] = useState(true); + + const runtimeUrl = isStreaming + ? "http://localhost:3001/api/copilot/stream" + : "http://localhost:3001/api/copilot/chat"; + + return ( +
+
+
+
+

+ Express Demo - Full Tools Test +

+

+ Server:{" "} + search_knowledge_base (hidden), get_current_time + {" | "} + Frontend:{" "} + calculate_expression, get_user_location +

+
+
+ + +
+
+
+ {runtimeUrl} +
+
+ +
+ + + + +
+
+ ); +} diff --git a/examples/experimental/app/test-max-iterations/page.tsx b/examples/experimental/app/test-max-iterations/page.tsx index 486db26..ec229f4 100644 --- a/examples/experimental/app/test-max-iterations/page.tsx +++ b/examples/experimental/app/test-max-iterations/page.tsx @@ -46,7 +46,6 @@ function TestTools() { useTools({ // Step 1: Initialize a process step1_initialize: { - name: "step1_initialize", location: "client", description: "Step 1: Initialize a multi-step process. MUST be called first before step2.", @@ -71,7 +70,6 @@ function TestTools() { // Step 2: Process step2_process: { - name: "step2_process", location: "client", description: "Step 2: Process the initialized task. Call this after step1_initialize.", @@ -96,7 +94,6 @@ function TestTools() { // Step 3: Finalize step3_finalize: { - name: "step3_finalize", location: "client", description: "Step 3: Finalize and complete the process. Call this after step2_process.", diff --git a/examples/express-demo/.env.example b/examples/express-demo/.env.example index 6494087..a3947e3 100644 --- a/examples/express-demo/.env.example +++ b/examples/express-demo/.env.example @@ -1,2 +1,5 @@ +# Choose one provider (Anthropic preferred for server-side tools) +ANTHROPIC_API_KEY=sk-ant-your-key-here OPENAI_API_KEY=sk-your-key-here + PORT=3001 diff --git a/examples/express-demo/package.json b/examples/express-demo/package.json index c771188..b8c30c0 100644 --- a/examples/express-demo/package.json +++ b/examples/express-demo/package.json @@ -8,6 +8,7 @@ "start": "tsx src/index.ts" }, "dependencies": { + "@anthropic-ai/sdk": "^0.39.0", "@yourgpt/llm-sdk": "workspace:*", "cors": "^2.8.5", "dotenv": "^16.4.0", diff --git a/examples/express-demo/src/index.ts b/examples/express-demo/src/index.ts index 1f29bef..a289289 100644 --- a/examples/express-demo/src/index.ts +++ b/examples/express-demo/src/index.ts @@ -1,42 +1,214 @@ import "dotenv/config"; import express from "express"; import cors from "cors"; -import { createRuntime } from "@yourgpt/llm-sdk"; +import { createRuntime, type ToolDefinition } from "@yourgpt/llm-sdk"; +import { createAnthropic } from "@yourgpt/llm-sdk/anthropic"; import { createOpenAI } from "@yourgpt/llm-sdk/openai"; const app = express(); app.use(cors()); app.use(express.json()); -// Create runtime once at startup +// ============================================ +// DUMMY KNOWLEDGE BASE DATA +// ============================================ + +const KNOWLEDGE_BASE_DATA = [ + { + doc_id: "kb_001", + content: + "YourGPT Copilot SDK is a powerful toolkit for building AI-powered chat interfaces. It supports multiple LLM providers including OpenAI, Anthropic Claude, Google Gemini, and local models via Ollama.", + score: 0.95, + }, + { + doc_id: "kb_002", + content: + "To install the SDK, run: npm install @yourgpt/llm-sdk. The SDK provides createRuntime() for server-side usage and CopilotProvider for React clients.", + score: 0.92, + }, + { + doc_id: "kb_003", + content: + "Server-side tools are defined with location: 'server' and include a handler function. The handler is executed on the server and results are sent back to the LLM for processing.", + score: 0.89, + }, + { + doc_id: "kb_004", + content: + "The agent loop automatically handles multi-turn tool calls. Configure maxIterations in agentLoop config to limit the number of turns. Default is 20 iterations.", + score: 0.87, + }, + { + doc_id: "kb_005", + content: + "For billing and usage tracking, use the onFinish callback which provides token usage data. Usage data is available server-side only and is stripped before sending to clients.", + score: 0.85, + }, + { + doc_id: "kb_006", + content: + "Pricing for YourGPT services: Basic plan is $29/month with 100k tokens. Pro plan is $99/month with 1M tokens. Enterprise plans with custom limits available.", + score: 0.82, + }, + { + doc_id: "kb_007", + content: + "To enable debug logging, set debug: true in createRuntime config. This will log all tool calls, LLM requests, and streaming events to the console.", + score: 0.8, + }, +]; + +// ============================================ +// SERVER-SIDE TOOL: SEARCH KNOWLEDGE BASE +// ============================================ + +const serverTools: ToolDefinition[] = [ + { + name: "search_knowledge_base", + description: + "Search the knowledge base for relevant documents. Use this when the user asks questions about YourGPT, the SDK, pricing, features, or how to use the product.", + location: "server", + // HIDDEN: This tool runs silently - user won't see it in the chat UI + hidden: true, + inputSchema: { + type: "object", + properties: { + query: { + type: "string", + description: "The search query to find relevant documents", + }, + limit: { + type: "number", + description: "Maximum number of results to return (1-10)", + }, + }, + required: ["query"], + }, + handler: async (params) => { + const args = params as { query: string; limit?: number }; + const searchLimit = Math.min(Math.max(args.limit || 3, 1), 10); + + console.log( + `\n[search_knowledge_base] Query: "${args.query}", Limit: ${searchLimit}`, + ); + + // Simulate search delay + await new Promise((resolve) => setTimeout(resolve, 100)); + + // Simple keyword matching for demo + const query = args.query.toLowerCase(); + const results = KNOWLEDGE_BASE_DATA.filter((doc) => + doc.content.toLowerCase().includes(query.split(" ")[0]), + ) + .slice(0, searchLimit) + .map((doc) => ({ + content: doc.content, + score: doc.score, + doc_id: doc.doc_id, + })); + + // If no matches, return top results + if (results.length === 0) { + const topResults = KNOWLEDGE_BASE_DATA.slice(0, searchLimit).map( + (doc) => ({ + content: doc.content, + score: doc.score, + doc_id: doc.doc_id, + }), + ); + console.log( + `[search_knowledge_base] No exact matches, returning top ${topResults.length} results`, + ); + return { results: topResults }; + } + + console.log(`[search_knowledge_base] Found ${results.length} results`); + return { results }; + }, + }, + { + name: "get_current_time", + description: "Get the current server time", + location: "server", + // VISIBLE: This tool will show in the chat UI (hidden: false is default) + inputSchema: { + type: "object", + properties: {}, + }, + handler: async () => { + const now = new Date(); + console.log(`\n[get_current_time] ${now.toISOString()}`); + return { + time: now.toISOString(), + timezone: Intl.DateTimeFormat().resolvedOptions().timeZone, + }; + }, + }, +]; + +// ============================================ +// CREATE PROVIDERS +// ============================================ + +// Anthropic Provider (preferred for server-side tools) +const anthropic = createAnthropic({ + apiKey: process.env.ANTHROPIC_API_KEY, +}); + +// OpenAI Provider (fallback) const openai = createOpenAI({ apiKey: process.env.OPENAI_API_KEY, }); +// Choose provider based on env +const provider = process.env.ANTHROPIC_API_KEY ? anthropic : openai; +const model = process.env.ANTHROPIC_API_KEY + ? "claude-haiku-4-5" + : "gpt-4o-mini"; + +console.log( + `Using provider: ${process.env.ANTHROPIC_API_KEY ? "Anthropic" : "OpenAI"}`, +); +console.log(`Using model: ${model}`); + +// ============================================ +// CREATE RUNTIME WITH SERVER-SIDE TOOLS +// ============================================ + const runtime = createRuntime({ - provider: openai, - model: "gpt-5.2", - systemPrompt: "You are a helpful AI assistant. Keep responses concise.", + provider, + model, + systemPrompt: `You are a helpful AI assistant for YourGPT. You have access to a knowledge base tool to search for information about YourGPT products, SDK, pricing, and features. + +IMPORTANT: When the user asks about YourGPT, the SDK, pricing, or any product-related question, ALWAYS use the search_knowledge_base tool first to find accurate information before responding. + +Be helpful, concise, and accurate. If the knowledge base doesn't have the answer, say so.`, + debug: true, + tools: serverTools, + agentLoop: { + enabled: true, + maxIterations: 5, + debug: true, + }, }); // ============================================ // COPILOT SDK COMPATIBLE ENDPOINTS -// Use these with CopilotProvider // ============================================ /** * Streaming (SSE) - Primary endpoint for Copilot SDK - * Returns: text/event-stream with SSE events */ app.post("/api/copilot/stream", async (req, res) => { - console.log("[/api/copilot/stream] SSE streaming"); - //log headers - console.log("Headers:", req.headers); + console.log("\n========================================"); + console.log("[/api/copilot/stream] SSE streaming request"); + console.log("Messages:", JSON.stringify(req.body.messages, null, 2)); + console.log("========================================\n"); await runtime .stream(req.body, { onFinish: ({ messages, usage }) => { - console.log("\n=== onFinish ==="); + console.log("\n=== Stream Complete ==="); console.log("Messages:", messages.length); console.log("Usage:", usage); }, @@ -46,47 +218,92 @@ app.post("/api/copilot/stream", async (req, res) => { /** * Non-streaming - For Copilot SDK with streaming={false} - * Returns: { text, messages, toolCalls } (usage stripped) */ app.post("/api/copilot/chat", async (req, res) => { - console.log("[/api/copilot/chat] Non-streaming JSON"); + console.log("\n========================================"); + console.log("[/api/copilot/chat] Non-streaming request"); + console.log("Messages:", JSON.stringify(req.body.messages, null, 2)); + console.log("========================================\n"); - // Usage is now included in result - strip before sending to client const { usage, ...clientResult } = await runtime.chat(req.body); - // Log usage server-side for billing if (usage) { - console.log("[/api/copilot/chat] Usage:", usage); + console.log("\n=== Chat Complete ==="); + console.log("Usage:", usage); } - // Send to client without usage res.json(clientResult); }); /** * Express handler - One-liner alternative - * Returns: text/event-stream with SSE events */ app.post("/api/copilot/handler", runtime.expressHandler()); // ============================================ -// RAW STREAMING ENDPOINTS -// For custom clients (NOT Copilot SDK) +// DIRECT TEST ENDPOINT (bypasses CopilotProvider format) // ============================================ /** - * Raw text stream - Plain text chunks - * Returns: text/plain stream + * Direct test endpoint that mimics the CopilotController pattern + * Similar to what's used in the real YourGPT Copilot backend */ +app.post("/api/test/copilot-response", async (req, res) => { + console.log("\n========================================"); + console.log("[/api/test/copilot-response] Direct API test"); + console.log("Body:", JSON.stringify(req.body, null, 2)); + console.log("========================================\n"); + + try { + const { messages, system, temperature, max_tokens } = req.body; + + if (!messages || !Array.isArray(messages) || messages.length === 0) { + return res.status(400).json({ + type: "RXERROR", + message: "Invalid params: messages array is required", + }); + } + + // Use runtime.chat() for non-streaming response (like the original code) + const result = await runtime.chat({ + messages, + systemPrompt: system, + config: { + temperature, + maxTokens: max_tokens || 4096, + }, + }); + + console.log("\n=== Response Complete ==="); + console.log("Text length:", result.text?.length || 0); + console.log("Tool calls:", result.toolCalls?.length || 0); + console.log("Messages:", result.messages?.length || 0); + + // Return in the format expected by the original controller + return res.status(200).json({ + text: result.text, + messages: result.messages, + toolCalls: result.toolCalls, + }); + } catch (error) { + console.error("[/api/test/copilot-response] Error:", error); + return res.status(500).json({ + type: "RXERROR", + message: "Failed to process copilot request", + error: error instanceof Error ? error.message : "Unknown error", + }); + } +}); + +// ============================================ +// RAW STREAMING ENDPOINTS +// ============================================ + app.post("/api/raw/stream/text", async (req, res) => { console.log("[/api/raw/stream/text] Plain text streaming"); await runtime.stream(req.body).pipeTextToResponse(res); }); -/** - * Raw stream with events - Stream with logging - * Returns: text/event-stream (but logs events server-side) - */ app.post("/api/raw/stream/events", async (req, res) => { console.log("[/api/raw/stream/events] Streaming with event handlers"); @@ -110,23 +327,14 @@ app.post("/api/raw/stream/events", async (req, res) => { // ============================================ // RAW NON-STREAMING ENDPOINTS -// For custom clients (NOT Copilot SDK) // ============================================ -/** - * Generate text only - * Returns: { text: string } - */ app.post("/api/raw/generate/text", async (req, res) => { console.log("[/api/raw/generate/text] Text only response"); const text = await runtime.stream(req.body).text(); res.json({ text }); }); -/** - * Generate full response (raw - includes usage) - * Returns: { text, messages, toolCalls, usage } - */ app.post("/api/raw/generate/full", async (req, res) => { console.log("[/api/raw/generate/full] Full response data"); const { text, messages, toolCalls, usage } = await runtime @@ -135,22 +343,6 @@ app.post("/api/raw/generate/full", async (req, res) => { res.json({ text, messages, toolCalls, usage }); }); -/** - * Generate with metadata - * Returns: { text, messageCount, toolCallCount } - */ -app.post("/api/raw/generate/summary", async (req, res) => { - console.log("[/api/raw/generate/summary] Summary response"); - const { text, messages, toolCalls } = await runtime - .stream(req.body) - .collect(); - res.json({ - text, - messageCount: messages.length, - toolCallCount: toolCalls.length, - }); -}); - // ============================================ // HEALTH CHECK // ============================================ @@ -158,20 +350,25 @@ app.post("/api/raw/generate/summary", async (req, res) => { app.get("/api/health", (_req, res) => { res.json({ status: "ok", - copilotEndpoints: [ - "POST /api/copilot/stream - SSE streaming (CopilotProvider default)", - "POST /api/copilot/chat - Non-streaming JSON (streaming={false})", - "POST /api/copilot/handler - Express handler one-liner", - ], - rawStreamEndpoints: [ - "POST /api/raw/stream/text - Plain text stream", - "POST /api/raw/stream/events - Stream with server-side event logging", - ], - rawGenerateEndpoints: [ - "POST /api/raw/generate/text - Returns { text }", - "POST /api/raw/generate/full - Returns { text, messages, toolCalls, usage }", - "POST /api/raw/generate/summary - Returns { text, messageCount, toolCallCount }", - ], + provider: process.env.ANTHROPIC_API_KEY ? "anthropic" : "openai", + model, + serverTools: serverTools.map((t) => t.name), + endpoints: { + copilot: [ + "POST /api/copilot/stream - SSE streaming", + "POST /api/copilot/chat - Non-streaming JSON", + "POST /api/copilot/handler - Express handler", + ], + test: [ + "POST /api/test/copilot-response - Direct API test (mimics CopilotController)", + ], + raw: [ + "POST /api/raw/stream/text - Plain text stream", + "POST /api/raw/stream/events - Stream with event logging", + "POST /api/raw/generate/text - Returns { text }", + "POST /api/raw/generate/full - Returns { text, messages, toolCalls, usage }", + ], + }, }); }); @@ -182,37 +379,38 @@ app.get("/api/health", (_req, res) => { const port = process.env.PORT || 3001; app.listen(port, () => { console.log(` -Express Demo Server running on http://localhost:${port} +╔══════════════════════════════════════════════════════════════╗ +║ Express Demo - Server-Side Tools ║ +╠══════════════════════════════════════════════════════════════╣ +║ Server: http://localhost:${port} ║ +║ Provider: ${(process.env.ANTHROPIC_API_KEY ? "Anthropic" : "OpenAI").padEnd(47)}║ +║ Model: ${model.padEnd(47)}║ +╚══════════════════════════════════════════════════════════════╝ -=== COPILOT SDK ENDPOINTS === - POST /api/copilot/stream - SSE streaming (default) - POST /api/copilot/chat - Non-streaming JSON - POST /api/copilot/handler - Express handler - -=== RAW STREAMING === - POST /api/raw/stream/text - Plain text stream - POST /api/raw/stream/events - Stream with event logging (includes usage) - -=== RAW NON-STREAMING === - POST /api/raw/generate/text - Returns { text } - POST /api/raw/generate/full - Returns { text, messages, toolCalls, usage } - POST /api/raw/generate/summary - Returns { text, messageCount, toolCallCount } +Server-side Tools: + - search_knowledge_base: Search dummy KB data + - get_current_time: Get server time === TEST CURLS === -# Copilot SDK - Streaming +# Test knowledge base search (server-side tool) +curl -X POST http://localhost:${port}/api/copilot/chat \\ + -H "Content-Type: application/json" \\ + -d '{"messages":[{"role":"user","content":"What is YourGPT SDK?"}]}' + +# Test streaming with tools curl -X POST http://localhost:${port}/api/copilot/stream \\ -H "Content-Type: application/json" \\ - -d '{"messages":[{"role":"user","content":"Say hello"}]}' + -d '{"messages":[{"role":"user","content":"Tell me about SDK pricing"}]}' -# Copilot SDK - Non-streaming -curl -X POST http://localhost:${port}/api/copilot/chat \\ +# Test direct API (mimics CopilotController) +curl -X POST http://localhost:${port}/api/test/copilot-response \\ -H "Content-Type: application/json" \\ - -d '{"messages":[{"role":"user","content":"Say hello"}]}' + -d '{"messages":[{"role":"user","content":"How do I install the SDK?"}]}' -# Raw - Full response with usage -curl -X POST http://localhost:${port}/api/raw/generate/full \\ +# Test time tool +curl -X POST http://localhost:${port}/api/copilot/chat \\ -H "Content-Type: application/json" \\ - -d '{"messages":[{"role":"user","content":"Say hello"}]}' + -d '{"messages":[{"role":"user","content":"What time is it?"}]}' `); }); diff --git a/packages/copilot-sdk/package.json b/packages/copilot-sdk/package.json index ace9889..cd0669a 100644 --- a/packages/copilot-sdk/package.json +++ b/packages/copilot-sdk/package.json @@ -1,6 +1,6 @@ { "name": "@yourgpt/copilot-sdk", - "version": "2.1.1", + "version": "2.1.3", "description": "Copilot SDK for building Production-ready AI Copilots for any product. Connect any LLM, deploy on your infrastructure, own your data.", "type": "module", "types": "./dist/core/index.d.ts", diff --git a/packages/copilot-sdk/src/chat/AbstractAgentLoop.ts b/packages/copilot-sdk/src/chat/AbstractAgentLoop.ts index ef9de10..470cdaf 100644 --- a/packages/copilot-sdk/src/chat/AbstractAgentLoop.ts +++ b/packages/copilot-sdk/src/chat/AbstractAgentLoop.ts @@ -170,6 +170,15 @@ export class AbstractAgentLoop implements AgentLoopActions { } private addToolExecution(execution: ToolExecution): void { + // Check for duplicate by ID - skip if already exists + const existingIndex = this._toolExecutions.findIndex( + (e) => e.id === execution.id, + ); + if (existingIndex !== -1) { + // Skip duplicate - don't add or merge + return; + } + this._toolExecutions = [...this._toolExecutions, execution]; // Prune old executions if over limit (prevents memory leak) @@ -182,10 +191,10 @@ export class AbstractAgentLoop implements AgentLoopActions { this.callbacks.onExecutionsChange?.(this._toolExecutions); } - private updateToolExecution( - id: string, - update: Partial, - ): void { + /** + * Update a tool execution with partial data + */ + updateToolExecution(id: string, update: Partial): void { this._toolExecutions = this._toolExecutions.map((exec) => exec.id === id ? { ...exec, ...update } : exec, ); @@ -307,6 +316,7 @@ export class AbstractAgentLoop implements AgentLoopActions { status: "pending", approvalStatus: "none", startedAt: new Date(), + hidden: tool?.hidden, }; this.addToolExecution(execution); @@ -493,6 +503,55 @@ export class AbstractAgentLoop implements AgentLoopActions { this._maxIterationsReached = false; } + // ============================================ + // Server-Side Tool Tracking + // ============================================ + + /** + * Add a server-side tool execution (from streaming action:start event) + * Used to track tools executed on the server (not client-side) + */ + addServerToolExecution(info: { + id: string; + name: string; + hidden?: boolean; + }): void { + const execution: ToolExecution = { + id: info.id, + toolCallId: info.id, + name: info.name, + args: {}, + status: "executing", + approvalStatus: "none", + startedAt: new Date(), + hidden: info.hidden, + }; + this.addToolExecution(execution); + } + + /** + * Update a server-side tool execution with args (from action:args event) + */ + updateServerToolArgs(id: string, args: Record): void { + this.updateToolExecution(id, { args }); + } + + /** + * Complete a server-side tool execution (from action:end event) + */ + completeServerToolExecution(info: { + id: string; + result?: unknown; + error?: string; + }): void { + this.updateToolExecution(info.id, { + status: info.error ? "failed" : "completed", + result: info.result, + error: info.error, + completedAt: new Date(), + }); + } + /** * Cancel all pending and executing tools * This will: diff --git a/packages/copilot-sdk/src/chat/ChatWithTools.ts b/packages/copilot-sdk/src/chat/ChatWithTools.ts index 766c107..555f258 100644 --- a/packages/copilot-sdk/src/chat/ChatWithTools.ts +++ b/packages/copilot-sdk/src/chat/ChatWithTools.ts @@ -146,6 +146,66 @@ export class ChatWithTools { onMessageFinish: callbacks.onMessageFinish, onToolCalls: callbacks.onToolCalls, onFinish: callbacks.onFinish, + // Server-side tool callbacks - track in agentLoop for UI display + // IMPORTANT: Only track tools that are NOT registered client-side + // Client-side tools are tracked via executeToolCalls() path + onServerToolStart: (info) => { + // Check if execution with this ID already exists + const existingExecution = this.agentLoop.toolExecutions.find( + (e) => e.id === info.id, + ); + if (existingExecution) { + // Update hidden flag if this event has it (agent-loop sends hidden, adapter doesn't) + if ( + info.hidden !== undefined && + existingExecution.hidden !== info.hidden + ) { + this.debug( + "Updating hidden flag for existing execution:", + info.name, + info.hidden, + ); + this.agentLoop.updateToolExecution(info.id, { + hidden: info.hidden, + }); + } + return; + } + // Skip if this tool is registered client-side (will be tracked via executeToolCalls) + const isClientTool = this.agentLoop.tools.some( + (t) => t.name === info.name && t.location === "client", + ); + if (isClientTool) { + this.debug("Skipping server tracking for client tool:", info.name); + return; + } + this.debug("Server tool started:", info.name, { + hidden: info.hidden, + id: info.id, + }); + this.agentLoop.addServerToolExecution(info); + }, + onServerToolArgs: (info) => { + // Skip if this tool is registered client-side + const isClientTool = this.agentLoop.tools.some( + (t) => t.name === info.name && t.location === "client", + ); + if (isClientTool) return; + this.debug("Server tool args:", info.name, info.args); + this.agentLoop.updateServerToolArgs(info.id, info.args ?? {}); + }, + onServerToolEnd: (info) => { + // Skip if this tool is registered client-side + const isClientTool = this.agentLoop.tools.some( + (t) => t.name === info.name && t.location === "client", + ); + if (isClientTool) return; + this.debug("Server tool ended:", info.name, { + error: info.error, + hasResult: !!info.result, + }); + this.agentLoop.completeServerToolExecution(info); + }, }, }); diff --git a/packages/copilot-sdk/src/chat/adapters/HttpTransport.ts b/packages/copilot-sdk/src/chat/adapters/HttpTransport.ts index 07cb1de..63a500a 100644 --- a/packages/copilot-sdk/src/chat/adapters/HttpTransport.ts +++ b/packages/copilot-sdk/src/chat/adapters/HttpTransport.ts @@ -72,20 +72,12 @@ export class HttpTransport implements ChatTransport { // Resolve dynamic values at request time (not constructor time) // This ensures fresh values on every request // Optimized: skips async overhead if all values are static - console.log( - "[HttpTransport] Config headers type:", - typeof this.config.headers, - ); - console.log("[HttpTransport] Config headers:", this.config.headers); - const resolved = await resolveValues({ url: this.config.url, headers: this.config.headers ?? {}, configBody: this.config.body ?? {}, }); - console.log("[HttpTransport] Resolved headers:", resolved.headers); - const response = await fetch(resolved.url as string, { method: "POST", headers: { diff --git a/packages/copilot-sdk/src/chat/classes/AbstractChat.ts b/packages/copilot-sdk/src/chat/classes/AbstractChat.ts index caa4dca..2a47223 100644 --- a/packages/copilot-sdk/src/chat/classes/AbstractChat.ts +++ b/packages/copilot-sdk/src/chat/classes/AbstractChat.ts @@ -754,9 +754,94 @@ export class AbstractChat { return; } + // Handle message:end mid-stream (server-side agent loop turn completed) + // This creates separate messages for each turn instead of combining them + if (chunk.type === "message:end" && this.streamState?.content) { + this.debug("message:end mid-stream - finalizing current turn"); + + // Finalize current message with its content and tool calls + const turnMessage = streamStateToMessage(this.streamState) as T; + + // Add toolCallsHidden metadata if applicable + const toolCallsHidden: Record = {}; + for (const [id, result] of this.streamState.toolResults) { + if (result.hidden !== undefined) { + toolCallsHidden[id] = result.hidden; + } + } + if ( + turnMessage.toolCalls?.length && + Object.keys(toolCallsHidden).length > 0 + ) { + (turnMessage as T & { metadata?: Record }).metadata = + { + ...(turnMessage as T & { metadata?: Record }) + .metadata, + toolCallsHidden, + }; + } + + this.state.updateMessageById( + this.streamState.messageId, + () => turnMessage, + ); + this.callbacks.onMessageFinish?.(turnMessage); + + // Reset stream state for next turn - will be initialized on next message:start + this.streamState = null; + continue; + } + + // Handle message:start after a mid-stream finalization + if (chunk.type === "message:start" && this.streamState === null) { + this.debug("message:start after mid-stream end - creating new message"); + const newMessage = createEmptyAssistantMessage() as T; + this.state.pushMessage(newMessage); + this.streamState = createStreamState(newMessage.id); + this.callbacks.onMessageStart?.(newMessage.id); + continue; + } + // Update stream state (pure function) + // Skip if streamState is null (shouldn't happen but be safe) + if (!this.streamState) { + this.debug("warning", "streamState is null, skipping chunk"); + continue; + } this.streamState = processStreamChunk(chunk, this.streamState); + // Emit server tool callbacks for action events + if (chunk.type === "action:start") { + this.callbacks.onServerToolStart?.({ + id: chunk.id, + name: chunk.name, + hidden: chunk.hidden, + }); + } else if (chunk.type === "action:args") { + let args: Record = {}; + try { + args = JSON.parse(chunk.args); + } catch { + // Keep empty args + } + // Get name from toolResults (set by action:start) + const existingResult = this.streamState?.toolResults.get(chunk.id); + if (existingResult) { + this.callbacks.onServerToolArgs?.({ + id: chunk.id, + name: existingResult.name, + args, + }); + } + } else if (chunk.type === "action:end") { + this.callbacks.onServerToolEnd?.({ + id: chunk.id, + name: chunk.name, + result: chunk.result, + error: chunk.error, + }); + } + // Update message in state BY ID (not last position) // This is critical: when tool calls trigger nested streams, // updateLastMessage would update the wrong message @@ -781,28 +866,105 @@ export class AbstractChat { // Check for completion if (isStreamDone(chunk)) { this.debug("streamDone", { chunk }); + + // CRITICAL: Process messages from done event (server-side tool results) + // Without this, tool_call_id is lost and causes Anthropic API errors + if (chunk.type === "done" && chunk.messages?.length) { + this.debug("processDoneMessages", { + count: chunk.messages.length, + }); + + // Build hidden map from stream state's toolResults + const toolCallsHidden: Record = {}; + if (this.streamState?.toolResults) { + for (const [id, result] of this.streamState.toolResults) { + if (result.hidden !== undefined) { + toolCallsHidden[id] = result.hidden; + } + } + } + + for (const msg of chunk.messages) { + // Skip ALL assistant messages - they're handled via streaming + // (message:end/message:start events create separate messages for each turn) + if (msg.role === "assistant") { + continue; + } + + // For assistant messages with tool_calls, add hidden metadata + let metadata: Record | undefined; + if ( + msg.role === "assistant" && + msg.tool_calls?.length && + Object.keys(toolCallsHidden).length > 0 + ) { + metadata = { toolCallsHidden }; + } + + const message = { + id: generateMessageId(), + role: msg.role as T["role"], + content: msg.content ?? "", + toolCalls: msg.tool_calls as T["toolCalls"], + toolCallId: msg.tool_call_id, + createdAt: new Date(), + metadata, + } as T; + + this.state.pushMessage(message); + } + } + break; } } this.debug("handleStreamResponse", `Processed ${chunkCount} chunks`); - // Finalize - update by ID to ensure we update the correct message - const finalMessage = streamStateToMessage(this.streamState) as T; - this.state.updateMessageById( - this.streamState.messageId, - () => finalMessage, - ); + // If streamState was already finalized (via message:end mid-stream), skip finalization + if (!this.streamState) { + this.debug("streamState already finalized via message:end"); + } else { + // Build hidden map from stream state's toolResults for final message metadata + const toolCallsHidden: Record = {}; + if (this.streamState.toolResults) { + for (const [id, result] of this.streamState.toolResults) { + if (result.hidden !== undefined) { + toolCallsHidden[id] = result.hidden; + } + } + } + + // Finalize - update by ID to ensure we update the correct message + const finalMessage = streamStateToMessage(this.streamState) as T; + + // Add toolCallsHidden metadata if we have tool calls with hidden flags + if ( + finalMessage.toolCalls?.length && + Object.keys(toolCallsHidden).length > 0 + ) { + (finalMessage as T & { metadata?: Record }).metadata = + { + ...(finalMessage as T & { metadata?: Record }) + .metadata, + toolCallsHidden, + }; + } + + this.state.updateMessageById( + this.streamState.messageId, + () => finalMessage, + ); - // Check if we got any content - if ( - !finalMessage.content && - (!finalMessage.toolCalls || finalMessage.toolCalls.length === 0) - ) { - this.debug("warning", "Empty response - no content and no tool calls"); + // Check if we got any content + if ( + !finalMessage.content && + (!finalMessage.toolCalls || finalMessage.toolCalls.length === 0) + ) { + this.debug("warning", "Empty response - no content and no tool calls"); + } } - this.callbacks.onMessageFinish?.(finalMessage); this.callbacks.onMessagesChange?.(this.state.messages); // Only set status to "ready" if NO tool calls were emitted @@ -822,14 +984,46 @@ export class AbstractChat { * Handle JSON (non-streaming) response */ protected handleJsonResponse(response: ChatResponse): void { + // Build a map of tool call hidden flags from response.toolCalls + const toolCallHiddenMap = new Map(); + if (response.toolCalls) { + for (const tc of response.toolCalls) { + if (tc.hidden !== undefined) { + toolCallHiddenMap.set(tc.id, tc.hidden); + } + } + } + // Add response messages for (const msg of response.messages ?? []) { + // For assistant messages with tool_calls, add hidden info to metadata + let metadata: Record | undefined; + if ( + msg.role === "assistant" && + msg.tool_calls && + toolCallHiddenMap.size > 0 + ) { + const toolCallsHidden: Record = {}; + for (const tc of msg.tool_calls as Array<{ id: string }>) { + const hidden = toolCallHiddenMap.get(tc.id); + if (hidden !== undefined) { + toolCallsHidden[tc.id] = hidden; + } + } + if (Object.keys(toolCallsHidden).length > 0) { + metadata = { toolCallsHidden }; + } + } + const message = { id: generateMessageId(), role: msg.role as T["role"], content: msg.content ?? "", toolCalls: msg.tool_calls as T["toolCalls"], + // CRITICAL: Preserve toolCallId for tool messages (fixes Anthropic API errors) + toolCallId: msg.tool_call_id, createdAt: new Date(), + metadata, } as T; this.state.pushMessage(message); diff --git a/packages/copilot-sdk/src/chat/functions/stream/processChunk.ts b/packages/copilot-sdk/src/chat/functions/stream/processChunk.ts index f2adef1..7daebd7 100644 --- a/packages/copilot-sdk/src/chat/functions/stream/processChunk.ts +++ b/packages/copilot-sdk/src/chat/functions/stream/processChunk.ts @@ -78,6 +78,7 @@ export function processStreamChunk( id: chunk.id, name: chunk.name, status: "executing", + hidden: chunk.hidden, }); return { ...state, toolResults: newResults }; } diff --git a/packages/copilot-sdk/src/chat/interfaces/ChatTransport.ts b/packages/copilot-sdk/src/chat/interfaces/ChatTransport.ts index ab7abe5..59e8b31 100644 --- a/packages/copilot-sdk/src/chat/interfaces/ChatTransport.ts +++ b/packages/copilot-sdk/src/chat/interfaces/ChatTransport.ts @@ -43,9 +43,18 @@ export interface ChatResponse { role: string; content: string | null; tool_calls?: unknown[]; + /** Tool call ID for tool result messages */ + tool_call_id?: string; }>; /** Whether client needs to execute tools */ requiresAction?: boolean; + /** Tool calls with metadata (includes hidden flag for server-side tools) */ + toolCalls?: Array<{ + id: string; + name: string; + args: Record; + hidden?: boolean; + }>; } /** @@ -87,9 +96,18 @@ export type StreamChunk = | { type: "tool_calls"; toolCalls: unknown[]; assistantMessage: unknown } | { type: "source:add"; source: unknown } | { type: "error"; message: string } - | { type: "done"; messages?: unknown[]; requiresAction?: boolean } + | { + type: "done"; + messages?: Array<{ + role: string; + content: string | null; + tool_calls?: unknown[]; + tool_call_id?: string; + }>; + requiresAction?: boolean; + } // Tool action events (from llm-sdk agent-loop) - | { type: "action:start"; id: string; name: string } + | { type: "action:start"; id: string; name: string; hidden?: boolean } | { type: "action:args"; id: string; args: string } | { type: "action:end"; diff --git a/packages/copilot-sdk/src/chat/types/chat.ts b/packages/copilot-sdk/src/chat/types/chat.ts index f59c1d9..f2f14a4 100644 --- a/packages/copilot-sdk/src/chat/types/chat.ts +++ b/packages/copilot-sdk/src/chat/types/chat.ts @@ -66,6 +66,16 @@ export interface ChatRequestOptions { metadata?: Record; } +/** + * Server-side tool execution info (from streaming action events) + */ +export interface ServerToolInfo { + id: string; + name: string; + args?: Record; + hidden?: boolean; +} + /** * Chat callbacks for state updates */ @@ -86,6 +96,14 @@ export interface ChatCallbacks { onToolCalls?: (toolCalls: T["toolCalls"]) => void; /** Called when generation is complete */ onFinish?: (messages: T[]) => void; + /** Called when a server-side tool starts executing (action:start event) */ + onServerToolStart?: (info: ServerToolInfo) => void; + /** Called when a server-side tool receives args (action:args event) */ + onServerToolArgs?: (info: ServerToolInfo) => void; + /** Called when a server-side tool finishes (action:end event) */ + onServerToolEnd?: ( + info: ServerToolInfo & { result?: unknown; error?: string }, + ) => void; } /** diff --git a/packages/copilot-sdk/src/chat/types/index.ts b/packages/copilot-sdk/src/chat/types/index.ts index c5d6e26..2b78396 100644 --- a/packages/copilot-sdk/src/chat/types/index.ts +++ b/packages/copilot-sdk/src/chat/types/index.ts @@ -19,6 +19,7 @@ export type { ChatCallbacks, ChatInit, SendMessageOptions, + ServerToolInfo, } from "./chat"; // Tool types diff --git a/packages/copilot-sdk/src/chat/types/message.ts b/packages/copilot-sdk/src/chat/types/message.ts index 47ea564..514b141 100644 --- a/packages/copilot-sdk/src/chat/types/message.ts +++ b/packages/copilot-sdk/src/chat/types/message.ts @@ -52,6 +52,8 @@ export interface StreamToolResult { args?: Record; result?: unknown; error?: string; + /** Whether this tool should be hidden from UI */ + hidden?: boolean; } /** diff --git a/packages/copilot-sdk/src/chat/types/tool.ts b/packages/copilot-sdk/src/chat/types/tool.ts index 6b5495a..3f2d142 100644 --- a/packages/copilot-sdk/src/chat/types/tool.ts +++ b/packages/copilot-sdk/src/chat/types/tool.ts @@ -65,6 +65,11 @@ export interface ToolExecution { approvalMessage?: string; /** Data passed from user's approval action (e.g., selected supervisor) */ approvalData?: Record; + /** + * Whether this tool execution should be hidden from the UI. + * When true, the tool won't appear in the chat, but still executes normally. + */ + hidden?: boolean; } /** diff --git a/packages/copilot-sdk/src/core/index.ts b/packages/copilot-sdk/src/core/index.ts index 3821b05..da66e12 100644 --- a/packages/copilot-sdk/src/core/index.ts +++ b/packages/copilot-sdk/src/core/index.ts @@ -168,6 +168,7 @@ export type { ToolDefinition, ToolConfig, ToolSet, + ToolSetEntry, UnifiedToolCall, UnifiedToolResult, ToolExecutionStatus, diff --git a/packages/copilot-sdk/src/core/types/tools.ts b/packages/copilot-sdk/src/core/types/tools.ts index 776b32d..ae785d4 100644 --- a/packages/copilot-sdk/src/core/types/tools.ts +++ b/packages/copilot-sdk/src/core/types/tools.ts @@ -651,6 +651,12 @@ export interface ToolExecution { approvalMessage?: string; /** Timestamp when user responded to approval request */ approvalTimestamp?: number; + + /** + * Whether this tool execution should be hidden from the UI. + * Server-side tools can set this to hide internal operations from users. + */ + hidden?: boolean; } // ============================================ @@ -689,18 +695,36 @@ export interface AgentLoopState { // ToolSet Type (Vercel AI SDK pattern) // ============================================ +/** + * A tool definition without the name (name is derived from the key in ToolSet) + */ +export type ToolSetEntry> = Omit< + ToolDefinition, + "name" +>; + /** * A set of tools, keyed by tool name * + * The key becomes the tool name, so tool definitions don't need a name property. + * Use with the `tool()` helper for clean syntax. + * * @example * ```typescript * const myTools: ToolSet = { - * capture_screenshot: screenshotTool, - * get_weather: weatherTool, + * capture_screenshot: tool({ + * description: 'Capture screenshot', + * handler: async () => ({ success: true }), + * }), + * get_weather: tool({ + * description: 'Get weather', + * inputSchema: { type: 'object', properties: { city: { type: 'string' } } }, + * handler: async ({ city }) => ({ success: true, data: { temp: 72 } }), + * }), * }; * ``` */ -export type ToolSet = Record; +export type ToolSet = Record; // ============================================ // Tool Helper Function (Vercel AI SDK pattern) diff --git a/packages/copilot-sdk/src/react/hooks/useToolExecutor.ts b/packages/copilot-sdk/src/react/hooks/useToolExecutor.ts index a009f48..d7bbaa4 100644 --- a/packages/copilot-sdk/src/react/hooks/useToolExecutor.ts +++ b/packages/copilot-sdk/src/react/hooks/useToolExecutor.ts @@ -101,6 +101,7 @@ export function useToolExecutor(): UseToolExecutorReturn { status: "executing", timestamp: Date.now(), approvalStatus: "none", + hidden: tool.hidden, }; // Add to execution list diff --git a/packages/copilot-sdk/src/react/index.ts b/packages/copilot-sdk/src/react/index.ts index 16b3344..ed433b4 100644 --- a/packages/copilot-sdk/src/react/index.ts +++ b/packages/copilot-sdk/src/react/index.ts @@ -189,9 +189,16 @@ export type { ToolExecutionStatus, UnifiedToolCall, AgentLoopConfig, + // ToolSet types (for useTools) + ToolSet, + ToolSetEntry, + ToolConfig, // Permission types PermissionLevel, ToolPermission, PermissionStorageConfig, PermissionStorageAdapter, } from "../core"; + +// Re-export tool helper function (Vercel AI SDK pattern) +export { tool } from "../core"; diff --git a/packages/copilot-sdk/src/ui/components/composed/chat/default-message.tsx b/packages/copilot-sdk/src/ui/components/composed/chat/default-message.tsx index 09edafa..e2d31c2 100644 --- a/packages/copilot-sdk/src/ui/components/composed/chat/default-message.tsx +++ b/packages/copilot-sdk/src/ui/components/composed/chat/default-message.tsx @@ -275,18 +275,22 @@ export function DefaultMessage({ ); } - // Helper: check if a tool is hidden (shouldn't appear in UI) - const isToolHidden = (toolName: string): boolean => { - const toolDef = registeredTools?.find((t) => t.name === toolName); + // Helper: check if a tool execution is hidden (shouldn't appear in UI) + // Checks both: 1) execution's hidden flag (from server), 2) registered tool's hidden flag + const isToolHidden = (exec: { name: string; hidden?: boolean }): boolean => { + // Check execution's own hidden flag first (from server's action:start event) + if (exec.hidden === true) return true; + // Then check registered tool definition + const toolDef = registeredTools?.find((t) => t.name === exec.name); return toolDef?.hidden === true; }; // Separate tool executions into categories (excluding hidden tools) const pendingApprovalTools = message.toolExecutions?.filter( - (exec) => exec.approvalStatus === "required" && !isToolHidden(exec.name), + (exec) => exec.approvalStatus === "required" && !isToolHidden(exec), ); const completedTools = message.toolExecutions?.filter( - (exec) => exec.approvalStatus !== "required" && !isToolHidden(exec.name), + (exec) => exec.approvalStatus !== "required" && !isToolHidden(exec), ); // Helper: check if tool has any custom render (toolRenderers, mcpToolRenderer, or tool.render) @@ -487,7 +491,7 @@ export function DefaultMessage({ {/* MCP-UI Resources - Interactive components from MCP tools (excluding hidden) */} {message.toolExecutions - ?.filter((exec) => !isToolHidden(exec.name)) + ?.filter((exec) => !isToolHidden(exec)) .map((exec) => { const uiResources = exec.result?._uiResources; if (!uiResources || uiResources.length === 0) return null; diff --git a/packages/copilot-sdk/src/ui/components/composed/connected-chat.tsx b/packages/copilot-sdk/src/ui/components/composed/connected-chat.tsx index 68698a7..cb3e0a3 100644 --- a/packages/copilot-sdk/src/ui/components/composed/connected-chat.tsx +++ b/packages/copilot-sdk/src/ui/components/composed/connected-chat.tsx @@ -337,6 +337,7 @@ function CopilotChatBase( error: exec.error, timestamp: exec.startedAt ? exec.startedAt.getTime() : Date.now(), approvalStatus: exec.approvalStatus, + hidden: exec.hidden, }), ); @@ -385,6 +386,11 @@ function CopilotChatBase( ); } else { // Build from stored tool_calls + tool messages (historical) + // Get hidden info from message metadata (set by handleJsonResponse) + const toolCallsHidden = ( + m.metadata as { toolCallsHidden?: Record } + )?.toolCallsHidden; + messageToolExecutions = m.toolCalls.map( (tc: { id: string; @@ -405,6 +411,15 @@ function CopilotChatBase( } catch { // Keep empty args } + // Check hidden from metadata first (from server response), + // then fall back to registeredTools + let hidden = toolCallsHidden?.[tc.id]; + if (hidden === undefined) { + const toolDef = registeredTools?.find( + (t) => t.name === tc.function.name, + ); + hidden = toolDef?.hidden; + } return { id: tc.id, name: tc.function.name, @@ -414,6 +429,7 @@ function CopilotChatBase( : "pending") as ToolExecutionData["status"], result, timestamp: Date.now(), // Historical - use current time + hidden, }; }, ); @@ -432,6 +448,11 @@ function CopilotChatBase( messageToolExecutions = savedExecutions; } + // Filter out hidden tool executions for the message + const visibleToolExecutions = messageToolExecutions?.filter( + (exec) => !exec.hidden, + ); + return { id: m.id, role: m.role as "user" | "assistant" | "system", @@ -441,11 +462,27 @@ function CopilotChatBase( attachments: m.attachments, // Include tool_calls for assistant messages tool_calls: m.toolCalls, - // Attach matched tool executions to assistant messages - toolExecutions: messageToolExecutions, + // Attach matched tool executions to assistant messages (only visible ones) + toolExecutions: visibleToolExecutions, // Include metadata (citations from native web search, etc.) metadata: m.metadata, + // Mark if this message had only hidden tools (for filtering empty bubbles) + _hasOnlyHiddenTools: + messageToolExecutions && + messageToolExecutions.length > 0 && + (!visibleToolExecutions || visibleToolExecutions.length === 0), }; + }) + // Filter out empty assistant messages that only had hidden tools + .filter((m) => { + if ( + m.role === "assistant" && + !m.content && + (m as { _hasOnlyHiddenTools?: boolean })._hasOnlyHiddenTools + ) { + return false; + } + return true; }); // Show suggestions only when no messages diff --git a/packages/copilot-sdk/src/ui/components/composed/tools/tool-execution-list.tsx b/packages/copilot-sdk/src/ui/components/composed/tools/tool-execution-list.tsx index 37ee341..e24f91f 100644 --- a/packages/copilot-sdk/src/ui/components/composed/tools/tool-execution-list.tsx +++ b/packages/copilot-sdk/src/ui/components/composed/tools/tool-execution-list.tsx @@ -79,6 +79,11 @@ export interface ToolExecutionData { approvalMessage?: string; /** Data passed from user's approval action (e.g., selected item) */ approvalData?: Record; + /** + * Whether this tool execution should be hidden from the UI. + * Server-side tools can set this to hide internal operations from users. + */ + hidden?: boolean; } /** diff --git a/packages/llm-sdk/package.json b/packages/llm-sdk/package.json index b8948a9..f9fefa9 100644 --- a/packages/llm-sdk/package.json +++ b/packages/llm-sdk/package.json @@ -1,6 +1,6 @@ { "name": "@yourgpt/llm-sdk", - "version": "2.1.0", + "version": "2.1.3", "description": "AI SDK for building AI Agents with any LLM", "main": "./dist/index.js", "module": "./dist/index.mjs", diff --git a/packages/llm-sdk/src/adapters/anthropic.ts b/packages/llm-sdk/src/adapters/anthropic.ts index 14a021e..c646fce 100644 --- a/packages/llm-sdk/src/adapters/anthropic.ts +++ b/packages/llm-sdk/src/adapters/anthropic.ts @@ -90,6 +90,10 @@ export class AnthropicAdapter implements LLMAdapter { const pendingToolResults: Array<{ tool_use_id: string; content: string }> = []; + // Track tool_use ids from assistant messages for inference + let lastToolCallIds: string[] = []; + let toolResultIndex = 0; + for (const msg of rawMessages) { // Skip system messages (handled separately) if (msg.role === "system") continue; @@ -110,6 +114,10 @@ export class AnthropicAdapter implements LLMAdapter { })), }); pendingToolResults.length = 0; + // Clear tracking - tool results have been flushed, any subsequent + // tool results without a new tool_use are orphaned + lastToolCallIds = []; + toolResultIndex = 0; } // Convert assistant message with potential tool_calls @@ -134,6 +142,10 @@ export class AnthropicAdapter implements LLMAdapter { | undefined; if (toolCalls && toolCalls.length > 0) { + // Track tool call IDs for inferring missing tool_call_id in tool messages + lastToolCallIds = toolCalls.map((tc) => tc.id); + toolResultIndex = 0; + for (const tc of toolCalls) { let input = {}; try { @@ -156,8 +168,44 @@ export class AnthropicAdapter implements LLMAdapter { } } else if (msg.role === "tool") { // Collect tool results to be bundled into a user message + let toolCallId = msg.tool_call_id as string | undefined; + + // If tool_call_id is missing, try to infer from preceding assistant's tool_calls + if (!toolCallId && lastToolCallIds.length > 0) { + toolCallId = lastToolCallIds[toolResultIndex]; + toolResultIndex++; + console.warn( + `[llm-sdk] Tool message missing tool_call_id, inferred: ${toolCallId}`, + ); + } + + if (!toolCallId) { + console.warn( + "[llm-sdk] Skipping tool message with missing tool_call_id (no inference possible):", + msg, + ); + continue; + } + + // Skip orphaned tool results (no pending tool_use to match) + // This happens when there's a duplicate/stale tool result in the conversation + if (lastToolCallIds.length === 0) { + console.warn( + `[llm-sdk] Skipping orphaned tool result (no pending tool_use): ${toolCallId}`, + ); + continue; + } + + // Skip if this tool_call_id is not in the expected list + if (!lastToolCallIds.includes(toolCallId)) { + console.warn( + `[llm-sdk] Skipping tool result with unexpected tool_call_id: ${toolCallId}`, + ); + continue; + } + pendingToolResults.push({ - tool_use_id: msg.tool_call_id as string, + tool_use_id: toolCallId, content: typeof msg.content === "string" ? msg.content diff --git a/packages/llm-sdk/src/core/stream-events.ts b/packages/llm-sdk/src/core/stream-events.ts index 5b61039..259eb94 100644 --- a/packages/llm-sdk/src/core/stream-events.ts +++ b/packages/llm-sdk/src/core/stream-events.ts @@ -83,6 +83,8 @@ export interface ActionStartEvent extends BaseEvent { type: "action:start"; id: string; name: string; + /** Whether this tool should be hidden from UI */ + hidden?: boolean; } /** @@ -121,6 +123,8 @@ export interface ToolCallInfo { id: string; name: string; args: Record; + /** Whether this tool should be hidden from UI */ + hidden?: boolean; } /** @@ -431,6 +435,13 @@ export interface ToolDefinition> { ) => unknown | Promise; render?: (props: unknown) => unknown; available?: boolean; + /** + * Hide this tool's execution from the chat UI. + * When true, tool calls and results won't be displayed to the user, + * but the tool will still execute normally. + * @default false + */ + hidden?: boolean; needsApproval?: boolean; approvalMessage?: string | ((params: TParams) => string); /** AI response mode for this tool (none, brief, full) */ diff --git a/packages/llm-sdk/src/providers/anthropic/provider.ts b/packages/llm-sdk/src/providers/anthropic/provider.ts index ea65af8..4b48b6e 100644 --- a/packages/llm-sdk/src/providers/anthropic/provider.ts +++ b/packages/llm-sdk/src/providers/anthropic/provider.ts @@ -369,38 +369,45 @@ function formatMessagesForAnthropic(messages: CoreMessage[]): { const formatted: any[] = []; const pendingToolResults: any[] = []; - for (const msg of messages) { - if (msg.role === "system") { - system += (system ? "\n" : "") + msg.content; - continue; - } + // Helper to flush pending tool results with validation + const flushToolResults = () => { + if (pendingToolResults.length === 0) return; + + const validResults = pendingToolResults.filter((tr) => { + if (!tr.toolCallId) { + console.warn("[llm-sdk] Skipping tool result with missing toolCallId"); + return false; + } + return true; + }); - // Flush pending tool results before adding assistant messages - if (msg.role === "assistant" && pendingToolResults.length > 0) { + if (validResults.length > 0) { formatted.push({ role: "user", - content: pendingToolResults.map((tr) => ({ + content: validResults.map((tr) => ({ type: "tool_result", tool_use_id: tr.toolCallId, content: tr.content, })), }); - pendingToolResults.length = 0; + } + pendingToolResults.length = 0; + }; + + for (const msg of messages) { + if (msg.role === "system") { + system += (system ? "\n" : "") + msg.content; + continue; + } + + // Flush pending tool results before adding assistant messages + if (msg.role === "assistant" && pendingToolResults.length > 0) { + flushToolResults(); } if (msg.role === "user") { // Flush pending tool results first - if (pendingToolResults.length > 0) { - formatted.push({ - role: "user", - content: pendingToolResults.map((tr) => ({ - type: "tool_result", - tool_use_id: tr.toolCallId, - content: tr.content, - })), - }); - pendingToolResults.length = 0; - } + flushToolResults(); if (typeof msg.content === "string") { formatted.push({ role: "user", content: msg.content }); @@ -460,24 +467,27 @@ function formatMessagesForAnthropic(messages: CoreMessage[]): { formatted.push({ role: "assistant", content }); } } else if (msg.role === "tool") { + // Handle both camelCase (SDK format) and snake_case (OpenAI format) + const toolCallId = + msg.toolCallId ?? (msg as any).tool_call_id ?? (msg as any).toolUseId; + + if (!toolCallId) { + console.warn( + "[llm-sdk] Tool message missing toolCallId, skipping:", + msg, + ); + continue; + } + pendingToolResults.push({ - toolCallId: msg.toolCallId, + toolCallId, content: msg.content, }); } } // Flush any remaining tool results - if (pendingToolResults.length > 0) { - formatted.push({ - role: "user", - content: pendingToolResults.map((tr) => ({ - type: "tool_result", - tool_use_id: tr.toolCallId, - content: tr.content, - })), - }); - } + flushToolResults(); return { system, messages: formatted }; } diff --git a/packages/llm-sdk/src/server/agent-loop.ts b/packages/llm-sdk/src/server/agent-loop.ts index 3eaa058..7dbd121 100644 --- a/packages/llm-sdk/src/server/agent-loop.ts +++ b/packages/llm-sdk/src/server/agent-loop.ts @@ -345,11 +345,12 @@ async function executeToolCalls( continue; } - // Emit action start + // Emit action start (include hidden flag for client-side filtering) emitEvent?.({ type: "action:start", id: toolCall.id, name: toolCall.name, + hidden: tool.hidden ?? false, }); // Emit arguments diff --git a/packages/llm-sdk/src/server/runtime.ts b/packages/llm-sdk/src/server/runtime.ts index 7df99dc..ada24d3 100644 --- a/packages/llm-sdk/src/server/runtime.ts +++ b/packages/llm-sdk/src/server/runtime.ts @@ -982,6 +982,11 @@ export class Runtime { messages: messagesWithResults as ChatRequest["messages"], }; + // Signal end of current message turn before continuing + // This tells the client to finalize the current assistant message + // The recursive call will emit a new message:start for the next turn + yield { type: "message:end" } as StreamEvent; + // Continue the agent loop - pass accumulated messages and HTTP request for await (const event of this.processChatWithLoop( nextRequest, @@ -1208,10 +1213,12 @@ export class Runtime { // Emit tool call events for (const tc of result.toolCalls) { + const tool = allTools.find((t) => t.name === tc.name); yield { type: "action:start", id: tc.id, name: tc.name, + hidden: tool?.hidden ?? false, } as StreamEvent; yield { type: "action:args", diff --git a/packages/llm-sdk/src/server/stream-result.ts b/packages/llm-sdk/src/server/stream-result.ts index 39c23d2..f02edd2 100644 --- a/packages/llm-sdk/src/server/stream-result.ts +++ b/packages/llm-sdk/src/server/stream-result.ts @@ -529,6 +529,29 @@ export class StreamResult { collected.toolCalls.push(...event.toolCalls); break; + case "action:start": + // Capture tool call with hidden flag (for server-side tools) + collected.toolCalls.push({ + id: event.id, + name: event.name, + args: {}, + hidden: event.hidden, + }); + break; + + case "action:args": { + // Update args for the tool call + const tc = collected.toolCalls.find((t) => t.id === event.id); + if (tc) { + try { + tc.args = JSON.parse(event.args || "{}"); + } catch { + tc.args = {}; + } + } + break; + } + case "done": if (event.messages) { collected.messages.push(...event.messages); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 3d2793c..9c7152e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -386,6 +386,9 @@ importers: examples/express-demo: dependencies: + '@anthropic-ai/sdk': + specifier: ^0.39.0 + version: 0.39.0 '@yourgpt/llm-sdk': specifier: workspace:* version: link:../../packages/llm-sdk From 1054b8ab41b62de62f56ef71c62865ba4436edc2 Mon Sep 17 00:00:00 2001 From: Sahil Date: Mon, 9 Mar 2026 16:07:28 +0530 Subject: [PATCH 03/72] feat: update Copilot SDK version and enhance Zod to JSON Schema conversion - Bump version from 2.1.3 to 2.1.4 in package.json. - Improve zodToJsonSchema function to support ToolInputSchema compatibility. - Refactor internal schema conversion logic for better clarity and maintainability. - Enhance useTool hook to support both Zod schemas and JSON schemas for input parameters. --- packages/copilot-sdk/package.json | 2 +- .../src/core/utils/zod-to-json-schema.ts | 38 +++++++-- .../copilot-sdk/src/react/hooks/useTool.ts | 80 +++++++++++++++---- 3 files changed, 94 insertions(+), 26 deletions(-) diff --git a/packages/copilot-sdk/package.json b/packages/copilot-sdk/package.json index cd0669a..205fa35 100644 --- a/packages/copilot-sdk/package.json +++ b/packages/copilot-sdk/package.json @@ -1,6 +1,6 @@ { "name": "@yourgpt/copilot-sdk", - "version": "2.1.3", + "version": "2.1.4", "description": "Copilot SDK for building Production-ready AI Copilots for any product. Connect any LLM, deploy on your infrastructure, own your data.", "type": "module", "types": "./dist/core/index.d.ts", diff --git a/packages/copilot-sdk/src/core/utils/zod-to-json-schema.ts b/packages/copilot-sdk/src/core/utils/zod-to-json-schema.ts index e3fc8e6..01f9ff3 100644 --- a/packages/copilot-sdk/src/core/utils/zod-to-json-schema.ts +++ b/packages/copilot-sdk/src/core/utils/zod-to-json-schema.ts @@ -188,9 +188,29 @@ function getZodEnumValues( // ============================================ /** - * Convert a Zod schema to JSON Schema property (supports Zod 3 and 4) + * Convert a Zod schema to JSON Schema property (supports Zod 3 and 4). + * When used with z.object(), the result is compatible with ToolInputSchema. + * + * @example + * ```ts + * // For tool input schemas + * useTool({ + * inputSchema: zodToJsonSchema(z.object({ + * name: z.string().describe("User name"), + * })), + * }); + * ``` + */ +export function zodToJsonSchema(schema: unknown): ToolInputSchema { + const result = _zodToJsonSchemaInternal(schema); + // Cast to ToolInputSchema - callers should only pass z.object() schemas for tool inputs + return result as unknown as ToolInputSchema; +} + +/** + * Internal implementation for recursive schema conversion */ -export function zodToJsonSchema(schema: unknown): JSONSchemaProperty { +function _zodToJsonSchemaInternal(schema: unknown): JSONSchemaProperty { if (!isZodSchema(schema)) { return { type: "string" }; } @@ -240,7 +260,9 @@ export function zodToJsonSchema(schema: unknown): JSONSchemaProperty { const innerType = getZodInnerType(schema); const result: JSONSchemaProperty = { type: "array", - items: innerType ? zodToJsonSchema(innerType) : { type: "string" }, + items: innerType + ? _zodToJsonSchemaInternal(innerType) + : { type: "string" }, }; if (description) result.description = description; return result; @@ -256,7 +278,7 @@ export function zodToJsonSchema(schema: unknown): JSONSchemaProperty { const required: string[] = []; for (const [key, value] of Object.entries(shapeObj)) { - properties[key] = zodToJsonSchema(value); + properties[key] = _zodToJsonSchemaInternal(value); // Check if the field is required (not optional/nullable) const fieldTypeName = getZodTypeName(value); @@ -281,7 +303,7 @@ export function zodToJsonSchema(schema: unknown): JSONSchemaProperty { case "ZodNullable": { const innerType = getZodInnerType(schema); if (innerType) { - return zodToJsonSchema(innerType); + return _zodToJsonSchemaInternal(innerType); } return { type: "string", description }; } @@ -289,7 +311,7 @@ export function zodToJsonSchema(schema: unknown): JSONSchemaProperty { case "ZodDefault": { const innerType = getZodInnerType(schema); if (innerType) { - const result = zodToJsonSchema(innerType); + const result = _zodToJsonSchemaInternal(innerType); // Note: Default value extraction is complex in Zod 4, skip for now return result; } @@ -342,7 +364,7 @@ export function zodToJsonSchema(schema: unknown): JSONSchemaProperty { } if (options && options.length > 0) { - return zodToJsonSchema(options[0]); + return _zodToJsonSchemaInternal(options[0]); } return { type: "string", description }; } @@ -362,7 +384,7 @@ export function zodToJsonSchema(schema: unknown): JSONSchemaProperty { * This fallback implementation is for older Zod versions. */ export function zodObjectToInputSchema(schema: unknown): ToolInputSchema { - const jsonSchema = zodToJsonSchema(schema); + const jsonSchema = _zodToJsonSchemaInternal(schema); if (jsonSchema.type !== "object" || !jsonSchema.properties) { const typeName = getZodTypeName(schema); diff --git a/packages/copilot-sdk/src/react/hooks/useTool.ts b/packages/copilot-sdk/src/react/hooks/useTool.ts index d66081c..028833d 100644 --- a/packages/copilot-sdk/src/react/hooks/useTool.ts +++ b/packages/copilot-sdk/src/react/hooks/useTool.ts @@ -1,29 +1,47 @@ "use client"; -import { useEffect, useRef } from "react"; +import { useEffect, useRef, useMemo } from "react"; import type { ToolDefinition, ToolResponse, ToolContext, ToolRenderProps, ToolSet, + ToolInputSchema, } from "../../core"; +import { zodToJsonSchema } from "../../core"; import { useCopilot } from "../provider/CopilotProvider"; /** - * Configuration for registering a tool (legacy format) + * Check if value is a Zod schema + */ +function isZodSchema(value: unknown): boolean { + if (value === null || typeof value !== "object") return false; + const obj = value as Record; + return ( + ("_def" in obj && typeof obj._def === "object") || + ("_zod" in obj && typeof obj._zod === "object") || + "~standard" in obj + ); +} + +/** + * Configuration for registering a tool */ export interface UseToolConfig> { /** Unique tool name */ name: string; /** Tool description for LLM */ description: string; - /** JSON Schema for input parameters */ - inputSchema: { - type: "object"; - properties: Record; - required?: string[]; - }; + /** + * Input schema - accepts either: + * - Zod schema: z.object({ name: z.string() }) + * - JSON Schema: { type: "object", properties: { name: { type: "string" } } } + * + * Zod schemas are automatically converted to JSON Schema at runtime. + */ + // eslint-disable-next-line @typescript-eslint/no-explicit-any + inputSchema: any; /** Handler function */ handler: ( params: TParams, @@ -51,22 +69,37 @@ export interface UseToolConfig> { * This hook registers a tool that can be called by the AI during a conversation. * The tool will execute on the client side. * + * Supports both Zod schemas and JSON schemas for inputSchema. + * * @example * ```tsx + * // Using Zod schema (recommended) + * import { z } from "zod"; + * * useTool({ * name: "navigate_to_page", * description: "Navigate to a specific page in the app", + * inputSchema: z.object({ + * path: z.string().describe("The path to navigate to"), + * }), + * handler: async ({ path }) => { + * router.push(path); + * return { success: true, message: `Navigated to ${path}` }; + * }, + * }); + * + * // Using JSON Schema + * useTool({ + * name: "open_modal", + * description: "Open a modal dialog", * inputSchema: { * type: "object", * properties: { - * path: { type: "string", description: "The path to navigate to" }, + * modalId: { type: "string" }, * }, - * required: ["path"], - * }, - * handler: async ({ path }) => { - * router.push(path); - * return { success: true, message: `Navigated to ${path}` }; + * required: ["modalId"], * }, + * handler: async ({ modalId }) => { ... }, * }); * ``` */ @@ -80,13 +113,21 @@ export function useTool>( // Update ref when config changes configRef.current = config; + // Convert Zod schema to JSON Schema if needed (memoized) + const inputSchema = useMemo(() => { + if (isZodSchema(config.inputSchema)) { + return zodToJsonSchema(config.inputSchema); + } + return config.inputSchema as ToolInputSchema; + }, [config.inputSchema]); + useEffect(() => { // Create tool definition const tool: ToolDefinition = { name: config.name, description: config.description, location: "client", - inputSchema: config.inputSchema as ToolDefinition["inputSchema"], + inputSchema, handler: async (params, context) => { return configRef.current.handler(params as TParams, context); }, @@ -106,7 +147,7 @@ export function useTool>( unregisterTool(config.name); }; // eslint-disable-next-line react-hooks/exhaustive-deps - }, [config.name, ...dependencies]); + }, [config.name, inputSchema, ...dependencies]); } /** @@ -219,11 +260,16 @@ export function useToolsArray>( const toolNames: string[] = []; for (const config of tools) { + // Convert Zod schema if needed + const inputSchema = isZodSchema(config.inputSchema) + ? zodToJsonSchema(config.inputSchema) + : (config.inputSchema as ToolInputSchema); + const tool: ToolDefinition = { name: config.name, description: config.description, location: "client", - inputSchema: config.inputSchema as ToolDefinition["inputSchema"], + inputSchema, handler: async (params, context) => { const currentConfig = toolsRef.current.find( (t) => t.name === config.name, From 70041a7c6ea8b0d4a910c64d84b50774c021e1fe Mon Sep 17 00:00:00 2001 From: Sahil Date: Wed, 11 Mar 2026 11:20:47 +0530 Subject: [PATCH 04/72] feat: enhance Express demo with minimal runtime and response endpoints - Introduced a BODY_SIZE_LIMIT environment variable for request size management. - Added minimal runtime for a simple AI assistant prompt. - Implemented two new endpoints: a streaming endpoint and a non-streaming endpoint for copilot responses. --- examples/express-demo/src/index.ts | 34 +++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/examples/express-demo/src/index.ts b/examples/express-demo/src/index.ts index a289289..165ca0c 100644 --- a/examples/express-demo/src/index.ts +++ b/examples/express-demo/src/index.ts @@ -6,8 +6,11 @@ import { createAnthropic } from "@yourgpt/llm-sdk/anthropic"; import { createOpenAI } from "@yourgpt/llm-sdk/openai"; const app = express(); +const BODY_SIZE_LIMIT = process.env.BODY_SIZE_LIMIT || "100mb"; + app.use(cors()); -app.use(express.json()); +app.use(express.json({ limit: BODY_SIZE_LIMIT })); +app.use(express.urlencoded({ extended: true, limit: BODY_SIZE_LIMIT })); // ============================================ // DUMMY KNOWLEDGE BASE DATA @@ -192,6 +195,35 @@ Be helpful, concise, and accurate. If the knowledge base doesn't have the answer }, }); +// ============================================ +// MINIMAL RUNTIME (No tools, simple prompt) +// ============================================ + +const minimalRuntime = createRuntime({ + provider, + model, + systemPrompt: "You are a helpful AI assistant.", +}); + +// ============================================ +// MINIMAL COPILOT RESPONSE ENDPOINT +// ============================================ + +/** + * Minimal streaming endpoint - no tools, simple prompt + */ +app.post("/api/copilot-response", async (req, res) => { + await minimalRuntime.stream(req.body).pipeToResponse(res); +}); + +/** + * Minimal non-streaming endpoint - no tools, simple prompt + */ +app.post("/api/copilot-response/chat", async (req, res) => { + const result = await minimalRuntime.chat(req.body); + res.json(result); +}); + // ============================================ // COPILOT SDK COMPATIBLE ENDPOINTS // ============================================ From b26f06e83621907b6490e19fca95c385d6fc8d79 Mon Sep 17 00:00:00 2001 From: Sahil Date: Wed, 11 Mar 2026 16:54:41 +0530 Subject: [PATCH 05/72] Add tool management, deferred loading, and native search support --- WEB_SEARCH_IMPLEMENTATION.md | 385 ----- examples/experimental/README.md | 24 +- .../app/api/chat/tool-scale/route.ts | 141 ++ examples/experimental/app/page.tsx | 5 + examples/experimental/app/tool-scale/page.tsx | 286 ++++ .../experimental/lib/tool-scale/catalog.ts | 827 ++++++++++ .../lib/tool-scale/client-tools.ts | 58 + .../lib/tool-scale/server-tools.ts | 51 + examples/express-demo/README.md | 40 + examples/express-demo/src/index.ts | 44 + .../copilot-sdk/src/chat/ChatWithTools.ts | 29 + .../src/chat/adapters/HttpTransport.ts | 1 + .../src/chat/classes/AbstractChat.ts | 217 +-- packages/copilot-sdk/src/chat/index.ts | 1 + .../src/chat/interfaces/ChatTransport.ts | 2 + .../copilot-sdk/src/chat/optimizations.ts | 1352 +++++++++++++++++ packages/copilot-sdk/src/chat/types/chat.ts | 12 +- packages/copilot-sdk/src/core/index.ts | 13 + packages/copilot-sdk/src/core/types/tools.ts | 179 +++ packages/llm-sdk/README.md | 59 + packages/llm-sdk/src/adapters/anthropic.ts | 135 +- packages/llm-sdk/src/adapters/azure.ts | 20 +- packages/llm-sdk/src/adapters/base.ts | 113 ++ packages/llm-sdk/src/adapters/google.ts | 42 +- packages/llm-sdk/src/adapters/ollama.ts | 19 +- packages/llm-sdk/src/adapters/openai.ts | 343 ++++- packages/llm-sdk/src/adapters/xai.ts | 20 +- packages/llm-sdk/src/core/stream-events.ts | 116 ++ packages/llm-sdk/src/index.ts | 15 + packages/llm-sdk/src/providers/anthropic.ts | 8 +- packages/llm-sdk/src/providers/gemini.ts | 8 +- packages/llm-sdk/src/providers/openai.ts | 8 +- packages/llm-sdk/src/server/agent-loop.ts | 122 +- packages/llm-sdk/src/server/index.ts | 9 + packages/llm-sdk/src/server/runtime.ts | 362 ++++- packages/llm-sdk/src/server/tool-selection.ts | 587 +++++++ packages/llm-sdk/src/server/types.ts | 22 + tool-search-implementation.md | 253 +++ 38 files changed, 5332 insertions(+), 596 deletions(-) delete mode 100644 WEB_SEARCH_IMPLEMENTATION.md create mode 100644 examples/experimental/app/api/chat/tool-scale/route.ts create mode 100644 examples/experimental/app/tool-scale/page.tsx create mode 100644 examples/experimental/lib/tool-scale/catalog.ts create mode 100644 examples/experimental/lib/tool-scale/client-tools.ts create mode 100644 examples/experimental/lib/tool-scale/server-tools.ts create mode 100644 packages/copilot-sdk/src/chat/optimizations.ts create mode 100644 packages/llm-sdk/src/server/tool-selection.ts create mode 100644 tool-search-implementation.md diff --git a/WEB_SEARCH_IMPLEMENTATION.md b/WEB_SEARCH_IMPLEMENTATION.md deleted file mode 100644 index eb27ae8..0000000 --- a/WEB_SEARCH_IMPLEMENTATION.md +++ /dev/null @@ -1,385 +0,0 @@ -# Web Search Implementation - Technical Documentation - -> Temporary documentation for the native web search feature implementation. - ---- - -## Current Implementation Status - -### Completed - -- [x] Native web search for all 3 LLM providers (OpenAI, Google, Anthropic) -- [x] Single API call (was 2 calls before - LLM + search provider) -- [x] Citations displayed as chips below messages (like Perplexity/ChatGPT) -- [x] Tree-shakeable subpath exports (~3KB per provider vs ~50KB for all) -- [x] Unified Citation format across all providers -- [x] HoverCard preview for citations with favicon and domain -- [x] Hidden "Web search" tool step when native citations exist -- [x] Debug logs cleaned up -- [x] Simplified naming (removed "-native" suffix) - ---- - -## Architecture - -### Package Structure - -``` -@yourgpt/copilot-sdk -├── /core # Main exports -├── /react # React hooks -├── /ui # UI components -└── /tools # Tree-shakeable tool exports - ├── /web-search # Shared types + utilities - ├── /openai # openaiSearch() - ├── /google # googleSearch() - ├── /anthropic # anthropicSearch() - ├── /tavily # tavilySearch() - ├── /serper # serperSearch() - ├── /brave # braveSearch() - ├── /exa # exaSearch() - └── /searxng # searxngSearch() -``` - -### Provider Implementation Files - -``` -packages/copilot-sdk/src/core/tools/webSearch/providers/ -├── openai.ts # OpenAI Responses API with web_search tool -├── google.ts # Gemini API with google_search grounding -├── anthropic.ts # Anthropic Messages API with web_search_20250305 -├── tavily.ts # Tavily API -├── serper.ts # Serper (Google) API -├── brave.ts # Brave Search API -├── exa.ts # Exa (semantic) API -└── searxng.ts # Self-hosted SearXNG -``` - -### LLM Adapters with Native Web Search - -``` -packages/llm-sdk/src/adapters/ -├── openai.ts # webSearch config → web_search_preview tool -├── google.ts # webSearch config → google_search grounding -└── anthropic.ts # webSearch config → web_search_20260209 tool -``` - ---- - -## Unified Citation Format - -All providers normalize to this format: - -```typescript -interface Citation { - index: number; // 1-based index - url: string; // Source URL - title: string; // Page title or domain - domain?: string; // Extracted domain (e.g., "example.com") - favicon?: string; // Google favicon URL - citedText?: string; // Relevant excerpt (Anthropic only) -} -``` - -### Stream Event - -```typescript -yield { type: "citation", citations: Citation[] }; -``` - -### Message Metadata - -Citations are stored in message metadata: - -```typescript -message.metadata.citations: Citation[] -``` - ---- - -## Provider-Specific Details - -### OpenAI - -**Tool Type:** `web_search_preview` -**API:** Chat Completions (streaming) -**Citations:** `delta.annotations[]` with `type: "url_citation"` - -```typescript -// Adapter config -webSearch: true | WebSearchConfig; - -// Emits during stream -if (annotation.type === "url_citation") { - collectedCitations.push({ - url: annotation.url_citation.url, - title: annotation.url_citation.title, - }); -} -``` - -### Google (Gemini) - -**Tool Type:** `{ google_search: {} }` -**API:** generateContent (streaming) -**Citations:** `candidate.groundingMetadata.groundingChunks[]` - -```typescript -// Grounding metadata -groundingMetadata: { - groundingChunks: [ - { web: { uri: string, title?: string } } - ] -} -``` - -### Anthropic - -**Tool Type:** `web_search_20260209` (streaming adapter) / `web_search_20250305` (standalone) -**API:** Messages (streaming) -**Citations:** `content[].citations[]` with `type: "web_search_result_location"` - -```typescript -// Citation format -{ - type: "web_search_result_location", - url: string, - title: string, - cited_text?: string, // Unique to Anthropic -} -``` - -**Note:** Anthropic provides `cited_text` - the actual text from the page that was cited. - ---- - -## UI Components - -### SourceGroup (`source.tsx`) - -Displays citations as chips with hover preview. - -```tsx - -``` - -### Source (individual chip) - -```tsx - -``` - -### HoverCard - -Uses `@radix-ui/react-hover-card` for preview on hover. -Animation requires `tw-animate-css` (Tailwind v4) in user's project. - ---- - -## Known Issues & Fixes Applied - -### 1. Citations Lost After Stream Ends - -**Problem:** `useInternalThreadManager` was calling `setMessages()` even without persistence adapter, overwriting metadata. - -**Fix:** Added `!adapter` check: - -```typescript -useEffect(() => { - if (!adapter) return; // Skip sync when no persistence - // ... -}, [adapter, messages]); -``` - -**File:** `packages/copilot-sdk/src/ui/hooks/useInternalThreadManager.ts` - -### 2. Tool Step Showing During Native Search - -**Problem:** "Web search" tool step was showing during streaming for native web search. - -**Fix:** Don't emit `action:start`/`action:end` for `web_search` tool: - -```typescript -if (currentToolUse.name !== "web_search") { - yield { type: "action:start", ... }; -} -``` - -**File:** `packages/llm-sdk/src/adapters/anthropic.ts` - -### 3. Citations Layout - -**Problem:** SourceGroup was rendering to the right of message content. - -**Fix:** Moved SourceGroup inside the content div in `default-message.tsx`. - -### 4. HoverCard Animations - -**Problem:** No transition on hover card. - -**Solution:** Users need to add `tw-animate-css` package (Tailwind v4): - -```bash -pnpm add tw-animate-css -``` - -```css -@import "tailwindcss"; -@import "tw-animate-css"; -``` - ---- - -## Suggestions for Future Improvements - -### 1. Extract Duplicate Utilities - -The `extractDomain` function is duplicated in: - -- `adapters/openai.ts` -- `adapters/google.ts` -- `adapters/anthropic.ts` -- `ui/components/ui/source.tsx` - -**Suggestion:** Create shared `packages/llm-sdk/src/utils/url.ts` - -### 2. Add Anthropic to Documentation Tabs - -The `web-search.mdx` docs are missing Anthropic tab in provider examples. - -### 3. Citation Loading State - -Currently citations appear after stream ends. Consider showing a subtle "Searching..." indicator during streaming. - -### 4. Consolidate Citation Components - -Both `citations.tsx` and `source.tsx` exist. Consider: - -- Deprecating one, or -- Clearly documenting when to use each - -### 5. Error Boundary for Citations - -Add graceful fallback if favicon fails to load (currently just hides). - -### 6. Version Consistency - -Ensure Anthropic web search version is consistent: - -- Adapter: `web_search_20260209` -- Standalone: `web_search_20250305` - -Pick one version and use consistently. - ---- - -## Usage Examples - -### Native Web Search (Recommended) - -```typescript -// In adapter config - single API call -const adapter = createAnthropicAdapter({ - apiKey: process.env.ANTHROPIC_API_KEY, - model: "claude-sonnet-4-20250514", - webSearch: true, // Enable native web search -}); -``` - -### Tree-Shakeable Tool Import - -```typescript -import { openaiSearch } from "@yourgpt/copilot-sdk/tools/openai"; - -const webSearch = openaiSearch({ - apiKey: process.env.OPENAI_API_KEY, - maxResults: 5, -}); - -const runtime = createRuntime({ - provider: openai, - model: "gpt-4o", - tools: [webSearch], -}); -``` - -### Legacy Import (All Providers) - -```typescript -import { createWebSearchTool } from "@yourgpt/copilot-sdk/core"; - -const webSearch = createWebSearchTool({ - provider: "anthropic", - apiKey: process.env.ANTHROPIC_API_KEY, -}); -``` - ---- - -## Bundle Size - -| Import Pattern | Size | -| -------------------------------------- | ------ | -| `@yourgpt/copilot-sdk/tools/openai` | ~2.5KB | -| `@yourgpt/copilot-sdk/tools/google` | ~2.5KB | -| `@yourgpt/copilot-sdk/tools/anthropic` | ~3KB | -| `@yourgpt/copilot-sdk/tools/tavily` | ~3KB | -| `@yourgpt/copilot-sdk/core` (all) | ~50KB | - -**~85% reduction** when using single provider import. - ---- - -## Testing - -### Demo App - -```bash -cd examples/web-search-demo -pnpm dev -# Open http://localhost:3009 -``` - -### Test Queries - -- "What are the latest AI news?" -- "What's the weather in New York?" -- "Who won the most recent Super Bowl?" -- "What's the current price of Bitcoin?" - ---- - -## Files Modified in This Feature - -### New Files - -- `packages/copilot-sdk/src/tools/*/index.ts` (8 tool exports) -- `packages/copilot-sdk/src/core/tools/webSearch/providers/*.ts` (8 providers) -- `packages/copilot-sdk/src/ui/components/ui/source.tsx` -- `packages/copilot-sdk/src/ui/components/ui/citations.tsx` -- `examples/web-search-demo/` (entire demo app) -- `apps/docs/content/docs/tools/built-in/web-search.mdx` - -### Modified Files - -- `packages/llm-sdk/src/adapters/openai.ts` (webSearch support) -- `packages/llm-sdk/src/adapters/google.ts` (webSearch support) -- `packages/llm-sdk/src/adapters/anthropic.ts` (webSearch support) -- `packages/copilot-sdk/src/ui/components/composed/chat/default-message.tsx` -- `packages/copilot-sdk/src/ui/hooks/useInternalThreadManager.ts` -- `packages/copilot-sdk/package.json` (subpath exports) -- `packages/copilot-sdk/tsup.config.ts` (entry points) - ---- - -_Last updated: 2026-02-23_ diff --git a/examples/experimental/README.md b/examples/experimental/README.md index 999af41..7f7ff49 100644 --- a/examples/experimental/README.md +++ b/examples/experimental/README.md @@ -8,14 +8,15 @@ This directory contains experimental and raw examples for testing various SDK fe ## Available Demos -| Demo | Path | Description | -| ----------------------- | ------------------ | ------------------------------------------------- | -| **Non-Streaming** | `/non-streaming` | `runtime.generate()` with CopilotChat | -| **Theme Demo** | `/theme-demo` | 9 theme presets with live preview | -| **Multi-Provider** | `/providers` | OpenAI, Anthropic, Google side-by-side | -| **Compound Components** | `/compound-test` | Custom home screen with `Chat.Home`, `Chat.Input` | -| **Tool Types** | `/tool-types-demo` | Different tool rendering patterns | -| **Widgets** | `/widgets` | Standalone UI components | +| Demo | Path | Description | +| ----------------------- | ------------------ | ----------------------------------------------------- | +| **Non-Streaming** | `/non-streaming` | `runtime.generate()` with CopilotChat | +| **Theme Demo** | `/theme-demo` | 9 theme presets with live preview | +| **Multi-Provider** | `/providers` | OpenAI, Anthropic, Google side-by-side | +| **Compound Components** | `/compound-test` | Custom home screen with `Chat.Home`, `Chat.Input` | +| **Tool Types** | `/tool-types-demo` | Different tool rendering patterns | +| **Tool Scale Lab** | `/tool-scale` | 100 tools with profiles, search, and deferred loading | +| **Widgets** | `/widgets` | Standalone UI components | ## Quick Start @@ -79,19 +80,22 @@ experimental/ │ ├── providers/ # Multi-provider test │ ├── compound-test/ # Compound components │ ├── tool-types-demo/ # Tool rendering patterns +│ ├── tool-scale/ # 100-tool selection/profile lab │ ├── widgets/ # Standalone widgets │ └── api/ │ └── chat/ │ ├── openai/route.ts │ ├── anthropic/route.ts -│ └── google/route.ts +│ ├── google/route.ts +│ └── tool-scale/route.ts ├── components/ │ ├── theme-provider.tsx │ ├── provider-card.tsx │ └── tools/ ├── lib/ │ ├── utils.ts -│ └── tools/ +│ ├── tools/ +│ └── tool-scale/ └── README.md ``` diff --git a/examples/experimental/app/api/chat/tool-scale/route.ts b/examples/experimental/app/api/chat/tool-scale/route.ts new file mode 100644 index 0000000..12d9cc6 --- /dev/null +++ b/examples/experimental/app/api/chat/tool-scale/route.ts @@ -0,0 +1,141 @@ +import { createRuntime } from "@yourgpt/llm-sdk"; +import { createAnthropic } from "@yourgpt/llm-sdk/anthropic"; +import { createOpenAI } from "@yourgpt/llm-sdk/openai"; + +import { toolScaleServerTools } from "@/lib/tool-scale/server-tools"; + +function resolveProvider() { + if (process.env.ANTHROPIC_API_KEY) { + return { + providerName: "Anthropic", + provider: createAnthropic({ + apiKey: process.env.ANTHROPIC_API_KEY, + }), + model: "claude-haiku-4-5", + }; + } + + // if (process.env.OPENAI_API_KEY) { + // return { + // providerName: "OpenAI", + // provider: createOpenAI({ + // apiKey: process.env.OPENAI_API_KEY, + // }), + // model: "gpt-5-mini-2025-08-07", + // // model: "gpt-5.4", + // }; + // } + + throw new Error( + "Set ANTHROPIC_API_KEY or OPENAI_API_KEY to run the tool scale lab example.", + ); +} + +const { providerName, provider, model } = resolveProvider(); + +const runtime = createRuntime({ + provider, + model, + debug: process.env.NODE_ENV === "development", + systemPrompt: `You are the Tool Scale Lab assistant. + +You are testing a project with 100 tools: 30 server-side and 70 client-side. +Use tools sparingly and intentionally. + +When tools are missing, rely on the search_tools meta-tool to discover deferred tools rather than guessing. +Keep answers short and explain which class of tools you used when it helps the user understand tool selection behavior.`, + tools: toolScaleServerTools, + agentLoop: { + enabled: true, + maxIterations: 6, + debug: process.env.NODE_ENV === "development", + toolSelection: { + enabled: true, + defaultProfile: "support", + includeUnprofiled: false, + search: { + enabled: true, + maxResults: 6, + minScore: 0.15, + exposeWhenToolCountExceeds: 12, + metaToolName: "search_tools", + strictDeferredLoading: true, + }, + dynamicSelection: { + enabled: true, + maxTools: 6, + }, + profiles: { + support: { + include: [ + "profile:support", + "category:knowledge", + "category:billing", + "category:browser", + "category:utility", + ], + exclude: ["group:admin"], + }, + workspace: { + include: [ + "profile:workspace", + "category:workspace", + "category:browser", + "category:analytics", + "category:utility", + ], + }, + commerce: { + include: [ + "profile:commerce", + "category:commerce", + "category:billing", + "group:actions", + ], + }, + admin: { + include: [ + "profile:admin", + "category:operations", + "category:analytics", + "category:utility", + ], + }, + }, + nativeProviderHints: { + anthropic: { + toolChoice: "auto", + disableParallelToolUse: true, + }, + openai: { + toolChoice: "auto", + parallelToolCalls: false, + }, + }, + }, + }, +}); + +export async function POST(request: Request) { + try { + return await runtime.handleRequest(request); + } catch (error) { + return Response.json( + { + error: error instanceof Error ? error.message : "Unknown error", + }, + { status: 500 }, + ); + } +} + +export async function GET() { + return Response.json({ + status: "ok", + provider: providerName, + model, + toolCount: { + server: toolScaleServerTools.length, + }, + }); +} diff --git a/examples/experimental/app/page.tsx b/examples/experimental/app/page.tsx index a87f4b1..fa3a9a7 100644 --- a/examples/experimental/app/page.tsx +++ b/examples/experimental/app/page.tsx @@ -26,6 +26,11 @@ const demos = [ href: "/tool-types-demo", description: "Different tool rendering patterns", }, + { + name: "Tool Scale Lab", + href: "/tool-scale", + description: "100-tool profile, search, and deferred-loading demo", + }, { name: "Widgets", href: "/widgets", diff --git a/examples/experimental/app/tool-scale/page.tsx b/examples/experimental/app/tool-scale/page.tsx new file mode 100644 index 0000000..4178a3f --- /dev/null +++ b/examples/experimental/app/tool-scale/page.tsx @@ -0,0 +1,286 @@ +"use client"; + +import { useState } from "react"; +import { CopilotProvider, useTools } from "@yourgpt/copilot-sdk/react"; +import { CopilotChat } from "@yourgpt/copilot-sdk/ui"; +import "@yourgpt/copilot-sdk/ui/styles.css"; +import "@yourgpt/copilot-sdk/ui/themes/claude.css"; + +import { toolScaleClientTools } from "@/lib/tool-scale/client-tools"; +import { + getProfileToolStats, + toolScaleCounts, + toolScaleProfiles, +} from "@/lib/tool-scale/catalog"; + +function ToolScaleClientRegistration() { + useTools(toolScaleClientTools); + return null; +} + +const profilePrompts: Record = { + support: [ + "What refund policy and SLA guidance should I give a customer asking about delayed support?", + "Search the right docs and pricing tools for an enterprise plan migration question.", + ], + workspace: [ + "Help me summarize the current workspace layout and find blocked tasks for tomorrow.", + "What document and scheduling tools should you use to inspect upcoming deadlines?", + ], + commerce: [ + "Check checkout issues, promo codes, and shipping details for an abandoned cart flow.", + "Which billing and commerce tools would you use for a failed payment complaint?", + ], + admin: [ + "Inspect incident status, audit signals, and dashboard metrics for an operations review.", + "What tools are relevant for a compliance and analytics triage session?", + ], +}; + +const requestSnippet = `{ + "toolProfile": "support", + "messages": [ + { + "role": "user", + "content": "Help me answer a pricing and SLA question" + } + ] +}`; + +const selectionSnippet = `toolSelection: { + enabled: true, + defaultProfile: "support", + includeUnprofiled: false, + search: { + enabled: true, + maxResults: 6, + exposeWhenToolCountExceeds: 12, + metaToolName: "search_tools", + strictDeferredLoading: true, + }, + dynamicSelection: { + enabled: true, + maxTools: 6, + }, +}`; + +export default function ToolScalePage() { + const [activeProfile, setActiveProfile] = useState("support"); + const stats = getProfileToolStats(activeProfile); + + return ( +
+
+
+
+
+

+ Experimental Tool Scale Lab +

+

+ 100-tool mixed runtime test +

+

+ This example simulates a project with {toolScaleCounts.total}{" "} + tools: {toolScaleCounts.server} server-side and{" "} + {toolScaleCounts.client} client-side. Most tools are deferred, + so the model sees a small profile-specific slice up front and + discovers the rest through `search_tools`. +

+
+ +
+
+
Total
+
+ {toolScaleCounts.total} +
+
+
+
Server
+
+ {toolScaleCounts.server} +
+
+
+
Client
+
+ {toolScaleCounts.client} +
+
+
+
Deferred
+
+ {toolScaleCounts.deferred} +
+
+
+
+
+ +
+ + +
+
+

Run the scenario

+

+ Switch profiles, then ask for something that should require + docs, billing, workspace, checkout, or operations tools. The + runtime will receive `toolProfile: "{activeProfile}"`. +

+
+ {profilePrompts[activeProfile].map((prompt) => ( + + {prompt} + + ))} +
+
+ +
+ + + + +
+
+
+
+
+ ); +} diff --git a/examples/experimental/lib/tool-scale/catalog.ts b/examples/experimental/lib/tool-scale/catalog.ts new file mode 100644 index 0000000..1e25207 --- /dev/null +++ b/examples/experimental/lib/tool-scale/catalog.ts @@ -0,0 +1,827 @@ +export type ToolLocation = "server" | "client"; + +export interface ToolSeed { + name: string; + title: string; + description: string; + location: ToolLocation; + category: string; + group: string; + profiles: string[]; + deferLoading: boolean; + searchKeywords: string[]; +} + +interface ClusterDefinition { + prefix: string; + location: ToolLocation; + category: string; + group: string; + profiles: string[]; + immediateCount: number; + items: Array<{ + slug: string; + title: string; + description: string; + keywords: string[]; + }>; +} + +function createClusterTools(cluster: ClusterDefinition): ToolSeed[] { + return cluster.items.map((item, index) => ({ + name: `${cluster.prefix}_${item.slug}`, + title: item.title, + description: item.description, + location: cluster.location, + category: cluster.category, + group: cluster.group, + profiles: cluster.profiles, + deferLoading: true, + // deferLoading: index >= cluster.immediateCount, + searchKeywords: [ + cluster.category, + cluster.group, + ...cluster.profiles, + ...item.keywords, + ], + })); +} + +const SERVER_CLUSTERS: ClusterDefinition[] = [ + { + prefix: "support", + location: "server", + category: "knowledge", + group: "support", + profiles: ["support", "research"], + immediateCount: 2, + items: [ + { + slug: "search_product_docs", + title: "Search product docs", + description: "Find official product documentation for support answers.", + keywords: ["docs", "product", "guide"], + }, + { + slug: "search_api_reference", + title: "Search API reference", + description: + "Find API contracts, request formats, and SDK reference material.", + keywords: ["api", "sdk", "reference"], + }, + { + slug: "find_setup_guides", + title: "Find setup guides", + description: + "Locate setup and onboarding walkthroughs for new integrations.", + keywords: ["setup", "onboarding", "installation"], + }, + { + slug: "find_migration_notes", + title: "Find migration notes", + description: "Locate migration checklists and breaking-change notes.", + keywords: ["migration", "upgrade", "release"], + }, + { + slug: "lookup_security_faq", + title: "Lookup security FAQ", + description: + "Retrieve security, privacy, and compliance answers for customers.", + keywords: ["security", "privacy", "compliance"], + }, + { + slug: "lookup_pricing_matrix", + title: "Lookup pricing matrix", + description: "Find plan tiers, usage caps, and pricing notes.", + keywords: ["pricing", "plans", "limits"], + }, + { + slug: "search_release_notes", + title: "Search release notes", + description: "Find release highlights, feature launches, and fixes.", + keywords: ["release", "changelog", "feature"], + }, + { + slug: "find_troubleshooting_playbooks", + title: "Find troubleshooting playbooks", + description: + "Retrieve support playbooks for common incidents and errors.", + keywords: ["troubleshooting", "incident", "errors"], + }, + { + slug: "search_integration_cookbook", + title: "Search integration cookbook", + description: "Find integration recipes for common product setups.", + keywords: ["integration", "cookbook", "recipes"], + }, + { + slug: "lookup_sla_policies", + title: "Lookup SLA policies", + description: "Find SLA, uptime, and support response policy details.", + keywords: ["sla", "uptime", "policy"], + }, + ], + }, + { + prefix: "finance", + location: "server", + category: "billing", + group: "finance", + profiles: ["support", "sales", "finance"], + immediateCount: 2, + items: [ + { + slug: "get_invoice_status", + title: "Get invoice status", + description: "Check invoice status, payment attempts, and due dates.", + keywords: ["invoice", "payment", "status"], + }, + { + slug: "get_plan_entitlements", + title: "Get plan entitlements", + description: "Inspect plan entitlements, seats, and included limits.", + keywords: ["plan", "entitlements", "seats"], + }, + { + slug: "lookup_credit_balance", + title: "Lookup credit balance", + description: + "Find remaining credits and rollover details for an account.", + keywords: ["credits", "balance", "usage"], + }, + { + slug: "find_refund_policy", + title: "Find refund policy", + description: + "Retrieve refund rules, eligibility criteria, and time windows.", + keywords: ["refund", "policy", "eligibility"], + }, + { + slug: "get_contract_terms", + title: "Get contract terms", + description: "Inspect contract renewal dates and commercial terms.", + keywords: ["contract", "renewal", "terms"], + }, + { + slug: "lookup_discount_rules", + title: "Lookup discount rules", + description: + "Find discounting rules and approved commercial exceptions.", + keywords: ["discount", "commercial", "pricing"], + }, + { + slug: "find_tax_guidance", + title: "Find tax guidance", + description: + "Retrieve region-specific tax handling and invoicing notes.", + keywords: ["tax", "region", "invoice"], + }, + { + slug: "get_overage_breakdown", + title: "Get overage breakdown", + description: "Inspect overage drivers and top consumption buckets.", + keywords: ["overage", "consumption", "usage"], + }, + { + slug: "lookup_checkout_rules", + title: "Lookup checkout rules", + description: "Find checkout, trial, and subscription conversion rules.", + keywords: ["checkout", "trial", "subscription"], + }, + { + slug: "find_procurement_packet", + title: "Find procurement packet", + description: + "Locate procurement, vendor, and approval packet material.", + keywords: ["procurement", "vendor", "security"], + }, + ], + }, + { + prefix: "ops", + location: "server", + category: "operations", + group: "admin", + profiles: ["ops", "admin"], + immediateCount: 2, + items: [ + { + slug: "check_incident_status", + title: "Check incident status", + description: "Review current incident state and impacted systems.", + keywords: ["incident", "status", "systems"], + }, + { + slug: "get_usage_snapshot", + title: "Get usage snapshot", + description: "Inspect current usage and system-level traffic shape.", + keywords: ["usage", "traffic", "snapshot"], + }, + { + slug: "lookup_rate_limit_state", + title: "Lookup rate limit state", + description: "Check rate-limit windows and throttling activity.", + keywords: ["rate", "limit", "throttle"], + }, + { + slug: "find_feature_flags", + title: "Find feature flags", + description: "Inspect active feature flags for an environment.", + keywords: ["feature", "flags", "environment"], + }, + { + slug: "resolve_workspace_owner", + title: "Resolve workspace owner", + description: "Find workspace ownership and escalation contacts.", + keywords: ["workspace", "owner", "escalation"], + }, + { + slug: "inspect_team_roles", + title: "Inspect team roles", + description: "List team roles, permissions, and admin assignments.", + keywords: ["team", "roles", "permissions"], + }, + { + slug: "review_audit_events", + title: "Review audit events", + description: "Search recent audit events and security changes.", + keywords: ["audit", "security", "events"], + }, + { + slug: "lookup_region_status", + title: "Lookup region status", + description: "Check service health and capacity by region.", + keywords: ["region", "health", "capacity"], + }, + { + slug: "find_data_retention_rules", + title: "Find data retention rules", + description: "Inspect retention windows and deletion policies.", + keywords: ["retention", "deletion", "policy"], + }, + { + slug: "get_compliance_controls", + title: "Get compliance controls", + description: "Retrieve compliance control mappings and attestations.", + keywords: ["compliance", "controls", "attestation"], + }, + ], + }, +]; + +const CLIENT_CLUSTERS: ClusterDefinition[] = [ + { + prefix: "browser", + location: "client", + category: "browser", + group: "inspection", + profiles: ["support", "workspace"], + immediateCount: 2, + items: [ + { + slug: "inspect_dom_outline", + title: "Inspect DOM outline", + description: "Inspect the current page structure and headings.", + keywords: ["dom", "html", "headings"], + }, + { + slug: "capture_visible_text", + title: "Capture visible text", + description: "Capture visible text content from the active page.", + keywords: ["text", "content", "page"], + }, + { + slug: "find_primary_actions", + title: "Find primary actions", + description: "Locate the main buttons and calls to action on the page.", + keywords: ["buttons", "cta", "actions"], + }, + { + slug: "find_form_fields", + title: "Find form fields", + description: "List form fields and labels available on the page.", + keywords: ["form", "fields", "labels"], + }, + { + slug: "inspect_error_banner", + title: "Inspect error banner", + description: "Check for visible alert, toast, or error banners.", + keywords: ["error", "alert", "toast"], + }, + { + slug: "extract_help_links", + title: "Extract help links", + description: "Collect help center and support links from the UI.", + keywords: ["help", "support", "links"], + }, + { + slug: "scan_table_headers", + title: "Scan table headers", + description: "Inspect visible table headers and summary labels.", + keywords: ["table", "headers", "data"], + }, + { + slug: "read_navigation_labels", + title: "Read navigation labels", + description: "List current navigation items and sidebar labels.", + keywords: ["navigation", "sidebar", "menu"], + }, + { + slug: "detect_modal_state", + title: "Detect modal state", + description: "Check whether a modal or drawer is currently open.", + keywords: ["modal", "drawer", "dialog"], + }, + { + slug: "inspect_page_metadata", + title: "Inspect page metadata", + description: "Read page title, URL path, and language metadata.", + keywords: ["metadata", "url", "language"], + }, + ], + }, + { + prefix: "browser", + location: "client", + category: "browser", + group: "actions", + profiles: ["support", "commerce"], + immediateCount: 2, + items: [ + { + slug: "focus_search_box", + title: "Focus search box", + description: "Find and focus the main search input in the UI.", + keywords: ["search", "input", "focus"], + }, + { + slug: "scroll_to_section", + title: "Scroll to section", + description: "Scroll the current page to a matching section.", + keywords: ["scroll", "section", "page"], + }, + { + slug: "expand_accordion", + title: "Expand accordion", + description: "Expand a collapsed accordion or disclosure element.", + keywords: ["accordion", "expand", "collapse"], + }, + { + slug: "copy_selected_text", + title: "Copy selected text", + description: "Copy highlighted or matched text from the page.", + keywords: ["copy", "text", "selection"], + }, + { + slug: "highlight_form_errors", + title: "Highlight form errors", + description: + "Identify invalid form fields and focus them for the user.", + keywords: ["form", "errors", "validation"], + }, + { + slug: "open_help_center", + title: "Open help center", + description: "Open the help center from the current experience.", + keywords: ["help", "center", "support"], + }, + { + slug: "dismiss_banner", + title: "Dismiss banner", + description: "Dismiss the active banner, toast, or notice if present.", + keywords: ["dismiss", "toast", "banner"], + }, + { + slug: "toggle_preview_panel", + title: "Toggle preview panel", + description: "Toggle a preview or detail side panel in the interface.", + keywords: ["preview", "panel", "toggle"], + }, + { + slug: "jump_to_checkout_step", + title: "Jump to checkout step", + description: "Move to a matching step in a checkout or wizard flow.", + keywords: ["checkout", "wizard", "step"], + }, + { + slug: "activate_primary_tab", + title: "Activate primary tab", + description: "Switch to the primary or requested tab in a tab set.", + keywords: ["tab", "switch", "navigation"], + }, + ], + }, + { + prefix: "workspace", + location: "client", + category: "workspace", + group: "documents", + profiles: ["workspace", "support"], + immediateCount: 2, + items: [ + { + slug: "open_doc_outline", + title: "Open doc outline", + description: "Open or summarize the current document outline.", + keywords: ["document", "outline", "doc"], + }, + { + slug: "list_recent_files", + title: "List recent files", + description: "List the most recent files visible in the workspace.", + keywords: ["recent", "files", "workspace"], + }, + { + slug: "find_comment_threads", + title: "Find comment threads", + description: "Inspect recent comment threads and unresolved notes.", + keywords: ["comments", "threads", "notes"], + }, + { + slug: "detect_unpublished_changes", + title: "Detect unpublished changes", + description: "Check whether there are unpublished or unsaved edits.", + keywords: ["publish", "draft", "changes"], + }, + { + slug: "read_doc_permissions", + title: "Read doc permissions", + description: + "Inspect sharing and permission hints for the current doc.", + keywords: ["sharing", "permissions", "access"], + }, + { + slug: "open_command_palette", + title: "Open command palette", + description: "Open the command palette for quick workspace actions.", + keywords: ["command", "palette", "shortcut"], + }, + { + slug: "search_workspace_mentions", + title: "Search workspace mentions", + description: "Find mentions, assignments, and @references in the UI.", + keywords: ["mentions", "assignments", "workspace"], + }, + { + slug: "inspect_publish_checks", + title: "Inspect publish checks", + description: "Inspect publishing checks, blockers, and warnings.", + keywords: ["publish", "checks", "warnings"], + }, + { + slug: "find_content_templates", + title: "Find content templates", + description: "Find reusable templates and starter documents.", + keywords: ["templates", "content", "starter"], + }, + { + slug: "review_editor_panels", + title: "Review editor panels", + description: + "List editor panels, drawers, and sidebars currently visible.", + keywords: ["editor", "panels", "sidebar"], + }, + ], + }, + { + prefix: "workspace", + location: "client", + category: "workspace", + group: "scheduling", + profiles: ["workspace"], + immediateCount: 2, + items: [ + { + slug: "list_calendar_slots", + title: "List calendar slots", + description: + "Read available meeting slots in the current scheduling view.", + keywords: ["calendar", "slots", "schedule"], + }, + { + slug: "find_upcoming_deadlines", + title: "Find upcoming deadlines", + description: "Find upcoming deadlines and due dates in the UI.", + keywords: ["deadlines", "due", "dates"], + }, + { + slug: "inspect_task_board_columns", + title: "Inspect task board columns", + description: "Inspect the current task board lanes and counts.", + keywords: ["task", "board", "kanban"], + }, + { + slug: "read_assignee_filters", + title: "Read assignee filters", + description: "Inspect active assignee and owner filters.", + keywords: ["assignee", "owner", "filters"], + }, + { + slug: "find_blocked_tasks", + title: "Find blocked tasks", + description: "Locate blocked tasks or status badges in the board.", + keywords: ["blocked", "tasks", "status"], + }, + { + slug: "open_meeting_notes", + title: "Open meeting notes", + description: "Open or summarize linked meeting notes.", + keywords: ["meeting", "notes", "agenda"], + }, + { + slug: "read_project_milestones", + title: "Read project milestones", + description: "Inspect milestone labels and delivery checkpoints.", + keywords: ["milestones", "delivery", "project"], + }, + { + slug: "find_status_updates", + title: "Find status updates", + description: "Collect recent project status updates from the UI.", + keywords: ["status", "updates", "project"], + }, + { + slug: "inspect_backlog_filters", + title: "Inspect backlog filters", + description: "Review active backlog filters and search chips.", + keywords: ["backlog", "filters", "search"], + }, + { + slug: "read_capacity_view", + title: "Read capacity view", + description: "Inspect team capacity and planned workload signals.", + keywords: ["capacity", "planning", "workload"], + }, + ], + }, + { + prefix: "commerce", + location: "client", + category: "commerce", + group: "checkout", + profiles: ["commerce", "support"], + immediateCount: 2, + items: [ + { + slug: "read_cart_summary", + title: "Read cart summary", + description: "Read cart totals, quantities, and current items.", + keywords: ["cart", "summary", "totals"], + }, + { + slug: "read_shipping_options", + title: "Read shipping options", + description: "Read current shipping methods and estimates.", + keywords: ["shipping", "delivery", "estimates"], + }, + { + slug: "inspect_promo_field", + title: "Inspect promo field", + description: "Inspect the coupon or promotional code field state.", + keywords: ["promo", "coupon", "discount"], + }, + { + slug: "find_payment_errors", + title: "Find payment errors", + description: + "Check for visible payment failures or validation messages.", + keywords: ["payment", "errors", "checkout"], + }, + { + slug: "read_subscription_selector", + title: "Read subscription selector", + description: "Inspect subscription plans visible in the purchase flow.", + keywords: ["subscription", "plans", "purchase"], + }, + { + slug: "locate_tax_breakdown", + title: "Locate tax breakdown", + description: "Locate tax lines and fee breakdown in checkout.", + keywords: ["tax", "fees", "breakdown"], + }, + { + slug: "capture_return_policy_banner", + title: "Capture return policy banner", + description: + "Capture return or cancellation policy text from the page.", + keywords: ["return", "cancellation", "policy"], + }, + { + slug: "find_saved_cards", + title: "Find saved cards", + description: "Inspect the saved payment methods shown in the UI.", + keywords: ["cards", "payment", "saved"], + }, + { + slug: "read_checkout_steps", + title: "Read checkout steps", + description: "List the current steps in the checkout wizard.", + keywords: ["checkout", "steps", "wizard"], + }, + { + slug: "inspect_order_notes", + title: "Inspect order notes", + description: "Inspect order notes and delivery instructions fields.", + keywords: ["order", "notes", "delivery"], + }, + ], + }, + { + prefix: "analytics", + location: "client", + category: "analytics", + group: "dashboard", + profiles: ["admin", "workspace"], + immediateCount: 2, + items: [ + { + slug: "read_kpi_strip", + title: "Read KPI strip", + description: "Read the top KPI numbers visible in the dashboard.", + keywords: ["kpi", "dashboard", "metrics"], + }, + { + slug: "inspect_chart_legend", + title: "Inspect chart legend", + description: "Inspect chart legends and visible series labels.", + keywords: ["chart", "legend", "series"], + }, + { + slug: "find_date_filters", + title: "Find date filters", + description: "Inspect date-range filters applied in analytics views.", + keywords: ["date", "filters", "analytics"], + }, + { + slug: "read_growth_badges", + title: "Read growth badges", + description: + "Read trend indicators and growth badges in analytics cards.", + keywords: ["growth", "trends", "badges"], + }, + { + slug: "inspect_funnel_steps", + title: "Inspect funnel steps", + description: "Inspect funnel stages and conversion labels.", + keywords: ["funnel", "conversion", "stages"], + }, + { + slug: "capture_table_rows", + title: "Capture table rows", + description: "Capture the visible analytics table rows for analysis.", + keywords: ["table", "rows", "analytics"], + }, + { + slug: "read_segment_chips", + title: "Read segment chips", + description: "Inspect active segments and cohort chips.", + keywords: ["segments", "cohorts", "chips"], + }, + { + slug: "inspect_alert_thresholds", + title: "Inspect alert thresholds", + description: + "Inspect threshold or anomaly settings in analytics widgets.", + keywords: ["alerts", "thresholds", "anomaly"], + }, + { + slug: "find_export_actions", + title: "Find export actions", + description: "Locate CSV, export, or share actions for dashboards.", + keywords: ["export", "csv", "share"], + }, + { + slug: "inspect_dashboard_tabs", + title: "Inspect dashboard tabs", + description: "Inspect top-level tabs and grouped analytics views.", + keywords: ["tabs", "dashboard", "views"], + }, + ], + }, + { + prefix: "utility", + location: "client", + category: "utility", + group: "capture", + profiles: ["support", "workspace", "admin"], + immediateCount: 2, + items: [ + { + slug: "get_browser_locale", + title: "Get browser locale", + description: + "Read the current browser locale and timezone information.", + keywords: ["locale", "timezone", "browser"], + }, + { + slug: "get_window_dimensions", + title: "Get window dimensions", + description: "Read the current viewport and window size.", + keywords: ["viewport", "window", "screen"], + }, + { + slug: "capture_selection_context", + title: "Capture selection context", + description: + "Capture the current text selection and surrounding context.", + keywords: ["selection", "context", "highlight"], + }, + { + slug: "read_clipboard_preview", + title: "Read clipboard preview", + description: "Read clipboard text preview when available.", + keywords: ["clipboard", "copy", "paste"], + }, + { + slug: "capture_console_summary", + title: "Capture console summary", + description: "Capture a lightweight console message summary.", + keywords: ["console", "logs", "errors"], + }, + { + slug: "inspect_network_summary", + title: "Inspect network summary", + description: + "Inspect a lightweight summary of captured network requests.", + keywords: ["network", "requests", "summary"], + }, + { + slug: "read_session_flags", + title: "Read session flags", + description: + "Read temporary session flags relevant to the current page.", + keywords: ["session", "flags", "state"], + }, + { + slug: "capture_page_snapshot", + title: "Capture page snapshot", + description: "Capture a lightweight page snapshot for later reasoning.", + keywords: ["snapshot", "page", "capture"], + }, + { + slug: "inspect_focus_state", + title: "Inspect focus state", + description: "Inspect the currently focused element and nearby labels.", + keywords: ["focus", "element", "labels"], + }, + { + slug: "read_keyboard_shortcuts", + title: "Read keyboard shortcuts", + description: "Read visible keyboard shortcuts or hotkey hints.", + keywords: ["keyboard", "shortcuts", "hotkeys"], + }, + ], + }, +]; + +export const serverToolSeeds = SERVER_CLUSTERS.flatMap(createClusterTools); +export const clientToolSeeds = CLIENT_CLUSTERS.flatMap(createClusterTools); +export const toolScaleSeeds = [...serverToolSeeds, ...clientToolSeeds]; + +export const toolScaleCounts = { + total: toolScaleSeeds.length, + server: serverToolSeeds.length, + client: clientToolSeeds.length, + deferred: toolScaleSeeds.filter((tool) => tool.deferLoading).length, + immediate: toolScaleSeeds.filter((tool) => !tool.deferLoading).length, +}; + +export const toolScaleProfiles = [ + { + id: "support", + label: "Support", + description: + "Customer support, docs, billing, browser inspection, utility capture.", + }, + { + id: "workspace", + label: "Workspace", + description: + "Project, document, scheduling, browser, and analytics collaboration tools.", + }, + { + id: "commerce", + label: "Commerce", + description: "Checkout, pricing, purchase, and customer-flow tools.", + }, + { + id: "admin", + label: "Admin", + description: "Operations, compliance, analytics, and governance tools.", + }, +]; + +export function getProfileToolStats(profile: string) { + const tools = toolScaleSeeds.filter((tool) => + tool.profiles.includes(profile), + ); + return { + total: tools.length, + immediate: tools.filter((tool) => !tool.deferLoading).length, + deferred: tools.filter((tool) => tool.deferLoading).length, + server: tools.filter((tool) => tool.location === "server").length, + client: tools.filter((tool) => tool.location === "client").length, + categories: Array.from(new Set(tools.map((tool) => tool.category))).sort(), + groups: Array.from(new Set(tools.map((tool) => tool.group))).sort(), + }; +} diff --git a/examples/experimental/lib/tool-scale/client-tools.ts b/examples/experimental/lib/tool-scale/client-tools.ts new file mode 100644 index 0000000..9d41a00 --- /dev/null +++ b/examples/experimental/lib/tool-scale/client-tools.ts @@ -0,0 +1,58 @@ +"use client"; + +import type { ToolSet } from "@yourgpt/copilot-sdk/react"; + +import { clientToolSeeds } from "@/lib/tool-scale/catalog"; + +const sharedInputSchema = { + type: "object" as const, + properties: { + task: { + type: "string" as const, + description: "What the tool should help with in this simulation.", + }, + target: { + type: "string" as const, + description: "Optional page element, record, or area of interest.", + }, + }, +}; + +export const toolScaleClientTools: ToolSet = Object.fromEntries( + clientToolSeeds.map((seed) => [ + seed.name, + { + description: seed.description, + location: "client", + category: seed.category, + group: seed.group, + profiles: seed.profiles, + deferLoading: seed.deferLoading, + searchKeywords: seed.searchKeywords, + inputSchema: sharedInputSchema, + handler: async (params) => { + const args = (params ?? {}) as { task?: string; target?: string }; + + return { + success: true, + tool: seed.name, + title: seed.title, + location: seed.location, + category: seed.category, + group: seed.group, + matchedProfiles: seed.profiles, + deferred: seed.deferLoading, + requestedTask: args.task ?? "general assistance", + target: args.target ?? "active page", + browserContext: { + path: + typeof window === "undefined" ? "/" : window.location.pathname, + locale: + typeof navigator === "undefined" ? "unknown" : navigator.language, + }, + summary: `${seed.title} returned a simulated browser-side result for the scale-test example.`, + }; + }, + }, + ]), +) as ToolSet; diff --git a/examples/experimental/lib/tool-scale/server-tools.ts b/examples/experimental/lib/tool-scale/server-tools.ts new file mode 100644 index 0000000..ffc6f06 --- /dev/null +++ b/examples/experimental/lib/tool-scale/server-tools.ts @@ -0,0 +1,51 @@ +import type { ToolDefinition } from "@yourgpt/llm-sdk"; + +import { serverToolSeeds } from "@/lib/tool-scale/catalog"; + +const sharedInputSchema = { + type: "object" as const, + properties: { + task: { + type: "string", + description: "What the tool should help with in this simulation.", + }, + target: { + type: "string", + description: "Optional object, account, page, or entity to inspect.", + }, + }, +}; + +export const toolScaleServerTools: ToolDefinition[] = serverToolSeeds.map( + (seed) => ({ + name: seed.name, + description: seed.description, + location: "server", + category: seed.category, + group: seed.group, + profiles: seed.profiles, + deferLoading: seed.deferLoading, + searchKeywords: seed.searchKeywords, + inputSchema: sharedInputSchema, + handler: async (params) => { + const args = (params ?? {}) as { task?: string; target?: string }; + + return { + tool: seed.name, + title: seed.title, + location: seed.location, + category: seed.category, + group: seed.group, + matchedProfiles: seed.profiles, + deferred: seed.deferLoading, + requestedTask: args.task ?? "general assistance", + target: args.target ?? "current context", + summary: `${seed.title} returned a simulated ${seed.category} response for the scale-test example.`, + guidance: [ + `Use ${seed.title.toLowerCase()} when the user needs ${seed.group} help.`, + `This tool belongs to the ${seed.category} category and is tagged for ${seed.profiles.join(", ")} profiles.`, + ], + }; + }, + }), +); diff --git a/examples/express-demo/README.md b/examples/express-demo/README.md index b60ee63..e4fc352 100644 --- a/examples/express-demo/README.md +++ b/examples/express-demo/README.md @@ -19,6 +19,8 @@ - Error handling - CORS configuration - Request/response streaming +- Tool profiles and selective loading +- Provider-aware tool hints for Anthropic/OpenAI ## Quick Start @@ -75,6 +77,44 @@ export OPENAI_API_KEY=your-api-key-here | `/api/chat/events` | Event handlers | With `on('text', ...)` | | `/api/chat/web` | `toResponse()` | Web Response conversion | +## Tool Selection Demo + +The demo runtime tags tools with profiles and categories, then enables `agentLoop.toolSelection`. + +- Default profile: `support` +- Alternate profile: `utility` +- Tool search over deferred tools: enabled +- Dynamic selection: enabled +- Native hints: + - Anthropic: single-tool preference + no parallel tool use + - OpenAI: single-tool preference + parallel tool calls disabled + +Example request using the utility profile: + +```bash +curl -X POST http://localhost:3001/api/copilot/chat \ + -H "Content-Type: application/json" \ + -d '{ + "toolProfile": "utility", + "messages": [ + { "role": "user", "content": "What time is it on the server?" } + ] + }' +``` + +Example request using the support profile: + +```bash +curl -X POST http://localhost:3001/api/copilot/chat \ + -H "Content-Type: application/json" \ + -d '{ + "toolProfile": "support", + "messages": [ + { "role": "user", "content": "What does the Copilot SDK support?" } + ] + }' +``` + ## Test Commands ### SSE Streaming (default) diff --git a/examples/express-demo/src/index.ts b/examples/express-demo/src/index.ts index 165ca0c..88ba6c7 100644 --- a/examples/express-demo/src/index.ts +++ b/examples/express-demo/src/index.ts @@ -71,6 +71,10 @@ const serverTools: ToolDefinition[] = [ description: "Search the knowledge base for relevant documents. Use this when the user asks questions about YourGPT, the SDK, pricing, features, or how to use the product.", location: "server", + category: "knowledge", + group: "search", + profiles: ["support", "research"], + searchKeywords: ["docs", "pricing", "features", "sdk", "yourgpt"], // HIDDEN: This tool runs silently - user won't see it in the chat UI hidden: true, inputSchema: { @@ -133,6 +137,11 @@ const serverTools: ToolDefinition[] = [ name: "get_current_time", description: "Get the current server time", location: "server", + category: "utility", + group: "time", + profiles: ["utility"], + deferLoading: true, + searchKeywords: ["time", "clock", "timezone", "date"], // VISIBLE: This tool will show in the chat UI (hidden: false is default) inputSchema: { type: "object", @@ -192,6 +201,39 @@ Be helpful, concise, and accurate. If the knowledge base doesn't have the answer enabled: true, maxIterations: 5, debug: true, + toolSelection: { + enabled: true, + defaultProfile: "support", + includeUnprofiled: true, + search: { + enabled: true, + maxResults: 3, + exposeWhenToolCountExceeds: 1, + }, + dynamicSelection: { + enabled: true, + maxTools: 2, + }, + profiles: { + support: { + include: ["category:knowledge", "search_knowledge_base"], + exclude: ["group:time"], + }, + utility: { + include: ["category:utility", "get_current_time"], + }, + }, + nativeProviderHints: { + anthropic: { + toolChoice: "single", + disableParallelToolUse: true, + }, + openai: { + toolChoice: "single", + parallelToolCalls: false, + }, + }, + }, }, }); @@ -234,6 +276,7 @@ app.post("/api/copilot-response/chat", async (req, res) => { app.post("/api/copilot/stream", async (req, res) => { console.log("\n========================================"); console.log("[/api/copilot/stream] SSE streaming request"); + console.log("Tool profile:", req.body.toolProfile || "default"); console.log("Messages:", JSON.stringify(req.body.messages, null, 2)); console.log("========================================\n"); @@ -254,6 +297,7 @@ app.post("/api/copilot/stream", async (req, res) => { app.post("/api/copilot/chat", async (req, res) => { console.log("\n========================================"); console.log("[/api/copilot/chat] Non-streaming request"); + console.log("Tool profile:", req.body.toolProfile || "default"); console.log("Messages:", JSON.stringify(req.body.messages, null, 2)); console.log("========================================\n"); diff --git a/packages/copilot-sdk/src/chat/ChatWithTools.ts b/packages/copilot-sdk/src/chat/ChatWithTools.ts index 555f258..501c6cd 100644 --- a/packages/copilot-sdk/src/chat/ChatWithTools.ts +++ b/packages/copilot-sdk/src/chat/ChatWithTools.ts @@ -12,6 +12,7 @@ */ import type { + ContextUsage, ToolDefinition, MessageAttachment, PermissionLevel, @@ -54,6 +55,8 @@ export interface ChatWithToolsConfig { tools?: ToolDefinition[]; /** Max tool execution iterations (default: 20) */ maxIterations?: number; + /** Optional prompt/tool optimization controls */ + optimization?: ChatConfig["optimization"]; /** Custom error message when max iterations reached (sent to AI as tool result) */ maxIterationsMessage?: string; /** State implementation (injected by framework adapter) */ @@ -70,6 +73,8 @@ export interface ChatWithToolsCallbacks extends ChatCallbacks { onToolExecutionsChange?: (executions: ToolExecution[]) => void; /** Called when a tool requires approval */ onApprovalRequired?: (execution: ToolExecution) => void; + /** Called when prompt context usage changes */ + onContextUsageChange?: (usage: ContextUsage) => void; } /** @@ -132,6 +137,7 @@ export class ChatWithTools { streaming: config.streaming, headers: config.headers, body: config.body, + optimization: config.optimization, threadId: config.threadId, debug: config.debug, initialMessages: config.initialMessages, @@ -146,6 +152,7 @@ export class ChatWithTools { onMessageFinish: callbacks.onMessageFinish, onToolCalls: callbacks.onToolCalls, onFinish: callbacks.onFinish, + onContextUsageChange: callbacks.onContextUsageChange, // Server-side tool callbacks - track in agentLoop for UI display // IMPORTANT: Only track tools that are NOT registered client-side // Client-side tools are tracked via executeToolCalls() path @@ -425,6 +432,28 @@ export class ChatWithTools { this.chat.setTools(tools); } + /** + * Update prompt/tool optimization controls. + */ + setOptimizationConfig(config?: ChatConfig["optimization"]): void { + this.config.optimization = config; + this.chat.setOptimizationConfig(config); + } + + /** + * Set the active tool profile used for request-time tool selection. + */ + setToolProfile(profile?: string): void { + this.chat.setToolProfile(profile); + } + + /** + * Get the most recent prompt context usage snapshot. + */ + getContextUsage(): ContextUsage | null { + return this.chat.getContextUsage(); + } + /** * Set dynamic context (from useAIContext hook) */ diff --git a/packages/copilot-sdk/src/chat/adapters/HttpTransport.ts b/packages/copilot-sdk/src/chat/adapters/HttpTransport.ts index 63a500a..3e74624 100644 --- a/packages/copilot-sdk/src/chat/adapters/HttpTransport.ts +++ b/packages/copilot-sdk/src/chat/adapters/HttpTransport.ts @@ -90,6 +90,7 @@ export class HttpTransport implements ChatTransport { systemPrompt: request.systemPrompt, llm: request.llm, tools: request.tools, + toolCatalog: request.toolCatalog, actions: request.actions, streaming: this.config.streaming, ...(resolved.configBody as Record), diff --git a/packages/copilot-sdk/src/chat/classes/AbstractChat.ts b/packages/copilot-sdk/src/chat/classes/AbstractChat.ts index 2a47223..859f19c 100644 --- a/packages/copilot-sdk/src/chat/classes/AbstractChat.ts +++ b/packages/copilot-sdk/src/chat/classes/AbstractChat.ts @@ -11,10 +11,12 @@ */ import type { + ContextUsage, MessageAttachment, AIResponseMode, ToolResponse, ToolDefinition, + ToolOptimizationConfig, } from "../../core"; import type { ChatState } from "../interfaces/ChatState"; import type { @@ -40,9 +42,9 @@ import { createStreamState, processStreamChunk, isStreamDone, - requiresToolExecution, } from "../functions/stream"; import { SimpleChatState } from "../interfaces/ChatState"; +import { ChatContextOptimizer } from "../optimizations"; // ============================================ // AI Response Control Helper @@ -165,6 +167,8 @@ export class AbstractChat { protected transport: ChatTransport; protected config: ChatConfig; protected callbacks: ChatCallbacks; + protected optimizer: ChatContextOptimizer; + protected lastContextUsage: ContextUsage | null = null; // Event handlers private eventHandlers = new Map< @@ -185,6 +189,7 @@ export class AbstractChat { body: init.body, threadId: init.threadId, debug: init.debug, + optimization: init.optimization, }; // Use provided state or create default @@ -205,6 +210,7 @@ export class AbstractChat { // Store callbacks this.callbacks = init.callbacks ?? {}; + this.optimizer = new ChatContextOptimizer(init.optimization); // Set initial messages if (init.initialMessages?.length) { @@ -557,6 +563,28 @@ export class AbstractChat { this.config.tools = tools; } + /** + * Update prompt/tool optimization behavior. + */ + setOptimizationConfig(config?: ToolOptimizationConfig): void { + this.config.optimization = config; + this.optimizer.updateConfig(config); + } + + /** + * Select the active tool profile for future requests. + */ + setToolProfile(profile?: string): void { + this.optimizer.setActiveProfile(profile); + } + + /** + * Get the most recent prompt context usage snapshot. + */ + getContextUsage(): ContextUsage | null { + return this.lastContextUsage; + } + /** * Dynamic context from useAIContext hook */ @@ -619,102 +647,35 @@ export class AbstractChat { * Build the request payload */ protected buildRequest() { - // Send tools in SDK format - runtime handles conversion to LLM format - // Filter out tools that are marked as unavailable - const tools = this.config.tools - ?.filter((tool) => tool.available !== false) - .map((tool) => ({ - name: tool.name, - description: tool.description, - inputSchema: tool.inputSchema, - })); - - // Build a map of toolCallId -> { toolName, args } from assistant messages - const toolCallMap = new Map< - string, - { toolName: string; args: Record } - >(); - for (const msg of this.state.messages) { - if (msg.role === "assistant" && msg.toolCalls) { - for (const tc of msg.toolCalls) { - try { - const args = tc.function?.arguments - ? JSON.parse(tc.function.arguments) - : {}; - toolCallMap.set(tc.id, { toolName: tc.function.name, args }); - } catch { - toolCallMap.set(tc.id, { toolName: tc.function.name, args: {} }); - } - } - } - } - - // Create a lookup for tool definitions by name - const toolDefMap = new Map(); - if (this.config.tools) { - for (const tool of this.config.tools) { - toolDefMap.set(tool.name, { - name: tool.name, - aiResponseMode: tool.aiResponseMode, - aiContext: tool.aiContext, - }); - } - } + const systemPrompt = this.dynamicContext + ? `${this.config.systemPrompt || ""}\n\n## Current App Context:\n${this.dynamicContext}`.trim() + : this.config.systemPrompt; + const optimized = this.optimizer.prepare({ + messages: this.state.messages, + tools: this.config.tools, + systemPrompt, + }); + this.lastContextUsage = optimized.contextUsage; + this.callbacks.onContextUsageChange?.(optimized.contextUsage); return { - messages: this.state.messages.map((m) => { - // For tool messages, transform based on aiResponseMode at SEND time - // This preserves full data in storage while sending brief to AI - if (m.role === "tool" && m.content && m.toolCallId) { - try { - const fullResult = JSON.parse(m.content); - - // Look up the tool name and args from the tool call - const toolCallInfo = toolCallMap.get(m.toolCallId); - const toolDef = toolCallInfo - ? toolDefMap.get(toolCallInfo.toolName) - : undefined; - const toolArgs = toolCallInfo?.args; - - const transformedContent = buildToolResultContentForAI( - fullResult, - toolDef, - toolArgs, - ); - return { - role: m.role, - content: transformedContent, - tool_call_id: m.toolCallId, - }; - } catch (e) { - // If not JSON, send as-is (log in debug mode) - this.debug("Failed to parse tool message JSON", { - content: m.content?.slice(0, 100), - error: e instanceof Error ? e.message : String(e), - }); - return { - role: m.role, - content: m.content, - tool_call_id: m.toolCallId, - }; - } - } - - // Other messages unchanged - return { - role: m.role, - content: m.content, - tool_calls: m.toolCalls, - tool_call_id: m.toolCallId, - attachments: m.attachments, - }; - }), + messages: optimized.messages, threadId: this.config.threadId, - systemPrompt: this.dynamicContext - ? `${this.config.systemPrompt || ""}\n\n## Current App Context:\n${this.dynamicContext}`.trim() - : this.config.systemPrompt, + systemPrompt, llm: this.config.llm, - tools: tools?.length ? tools : undefined, + tools: optimized.tools?.length ? optimized.tools : undefined, + toolCatalog: this.config.tools?.length + ? this.config.tools.map((tool) => ({ + name: tool.name, + description: tool.description, + category: tool.category, + group: tool.group, + deferLoading: tool.deferLoading, + profiles: tool.profiles, + searchKeywords: tool.searchKeywords, + inputSchema: tool.inputSchema, + })) + : undefined, }; } @@ -856,13 +817,6 @@ export class AbstractChat { this.callbacks.onMessageDelta?.(assistantMessage.id, chunk.content); } - // Check for tool calls - only emit once per stream - if (requiresToolExecution(chunk) && !toolCallsEmitted) { - toolCallsEmitted = true; - this.debug("toolCalls", { toolCalls: updatedMessage.toolCalls }); - this.emit("toolCalls", { toolCalls: updatedMessage.toolCalls }); - } - // Check for completion if (isStreamDone(chunk)) { this.debug("streamDone", { chunk }); @@ -874,6 +828,11 @@ export class AbstractChat { count: chunk.messages.length, }); + const currentStreamToolCallIds = new Set( + this.streamState?.toolCalls?.map((toolCall) => toolCall.id) ?? [], + ); + const messagesToInsert: T[] = []; + // Build hidden map from stream state's toolResults const toolCallsHidden: Record = {}; if (this.streamState?.toolResults) { @@ -885,9 +844,26 @@ export class AbstractChat { } for (const msg of chunk.messages) { - // Skip ALL assistant messages - they're handled via streaming - // (message:end/message:start events create separate messages for each turn) - if (msg.role === "assistant") { + // Skip plain assistant text messages because they are already represented + // by streamed message:start/message:delta/message:end events. Preserve + // assistant messages that carry tool_calls so tool results keep a valid + // preceding assistant tool_call message in local state. + if (msg.role === "assistant" && !msg.tool_calls?.length) { + continue; + } + + // The current streamed turn already becomes an assistant message from + // streamState/tool_calls handling. Skip the duplicate copy from the + // done payload, but keep assistant tool_call messages from earlier + // recursive turns (for example search_tools followed by a later client + // tool call). + if ( + msg.role === "assistant" && + msg.tool_calls?.length && + (msg.tool_calls as Array<{ id: string }>).every((toolCall) => + currentStreamToolCallIds.has(toolCall.id), + ) + ) { continue; } @@ -911,7 +887,40 @@ export class AbstractChat { metadata, } as T; - this.state.pushMessage(message); + messagesToInsert.push(message); + } + + if (messagesToInsert.length > 0) { + const currentMessages = this.state.messages; + const currentStreamIndex = this.streamState + ? currentMessages.findIndex( + (message) => message.id === this.streamState!.messageId, + ) + : -1; + + if (currentStreamIndex === -1) { + this.state.setMessages([...currentMessages, ...messagesToInsert]); + } else { + this.state.setMessages([ + ...currentMessages.slice(0, currentStreamIndex), + ...messagesToInsert, + ...currentMessages.slice(currentStreamIndex), + ]); + } + } + + // Only execute client tools once the full done payload has been + // merged into local state. Emitting earlier on the first tool_calls + // chunk can race with recursive server-tool turns and produce an + // invalid continuation order for OpenAI-compatible providers. + if ( + chunk.requiresAction && + !toolCallsEmitted && + updatedMessage.toolCalls?.length + ) { + toolCallsEmitted = true; + this.debug("toolCalls", { toolCalls: updatedMessage.toolCalls }); + this.emit("toolCalls", { toolCalls: updatedMessage.toolCalls }); } } diff --git a/packages/copilot-sdk/src/chat/index.ts b/packages/copilot-sdk/src/chat/index.ts index f36494e..eddc84b 100644 --- a/packages/copilot-sdk/src/chat/index.ts +++ b/packages/copilot-sdk/src/chat/index.ts @@ -75,6 +75,7 @@ export { AbstractChat, type ChatEvent, type ChatEventHandler } from "./classes"; // AbstractAgentLoop (tool execution) export { AbstractAgentLoop } from "./AbstractAgentLoop"; +export { ChatContextOptimizer } from "./optimizations"; // ChatWithTools (coordinated chat + tools - recommended) export { diff --git a/packages/copilot-sdk/src/chat/interfaces/ChatTransport.ts b/packages/copilot-sdk/src/chat/interfaces/ChatTransport.ts index 59e8b31..9ea76f4 100644 --- a/packages/copilot-sdk/src/chat/interfaces/ChatTransport.ts +++ b/packages/copilot-sdk/src/chat/interfaces/ChatTransport.ts @@ -28,6 +28,8 @@ export interface ChatRequest { llm?: Record; /** Tool definitions */ tools?: unknown[]; + /** Full client-side tool catalog for server-side selection/search */ + toolCatalog?: unknown[]; /** Action definitions */ actions?: unknown[]; /** Additional body properties */ diff --git a/packages/copilot-sdk/src/chat/optimizations.ts b/packages/copilot-sdk/src/chat/optimizations.ts new file mode 100644 index 0000000..9cced3f --- /dev/null +++ b/packages/copilot-sdk/src/chat/optimizations.ts @@ -0,0 +1,1352 @@ +import type { + ContextUsage, + ContextSummarizationConfig, + ToolDefinition, + ToolOptimizationConfig, + ToolResultTruncationConfig, + ToolTruncationStrategy, +} from "../core"; +import type { ChatRequest } from "./interfaces"; +import type { UIMessage } from "./types"; + +const DEFAULT_CHARS_PER_TOKEN = 4; +const DEFAULT_SAFETY_MARGIN = 1.2; +const DEFAULT_INPUT_HEADROOM_RATIO = 0.75; +const DEFAULT_SYSTEM_PROMPT_SHARE = 0.15; +const DEFAULT_HISTORY_SHARE = 0.5; +const DEFAULT_TOOL_RESULTS_SHARE = 0.3; +const DEFAULT_TOOL_DEFINITIONS_SHARE = 0.05; +const DEFAULT_MAX_TOOL_RESULT_CONTEXT_SHARE = 0.3; +const DEFAULT_TOOL_RESULT_HARD_MAX_CHARS = 400_000; +const DEFAULT_TOOL_RESULT_MIN_KEEP_CHARS = 2_000; +const DEFAULT_TOOL_RESULT_STRATEGY: ToolTruncationStrategy = "head-tail"; +const DEFAULT_RECENT_HISTORY_PRESERVE = 6; +const TOOL_RESULT_TRUNCATION_NOTICE = + "\n\n[tool result truncated to fit prompt budget]"; +const TOOL_RESULT_COMPACTION_NOTICE = + "[tool result compacted to preserve context budget]"; +const SYSTEM_PROMPT_TRUNCATION_NOTICE = + "\n\n[system prompt truncated to fit prompt budget]"; +const HISTORY_SUMMARY_HEADER = "Conversation summary of earlier context:"; +const HISTORY_SUMMARY_COMPACTION_NOTICE = + "\n\n[summary compacted to preserve context continuity]"; +const DEFAULT_SUMMARY_TRIGGER = 12; +const DEFAULT_SUMMARY_CHUNK_SIZE = 10; +const DEFAULT_SUMMARY_MAX_CHARS = 1_600; +const SUMMARY_STOP_WORDS = new Set([ + "about", + "after", + "again", + "also", + "because", + "been", + "before", + "being", + "could", + "from", + "have", + "into", + "just", + "more", + "need", + "only", + "over", + "same", + "some", + "than", + "that", + "their", + "them", + "then", + "there", + "these", + "they", + "this", + "those", + "through", + "under", + "very", + "want", + "were", + "what", + "when", + "where", + "which", + "while", + "with", + "would", + "your", +]); + +type RequestMessage = ChatRequest["messages"][number]; +type RequestTool = { + name: string; + description: string; + category?: string; + group?: string; + deferLoading?: boolean; + profiles?: string[]; + searchKeywords?: string[]; + inputSchema: unknown; +}; + +type PreparedBuckets = { + systemPrompt: string | undefined; + transformedMessages: RequestMessage[]; + historyMessages: RequestMessage[]; + toolResultMessages: RequestMessage[]; + requestTools: RequestTool[] | undefined; +}; + +function clampRatio(value: number | undefined, fallback: number): number { + if (!Number.isFinite(value)) { + return fallback; + } + return Math.min(1, Math.max(0, value as number)); +} + +function unique(values: T[]): T[] { + return [...new Set(values)]; +} + +function stringifyContent(value: unknown): string { + if (typeof value === "string") { + return value; + } + if (value == null) { + return ""; + } + try { + return JSON.stringify(value); + } catch { + return String(value); + } +} + +function normalizeWhitespace(text: string): string { + return text.replace(/\s+/g, " ").trim(); +} + +function abbreviateText(text: string, maxChars = 220): string { + const normalized = normalizeWhitespace(text); + if (!normalized) { + return ""; + } + if (normalized.length <= maxChars) { + return normalized; + } + return `${normalized.slice(0, Math.max(1, maxChars - 3)).trimEnd()}...`; +} + +function tokenize(text: string): string[] { + return text + .toLowerCase() + .replace(/[^a-z0-9_\s-]/g, " ") + .split(/\s+/) + .filter((token) => token.length > 1); +} + +function estimateTokens( + text: string, + charsPerToken = DEFAULT_CHARS_PER_TOKEN, +): number { + if (!text) { + return 0; + } + return Math.ceil(text.length / Math.max(1, charsPerToken)); +} + +function estimateMessageTokens( + message: RequestMessage, + charsPerToken = DEFAULT_CHARS_PER_TOKEN, +): number { + const content = + typeof message.content === "string" + ? message.content + : JSON.stringify(message.content ?? ""); + const toolCalls = message.tool_calls + ? JSON.stringify(message.tool_calls) + : ""; + const attachments = message.attachments + ? JSON.stringify(message.attachments) + : ""; + return estimateTokens( + `${message.role}\n${content}\n${toolCalls}\n${attachments}`, + charsPerToken, + ); +} + +function estimateToolTokens( + tool: RequestTool, + charsPerToken = DEFAULT_CHARS_PER_TOKEN, +): number { + return estimateTokens(JSON.stringify(tool), charsPerToken); +} + +function buildToolQuery(messages: UIMessage[]): string { + return messages + .filter( + (message) => message.role === "user" || message.role === "assistant", + ) + .slice(-3) + .map((message) => message.content) + .filter(Boolean) + .join(" "); +} + +function matchesSelector( + tool: ToolDefinition, + selector: string, + activeProfile?: string, +): boolean { + const normalized = selector.trim().toLowerCase(); + if (!normalized) { + return false; + } + + if (normalized === "*" || normalized === "all") { + return true; + } + if (normalized === tool.name.toLowerCase()) { + return true; + } + if (normalized.startsWith("group:")) { + return (tool.group ?? "").toLowerCase() === normalized.slice(6); + } + if (normalized.startsWith("category:")) { + return (tool.category ?? "").toLowerCase() === normalized.slice(9); + } + if (normalized.startsWith("profile:")) { + return (tool.profiles ?? []) + .map((value) => value.toLowerCase()) + .includes(normalized.slice(8)); + } + if (activeProfile && normalized === activeProfile.toLowerCase()) { + return (tool.profiles ?? []) + .map((value) => value.toLowerCase()) + .includes(normalized); + } + return false; +} + +function scoreTool( + tool: ToolDefinition, + queryTokens: string[], + activeProfile?: string, +): number { + const haystack = [ + tool.name, + tool.description, + tool.category, + tool.group, + ...(tool.profiles ?? []), + ...(tool.searchKeywords ?? []), + ] + .filter(Boolean) + .join(" ") + .toLowerCase(); + + let score = tool.deferLoading ? 0 : 2; + if (activeProfile && tool.profiles?.includes(activeProfile)) { + score += 2; + } + for (const token of queryTokens) { + if (tool.name.toLowerCase() === token) { + score += 6; + } else if (tool.name.toLowerCase().includes(token)) { + score += 4; + } else if (haystack.includes(token)) { + score += 2; + } + } + return score; +} + +function truncateText( + text: string, + maxChars: number, + strategy: ToolTruncationStrategy, + notice = TOOL_RESULT_TRUNCATION_NOTICE, +): string { + if (text.length <= maxChars) { + return text; + } + + const bodyBudget = Math.max(1, maxChars - notice.length); + if (strategy === "head") { + return text.slice(0, bodyBudget) + notice; + } + + if (strategy === "head-tail" || strategy === "smart") { + const tailLooksImportant = + strategy === "smart" + ? /\b(error|exception|failed|traceback|summary|result|done|complete)\b/i.test( + text.slice(-2000), + ) + : true; + if (tailLooksImportant && bodyBudget > 32) { + const tailBudget = Math.min(Math.floor(bodyBudget * 0.3), 4_000); + const headBudget = Math.max(1, bodyBudget - tailBudget - 32); + return ( + text.slice(0, headBudget) + + "\n\n[... omitted ...]\n\n" + + text.slice(-tailBudget) + + notice + ); + } + } + + return text.slice(0, bodyBudget) + notice; +} + +function isHistorySummaryMessage(message: RequestMessage): boolean { + return ( + message.role === "system" && + typeof message.content === "string" && + message.content.startsWith(HISTORY_SUMMARY_HEADER) + ); +} + +function collectTopKeywords(messages: RequestMessage[]): string[] { + const counts = new Map(); + for (const message of messages) { + if (message.role !== "user") { + continue; + } + for (const token of tokenize(stringifyContent(message.content))) { + if (token.length < 3 || SUMMARY_STOP_WORDS.has(token)) { + continue; + } + counts.set(token, (counts.get(token) ?? 0) + 1); + } + } + + return [...counts.entries()] + .sort((left, right) => { + const countDiff = right[1] - left[1]; + if (countDiff !== 0) { + return countDiff; + } + return left[0].localeCompare(right[0]); + }) + .slice(0, 5) + .map(([token]) => token); +} + +function collectToolCallNames(messages: RequestMessage[]): Map { + const toolCallNames = new Map(); + for (const message of messages) { + if (message.role !== "assistant" || !message.tool_calls?.length) { + continue; + } + for (const toolCall of message.tool_calls) { + const parsedToolCall = toolCall as { + id?: string; + function?: { name?: string }; + }; + if (parsedToolCall.id && parsedToolCall.function?.name) { + toolCallNames.set(parsedToolCall.id, parsedToolCall.function.name); + } + } + } + return toolCallNames; +} + +function compressSummaryContent( + content: string, + maxChars: number, + fallbackBehavior: ContextSummarizationConfig["fallbackBehavior"] | undefined, +): string { + if (content.length <= maxChars) { + return content; + } + if (fallbackBehavior === "error") { + throw new Error("History summary exceeded configured continuity budget."); + } + if (fallbackBehavior === "statistical") { + const lines = content.split("\n"); + const retained = lines.filter((line) => + /^(Conversation summary|Stats:|- Messages compacted:|- User turns compacted:|- Assistant turns compacted:|- Tool results compacted:|- Latest user request before preserved window:|- Latest assistant response before preserved window:)/.test( + line, + ), + ); + const statistical = retained.join("\n"); + if (statistical.length <= maxChars) { + return statistical; + } + } + return truncateText( + content, + maxChars, + "head", + HISTORY_SUMMARY_COMPACTION_NOTICE, + ); +} + +function buildHistorySummary( + messages: RequestMessage[], + summarization?: ContextSummarizationConfig, + maxChars = DEFAULT_SUMMARY_MAX_CHARS, +): RequestMessage | null { + if (messages.length === 0) { + return null; + } + + if (summarization?.enabled === false) { + return null; + } + + const previousSummaries = messages.filter(isHistorySummaryMessage); + const rawMessages = messages.filter( + (message) => !isHistorySummaryMessage(message), + ); + const focusWindowSize = Math.max( + 1, + summarization?.chunkSize ?? DEFAULT_SUMMARY_CHUNK_SIZE, + ); + const detailedThreshold = Math.max( + 1, + summarization?.triggerAt ?? DEFAULT_SUMMARY_TRIGGER, + ); + const focusMessages = rawMessages.slice(-focusWindowSize); + const userMessages = rawMessages.filter((message) => message.role === "user"); + const assistantMessages = rawMessages.filter( + (message) => message.role === "assistant", + ); + const toolMessages = rawMessages.filter((message) => message.role === "tool"); + const recentUser = + abbreviateText(stringifyContent(userMessages.at(-1)?.content), 240) || + "n/a"; + const recentAssistant = + abbreviateText(stringifyContent(assistantMessages.at(-1)?.content), 240) || + "n/a"; + const recentUserGoals = userMessages + .slice(-3) + .map((message) => abbreviateText(stringifyContent(message.content), 180)) + .filter(Boolean); + const recentAssistantNotes = assistantMessages + .map((message) => abbreviateText(stringifyContent(message.content), 180)) + .filter(Boolean) + .slice(-2); + const toolCallNames = collectToolCallNames(rawMessages); + const toolActivity = unique([ + ...focusMessages + .filter((message) => message.role === "assistant") + .flatMap((message) => + (message.tool_calls ?? []) + .map((toolCall) => { + const parsedToolCall = toolCall as { + function?: { name?: string }; + }; + return parsedToolCall.function?.name; + }) + .filter((name): name is string => Boolean(name)), + ), + ...toolMessages + .slice(-2) + .map((message) => { + const toolName = message.tool_call_id + ? toolCallNames.get(message.tool_call_id) + : undefined; + const snippet = abbreviateText(stringifyContent(message.content), 120); + return toolName && snippet + ? `${toolName}: ${snippet}` + : (toolName ?? snippet); + }) + .filter(Boolean), + ]).slice(-4); + const priorSummaryCarryForward = previousSummaries + .map((message) => + abbreviateText( + stringifyContent(message.content).replace( + `${HISTORY_SUMMARY_HEADER}\n`, + "", + ), + 180, + ), + ) + .filter(Boolean) + .slice(-2); + const topKeywords = + rawMessages.length >= detailedThreshold + ? collectTopKeywords( + focusMessages.length > 0 ? focusMessages : rawMessages, + ) + : []; + + const lines = [ + HISTORY_SUMMARY_HEADER, + "Stats:", + `- Messages compacted: ${messages.length}`, + `- User turns compacted: ${userMessages.length}`, + `- Assistant turns compacted: ${assistantMessages.length}`, + `- Tool results compacted: ${toolMessages.length}`, + ]; + + if (previousSummaries.length > 0) { + lines.push(`- Previous summaries merged: ${previousSummaries.length}`); + } + if (rawMessages.length > focusMessages.length) { + lines.push( + `- Older compacted messages outside the detailed window: ${rawMessages.length - focusMessages.length}`, + ); + } + if (topKeywords.length > 0) { + lines.push(`- Recurring user topics: ${topKeywords.join(", ")}`); + } + if (recentUser !== "n/a") { + lines.push(`- Latest user request before preserved window: ${recentUser}`); + } + if (recentAssistant !== "n/a") { + lines.push( + `- Latest assistant response before preserved window: ${recentAssistant}`, + ); + } + if (recentUserGoals.length > 0) { + lines.push("Carry forward user goals:"); + for (const goal of recentUserGoals) { + lines.push(`- ${goal}`); + } + } + if (recentAssistantNotes.length > 0) { + lines.push("Carry forward assistant commitments:"); + for (const note of recentAssistantNotes) { + lines.push(`- ${note}`); + } + } + if (toolActivity.length > 0) { + lines.push("Recent tool activity:"); + for (const item of toolActivity) { + lines.push(`- ${item}`); + } + } + if (priorSummaryCarryForward.length > 0) { + lines.push("Earlier carried-forward context:"); + for (const item of priorSummaryCarryForward) { + lines.push(`- ${item}`); + } + } + + const content = compressSummaryContent( + lines.join("\n"), + maxChars, + summarization?.fallbackBehavior, + ); + + return { + role: "system", + content, + }; +} + +function buildToolDefinitions( + selectedTools: ToolDefinition[], +): RequestTool[] | undefined { + if (selectedTools.length === 0) { + return undefined; + } + + return selectedTools.map((tool) => ({ + name: tool.name, + description: tool.description, + category: tool.category, + group: tool.group, + deferLoading: tool.deferLoading, + profiles: tool.profiles, + searchKeywords: tool.searchKeywords, + inputSchema: tool.inputSchema, + })); +} + +function resolveTruncationConfig(params: { + tool?: ToolDefinition; + config?: ToolOptimizationConfig; +}): Required { + const charsPerToken = + params.config?.contextManagement?.tokenEstimation?.charsPerToken ?? + DEFAULT_CHARS_PER_TOKEN; + const contextWindowTokens = + params.config?.contextBudget?.budget?.contextWindowTokens; + const globalConfig = params.config?.toolResultConfig?.truncation; + const perToolConfig = params.tool?.resultConfig?.truncation; + const merged = { ...globalConfig, ...perToolConfig }; + const hardMaxChars = + merged.hardMaxChars ?? + (contextWindowTokens + ? Math.floor( + contextWindowTokens * + clampRatio( + merged.maxContextShare, + DEFAULT_MAX_TOOL_RESULT_CONTEXT_SHARE, + ) * + charsPerToken, + ) + : DEFAULT_TOOL_RESULT_HARD_MAX_CHARS); + + return { + enabled: merged.enabled ?? true, + maxContextShare: clampRatio( + merged.maxContextShare, + DEFAULT_MAX_TOOL_RESULT_CONTEXT_SHARE, + ), + hardMaxChars: Math.max(1, hardMaxChars), + minKeepChars: Math.max( + 256, + merged.minKeepChars ?? DEFAULT_TOOL_RESULT_MIN_KEEP_CHARS, + ), + strategy: merged.strategy ?? DEFAULT_TOOL_RESULT_STRATEGY, + preserveErrors: merged.preserveErrors ?? true, + }; +} + +function buildToolResultContent( + result: unknown, + tool?: ToolDefinition, + args?: Record, +): string { + if (typeof result === "string") { + return result; + } + + const typedResult = (result ?? null) as + | ({ + _aiResponseMode?: "none" | "brief" | "full"; + _aiContext?: string; + _aiContent?: unknown; + _uiResources?: unknown; + } & Record) + | null; + const responseMode = + typedResult?._aiResponseMode ?? tool?.aiResponseMode ?? "full"; + + if (typedResult?._aiContent) { + return JSON.stringify(typedResult._aiContent); + } + + let aiContext = typedResult?._aiContext; + if (!aiContext && tool?.aiContext) { + aiContext = + typeof tool.aiContext === "function" + ? tool.aiContext( + (typedResult ?? { success: true }) as never, + args ?? {}, + ) + : tool.aiContext; + } + + switch (responseMode) { + case "none": + return aiContext ?? "[Result displayed to user]"; + case "brief": + return aiContext ?? "[Tool executed successfully]"; + case "full": + default: { + if (aiContext) { + const { + _aiResponseMode, + _aiContext, + _aiContent, + _uiResources, + ...dataOnly + } = typedResult ?? {}; + return `${aiContext}\n\nFull data: ${JSON.stringify(dataOnly)}`; + } + if (typedResult?._uiResources) { + const { _uiResources, ...dataOnly } = typedResult; + return JSON.stringify(dataOnly); + } + return JSON.stringify(result); + } + } +} + +export function buildToolResultContentForPrompt( + result: unknown, + tool: ToolDefinition | undefined, + args: Record, + config: ToolOptimizationConfig | undefined, +): string { + const text = buildToolResultContent(result, tool, args); + const truncation = resolveTruncationConfig({ tool, config }); + if (!truncation.enabled) { + return text; + } + + if ( + truncation.preserveErrors && + typeof result === "object" && + result !== null && + "error" in result && + typeof (result as { error?: unknown }).error === "string" + ) { + return text; + } + + const maxChars = Math.max(truncation.minKeepChars, truncation.hardMaxChars); + return truncateText(text, maxChars, truncation.strategy); +} + +function sliceHistoryToMaxMessages(params: { + historyMessages: RequestMessage[]; + maxMessages: number | undefined; + pruneStrategy: "oldest" | "least-relevant" | "summarize" | undefined; + summarization?: ContextSummarizationConfig; +}): RequestMessage[] { + const { historyMessages, maxMessages, pruneStrategy, summarization } = params; + if (!maxMessages || historyMessages.length <= maxMessages) { + return historyMessages; + } + + const dropped = historyMessages.slice( + 0, + historyMessages.length - maxMessages, + ); + const kept = historyMessages.slice(-maxMessages); + if (pruneStrategy === "summarize") { + const summary = buildHistorySummary(dropped, summarization); + return summary ? [summary, ...kept] : kept; + } + return kept; +} + +function compactHistoryToTokenBudget(params: { + historyMessages: RequestMessage[]; + maxTokens: number | undefined; + preserveRecent: number; + charsPerToken: number; + pruneStrategy: "oldest" | "least-relevant" | "summarize" | undefined; + summarization?: ContextSummarizationConfig; +}): RequestMessage[] { + const { + maxTokens, + preserveRecent, + charsPerToken, + pruneStrategy, + summarization, + } = params; + let historyMessages = params.historyMessages; + if (!maxTokens) { + return historyMessages; + } + + const getHistoryTokens = () => + historyMessages.reduce( + (sum, message) => sum + estimateMessageTokens(message, charsPerToken), + 0, + ); + + while (historyMessages.length > 1 && getHistoryTokens() > maxTokens) { + const prunableCount = Math.max(0, historyMessages.length - preserveRecent); + if (prunableCount <= 0) { + const firstMessage = historyMessages[0]; + if ( + isHistorySummaryMessage(firstMessage) && + typeof firstMessage.content === "string" + ) { + const compactedSummary = compressSummaryContent( + firstMessage.content, + Math.max(400, Math.floor(maxTokens * charsPerToken * 0.25)), + summarization?.fallbackBehavior, + ); + if (compactedSummary !== firstMessage.content) { + historyMessages = [ + { ...firstMessage, content: compactedSummary }, + ...historyMessages.slice(1), + ]; + continue; + } + } + historyMessages = historyMessages.slice(1); + continue; + } + + const pruned = historyMessages.slice(0, prunableCount); + const kept = historyMessages.slice(prunableCount); + if (pruneStrategy === "summarize") { + const summary = buildHistorySummary( + pruned, + summarization, + Math.max(500, Math.floor(maxTokens * charsPerToken * 0.35)), + ); + historyMessages = summary ? [summary, ...kept] : kept; + } else { + historyMessages = kept; + } + } + + return historyMessages; +} + +function compactToolResultsToBudget(params: { + toolResultMessages: RequestMessage[]; + maxTokens: number | undefined; + charsPerToken: number; +}): RequestMessage[] { + let toolResultMessages = params.toolResultMessages; + if (!params.maxTokens) { + return toolResultMessages; + } + + const getToolResultTokens = () => + toolResultMessages.reduce( + (sum, message) => + sum + estimateMessageTokens(message, params.charsPerToken), + 0, + ); + + while ( + toolResultMessages.length > 0 && + getToolResultTokens() > params.maxTokens + ) { + const index = toolResultMessages.findIndex( + (message) => message.content !== TOOL_RESULT_COMPACTION_NOTICE, + ); + if (index === -1) { + break; + } + + toolResultMessages = toolResultMessages.map((message, currentIndex) => + currentIndex === index + ? { ...message, content: TOOL_RESULT_COMPACTION_NOTICE } + : message, + ); + } + + return toolResultMessages; +} + +function fitToolsToBudget(params: { + tools: RequestTool[] | undefined; + maxTokens: number | undefined; + charsPerToken: number; +}): RequestTool[] | undefined { + let tools = params.tools; + if (!tools?.length || !params.maxTokens) { + return tools; + } + + const getToolTokens = () => + tools!.reduce( + (sum, tool) => sum + estimateToolTokens(tool, params.charsPerToken), + 0, + ); + + while (tools.length > 0 && getToolTokens() > params.maxTokens) { + tools = tools.slice(0, -1); + } + + return tools; +} + +function truncateSystemPromptToBudget(params: { + systemPrompt: string | undefined; + maxTokens: number | undefined; + charsPerToken: number; +}): string | undefined { + const { systemPrompt, maxTokens, charsPerToken } = params; + if (!systemPrompt || !maxTokens) { + return systemPrompt; + } + + const maxChars = maxTokens * charsPerToken; + if (systemPrompt.length <= maxChars) { + return systemPrompt; + } + + return truncateText( + systemPrompt, + maxChars, + "head", + SYSTEM_PROMPT_TRUNCATION_NOTICE, + ); +} + +function calculateBuckets(params: { + systemPrompt: string | undefined; + historyMessages: RequestMessage[]; + toolResultMessages: RequestMessage[]; + requestTools: RequestTool[] | undefined; + charsPerToken: number; + availableBudget: number; + warnings: string[]; +}): ContextUsage { + const systemPromptTokens = estimateTokens( + params.systemPrompt ?? "", + params.charsPerToken, + ); + const historyTokens = params.historyMessages.reduce( + (sum, message) => + sum + estimateMessageTokens(message, params.charsPerToken), + 0, + ); + const toolResultsTokens = params.toolResultMessages.reduce( + (sum, message) => + sum + estimateMessageTokens(message, params.charsPerToken), + 0, + ); + const toolDefinitionTokens = (params.requestTools ?? []).reduce( + (sum, tool) => sum + estimateToolTokens(tool, params.charsPerToken), + 0, + ); + const total = + systemPromptTokens + + historyTokens + + toolResultsTokens + + toolDefinitionTokens; + const budget = Number.isFinite(params.availableBudget) + ? params.availableBudget + : total; + const toPart = (tokens: number) => ({ + tokens, + percent: budget > 0 ? Number(((tokens / budget) * 100).toFixed(2)) : 0, + }); + + return { + total: toPart(total), + breakdown: { + systemPrompt: toPart(systemPromptTokens), + history: toPart(historyTokens), + toolResults: toPart(toolResultsTokens), + tools: toPart(toolDefinitionTokens), + }, + budget: { + available: budget, + remaining: Math.max(0, budget - total), + }, + warnings: unique(params.warnings), + }; +} + +function mergeBucketsInOriginalOrder(params: { + transformedMessages: RequestMessage[]; + historyMessages: RequestMessage[]; + toolResultMessages: RequestMessage[]; +}): RequestMessage[] { + const historyQueue = [...params.historyMessages]; + const toolQueue = [...params.toolResultMessages]; + + return params.transformedMessages.flatMap((message) => { + if (message.role === "tool") { + const nextTool = toolQueue.shift(); + return nextTool ? [nextTool] : []; + } + const nextHistory = historyQueue.shift(); + return nextHistory ? [nextHistory] : []; + }); +} + +export class ChatContextOptimizer { + private config: ToolOptimizationConfig | undefined; + private activeProfile: string | undefined; + private lastContextUsage: ContextUsage | null = null; + + constructor(config?: ToolOptimizationConfig) { + this.config = config; + this.activeProfile = config?.toolProfiles?.defaultProfile; + } + + updateConfig(config?: ToolOptimizationConfig): void { + this.config = config; + if (!this.activeProfile) { + this.activeProfile = config?.toolProfiles?.defaultProfile; + } + } + + setActiveProfile(profile?: string): void { + this.activeProfile = profile?.trim() || undefined; + } + + getContextUsage(): ContextUsage | null { + return this.lastContextUsage; + } + + prepare(params: { + messages: UIMessage[]; + tools?: ToolDefinition[]; + systemPrompt?: string; + }): { + messages: RequestMessage[]; + tools?: RequestTool[]; + contextUsage: ContextUsage; + warnings: string[]; + } { + const charsPerToken = + this.config?.contextManagement?.tokenEstimation?.charsPerToken ?? + DEFAULT_CHARS_PER_TOKEN; + const safetyMargin = + this.config?.contextManagement?.tokenEstimation?.safetyMargin ?? + DEFAULT_SAFETY_MARGIN; + const warnings: string[] = []; + const contextManagement = this.config?.contextManagement; + const contextBudget = this.config?.contextBudget; + const allTools = params.tools ?? []; + const selectedTools = this.selectTools(allTools, params.messages); + const transformedMessages = this.transformMessages( + params.messages, + allTools, + ); + const preserveRecent = + contextManagement?.summarization?.preserveRecent ?? + DEFAULT_RECENT_HISTORY_PRESERVE; + + let buckets: PreparedBuckets = { + systemPrompt: params.systemPrompt, + transformedMessages, + historyMessages: transformedMessages.filter( + (message) => message.role !== "tool", + ), + toolResultMessages: transformedMessages.filter( + (message) => message.role === "tool", + ), + requestTools: buildToolDefinitions(selectedTools), + }; + + if (contextManagement?.enabled) { + buckets.historyMessages = sliceHistoryToMaxMessages({ + historyMessages: buckets.historyMessages, + maxMessages: contextManagement.history?.maxMessages, + pruneStrategy: contextManagement.history?.pruneStrategy, + summarization: contextManagement?.summarization, + }); + } + + const budgetConfig = contextBudget?.budget; + const contextWindowTokens = budgetConfig?.contextWindowTokens; + const inputHeadroomRatio = clampRatio( + budgetConfig?.inputHeadroomRatio, + DEFAULT_INPUT_HEADROOM_RATIO, + ); + const availableBudget = contextWindowTokens + ? Math.max(1, Math.floor(contextWindowTokens * inputHeadroomRatio)) + : Number.POSITIVE_INFINITY; + + const sharedBudget = Number.isFinite(availableBudget) + ? availableBudget + : undefined; + const systemPromptBudget = sharedBudget + ? Math.max( + 1, + Math.floor( + sharedBudget * + clampRatio( + budgetConfig?.systemPromptShare, + DEFAULT_SYSTEM_PROMPT_SHARE, + ), + ), + ) + : undefined; + const historyBudgetByShare = sharedBudget + ? Math.max( + 1, + Math.floor( + sharedBudget * + clampRatio(budgetConfig?.historyShare, DEFAULT_HISTORY_SHARE), + ), + ) + : undefined; + const historyBudgetByConfig = + contextManagement?.enabled && contextManagement.history?.maxTokens + ? Math.floor(contextManagement.history.maxTokens / safetyMargin) + : undefined; + const historyBudget = + historyBudgetByShare && historyBudgetByConfig + ? Math.min(historyBudgetByShare, historyBudgetByConfig) + : (historyBudgetByShare ?? historyBudgetByConfig); + const toolResultsBudget = sharedBudget + ? Math.max( + 1, + Math.floor( + sharedBudget * + clampRatio( + budgetConfig?.toolResultsShare, + DEFAULT_TOOL_RESULTS_SHARE, + ), + ), + ) + : undefined; + const toolDefinitionsBudget = sharedBudget + ? Math.max( + 1, + Math.floor( + sharedBudget * + clampRatio( + budgetConfig?.toolDefinitionsShare, + DEFAULT_TOOL_DEFINITIONS_SHARE, + ), + ), + ) + : undefined; + + if (contextBudget?.enabled) { + buckets.systemPrompt = truncateSystemPromptToBudget({ + systemPrompt: buckets.systemPrompt, + maxTokens: systemPromptBudget, + charsPerToken, + }); + } + + buckets.historyMessages = compactHistoryToTokenBudget({ + historyMessages: buckets.historyMessages, + maxTokens: historyBudget, + preserveRecent, + charsPerToken, + pruneStrategy: contextManagement?.history?.pruneStrategy, + summarization: contextManagement?.summarization, + }); + + buckets.toolResultMessages = compactToolResultsToBudget({ + toolResultMessages: buckets.toolResultMessages, + maxTokens: toolResultsBudget, + charsPerToken, + }); + + buckets.requestTools = fitToolsToBudget({ + tools: buckets.requestTools, + maxTokens: toolDefinitionsBudget, + charsPerToken, + }); + + let usage = calculateBuckets({ + ...buckets, + charsPerToken, + availableBudget, + warnings, + }); + + if ( + Number.isFinite(availableBudget) && + usage.total.tokens > availableBudget + ) { + // Final global fallback: preserve recent history, compact tool results first, then trim history and tools. + buckets.toolResultMessages = compactToolResultsToBudget({ + toolResultMessages: buckets.toolResultMessages, + maxTokens: Math.max( + 1, + usage.breakdown.toolResults.tokens - + usage.total.tokens + + availableBudget, + ), + charsPerToken, + }); + + usage = calculateBuckets({ + ...buckets, + charsPerToken, + availableBudget, + warnings, + }); + + if (usage.total.tokens > availableBudget) { + const overflow = usage.total.tokens - availableBudget; + buckets.historyMessages = compactHistoryToTokenBudget({ + historyMessages: buckets.historyMessages, + maxTokens: Math.max(1, usage.breakdown.history.tokens - overflow), + preserveRecent, + charsPerToken, + pruneStrategy: contextManagement?.history?.pruneStrategy, + }); + usage = calculateBuckets({ + ...buckets, + charsPerToken, + availableBudget, + warnings, + }); + } + + if (usage.total.tokens > availableBudget) { + buckets.requestTools = fitToolsToBudget({ + tools: buckets.requestTools, + maxTokens: Math.max( + 1, + usage.breakdown.tools.tokens - + (usage.total.tokens - availableBudget), + ), + charsPerToken, + }); + usage = calculateBuckets({ + ...buckets, + charsPerToken, + availableBudget, + warnings, + }); + } + } + + if ( + Number.isFinite(availableBudget) && + usage.total.tokens > availableBudget + ) { + warnings.push( + `Prompt budget exceeded: using ${usage.total.tokens} tokens of ${availableBudget}.`, + ); + usage = { + ...usage, + warnings: unique(warnings), + }; + if (contextBudget?.enforcement?.mode === "error") { + throw new Error(warnings[warnings.length - 1]); + } + contextBudget?.enforcement?.onBudgetExceeded?.(usage); + } else { + usage = { + ...usage, + warnings: unique(warnings), + }; + } + + contextBudget?.monitoring?.onUsageUpdate?.(usage); + this.lastContextUsage = usage; + + return { + messages: mergeBucketsInOriginalOrder(buckets), + tools: buckets.requestTools, + contextUsage: usage, + warnings: usage.warnings, + }; + } + + private selectTools( + tools: ToolDefinition[], + messages: UIMessage[], + ): ToolDefinition[] { + if (!tools.length) { + return []; + } + + const available = tools.filter((tool) => tool.available !== false); + const profileConfig = this.config?.toolProfiles; + if (!profileConfig?.enabled) { + return available; + } + + const activeProfile = this.activeProfile ?? profileConfig.defaultProfile; + const includeUnprofiled = profileConfig.includeUnprofiled ?? true; + const profile = activeProfile + ? profileConfig.profiles?.[activeProfile] + : undefined; + let filtered = available; + + if (profile?.include?.length) { + filtered = filtered.filter( + (tool) => + profile.include!.some((selector) => + matchesSelector(tool, selector, activeProfile), + ) || + (!!activeProfile && tool.profiles?.includes(activeProfile)), + ); + } else if (activeProfile) { + filtered = filtered.filter((tool) => { + if (tool.profiles?.length) { + return tool.profiles.includes(activeProfile); + } + return includeUnprofiled; + }); + } + + if (profile?.exclude?.length) { + filtered = filtered.filter( + (tool) => + !profile.exclude!.some((selector) => + matchesSelector(tool, selector, activeProfile), + ), + ); + } + + if (!profileConfig.dynamicSelection?.enabled) { + return filtered; + } + + const maxTools = Math.max( + 1, + Math.min( + profileConfig.dynamicSelection.maxTools ?? filtered.length, + filtered.length, + ), + ); + const queryTokens = tokenize(buildToolQuery(messages)); + return [...filtered] + .sort((left, right) => { + const scoreDiff = + scoreTool(right, queryTokens, activeProfile) - + scoreTool(left, queryTokens, activeProfile); + if (scoreDiff !== 0) { + return scoreDiff; + } + return left.name.localeCompare(right.name); + }) + .slice(0, maxTools); + } + + private transformMessages( + messages: UIMessage[], + allTools: ToolDefinition[], + ): RequestMessage[] { + const toolCallMap = new Map< + string, + { toolName: string; args: Record } + >(); + for (const message of messages) { + if (message.role !== "assistant" || !message.toolCalls?.length) { + continue; + } + for (const toolCall of message.toolCalls) { + try { + toolCallMap.set(toolCall.id, { + toolName: toolCall.function.name, + args: JSON.parse(toolCall.function.arguments), + }); + } catch { + toolCallMap.set(toolCall.id, { + toolName: toolCall.function.name, + args: {}, + }); + } + } + } + + const toolDefMap = new Map( + allTools.map((tool) => [tool.name, tool] as const), + ); + + return messages.map((message) => { + if (message.role !== "tool") { + return { + role: message.role, + content: message.content, + tool_calls: message.toolCalls, + tool_call_id: message.toolCallId, + attachments: message.attachments, + }; + } + + const toolCall = message.toolCallId + ? toolCallMap.get(message.toolCallId) + : undefined; + const tool = toolCall ? toolDefMap.get(toolCall.toolName) : undefined; + let content = message.content; + + try { + const parsed = JSON.parse(message.content); + content = buildToolResultContentForPrompt( + parsed, + tool, + toolCall?.args ?? {}, + this.config, + ); + } catch { + content = buildToolResultContentForPrompt( + message.content, + tool, + toolCall?.args ?? {}, + this.config, + ); + } + + return { + role: message.role, + content, + tool_call_id: message.toolCallId, + }; + }); + } +} diff --git a/packages/copilot-sdk/src/chat/types/chat.ts b/packages/copilot-sdk/src/chat/types/chat.ts index f2f14a4..1c536ae 100644 --- a/packages/copilot-sdk/src/chat/types/chat.ts +++ b/packages/copilot-sdk/src/chat/types/chat.ts @@ -4,7 +4,13 @@ * Configuration and status types for chat functionality. */ -import type { LLMConfig, MessageAttachment, ToolDefinition } from "../../core"; +import type { + ContextUsage, + LLMConfig, + MessageAttachment, + ToolDefinition, + ToolOptimizationConfig, +} from "../../core"; import type { Resolvable } from "../../core/utils/resolvable"; import type { UIMessage } from "./message"; @@ -52,6 +58,8 @@ export interface ChatConfig { debug?: boolean; /** Available tools (passed to LLM) */ tools?: ToolDefinition[]; + /** Optional prompt/tool optimization controls */ + optimization?: ToolOptimizationConfig; } /** @@ -96,6 +104,8 @@ export interface ChatCallbacks { onToolCalls?: (toolCalls: T["toolCalls"]) => void; /** Called when generation is complete */ onFinish?: (messages: T[]) => void; + /** Called when prompt context usage changes */ + onContextUsageChange?: (usage: ContextUsage) => void; /** Called when a server-side tool starts executing (action:start event) */ onServerToolStart?: (info: ServerToolInfo) => void; /** Called when a server-side tool receives args (action:args event) */ diff --git a/packages/copilot-sdk/src/core/index.ts b/packages/copilot-sdk/src/core/index.ts index da66e12..dcc7d36 100644 --- a/packages/copilot-sdk/src/core/index.ts +++ b/packages/copilot-sdk/src/core/index.ts @@ -167,6 +167,19 @@ export type { ToolRenderProps, ToolDefinition, ToolConfig, + ToolResultConfig, + ToolResultTruncationConfig, + ToolTruncationStrategy, + ToolProfile, + ToolProfileConfig, + ContextHistoryConfig, + ContextSummarizationConfig, + TokenEstimationConfig, + ContextManagementConfig, + ContextUsagePart, + ContextUsage, + ContextBudgetConfig, + ToolOptimizationConfig, ToolSet, ToolSetEntry, UnifiedToolCall, diff --git a/packages/copilot-sdk/src/core/types/tools.ts b/packages/copilot-sdk/src/core/types/tools.ts index ae785d4..b10c4d5 100644 --- a/packages/copilot-sdk/src/core/types/tools.ts +++ b/packages/copilot-sdk/src/core/types/tools.ts @@ -174,6 +174,153 @@ export type AIContent = | { type: "image"; data: string; mediaType: string } | { type: "text"; text: string }; +/** + * How large tool results should be trimmed before they are sent back to the AI. + */ +export type ToolTruncationStrategy = "head" | "head-tail" | "smart"; + +/** + * Truncation controls for tool results. + */ +export interface ToolResultTruncationConfig { + enabled?: boolean; + maxContextShare?: number; + hardMaxChars?: number; + minKeepChars?: number; + strategy?: ToolTruncationStrategy; + preserveErrors?: boolean; +} + +/** + * Global or per-tool controls for how tool results are represented in prompts. + */ +export interface ToolResultConfig { + truncation?: ToolResultTruncationConfig; +} + +/** + * Named tool profile for selective loading. + */ +export interface ToolProfile { + name: string; + description?: string; + include?: string[]; + exclude?: string[]; +} + +/** + * Tool profile configuration. + */ +export interface ToolProfileConfig { + enabled?: boolean; + defaultProfile?: string; + profiles?: Record; + /** When false, active profiles exclude tools that do not declare profile membership. */ + includeUnprofiled?: boolean; + dynamicSelection?: { + enabled?: boolean; + maxTools?: number; + }; +} + +/** + * History compaction behavior for long-running sessions. + */ +export interface ContextHistoryConfig { + maxMessages?: number; + maxTokens?: number; + maxContextShare?: number; + pruneStrategy?: "oldest" | "least-relevant" | "summarize"; +} + +/** + * Optional summarization controls used during history compaction. + */ +export interface ContextSummarizationConfig { + enabled?: boolean; + triggerAt?: number; + chunkSize?: number; + preserveRecent?: number; + fallbackBehavior?: "truncate" | "statistical" | "error"; +} + +/** + * Token estimation controls. + */ +export interface TokenEstimationConfig { + safetyMargin?: number; + charsPerToken?: number; +} + +/** + * Conversation context management. + */ +export interface ContextManagementConfig { + enabled?: boolean; + history?: ContextHistoryConfig; + summarization?: ContextSummarizationConfig; + tokenEstimation?: TokenEstimationConfig; +} + +/** + * One budget bucket in the prompt context. + */ +export interface ContextUsagePart { + tokens: number; + percent: number; +} + +/** + * Prompt context usage snapshot. + */ +export interface ContextUsage { + total: ContextUsagePart; + breakdown: { + systemPrompt: ContextUsagePart; + history: ContextUsagePart; + toolResults: ContextUsagePart; + tools: ContextUsagePart; + }; + budget: { + available: number; + remaining: number; + }; + warnings: string[]; +} + +/** + * Real-time context budget configuration. + */ +export interface ContextBudgetConfig { + enabled?: boolean; + budget?: { + contextWindowTokens?: number; + inputHeadroomRatio?: number; + systemPromptShare?: number; + historyShare?: number; + toolResultsShare?: number; + toolDefinitionsShare?: number; + }; + enforcement?: { + mode?: "warn" | "truncate" | "error"; + onBudgetExceeded?: (info: ContextUsage) => void; + }; + monitoring?: { + enabled?: boolean; + onUsageUpdate?: (usage: ContextUsage) => void; + }; +} + +/** + * Framework-agnostic optimization controls for tool-heavy chat sessions. + */ +export interface ToolOptimizationConfig { + toolProfiles?: ToolProfileConfig; + toolResultConfig?: ToolResultConfig; + contextManagement?: ContextManagementConfig; + contextBudget?: ContextBudgetConfig; +} + /** * Tool response format */ @@ -368,6 +515,18 @@ export interface ToolDefinition> { * @default "custom" */ source?: ToolSource; + /** Optional category for search, filtering, and budgets */ + category?: string; + /** Optional group for profile-based tool selection */ + group?: string; + /** Deferred tools are discoverable but need not be sent on every request */ + deferLoading?: boolean; + /** Profile memberships for selective tool loading */ + profiles?: string[]; + /** Extra keywords for dynamic tool selection */ + searchKeywords?: string[]; + /** Per-tool prompt/result shaping controls */ + resultConfig?: ToolResultConfig; // ============================================ // Display Configuration @@ -673,6 +832,8 @@ export interface AgentLoopConfig { debug?: boolean; /** Whether to enable the agentic loop (default: true) */ enabled?: boolean; + /** Optional prompt/tool optimization controls */ + optimization?: ToolOptimizationConfig; } /** @@ -738,6 +899,18 @@ export interface ToolConfig> { description: string; /** Where the tool executes (default: 'client') */ location?: ToolLocation; + /** Optional category for search, filtering, and budgets */ + category?: string; + /** Optional group for profile-based tool selection */ + group?: string; + /** Deferred tools are discoverable but omitted from the default prompt */ + deferLoading?: boolean; + /** Profile memberships for selective tool loading */ + profiles?: string[]; + /** Extra keywords for dynamic tool selection */ + searchKeywords?: string[]; + /** Per-tool prompt/result shaping controls */ + resultConfig?: ToolResultConfig; // Display Configuration /** Human-readable title for UI display */ @@ -799,6 +972,12 @@ export function tool>( return { description: config.description, location: config.location ?? "client", + category: config.category, + group: config.group, + deferLoading: config.deferLoading, + profiles: config.profiles, + searchKeywords: config.searchKeywords, + resultConfig: config.resultConfig, // Display configuration title: config.title, executingTitle: config.executingTitle, diff --git a/packages/llm-sdk/README.md b/packages/llm-sdk/README.md index e546c5d..c6f7c50 100644 --- a/packages/llm-sdk/README.md +++ b/packages/llm-sdk/README.md @@ -55,6 +55,65 @@ export async function POST(req: Request) { } ``` +## Selective Tool Loading + +`llm-sdk` can now narrow tools before they reach the provider. This is opt-in and works with both local ranking and provider-native hints. + +```ts +import { createRuntime, type ToolDefinition } from "@yourgpt/llm-sdk"; +import { createOpenAI } from "@yourgpt/llm-sdk/openai"; + +const tools: ToolDefinition[] = [ + { + name: "search_docs", + description: "Search product docs", + location: "server", + category: "knowledge", + profiles: ["support", "research"], + searchKeywords: ["docs", "kb", "help"], + inputSchema: { type: "object", properties: { query: { type: "string" } } }, + handler: async ({ query }) => ({ query }), + }, + { + name: "get_time", + description: "Get current time", + location: "server", + category: "utility", + profiles: ["utility"], + deferLoading: true, + inputSchema: { type: "object", properties: {} }, + handler: async () => ({ now: new Date().toISOString() }), + }, +]; + +const runtime = createRuntime({ + provider: createOpenAI({ apiKey: process.env.OPENAI_API_KEY }), + model: "gpt-4o-mini", + tools, + agentLoop: { + enabled: true, + toolSelection: { + enabled: true, + defaultProfile: "support", + search: { + enabled: true, + maxResults: 4, + exposeWhenToolCountExceeds: 1, + }, + dynamicSelection: { enabled: true, maxTools: 2 }, + nativeProviderHints: { + openai: { toolChoice: "single", parallelToolCalls: false }, + }, + }, + }, +}); + +// Request body can override the active profile: +// { "messages": [...], "toolProfile": "utility" } +``` + +When `search.enabled` is on, deferred tools can be discovered through a hidden `search_tools` server tool. Matching tools are loaded into the next loop iteration instead of sending every deferred tool definition up front. + ## Documentation Visit **[copilot-sdk.yourgpt.ai](https://copilot-sdk.yourgpt.ai)** for full documentation: diff --git a/packages/llm-sdk/src/adapters/anthropic.ts b/packages/llm-sdk/src/adapters/anthropic.ts index c646fce..6c84361 100644 --- a/packages/llm-sdk/src/adapters/anthropic.ts +++ b/packages/llm-sdk/src/adapters/anthropic.ts @@ -3,6 +3,7 @@ import type { StreamEvent, WebSearchConfig, Citation, + ToolDefinition, } from "../core/stream-events"; import { generateMessageId } from "../core/utils"; import type { @@ -13,6 +14,7 @@ import type { import { formatMessagesForAnthropic, messageToAnthropicContent, + logProviderPayload, type AnthropicContentBlock, } from "./base"; @@ -337,6 +339,37 @@ export class AnthropicAdapter implements LLMAdapter { return messages; } + private buildNativeSearchTools( + tools: ToolDefinition[], + variant: "bm25" | "regex" = "bm25", + ): Array> { + const nativeSearchTool = + variant === "regex" + ? { + type: "tool_search_tool_regex_20251119", + name: "tool_search_tool_regex", + } + : { + type: "tool_search_tool_bm25_20251119", + name: "tool_search_tool_bm25", + }; + + const providerTools = tools + .filter((tool) => tool.available !== false) + .map((tool) => ({ + name: tool.name, + description: tool.description, + input_schema: tool.inputSchema ?? { + type: "object" as const, + properties: {}, + required: [], + }, + defer_loading: tool.deferLoading === true, + })); + + return [nativeSearchTool, ...providerTools]; + } + /** * Build common request options for both streaming and non-streaming */ @@ -358,32 +391,38 @@ export class AnthropicAdapter implements LLMAdapter { messages = formatted.messages as Array>; } - // Convert actions to Anthropic tool format - const tools: Array> = - request.actions?.map((action) => ({ - name: action.name, - description: action.description, - input_schema: { - type: "object" as const, - properties: action.parameters - ? Object.fromEntries( - Object.entries(action.parameters).map(([key, param]) => [ - key, - { - type: param.type, - description: param.description, - enum: param.enum, - }, - ]), - ) - : {}, - required: action.parameters - ? Object.entries(action.parameters) - .filter(([, param]) => param.required) - .map(([key]) => key) - : [], - }, - })) || []; + const anthropicNativeSearch = + request.providerToolOptions?.anthropic?.nativeToolSearch; + + const tools: Array> = anthropicNativeSearch?.enabled + ? this.buildNativeSearchTools( + request.toolDefinitions ?? [], + anthropicNativeSearch.variant, + ) + : request.actions?.map((action) => ({ + name: action.name, + description: action.description, + input_schema: { + type: "object" as const, + properties: action.parameters + ? Object.fromEntries( + Object.entries(action.parameters).map(([key, param]) => [ + key, + { + type: param.type, + description: param.description, + enum: param.enum, + }, + ]), + ) + : {}, + required: action.parameters + ? Object.entries(action.parameters) + .filter(([, param]) => param.required) + .map(([key]) => key) + : [], + }, + })) || []; // Check for web search configuration (from request or adapter config) const webSearchConfig = request.webSearch ?? this.config.webSearch; @@ -436,6 +475,31 @@ export class AnthropicAdapter implements LLMAdapter { tools: tools.length ? tools : undefined, }; + const anthropicToolOptions = request.providerToolOptions?.anthropic; + if (tools.length > 0 && anthropicToolOptions) { + if ( + anthropicToolOptions.toolChoice || + anthropicToolOptions.disableParallelToolUse !== undefined + ) { + const toolChoice: Record = + typeof anthropicToolOptions.toolChoice === "object" + ? { + type: "tool", + name: anthropicToolOptions.toolChoice.name, + } + : anthropicToolOptions.toolChoice + ? { type: anthropicToolOptions.toolChoice } + : { type: "auto" }; + + if (anthropicToolOptions.disableParallelToolUse !== undefined) { + toolChoice.disable_parallel_tool_use = + anthropicToolOptions.disableParallelToolUse; + } + + options.tool_choice = toolChoice; + } + } + // Add server tool configuration for web search if (serverToolConfiguration) { options.server_tool_configuration = serverToolConfiguration; @@ -466,7 +530,19 @@ export class AnthropicAdapter implements LLMAdapter { } as Record & { stream: false }; try { + logProviderPayload( + "anthropic", + "request payload", + nonStreamingOptions, + request.debug, + ); const response = await client.messages.create(nonStreamingOptions); + logProviderPayload( + "anthropic", + "response payload", + response, + request.debug, + ); // Parse response let content = ""; @@ -512,6 +588,12 @@ export class AnthropicAdapter implements LLMAdapter { yield { type: "message:start", id: messageId }; try { + logProviderPayload( + "anthropic", + "request payload", + options, + request.debug, + ); const stream = await client.messages.stream(options); let currentToolUse: { @@ -536,6 +618,7 @@ export class AnthropicAdapter implements LLMAdapter { | undefined; for await (const event of stream) { + logProviderPayload("anthropic", "stream event", event, request.debug); // Check for abort if (request.signal?.aborted) { break; diff --git a/packages/llm-sdk/src/adapters/azure.ts b/packages/llm-sdk/src/adapters/azure.ts index 4de7071..ea7f1db 100644 --- a/packages/llm-sdk/src/adapters/azure.ts +++ b/packages/llm-sdk/src/adapters/azure.ts @@ -15,7 +15,11 @@ import type { ChatCompletionRequest, CompletionResult, } from "./base"; -import { formatMessagesForOpenAI, formatTools } from "./base"; +import { + formatMessagesForOpenAI, + formatTools, + logProviderPayload, +} from "./base"; // ============================================ // Types @@ -177,7 +181,7 @@ export class AzureAdapter implements LLMAdapter { yield { type: "message:start", id: messageId }; try { - const stream = await client.chat.completions.create({ + const payload = { // Azure uses deployment name, not model name model: this.config.deploymentName, messages, @@ -185,7 +189,9 @@ export class AzureAdapter implements LLMAdapter { temperature: request.config?.temperature ?? this.config.temperature, max_tokens: request.config?.maxTokens ?? this.config.maxTokens, stream: true, - }); + }; + logProviderPayload("azure", "request payload", payload, request.debug); + const stream = await client.chat.completions.create(payload); let currentToolCall: { id: string; @@ -194,6 +200,7 @@ export class AzureAdapter implements LLMAdapter { } | null = null; for await (const chunk of stream) { + logProviderPayload("azure", "stream chunk", chunk, request.debug); // Check for abort if (request.signal?.aborted) { break; @@ -292,13 +299,16 @@ export class AzureAdapter implements LLMAdapter { ? formatTools(request.actions) : undefined; - const response = await client.chat.completions.create({ + const payload = { model: this.config.deploymentName, messages, tools, temperature: request.config?.temperature ?? this.config.temperature, max_tokens: request.config?.maxTokens ?? this.config.maxTokens, - }); + }; + logProviderPayload("azure", "request payload", payload, request.debug); + const response = await client.chat.completions.create(payload); + logProviderPayload("azure", "response payload", response, request.debug); const choice = response.choices[0]; const message = choice?.message; diff --git a/packages/llm-sdk/src/adapters/base.ts b/packages/llm-sdk/src/adapters/base.ts index 11d1863..e788b67 100644 --- a/packages/llm-sdk/src/adapters/base.ts +++ b/packages/llm-sdk/src/adapters/base.ts @@ -4,7 +4,9 @@ import type { ActionDefinition, StreamEvent, LLMConfig, + ToolDefinition, WebSearchConfig, + ProviderToolRuntimeOptions, } from "../core/stream-events"; import type { TokenUsage } from "../core/types"; @@ -31,6 +33,8 @@ export interface ChatCompletionRequest { rawMessages?: Array>; /** Available actions/tools */ actions?: ActionDefinition[]; + /** Full tool definitions for provider-native tool search / deferred loading paths. */ + toolDefinitions?: ToolDefinition[]; /** System prompt */ systemPrompt?: string; /** LLM configuration overrides */ @@ -42,6 +46,10 @@ export interface ChatCompletionRequest { * When true or configured, the provider's native search is enabled. */ webSearch?: boolean | WebSearchConfig; + /** Optional provider-specific tool policy hints derived from runtime selection. */ + providerToolOptions?: ProviderToolRuntimeOptions; + /** Enable adapter-level provider payload logging. */ + debug?: boolean; } /** @@ -86,6 +94,55 @@ export interface LLMAdapter { */ export type AdapterFactory = (config: LLMConfig) => LLMAdapter; +function stringifyForDebug(value: unknown): string { + return JSON.stringify( + value, + (_key, currentValue) => { + if (typeof currentValue === "bigint") { + return currentValue.toString(); + } + if (currentValue instanceof Error) { + return { + name: currentValue.name, + message: currentValue.message, + stack: currentValue.stack, + }; + } + return currentValue; + }, + 2, + ); +} + +export function logProviderPayload( + provider: string, + label: string, + payload: unknown, + enabled?: boolean, +): void { + if (!enabled) { + return; + } + + // Stream chunks/events are too noisy for regular debug output and can flood + // terminal context. Keep request/response payload logging, but suppress the + // per-event stream logs unless we add a separate verbose flag later. + if (label.toLowerCase().includes("stream ")) { + return; + } + + try { + console.log( + `[llm-sdk:${provider}] ${label}\n${stringifyForDebug(payload)}`, + ); + } catch (error) { + console.log( + `[llm-sdk:${provider}] ${label} (failed to stringify payload)`, + error, + ); + } +} + /** * Convert messages to provider format (simple text only) */ @@ -162,11 +219,66 @@ function parameterToJsonSchema(param: { ), ]), ); + schema.additionalProperties = false; } return schema; } +export function normalizeObjectJsonSchema( + schema: Record | undefined, +): Record { + if (!schema || typeof schema !== "object") { + return { + type: "object", + properties: {}, + required: [], + additionalProperties: false, + }; + } + + const normalized: Record = { ...schema }; + const type = normalized.type; + + if (type === "object") { + const properties = + normalized.properties && + typeof normalized.properties === "object" && + !Array.isArray(normalized.properties) + ? (normalized.properties as Record) + : {}; + + normalized.properties = Object.fromEntries( + Object.entries(properties).map(([key, value]) => [ + key, + normalizeObjectJsonSchema(value as Record), + ]), + ); + + const propertyKeys = Object.keys(properties); + const required = Array.isArray(normalized.required) + ? normalized.required.filter( + (value): value is string => typeof value === "string", + ) + : []; + normalized.required = Array.from(new Set([...required, ...propertyKeys])); + + if (normalized.additionalProperties === undefined) { + normalized.additionalProperties = false; + } + } else if ( + type === "array" && + normalized.items && + typeof normalized.items === "object" + ) { + normalized.items = normalizeObjectJsonSchema( + normalized.items as Record, + ); + } + + return normalized; +} + /** * Convert actions to OpenAI tool format */ @@ -198,6 +310,7 @@ export function formatTools(actions: ActionDefinition[]): Array<{ .filter(([, param]) => param.required) .map(([key]) => key) : [], + additionalProperties: false, }, }, })); diff --git a/packages/llm-sdk/src/adapters/google.ts b/packages/llm-sdk/src/adapters/google.ts index 34c9d34..520d43e 100644 --- a/packages/llm-sdk/src/adapters/google.ts +++ b/packages/llm-sdk/src/adapters/google.ts @@ -18,7 +18,7 @@ import type { ChatCompletionRequest, CompletionResult, } from "./base"; -import { formatTools } from "./base"; +import { formatTools, logProviderPayload } from "./base"; // ============================================ // Types @@ -373,6 +373,24 @@ export class GoogleAdapter implements LLMAdapter { yield { type: "message:start", id: messageId }; try { + logProviderPayload( + "google", + "request payload", + { + model: modelId, + history: mergedContents.slice(0, -1), + systemInstruction: systemInstruction + ? { parts: [{ text: systemInstruction }] } + : undefined, + tools: toolsArray.length > 0 ? toolsArray : undefined, + generationConfig: { + temperature: request.config?.temperature ?? this.config.temperature, + maxOutputTokens: request.config?.maxTokens ?? this.config.maxTokens, + }, + messageParts: mergedContents[mergedContents.length - 1]?.parts, + }, + request.debug, + ); // Start chat session with system instruction const chat = model.startChat({ history: mergedContents.slice(0, -1), // All but the last message @@ -402,6 +420,7 @@ export class GoogleAdapter implements LLMAdapter { const collectedCitations: Citation[] = []; for await (const chunk of result.stream) { + logProviderPayload("google", "stream chunk", chunk, request.debug); // Check for abort if (request.signal?.aborted) { break; @@ -501,6 +520,12 @@ export class GoogleAdapter implements LLMAdapter { try { const response = await result.response; + logProviderPayload( + "google", + "response payload", + response, + request.debug, + ); if (response.usageMetadata) { usage = { prompt_tokens: response.usageMetadata.promptTokenCount || 0, @@ -611,6 +636,20 @@ export class GoogleAdapter implements LLMAdapter { const tools = formatToolsForGemini(request.actions); + const payload = { + model: modelId, + history: mergedContents.slice(0, -1), + systemInstruction: systemInstruction + ? { parts: [{ text: systemInstruction }] } + : undefined, + tools: tools ? [tools] : undefined, + generationConfig: { + temperature: request.config?.temperature ?? this.config.temperature, + maxOutputTokens: request.config?.maxTokens ?? this.config.maxTokens, + }, + messageParts: mergedContents[mergedContents.length - 1]?.parts, + }; + logProviderPayload("google", "request payload", payload, request.debug); const chat = model.startChat({ history: mergedContents.slice(0, -1), systemInstruction: systemInstruction @@ -626,6 +665,7 @@ export class GoogleAdapter implements LLMAdapter { const lastMessage = mergedContents[mergedContents.length - 1]; const result = await chat.sendMessage(lastMessage.parts); const response = result.response; + logProviderPayload("google", "response payload", response, request.debug); // Extract content and tool calls let textContent = ""; diff --git a/packages/llm-sdk/src/adapters/ollama.ts b/packages/llm-sdk/src/adapters/ollama.ts index 9d56f26..a06dad3 100644 --- a/packages/llm-sdk/src/adapters/ollama.ts +++ b/packages/llm-sdk/src/adapters/ollama.ts @@ -5,7 +5,7 @@ import type { } from "../core/stream-events"; import { generateMessageId, generateToolCallId } from "../core/utils"; import type { LLMAdapter, ChatCompletionRequest } from "./base"; -import { formatMessages, formatTools } from "./base"; +import { formatMessages, formatTools, logProviderPayload } from "./base"; import type { OllamaModelOptions } from "../providers/types"; /** @@ -288,18 +288,20 @@ export class OllamaAdapter implements LLMAdapter { Object.assign(ollamaOptions, this.config.options); } + const payload = { + model: request.config?.model || this.model, + messages, + tools, + stream: true, + options: ollamaOptions, + }; + logProviderPayload("ollama", "request payload", payload, request.debug); const response = await fetch(`${this.baseUrl}/api/chat`, { method: "POST", headers: { "Content-Type": "application/json", }, - body: JSON.stringify({ - model: request.config?.model || this.model, - messages, - tools, - stream: true, - options: ollamaOptions, - }), + body: JSON.stringify(payload), signal: request.signal, }); @@ -336,6 +338,7 @@ export class OllamaAdapter implements LLMAdapter { try { const chunk = JSON.parse(line); + logProviderPayload("ollama", "stream chunk", chunk, request.debug); // Handle content if (chunk.message?.content) { diff --git a/packages/llm-sdk/src/adapters/openai.ts b/packages/llm-sdk/src/adapters/openai.ts index b910065..36d96fd 100644 --- a/packages/llm-sdk/src/adapters/openai.ts +++ b/packages/llm-sdk/src/adapters/openai.ts @@ -3,10 +3,20 @@ import type { StreamEvent, WebSearchConfig, Citation, + ToolDefinition, } from "../core/stream-events"; import { generateMessageId, generateToolCallId } from "../core/utils"; -import type { LLMAdapter, ChatCompletionRequest } from "./base"; -import { formatMessagesForOpenAI, formatTools } from "./base"; +import type { + LLMAdapter, + ChatCompletionRequest, + CompletionResult, +} from "./base"; +import { + formatMessagesForOpenAI, + formatTools, + logProviderPayload, + normalizeObjectJsonSchema, +} from "./base"; /** * OpenAI adapter configuration @@ -53,7 +63,230 @@ export class OpenAIAdapter implements LLMAdapter { return this.client; } + private shouldUseResponsesApi(request: ChatCompletionRequest): boolean { + return ( + request.providerToolOptions?.openai?.nativeToolSearch?.enabled === true && + request.providerToolOptions.openai.nativeToolSearch.useResponsesApi !== + false && + Array.isArray(request.toolDefinitions) && + request.toolDefinitions.length > 0 + ); + } + + private buildResponsesInput( + request: ChatCompletionRequest, + ): Array> { + const sourceMessages = + request.rawMessages && request.rawMessages.length > 0 + ? request.rawMessages + : (formatMessagesForOpenAI(request.messages, undefined) as Array< + Record + >); + const input: Array> = []; + + for (const message of sourceMessages) { + if (message.role === "system") { + continue; + } + + if (message.role === "assistant") { + const content = + typeof message.content === "string" + ? message.content + : Array.isArray(message.content) + ? message.content + : message.content + ? JSON.stringify(message.content) + : ""; + + if (content) { + input.push({ + type: "message", + role: "assistant", + content, + }); + } + + const toolCalls = Array.isArray(message.tool_calls) + ? (message.tool_calls as Array<{ + id: string; + function?: { name?: string; arguments?: string }; + }>) + : []; + + for (const toolCall of toolCalls) { + input.push({ + type: "function_call", + call_id: toolCall.id, + name: toolCall.function?.name, + arguments: toolCall.function?.arguments ?? "{}", + }); + } + continue; + } + + if (message.role === "tool") { + input.push({ + type: "function_call_output", + call_id: message.tool_call_id, + output: + typeof message.content === "string" + ? message.content + : JSON.stringify(message.content ?? null), + }); + continue; + } + + input.push({ + type: "message", + role: message.role === "developer" ? "developer" : "user", + content: + typeof message.content === "string" + ? message.content + : Array.isArray(message.content) + ? message.content + : JSON.stringify(message.content ?? ""), + }); + } + + return input; + } + + private buildResponsesTools( + tools: ToolDefinition[], + ): Array> { + const nativeTools = tools + .filter((tool) => tool.available !== false) + .map((tool) => ({ + type: "function", + name: tool.name, + description: tool.description, + parameters: normalizeObjectJsonSchema( + (tool.inputSchema as Record | undefined) ?? { + type: "object", + properties: {}, + required: [], + }, + ), + strict: true, + defer_loading: tool.deferLoading === true, + })); + + return [{ type: "tool_search" }, ...nativeTools]; + } + + private parseResponsesResult(response: any): CompletionResult { + const content = + typeof response?.output_text === "string" ? response.output_text : ""; + const toolCalls = Array.isArray(response?.output) + ? response.output + .filter((item: any) => item?.type === "function_call") + .map((item: any) => ({ + id: item.call_id ?? item.id ?? generateToolCallId(), + name: item.name, + args: (() => { + try { + return JSON.parse(item.arguments ?? "{}"); + } catch { + return {}; + } + })(), + })) + : []; + + return { + content, + toolCalls, + usage: response?.usage + ? { + promptTokens: response.usage.input_tokens ?? 0, + completionTokens: response.usage.output_tokens ?? 0, + totalTokens: + response.usage.total_tokens ?? + (response.usage.input_tokens ?? 0) + + (response.usage.output_tokens ?? 0), + } + : undefined, + rawResponse: response as Record, + }; + } + + private async completeWithResponses( + request: ChatCompletionRequest, + ): Promise { + const client = await this.getClient(); + const openaiToolOptions = request.providerToolOptions?.openai; + const payload = { + model: request.config?.model || this.model, + instructions: request.systemPrompt, + input: this.buildResponsesInput(request), + tools: this.buildResponsesTools(request.toolDefinitions ?? []), + tool_choice: + openaiToolOptions?.toolChoice === "required" + ? "required" + : openaiToolOptions?.toolChoice === "auto" + ? "auto" + : undefined, + parallel_tool_calls: openaiToolOptions?.parallelToolCalls, + temperature: request.config?.temperature ?? this.config.temperature, + max_output_tokens: request.config?.maxTokens ?? this.config.maxTokens, + stream: false, + }; + + logProviderPayload("openai", "request payload", payload, request.debug); + const response = await client.responses.create(payload); + logProviderPayload("openai", "response payload", response, request.debug); + + return this.parseResponsesResult(response); + } + async *stream(request: ChatCompletionRequest): AsyncGenerator { + if (this.shouldUseResponsesApi(request)) { + const messageId = generateMessageId(); + yield { type: "message:start", id: messageId }; + + try { + const result = await this.completeWithResponses(request); + + if (result.content) { + yield { type: "message:delta", content: result.content }; + } + + for (const toolCall of result.toolCalls) { + yield { + type: "action:start", + id: toolCall.id, + name: toolCall.name, + }; + yield { + type: "action:args", + id: toolCall.id, + args: JSON.stringify(toolCall.args), + }; + } + + yield { type: "message:end" }; + yield { + type: "done", + usage: result.usage + ? { + prompt_tokens: result.usage.promptTokens, + completion_tokens: result.usage.completionTokens, + total_tokens: result.usage.totalTokens, + } + : undefined, + }; + return; + } catch (error) { + yield { + type: "error", + message: error instanceof Error ? error.message : "Unknown error", + code: "OPENAI_RESPONSES_ERROR", + }; + return; + } + } + const client = await this.getClient(); // Use raw messages if provided (for agent loop with tool calls), otherwise format from Message[] @@ -163,15 +396,31 @@ export class OpenAIAdapter implements LLMAdapter { yield { type: "message:start", id: messageId }; try { - const stream = await client.chat.completions.create({ + const openaiToolOptions = request.providerToolOptions?.openai; + const toolChoice = + openaiToolOptions?.toolChoice && + typeof openaiToolOptions.toolChoice === "object" + ? { + type: "function" as const, + function: { + name: openaiToolOptions.toolChoice.name, + }, + } + : openaiToolOptions?.toolChoice; + const payload = { model: request.config?.model || this.model, messages, tools: tools.length > 0 ? tools : undefined, + tool_choice: tools.length > 0 ? toolChoice : undefined, + parallel_tool_calls: + tools.length > 0 ? openaiToolOptions?.parallelToolCalls : undefined, temperature: request.config?.temperature ?? this.config.temperature, max_tokens: request.config?.maxTokens ?? this.config.maxTokens, stream: true, stream_options: { include_usage: true }, - }); + }; + logProviderPayload("openai", "request payload", payload, request.debug); + const stream = await client.chat.completions.create(payload); let currentToolCall: { id: string; @@ -192,6 +441,7 @@ export class OpenAIAdapter implements LLMAdapter { | undefined; for await (const chunk of stream) { + logProviderPayload("openai", "stream chunk", chunk, request.debug); // Check for abort if (request.signal?.aborted) { break; @@ -308,6 +558,91 @@ export class OpenAIAdapter implements LLMAdapter { }; } } + + async complete(request: ChatCompletionRequest): Promise { + if (this.shouldUseResponsesApi(request)) { + return this.completeWithResponses(request); + } + + const client = await this.getClient(); + + let messages: Array>; + if (request.rawMessages && request.rawMessages.length > 0) { + messages = request.rawMessages; + if ( + request.systemPrompt && + !messages.some((message) => message.role === "system") + ) { + messages = [ + { role: "system", content: request.systemPrompt }, + ...messages, + ]; + } + } else { + messages = formatMessagesForOpenAI( + request.messages, + request.systemPrompt, + ) as Array>; + } + + const tools: Array> = request.actions?.length + ? formatTools(request.actions) + : []; + + const openaiToolOptions = request.providerToolOptions?.openai; + const toolChoice = + openaiToolOptions?.toolChoice && + typeof openaiToolOptions.toolChoice === "object" + ? { + type: "function" as const, + function: { + name: openaiToolOptions.toolChoice.name, + }, + } + : openaiToolOptions?.toolChoice; + + const payload = { + model: request.config?.model || this.model, + messages, + tools: tools.length > 0 ? tools : undefined, + tool_choice: tools.length > 0 ? toolChoice : undefined, + parallel_tool_calls: + tools.length > 0 ? openaiToolOptions?.parallelToolCalls : undefined, + temperature: request.config?.temperature ?? this.config.temperature, + max_tokens: request.config?.maxTokens ?? this.config.maxTokens, + stream: false, + }; + + logProviderPayload("openai", "request payload", payload, request.debug); + const response = await client.chat.completions.create(payload); + logProviderPayload("openai", "response payload", response, request.debug); + + const choice = response.choices?.[0]; + const message = choice?.message; + return { + content: message?.content ?? "", + toolCalls: + message?.tool_calls?.map((toolCall: any) => ({ + id: toolCall.id ?? generateToolCallId(), + name: toolCall.function?.name ?? "", + args: (() => { + try { + return JSON.parse(toolCall.function?.arguments ?? "{}"); + } catch { + return {}; + } + })(), + })) ?? [], + usage: response.usage + ? { + promptTokens: response.usage.prompt_tokens, + completionTokens: response.usage.completion_tokens, + totalTokens: response.usage.total_tokens, + } + : undefined, + rawResponse: response as Record, + }; + } } /** diff --git a/packages/llm-sdk/src/adapters/xai.ts b/packages/llm-sdk/src/adapters/xai.ts index 0e6adc2..e176fbe 100644 --- a/packages/llm-sdk/src/adapters/xai.ts +++ b/packages/llm-sdk/src/adapters/xai.ts @@ -15,7 +15,11 @@ import type { ChatCompletionRequest, CompletionResult, } from "./base"; -import { formatMessagesForOpenAI, formatTools } from "./base"; +import { + formatMessagesForOpenAI, + formatTools, + logProviderPayload, +} from "./base"; // ============================================ // Types @@ -147,14 +151,16 @@ export class XAIAdapter implements LLMAdapter { yield { type: "message:start", id: messageId }; try { - const stream = await client.chat.completions.create({ + const payload = { model: request.config?.model || this.model, messages, tools, temperature: request.config?.temperature ?? this.config.temperature, max_tokens: request.config?.maxTokens ?? this.config.maxTokens, stream: true, - }); + }; + logProviderPayload("xai", "request payload", payload, request.debug); + const stream = await client.chat.completions.create(payload); let currentToolCall: { id: string; @@ -163,6 +169,7 @@ export class XAIAdapter implements LLMAdapter { } | null = null; for await (const chunk of stream) { + logProviderPayload("xai", "stream chunk", chunk, request.debug); // Check for abort if (request.signal?.aborted) { break; @@ -261,13 +268,16 @@ export class XAIAdapter implements LLMAdapter { ? formatTools(request.actions) : undefined; - const response = await client.chat.completions.create({ + const payload = { model: request.config?.model || this.model, messages, tools, temperature: request.config?.temperature ?? this.config.temperature, max_tokens: request.config?.maxTokens ?? this.config.maxTokens, - }); + }; + logProviderPayload("xai", "request payload", payload, request.debug); + const response = await client.chat.completions.create(payload); + logProviderPayload("xai", "response payload", response, request.debug); const choice = response.choices[0]; const message = choice?.message; diff --git a/packages/llm-sdk/src/core/stream-events.ts b/packages/llm-sdk/src/core/stream-events.ts index 259eb94..9611dad 100644 --- a/packages/llm-sdk/src/core/stream-events.ts +++ b/packages/llm-sdk/src/core/stream-events.ts @@ -427,6 +427,10 @@ export interface ToolDefinition> { name: string; description: string; location: ToolLocation; + /** Optional logical category for tool search and selective loading. */ + category?: string; + /** Optional group label for related tools. */ + group?: string; title?: string | ((args: TParams) => string); inputSchema?: ToolInputSchema; handler?: ( @@ -450,6 +454,117 @@ export interface ToolDefinition> { aiContext?: | string | ((result: ToolResponse, args: Record) => string); + /** Hint that this tool should be loaded lazily when dynamic selection is active. */ + deferLoading?: boolean; + /** Named profiles this tool belongs to (for example "coding" or "search"). */ + profiles?: string[]; + /** Extra keywords used by lightweight tool search/ranking. */ + searchKeywords?: string[]; +} + +export interface ToolProfile { + include?: string[]; + exclude?: string[]; +} + +export interface ToolDynamicSelectionConfig { + enabled?: boolean; + maxTools?: number; +} + +export interface ToolSearchConfig { + enabled?: boolean; + /** + * Search execution mode. + * - auto: use native provider search when supported, otherwise fall back to manual search_tools + * - native: require provider-native search when supported, otherwise fall back to manual search_tools + * - manual: always use the SDK-managed search_tools fallback + */ + mode?: "auto" | "native" | "manual"; + metaToolName?: string; + maxResults?: number; + minScore?: number; + exposeWhenToolCountExceeds?: number; + /** Anthropic native tool search variant. Defaults to bm25. */ + anthropicVariant?: "bm25" | "regex"; + /** + * When true, tools marked with deferLoading stay hidden from the initial + * selected tool list and are only introduced after search_tools loads them. + */ + strictDeferredLoading?: boolean; +} + +export interface OpenAIToolSelectionHints { + /** + * "single" forces the selected tool when exactly one tool remains after selection. + * Otherwise the adapter falls back to automatic tool choice. + */ + toolChoice?: "auto" | "required" | "single"; + /** Set false to disable parallel tool calls on OpenAI-compatible providers. */ + parallelToolCalls?: boolean; +} + +export interface AnthropicToolSelectionHints { + /** + * "single" forces the selected tool when exactly one tool remains after selection. + * Otherwise the adapter falls back to Anthropic's automatic tool choice. + */ + toolChoice?: "auto" | "any" | "single"; + /** Disable parallel tool use when supported by the Anthropic API. */ + disableParallelToolUse?: boolean; +} + +export interface ToolNativeProviderHints { + openai?: OpenAIToolSelectionHints; + anthropic?: AnthropicToolSelectionHints; +} + +export interface ToolSelectionConfig { + enabled?: boolean; + defaultProfile?: string; + profiles?: Record; + /** When false, active profiles exclude tools without explicit profile membership. */ + includeUnprofiled?: boolean; + dynamicSelection?: ToolDynamicSelectionConfig; + /** Optional indexed search over deferred tools. */ + search?: ToolSearchConfig; + /** Optional provider-native hints layered on top of local tool selection. */ + nativeProviderHints?: ToolNativeProviderHints; +} + +export interface OpenAIProviderToolOptions { + toolChoice?: + | "auto" + | "required" + | { + type: "function"; + name: string; + }; + parallelToolCalls?: boolean; + nativeToolSearch?: { + enabled: boolean; + useResponsesApi?: boolean; + }; +} + +export interface AnthropicProviderToolOptions { + toolChoice?: + | "auto" + | "any" + | { + type: "tool"; + name: string; + }; + disableParallelToolUse?: boolean; + nativeToolSearch?: { + enabled: boolean; + variant: "bm25" | "regex"; + }; +} + +export interface ProviderToolRuntimeOptions { + openai?: OpenAIProviderToolOptions; + anthropic?: AnthropicProviderToolOptions; } /** @@ -459,6 +574,7 @@ export interface AgentLoopConfig { maxIterations?: number; debug?: boolean; enabled?: boolean; + toolSelection?: ToolSelectionConfig; } /** diff --git a/packages/llm-sdk/src/index.ts b/packages/llm-sdk/src/index.ts index 99bc56d..cc22c30 100644 --- a/packages/llm-sdk/src/index.ts +++ b/packages/llm-sdk/src/index.ts @@ -89,10 +89,15 @@ export { DEFAULT_CAPABILITIES } from "./core/types"; export { Runtime, createRuntime, + selectTools, + searchTools, + shouldExposeToolSearch, + buildProviderToolOptions, type RuntimeConfig, type ChatRequest, type ActionRequest, type RequestContext, + type ToolSearchMatch, } from "./server"; // StreamResult (Industry Standard Pattern) @@ -186,6 +191,16 @@ export type { UnifiedToolResult, ToolExecution, AgentLoopConfig, + ToolProfile, + ToolDynamicSelectionConfig, + ToolSearchConfig, + OpenAIToolSelectionHints, + AnthropicToolSelectionHints, + ToolNativeProviderHints, + ToolSelectionConfig, + OpenAIProviderToolOptions, + AnthropicProviderToolOptions, + ProviderToolRuntimeOptions, DoneEventMessage, ToolCallInfo, TokenUsageRaw, diff --git a/packages/llm-sdk/src/providers/anthropic.ts b/packages/llm-sdk/src/providers/anthropic.ts index 6aa7733..4880680 100644 --- a/packages/llm-sdk/src/providers/anthropic.ts +++ b/packages/llm-sdk/src/providers/anthropic.ts @@ -27,7 +27,13 @@ export function transformTools(tools: ToolDefinition[]): AnthropicTool[] { return tools.map((tool) => ({ name: tool.name, description: tool.description, - input_schema: tool.inputSchema, + input_schema: tool.inputSchema + ? { + type: "object", + properties: tool.inputSchema.properties ?? {}, + required: tool.inputSchema.required, + } + : { type: "object", properties: {} }, })); } diff --git a/packages/llm-sdk/src/providers/gemini.ts b/packages/llm-sdk/src/providers/gemini.ts index 1c7b141..41c49af 100644 --- a/packages/llm-sdk/src/providers/gemini.ts +++ b/packages/llm-sdk/src/providers/gemini.ts @@ -31,7 +31,13 @@ export function transformTools( functionDeclarations: tools.map((tool) => ({ name: tool.name, description: tool.description, - parameters: tool.inputSchema, + parameters: tool.inputSchema + ? { + type: "object", + properties: tool.inputSchema.properties ?? {}, + required: tool.inputSchema.required, + } + : { type: "object", properties: {} }, })), }, ]; diff --git a/packages/llm-sdk/src/providers/openai.ts b/packages/llm-sdk/src/providers/openai.ts index 8610ce6..72f8538 100644 --- a/packages/llm-sdk/src/providers/openai.ts +++ b/packages/llm-sdk/src/providers/openai.ts @@ -30,7 +30,13 @@ export function transformTools(tools: ToolDefinition[]): OpenAITool[] { function: { name: tool.name, description: tool.description, - parameters: tool.inputSchema, + parameters: tool.inputSchema + ? { + type: "object", + properties: tool.inputSchema.properties ?? {}, + required: tool.inputSchema.required, + } + : { type: "object", properties: {} }, }, })); } diff --git a/packages/llm-sdk/src/server/agent-loop.ts b/packages/llm-sdk/src/server/agent-loop.ts index 7dbd121..0363770 100644 --- a/packages/llm-sdk/src/server/agent-loop.ts +++ b/packages/llm-sdk/src/server/agent-loop.ts @@ -22,6 +22,12 @@ import type { import type { AIProvider } from "../providers/types"; import { generateToolCallId, generateMessageId } from "../core/utils"; import { getFormatter } from "../providers"; +import { + buildProviderToolOptions, + searchTools, + selectTools, + shouldExposeToolSearch, +} from "./tool-selection"; // ======================================== // Constants @@ -50,11 +56,17 @@ export interface AgentLoopOptions { signal?: AbortSignal; /** Loop configuration */ config?: AgentLoopConfig; + /** Optional active tool profile for selective loading. */ + toolProfile?: string; /** * LLM call function * Should call the LLM and return the raw response */ - callLLM: (messages: unknown[], tools: unknown[]) => Promise; + callLLM: ( + messages: unknown[], + tools: unknown[], + providerToolOptions?: ReturnType, + ) => Promise; /** * Server-side tool executor * Called when a server-side tool needs to be executed @@ -103,6 +115,7 @@ export async function* runAgentLoop( provider, signal, config, + toolProfile, callLLM, executeServerTool, waitForClientToolResult, @@ -111,15 +124,14 @@ export async function* runAgentLoop( const maxIterations = config?.maxIterations ?? DEFAULT_MAX_ITERATIONS; const debug = config?.debug ?? false; const formatter = getFormatter(provider.name); + const toolSearchMetaToolName = + config?.toolSelection?.search?.metaToolName ?? "search_tools"; // Separate server and client tools const serverTools = tools.filter((t) => t.location === "server"); const clientTools = tools.filter((t) => t.location === "client"); const allTools = [...serverTools, ...clientTools]; - // Transform tools to provider format - const providerTools = formatter.transformTools(allTools); - // Build conversation const conversation: ConversationMessage[] = buildConversation( messages, @@ -127,13 +139,15 @@ export async function* runAgentLoop( ); let iteration = 0; + let loadedToolNames = new Set(); if (debug) { console.log("[AgentLoop] Starting with", { messageCount: messages.length, - toolCount: allTools.length, + availableToolCount: allTools.length, serverToolCount: serverTools.length, clientToolCount: clientTools.length, + activeProfile: toolProfile ?? config?.toolSelection?.defaultProfile, maxIterations, }); } @@ -159,13 +173,84 @@ export async function* runAgentLoop( maxIterations, }; + const selectedTools = selectTools({ + tools: allTools, + messages, + config: config?.toolSelection, + activeProfile: toolProfile, + forceIncludeNames: [...loadedToolNames], + }); + const toolSearchTool = shouldExposeToolSearch({ + tools: allTools, + selectedTools, + config: config?.toolSelection, + }) + ? ({ + name: toolSearchMetaToolName, + description: + "Search available deferred tools and load the most relevant ones for the next step when the required tool is not currently exposed.", + location: "server", + hidden: true, + inputSchema: { + type: "object", + properties: { + query: { + type: "string", + description: "Describe the tool capability you need to find.", + }, + limit: { + type: "number", + description: "Maximum number of matching tools to load.", + }, + }, + required: ["query"], + }, + handler: async (params: Record) => { + const query = typeof params.query === "string" ? params.query : ""; + const limit = + typeof params.limit === "number" ? params.limit : undefined; + const results = searchTools({ + tools: allTools, + query, + config: config?.toolSelection, + activeProfile: toolProfile, + limit, + excludeNames: selectedTools.map((tool) => tool.name), + }); + return { + success: true, + query, + loadedTools: results.map((result) => result.name), + results, + }; + }, + } satisfies ToolDefinition) + : null; + const effectiveSelectedTools = toolSearchTool + ? [...selectedTools, toolSearchTool] + : selectedTools; + const providerToolOptions = buildProviderToolOptions({ + providerName: provider.name, + selectedTools: effectiveSelectedTools, + config: config?.toolSelection, + metaToolName: toolSearchMetaToolName, + }); + const providerTools = formatter.transformTools(effectiveSelectedTools); + if (debug) { - console.log(`[AgentLoop] Iteration ${iteration}/${maxIterations}`); + console.log(`[AgentLoop] Iteration ${iteration}/${maxIterations}`, { + selectedToolCount: effectiveSelectedTools.length, + loadedDeferredTools: [...loadedToolNames], + }); } try { // Call LLM - const response = await callLLM(conversation, providerTools); + const response = await callLLM( + conversation, + providerTools, + providerToolOptions, + ); // Parse tool calls and text from response const toolCalls = formatter.parseToolCalls(response); @@ -191,7 +276,7 @@ export async function* runAgentLoop( // Execute tools const results = await executeToolCalls( toolCalls, - tools, + effectiveSelectedTools, executeServerTool, waitForClientToolResult, function* (event: StreamEvent) { @@ -213,6 +298,27 @@ export async function* runAgentLoop( } } + for (const result of results) { + const toolCall = toolCalls.find((tc) => tc.id === result.toolCallId); + if (!toolCall || toolCall.name !== toolSearchMetaToolName) { + continue; + } + try { + const parsed = JSON.parse(result.content) as { + loadedTools?: unknown; + }; + if (Array.isArray(parsed.loadedTools)) { + for (const toolName of parsed.loadedTools) { + if (typeof toolName === "string" && toolName) { + loadedToolNames.add(toolName); + } + } + } + } catch { + // Ignore malformed tool search result payloads. + } + } + // Add assistant message with tool calls to conversation const assistantMessage = formatter.buildAssistantToolMessage( toolCalls, diff --git a/packages/llm-sdk/src/server/index.ts b/packages/llm-sdk/src/server/index.ts index 3729d9b..c7258fc 100644 --- a/packages/llm-sdk/src/server/index.ts +++ b/packages/llm-sdk/src/server/index.ts @@ -52,3 +52,12 @@ export { DEFAULT_MAX_ITERATIONS, type AgentLoopOptions, } from "./agent-loop"; + +// Tool selection +export { + selectTools, + searchTools, + shouldExposeToolSearch, + buildProviderToolOptions, +} from "./tool-selection"; +export type { ToolSearchMatch } from "./tool-selection"; diff --git a/packages/llm-sdk/src/server/runtime.ts b/packages/llm-sdk/src/server/runtime.ts index ada24d3..23c0c52 100644 --- a/packages/llm-sdk/src/server/runtime.ts +++ b/packages/llm-sdk/src/server/runtime.ts @@ -26,6 +26,36 @@ import type { import { createSSEResponse } from "./streaming"; import { StreamResult, type CollectedResult } from "./stream-result"; import { GenerateResult } from "./generate-result"; +import { + buildProviderToolOptions, + filterToolsByProfile, + resolveNativeToolSearch, + searchTools, + selectTools, + shouldExposeToolSearch, +} from "./tool-selection"; + +type ToolSearchState = { + loadedToolNames: string[]; +}; + +type NativeToolSearchState = ReturnType; + +type ToolSearchResult = { + success: true; + query: string; + loadedTools: string[]; + results: Array<{ + name: string; + description: string; + location?: ToolDefinition["location"]; + category?: string; + group?: string; + profiles?: string[]; + searchKeywords?: string[]; + score: number; + }>; +}; // ============================================ // AI Response Control @@ -244,6 +274,7 @@ export class Runtime { config: request.config, signal, webSearch: this.getWebSearchConfig(), + debug: this.config.debug, }; // Stream response from adapter @@ -311,7 +342,15 @@ export class Runtime { const body = (await request.json()) as ChatRequest; if (this.config.debug) { - console.log("[Copilot SDK] Request:", JSON.stringify(body, null, 2)); + console.log("[Copilot SDK] Request:", { + messageCount: body.messages?.length ?? 0, + toolCount: body.tools?.length ?? 0, + toolCatalogCount: body.toolCatalog?.length ?? 0, + hasSystemPrompt: Boolean(body.systemPrompt), + threadId: body.threadId, + streaming: body.streaming !== false, + toolProfile: body.toolProfile, + }); } // Create abort controller from request signal @@ -644,6 +683,195 @@ export class Runtime { return undefined; } + private collectToolsForRequest(request: ChatRequest): ToolDefinition[] { + const allTools: ToolDefinition[] = [...this.tools.values()]; + + const clientTools = + this.config.agentLoop?.toolSelection?.enabled && + request.toolCatalog?.length + ? request.toolCatalog + : request.tools; + + if (clientTools) { + for (const tool of clientTools) { + allTools.push({ + name: tool.name, + description: tool.description, + location: "client", + category: tool.category, + group: tool.group, + deferLoading: tool.deferLoading, + profiles: tool.profiles, + searchKeywords: tool.searchKeywords, + inputSchema: tool.inputSchema as ToolDefinition["inputSchema"], + }); + } + } + + return allTools; + } + + private selectToolsForRequest( + request: ChatRequest, + allTools: ToolDefinition[], + toolSearchState?: ToolSearchState, + ): ToolDefinition[] { + return selectTools({ + tools: allTools, + messages: request.messages, + config: this.config.agentLoop?.toolSelection, + activeProfile: request.toolProfile, + forceIncludeNames: toolSearchState?.loadedToolNames, + }); + } + + private resolveNativeToolSearchForRequest(): NativeToolSearchState { + return resolveNativeToolSearch({ + providerName: this.adapter.provider, + modelName: this.getModel(), + config: this.config.agentLoop?.toolSelection, + }); + } + + private buildNativeToolCatalogForRequest( + request: ChatRequest, + allTools: ToolDefinition[], + ): ToolDefinition[] { + return filterToolsByProfile({ + tools: allTools, + config: this.config.agentLoop?.toolSelection, + activeProfile: request.toolProfile, + }); + } + + private buildProviderToolOptionsForRequest(selectedTools: ToolDefinition[]) { + return buildProviderToolOptions({ + providerName: this.adapter.provider, + modelName: this.getModel(), + selectedTools, + config: this.config.agentLoop?.toolSelection, + metaToolName: this.getToolSearchMetaToolName(), + }); + } + + private getToolSearchMetaToolName(): string { + return ( + this.config.agentLoop?.toolSelection?.search?.metaToolName ?? + "search_tools" + ); + } + + private createToolSearchTool( + request: ChatRequest, + allTools: ToolDefinition[], + selectedTools: ToolDefinition[], + ): ToolDefinition | null { + if ( + !shouldExposeToolSearch({ + tools: allTools, + selectedTools, + config: this.config.agentLoop?.toolSelection, + }) + ) { + return null; + } + + const toolName = this.getToolSearchMetaToolName(); + const excludedNames = selectedTools.map((tool) => tool.name); + + return { + name: toolName, + description: + "Search available deferred tools and load the most relevant ones for the next step when the right tool is not currently exposed.", + location: "server", + hidden: true, + inputSchema: { + type: "object", + properties: { + query: { + type: "string", + description: "Describe the tool capability you need to find.", + }, + limit: { + type: "number", + description: "Maximum number of matching tools to load.", + }, + }, + required: ["query"], + }, + handler: async (params) => { + const args = params as { query: string; limit?: number }; + const results = searchTools({ + tools: allTools, + query: args.query, + config: this.config.agentLoop?.toolSelection, + activeProfile: request.toolProfile, + limit: args.limit, + excludeNames: excludedNames, + }); + + if (this.config.debug || this.config.agentLoop?.debug) { + console.log("[Copilot SDK] search_tools result:", { + query: args.query, + activeProfile: + request.toolProfile ?? + this.config.agentLoop?.toolSelection?.defaultProfile, + selectedToolCount: selectedTools.length, + catalogCount: allTools.length, + loadedTools: results.map((result) => result.name), + results: results.map((result) => ({ + name: result.name, + location: result.location, + category: result.category, + group: result.group, + score: result.score, + })), + }); + } + + return { + success: true, + query: args.query, + loadedTools: results.map((result) => result.name), + results, + } satisfies ToolSearchResult; + }, + }; + } + + private extendLoadedToolNames( + current: ToolSearchState | undefined, + results: Array<{ name: string; result: unknown }>, + ): ToolSearchState | undefined { + const loaded = new Set(current?.loadedToolNames ?? []); + const searchToolName = this.getToolSearchMetaToolName(); + + for (const result of results) { + if (result.name !== searchToolName) { + continue; + } + const typedResult = result.result as { + loadedTools?: unknown; + } | null; + if (!Array.isArray(typedResult?.loadedTools)) { + continue; + } + for (const toolName of typedResult.loadedTools) { + if (typeof toolName === "string" && toolName) { + loaded.add(toolName); + } + } + } + + if (loaded.size === 0) { + return current; + } + + return { + loadedToolNames: [...loaded], + }; + } + /** * Process a chat request with tool support (Vercel AI SDK pattern) * @@ -663,6 +891,7 @@ export class Runtime { _isRecursive?: boolean, // HTTP request for extracting headers (auth context) _httpRequest?: Request, + _toolSearchState?: ToolSearchState, ): AsyncGenerator { const debug = this.config.debug || this.config.agentLoop?.debug; @@ -679,6 +908,7 @@ export class Runtime { _accumulatedMessages, _isRecursive, _httpRequest, + _toolSearchState, )) { yield event; } @@ -689,25 +919,44 @@ export class Runtime { const newMessages: DoneEventMessage[] = _accumulatedMessages || []; const maxIterations = this.config.agentLoop?.maxIterations || 20; - // Collect all tools (server + client from request) - const allTools: ToolDefinition[] = [...this.tools.values()]; - - // Add client tools from request - if (request.tools) { - for (const tool of request.tools) { - allTools.push({ - name: tool.name, - description: tool.description, - location: "client", - inputSchema: tool.inputSchema as ToolDefinition["inputSchema"], - }); - } - } + const allTools = this.collectToolsForRequest(request); + const nativeToolSearch = this.resolveNativeToolSearchForRequest(); + const nativeToolCatalog = nativeToolSearch + ? this.buildNativeToolCatalogForRequest(request, allTools) + : null; + const selectedTools = + nativeToolCatalog ?? + this.selectToolsForRequest(request, allTools, _toolSearchState); + const toolSearchTool = nativeToolSearch + ? null + : this.createToolSearchTool(request, allTools, selectedTools); + const effectiveSelectedTools = nativeToolCatalog + ? nativeToolCatalog + : toolSearchTool + ? [...selectedTools, toolSearchTool] + : selectedTools; + const providerToolOptions = this.buildProviderToolOptionsForRequest( + effectiveSelectedTools, + ); + const selectedToolMap = new Map( + effectiveSelectedTools.map((tool) => [tool.name, tool] as const), + ); if (debug) { console.log( `[Copilot SDK] Processing chat with ${allTools.length} tools`, ); + if (effectiveSelectedTools.length !== allTools.length) { + console.log( + `[Copilot SDK] Tool selection active: ${effectiveSelectedTools.length}/${allTools.length} tools`, + { + activeProfile: + request.toolProfile ?? + this.config.agentLoop?.toolSelection?.defaultProfile, + nativeSearch: nativeToolSearch?.provider ?? null, + }, + ); + } // Log messages with attachments for debugging vision support for (let i = 0; i < request.messages.length; i++) { const msg = request.messages[i]; @@ -747,11 +996,16 @@ export class Runtime { const completionRequest: ChatCompletionRequest = { messages: [], // Not used when rawMessages is provided rawMessages: request.messages as Array>, - actions: this.convertToolsToActions(allTools), + actions: nativeToolSearch + ? undefined + : this.convertToolsToActions(effectiveSelectedTools), + toolDefinitions: nativeToolSearch ? effectiveSelectedTools : undefined, systemPrompt: systemPrompt, config: request.config, signal, webSearch: this.getWebSearchConfig(), + providerToolOptions, + debug, }; // Stream from adapter @@ -846,7 +1100,7 @@ export class Runtime { const clientToolCalls: ToolCallInfo[] = []; for (const tc of toolCalls) { - const tool = allTools.find((t) => t.name === tc.name); + const tool = selectedToolMap.get(tc.name); if (tool?.location === "server" && tool.handler) { serverToolCalls.push(tc); } else { @@ -868,7 +1122,7 @@ export class Runtime { "toolContext" in this.config ? this.config.toolContext : undefined; for (const tc of serverToolCalls) { - const tool = allTools.find((t) => t.name === tc.name); + const tool = selectedToolMap.get(tc.name); if (tool?.handler) { if (debug) { console.log(`[Copilot SDK] Executing server-side tool: ${tc.name}`); @@ -896,6 +1150,7 @@ export class Runtime { yield { type: "action:end", id: tc.id, + name: tc.name, result, } as StreamEvent; } catch (error) { @@ -917,6 +1172,7 @@ export class Runtime { yield { type: "action:end", id: tc.id, + name: tc.name, error: error instanceof Error ? error.message @@ -981,6 +1237,13 @@ export class Runtime { ...request, messages: messagesWithResults as ChatRequest["messages"], }; + const nextToolSearchState = this.extendLoadedToolNames( + _toolSearchState, + serverToolResults.map((result) => ({ + name: result.name, + result: result.result, + })), + ); // Signal end of current message turn before continuing // This tells the client to finalize the current assistant message @@ -994,6 +1257,7 @@ export class Runtime { newMessages, true, // Mark as recursive _httpRequest, + nextToolSearchState, )) { yield event; } @@ -1076,6 +1340,7 @@ export class Runtime { _accumulatedMessages?: DoneEventMessage[], _isRecursive?: boolean, _httpRequest?: Request, + _toolSearchState?: ToolSearchState, ): AsyncGenerator { const newMessages: DoneEventMessage[] = _accumulatedMessages || []; const debug = this.config.debug || this.config.agentLoop?.debug; @@ -1087,20 +1352,9 @@ export class Runtime { total_tokens: number; } = { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }; - // Collect all tools (server + client from request) - const allTools: ToolDefinition[] = [...this.tools.values()]; - - // Add client tools from request - if (request.tools) { - for (const tool of request.tools) { - allTools.push({ - name: tool.name, - description: tool.description, - location: "client", - inputSchema: tool.inputSchema as ToolDefinition["inputSchema"], - }); - } - } + const allTools = this.collectToolsForRequest(request); + const nativeToolSearch = this.resolveNativeToolSearchForRequest(); + let toolSearchState = _toolSearchState; // Build system prompt const systemPrompt = request.systemPrompt || this.config.systemPrompt || ""; @@ -1144,21 +1398,48 @@ export class Runtime { _accumulatedMessages, _isRecursive, _httpRequest, + toolSearchState, )) { yield event; } return; } + const nativeToolCatalog = nativeToolSearch + ? this.buildNativeToolCatalogForRequest(request, allTools) + : null; + const selectedTools = + nativeToolCatalog ?? + this.selectToolsForRequest(request, allTools, toolSearchState); + const toolSearchTool = nativeToolSearch + ? null + : this.createToolSearchTool(request, allTools, selectedTools); + const effectiveSelectedTools = nativeToolCatalog + ? nativeToolCatalog + : toolSearchTool + ? [...selectedTools, toolSearchTool] + : selectedTools; + const providerToolOptions = this.buildProviderToolOptionsForRequest( + effectiveSelectedTools, + ); + const selectedToolMap = new Map( + effectiveSelectedTools.map((tool) => [tool.name, tool] as const), + ); + // Create completion request const completionRequest: ChatCompletionRequest = { messages: [], rawMessages: conversationMessages, - actions: this.convertToolsToActions(allTools), + actions: nativeToolSearch + ? undefined + : this.convertToolsToActions(effectiveSelectedTools), + toolDefinitions: nativeToolSearch ? effectiveSelectedTools : undefined, systemPrompt: systemPrompt, config: request.config, signal, webSearch: this.getWebSearchConfig(), + providerToolOptions, + debug, }; try { @@ -1199,7 +1480,7 @@ export class Runtime { const clientToolCalls: ToolCallInfo[] = []; for (const tc of result.toolCalls) { - const tool = allTools.find((t) => t.name === tc.name); + const tool = selectedToolMap.get(tc.name); if (tool?.location === "server" && tool.handler) { serverToolCalls.push(tc); } else { @@ -1213,7 +1494,7 @@ export class Runtime { // Emit tool call events for (const tc of result.toolCalls) { - const tool = allTools.find((t) => t.name === tc.name); + const tool = selectedToolMap.get(tc.name); yield { type: "action:start", id: tc.id, @@ -1241,7 +1522,7 @@ export class Runtime { "toolContext" in this.config ? this.config.toolContext : undefined; for (const tc of serverToolCalls) { - const tool = allTools.find((t) => t.name === tc.name); + const tool = selectedToolMap.get(tc.name); if (tool?.handler) { if (debug) { console.log(`[Copilot SDK] Executing tool: ${tc.name}`); @@ -1268,6 +1549,7 @@ export class Runtime { yield { type: "action:end", id: tc.id, + name: tc.name, result: toolResult, } as StreamEvent; } catch (error) { @@ -1288,6 +1570,7 @@ export class Runtime { yield { type: "action:end", id: tc.id, + name: tc.name, error: error instanceof Error ? error.message @@ -1344,6 +1627,13 @@ export class Runtime { Record >), ]; + toolSearchState = this.extendLoadedToolNames( + toolSearchState, + serverToolResults.map((toolResult) => ({ + name: toolResult.name, + result: toolResult.result, + })), + ); // Continue loop continue; diff --git a/packages/llm-sdk/src/server/tool-selection.ts b/packages/llm-sdk/src/server/tool-selection.ts new file mode 100644 index 0000000..a12cd3a --- /dev/null +++ b/packages/llm-sdk/src/server/tool-selection.ts @@ -0,0 +1,587 @@ +import type { + AnthropicProviderToolOptions, + OpenAIProviderToolOptions, + ProviderToolRuntimeOptions, + ToolDefinition, + ToolSearchConfig, + ToolSelectionConfig, +} from "../core/stream-events"; + +type ToolSelectionMessage = { + role: string; + content?: unknown; +}; + +export interface ToolSearchMatch { + name: string; + description: string; + location?: ToolDefinition["location"]; + category?: string; + group?: string; + profiles?: string[]; + searchKeywords?: string[]; + score: number; +} + +export interface ResolvedNativeToolSearch { + provider: "anthropic" | "openai"; + variant?: "bm25" | "regex"; + useResponsesApi?: boolean; +} + +const BM25_K1 = 1.2; +const BM25_B = 0.75; + +function unique(values: T[]): T[] { + return [...new Set(values)]; +} + +function tokenize(text: string): string[] { + return text + .toLowerCase() + .replace(/[^a-z0-9_\s-]/g, " ") + .split(/\s+/) + .filter((token) => token.length > 1); +} + +function stringifyContent(content: unknown): string { + if (typeof content === "string") { + return content; + } + if (!content) { + return ""; + } + try { + return JSON.stringify(content); + } catch { + return String(content); + } +} + +function buildToolQuery(messages: ToolSelectionMessage[]): string { + return messages + .filter( + (message) => message.role === "user" || message.role === "assistant", + ) + .slice(-3) + .map((message) => stringifyContent(message.content)) + .filter(Boolean) + .join(" "); +} + +function buildSearchText(tool: ToolDefinition): string { + return [ + tool.name.replace(/_/g, " "), + tool.description, + tool.category ?? "", + tool.group ?? "", + ...(tool.profiles ?? []), + ...(tool.searchKeywords ?? []), + ] + .filter(Boolean) + .join(" "); +} + +function matchesSelector( + tool: ToolDefinition, + selector: string, + activeProfile?: string, +): boolean { + const normalized = selector.trim().toLowerCase(); + if (!normalized) { + return false; + } + + if (normalized === "*" || normalized === "all") { + return true; + } + if (normalized === tool.name.toLowerCase()) { + return true; + } + if (normalized.startsWith("group:")) { + return (tool.group ?? "").toLowerCase() === normalized.slice(6); + } + if (normalized.startsWith("category:")) { + return (tool.category ?? "").toLowerCase() === normalized.slice(9); + } + if (normalized.startsWith("profile:")) { + return (tool.profiles ?? []) + .map((value) => value.toLowerCase()) + .includes(normalized.slice(8)); + } + if (activeProfile && normalized === activeProfile.toLowerCase()) { + return (tool.profiles ?? []) + .map((value) => value.toLowerCase()) + .includes(normalized); + } + return false; +} + +function scoreTool( + tool: ToolDefinition, + queryTokens: string[], + activeProfile?: string, +): number { + const haystack = [ + tool.name, + tool.description, + tool.category, + tool.group, + ...(tool.profiles ?? []), + ...(tool.searchKeywords ?? []), + ] + .filter(Boolean) + .join(" ") + .toLowerCase(); + + let score = tool.deferLoading ? 0 : 2; + if (activeProfile && tool.profiles?.includes(activeProfile)) { + score += 2; + } + + for (const token of queryTokens) { + if (tool.name.toLowerCase() === token) { + score += 6; + } else if (tool.name.toLowerCase().includes(token)) { + score += 4; + } else if (haystack.includes(token)) { + score += 2; + } + } + + return score; +} + +export function filterToolsByProfile(params: { + tools: ToolDefinition[]; + config?: ToolSelectionConfig; + activeProfile?: string; +}): ToolDefinition[] { + const available = params.tools.filter((tool) => tool.available !== false); + const config = params.config; + if (!config?.enabled) { + return available; + } + + const activeProfile = params.activeProfile ?? config.defaultProfile; + const includeUnprofiled = config.includeUnprofiled ?? true; + const profile = activeProfile ? config.profiles?.[activeProfile] : undefined; + let filtered = available; + + if (profile?.include?.length) { + filtered = filtered.filter( + (tool) => + profile.include!.some((selector) => + matchesSelector(tool, selector, activeProfile), + ) || + (!!activeProfile && tool.profiles?.includes(activeProfile)), + ); + } else if (activeProfile) { + filtered = filtered.filter((tool) => { + if (tool.profiles?.length) { + return tool.profiles.includes(activeProfile); + } + return includeUnprofiled; + }); + } + + if (profile?.exclude?.length) { + filtered = filtered.filter( + (tool) => + !profile.exclude!.some((selector) => + matchesSelector(tool, selector, activeProfile), + ), + ); + } + + return filtered; +} + +function calculateBM25Score( + tool: ToolDefinition, + queryTerms: string[], + idf: Map, + avgDocLength: number, + activeProfile?: string, +): number { + const text = buildSearchText(tool); + const tokens = tokenize(text); + const docLength = Math.max(1, tokens.length); + + let score = 0; + + for (const term of queryTerms) { + const termFreq = tokens.filter((token) => token === term).length; + if (termFreq === 0) { + continue; + } + + const termIDF = idf.get(term) ?? 0; + const numerator = termFreq * (BM25_K1 + 1); + const denominator = + termFreq + BM25_K1 * (1 - BM25_B + BM25_B * (docLength / avgDocLength)); + score += termIDF * (numerator / denominator); + } + + const nameLower = tool.name.toLowerCase(); + for (const term of queryTerms) { + if (nameLower === term) { + score += 3; + } else if (nameLower.includes(term)) { + score += 1.5; + } + } + + if (activeProfile && tool.profiles?.includes(activeProfile)) { + score += 0.75; + } + + return score; +} + +export function selectTools(params: { + tools: ToolDefinition[]; + messages: ToolSelectionMessage[]; + config?: ToolSelectionConfig; + activeProfile?: string; + forceIncludeNames?: string[]; +}): ToolDefinition[] { + const config = params.config; + const available = filterToolsByProfile({ + tools: params.tools, + config, + activeProfile: params.activeProfile, + }); + if (!config?.enabled) { + return available; + } + const activeProfile = params.activeProfile ?? config.defaultProfile; + const forceIncludeNames = new Set(params.forceIncludeNames ?? []); + let filtered = available; + const strictDeferredLoading = + config.search?.enabled && config.search.strictDeferredLoading === true; + + if (strictDeferredLoading) { + filtered = filtered.filter( + (tool) => !tool.deferLoading || forceIncludeNames.has(tool.name), + ); + } + + if (!config.dynamicSelection?.enabled) { + if (forceIncludeNames.size === 0) { + return filtered; + } + const merged = new Map(filtered.map((tool) => [tool.name, tool])); + for (const tool of available) { + if (forceIncludeNames.has(tool.name)) { + merged.set(tool.name, tool); + } + } + return [...merged.values()]; + } + + if (filtered.length === 0) { + return filtered; + } + + const maxTools = Math.max( + 1, + Math.min( + config.dynamicSelection.maxTools ?? filtered.length, + filtered.length, + ), + ); + const queryTokens = unique(tokenize(buildToolQuery(params.messages))); + const ranked = [...filtered].sort((left, right) => { + const scoreDiff = + scoreTool(right, queryTokens, activeProfile) - + scoreTool(left, queryTokens, activeProfile); + if (scoreDiff !== 0) { + return scoreDiff; + } + return left.name.localeCompare(right.name); + }); + + if (forceIncludeNames.size === 0) { + return ranked.slice(0, maxTools); + } + + const forced = ranked.filter((tool) => forceIncludeNames.has(tool.name)); + const others = ranked.filter((tool) => !forceIncludeNames.has(tool.name)); + const remainingSlots = Math.max(0, maxTools - forced.length); + return [...forced, ...others.slice(0, remainingSlots)]; +} + +export function searchTools(params: { + tools: ToolDefinition[]; + query: string; + config?: ToolSelectionConfig; + activeProfile?: string; + limit?: number; + excludeNames?: string[]; + includeSelected?: boolean; +}): ToolSearchMatch[] { + const queryTerms = unique(tokenize(params.query)); + if (queryTerms.length === 0) { + return []; + } + + const candidates = filterToolsByProfile({ + tools: params.tools, + config: params.config, + activeProfile: params.activeProfile, + }).filter((tool) => { + if ((params.excludeNames ?? []).includes(tool.name)) { + return false; + } + if (params.includeSelected) { + return true; + } + return tool.deferLoading === true; + }); + + if (candidates.length === 0) { + return []; + } + + const docs = candidates.map((tool) => tokenize(buildSearchText(tool))); + const avgDocLength = + docs.reduce((sum, tokens) => sum + Math.max(1, tokens.length), 0) / + docs.length; + const idf = new Map(); + + for (const term of queryTerms) { + const docFreq = docs.reduce( + (count, tokens) => count + (tokens.includes(term) ? 1 : 0), + 0, + ); + idf.set( + term, + Math.log((docs.length - docFreq + 0.5) / (docFreq + 0.5) + 1), + ); + } + + const minScore = params.config?.search?.minScore ?? 0.1; + const limit = Math.max( + 1, + params.limit ?? params.config?.search?.maxResults ?? 5, + ); + const activeProfile = params.activeProfile ?? params.config?.defaultProfile; + + return candidates + .map((tool) => ({ + tool, + score: calculateBM25Score( + tool, + queryTerms, + idf, + avgDocLength, + activeProfile, + ), + })) + .filter((entry) => entry.score >= minScore) + .sort((left, right) => { + const scoreDiff = right.score - left.score; + if (scoreDiff !== 0) { + return scoreDiff; + } + return left.tool.name.localeCompare(right.tool.name); + }) + .slice(0, limit) + .map(({ tool, score }) => ({ + name: tool.name, + description: tool.description, + location: tool.location, + category: tool.category, + group: tool.group, + profiles: tool.profiles, + searchKeywords: tool.searchKeywords, + score: Number(score.toFixed(4)), + })); +} + +function normalizeModelName(modelName?: string): string { + return (modelName ?? "").trim().toLowerCase(); +} + +export function supportsAnthropicNativeToolSearch(modelName?: string): boolean { + const model = normalizeModelName(modelName); + if (!model) { + return false; + } + + if (model.includes("haiku")) { + return false; + } + + return ( + /(?:^|[-_ ])(?:sonnet|opus)[-_ ]?4(?:$|[-_. ])/.test(model) || + /claude[-_ ](?:sonnet|opus)[-_ ]?4/.test(model) || + /claude[-_ ]?4[-_ ](?:sonnet|opus)/.test(model) + ); +} + +export function supportsOpenAINativeToolSearch(modelName?: string): boolean { + const model = normalizeModelName(modelName); + if (!model) { + return false; + } + + const match = model.match(/^gpt-5(?:[._-](\d+))?(?:$|[._-])/); + if (!match) { + return false; + } + + const minorVersion = match[1] ? Number.parseInt(match[1], 10) : Number.NaN; + if (!Number.isFinite(minorVersion)) { + return false; + } + + return minorVersion >= 4; +} + +export function resolveNativeToolSearch(params: { + providerName: string; + modelName?: string; + config?: ToolSelectionConfig; +}): ResolvedNativeToolSearch | null { + const searchConfig = params.config?.search; + if (!searchConfig?.enabled) { + return null; + } + + const mode = searchConfig.mode ?? "auto"; + if (mode === "manual") { + return null; + } + + if ( + params.providerName === "anthropic" && + supportsAnthropicNativeToolSearch(params.modelName) + ) { + return { + provider: "anthropic", + variant: searchConfig.anthropicVariant ?? "bm25", + }; + } + + if ( + params.providerName === "openai" && + supportsOpenAINativeToolSearch(params.modelName) + ) { + return { + provider: "openai", + useResponsesApi: true, + }; + } + + return mode === "native" ? null : null; +} + +export function shouldExposeToolSearch(params: { + tools: ToolDefinition[]; + selectedTools: ToolDefinition[]; + config?: ToolSelectionConfig; +}): boolean { + const searchConfig = params.config?.search; + if (!searchConfig?.enabled) { + return false; + } + + const deferredCount = params.tools.filter((tool) => tool.deferLoading).length; + if (deferredCount === 0) { + return false; + } + + const threshold = searchConfig.exposeWhenToolCountExceeds ?? 8; + return ( + params.tools.length >= threshold || + deferredCount > Math.max(0, params.selectedTools.length) + ); +} + +export function buildProviderToolOptions(params: { + providerName: string; + modelName?: string; + selectedTools: ToolDefinition[]; + config?: ToolSelectionConfig; + metaToolName?: string; +}): ProviderToolRuntimeOptions | undefined { + const nativeHints = params.config?.nativeProviderHints; + const resolvedNativeSearch = resolveNativeToolSearch({ + providerName: params.providerName, + modelName: params.modelName, + config: params.config, + }); + const effectiveTools = params.metaToolName + ? params.selectedTools.filter((tool) => tool.name !== params.metaToolName) + : params.selectedTools; + + if (params.providerName === "openai") { + const hints = nativeHints?.openai; + if (!hints && !resolvedNativeSearch) { + return undefined; + } + + let toolChoice: OpenAIProviderToolOptions["toolChoice"]; + if (hints?.toolChoice === "required") { + toolChoice = "required"; + } else if (hints?.toolChoice === "single" && effectiveTools.length === 1) { + toolChoice = { + type: "function", + name: effectiveTools[0].name, + }; + } else if (hints?.toolChoice === "auto") { + toolChoice = "auto"; + } + + return { + openai: { + toolChoice, + parallelToolCalls: hints?.parallelToolCalls, + nativeToolSearch: + resolvedNativeSearch?.provider === "openai" + ? { + enabled: true, + useResponsesApi: resolvedNativeSearch.useResponsesApi, + } + : undefined, + }, + }; + } + + if (params.providerName === "anthropic") { + const hints = nativeHints?.anthropic; + if (!hints && !resolvedNativeSearch) { + return undefined; + } + + let toolChoice: AnthropicProviderToolOptions["toolChoice"]; + if (hints?.toolChoice === "any") { + toolChoice = "any"; + } else if (hints?.toolChoice === "single" && effectiveTools.length === 1) { + toolChoice = { + type: "tool", + name: effectiveTools[0].name, + }; + } else if (hints?.toolChoice === "auto") { + toolChoice = "auto"; + } + + return { + anthropic: { + toolChoice, + disableParallelToolUse: hints?.disableParallelToolUse, + nativeToolSearch: + resolvedNativeSearch?.provider === "anthropic" + ? { + enabled: true, + variant: resolvedNativeSearch.variant ?? "bm25", + } + : undefined, + }, + }; + } + + return undefined; +} diff --git a/packages/llm-sdk/src/server/types.ts b/packages/llm-sdk/src/server/types.ts index af720d0..7ca15ed 100644 --- a/packages/llm-sdk/src/server/types.ts +++ b/packages/llm-sdk/src/server/types.ts @@ -143,12 +143,34 @@ export interface ChatRequest { tools?: Array<{ name: string; description: string; + category?: string; + group?: string; + deferLoading?: boolean; + profiles?: string[]; + searchKeywords?: string[]; inputSchema: { type: "object"; properties: Record; required?: string[]; }; }>; + /** Full client tool catalog used for server-side tool selection and deferred search. */ + toolCatalog?: Array<{ + name: string; + description: string; + category?: string; + group?: string; + deferLoading?: boolean; + profiles?: string[]; + searchKeywords?: string[]; + inputSchema: { + type: "object"; + properties: Record; + required?: string[]; + }; + }>; + /** Active tool profile to apply when agentLoop.toolSelection is enabled. */ + toolProfile?: string; /** Enable agentic loop mode */ useAgentLoop?: boolean; /** Enable streaming responses (default: true). Set to false for non-streaming mode. */ diff --git a/tool-search-implementation.md b/tool-search-implementation.md new file mode 100644 index 0000000..2dd6941 --- /dev/null +++ b/tool-search-implementation.md @@ -0,0 +1,253 @@ +# Tool Management Branch Summary + +Branch: `codex/tool-management-core` + +## Scope + +This branch adds the first full tool-management stack across `copilot-sdk`, `llm-sdk`, and the experimental demos. + +It covers: + +- tool profiles and selective loading +- deferred tool loading +- manual tool search fallback +- native provider tool search hooks for Anthropic and OpenAI +- prompt-side tool result truncation and context compaction groundwork +- mixed client/server tool catalog support +- provider payload debug logging +- experimental scale-testing demo with 100 tools + +## Main Features Added + +### 1. Framework-agnostic prompt/tool optimization (`copilot-sdk`) + +Added shared optimization support in the chat/core layer: + +- tool profile selection +- dynamic tool narrowing +- tool result truncation controls +- context budget reporting +- history compaction with continuity summaries + +Public APIs added: + +- `setOptimizationConfig(...)` +- `setToolProfile(...)` +- `getContextUsage()` + +Main files: + +- `packages/copilot-sdk/src/chat/optimizations.ts` +- `packages/copilot-sdk/src/chat/ChatWithTools.ts` +- `packages/copilot-sdk/src/chat/classes/AbstractChat.ts` +- `packages/copilot-sdk/src/core/types/tools.ts` + +### 2. Tool metadata and selection pipeline (`llm-sdk`) + +Added richer tool metadata and request-time selection: + +- `category` +- `group` +- `profiles` +- `searchKeywords` +- `deferLoading` + +Selection features: + +- profile-based filtering +- include/exclude selectors +- dynamic ranking by recent query/context +- strict deferred loading mode +- request-level `toolProfile` + +Main files: + +- `packages/llm-sdk/src/core/stream-events.ts` +- `packages/llm-sdk/src/server/tool-selection.ts` +- `packages/llm-sdk/src/server/runtime.ts` +- `packages/llm-sdk/src/server/agent-loop.ts` + +### 3. Manual deferred tool search fallback + +Added SDK-managed `search_tools` fallback for providers/models without native search support. + +Behavior: + +- full tool catalog stays on the server +- deferred tools stay out of the initial model-facing tool list +- model can call `search_tools` +- runtime loads matching deferred tools into the next loop iteration + +Supports: + +- mixed server tools + client tools +- profile-aware search +- BM25-style ranking + +Main files: + +- `packages/llm-sdk/src/server/tool-selection.ts` +- `packages/llm-sdk/src/server/runtime.ts` + +### 4. Native provider tool search support + +Added provider-aware search mode selection: + +- `search.mode = "auto" | "native" | "manual"` + +Current behavior: + +- Anthropic Sonnet 4 / Opus 4 supported models -> native Anthropic search path +- OpenAI `gpt-5.4+` supported models -> internal OpenAI Responses-based native path +- all other providers/models -> manual `search_tools` fallback + +Anthropic native path: + +- adds `tool_search_tool_bm25_20251119` or regex variant +- passes deferred tools with `defer_loading: true` + +OpenAI native path: + +- uses internal Responses-based adapter branch +- keeps public SDK/frontend usage unchanged + +Main files: + +- `packages/llm-sdk/src/adapters/anthropic.ts` +- `packages/llm-sdk/src/adapters/openai.ts` +- `packages/llm-sdk/src/server/tool-selection.ts` + +### 5. Mixed client/server catalog support + +Added `toolCatalog` transport support so the runtime can search/select from the full catalog: + +- server tools from runtime config +- client tools registered in the browser + +This allows deferred client tools to be discovered by search even when they are not initially exposed to the model. + +Main files: + +- `packages/copilot-sdk/src/chat/interfaces/ChatTransport.ts` +- `packages/copilot-sdk/src/chat/adapters/HttpTransport.ts` +- `packages/copilot-sdk/src/chat/classes/AbstractChat.ts` +- `packages/llm-sdk/src/server/types.ts` +- `packages/llm-sdk/src/server/runtime.ts` + +### 6. Provider payload logging + +Added adapter-level debug payload logging for request/response inspection. + +Supported across: + +- OpenAI +- Anthropic +- Azure +- Google +- xAI +- Ollama + +Current behavior: + +- logs request payloads +- logs final provider responses +- suppresses per-event stream spam + +Main file: + +- `packages/llm-sdk/src/adapters/base.ts` + +### 7. Experimental Tool Scale Lab + +Added a dedicated experimental demo for scale testing: + +- 100 tools total +- 30 server tools +- 70 client tools +- profile switching +- deferred loading +- manual/native search path testing +- provider behavior testing + +Main files: + +- `examples/experimental/app/tool-scale/page.tsx` +- `examples/experimental/app/api/chat/tool-scale/route.ts` +- `examples/experimental/lib/tool-scale/catalog.ts` +- `examples/experimental/lib/tool-scale/server-tools.ts` +- `examples/experimental/lib/tool-scale/client-tools.ts` + +## Config Examples + +### Runtime tool selection + +```ts +agentLoop: { + enabled: true, + toolSelection: { + enabled: true, + defaultProfile: "support", + includeUnprofiled: false, + dynamicSelection: { + enabled: true, + maxTools: 6, + }, + search: { + enabled: true, + mode: "auto", + strictDeferredLoading: true, + maxResults: 6, + metaToolName: "search_tools", + anthropicVariant: "bm25", + }, + }, +} +``` + +### Client-side optimization + +```ts +optimization: { + toolProfiles: { + enabled: true, + defaultProfile: "support", + }, + toolResultConfig: { + truncation: { + enabled: true, + strategy: "smart", + hardMaxChars: 12000, + }, + }, + contextManagement: { + enabled: true, + history: { + maxMessages: 20, + pruneStrategy: "summarize", + }, + }, + contextBudget: { + enabled: true, + budget: { + contextWindowTokens: 128000, + toolResultsShare: 0.3, + }, + }, +} +``` + +## Current Known Caveats + +These are not fully closed out yet: + +- mixed same-turn server + client tool calls still need more hardening in the runtime loop +- OpenAI manual fallback + continuation path needs more validation +- OpenAI native Responses path currently preserves the SDK contract, but is not full event-by-event Responses streaming yet +- no dedicated automated tests were added in this branch yet + +## Suggested Next Steps + +- add tests for tool selection, deferred loading, and continuation ordering +- tighten manual search scoring so profile-only matches do not leak through +- harden mixed same-turn server/client tool execution ordering +- improve OpenAI Responses-native streaming parity From a4ff7339ad79c5be72975bf2244e728765603136 Mon Sep 17 00:00:00 2001 From: Sahil Date: Thu, 12 Mar 2026 12:22:26 +0530 Subject: [PATCH 06/72] refactor(sdk): flatten agentLoop API, auto-detect deferred tools, unified tool search Breaking: removed agentLoop config block (use flat maxIterations + toolSearch), removed toolCatalog from ChatRequest (use tools only), removed AgentLoopConfig / ToolSelectionConfig / ToolDynamicSelectionConfig from public API, removed strictDeferredLoading and providerHints (use flat toolChoice/parallelCalls in ToolSearchConfig). New: auto-detection of deferred tools (zero config, deferLoading flag on tool is enough), flat ToolSearchConfig with description/name/maxEagerTools/maxResults/ exposeWhenExceeds/toolChoice/parallelCalls/defaultProfile/profiles/includeUnprofiled, native provider tool search auto-detected for Anthropic Sonnet/Opus 4+ and GPT-5.4+, ToolSearchConfig now exported from server index, full BM25 rewrite in tool-selection.ts (selectTools, searchTools, filterToolsByProfile, shouldExposeToolSearch, resolveNativeToolSearch, buildProviderToolOptions), AgentLoopOptions flattened to maxIterations/debug/toolSelectionConfig. copilot-sdk: useTool hook gains deferLoading/profiles/searchKeywords/group/category/ resultConfig/title/executingTitle/completedTitle/aiResponseMode/aiContext props; CopilotProvider maxIterations promoted to top-level; AbstractChat.buildRequest sends full tools array with deferLoading flags intact for server-side deferred management; HttpTransport drops toolCatalog from request body. Examples: express-demo migrated to flat API with profiles and debug payload capture; tool-scale example updated for 100-tool deferred loading; other routes updated. Co-Authored-By: Claude Sonnet 4.6 --- .../experimental/app/api/chat/openai/route.ts | 4 +- .../app/api/chat/test-server-tools/route.ts | 4 +- .../app/api/chat/tool-scale/route.ts | 104 +++--- examples/experimental/app/tool-scale/page.tsx | 19 +- examples/express-demo/src/index.ts | 209 +++++++++-- .../src/chat/adapters/HttpTransport.ts | 1 - .../src/chat/classes/AbstractChat.ts | 90 +---- .../src/chat/interfaces/ChatTransport.ts | 2 - .../copilot-sdk/src/react/hooks/useTool.ts | 37 ++ .../src/react/provider/CopilotProvider.tsx | 5 + packages/llm-sdk/src/core/stream-events.ts | 50 --- packages/llm-sdk/src/index.ts | 4 - packages/llm-sdk/src/server/agent-loop.ts | 32 +- packages/llm-sdk/src/server/index.ts | 1 + packages/llm-sdk/src/server/runtime.ts | 100 ++++-- packages/llm-sdk/src/server/tool-selection.ts | 339 ++++++------------ packages/llm-sdk/src/server/types.ts | 96 +++-- 17 files changed, 531 insertions(+), 566 deletions(-) diff --git a/examples/experimental/app/api/chat/openai/route.ts b/examples/experimental/app/api/chat/openai/route.ts index 402408d..f19714a 100644 --- a/examples/experimental/app/api/chat/openai/route.ts +++ b/examples/experimental/app/api/chat/openai/route.ts @@ -12,9 +12,7 @@ const runtime = createRuntime({ "You are a helpful assistant powered by OpenAI GPT-4o. Be concise and helpful.", debug: process.env.NODE_ENV === "development", // For testing max iterations - set to 2 to easily trigger the limit - agentLoop: { - maxIterations: 2, - }, + maxIterations: 2, }); export async function POST(request: Request) { diff --git a/examples/experimental/app/api/chat/test-server-tools/route.ts b/examples/experimental/app/api/chat/test-server-tools/route.ts index de2e4f3..dbfaca8 100644 --- a/examples/experimental/app/api/chat/test-server-tools/route.ts +++ b/examples/experimental/app/api/chat/test-server-tools/route.ts @@ -43,9 +43,7 @@ const runtime = createRuntime({ "You are a helpful assistant. Use the get_random_number tool when asked for random numbers.", debug: true, tools: serverTools, - agentLoop: { - maxIterations: 2, // Low limit for testing - }, + maxIterations: 2, // Low limit for testing }); export async function POST(request: Request) { diff --git a/examples/experimental/app/api/chat/tool-scale/route.ts b/examples/experimental/app/api/chat/tool-scale/route.ts index 12d9cc6..be8c8ea 100644 --- a/examples/experimental/app/api/chat/tool-scale/route.ts +++ b/examples/experimental/app/api/chat/tool-scale/route.ts @@ -45,74 +45,52 @@ Use tools sparingly and intentionally. When tools are missing, rely on the search_tools meta-tool to discover deferred tools rather than guessing. Keep answers short and explain which class of tools you used when it helps the user understand tool selection behavior.`, tools: toolScaleServerTools, - agentLoop: { - enabled: true, - maxIterations: 6, - debug: process.env.NODE_ENV === "development", - toolSelection: { - enabled: true, - defaultProfile: "support", - includeUnprofiled: false, - search: { - enabled: true, - maxResults: 6, - minScore: 0.15, - exposeWhenToolCountExceeds: 12, - metaToolName: "search_tools", - strictDeferredLoading: true, + maxIterations: 6, + toolSearch: { + maxResults: 6, + exposeWhenExceeds: 12, + maxEagerTools: 6, + defaultProfile: "support", + includeUnprofiled: false, + profiles: { + support: { + include: [ + "profile:support", + "category:knowledge", + "category:billing", + "category:browser", + "category:utility", + ], + exclude: ["group:admin"], }, - dynamicSelection: { - enabled: true, - maxTools: 6, + workspace: { + include: [ + "profile:workspace", + "category:workspace", + "category:browser", + "category:analytics", + "category:utility", + ], }, - profiles: { - support: { - include: [ - "profile:support", - "category:knowledge", - "category:billing", - "category:browser", - "category:utility", - ], - exclude: ["group:admin"], - }, - workspace: { - include: [ - "profile:workspace", - "category:workspace", - "category:browser", - "category:analytics", - "category:utility", - ], - }, - commerce: { - include: [ - "profile:commerce", - "category:commerce", - "category:billing", - "group:actions", - ], - }, - admin: { - include: [ - "profile:admin", - "category:operations", - "category:analytics", - "category:utility", - ], - }, + commerce: { + include: [ + "profile:commerce", + "category:commerce", + "category:billing", + "group:actions", + ], }, - nativeProviderHints: { - anthropic: { - toolChoice: "auto", - disableParallelToolUse: true, - }, - openai: { - toolChoice: "auto", - parallelToolCalls: false, - }, + admin: { + include: [ + "profile:admin", + "category:operations", + "category:analytics", + "category:utility", + ], }, }, + toolChoice: "auto", + parallelCalls: false, }, }); diff --git a/examples/experimental/app/tool-scale/page.tsx b/examples/experimental/app/tool-scale/page.tsx index 4178a3f..2c4a6ae 100644 --- a/examples/experimental/app/tool-scale/page.tsx +++ b/examples/experimental/app/tool-scale/page.tsx @@ -47,20 +47,15 @@ const requestSnippet = `{ ] }`; -const selectionSnippet = `toolSelection: { - enabled: true, +const selectionSnippet = `toolSearch: { + maxResults: 6, + exposeWhenExceeds: 12, + maxEagerTools: 6, defaultProfile: "support", includeUnprofiled: false, - search: { - enabled: true, - maxResults: 6, - exposeWhenToolCountExceeds: 12, - metaToolName: "search_tools", - strictDeferredLoading: true, - }, - dynamicSelection: { - enabled: true, - maxTools: 6, + profiles: { + support: { include: ["profile:support", "category:knowledge"] }, + workspace: { include: ["profile:workspace", "category:workspace"] }, }, }`; diff --git a/examples/express-demo/src/index.ts b/examples/express-demo/src/index.ts index 88ba6c7..2a28d85 100644 --- a/examples/express-demo/src/index.ts +++ b/examples/express-demo/src/index.ts @@ -197,43 +197,23 @@ IMPORTANT: When the user asks about YourGPT, the SDK, pricing, or any product-re Be helpful, concise, and accurate. If the knowledge base doesn't have the answer, say so.`, debug: true, tools: serverTools, - agentLoop: { - enabled: true, - maxIterations: 5, - debug: true, - toolSelection: { - enabled: true, - defaultProfile: "support", - includeUnprofiled: true, - search: { - enabled: true, - maxResults: 3, - exposeWhenToolCountExceeds: 1, + maxIterations: 5, + toolSearch: { + maxResults: 3, + exposeWhenExceeds: 1, + maxEagerTools: 2, + defaultProfile: "support", + includeUnprofiled: true, + profiles: { + support: { + include: ["category:knowledge", "search_knowledge_base"], + exclude: ["group:time"], }, - dynamicSelection: { - enabled: true, - maxTools: 2, - }, - profiles: { - support: { - include: ["category:knowledge", "search_knowledge_base"], - exclude: ["group:time"], - }, - utility: { - include: ["category:utility", "get_current_time"], - }, - }, - nativeProviderHints: { - anthropic: { - toolChoice: "single", - disableParallelToolUse: true, - }, - openai: { - toolChoice: "single", - parallelToolCalls: false, - }, + utility: { + include: ["category:utility", "get_current_time"], }, }, + parallelCalls: false, }, }); @@ -242,28 +222,179 @@ Be helpful, concise, and accurate. If the knowledge base doesn't have the answer // ============================================ const minimalRuntime = createRuntime({ - provider, - model, + // provider: openai, + provider: anthropic, + model: "claude-haiku-4-5", + // model: "gpt-5.4", systemPrompt: "You are a helpful AI assistant.", + debug: true, // enables logProviderPayload() calls in adapters }); +// ============================================ +// PER-REQUEST DEBUG LOG CAPTURE +// Captures what the SDK sends to Anthropic/OpenAI +// ============================================ + +interface CapturedLog { + label: string; + payload: unknown; +} + +function captureProviderLogs(fn: () => Promise): Promise { + const captured: CapturedLog[] = []; + const origLog = console.log; + + console.log = (...args: unknown[]) => { + origLog(...args); // still print to terminal + const line = args.map((a) => (typeof a === "string" ? a : "")).join(" "); + // SDK logs format: "[llm-sdk:anthropic] request payload\n{...json...}" + const match = line.match(/^\[llm-sdk:[^\]]+\] (.+?)\n([\s\S]*)$/); + if (match) { + const label = match[1].trim(); + const raw = match[2].trim(); + try { + captured.push({ label, payload: JSON.parse(raw) }); + } catch { + captured.push({ label, payload: raw }); + } + } + }; + + return fn() + .catch((err) => { + throw err; + }) + .finally(() => { + console.log = origLog; + }) + .then(() => captured); +} + // ============================================ // MINIMAL COPILOT RESPONSE ENDPOINT // ============================================ +// ============================================ +// LOGGING HELPERS +// ============================================ + +function logRequest(endpoint: string, body: Record) { + const { tools, messages, systemPrompt, ...rest } = body; + console.log(`\n${"═".repeat(60)}`); + console.log(`▶ REQUEST ${endpoint}`); + console.log(`${"─".repeat(60)}`); + if (systemPrompt) { + console.log(`[systemPrompt]\n${systemPrompt}`); + console.log(`${"─".repeat(60)}`); + } + if (Array.isArray(messages)) { + console.log(`[messages] (${messages.length})`); + for (const m of messages as Record[]) { + const role = String(m.role ?? "?").padEnd(10); + const content = + typeof m.content === "string" + ? m.content.slice(0, 300) + (m.content.length > 300 ? "…" : "") + : JSON.stringify(m.content ?? "").slice(0, 300); + console.log(` ${role} ${content}`); + } + console.log(`${"─".repeat(60)}`); + } + if (Array.isArray(tools) && tools.length > 0) { + const toolNames = (tools as Record[]).map( + (t) => t.name ?? "?", + ); + console.log(`[tools] (${tools.length}): ${toolNames.join(", ")}`); + console.log(`${"─".repeat(60)}`); + } + if (Object.keys(rest).length > 0) { + console.log(`[config] ${JSON.stringify(rest, null, 2)}`); + console.log(`${"─".repeat(60)}`); + } +} + +function logResponse(result: Record) { + console.log(`${"─".repeat(60)}`); + console.log(`◀ RESPONSE`); + console.log(`${"─".repeat(60)}`); + if (result.text) { + const text = String(result.text); + console.log( + `[text]\n${text.slice(0, 800)}${text.length > 800 ? "\n…(truncated)" : ""}`, + ); + } + if (Array.isArray(result.toolCalls) && result.toolCalls.length > 0) { + console.log(`[toolCalls]`); + console.log(JSON.stringify(result.toolCalls, null, 2)); + } + if (result.usage) { + console.log(`[usage] ${JSON.stringify(result.usage)}`); + } + console.log(`${"═".repeat(60)}\n`); +} + /** * Minimal streaming endpoint - no tools, simple prompt */ app.post("/api/copilot-response", async (req, res) => { - await minimalRuntime.stream(req.body).pipeToResponse(res); + logRequest("/api/copilot-response (stream)", req.body); + + let fullText = ""; + const stream = minimalRuntime.stream(req.body); + stream.on("text", (chunk: string) => { + fullText += chunk; + }); + stream.on("error", (err: Error) => { + console.error(`[/api/copilot-response] Stream error:`, err.message); + }); + + await stream.pipeToResponse(res); + + // Log assembled response after stream completes + logResponse({ text: fullText }); }); /** * Minimal non-streaming endpoint - no tools, simple prompt */ app.post("/api/copilot-response/chat", async (req, res) => { - const result = await minimalRuntime.chat(req.body); - res.json(result); + logRequest("/api/copilot-response/chat", req.body); + + let result: Awaited>; + const { tools, messages, systemPrompt, ...restConfig } = req.body; + + const providerLogs = await captureProviderLogs(async () => { + result = await minimalRuntime.chat(req.body); + }); + + logResponse(result! as unknown as Record); + + // Split captured logs into request/response pairs + const aiRequest = + providerLogs.find((l) => l.label.includes("request payload"))?.payload ?? + null; + const aiResponse = + providerLogs.find((l) => l.label.includes("response"))?.payload ?? null; + + res.json({ + ...result!, + _debug: { + // What the SDK client sent to this server + sdkRequest: { + systemPrompt: systemPrompt ?? null, + messageCount: Array.isArray(messages) ? messages.length : 0, + messages: messages ?? [], + toolCount: Array.isArray(tools) ? tools.length : 0, + toolNames: Array.isArray(tools) + ? (tools as { name?: string }[]).map((t) => t.name) + : [], + config: restConfig, + }, + // Raw payload this server sent to Anthropic/OpenAI + aiProviderRequest: aiRequest, + // Raw response back from Anthropic/OpenAI + aiProviderResponse: aiResponse, + }, + }); }); // ============================================ diff --git a/packages/copilot-sdk/src/chat/adapters/HttpTransport.ts b/packages/copilot-sdk/src/chat/adapters/HttpTransport.ts index 3e74624..63a500a 100644 --- a/packages/copilot-sdk/src/chat/adapters/HttpTransport.ts +++ b/packages/copilot-sdk/src/chat/adapters/HttpTransport.ts @@ -90,7 +90,6 @@ export class HttpTransport implements ChatTransport { systemPrompt: request.systemPrompt, llm: request.llm, tools: request.tools, - toolCatalog: request.toolCatalog, actions: request.actions, streaming: this.config.streaming, ...(resolved.configBody as Record), diff --git a/packages/copilot-sdk/src/chat/classes/AbstractChat.ts b/packages/copilot-sdk/src/chat/classes/AbstractChat.ts index 859f19c..3a242b2 100644 --- a/packages/copilot-sdk/src/chat/classes/AbstractChat.ts +++ b/packages/copilot-sdk/src/chat/classes/AbstractChat.ts @@ -13,8 +13,6 @@ import type { ContextUsage, MessageAttachment, - AIResponseMode, - ToolResponse, ToolDefinition, ToolOptimizationConfig, } from "../../core"; @@ -46,89 +44,6 @@ import { import { SimpleChatState } from "../interfaces/ChatState"; import { ChatContextOptimizer } from "../optimizations"; -// ============================================ -// AI Response Control Helper -// ============================================ - -/** - * Tool definition with AI response control fields - */ -interface ToolWithAIConfig { - name: string; - aiResponseMode?: AIResponseMode; - aiContext?: - | string - | ((result: ToolResponse, args: Record) => string); -} - -/** - * Build tool result content for AI based on aiResponseMode and aiContext - * This transforms client-side tool results before sending to the LLM - * - * Priority for responseMode: result._aiResponseMode > tool.aiResponseMode > "full" - * Priority for context: result._aiContext > tool.aiContext > undefined - * - * @param result - The tool result (may include _aiResponseMode, _aiContext, _aiContent) - * @param tool - Optional tool definition with aiResponseMode and aiContext - * @param args - Tool arguments (for dynamic aiContext functions) - * @returns The content string to send to the AI - */ -function buildToolResultContentForAI( - result: unknown, - tool?: ToolWithAIConfig, - args?: Record, -): string { - if (typeof result === "string") return result; - - const typedResult = result as ToolResponse | null; - - // Priority: result._aiResponseMode > tool.aiResponseMode > "full" - const responseMode = - typedResult?._aiResponseMode ?? tool?.aiResponseMode ?? "full"; - - // Check for multimodal content - if (typedResult?._aiContent) { - return JSON.stringify(typedResult._aiContent); - } - - // Get AI context: result._aiContext > tool.aiContext (string or function) - let aiContext: string | undefined = typedResult?._aiContext; - if (!aiContext && tool?.aiContext) { - aiContext = - typeof tool.aiContext === "function" - ? tool.aiContext(typedResult as ToolResponse, args ?? {}) - : tool.aiContext; - } - - switch (responseMode) { - case "none": - return aiContext ?? "[Result displayed to user]"; - - case "brief": - return aiContext ?? "[Tool executed successfully]"; - - case "full": - default: - if (aiContext) { - // Include context as prefix, then full data (without the control fields) - const { - _aiResponseMode, - _aiContext, - _aiContent, - _uiResources, - ...dataOnly - } = typedResult ?? {}; - return `${aiContext}\n\nFull data: ${JSON.stringify(dataOnly)}`; - } - // Strip UI resources from full result - they're for rendering, not for AI - if (typedResult?._uiResources) { - const { _uiResources, ...dataOnly } = typedResult; - return JSON.stringify(dataOnly); - } - return JSON.stringify(result); - } -} - /** * Event types emitted by AbstractChat */ @@ -663,8 +578,7 @@ export class AbstractChat { threadId: this.config.threadId, systemPrompt, llm: this.config.llm, - tools: optimized.tools?.length ? optimized.tools : undefined, - toolCatalog: this.config.tools?.length + tools: this.config.tools?.length ? this.config.tools.map((tool) => ({ name: tool.name, description: tool.description, @@ -699,7 +613,6 @@ export class AbstractChat { this.debug("handleStreamResponse", "Starting to process stream"); let chunkCount = 0; - let hasError = false; let toolCallsEmitted = false; // Guard to prevent emitting toolCalls twice // Process stream chunks @@ -709,7 +622,6 @@ export class AbstractChat { // Handle error chunks immediately if (chunk.type === "error") { - hasError = true; const error = new Error(chunk.message || "Stream error"); this.handleError(error); return; diff --git a/packages/copilot-sdk/src/chat/interfaces/ChatTransport.ts b/packages/copilot-sdk/src/chat/interfaces/ChatTransport.ts index 9ea76f4..59e8b31 100644 --- a/packages/copilot-sdk/src/chat/interfaces/ChatTransport.ts +++ b/packages/copilot-sdk/src/chat/interfaces/ChatTransport.ts @@ -28,8 +28,6 @@ export interface ChatRequest { llm?: Record; /** Tool definitions */ tools?: unknown[]; - /** Full client-side tool catalog for server-side selection/search */ - toolCatalog?: unknown[]; /** Action definitions */ actions?: unknown[]; /** Additional body properties */ diff --git a/packages/copilot-sdk/src/react/hooks/useTool.ts b/packages/copilot-sdk/src/react/hooks/useTool.ts index 028833d..49e6c04 100644 --- a/packages/copilot-sdk/src/react/hooks/useTool.ts +++ b/packages/copilot-sdk/src/react/hooks/useTool.ts @@ -8,6 +8,8 @@ import type { ToolRenderProps, ToolSet, ToolInputSchema, + AIResponseMode, + ToolResultConfig, } from "../../core"; import { zodToJsonSchema } from "../../core"; import { useCopilot } from "../provider/CopilotProvider"; @@ -61,6 +63,30 @@ export interface UseToolConfig> { * @default false */ hidden?: boolean; + /** Deferred tools stay out of the default request payload; discovered only when query matches */ + deferLoading?: boolean; + /** Profile memberships for selective tool loading */ + profiles?: string[]; + /** Extra keywords for dynamic tool-selection scoring */ + searchKeywords?: string[]; + /** Optional group for profile-based selection */ + group?: string; + /** Optional category for search, filtering, and budgets */ + category?: string; + /** Per-tool prompt/result shaping controls */ + resultConfig?: ToolResultConfig; + /** Human-readable title for UI display */ + title?: string | ((args: TParams) => string); + /** Title shown while executing */ + executingTitle?: string | ((args: TParams) => string); + /** Title shown after completion */ + completedTitle?: string | ((args: TParams) => string); + /** How the AI should respond when this tool's result is rendered as UI */ + aiResponseMode?: AIResponseMode; + /** Context/summary sent to AI instead of full result */ + aiContext?: + | string + | ((result: ToolResponse, args: Record) => string); } /** @@ -137,6 +163,17 @@ export function useTool>( approvalMessage: config.approvalMessage as ToolDefinition["approvalMessage"], hidden: config.hidden, + deferLoading: config.deferLoading, + profiles: config.profiles, + searchKeywords: config.searchKeywords, + group: config.group, + category: config.category, + resultConfig: config.resultConfig, + title: config.title as ToolDefinition["title"], + executingTitle: config.executingTitle as ToolDefinition["executingTitle"], + completedTitle: config.completedTitle as ToolDefinition["completedTitle"], + aiResponseMode: config.aiResponseMode, + aiContext: config.aiContext as ToolDefinition["aiContext"], }; // Register tool diff --git a/packages/copilot-sdk/src/react/provider/CopilotProvider.tsx b/packages/copilot-sdk/src/react/provider/CopilotProvider.tsx index 531d697..1ea9439 100644 --- a/packages/copilot-sdk/src/react/provider/CopilotProvider.tsx +++ b/packages/copilot-sdk/src/react/provider/CopilotProvider.tsx @@ -25,6 +25,7 @@ import type { ActionDefinition, MessageAttachment, PermissionLevel, + ToolOptimizationConfig, } from "../../core"; import type { MCPServerConfig } from "../../mcp/types"; @@ -104,6 +105,8 @@ export interface CopilotProviderProps { maxIterationsMessage?: string; /** MCP servers to connect to automatically */ mcpServers?: MCPServerConfig[]; + /** Optional prompt/tool optimization controls (tool profiles, context budgets, etc.) */ + optimization?: ToolOptimizationConfig; } export interface CopilotContextValue { @@ -196,6 +199,7 @@ export function CopilotProvider({ maxIterations, maxIterationsMessage, mcpServers, + optimization, }: CopilotProviderProps) { // Debug logger const debugLog = useCallback( @@ -261,6 +265,7 @@ export function CopilotProvider({ debug, maxIterations, maxIterationsMessage, + optimization, }, { onToolExecutionsChange: (executions) => { diff --git a/packages/llm-sdk/src/core/stream-events.ts b/packages/llm-sdk/src/core/stream-events.ts index 9611dad..a6bcba3 100644 --- a/packages/llm-sdk/src/core/stream-events.ts +++ b/packages/llm-sdk/src/core/stream-events.ts @@ -467,33 +467,6 @@ export interface ToolProfile { exclude?: string[]; } -export interface ToolDynamicSelectionConfig { - enabled?: boolean; - maxTools?: number; -} - -export interface ToolSearchConfig { - enabled?: boolean; - /** - * Search execution mode. - * - auto: use native provider search when supported, otherwise fall back to manual search_tools - * - native: require provider-native search when supported, otherwise fall back to manual search_tools - * - manual: always use the SDK-managed search_tools fallback - */ - mode?: "auto" | "native" | "manual"; - metaToolName?: string; - maxResults?: number; - minScore?: number; - exposeWhenToolCountExceeds?: number; - /** Anthropic native tool search variant. Defaults to bm25. */ - anthropicVariant?: "bm25" | "regex"; - /** - * When true, tools marked with deferLoading stay hidden from the initial - * selected tool list and are only introduced after search_tools loads them. - */ - strictDeferredLoading?: boolean; -} - export interface OpenAIToolSelectionHints { /** * "single" forces the selected tool when exactly one tool remains after selection. @@ -519,19 +492,6 @@ export interface ToolNativeProviderHints { anthropic?: AnthropicToolSelectionHints; } -export interface ToolSelectionConfig { - enabled?: boolean; - defaultProfile?: string; - profiles?: Record; - /** When false, active profiles exclude tools without explicit profile membership. */ - includeUnprofiled?: boolean; - dynamicSelection?: ToolDynamicSelectionConfig; - /** Optional indexed search over deferred tools. */ - search?: ToolSearchConfig; - /** Optional provider-native hints layered on top of local tool selection. */ - nativeProviderHints?: ToolNativeProviderHints; -} - export interface OpenAIProviderToolOptions { toolChoice?: | "auto" @@ -567,16 +527,6 @@ export interface ProviderToolRuntimeOptions { anthropic?: AnthropicProviderToolOptions; } -/** - * Agent loop configuration - */ -export interface AgentLoopConfig { - maxIterations?: number; - debug?: boolean; - enabled?: boolean; - toolSelection?: ToolSelectionConfig; -} - /** * Web search configuration for native provider search * diff --git a/packages/llm-sdk/src/index.ts b/packages/llm-sdk/src/index.ts index cc22c30..507f1d7 100644 --- a/packages/llm-sdk/src/index.ts +++ b/packages/llm-sdk/src/index.ts @@ -190,14 +190,10 @@ export type { UnifiedToolCall, UnifiedToolResult, ToolExecution, - AgentLoopConfig, ToolProfile, - ToolDynamicSelectionConfig, - ToolSearchConfig, OpenAIToolSelectionHints, AnthropicToolSelectionHints, ToolNativeProviderHints, - ToolSelectionConfig, OpenAIProviderToolOptions, AnthropicProviderToolOptions, ProviderToolRuntimeOptions, diff --git a/packages/llm-sdk/src/server/agent-loop.ts b/packages/llm-sdk/src/server/agent-loop.ts index 0363770..959a759 100644 --- a/packages/llm-sdk/src/server/agent-loop.ts +++ b/packages/llm-sdk/src/server/agent-loop.ts @@ -16,7 +16,6 @@ import type { UnifiedToolCall, UnifiedToolResult, ToolResponse, - AgentLoopConfig, Message, } from "../core/stream-events"; import type { AIProvider } from "../providers/types"; @@ -27,6 +26,7 @@ import { searchTools, selectTools, shouldExposeToolSearch, + type InternalToolSelectionConfig, } from "./tool-selection"; // ======================================== @@ -54,8 +54,12 @@ export interface AgentLoopOptions { provider: AIProvider; /** Abort signal for cancellation */ signal?: AbortSignal; - /** Loop configuration */ - config?: AgentLoopConfig; + /** Max agent loop iterations (default: 20) */ + maxIterations?: number; + /** Enable debug logging */ + debug?: boolean; + /** Internal tool selection config (from resolveEffectiveToolSelectionConfig) */ + toolSelectionConfig?: InternalToolSelectionConfig; /** Optional active tool profile for selective loading. */ toolProfile?: string; /** @@ -114,18 +118,19 @@ export async function* runAgentLoop( systemPrompt, provider, signal, - config, + maxIterations: optMaxIterations, + debug: optDebug, + toolSelectionConfig, toolProfile, callLLM, executeServerTool, waitForClientToolResult, } = options; - const maxIterations = config?.maxIterations ?? DEFAULT_MAX_ITERATIONS; - const debug = config?.debug ?? false; + const maxIterations = optMaxIterations ?? DEFAULT_MAX_ITERATIONS; + const debug = optDebug ?? false; const formatter = getFormatter(provider.name); - const toolSearchMetaToolName = - config?.toolSelection?.search?.metaToolName ?? "search_tools"; + const toolSearchMetaToolName = "search_tools"; // Separate server and client tools const serverTools = tools.filter((t) => t.location === "server"); @@ -147,7 +152,7 @@ export async function* runAgentLoop( availableToolCount: allTools.length, serverToolCount: serverTools.length, clientToolCount: clientTools.length, - activeProfile: toolProfile ?? config?.toolSelection?.defaultProfile, + activeProfile: toolProfile ?? toolSelectionConfig?.defaultProfile, maxIterations, }); } @@ -176,14 +181,13 @@ export async function* runAgentLoop( const selectedTools = selectTools({ tools: allTools, messages, - config: config?.toolSelection, + config: toolSelectionConfig, activeProfile: toolProfile, forceIncludeNames: [...loadedToolNames], }); const toolSearchTool = shouldExposeToolSearch({ tools: allTools, - selectedTools, - config: config?.toolSelection, + config: toolSelectionConfig, }) ? ({ name: toolSearchMetaToolName, @@ -212,7 +216,7 @@ export async function* runAgentLoop( const results = searchTools({ tools: allTools, query, - config: config?.toolSelection, + config: toolSelectionConfig, activeProfile: toolProfile, limit, excludeNames: selectedTools.map((tool) => tool.name), @@ -232,7 +236,7 @@ export async function* runAgentLoop( const providerToolOptions = buildProviderToolOptions({ providerName: provider.name, selectedTools: effectiveSelectedTools, - config: config?.toolSelection, + config: toolSelectionConfig, metaToolName: toolSearchMetaToolName, }); const providerTools = formatter.transformTools(effectiveSelectedTools); diff --git a/packages/llm-sdk/src/server/index.ts b/packages/llm-sdk/src/server/index.ts index c7258fc..9ed321f 100644 --- a/packages/llm-sdk/src/server/index.ts +++ b/packages/llm-sdk/src/server/index.ts @@ -2,6 +2,7 @@ export { Runtime, createRuntime } from "./runtime"; export type { RuntimeConfig, + ToolSearchConfig, ChatRequest, ActionRequest, RequestContext, diff --git a/packages/llm-sdk/src/server/runtime.ts b/packages/llm-sdk/src/server/runtime.ts index 23c0c52..48a63ac 100644 --- a/packages/llm-sdk/src/server/runtime.ts +++ b/packages/llm-sdk/src/server/runtime.ts @@ -33,6 +33,7 @@ import { searchTools, selectTools, shouldExposeToolSearch, + type InternalToolSelectionConfig, } from "./tool-selection"; type ToolSearchState = { @@ -345,7 +346,6 @@ export class Runtime { console.log("[Copilot SDK] Request:", { messageCount: body.messages?.length ?? 0, toolCount: body.tools?.length ?? 0, - toolCatalogCount: body.toolCatalog?.length ?? 0, hasSystemPrompt: Boolean(body.systemPrompt), threadId: body.threadId, streaming: body.streaming !== false, @@ -356,10 +356,10 @@ export class Runtime { // Create abort controller from request signal const signal = request.signal; - // Use agent loop if tools are present or explicitly enabled + // Use agent loop if tools are present const hasTools = (body.tools && body.tools.length > 0) || this.tools.size > 0; - const useAgentLoop = hasTools || this.config.agentLoop?.enabled; + const useAgentLoop = hasTools; // NON-STREAMING: Return JSON response instead of SSE if (body.streaming === false) { @@ -683,17 +683,41 @@ export class Runtime { return undefined; } + /** + * Resolve effective tool selection config for a request. + */ + private resolveEffectiveToolSelectionConfig( + request: ChatRequest, + ): InternalToolSelectionConfig | undefined { + const toolSearch = + "toolSearch" in this.config ? this.config.toolSearch : undefined; + + const hasDeferredServerTool = [...this.tools.values()].some( + (t) => t.deferLoading, + ); + const hasDeferredInRequest = request.tools?.some((t) => t.deferLoading); + + if (!hasDeferredServerTool && !hasDeferredInRequest && !toolSearch) { + return undefined; + } + + return { + maxEagerTools: toolSearch?.maxEagerTools ?? 20, + maxResults: toolSearch?.maxResults ?? 8, + exposeWhenExceeds: toolSearch?.exposeWhenExceeds ?? 8, + toolChoice: toolSearch?.toolChoice, + parallelCalls: toolSearch?.parallelCalls, + defaultProfile: toolSearch?.defaultProfile, + profiles: toolSearch?.profiles, + includeUnprofiled: toolSearch?.includeUnprofiled, + }; + } + private collectToolsForRequest(request: ChatRequest): ToolDefinition[] { const allTools: ToolDefinition[] = [...this.tools.values()]; - const clientTools = - this.config.agentLoop?.toolSelection?.enabled && - request.toolCatalog?.length - ? request.toolCatalog - : request.tools; - - if (clientTools) { - for (const tool of clientTools) { + if (request.tools) { + for (const tool of request.tools) { allTools.push({ name: tool.name, description: tool.description, @@ -719,17 +743,19 @@ export class Runtime { return selectTools({ tools: allTools, messages: request.messages, - config: this.config.agentLoop?.toolSelection, + config: this.resolveEffectiveToolSelectionConfig(request), activeProfile: request.toolProfile, forceIncludeNames: toolSearchState?.loadedToolNames, }); } - private resolveNativeToolSearchForRequest(): NativeToolSearchState { + private resolveNativeToolSearchForRequest( + request: ChatRequest, + ): NativeToolSearchState { return resolveNativeToolSearch({ providerName: this.adapter.provider, modelName: this.getModel(), - config: this.config.agentLoop?.toolSelection, + config: this.resolveEffectiveToolSelectionConfig(request), }); } @@ -739,26 +765,28 @@ export class Runtime { ): ToolDefinition[] { return filterToolsByProfile({ tools: allTools, - config: this.config.agentLoop?.toolSelection, + config: this.resolveEffectiveToolSelectionConfig(request), activeProfile: request.toolProfile, }); } - private buildProviderToolOptionsForRequest(selectedTools: ToolDefinition[]) { + private buildProviderToolOptionsForRequest( + selectedTools: ToolDefinition[], + request: ChatRequest, + ) { return buildProviderToolOptions({ providerName: this.adapter.provider, modelName: this.getModel(), selectedTools, - config: this.config.agentLoop?.toolSelection, + config: this.resolveEffectiveToolSelectionConfig(request), metaToolName: this.getToolSearchMetaToolName(), }); } private getToolSearchMetaToolName(): string { - return ( - this.config.agentLoop?.toolSelection?.search?.metaToolName ?? - "search_tools" - ); + const toolSearch = + "toolSearch" in this.config ? this.config.toolSearch : undefined; + return toolSearch?.name ?? "search_tools"; } private createToolSearchTool( @@ -769,8 +797,7 @@ export class Runtime { if ( !shouldExposeToolSearch({ tools: allTools, - selectedTools, - config: this.config.agentLoop?.toolSelection, + config: this.resolveEffectiveToolSelectionConfig(request), }) ) { return null; @@ -782,6 +809,7 @@ export class Runtime { return { name: toolName, description: + ("toolSearch" in this.config && this.config.toolSearch?.description) || "Search available deferred tools and load the most relevant ones for the next step when the right tool is not currently exposed.", location: "server", hidden: true, @@ -804,18 +832,16 @@ export class Runtime { const results = searchTools({ tools: allTools, query: args.query, - config: this.config.agentLoop?.toolSelection, + config: this.resolveEffectiveToolSelectionConfig(request), activeProfile: request.toolProfile, limit: args.limit, excludeNames: excludedNames, }); - if (this.config.debug || this.config.agentLoop?.debug) { + if (this.config.debug) { console.log("[Copilot SDK] search_tools result:", { query: args.query, - activeProfile: - request.toolProfile ?? - this.config.agentLoop?.toolSelection?.defaultProfile, + activeProfile: request.toolProfile, selectedToolCount: selectedTools.length, catalogCount: allTools.length, loadedTools: results.map((result) => result.name), @@ -893,7 +919,7 @@ export class Runtime { _httpRequest?: Request, _toolSearchState?: ToolSearchState, ): AsyncGenerator { - const debug = this.config.debug || this.config.agentLoop?.debug; + const debug = this.config.debug; // Check if non-streaming mode is requested // Use non-streaming for better comparison with original studio-ai behavior @@ -917,10 +943,10 @@ export class Runtime { // Track new messages created during this request const newMessages: DoneEventMessage[] = _accumulatedMessages || []; - const maxIterations = this.config.agentLoop?.maxIterations || 20; + const maxIterations = this.config.maxIterations ?? 20; const allTools = this.collectToolsForRequest(request); - const nativeToolSearch = this.resolveNativeToolSearchForRequest(); + const nativeToolSearch = this.resolveNativeToolSearchForRequest(request); const nativeToolCatalog = nativeToolSearch ? this.buildNativeToolCatalogForRequest(request, allTools) : null; @@ -937,6 +963,7 @@ export class Runtime { : selectedTools; const providerToolOptions = this.buildProviderToolOptionsForRequest( effectiveSelectedTools, + request, ); const selectedToolMap = new Map( effectiveSelectedTools.map((tool) => [tool.name, tool] as const), @@ -950,9 +977,7 @@ export class Runtime { console.log( `[Copilot SDK] Tool selection active: ${effectiveSelectedTools.length}/${allTools.length} tools`, { - activeProfile: - request.toolProfile ?? - this.config.agentLoop?.toolSelection?.defaultProfile, + activeProfile: request.toolProfile, nativeSearch: nativeToolSearch?.provider ?? null, }, ); @@ -1343,8 +1368,8 @@ export class Runtime { _toolSearchState?: ToolSearchState, ): AsyncGenerator { const newMessages: DoneEventMessage[] = _accumulatedMessages || []; - const debug = this.config.debug || this.config.agentLoop?.debug; - const maxIterations = this.config.agentLoop?.maxIterations || 20; + const debug = this.config.debug; + const maxIterations = this.config.maxIterations ?? 20; // Track accumulated usage across iterations (for onFinish callback) let accumulatedUsage: { prompt_tokens: number; @@ -1353,7 +1378,7 @@ export class Runtime { } = { prompt_tokens: 0, completion_tokens: 0, total_tokens: 0 }; const allTools = this.collectToolsForRequest(request); - const nativeToolSearch = this.resolveNativeToolSearchForRequest(); + const nativeToolSearch = this.resolveNativeToolSearchForRequest(request); let toolSearchState = _toolSearchState; // Build system prompt @@ -1421,6 +1446,7 @@ export class Runtime { : selectedTools; const providerToolOptions = this.buildProviderToolOptionsForRequest( effectiveSelectedTools, + request, ); const selectedToolMap = new Map( effectiveSelectedTools.map((tool) => [tool.name, tool] as const), diff --git a/packages/llm-sdk/src/server/tool-selection.ts b/packages/llm-sdk/src/server/tool-selection.ts index a12cd3a..616e423 100644 --- a/packages/llm-sdk/src/server/tool-selection.ts +++ b/packages/llm-sdk/src/server/tool-selection.ts @@ -1,10 +1,9 @@ import type { + ToolDefinition, + ToolProfile, AnthropicProviderToolOptions, OpenAIProviderToolOptions, ProviderToolRuntimeOptions, - ToolDefinition, - ToolSearchConfig, - ToolSelectionConfig, } from "../core/stream-events"; type ToolSelectionMessage = { @@ -29,8 +28,25 @@ export interface ResolvedNativeToolSearch { useResponsesApi?: boolean; } +/** + * Internal tool selection configuration. + * Built by resolveEffectiveToolSelectionConfig in runtime.ts. + * Not part of the public API. + */ +export interface InternalToolSelectionConfig { + maxEagerTools: number; + maxResults: number; + exposeWhenExceeds: number; + defaultProfile?: string; + profiles?: Record; + includeUnprofiled?: boolean; + toolChoice?: "auto" | "required"; + parallelCalls?: boolean; +} + const BM25_K1 = 1.2; const BM25_B = 0.75; +const MIN_SCORE = 0.1; function unique(values: T[]): T[] { return [...new Set(values)]; @@ -45,12 +61,8 @@ function tokenize(text: string): string[] { } function stringifyContent(content: unknown): string { - if (typeof content === "string") { - return content; - } - if (!content) { - return ""; - } + if (typeof content === "string") return content; + if (!content) return ""; try { return JSON.stringify(content); } catch { @@ -60,11 +72,9 @@ function stringifyContent(content: unknown): string { function buildToolQuery(messages: ToolSelectionMessage[]): string { return messages - .filter( - (message) => message.role === "user" || message.role === "assistant", - ) + .filter((m) => m.role === "user" || m.role === "assistant") .slice(-3) - .map((message) => stringifyContent(message.content)) + .map((m) => stringifyContent(m.content)) .filter(Boolean) .join(" "); } @@ -88,16 +98,9 @@ function matchesSelector( activeProfile?: string, ): boolean { const normalized = selector.trim().toLowerCase(); - if (!normalized) { - return false; - } - - if (normalized === "*" || normalized === "all") { - return true; - } - if (normalized === tool.name.toLowerCase()) { - return true; - } + if (!normalized) return false; + if (normalized === "*" || normalized === "all") return true; + if (normalized === tool.name.toLowerCase()) return true; if (normalized.startsWith("group:")) { return (tool.group ?? "").toLowerCase() === normalized.slice(6); } @@ -106,12 +109,12 @@ function matchesSelector( } if (normalized.startsWith("profile:")) { return (tool.profiles ?? []) - .map((value) => value.toLowerCase()) + .map((v) => v.toLowerCase()) .includes(normalized.slice(8)); } if (activeProfile && normalized === activeProfile.toLowerCase()) { return (tool.profiles ?? []) - .map((value) => value.toLowerCase()) + .map((v) => v.toLowerCase()) .includes(normalized); } return false; @@ -135,18 +138,12 @@ function scoreTool( .toLowerCase(); let score = tool.deferLoading ? 0 : 2; - if (activeProfile && tool.profiles?.includes(activeProfile)) { - score += 2; - } + if (activeProfile && tool.profiles?.includes(activeProfile)) score += 2; for (const token of queryTokens) { - if (tool.name.toLowerCase() === token) { - score += 6; - } else if (tool.name.toLowerCase().includes(token)) { - score += 4; - } else if (haystack.includes(token)) { - score += 2; - } + if (tool.name.toLowerCase() === token) score += 6; + else if (tool.name.toLowerCase().includes(token)) score += 4; + else if (haystack.includes(token)) score += 2; } return score; @@ -154,14 +151,12 @@ function scoreTool( export function filterToolsByProfile(params: { tools: ToolDefinition[]; - config?: ToolSelectionConfig; + config?: InternalToolSelectionConfig; activeProfile?: string; }): ToolDefinition[] { const available = params.tools.filter((tool) => tool.available !== false); const config = params.config; - if (!config?.enabled) { - return available; - } + if (!config) return available; const activeProfile = params.activeProfile ?? config.defaultProfile; const includeUnprofiled = config.includeUnprofiled ?? true; @@ -178,9 +173,7 @@ export function filterToolsByProfile(params: { ); } else if (activeProfile) { filtered = filtered.filter((tool) => { - if (tool.profiles?.length) { - return tool.profiles.includes(activeProfile); - } + if (tool.profiles?.length) return tool.profiles.includes(activeProfile); return includeUnprofiled; }); } @@ -209,13 +202,9 @@ function calculateBM25Score( const docLength = Math.max(1, tokens.length); let score = 0; - for (const term of queryTerms) { - const termFreq = tokens.filter((token) => token === term).length; - if (termFreq === 0) { - continue; - } - + const termFreq = tokens.filter((t) => t === term).length; + if (termFreq === 0) continue; const termIDF = idf.get(term) ?? 0; const numerator = termFreq * (BM25_K1 + 1); const denominator = @@ -225,16 +214,11 @@ function calculateBM25Score( const nameLower = tool.name.toLowerCase(); for (const term of queryTerms) { - if (nameLower === term) { - score += 3; - } else if (nameLower.includes(term)) { - score += 1.5; - } + if (nameLower === term) score += 3; + else if (nameLower.includes(term)) score += 1.5; } - if (activeProfile && tool.profiles?.includes(activeProfile)) { - score += 0.75; - } + if (activeProfile && tool.profiles?.includes(activeProfile)) score += 0.75; return score; } @@ -242,7 +226,7 @@ function calculateBM25Score( export function selectTools(params: { tools: ToolDefinition[]; messages: ToolSelectionMessage[]; - config?: ToolSelectionConfig; + config?: InternalToolSelectionConfig; activeProfile?: string; forceIncludeNames?: string[]; }): ToolDefinition[] { @@ -252,97 +236,60 @@ export function selectTools(params: { config, activeProfile: params.activeProfile, }); - if (!config?.enabled) { - return available; - } + + // No config means no selection — return all available tools + if (!config) return available; + const activeProfile = params.activeProfile ?? config.defaultProfile; const forceIncludeNames = new Set(params.forceIncludeNames ?? []); - let filtered = available; - const strictDeferredLoading = - config.search?.enabled && config.search.strictDeferredLoading === true; - if (strictDeferredLoading) { - filtered = filtered.filter( - (tool) => !tool.deferLoading || forceIncludeNames.has(tool.name), - ); - } - - if (!config.dynamicSelection?.enabled) { - if (forceIncludeNames.size === 0) { - return filtered; - } - const merged = new Map(filtered.map((tool) => [tool.name, tool])); - for (const tool of available) { - if (forceIncludeNames.has(tool.name)) { - merged.set(tool.name, tool); - } - } - return [...merged.values()]; - } + // Always strip deferred tools from initial context (they're loaded via search) + let filtered = available.filter( + (tool) => !tool.deferLoading || forceIncludeNames.has(tool.name), + ); - if (filtered.length === 0) { - return filtered; - } + if (filtered.length === 0) return filtered; - const maxTools = Math.max( - 1, - Math.min( - config.dynamicSelection.maxTools ?? filtered.length, - filtered.length, - ), - ); + const maxTools = Math.max(1, Math.min(config.maxEagerTools, filtered.length)); const queryTokens = unique(tokenize(buildToolQuery(params.messages))); - const ranked = [...filtered].sort((left, right) => { - const scoreDiff = - scoreTool(right, queryTokens, activeProfile) - - scoreTool(left, queryTokens, activeProfile); - if (scoreDiff !== 0) { - return scoreDiff; - } - return left.name.localeCompare(right.name); + const ranked = [...filtered].sort((a, b) => { + const diff = + scoreTool(b, queryTokens, activeProfile) - + scoreTool(a, queryTokens, activeProfile); + return diff !== 0 ? diff : a.name.localeCompare(b.name); }); - if (forceIncludeNames.size === 0) { - return ranked.slice(0, maxTools); - } + if (forceIncludeNames.size === 0) return ranked.slice(0, maxTools); - const forced = ranked.filter((tool) => forceIncludeNames.has(tool.name)); - const others = ranked.filter((tool) => !forceIncludeNames.has(tool.name)); - const remainingSlots = Math.max(0, maxTools - forced.length); - return [...forced, ...others.slice(0, remainingSlots)]; + const forced = ranked.filter((t) => forceIncludeNames.has(t.name)); + const others = ranked.filter((t) => !forceIncludeNames.has(t.name)); + const remaining = Math.max(0, maxTools - forced.length); + return [...forced, ...others.slice(0, remaining)]; } export function searchTools(params: { tools: ToolDefinition[]; query: string; - config?: ToolSelectionConfig; + config?: InternalToolSelectionConfig; activeProfile?: string; limit?: number; excludeNames?: string[]; includeSelected?: boolean; }): ToolSearchMatch[] { const queryTerms = unique(tokenize(params.query)); - if (queryTerms.length === 0) { - return []; - } + if (queryTerms.length === 0) return []; const candidates = filterToolsByProfile({ tools: params.tools, config: params.config, activeProfile: params.activeProfile, }).filter((tool) => { - if ((params.excludeNames ?? []).includes(tool.name)) { - return false; - } - if (params.includeSelected) { - return true; - } + if ((params.excludeNames ?? []).includes(tool.name)) return false; + if (params.includeSelected) return true; return tool.deferLoading === true; }); - if (candidates.length === 0) { - return []; - } + if (candidates.length === 0) return []; const docs = candidates.map((tool) => tokenize(buildSearchText(tool))); const avgDocLength = @@ -361,11 +308,7 @@ export function searchTools(params: { ); } - const minScore = params.config?.search?.minScore ?? 0.1; - const limit = Math.max( - 1, - params.limit ?? params.config?.search?.maxResults ?? 5, - ); + const limit = Math.max(1, params.limit ?? params.config?.maxResults ?? 8); const activeProfile = params.activeProfile ?? params.config?.defaultProfile; return candidates @@ -379,13 +322,10 @@ export function searchTools(params: { activeProfile, ), })) - .filter((entry) => entry.score >= minScore) - .sort((left, right) => { - const scoreDiff = right.score - left.score; - if (scoreDiff !== 0) { - return scoreDiff; - } - return left.tool.name.localeCompare(right.tool.name); + .filter((entry) => entry.score >= MIN_SCORE) + .sort((a, b) => { + const diff = b.score - a.score; + return diff !== 0 ? diff : a.tool.name.localeCompare(b.tool.name); }) .slice(0, limit) .map(({ tool, score }) => ({ @@ -406,14 +346,7 @@ function normalizeModelName(modelName?: string): string { export function supportsAnthropicNativeToolSearch(modelName?: string): boolean { const model = normalizeModelName(modelName); - if (!model) { - return false; - } - - if (model.includes("haiku")) { - return false; - } - + if (!model || model.includes("haiku")) return false; return ( /(?:^|[-_ ])(?:sonnet|opus)[-_ ]?4(?:$|[-_. ])/.test(model) || /claude[-_ ](?:sonnet|opus)[-_ ]?4/.test(model) || @@ -423,122 +356,79 @@ export function supportsAnthropicNativeToolSearch(modelName?: string): boolean { export function supportsOpenAINativeToolSearch(modelName?: string): boolean { const model = normalizeModelName(modelName); - if (!model) { - return false; - } - + if (!model) return false; const match = model.match(/^gpt-5(?:[._-](\d+))?(?:$|[._-])/); - if (!match) { - return false; - } - + if (!match) return false; const minorVersion = match[1] ? Number.parseInt(match[1], 10) : Number.NaN; - if (!Number.isFinite(minorVersion)) { - return false; - } - + if (!Number.isFinite(minorVersion)) return false; return minorVersion >= 4; } export function resolveNativeToolSearch(params: { providerName: string; modelName?: string; - config?: ToolSelectionConfig; + config?: InternalToolSelectionConfig; }): ResolvedNativeToolSearch | null { - const searchConfig = params.config?.search; - if (!searchConfig?.enabled) { - return null; - } - - const mode = searchConfig.mode ?? "auto"; - if (mode === "manual") { - return null; - } + // No config means no deferred tools — no need for native search + if (!params.config) return null; if ( params.providerName === "anthropic" && supportsAnthropicNativeToolSearch(params.modelName) ) { - return { - provider: "anthropic", - variant: searchConfig.anthropicVariant ?? "bm25", - }; + return { provider: "anthropic", variant: "bm25" }; } if ( params.providerName === "openai" && supportsOpenAINativeToolSearch(params.modelName) ) { - return { - provider: "openai", - useResponsesApi: true, - }; + return { provider: "openai", useResponsesApi: true }; } - return mode === "native" ? null : null; + return null; } export function shouldExposeToolSearch(params: { tools: ToolDefinition[]; - selectedTools: ToolDefinition[]; - config?: ToolSelectionConfig; + config?: InternalToolSelectionConfig; }): boolean { - const searchConfig = params.config?.search; - if (!searchConfig?.enabled) { - return false; - } - - const deferredCount = params.tools.filter((tool) => tool.deferLoading).length; - if (deferredCount === 0) { - return false; - } - - const threshold = searchConfig.exposeWhenToolCountExceeds ?? 8; - return ( - params.tools.length >= threshold || - deferredCount > Math.max(0, params.selectedTools.length) - ); + if (!params.config) return false; + const deferredCount = params.tools.filter((t) => t.deferLoading).length; + if (deferredCount === 0) return false; + return params.tools.length >= params.config.exposeWhenExceeds; } export function buildProviderToolOptions(params: { providerName: string; modelName?: string; selectedTools: ToolDefinition[]; - config?: ToolSelectionConfig; + config?: InternalToolSelectionConfig; metaToolName?: string; }): ProviderToolRuntimeOptions | undefined { - const nativeHints = params.config?.nativeProviderHints; + const { toolChoice, parallelCalls } = params.config ?? {}; const resolvedNativeSearch = resolveNativeToolSearch({ providerName: params.providerName, modelName: params.modelName, config: params.config, }); - const effectiveTools = params.metaToolName - ? params.selectedTools.filter((tool) => tool.name !== params.metaToolName) - : params.selectedTools; if (params.providerName === "openai") { - const hints = nativeHints?.openai; - if (!hints && !resolvedNativeSearch) { + if ( + toolChoice === undefined && + parallelCalls === undefined && + !resolvedNativeSearch + ) { return undefined; } - - let toolChoice: OpenAIProviderToolOptions["toolChoice"]; - if (hints?.toolChoice === "required") { - toolChoice = "required"; - } else if (hints?.toolChoice === "single" && effectiveTools.length === 1) { - toolChoice = { - type: "function", - name: effectiveTools[0].name, - }; - } else if (hints?.toolChoice === "auto") { - toolChoice = "auto"; - } + let oaiToolChoice: OpenAIProviderToolOptions["toolChoice"]; + if (toolChoice === "required") oaiToolChoice = "required"; + else if (toolChoice === "auto") oaiToolChoice = "auto"; return { openai: { - toolChoice, - parallelToolCalls: hints?.parallelToolCalls, + toolChoice: oaiToolChoice, + parallelToolCalls: parallelCalls, nativeToolSearch: resolvedNativeSearch?.provider === "openai" ? { @@ -551,33 +441,26 @@ export function buildProviderToolOptions(params: { } if (params.providerName === "anthropic") { - const hints = nativeHints?.anthropic; - if (!hints && !resolvedNativeSearch) { + if ( + toolChoice === undefined && + parallelCalls === undefined && + !resolvedNativeSearch + ) { return undefined; } - - let toolChoice: AnthropicProviderToolOptions["toolChoice"]; - if (hints?.toolChoice === "any") { - toolChoice = "any"; - } else if (hints?.toolChoice === "single" && effectiveTools.length === 1) { - toolChoice = { - type: "tool", - name: effectiveTools[0].name, - }; - } else if (hints?.toolChoice === "auto") { - toolChoice = "auto"; - } + let anthropicToolChoice: AnthropicProviderToolOptions["toolChoice"]; + // "required" maps to Anthropic's "any" (force tool use) + if (toolChoice === "required") anthropicToolChoice = "any"; + else if (toolChoice === "auto") anthropicToolChoice = "auto"; return { anthropic: { - toolChoice, - disableParallelToolUse: hints?.disableParallelToolUse, + toolChoice: anthropicToolChoice, + // parallelCalls: false → disableParallelToolUse: true + disableParallelToolUse: parallelCalls === false ? true : undefined, nativeToolSearch: resolvedNativeSearch?.provider === "anthropic" - ? { - enabled: true, - variant: resolvedNativeSearch.variant ?? "bm25", - } + ? { enabled: true, variant: resolvedNativeSearch.variant ?? "bm25" } : undefined, }, }; diff --git a/packages/llm-sdk/src/server/types.ts b/packages/llm-sdk/src/server/types.ts index 7ca15ed..a438de4 100644 --- a/packages/llm-sdk/src/server/types.ts +++ b/packages/llm-sdk/src/server/types.ts @@ -2,12 +2,67 @@ import type { ActionDefinition, KnowledgeBaseConfig, ToolDefinition, - AgentLoopConfig, + ToolProfile, WebSearchConfig, } from "../core/stream-events"; import type { LLMAdapter } from "../adapters"; import type { AIProvider } from "../providers/types"; +/** + * Tool search/discovery configuration. + * Controls the `search_tools` meta-tool that lets the AI discover deferred tools. + * + * Tools marked with `deferLoading: true` are excluded from the default context + * and loaded on demand when the AI calls `search_tools`. + */ +export interface ToolSearchConfig { + /** + * Custom description for the search_tools meta-tool shown to the AI. + */ + description?: string; + /** + * Custom name for the search meta-tool (default: "search_tools"). + */ + name?: string; + /** + * Max eager tools sent to the AI per request (default: 20). + * Tools beyond this limit are deferred and discoverable via search. + */ + maxEagerTools?: number; + /** + * Max deferred tools returned per search query (default: 8). + */ + maxResults?: number; + /** + * Expose the search tool when total tool count exceeds this number (default: 8). + */ + exposeWhenExceeds?: number; + /** + * How the AI should choose tools. + * - "auto": model decides whether to use a tool (default) + * - "required": model must call at least one tool + */ + toolChoice?: "auto" | "required"; + /** + * Allow the model to call multiple tools in a single turn (default: true). + * Set false to force one tool call at a time. + */ + parallelCalls?: boolean; + /** + * Default active profile when none is provided in the request. + */ + defaultProfile?: string; + /** + * Named tool profiles with include/exclude selectors. + * Profiles filter which tools are visible to the AI per request. + */ + profiles?: Record; + /** + * When a profile is active, include tools with no profile membership (default: true). + */ + includeUnprofiled?: boolean; +} + /** * Runtime configuration with adapter (advanced usage) */ @@ -20,8 +75,15 @@ export interface RuntimeConfigWithAdapter { actions?: ActionDefinition[]; /** Available tools (new - supports location: server/client) */ tools?: ToolDefinition[]; - /** Agent loop configuration */ - agentLoop?: AgentLoopConfig; + /** + * Max agent loop iterations before stopping (default: 20). + */ + maxIterations?: number; + /** + * Configure deferred tool discovery. Tools with `deferLoading: true` are + * excluded from the default context and discoverable via the search meta-tool. + */ + toolSearch?: ToolSearchConfig; /** Knowledge base configuration (enables search_knowledge tool) */ knowledgeBase?: KnowledgeBaseConfig; /** Enable debug logging */ @@ -60,8 +122,15 @@ export interface RuntimeConfigWithProvider { actions?: ActionDefinition[]; /** Available tools (new - supports location: server/client) */ tools?: ToolDefinition[]; - /** Agent loop configuration */ - agentLoop?: AgentLoopConfig; + /** + * Max agent loop iterations before stopping (default: 20). + */ + maxIterations?: number; + /** + * Configure deferred tool discovery. Tools with `deferLoading: true` are + * excluded from the default context and discoverable via the search meta-tool. + */ + toolSearch?: ToolSearchConfig; /** Knowledge base configuration (enables search_knowledge tool) */ knowledgeBase?: KnowledgeBaseConfig; /** Enable debug logging */ @@ -154,22 +223,7 @@ export interface ChatRequest { required?: string[]; }; }>; - /** Full client tool catalog used for server-side tool selection and deferred search. */ - toolCatalog?: Array<{ - name: string; - description: string; - category?: string; - group?: string; - deferLoading?: boolean; - profiles?: string[]; - searchKeywords?: string[]; - inputSchema: { - type: "object"; - properties: Record; - required?: string[]; - }; - }>; - /** Active tool profile to apply when agentLoop.toolSelection is enabled. */ + /** Active tool profile to apply (filters tools by profile when toolSearch is configured). */ toolProfile?: string; /** Enable agentic loop mode */ useAgentLoop?: boolean; From 154e904bc7c31e3a6ca5164f45cd4ad53b8c16b8 Mon Sep 17 00:00:00 2001 From: Sahil Date: Thu, 12 Mar 2026 13:53:02 +0530 Subject: [PATCH 07/72] feat(sdk): add client-side skills system MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add skill-system/ (types, registry, load-skills, frontmatter) for defining and resolving skills - Add react/skill/ (SkillProvider, SkillContext, define-skill) for React integration - Add useSkill hook for per-component skill registration/unregistration - Add useSkillStatus hook for querying registered skill state - Add setInlineSkills to ChatWithTools (delegates to AbstractChat) - Add setInlineSkills + skills prop to CopilotProvider - Export all skills APIs from react/index.ts - Rename from skills/ to skill-system/ and react/skill/ to avoid .gitignore conflict with Claude Code skill folders - Fix source type mapping in SkillProvider (inline→client-inline, url→remote-url, file→server-dir) Co-Authored-By: Claude Sonnet 4.6 --- .../copilot-sdk/src/chat/ChatWithTools.ts | 14 + .../src/chat/classes/AbstractChat.ts | 26 ++ packages/copilot-sdk/src/react/hooks/index.ts | 4 + .../copilot-sdk/src/react/hooks/useSkill.ts | 78 +++++ .../src/react/hooks/useSkillStatus.ts | 51 +++ packages/copilot-sdk/src/react/index.ts | 16 + .../src/react/provider/CopilotProvider.tsx | 44 ++- .../src/react/skill/SkillContext.tsx | 34 ++ .../src/react/skill/SkillProvider.tsx | 230 +++++++++++++ .../src/react/skill/define-skill.ts | 29 ++ .../src/skill-system/frontmatter.ts | 98 ++++++ .../copilot-sdk/src/skill-system/index.ts | 21 ++ .../src/skill-system/load-skills.ts | 316 ++++++++++++++++++ .../copilot-sdk/src/skill-system/registry.ts | 109 ++++++ .../copilot-sdk/src/skill-system/types.ts | 94 ++++++ 15 files changed, 1163 insertions(+), 1 deletion(-) create mode 100644 packages/copilot-sdk/src/react/hooks/useSkill.ts create mode 100644 packages/copilot-sdk/src/react/hooks/useSkillStatus.ts create mode 100644 packages/copilot-sdk/src/react/skill/SkillContext.tsx create mode 100644 packages/copilot-sdk/src/react/skill/SkillProvider.tsx create mode 100644 packages/copilot-sdk/src/react/skill/define-skill.ts create mode 100644 packages/copilot-sdk/src/skill-system/frontmatter.ts create mode 100644 packages/copilot-sdk/src/skill-system/index.ts create mode 100644 packages/copilot-sdk/src/skill-system/load-skills.ts create mode 100644 packages/copilot-sdk/src/skill-system/registry.ts create mode 100644 packages/copilot-sdk/src/skill-system/types.ts diff --git a/packages/copilot-sdk/src/chat/ChatWithTools.ts b/packages/copilot-sdk/src/chat/ChatWithTools.ts index 501c6cd..1cc6eb4 100644 --- a/packages/copilot-sdk/src/chat/ChatWithTools.ts +++ b/packages/copilot-sdk/src/chat/ChatWithTools.ts @@ -492,6 +492,20 @@ export class ChatWithTools { this.chat.setBody(body); } + /** + * Set inline skills (forwarded to underlying chat instance) + */ + setInlineSkills( + skills: Array<{ + name: string; + description: string; + content: string; + strategy?: string; + }>, + ): void { + this.chat.setInlineSkills(skills); + } + // ============================================ // Tool Registration // ============================================ diff --git a/packages/copilot-sdk/src/chat/classes/AbstractChat.ts b/packages/copilot-sdk/src/chat/classes/AbstractChat.ts index 3a242b2..0a6bb0b 100644 --- a/packages/copilot-sdk/src/chat/classes/AbstractChat.ts +++ b/packages/copilot-sdk/src/chat/classes/AbstractChat.ts @@ -500,6 +500,31 @@ export class AbstractChat { return this.lastContextUsage; } + /** + * Inline skills from the client (sent on every request for server to merge) + */ + protected inlineSkills: Array<{ + name: string; + description: string; + content: string; + strategy?: string; + }> = []; + + /** + * Set inline skills (called by SkillProvider via React layer) + */ + setInlineSkills( + skills: Array<{ + name: string; + description: string; + content: string; + strategy?: string; + }>, + ): void { + this.inlineSkills = skills; + this.debug("Inline skills updated", { count: skills.length }); + } + /** * Dynamic context from useAIContext hook */ @@ -590,6 +615,7 @@ export class AbstractChat { inputSchema: tool.inputSchema, })) : undefined, + __skills: this.inlineSkills.length ? this.inlineSkills : undefined, }; } diff --git a/packages/copilot-sdk/src/react/hooks/index.ts b/packages/copilot-sdk/src/react/hooks/index.ts index 52581c7..4f85820 100644 --- a/packages/copilot-sdk/src/react/hooks/index.ts +++ b/packages/copilot-sdk/src/react/hooks/index.ts @@ -96,3 +96,7 @@ export { type UseMCPUIIntentsConfig, type UseMCPUIIntentsReturn, } from "./useMCPUIIntents"; + +// Skills Hooks +export { useSkill } from "./useSkill"; +export { useSkillStatus, type UseSkillStatusReturn } from "./useSkillStatus"; diff --git a/packages/copilot-sdk/src/react/hooks/useSkill.ts b/packages/copilot-sdk/src/react/hooks/useSkill.ts new file mode 100644 index 0000000..1e17bdc --- /dev/null +++ b/packages/copilot-sdk/src/react/hooks/useSkill.ts @@ -0,0 +1,78 @@ +"use client"; + +/** + * useSkill — register a skill from a React component + * + * Registers the skill on mount, unregisters on unmount. + * Must be used inside . + * + * Only inline skills are supported client-side. + * For file/url skills use loadSkills() on the server. + * + * @example + * ```tsx + * function CheckoutPage() { + * useSkill({ + * name: "checkout-flow", + * description: "Guides the user through checkout", + * strategy: "auto", + * source: { + * type: "inline", + * content: "When helping with checkout...", + * }, + * }); + * + * return ; + * } + * ``` + */ + +import { useEffect } from "react"; +import { useSkillContext } from "../skill/SkillContext"; +import type { SkillDefinition, ResolvedSkill } from "../../skill-system/types"; + +const DEV_CONTENT_WARN_THRESHOLD = 2000; + +export function useSkill(skill: SkillDefinition): void { + const { register, unregister } = useSkillContext(); + + // Warn in development if inline content is too large + if ( + process.env.NODE_ENV !== "production" && + skill.source.type === "inline" && + skill.source.content.length > DEV_CONTENT_WARN_THRESHOLD + ) { + console.warn( + `[copilot-sdk/skills] Inline skill "${skill.name}" has ${skill.source.content.length} characters. ` + + `Inline skills are sent on every request — keep them under ${DEV_CONTENT_WARN_THRESHOLD} characters. ` + + `Consider using a file or URL skill instead.`, + ); + } + + useEffect(() => { + if (skill.source.type !== "inline") { + console.warn( + `[copilot-sdk/skills] useSkill only supports inline skills client-side. ` + + `Skill "${skill.name}" has source type "${skill.source.type}" and will be skipped.`, + ); + return; + } + + const resolved: ResolvedSkill = { + ...skill, + content: skill.source.content, + }; + + register(resolved); + + return () => { + unregister(skill.name); + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [ + skill.name, + skill.source.type === "inline" ? skill.source.content : "", + skill.strategy, + skill.description, + ]); +} diff --git a/packages/copilot-sdk/src/react/hooks/useSkillStatus.ts b/packages/copilot-sdk/src/react/hooks/useSkillStatus.ts new file mode 100644 index 0000000..151da64 --- /dev/null +++ b/packages/copilot-sdk/src/react/hooks/useSkillStatus.ts @@ -0,0 +1,51 @@ +"use client"; + +/** + * useSkillStatus — observe the current skill registry state + * + * Returns a reactive snapshot of registered skills. + * Must be used inside . + * + * @example + * ```tsx + * function DebugPanel() { + * const { skills, count, has } = useSkillStatus(); + * + * return ( + *
+ *

{count} skill(s) registered

+ * {has("code-review") &&

Code review skill active

} + *
+ * ); + * } + * ``` + */ + +import { useCallback } from "react"; +import { useSkillContext } from "../skill/SkillContext"; +import type { ResolvedSkill } from "../../skill-system/types"; + +export interface UseSkillStatusReturn { + /** All currently registered skills */ + skills: ResolvedSkill[]; + /** Number of registered skills */ + count: number; + /** Check if a skill with the given name is registered */ + has: (name: string) => boolean; +} + +export function useSkillStatus(): UseSkillStatusReturn { + const { skills, registry } = useSkillContext(); + + const has = useCallback( + (name: string) => registry.has(name), + // eslint-disable-next-line react-hooks/exhaustive-deps + [skills], + ); + + return { + skills, + count: skills.length, + has, + }; +} diff --git a/packages/copilot-sdk/src/react/index.ts b/packages/copilot-sdk/src/react/index.ts index ed433b4..dfc6d88 100644 --- a/packages/copilot-sdk/src/react/index.ts +++ b/packages/copilot-sdk/src/react/index.ts @@ -202,3 +202,19 @@ export type { // Re-export tool helper function (Vercel AI SDK pattern) export { tool } from "../core"; + +// Skills System +export { SkillProvider, type SkillProviderProps } from "./skill/SkillProvider"; +export { defineSkill } from "./skill/define-skill"; +export { useSkill } from "./hooks/useSkill"; +export { + useSkillStatus, + type UseSkillStatusReturn, +} from "./hooks/useSkillStatus"; +export type { + SkillDefinition, + SkillSource, + SkillStrategy, + ResolvedSkill, + ClientInlineSkill, +} from "../skill-system/types"; diff --git a/packages/copilot-sdk/src/react/provider/CopilotProvider.tsx b/packages/copilot-sdk/src/react/provider/CopilotProvider.tsx index 1ea9439..35d2150 100644 --- a/packages/copilot-sdk/src/react/provider/CopilotProvider.tsx +++ b/packages/copilot-sdk/src/react/provider/CopilotProvider.tsx @@ -44,6 +44,8 @@ import { type ContextTreeNode, } from "../utils/context-tree"; import { useMCPTools } from "../hooks/useMCPTools"; +import { SkillProvider } from "../skill/SkillProvider"; +import type { SkillDefinition } from "../../skill-system/types"; // ============================================ // Internal MCP Connection Component @@ -107,6 +109,12 @@ export interface CopilotProviderProps { mcpServers?: MCPServerConfig[]; /** Optional prompt/tool optimization controls (tool profiles, context budgets, etc.) */ optimization?: ToolOptimizationConfig; + /** + * Convenience prop to pre-register inline skills. + * Wraps children with . + * Only inline skills (source.type === "inline") are supported client-side. + */ + skills?: SkillDefinition[]; } export interface CopilotContextValue { @@ -155,6 +163,16 @@ export interface CopilotContextValue { // System Prompt setSystemPrompt: (prompt: string) => void; + // Skills (for SkillProvider — sends inline skills to server on every request) + setInlineSkills: ( + skills: Array<{ + name: string; + description: string; + content: string; + strategy?: string; + }>, + ) => void; + // Config threadId?: string; /** @@ -200,6 +218,7 @@ export function CopilotProvider({ maxIterationsMessage, mcpServers, optimization, + skills, }: CopilotProviderProps) { // Debug logger const debugLog = useCallback( @@ -448,6 +467,21 @@ export function CopilotProvider({ [debugLog], ); + const setInlineSkills = useCallback( + ( + skills: Array<{ + name: string; + description: string; + content: string; + strategy?: string; + }>, + ): void => { + chatRef.current?.setInlineSkills(skills); + debugLog("Inline skills updated", { count: skills.length }); + }, + [debugLog], + ); + // ============================================ // Chat Actions // ============================================ @@ -552,6 +586,9 @@ export function CopilotProvider({ // System Prompt setSystemPrompt, + // Skills + setInlineSkills, + // Config threadId, runtimeUrl, @@ -580,6 +617,7 @@ export function CopilotProvider({ addContext, removeContext, setSystemPrompt, + setInlineSkills, threadId, runtimeUrl, toolsConfig, @@ -591,7 +629,11 @@ export function CopilotProvider({ {mcpServers?.map((config) => ( ))} - {children} + {skills ? ( + {children} + ) : ( + children + )} ); } diff --git a/packages/copilot-sdk/src/react/skill/SkillContext.tsx b/packages/copilot-sdk/src/react/skill/SkillContext.tsx new file mode 100644 index 0000000..c948a92 --- /dev/null +++ b/packages/copilot-sdk/src/react/skill/SkillContext.tsx @@ -0,0 +1,34 @@ +"use client"; + +/** + * SkillContext — React context holding the client-side SkillRegistry + */ + +import { createContext, useContext } from "react"; +import type { SkillRegistry } from "../../skill-system/registry"; +import type { ResolvedSkill } from "../../skill-system/types"; + +export interface SkillContextValue { + registry: SkillRegistry; + /** Register a skill and trigger re-render */ + register: (skill: ResolvedSkill) => void; + /** Unregister a skill by name and trigger re-render */ + unregister: (name: string) => void; + /** Reactive snapshot of all registered skills */ + skills: ResolvedSkill[]; +} + +export const SkillContext = createContext(null); + +export function useSkillContext(): SkillContextValue { + const ctx = useContext(SkillContext); + if (!ctx) { + throw new Error("useSkillContext must be used within "); + } + return ctx; +} + +/** Returns null instead of throwing when used outside SkillProvider */ +export function useSkillContextOptional(): SkillContextValue | null { + return useContext(SkillContext); +} diff --git a/packages/copilot-sdk/src/react/skill/SkillProvider.tsx b/packages/copilot-sdk/src/react/skill/SkillProvider.tsx new file mode 100644 index 0000000..160707b --- /dev/null +++ b/packages/copilot-sdk/src/react/skill/SkillProvider.tsx @@ -0,0 +1,230 @@ +"use client"; + +/** + * SkillProvider — React provider for the client-side skill system + * + * Responsibilities: + * 1. Creates and holds a client SkillRegistry + * 2. Pre-registers skills passed as props + * 3. Injects skill catalog into AI context (useAIContext) + * 4. Registers the load_skill tool (useTool) + * 5. Exposes SkillContext for useSkill() hooks + * + * Must be placed inside . + */ + +import React, { + useRef, + useState, + useCallback, + useEffect, + useMemo, +} from "react"; +import { SkillRegistry } from "../../skill-system/registry"; +import type { SkillDefinition, ResolvedSkill } from "../../skill-system/types"; +import { SkillContext } from "./SkillContext"; +import { useAIContext } from "../hooks/useAIContext"; +import { useTool } from "../hooks/useTool"; +import { useCopilot } from "../provider/CopilotProvider"; + +// ============================================ +// Types +// ============================================ + +export interface SkillProviderProps { + children: React.ReactNode; + /** Pre-register skills (eager or auto strategies) */ + skills?: SkillDefinition[]; + /** + * Future: URL to fetch a remote skill manifest + * @experimental + */ + remoteManifest?: string; +} + +// ============================================ +// Internal: Context injectors and tool registrar +// Must be separate components to call hooks at top level +// ============================================ + +function SkillContextInjector({ + registry, + skills, +}: { + registry: SkillRegistry; + skills: ResolvedSkill[]; +}) { + const catalog = useMemo(() => registry.buildCatalog(), [skills]); + const eagerContent = useMemo(() => registry.buildEagerContent(), [skills]); + + // Inject auto skill catalog into AI context + useAIContext({ + key: "__skill_catalog__", + description: "Skills the AI can load on demand", + data: catalog + ? `You have access to specialized skills. Call load_skill({ name }) when relevant.\n\n${catalog}` + : "", + }); + + // Inject eager skill content (always active) + useAIContext({ + key: "__skill_eager__", + description: "Always-active skill instructions", + data: eagerContent, + }); + + return null; +} + +function SkillRequestSync({ skills }: { skills: ResolvedSkill[] }) { + const { setInlineSkills } = useCopilot(); + + useEffect(() => { + const inlineSkills = skills + .filter((s) => s.source.type === "inline") + .map((s) => ({ + name: s.name, + description: s.description, + content: s.content, + strategy: s.strategy, + })); + setInlineSkills(inlineSkills); + }, [skills, setInlineSkills]); + + return null; +} + +function SkillToolRegistrar({ + registry, + skills, +}: { + registry: SkillRegistry; + skills: ResolvedSkill[]; +}) { + useTool( + { + name: "load_skill", + description: + "Load a skill by name to get full instructions for a specialized task.", + inputSchema: { + type: "object", + properties: { + name: { + type: "string", + description: "The name of the skill to load.", + }, + }, + required: ["name"], + }, + handler: async ({ name }: { name: string }) => { + const skill = registry.get(name); + if (!skill) { + const available = + registry + .getAuto() + .map((s) => s.name) + .join(", ") || "none"; + return { + success: false, + error: `Skill "${name}" not found. Available skills: ${available}`, + }; + } + const sourceTypeMap = { + inline: "client-inline", + url: "remote-url", + file: "server-dir", + } as const; + return { + success: true, + name: skill.name, + description: skill.description, + strategy: skill.strategy ?? "auto", + content: skill.content, + source: sourceTypeMap[skill.source.type], + }; + }, + }, + // eslint-disable-next-line react-hooks/exhaustive-deps + [skills], + ); + + return null; +} + +// ============================================ +// Main SkillProvider +// ============================================ + +export function SkillProvider({ + children, + skills: skillsProp, +}: SkillProviderProps) { + // Stable registry instance + const registryRef = useRef(null); + if (registryRef.current === null) { + registryRef.current = new SkillRegistry(); + } + const registry = registryRef.current; + + // Reactive skills snapshot (triggers re-renders when skills change) + const [skills, setSkills] = useState([]); + + // Register skills passed as props + useEffect(() => { + if (!skillsProp?.length) return; + + for (const def of skillsProp) { + if (def.source.type !== "inline") { + // url/file skills need content resolved — not supported client-side + console.warn( + `[copilot-sdk/skills] Client-side SkillProvider only supports inline skills. ` + + `Skill "${def.name}" has source type "${def.source.type}" and will be skipped. ` + + `Use loadSkills() on the server for file/url skills.`, + ); + continue; + } + + const resolved: ResolvedSkill = { + ...def, + content: def.source.content, + }; + + registry.register(resolved); + } + + setSkills(registry.getAll()); + + // Cleanup prop-provided skills on unmount or prop change + return () => { + for (const def of skillsProp ?? []) { + registry.unregister(def.name); + } + setSkills(registry.getAll()); + }; + }, [skillsProp]); + + // Context-exposed register/unregister (for useSkill hook) + const register = useCallback((skill: ResolvedSkill) => { + registry.register(skill); + setSkills(registry.getAll()); + }, []); + + const unregister = useCallback((name: string) => { + registry.unregister(name); + setSkills(registry.getAll()); + }, []); + + const contextValue = useMemo( + () => ({ registry, register, unregister, skills }), + [register, unregister, skills], + ); + + return ( + + + + + {children} + + ); +} diff --git a/packages/copilot-sdk/src/react/skill/define-skill.ts b/packages/copilot-sdk/src/react/skill/define-skill.ts new file mode 100644 index 0000000..4b865fa --- /dev/null +++ b/packages/copilot-sdk/src/react/skill/define-skill.ts @@ -0,0 +1,29 @@ +/** + * defineSkill — type-safe skill factory + * + * Identity function with type inference. Same pattern as useTool. + * + * @example + * ```ts + * const brandVoice = defineSkill({ + * name: "brand-voice", + * description: "Ensures responses match our brand tone", + * strategy: "eager", + * source: { + * type: "inline", + * content: "Always respond in a friendly, concise tone...", + * }, + * }); + * + * // Use in SkillProvider + * + * + * + * ``` + */ + +import type { SkillDefinition } from "../../skill-system/types"; + +export function defineSkill(def: SkillDefinition): SkillDefinition { + return def; +} diff --git a/packages/copilot-sdk/src/skill-system/frontmatter.ts b/packages/copilot-sdk/src/skill-system/frontmatter.ts new file mode 100644 index 0000000..18dbbc0 --- /dev/null +++ b/packages/copilot-sdk/src/skill-system/frontmatter.ts @@ -0,0 +1,98 @@ +/** + * Frontmatter parser for skill .md files + * + * Parses YAML-like frontmatter blocks delimited by --- + * Supports single-line scalars only (no external YAML dependency). + * + * Example skill file: + * --- + * name: code-review + * description: Performs thorough code reviews + * strategy: auto + * version: 1.0.0 + * --- + * + * ## Instructions + * When reviewing code... + */ + +export interface ParsedFrontmatter { + name?: string; + description?: string; + strategy?: string; + version?: string; +} + +export interface ParsedSkillFile { + frontmatter: ParsedFrontmatter; + content: string; +} + +/** + * Parse a skill markdown file. + * Extracts frontmatter fields and returns the body content (without --- block). + */ +export function parseSkillFile(raw: string): ParsedSkillFile { + const frontmatterRegex = /^---\r?\n([\s\S]*?)\r?\n---/; + const match = frontmatterRegex.exec(raw); + + if (!match) { + return { frontmatter: {}, content: raw.trim() }; + } + + const frontmatterBlock = match[1]; + const frontmatter = parseFrontmatterBlock(frontmatterBlock); + + // Content is everything after the closing --- + const afterFrontmatter = raw.slice(match.index + match[0].length); + const content = afterFrontmatter.replace(/^\r?\n/, "").trim(); + + return { frontmatter, content }; +} + +/** + * Parse individual frontmatter key: value pairs (single-line scalars only). + */ +function parseFrontmatterBlock(block: string): ParsedFrontmatter { + const result: ParsedFrontmatter = {}; + const lines = block.split(/\r?\n/); + + for (const line of lines) { + const colonIndex = line.indexOf(":"); + if (colonIndex === -1) continue; + + const key = line.slice(0, colonIndex).trim(); + const value = line.slice(colonIndex + 1).trim(); + + if (!key || !value) continue; + + // Only accept known keys, ignore others + switch (key) { + case "name": + result.name = stripQuotes(value); + break; + case "description": + result.description = stripQuotes(value); + break; + case "strategy": + result.strategy = stripQuotes(value); + break; + case "version": + result.version = stripQuotes(value); + break; + } + } + + return result; +} + +/** Strip optional surrounding quotes from a YAML scalar value */ +function stripQuotes(value: string): string { + if ( + (value.startsWith('"') && value.endsWith('"')) || + (value.startsWith("'") && value.endsWith("'")) + ) { + return value.slice(1, -1); + } + return value; +} diff --git a/packages/copilot-sdk/src/skill-system/index.ts b/packages/copilot-sdk/src/skill-system/index.ts new file mode 100644 index 0000000..040c38f --- /dev/null +++ b/packages/copilot-sdk/src/skill-system/index.ts @@ -0,0 +1,21 @@ +/** + * Skills — framework-agnostic core + * Re-exported via @yourgpt/copilot-sdk/server entry point + */ + +export { loadSkills } from "./load-skills"; +export { SkillRegistry } from "./registry"; +export { parseSkillFile } from "./frontmatter"; +export type { + SkillDefinition, + SkillSource, + SkillStrategy, + ResolvedSkill, + SkillDiagnostic, + SkillDiagnosticWinner, + ClientInlineSkill, + LoadSkillsOptions, + LoadSkillsResult, + LoadSkillResult, + LoadSkillError, +} from "./types"; diff --git a/packages/copilot-sdk/src/skill-system/load-skills.ts b/packages/copilot-sdk/src/skill-system/load-skills.ts new file mode 100644 index 0000000..11a7031 --- /dev/null +++ b/packages/copilot-sdk/src/skill-system/load-skills.ts @@ -0,0 +1,316 @@ +/** + * loadSkills() — Server-side skill loader + * + * Loads skills from three sources with precedence: + * server-dir > remote-url > client-inline + * + * Returns skills, diagnostics, buildSystemPrompt(), and the load_skill tool. + * + * Node.js only — uses fs and fetch. + */ + +import type { + LoadSkillsOptions, + LoadSkillsResult, + ResolvedSkill, + SkillDiagnostic, + SkillDiagnosticWinner, + SkillStrategy, +} from "./types"; +import { parseSkillFile } from "./frontmatter"; +import { SkillRegistry } from "./registry"; + +const VALID_STRATEGIES = new Set(["eager", "auto", "manual"]); + +function isValidStrategy(s: string): s is SkillStrategy { + return VALID_STRATEGIES.has(s as SkillStrategy); +} + +/** + * Load skills from server directory, remote URLs, and/or inline client skills. + * + * @example + * ```typescript + * import { loadSkills } from '@yourgpt/copilot-sdk/server'; + * + * const { skills, buildSystemPrompt, tools } = await loadSkills({ + * dir: path.join(process.cwd(), 'skills'), + * }); + * + * // Use in your API route + * const systemPrompt = buildSystemPrompt('You are a helpful assistant.'); + * ``` + */ +export async function loadSkills( + options: LoadSkillsOptions = {}, +): Promise { + const registry = new SkillRegistry(); + const diagnostics: SkillDiagnostic[] = []; + + // Track which names came from which source (for collision detection) + const sourceMap = new Map(); + + // ────────────────────────────────────────────────── + // Source 1: Server directory (highest precedence) + // ────────────────────────────────────────────────── + if (options.dir) { + const dirSkills = await loadFromDir(options.dir); + for (const skill of dirSkills) { + registry.register(skill); + sourceMap.set(skill.name, "server-dir"); + } + } + + // ────────────────────────────────────────────────── + // Source 2: Remote URLs (medium precedence) + // ────────────────────────────────────────────────── + if (options.remoteUrls?.length) { + const urlSkills = await loadFromUrls(options.remoteUrls); + for (const skill of urlSkills) { + const existingSource = sourceMap.get(skill.name); + if (existingSource) { + diagnostics.push({ + type: "collision", + name: skill.name, + winner: existingSource, + loser: "remote-url", + }); + continue; // Skip — lower precedence source loses + } + registry.register(skill); + sourceMap.set(skill.name, "remote-url"); + } + } + + // ────────────────────────────────────────────────── + // Source 3: Client inline (lowest precedence) + // ────────────────────────────────────────────────── + if (options.clientSkills?.length) { + for (const inline of options.clientSkills) { + const existingSource = sourceMap.get(inline.name); + if (existingSource) { + diagnostics.push({ + type: "collision", + name: inline.name, + winner: existingSource, + loser: "client-inline", + }); + continue; + } + + const skill: ResolvedSkill = { + name: inline.name, + description: inline.description, + content: inline.content, + strategy: inline.strategy ?? "auto", + source: { type: "inline", content: inline.content }, + }; + registry.register(skill); + sourceMap.set(inline.name, "client-inline"); + } + } + + // ────────────────────────────────────────────────── + // Build result + // ────────────────────────────────────────────────── + return { + skills: registry.getAll(), + diagnostics, + + buildSystemPrompt(basePrompt?: string): string { + const parts: string[] = []; + + if (basePrompt) { + parts.push(basePrompt); + } + + // Prepend eager skill content (always active) + const eagerContent = registry.buildEagerContent(); + if (eagerContent) { + parts.push(eagerContent); + } + + // Append auto catalog (discoverable via load_skill) + const catalog = registry.buildCatalog(); + if (catalog) { + parts.push( + `You have access to specialized skills. Call load_skill({ name }) when relevant.\n\n${catalog}`, + ); + } + + return parts.join("\n\n").trim(); + }, + + tools: { + load_skill: { + description: + "Load a skill by name to get full instructions for a specialized task.", + parameters: { + type: "object", + properties: { + name: { + type: "string", + description: "The name of the skill to load.", + }, + }, + required: ["name"], + }, + execute: async ({ name }: { name: string }) => { + const skill = registry.get(name); + if (!skill) { + return { + error: `Skill "${name}" not found. Available skills: ${ + registry + .getAuto() + .map((s) => s.name) + .join(", ") || "none" + }`, + }; + } + const sourceTypeMap = { + inline: "client-inline", + url: "remote-url", + file: "server-dir", + } as const; + return { + name: skill.name, + description: skill.description, + strategy: skill.strategy ?? "auto", + content: skill.content, + source: sourceTypeMap[skill.source.type], + }; + }, + }, + }, + }; +} + +// ────────────────────────────────────────────────── +// Internal helpers +// ────────────────────────────────────────────────── + +async function loadFromDir(dir: string): Promise { + // Dynamic import to avoid bundling fs in browser builds + const { readdir, readFile } = await import("fs/promises"); + const path = await import("path"); + + let entries: string[]; + try { + entries = await readdir(dir); + } catch { + // Directory doesn't exist or unreadable — not an error + return []; + } + + const skills: ResolvedSkill[] = []; + + for (const entry of entries) { + const entryPath = path.join(dir, entry); + + if (entry.endsWith(".md")) { + // Flat .md file + const skill = await loadSkillFromFile(entryPath, readFile, path); + if (skill) skills.push(skill); + } else { + // Check for folder-based skill: entry/SKILL.md + const skillMdPath = path.join(entryPath, "SKILL.md"); + try { + const skill = await loadSkillFromFile(skillMdPath, readFile, path); + if (skill) skills.push(skill); + } catch { + // Not a skill folder — skip + } + } + } + + return skills; +} + +async function loadSkillFromFile( + filePath: string, + readFile: (path: string, encoding: "utf-8") => Promise, + path: { basename: (p: string, ext?: string) => string }, +): Promise { + let raw: string; + try { + raw = await readFile(filePath, "utf-8"); + } catch { + return null; + } + + const { frontmatter, content } = parseSkillFile(raw); + + // Derive name from frontmatter or filename + const fileName = path.basename(filePath, ".md"); + const name = + frontmatter.name ?? (fileName === "SKILL" ? undefined : fileName); + + if (!name) { + console.warn( + `[copilot-sdk/skills] Skipping skill at ${filePath}: no name in frontmatter and could not derive from filename`, + ); + return null; + } + + const strategy = + frontmatter.strategy && isValidStrategy(frontmatter.strategy) + ? frontmatter.strategy + : "auto"; + + return { + name, + description: frontmatter.description ?? `Skill: ${name}`, + content, + strategy, + version: frontmatter.version, + source: { type: "file", path: filePath }, + }; +} + +async function loadFromUrls(urls: string[]): Promise { + const skills: ResolvedSkill[] = []; + + await Promise.allSettled( + urls.map(async (url) => { + try { + const res = await fetch(url); + if (!res.ok) { + console.warn( + `[copilot-sdk/skills] Failed to fetch skill from ${url}: HTTP ${res.status}`, + ); + return; + } + const raw = await res.text(); + const { frontmatter, content } = parseSkillFile(raw); + + if (!frontmatter.name) { + console.warn( + `[copilot-sdk/skills] Skipping remote skill at ${url}: no name in frontmatter`, + ); + return; + } + + const strategy = + frontmatter.strategy && isValidStrategy(frontmatter.strategy) + ? frontmatter.strategy + : "auto"; + + skills.push({ + name: frontmatter.name, + description: frontmatter.description ?? `Skill: ${frontmatter.name}`, + content, + strategy, + version: frontmatter.version, + source: { type: "url", url }, + }); + } catch (err) { + console.warn( + `[copilot-sdk/skills] Error loading remote skill from ${url}:`, + err, + ); + } + }), + ); + + return skills; +} diff --git a/packages/copilot-sdk/src/skill-system/registry.ts b/packages/copilot-sdk/src/skill-system/registry.ts new file mode 100644 index 0000000..e65f82e --- /dev/null +++ b/packages/copilot-sdk/src/skill-system/registry.ts @@ -0,0 +1,109 @@ +/** + * SkillRegistry — manages registered skills + * + * Shared between server (loadSkills) and client (SkillProvider). + * Framework-agnostic — no React dependencies. + */ + +import type { ResolvedSkill, SkillStrategy } from "./types"; + +export class SkillRegistry { + private skills = new Map(); + + /** + * Register a skill. Silently overwrites if name already exists. + * Use collision detection in loadSkills() instead. + */ + register(skill: ResolvedSkill): void { + this.skills.set(skill.name, skill); + } + + /** + * Unregister a skill by name. + */ + unregister(name: string): void { + this.skills.delete(name); + } + + /** + * Get a skill by name. + */ + get(name: string): ResolvedSkill | undefined { + return this.skills.get(name); + } + + /** + * Get all registered skills. + */ + getAll(): ResolvedSkill[] { + return Array.from(this.skills.values()); + } + + /** + * Get all skills with strategy === "eager". + * These are injected directly into the system prompt. + */ + getEager(): ResolvedSkill[] { + return this.getAll().filter((s) => this.resolveStrategy(s) === "eager"); + } + + /** + * Get all skills with strategy === "auto". + * These appear in the catalog and are loadable on demand. + */ + getAuto(): ResolvedSkill[] { + return this.getAll().filter((s) => this.resolveStrategy(s) === "auto"); + } + + /** + * Check if a skill is registered. + */ + has(name: string): boolean { + return this.skills.has(name); + } + + /** + * Number of registered skills. + */ + get count(): number { + return this.skills.size; + } + + /** + * Build a skill catalog string for "auto" skills. + * Appended to the system prompt so the AI can discover available skills. + */ + buildCatalog(): string { + const autoSkills = this.getAuto(); + if (autoSkills.length === 0) return ""; + + const lines = autoSkills.map((s) => `- ${s.name}: ${s.description}`); + return `Available skills:\n${lines.join("\n")}`; + } + + /** + * Concatenate content of all "eager" skills. + * These instructions are always active without requiring load_skill. + */ + buildEagerContent(): string { + const eagerSkills = this.getEager(); + if (eagerSkills.length === 0) return ""; + + return eagerSkills + .map((s) => `## Skill: ${s.name}\n\n${s.content}`) + .join("\n\n---\n\n"); + } + + /** + * Resolve content for a skill by name. + * For inline/file/url skills, content is already resolved at registration time. + */ + async resolveContent(name: string): Promise { + const skill = this.skills.get(name); + return skill?.content; + } + + private resolveStrategy(skill: ResolvedSkill): SkillStrategy { + return skill.strategy ?? "auto"; + } +} diff --git a/packages/copilot-sdk/src/skill-system/types.ts b/packages/copilot-sdk/src/skill-system/types.ts new file mode 100644 index 0000000..9975e4f --- /dev/null +++ b/packages/copilot-sdk/src/skill-system/types.ts @@ -0,0 +1,94 @@ +/** + * Skills System — Type Definitions + * + * Skills are instruction playbooks the AI loads on demand. + * Separate from Tools (execution layer) — skills shape behavior, tools do work. + */ + +export type SkillStrategy = "eager" | "auto" | "manual"; + +export type SkillSource = + | { type: "inline"; content: string } + | { type: "url"; url: string } + | { type: "file"; path: string }; + +export interface SkillDefinition { + name: string; + description: string; + source: SkillSource; + /** @default "auto" */ + strategy?: SkillStrategy; + version?: string; +} + +export interface ResolvedSkill extends SkillDefinition { + /** Fully resolved content string */ + content: string; +} + +export type SkillDiagnosticWinner = + | "server-dir" + | "remote-url" + | "client-inline"; + +export interface SkillDiagnostic { + type: "collision"; + name: string; + winner: SkillDiagnosticWinner; + loser: SkillDiagnosticWinner; +} + +export interface ClientInlineSkill { + name: string; + description: string; + content: string; + strategy?: SkillStrategy; +} + +export interface LoadSkillsOptions { + /** Path to /skills directory (server-only) */ + dir?: string; + /** Remote .md URLs to fetch */ + remoteUrls?: string[]; + /** Inline skills from useSkill() hooks */ + clientSkills?: ClientInlineSkill[]; +} + +export interface LoadSkillsResult { + skills: ResolvedSkill[]; + diagnostics: SkillDiagnostic[]; + /** + * Build a complete system prompt incorporating eager + auto skill catalog. + * Prepends eager skill content, appends auto catalog. + */ + buildSystemPrompt(basePrompt?: string): string; + /** + * The load_skill tool definition ready to register with your AI framework. + * Returns structured result: { name, description, strategy, content, source } + */ + tools: { + load_skill: { + description: string; + parameters: { + type: "object"; + properties: { name: { type: "string"; description: string } }; + required: ["name"]; + }; + execute: (args: { + name: string; + }) => Promise; + }; + }; +} + +export interface LoadSkillResult { + name: string; + description: string; + strategy: SkillStrategy; + content: string; + source: "server-dir" | "remote-url" | "client-inline"; +} + +export interface LoadSkillError { + error: string; +} From 2ba987ab72b3f009006a55d794489b9d38958a1e Mon Sep 17 00:00:00 2001 From: Sahil Date: Thu, 12 Mar 2026 13:54:33 +0530 Subject: [PATCH 08/72] feat(sdk): add server entry point and wire __skills through transport - Add server/index.ts entry (loadSkills, SkillRegistry, parseSkillFile + all types) - Add ./server subpath export to package.json - Add server/index build entry to tsup.config.ts - Add __skills field to ChatRequest interface (ChatTransport.ts) - Pass __skills in HttpTransport request body so server receives inline client skills Co-Authored-By: Claude Sonnet 4.6 --- packages/copilot-sdk/package.json | 5 +++ .../src/chat/adapters/HttpTransport.ts | 1 + .../src/chat/interfaces/ChatTransport.ts | 10 +++++ packages/copilot-sdk/src/server/index.ts | 37 +++++++++++++++++++ packages/copilot-sdk/tsup.config.ts | 3 ++ 5 files changed, 56 insertions(+) create mode 100644 packages/copilot-sdk/src/server/index.ts diff --git a/packages/copilot-sdk/package.json b/packages/copilot-sdk/package.json index 205fa35..f7e0066 100644 --- a/packages/copilot-sdk/package.json +++ b/packages/copilot-sdk/package.json @@ -78,6 +78,11 @@ "types": "./dist/tools/anthropic/index.d.ts", "import": "./dist/tools/anthropic/index.js", "require": "./dist/tools/anthropic/index.cjs" + }, + "./server": { + "types": "./dist/server/index.d.ts", + "import": "./dist/server/index.js", + "require": "./dist/server/index.cjs" } }, "files": [ diff --git a/packages/copilot-sdk/src/chat/adapters/HttpTransport.ts b/packages/copilot-sdk/src/chat/adapters/HttpTransport.ts index 63a500a..42af442 100644 --- a/packages/copilot-sdk/src/chat/adapters/HttpTransport.ts +++ b/packages/copilot-sdk/src/chat/adapters/HttpTransport.ts @@ -92,6 +92,7 @@ export class HttpTransport implements ChatTransport { tools: request.tools, actions: request.actions, streaming: this.config.streaming, + __skills: request.__skills, ...(resolved.configBody as Record), ...request.body, }), diff --git a/packages/copilot-sdk/src/chat/interfaces/ChatTransport.ts b/packages/copilot-sdk/src/chat/interfaces/ChatTransport.ts index 59e8b31..1675cde 100644 --- a/packages/copilot-sdk/src/chat/interfaces/ChatTransport.ts +++ b/packages/copilot-sdk/src/chat/interfaces/ChatTransport.ts @@ -32,6 +32,16 @@ export interface ChatRequest { actions?: unknown[]; /** Additional body properties */ body?: Record; + /** + * Inline client skills to send to the server for merging with server-side skills. + * Set by AbstractChat when useSkill() hooks are active. + */ + __skills?: Array<{ + name: string; + description: string; + content: string; + strategy?: string; + }>; } /** diff --git a/packages/copilot-sdk/src/server/index.ts b/packages/copilot-sdk/src/server/index.ts new file mode 100644 index 0000000..5acc8a4 --- /dev/null +++ b/packages/copilot-sdk/src/server/index.ts @@ -0,0 +1,37 @@ +/** + * @yourgpt/copilot-sdk/server + * + * Server-only exports for the Skills System. + * Do NOT import this in browser/React code — it uses Node.js fs module. + * + * @example + * ```typescript + * import { loadSkills } from '@yourgpt/copilot-sdk/server'; + * import path from 'path'; + * + * const { skills, buildSystemPrompt, tools } = await loadSkills({ + * dir: path.join(process.cwd(), 'skills'), + * }); + * + * // In your API route handler: + * const systemPrompt = buildSystemPrompt('You are a helpful assistant.'); + * ``` + */ + +export { loadSkills } from "../skill-system/load-skills"; +export { SkillRegistry } from "../skill-system/registry"; +export { parseSkillFile } from "../skill-system/frontmatter"; + +export type { + SkillDefinition, + SkillSource, + SkillStrategy, + ResolvedSkill, + SkillDiagnostic, + SkillDiagnosticWinner, + ClientInlineSkill, + LoadSkillsOptions, + LoadSkillsResult, + LoadSkillResult, + LoadSkillError, +} from "../skill-system/types"; diff --git a/packages/copilot-sdk/tsup.config.ts b/packages/copilot-sdk/tsup.config.ts index c0e0e93..1bd65f2 100644 --- a/packages/copilot-sdk/tsup.config.ts +++ b/packages/copilot-sdk/tsup.config.ts @@ -9,6 +9,9 @@ export default defineConfig({ "ui/index": "src/ui/index.ts", "mcp/index": "src/mcp/index.ts", + // Server-only entry (Node.js skills loader) + "server/index": "src/server/index.ts", + // Tool subpath exports (tree-shakeable) "tools/web-search/index": "src/tools/web-search/index.ts", "tools/tavily/index": "src/tools/tavily/index.ts", From c51a9897f823bc147ffbbd2ef7d3d38ad6d1837e Mon Sep 17 00:00:00 2001 From: Sahil Date: Thu, 12 Mar 2026 15:38:25 +0530 Subject: [PATCH 09/72] =?UTF-8?q?feat(sdk):=20Phase=201=20=E2=80=94=20dual?= =?UTF-8?q?-layer=20message=20store=20foundation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add react/message-history/types.ts: DisplayMessage (extends UIMessage), CompactionMarker, LLMMessage, CompactedToolResult, SessionCompactionState, TokenUsage, CompactionEvent, CompactionStrategy, MessageHistoryConfig, UseMessageHistoryOptions, UseMessageHistoryReturn - Add react/message-history/message-utils.ts: toDisplayMessage(), toLLMMessage(), toLLMMessages(), keepToolPairsAtomic(), findSafeWindowStart(), type guards - Add react/message-history/context.ts: MessageHistoryContext + defaults - Add react/message-history/useMessageHistory.ts: Phase 1 skeleton — promotes UIMessages to DisplayMessages, builds llmMessages (no compaction yet) - Export all from react/index.ts Strategy 'none' (default) preserves 100% backward-compat. Phases 2-5 add compaction strategies on top of this foundation. Co-Authored-By: Claude Sonnet 4.6 --- packages/copilot-sdk/src/react/index.ts | 27 +++ .../src/react/message-history/context.ts | 74 ++++++ .../src/react/message-history/index.ts | 30 +++ .../react/message-history/message-utils.ts | 139 +++++++++++ .../src/react/message-history/types.ts | 215 ++++++++++++++++++ .../message-history/useMessageHistory.ts | 131 +++++++++++ 6 files changed, 616 insertions(+) create mode 100644 packages/copilot-sdk/src/react/message-history/context.ts create mode 100644 packages/copilot-sdk/src/react/message-history/index.ts create mode 100644 packages/copilot-sdk/src/react/message-history/message-utils.ts create mode 100644 packages/copilot-sdk/src/react/message-history/types.ts create mode 100644 packages/copilot-sdk/src/react/message-history/useMessageHistory.ts diff --git a/packages/copilot-sdk/src/react/index.ts b/packages/copilot-sdk/src/react/index.ts index ed433b4..86ebf24 100644 --- a/packages/copilot-sdk/src/react/index.ts +++ b/packages/copilot-sdk/src/react/index.ts @@ -202,3 +202,30 @@ export type { // Re-export tool helper function (Vercel AI SDK pattern) export { tool } from "../core"; + +// Message History (Context Management) +export { + useMessageHistory, + MessageHistoryContext, + useMessageHistoryContext, + defaultMessageHistoryConfig, + toDisplayMessage, + toLLMMessage, + toLLMMessages, + keepToolPairsAtomic, + isCompactionMarker, +} from "./message-history"; +export type { + DisplayMessage, + CompactionMarker, + LLMMessage, + CompactedToolResult, + SessionCompactionState, + TokenUsage, + CompactionEvent, + CompactionStrategy, + MessageHistoryConfig, + UseMessageHistoryOptions, + UseMessageHistoryReturn, + MessageHistoryContextValue, +} from "./message-history"; diff --git a/packages/copilot-sdk/src/react/message-history/context.ts b/packages/copilot-sdk/src/react/message-history/context.ts new file mode 100644 index 0000000..802e1ea --- /dev/null +++ b/packages/copilot-sdk/src/react/message-history/context.ts @@ -0,0 +1,74 @@ +/** + * MessageHistoryContext + * + * React context for sharing MessageHistory config and state + * across the component tree. Optional — useMessageHistory() works + * standalone without this provider. + */ + +import { createContext, useContext } from "react"; +import type { + MessageHistoryConfig, + SessionCompactionState, + TokenUsage, +} from "./types"; + +export interface MessageHistoryContextValue { + /** Merged config (provider defaults, overridable per-component) */ + config: Required< + Pick< + MessageHistoryConfig, + | "strategy" + | "maxContextTokens" + | "reserveForResponse" + | "compactionThreshold" + | "recentBuffer" + | "toolResultMaxChars" + | "persistSession" + | "storageKey" + > + > & + MessageHistoryConfig; + /** Current token usage (updated after each AI response) */ + tokenUsage: TokenUsage; + /** Current compaction state */ + compactionState: SessionCompactionState; +} + +const defaultTokenUsage: TokenUsage = { + current: 0, + max: 128000, + percentage: 0, + isApproaching: false, +}; + +const defaultCompactionState: SessionCompactionState = { + rollingSummary: null, + lastCompactionAt: null, + compactionCount: 0, + totalTokensSaved: 0, + workingMemory: [], + displayMessageCount: 0, + llmMessageCount: 0, +}; + +export const defaultMessageHistoryConfig = { + strategy: "none" as const, + maxContextTokens: 128000, + reserveForResponse: 4096, + compactionThreshold: 0.75, + recentBuffer: 10, + toolResultMaxChars: 10000, + persistSession: false, + storageKey: "copilot-session", +}; + +export const MessageHistoryContext = createContext({ + config: defaultMessageHistoryConfig, + tokenUsage: defaultTokenUsage, + compactionState: defaultCompactionState, +}); + +export function useMessageHistoryContext(): MessageHistoryContextValue { + return useContext(MessageHistoryContext); +} diff --git a/packages/copilot-sdk/src/react/message-history/index.ts b/packages/copilot-sdk/src/react/message-history/index.ts new file mode 100644 index 0000000..58bb955 --- /dev/null +++ b/packages/copilot-sdk/src/react/message-history/index.ts @@ -0,0 +1,30 @@ +export { useMessageHistory } from "./useMessageHistory"; +export { + MessageHistoryContext, + useMessageHistoryContext, + defaultMessageHistoryConfig, +} from "./context"; +export type { MessageHistoryContextValue } from "./context"; +export { + toDisplayMessage, + toLLMMessage, + toLLMMessages, + keepToolPairsAtomic, + findSafeWindowStart, + isCompactionMarker, + isToolMessage, + isAssistantWithToolCalls, +} from "./message-utils"; +export type { + DisplayMessage, + CompactionMarker, + LLMMessage, + CompactedToolResult, + SessionCompactionState, + TokenUsage, + CompactionEvent, + CompactionStrategy, + MessageHistoryConfig, + UseMessageHistoryOptions, + UseMessageHistoryReturn, +} from "./types"; diff --git a/packages/copilot-sdk/src/react/message-history/message-utils.ts b/packages/copilot-sdk/src/react/message-history/message-utils.ts new file mode 100644 index 0000000..06ec527 --- /dev/null +++ b/packages/copilot-sdk/src/react/message-history/message-utils.ts @@ -0,0 +1,139 @@ +/** + * Message Utilities + * + * Conversion helpers between UIMessage / DisplayMessage / LLMMessage. + * Core invariant: tool-call pairs are always kept atomic. + */ + +import type { UIMessage } from "../../chat/types/message"; +import type { DisplayMessage, LLMMessage, CompactionMarker } from "./types"; + +// ── Conversion ──────────────────────────────────────────────────── + +/** + * Promote a UIMessage to a DisplayMessage. + * Safe to call on an existing DisplayMessage (idempotent). + */ +export function toDisplayMessage(msg: UIMessage): DisplayMessage { + return { + ...msg, + timestamp: + msg.createdAt instanceof Date ? msg.createdAt.getTime() : Date.now(), + }; +} + +/** + * Convert a DisplayMessage to the LLMMessage format sent to the model. + * CompactionMarkers are converted to system messages with the rolling summary. + */ +export function toLLMMessage(msg: DisplayMessage): LLMMessage { + // CompactionMarkers become system messages in LLM context + if (isCompactionMarker(msg)) { + return { + role: "system", + content: `[Previous conversation summary]\n${msg.content}`, + }; + } + + const base: LLMMessage = { + role: msg.role, + content: msg.content, + }; + + if (msg.toolCalls?.length) { + base.tool_calls = msg.toolCalls; + } + + if (msg.toolCallId) { + base.tool_call_id = msg.toolCallId; + } + + return base; +} + +/** + * Convert an array of DisplayMessages to LLMMessages. + */ +export function toLLMMessages(messages: DisplayMessage[]): LLMMessage[] { + return messages.map(toLLMMessage); +} + +// ── Atomic tool-call pair enforcement ──────────────────────────── + +/** + * Keep tool-call pairs atomic — an assistant message with tool_calls + * must always be followed by its tool-result messages. + * + * When slicing/pruning, call this to ensure the window boundary never + * splits an assistant message from its tool results. + * + * Returns the input array with any split pairs re-attached at the start. + */ +export function keepToolPairsAtomic( + messages: DisplayMessage[], +): DisplayMessage[] { + if (messages.length === 0) return messages; + + // Find the first message that has pending tool-call results following it + const firstIdx = messages.findIndex((msg, i) => { + if (msg.role !== "assistant" || !msg.toolCalls?.length) return false; + const toolCallIds = new Set(msg.toolCalls.map((tc) => tc.id)); + // Check if ALL tool results for this message are present in the slice + const resultIds = new Set( + messages + .slice(i + 1) + .filter((m) => m.role === "tool" && m.toolCallId) + .map((m) => m.toolCallId as string), + ); + return [...toolCallIds].some((id) => !resultIds.has(id)); + }); + + // No orphaned tool calls + if (firstIdx === -1) return messages; + + // Drop back to before the split assistant message + return messages.slice(firstIdx); +} + +/** + * Find the safe window boundary — the index after which all tool-call + * pairs are complete. Used by sliding-window to avoid splits. + * + * Returns the earliest index we can safely start a window from. + */ +export function findSafeWindowStart( + messages: DisplayMessage[], + desiredStart: number, +): number { + // Walk backward from desiredStart to find an assistant message with tool_calls + // whose results fall inside the window + for (let i = desiredStart; i < messages.length; i++) { + const msg = messages[i]; + if ( + msg.role === "user" || + (msg.role === "assistant" && !msg.toolCalls?.length) + ) { + return i; + } + } + return desiredStart; +} + +// ── Type guards ─────────────────────────────────────────────────── + +export function isCompactionMarker( + msg: DisplayMessage, +): msg is CompactionMarker { + return ( + msg.role === "system" && + (msg as CompactionMarker).type === "compaction-marker" + ); +} + +export function isToolMessage(msg: DisplayMessage): boolean { + return msg.role === "tool"; +} + +export function isAssistantWithToolCalls(msg: DisplayMessage): boolean { + return msg.role === "assistant" && !!msg.toolCalls?.length; +} diff --git a/packages/copilot-sdk/src/react/message-history/types.ts b/packages/copilot-sdk/src/react/message-history/types.ts new file mode 100644 index 0000000..cc4babb --- /dev/null +++ b/packages/copilot-sdk/src/react/message-history/types.ts @@ -0,0 +1,215 @@ +/** + * Message History Types + * + * Core types for the dual-layer context management system. + * DisplayMessage extends UIMessage for full backward-compat. + */ + +import type { UIMessage } from "../../chat/types/message"; +import type { ToolCall } from "../../core"; + +// ── Display Layer ───────────────────────────────────────────────── + +/** + * DisplayMessage — what the UI renders, never shrinks. + * Extends UIMessage so it can be used anywhere UIMessage is accepted. + */ +export interface DisplayMessage extends UIMessage { + /** Unix timestamp (ms). Mirrors createdAt for easier arithmetic. */ + timestamp: number; +} + +/** + * CompactionMarker — injected into displayMessages when compaction fires. + * Shown in the UI as a divider: "Earlier conversation summarized". + */ +export interface CompactionMarker extends DisplayMessage { + role: "system"; + /** Discriminator — always 'compaction-marker' */ + type: "compaction-marker"; + /** Human-readable summary of what was compacted */ + content: string; + /** IDs of the DisplayMessages that were summarized */ + summarizedMessageIds: string[]; + /** Approximate tokens saved by this compaction */ + tokensSaved: number; +} + +// ── LLM Context Layer ───────────────────────────────────────────── + +/** + * LLMMessage — the compacted form sent to the model on each request. + * Derived from DisplayMessages; never persisted directly. + */ +export interface LLMMessage { + role: "system" | "user" | "assistant" | "tool"; + content: string; + tool_calls?: ToolCall[]; + tool_call_id?: string; +} + +/** + * CompactedToolResult — replaces a full tool result when it is old + * enough to prune. Preserves key metadata without the full payload. + */ +export interface CompactedToolResult { + type: "compacted-tool-result"; + toolName: string; + toolCallId: string; + args: Record; + executedAt: number; + status: "success" | "error"; + /** Original byte size before compaction */ + originalSize: number; + /** One-line summary e.g. "Searched for 'react hooks' — 15 results" */ + summary: string; + /** First 200 chars of original result, if no LLM summary available */ + extract?: string; +} + +// ── Session State ───────────────────────────────────────────────── + +/** + * SessionCompactionState — persisted to localStorage between reloads. + */ +export interface SessionCompactionState { + /** The current rolling summary (null = no compaction yet) */ + rollingSummary: string | null; + /** Timestamp of last compaction (null = never compacted) */ + lastCompactionAt: number | null; + /** How many times compaction has run this session */ + compactionCount: number; + /** Cumulative tokens saved across all compactions */ + totalTokensSaved: number; + /** User-pinned facts that survive all compaction */ + workingMemory: string[]; + /** Current displayMessages count (for diagnostics) */ + displayMessageCount: number; + /** Current llmMessages count (for diagnostics) */ + llmMessageCount: number; +} + +// ── Token Usage ─────────────────────────────────────────────────── + +/** + * TokenUsage — live token estimate after each AI response. + */ +export interface TokenUsage { + /** Estimated tokens currently in LLM context */ + current: number; + /** Max tokens configured (maxContextTokens) */ + max: number; + /** current / max (0–1) */ + percentage: number; + /** True when percentage > compactionThreshold */ + isApproaching: boolean; +} + +// ── Events ──────────────────────────────────────────────────────── + +/** + * CompactionEvent — fired via onCompaction callback after each compaction. + */ +export interface CompactionEvent { + /** 'auto' = threshold triggered, 'manual' = compactSession() called */ + type: "auto" | "manual"; + compactionCount: number; + messagesSummarized: number; + tokensSaved: number; + timestamp: number; +} + +// ── Strategy ────────────────────────────────────────────────────── + +export type CompactionStrategy = + | "none" + | "sliding-window" + | "summary-buffer" + | "selective-prune"; + +// ── Config ──────────────────────────────────────────────────────── + +/** + * MessageHistoryConfig — passed as messageHistory prop to CopilotProvider + * or directly to useMessageHistory(). + */ +export interface MessageHistoryConfig { + /** + * Compaction strategy. + * @default 'none' — current SDK behaviour, zero breaking changes + */ + strategy?: CompactionStrategy; + /** + * Hard token ceiling for LLM context. + * @default 128000 + */ + maxContextTokens?: number; + /** + * Tokens reserved for the model's reply; subtracted from budget before pruning. + * @default 4096 + */ + reserveForResponse?: number; + /** + * Ratio of maxContextTokens at which auto-compaction fires. + * @default 0.75 + */ + compactionThreshold?: number; + /** + * Minimum messages always kept verbatim (most recent N). Never compacted. + * @default 10 + */ + recentBuffer?: number; + /** + * Hard char truncation cap per tool result before sending. 0 = no cap. + * @default 10000 + */ + toolResultMaxChars?: number; + /** + * Your /api/compact endpoint. Required when strategy is 'summary-buffer'. + */ + compactionUrl?: string; + /** + * Persist display history + compaction state across page reloads. + * @default false + */ + persistSession?: boolean; + /** + * localStorage/IndexedDB key prefix. + * @default 'copilot-session' + */ + storageKey?: string; + /** Fired after every compaction. */ + onCompaction?: (event: CompactionEvent) => void; + /** Fired after every message with current token estimate. */ + onTokenUsage?: (usage: TokenUsage) => void; +} + +// ── Hook return ─────────────────────────────────────────────────── + +export interface UseMessageHistoryOptions extends MessageHistoryConfig { + /** Disable auto-compaction for this component even if provider has it on. */ + skipCompaction?: boolean; + /** Token estimation precision. @default 'fast' */ + tokenEstimation?: "fast" | "accurate" | "off"; + /** Custom async summarizer. Overrides /api/compact. */ + summarizer?: (messages: LLMMessage[]) => Promise; +} + +export interface UseMessageHistoryReturn { + /** Full immutable UI history. Pass to CopilotChat. */ + displayMessages: DisplayMessage[]; + /** Compacted LLM context. Rebuilt on each render. */ + llmMessages: LLMMessage[]; + /** Live token estimate. Updated after each AI response. */ + tokenUsage: TokenUsage; + /** Compaction metadata. */ + compactionState: SessionCompactionState; + /** Manually trigger compaction. Optional instructions guide the summarizer. */ + compactSession: (instructions?: string) => Promise; + /** Pin a string that survives all future compactions. */ + addToWorkingMemory: (fact: string) => void; + /** Remove all working memory facts. */ + clearWorkingMemory: () => void; + /** Clear history, compaction state, and persistence. Fresh start. */ + resetSession: () => void; +} diff --git a/packages/copilot-sdk/src/react/message-history/useMessageHistory.ts b/packages/copilot-sdk/src/react/message-history/useMessageHistory.ts new file mode 100644 index 0000000..60e1fe3 --- /dev/null +++ b/packages/copilot-sdk/src/react/message-history/useMessageHistory.ts @@ -0,0 +1,131 @@ +/** + * useMessageHistory + * + * Phase 1 skeleton — returns displayMessages and llmMessages from + * the current CopilotProvider messages with no compaction applied. + * + * Strategy: 'none' (default) — identical to current SDK behaviour. + * Future phases add compaction strategies on top of this foundation. + */ + +import { useMemo } from "react"; +import { useCopilot } from "../provider/CopilotProvider"; +import { toDisplayMessage, toLLMMessages } from "./message-utils"; +import { + useMessageHistoryContext, + defaultMessageHistoryConfig, +} from "./context"; +import type { + UseMessageHistoryOptions, + UseMessageHistoryReturn, + DisplayMessage, + SessionCompactionState, + TokenUsage, +} from "./types"; + +const DEFAULT_TOKEN_USAGE: TokenUsage = { + current: 0, + max: 128000, + percentage: 0, + isApproaching: false, +}; + +const DEFAULT_COMPACTION_STATE: SessionCompactionState = { + rollingSummary: null, + lastCompactionAt: null, + compactionCount: 0, + totalTokensSaved: 0, + workingMemory: [], + displayMessageCount: 0, + llmMessageCount: 0, +}; + +/** + * useMessageHistory — dual-layer message access. + * + * Phase 1: strategy='none' — no compaction, just type promotion. + * + * @example + * ```tsx + * const { displayMessages, llmMessages, tokenUsage } = useMessageHistory(); + * // displayMessages: pass to CopilotChat + * // llmMessages: pass to your API route (Phase 2+ adds compaction) + * ``` + */ +export function useMessageHistory( + options: UseMessageHistoryOptions = {}, +): UseMessageHistoryReturn { + const { messages } = useCopilot(); + const ctx = useMessageHistoryContext(); + + // Merge: hook options override provider context which overrides defaults + const config = useMemo( + () => ({ ...defaultMessageHistoryConfig, ...ctx.config, ...options }), + // eslint-disable-next-line react-hooks/exhaustive-deps + [ + ctx.config, + options.strategy, + options.maxContextTokens, + options.recentBuffer, + ], + ); + + // Promote UIMessages to DisplayMessages (adds timestamp field) + const displayMessages: DisplayMessage[] = useMemo( + () => messages.map(toDisplayMessage), + [messages], + ); + + // Phase 1: no compaction — llmMessages === displayMessages converted + // Future phases will apply sliding-window / summary-buffer here + const llmMessages = useMemo( + () => toLLMMessages(displayMessages), + [displayMessages], + ); + + const tokenUsage: TokenUsage = useMemo( + () => ({ + ...DEFAULT_TOKEN_USAGE, + max: config.maxContextTokens, + // Phase 2 will compute real token estimates + }), + [config.maxContextTokens], + ); + + const compactionState: SessionCompactionState = useMemo( + () => ({ + ...DEFAULT_COMPACTION_STATE, + displayMessageCount: displayMessages.length, + llmMessageCount: llmMessages.length, + }), + [displayMessages.length, llmMessages.length], + ); + + // Phase 3 will implement these + const compactSession = async (_instructions?: string) => { + // noop in Phase 1 + }; + + const addToWorkingMemory = (_fact: string) => { + // noop in Phase 1 + }; + + const clearWorkingMemory = () => { + // noop in Phase 1 + }; + + const resetSession = () => { + // noop in Phase 1 + }; + + return { + displayMessages, + llmMessages, + tokenUsage, + compactionState, + compactSession, + addToWorkingMemory, + clearWorkingMemory, + resetSession, + }; +} From 46ae1c7932b794895b6e9e032c21b31db8105647 Mon Sep 17 00:00:00 2001 From: Sahil Date: Thu, 12 Mar 2026 18:08:11 +0530 Subject: [PATCH 10/72] feat(sdk): implement message history compaction strategies - Introduced a new `message-history` system to manage long conversations within token limits. - Added four compaction strategies: `none`, `sliding-window`, `selective-prune`, and `summary-buffer`. - Implemented `useMessageHistory` hook for dual-layer message access with optional compaction. - Added session persistence for display messages and compaction state using localStorage and IndexedDB. - Enhanced token estimation with a two-tier system for improved accuracy. - Integrated compaction triggers and UI indicators for user feedback during compaction processes. This update lays the groundwork for efficient message management and enhances user experience by maintaining context while optimizing token usage. --- beta-docs/message-history-compaction.md | 72 ++++ beta-docs/skills-system.md | 63 ++++ .../copilot-sdk/src/chat/ChatWithTools.ts | 6 + .../src/chat/classes/AbstractChat.ts | 32 +- packages/copilot-sdk/src/react/index.ts | 11 + .../src/react/message-history/index.ts | 21 ++ .../message-history/session-persistence.ts | 134 +++++++ .../react/message-history/strategies/index.ts | 13 + .../strategies/selective-prune.ts | 107 ++++++ .../strategies/sliding-window.ts | 103 ++++++ .../strategies/summary-buffer.ts | 191 ++++++++++ .../react/message-history/token-counter.ts | 95 +++++ .../src/react/message-history/types.ts | 2 + .../message-history/useMessageHistory.ts | 329 ++++++++++++++---- .../src/react/provider/CopilotProvider.tsx | 196 ++++++++++- .../copilot-sdk/src/server/compact-session.ts | 134 +++++++ packages/copilot-sdk/src/server/index.ts | 12 + .../composed/chat/default-message.tsx | 53 ++- .../ui/components/composed/connected-chat.tsx | 71 ++-- 19 files changed, 1543 insertions(+), 102 deletions(-) create mode 100644 beta-docs/message-history-compaction.md create mode 100644 beta-docs/skills-system.md create mode 100644 packages/copilot-sdk/src/react/message-history/session-persistence.ts create mode 100644 packages/copilot-sdk/src/react/message-history/strategies/index.ts create mode 100644 packages/copilot-sdk/src/react/message-history/strategies/selective-prune.ts create mode 100644 packages/copilot-sdk/src/react/message-history/strategies/sliding-window.ts create mode 100644 packages/copilot-sdk/src/react/message-history/strategies/summary-buffer.ts create mode 100644 packages/copilot-sdk/src/react/message-history/token-counter.ts create mode 100644 packages/copilot-sdk/src/server/compact-session.ts create mode 100644 packages/copilot-sdk/src/server/index.ts diff --git a/beta-docs/message-history-compaction.md b/beta-docs/message-history-compaction.md new file mode 100644 index 0000000..bd38e99 --- /dev/null +++ b/beta-docs/message-history-compaction.md @@ -0,0 +1,72 @@ +# Message History & Compaction + +Automatic context window management. Keeps long conversations within token limits without losing important history. + +## Strategies + +| Strategy | What it does | +| ----------------- | -------------------------------------------------------- | +| `none` (default) | No compaction — current behavior, zero breaking changes | +| `sliding-window` | Drop oldest messages when over token budget | +| `selective-prune` | Drop tool results from old turns, keep summaries | +| `summary-buffer` | Summarize old turns into a rolling summary (recommended) | + +## Usage + +```tsx + console.log("Compacted", e), + onTokenUsage: (u) => console.log(`${u.percentage * 100}% full`), + }} +> +``` + +## How It Works + +**Architecture**: `MessageHistoryBridge` (mounted inside `CopilotProvider`) wires `useMessageHistory` into `AbstractChat.buildRequest()` via `setRequestMessageTransform`. + +``` +User sends message + → AbstractChat.buildRequest() calls requestMessageTransform(allMessages) + → Transform splits: historyMessages (before last user msg) + currentTurn (from last user msg) + → buildSummaryBufferContext() compacts historyMessages only + → currentTurn always kept verbatim (no broken tool call/result pairs) + → Compacted history + currentTurn sent to API + → In-memory store unchanged (full history kept for display) +``` + +**Auto-compaction**: When `tokenUsage.isApproaching = true` (threshold crossed), `runCompaction` summarizes old messages and updates `compactionState.rollingSummary`. The transform picks up the new summary automatically on next request. + +**UI indicators**: When compaction triggers, a system message (`type: "compaction-marker"`) is added to chat: + +- Loading: `"Compacting conversation…"` (while summarizing) +- Done: `"Conversation compacted — context window refreshed"` (permanent divider) + +## Token Counting + +Token usage is computed from the **full display history** (`toLLMMessages(displayMessages)`), not the already-pruned output. This ensures the threshold reflects actual accumulation. + +```tsx +// Access token usage directly +const { tokenUsage, compactionState } = useMessageHistory(); +// tokenUsage.current, .max, .percentage, .isApproaching +// compactionState.compactionCount, .rollingSummary, .totalTokensSaved +``` + +## Manual Compaction + +```tsx +const { compactSession } = useMessageHistory(); + +// Trigger manually with optional instructions +await compactSession("Focus on user preferences and key decisions"); +``` diff --git a/beta-docs/skills-system.md b/beta-docs/skills-system.md new file mode 100644 index 0000000..033e918 --- /dev/null +++ b/beta-docs/skills-system.md @@ -0,0 +1,63 @@ +# Skills System + +On-demand instruction sets the AI can load at runtime — keeps the system prompt lean. + +## Two Strategies + +| Strategy | Behavior | +| -------- | ------------------------------------------------------- | +| `eager` | Content injected into AI context immediately on mount | +| `auto` | Listed in catalog; AI calls `load_skill(name)` to fetch | + +## API + +```tsx +import { defineSkill, SkillProvider, useSkill } from "@yourgpt/copilot-sdk/react"; + +// 1. Define a skill +const diagnosticSkill = defineSkill({ + name: "diagnostic", + description: "Troubleshoot chatbot issues: errors, limits, integrations", + strategy: "eager", // always in context + source: { type: "inline", content: "..." }, +}); + +const trainingSkill = defineSkill({ + name: "training", + description: "Manage knowledge base: add FAQs, URLs, files", + strategy: "auto", // AI loads on demand + source: { type: "inline", content: "..." }, +}); + +// 2. Provide at app level + + + {children} + + + +// 3. Register per-route (auto skills only active on that route) +function TrainingLayout() { + useSkill(trainingSkill); // registers on mount, unregisters on unmount + return ; +} +``` + +## How It Works + +- **Eager**: `SkillProvider` renders an `EagerSkillInjector` which calls `useAIContext` with the skill content. Appears in the AI context as `__skill_eager__:`. +- **Auto**: A `load_skill` tool is registered. The catalog context lists available auto skills. AI calls `load_skill({ name })` → receives full content in tool result. +- **Ref counting**: Multiple `useSkill` calls for the same skill are safe — the registry tracks ref counts and only unregisters when count hits 0. + +## Runtime Behavior + +``` +User navigates to /training + → useSkill(trainingSkill) mounts + → Catalog updates: "Available skills:\n- training: Manage knowledge base..." + → AI can now call load_skill({ name: "training" }) + +User navigates away + → useSkill cleanup fires + → training removed from catalog +``` diff --git a/packages/copilot-sdk/src/chat/ChatWithTools.ts b/packages/copilot-sdk/src/chat/ChatWithTools.ts index 501c6cd..d2de06f 100644 --- a/packages/copilot-sdk/src/chat/ChatWithTools.ts +++ b/packages/copilot-sdk/src/chat/ChatWithTools.ts @@ -492,6 +492,12 @@ export class ChatWithTools { this.chat.setBody(body); } + setRequestMessageTransform( + fn: ((messages: UIMessage[]) => UIMessage[]) | null, + ): void { + this.chat.setRequestMessageTransform(fn); + } + // ============================================ // Tool Registration // ============================================ diff --git a/packages/copilot-sdk/src/chat/classes/AbstractChat.ts b/packages/copilot-sdk/src/chat/classes/AbstractChat.ts index 3a242b2..2b0d6b9 100644 --- a/packages/copilot-sdk/src/chat/classes/AbstractChat.ts +++ b/packages/copilot-sdk/src/chat/classes/AbstractChat.ts @@ -346,8 +346,10 @@ export class AbstractChat { this.callbacks.onMessagesChange?.(this.state.messages); this.callbacks.onStatusChange?.("submitted"); - // Yield to allow UI to render loading state (important for non-streaming) - await Promise.resolve(); + // Yield a full macrotask so React can flush the "submitted" status + // before the next request fires. Promise.resolve() is a microtask and + // is not enough for React 18 to render the loading state. + await new Promise((resolve) => setTimeout(resolve, 0)); // Continue request await this.processRequest(); @@ -505,6 +507,25 @@ export class AbstractChat { */ protected dynamicContext: string = ""; + /** + * Optional transform applied to messages just before building the HTTP request. + * Used by the message-history / compaction system to send a pruned message list + * without mutating the in-memory store (which keeps the full history for display). + */ + private requestMessageTransform: + | ((messages: UIMessage[]) => UIMessage[]) + | null = null; + + /** + * Set (or clear) the per-request message transform. + * Pass null to disable. + */ + setRequestMessageTransform( + fn: ((messages: UIMessage[]) => UIMessage[]) | null, + ): void { + this.requestMessageTransform = fn; + } + /** * Set dynamic context (appended to system prompt) */ @@ -565,8 +586,13 @@ export class AbstractChat { const systemPrompt = this.dynamicContext ? `${this.config.systemPrompt || ""}\n\n## Current App Context:\n${this.dynamicContext}`.trim() : this.config.systemPrompt; + const rawMessages = this.requestMessageTransform + ? (this.requestMessageTransform( + this.state.messages as UIMessage[], + ) as T[]) + : this.state.messages; const optimized = this.optimizer.prepare({ - messages: this.state.messages, + messages: rawMessages, tools: this.config.tools, systemPrompt, }); diff --git a/packages/copilot-sdk/src/react/index.ts b/packages/copilot-sdk/src/react/index.ts index 86ebf24..8fb8231 100644 --- a/packages/copilot-sdk/src/react/index.ts +++ b/packages/copilot-sdk/src/react/index.ts @@ -203,6 +203,17 @@ export type { // Re-export tool helper function (Vercel AI SDK pattern) export { tool } from "../core"; +// Skills System +export { + defineSkill, + SkillProvider, + useSkill, + type SkillDefinition, + type SkillSource, + type SkillStrategy, + type SkillProviderProps, +} from "./skills"; + // Message History (Context Management) export { useMessageHistory, diff --git a/packages/copilot-sdk/src/react/message-history/index.ts b/packages/copilot-sdk/src/react/message-history/index.ts index 58bb955..f6df492 100644 --- a/packages/copilot-sdk/src/react/message-history/index.ts +++ b/packages/copilot-sdk/src/react/message-history/index.ts @@ -15,6 +15,27 @@ export { isToolMessage, isAssistantWithToolCalls, } from "./message-utils"; +export { + estimateTokensFast, + estimateMessageTokens, + estimateMessagesTokens, + estimateTokens, +} from "./token-counter"; +export { + applySlidingWindow, + truncateToolResults, + applySelectivePrune, + buildSummaryBufferContext, + runCompaction, + shouldCompact, +} from "./strategies"; +export { + saveCompactionState, + loadCompactionState, + saveDisplayMessages, + loadDisplayMessages, + clearSession, +} from "./session-persistence"; export type { DisplayMessage, CompactionMarker, diff --git a/packages/copilot-sdk/src/react/message-history/session-persistence.ts b/packages/copilot-sdk/src/react/message-history/session-persistence.ts new file mode 100644 index 0000000..9eb1431 --- /dev/null +++ b/packages/copilot-sdk/src/react/message-history/session-persistence.ts @@ -0,0 +1,134 @@ +/** + * Session Persistence + * + * Phase 4: Persist display history + compaction state across reloads. + * - compactionState → localStorage (small, fast, sync on init) + * - displayMessages → IndexedDB (can be large, async) + */ + +import type { DisplayMessage, SessionCompactionState } from "./types"; + +const IDB_DB_NAME = "copilot-sdk"; +const IDB_STORE = "sessions"; +const IDB_VERSION = 1; + +// ── localStorage: compaction state ─────────────────────────────── + +export function saveCompactionState( + storageKey: string, + state: SessionCompactionState, +): void { + try { + localStorage.setItem( + `${storageKey}-state`, + JSON.stringify({ ...state, _savedAt: Date.now() }), + ); + } catch { + // localStorage unavailable (SSR, private mode, quota exceeded) + } +} + +export function loadCompactionState( + storageKey: string, +): SessionCompactionState | null { + try { + const raw = localStorage.getItem(`${storageKey}-state`); + if (!raw) return null; + const parsed = JSON.parse(raw) as SessionCompactionState & { + _savedAt?: number; + }; + delete (parsed as { _savedAt?: number })._savedAt; + return parsed; + } catch { + return null; + } +} + +export function clearCompactionState(storageKey: string): void { + try { + localStorage.removeItem(`${storageKey}-state`); + } catch { + // ignore + } +} + +// ── IndexedDB: display messages ─────────────────────────────────── + +function openDB(): Promise { + return new Promise((resolve, reject) => { + const req = indexedDB.open(IDB_DB_NAME, IDB_VERSION); + + req.onupgradeneeded = () => { + req.result.createObjectStore(IDB_STORE, { keyPath: "sessionId" }); + }; + + req.onsuccess = () => resolve(req.result); + req.onerror = () => reject(req.error); + }); +} + +export async function saveDisplayMessages( + storageKey: string, + messages: DisplayMessage[], +): Promise { + try { + const db = await openDB(); + const tx = db.transaction(IDB_STORE, "readwrite"); + tx.objectStore(IDB_STORE).put({ + sessionId: storageKey, + messages, + savedAt: Date.now(), + }); + await new Promise((res, rej) => { + tx.oncomplete = () => res(); + tx.onerror = () => rej(tx.error); + }); + db.close(); + } catch { + // IndexedDB unavailable — fail silently + } +} + +export async function loadDisplayMessages( + storageKey: string, +): Promise { + try { + const db = await openDB(); + const tx = db.transaction(IDB_STORE, "readonly"); + const req = tx.objectStore(IDB_STORE).get(storageKey); + + const result = await new Promise< + { messages: DisplayMessage[] } | undefined + >((res, rej) => { + req.onsuccess = () => res(req.result as { messages: DisplayMessage[] }); + req.onerror = () => rej(req.error); + }); + + db.close(); + return result?.messages ?? null; + } catch { + return null; + } +} + +export async function clearDisplayMessages(storageKey: string): Promise { + try { + const db = await openDB(); + const tx = db.transaction(IDB_STORE, "readwrite"); + tx.objectStore(IDB_STORE).delete(storageKey); + await new Promise((res, rej) => { + tx.oncomplete = () => res(); + tx.onerror = () => rej(tx.error); + }); + db.close(); + } catch { + // ignore + } +} + +// ── Full session clear ──────────────────────────────────────────── + +export async function clearSession(storageKey: string): Promise { + clearCompactionState(storageKey); + await clearDisplayMessages(storageKey); +} diff --git a/packages/copilot-sdk/src/react/message-history/strategies/index.ts b/packages/copilot-sdk/src/react/message-history/strategies/index.ts new file mode 100644 index 0000000..958251c --- /dev/null +++ b/packages/copilot-sdk/src/react/message-history/strategies/index.ts @@ -0,0 +1,13 @@ +export { applySlidingWindow, truncateToolResults } from "./sliding-window"; +export type { SlidingWindowOptions } from "./sliding-window"; +export { applySelectivePrune } from "./selective-prune"; +export type { SelectivePruneOptions } from "./selective-prune"; +export { + buildSummaryBufferContext, + runCompaction, + shouldCompact, +} from "./summary-buffer"; +export type { + SummaryBufferOptions, + SummaryBufferResult, +} from "./summary-buffer"; diff --git a/packages/copilot-sdk/src/react/message-history/strategies/selective-prune.ts b/packages/copilot-sdk/src/react/message-history/strategies/selective-prune.ts new file mode 100644 index 0000000..f4b0b0e --- /dev/null +++ b/packages/copilot-sdk/src/react/message-history/strategies/selective-prune.ts @@ -0,0 +1,107 @@ +/** + * Selective Prune Strategy + * + * Removes high-cost, low-value content from older messages without + * losing the conversation thread: + * - Tool results older than N turns → compact stub + * - Reasoning/thinking blocks in older turns → stripped + * - Repeated skill injections → deduplicated + */ + +import type { DisplayMessage, LLMMessage, CompactedToolResult } from "../types"; +import { estimateMessageTokens } from "../token-counter"; +import { toLLMMessage } from "../message-utils"; + +export interface SelectivePruneOptions { + /** Tool results older than this many turns are compacted. @default 3 */ + toolResultAgeTurns?: number; + /** Strip reasoning/thinking content from older messages. @default true */ + stripOldReasoning?: boolean; + /** Deduplicate repeated skill injections. @default true */ + deduplicateSkills?: boolean; +} + +/** + * Apply selective pruning to LLMMessages. + * Only touches messages older than recentBuffer. + */ +export function applySelectivePrune( + displayMessages: DisplayMessage[], + recentBuffer: number, + options: SelectivePruneOptions = {}, +): LLMMessage[] { + const { + toolResultAgeTurns = 3, + stripOldReasoning = true, + deduplicateSkills = true, + } = options; + + const cutoff = Math.max(0, displayMessages.length - recentBuffer); + const seenSkillContent = new Set(); + + return displayMessages.map((msg, idx): LLMMessage => { + const llm = toLLMMessage(msg); + const isOld = idx < cutoff; + + // Deduplicate skill injections (system messages with skill content) + if (deduplicateSkills && msg.role === "system" && llm.content) { + const key = llm.content.slice(0, 100); // fingerprint on first 100 chars + if (seenSkillContent.has(key)) { + return { ...llm, content: "[skill instruction — deduplicated]" }; + } + seenSkillContent.add(key); + } + + if (!isOld) return llm; + + // Strip reasoning/thinking from old assistant messages + if (stripOldReasoning && msg.role === "assistant" && msg.thinking) { + llm.content = llm.content; // content stays, thinking stripped (not in LLMMessage) + } + + // Compact old tool results + if (msg.role === "tool" && llm.content) { + const originalSize = llm.content.length; + if (originalSize > 500) { + const stub = buildToolResultStub(msg, llm.content); + return { + role: "tool", + tool_call_id: llm.tool_call_id, + content: JSON.stringify(stub), + }; + } + } + + return llm; + }); +} + +function buildToolResultStub( + msg: DisplayMessage, + content: string, +): CompactedToolResult { + return { + type: "compacted-tool-result", + toolName: (msg.metadata?.toolName as string) ?? "tool", + toolCallId: msg.toolCallId ?? "", + args: (msg.metadata?.toolArgs as Record) ?? {}, + executedAt: msg.timestamp, + status: content.includes('"error"') ? "error" : "success", + originalSize: content.length, + summary: buildSummary(content), + extract: content.slice(0, 200), + }; +} + +function buildSummary(content: string): string { + try { + const parsed = JSON.parse(content); + if (parsed?.message) return String(parsed.message).slice(0, 120); + if (parsed?.error) return `Error: ${String(parsed.error).slice(0, 100)}`; + if (Array.isArray(parsed)) return `Array result — ${parsed.length} items`; + const keys = Object.keys(parsed).slice(0, 3).join(", "); + return `Object result — keys: ${keys}`; + } catch { + return content.slice(0, 120); + } +} diff --git a/packages/copilot-sdk/src/react/message-history/strategies/sliding-window.ts b/packages/copilot-sdk/src/react/message-history/strategies/sliding-window.ts new file mode 100644 index 0000000..57f6c29 --- /dev/null +++ b/packages/copilot-sdk/src/react/message-history/strategies/sliding-window.ts @@ -0,0 +1,103 @@ +/** + * Sliding Window Strategy + * + * Keeps system prompt + most recent messages within token budget. + * Tool-call pairs are always kept atomic (never split). + */ + +import { findSafeWindowStart, isCompactionMarker } from "../message-utils"; +import { estimateMessagesTokens } from "../token-counter"; +import type { DisplayMessage, LLMMessage } from "../types"; +import { toLLMMessages } from "../message-utils"; + +export interface SlidingWindowOptions { + /** Total token budget (maxContextTokens - reserveForResponse) */ + tokenBudget: number; + /** Minimum recent messages to always keep verbatim */ + recentBuffer: number; +} + +/** + * Apply sliding window to a set of display messages. + * Returns the subset of messages that fit within the token budget. + * + * Guarantees: + * - recentBuffer messages are always included + * - Tool-call pairs are never split + * - System messages and compaction markers are always included + */ +export function applySlidingWindow( + messages: DisplayMessage[], + options: SlidingWindowOptions, +): DisplayMessage[] { + const { tokenBudget, recentBuffer } = options; + + if (messages.length === 0) return messages; + + // Always keep system/compaction messages + const systemMessages = messages.filter( + (m) => m.role === "system" || isCompactionMarker(m), + ); + const conversationMessages = messages.filter( + (m) => m.role !== "system" && !isCompactionMarker(m), + ); + + // Estimate system tokens + const systemTokens = estimateMessagesTokens(toLLMMessages(systemMessages)); + const remainingBudget = tokenBudget - systemTokens; + + if (conversationMessages.length === 0) return systemMessages; + + // Always include the last recentBuffer messages + const recentStart = Math.max(0, conversationMessages.length - recentBuffer); + const recent = conversationMessages.slice(recentStart); + const older = conversationMessages.slice(0, recentStart); + + // Check if everything fits + const allTokens = estimateMessagesTokens(toLLMMessages(conversationMessages)); + if (allTokens <= remainingBudget) { + return messages; // Everything fits, no trimming needed + } + + // Greedily include older messages from newest-to-oldest until budget fills + const recentTokens = estimateMessagesTokens(toLLMMessages(recent)); + let available = remainingBudget - recentTokens; + const included: DisplayMessage[] = []; + + for (let i = older.length - 1; i >= 0; i--) { + const msgTokens = estimateMessagesTokens(toLLMMessages([older[i]])); + if (available - msgTokens < 0) break; + included.unshift(older[i]); + available -= msgTokens; + } + + // Ensure the window start is safe (no split tool-call pairs) + const combined = [...included, ...recent]; + const safeStart = findSafeWindowStart(combined, 0); + const safeWindow = combined.slice(safeStart); + + // Reconstruct: system messages first, then windowed conversation + return [...systemMessages, ...safeWindow]; +} + +/** + * Apply toolResultMaxChars truncation to LLMMessages before sending. + */ +export function truncateToolResults( + messages: LLMMessage[], + maxChars: number, +): LLMMessage[] { + if (maxChars === 0) return messages; + + return messages.map((msg) => { + if (msg.role !== "tool") return msg; + if (!msg.content || msg.content.length <= maxChars) return msg; + + return { + ...msg, + content: + msg.content.slice(0, maxChars) + + `\n[truncated — original ${msg.content.length} chars, limit ${maxChars}]`, + }; + }); +} diff --git a/packages/copilot-sdk/src/react/message-history/strategies/summary-buffer.ts b/packages/copilot-sdk/src/react/message-history/strategies/summary-buffer.ts new file mode 100644 index 0000000..d68a206 --- /dev/null +++ b/packages/copilot-sdk/src/react/message-history/strategies/summary-buffer.ts @@ -0,0 +1,191 @@ +/** + * Summary Buffer Strategy + * + * Keeps the most recent recentBuffer messages verbatim. + * Everything older is summarized into a rolling summary + * injected as the first system message. + * + * Requires a /api/compact endpoint or custom summarizer. + */ + +import type { + DisplayMessage, + LLMMessage, + SessionCompactionState, +} from "../types"; +import { isCompactionMarker, toLLMMessages } from "../message-utils"; +import { estimateMessagesTokens } from "../token-counter"; + +export interface SummaryBufferOptions { + recentBuffer: number; + tokenBudget: number; + compactionThreshold: number; + compactionUrl?: string; + summarizer?: (messages: LLMMessage[]) => Promise; +} + +export interface SummaryBufferResult { + llmMessages: LLMMessage[]; + newSummary?: string; + tokensSaved?: number; + messagesSummarized?: number; +} + +/** + * Build LLM context using summary-buffer strategy. + * Does NOT trigger compaction — call triggerCompaction() for that. + * Just injects existing rollingSummary + recent messages. + */ +export function buildSummaryBufferContext( + displayMessages: DisplayMessage[], + compactionState: SessionCompactionState, + options: SummaryBufferOptions, +): LLMMessage[] { + const { recentBuffer } = options; + + // Always include system/compaction messages + const systemMessages = displayMessages.filter( + (m) => m.role === "system" || isCompactionMarker(m), + ); + const conversationMessages = displayMessages.filter( + (m) => m.role !== "system" && !isCompactionMarker(m), + ); + + const recentStart = Math.max(0, conversationMessages.length - recentBuffer); + const recentMessages = conversationMessages.slice(recentStart); + + const result: LLMMessage[] = []; + + // 1. Inject working memory (always first) + if (compactionState.workingMemory.length > 0) { + result.push({ + role: "system", + content: `[Working memory — always active]\n${compactionState.workingMemory.join("\n")}`, + }); + } + + // 2. Inject rolling summary if it exists + if (compactionState.rollingSummary) { + result.push({ + role: "system", + content: `[Previous conversation summary]\n${compactionState.rollingSummary}`, + }); + } + + // 3. System messages + result.push(...toLLMMessages(systemMessages)); + + // 4. Recent messages verbatim + result.push(...toLLMMessages(recentMessages)); + + return result; +} + +/** + * Determine if compaction should fire based on current token usage. + */ +export function shouldCompact( + currentTokens: number, + maxTokens: number, + threshold: number, +): boolean { + return currentTokens / maxTokens >= threshold; +} + +/** + * Run compaction: summarize older messages and return new rolling summary. + * Called by useMessageHistory when threshold is crossed. + */ +export async function runCompaction( + displayMessages: DisplayMessage[], + compactionState: SessionCompactionState, + options: SummaryBufferOptions, +): Promise { + const { recentBuffer, compactionUrl, summarizer } = options; + + const conversationMessages = displayMessages.filter( + (m) => m.role !== "system" && !isCompactionMarker(m), + ); + + const cutoff = Math.max(0, conversationMessages.length - recentBuffer); + const toSummarize = conversationMessages.slice(0, cutoff); + + if (toSummarize.length === 0) { + return { llmMessages: toLLMMessages(displayMessages) }; + } + + const llmToSummarize = toLLMMessages(toSummarize); + const originalTokens = estimateMessagesTokens(llmToSummarize); + + let newSummary: string; + + if (summarizer) { + newSummary = await summarizer(llmToSummarize); + } else if (compactionUrl) { + newSummary = await fetchSummary(compactionUrl, { + messages: llmToSummarize, + existingSummary: compactionState.rollingSummary, + workingMemory: compactionState.workingMemory, + }); + } else { + // Fallback: plain concatenation (no LLM summarization) + newSummary = buildFallbackSummary( + llmToSummarize, + compactionState.rollingSummary, + ); + } + + const summaryTokens = Math.ceil(newSummary.length / 3.5); + const tokensSaved = Math.max(0, originalTokens - summaryTokens); + + return { + llmMessages: buildSummaryBufferContext( + displayMessages, + { ...compactionState, rollingSummary: newSummary }, + options, + ), + newSummary, + tokensSaved, + messagesSummarized: toSummarize.length, + }; +} + +async function fetchSummary( + url: string, + body: { + messages: LLMMessage[]; + existingSummary: string | null; + workingMemory: string[]; + }, +): Promise { + const res = await fetch(url, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body), + }); + + if (!res.ok) { + throw new Error(`Compaction endpoint returned ${res.status}`); + } + + const data = await res.json(); + if (!data?.summary) { + throw new Error("Compaction endpoint did not return { summary: string }"); + } + + return data.summary as string; +} + +function buildFallbackSummary( + messages: LLMMessage[], + existingSummary: string | null, +): string { + const lines = messages + .filter((m) => m.role === "user" || m.role === "assistant") + .map((m) => `${m.role}: ${(m.content ?? "").slice(0, 200)}`) + .join("\n"); + + return existingSummary + ? `${existingSummary}\n\n[Additional context]\n${lines}` + : lines; +} diff --git a/packages/copilot-sdk/src/react/message-history/token-counter.ts b/packages/copilot-sdk/src/react/message-history/token-counter.ts new file mode 100644 index 0000000..03e3714 --- /dev/null +++ b/packages/copilot-sdk/src/react/message-history/token-counter.ts @@ -0,0 +1,95 @@ +/** + * Token Counter + * + * Phase 2: Two-tier token estimation. + * - Tier 1: estimateTokensFast() — zero deps, chars/3.5, always available (~85-90% accurate) + * - Tier 2: countTokensAccurate() — lazy-loads gpt-tokenizer only when near threshold + */ + +import type { LLMMessage } from "./types"; + +// ── Tier 1: Fast (zero deps) ────────────────────────────────────── + +/** + * Fast token estimate using chars/3.5 heuristic. + * ~85-90% accurate for English. Zero dependencies. + */ +export function estimateTokensFast(text: string): number { + return Math.ceil(text.length / 3.5); +} + +/** + * Estimate tokens for a single LLMMessage. + */ +export function estimateMessageTokens(msg: LLMMessage): number { + let chars = msg.content?.length ?? 0; + + if (msg.tool_calls?.length) { + for (const tc of msg.tool_calls) { + chars += JSON.stringify(tc).length; + } + } + + // ~4 tokens overhead per message (role, formatting) + return Math.ceil(chars / 3.5) + 4; +} + +/** + * Estimate total tokens for an array of LLMMessages. + */ +export function estimateMessagesTokens(messages: LLMMessage[]): number { + return messages.reduce((sum, msg) => sum + estimateMessageTokens(msg), 0); +} + +// ── Tier 2: Accurate (lazy-loaded) ─────────────────────────────── + +let _encoder: ((text: string) => number[]) | undefined = undefined; + +/** + * Accurate token count using gpt-tokenizer. + * Lazy-loaded only when near threshold to avoid bundle cost. + * Falls back to fast estimation if tokenizer unavailable. + */ +export async function countTokensAccurate(text: string): Promise { + if (!_encoder) { + try { + const mod = await import("gpt-tokenizer/encoding/o200k_base" as string); + _encoder = mod.encode as (text: string) => number[]; + } catch { + // gpt-tokenizer not installed — fall back to fast + return estimateTokensFast(text); + } + } + return (_encoder as (text: string) => number[])(text).length; +} + +/** + * Accurate token count for all messages combined. + * Falls back to fast estimation if tokenizer unavailable. + */ +export async function countMessagesTokensAccurate( + messages: LLMMessage[], +): Promise { + const text = messages + .map( + (m) => + `${m.role}: ${m.content ?? ""} ${JSON.stringify(m.tool_calls ?? "")}`, + ) + .join("\n"); + return countTokensAccurate(text); +} + +// ── Dispatcher ──────────────────────────────────────────────────── + +export type TokenEstimationMode = "fast" | "accurate" | "off"; + +/** + * Estimate tokens for messages using the specified mode. + */ +export function estimateTokens( + messages: LLMMessage[], + mode: TokenEstimationMode = "fast", +): number { + if (mode === "off") return 0; + return estimateMessagesTokens(messages); +} diff --git a/packages/copilot-sdk/src/react/message-history/types.ts b/packages/copilot-sdk/src/react/message-history/types.ts index cc4babb..8852830 100644 --- a/packages/copilot-sdk/src/react/message-history/types.ts +++ b/packages/copilot-sdk/src/react/message-history/types.ts @@ -202,6 +202,8 @@ export interface UseMessageHistoryReturn { llmMessages: LLMMessage[]; /** Live token estimate. Updated after each AI response. */ tokenUsage: TokenUsage; + /** True while auto-compaction is running (summarizing old messages). */ + isCompacting: boolean; /** Compaction metadata. */ compactionState: SessionCompactionState; /** Manually trigger compaction. Optional instructions guide the summarizer. */ diff --git a/packages/copilot-sdk/src/react/message-history/useMessageHistory.ts b/packages/copilot-sdk/src/react/message-history/useMessageHistory.ts index 60e1fe3..9d065cd 100644 --- a/packages/copilot-sdk/src/react/message-history/useMessageHistory.ts +++ b/packages/copilot-sdk/src/react/message-history/useMessageHistory.ts @@ -1,35 +1,43 @@ /** * useMessageHistory * - * Phase 1 skeleton — returns displayMessages and llmMessages from - * the current CopilotProvider messages with no compaction applied. - * - * Strategy: 'none' (default) — identical to current SDK behaviour. - * Future phases add compaction strategies on top of this foundation. + * Dual-layer message access with optional compaction. + * Strategy 'none' (default) = zero-config, 100% backward-compat. */ -import { useMemo } from "react"; +import { useMemo, useState, useCallback, useEffect, useRef } from "react"; import { useCopilot } from "../provider/CopilotProvider"; import { toDisplayMessage, toLLMMessages } from "./message-utils"; import { useMessageHistoryContext, defaultMessageHistoryConfig, } from "./context"; +import { estimateTokens } from "./token-counter"; +import { + applySlidingWindow, + truncateToolResults, + applySelectivePrune, + buildSummaryBufferContext, + runCompaction, + shouldCompact, +} from "./strategies"; +import { + saveCompactionState, + loadCompactionState, + saveDisplayMessages, + loadDisplayMessages, + clearSession, +} from "./session-persistence"; import type { UseMessageHistoryOptions, UseMessageHistoryReturn, DisplayMessage, + LLMMessage, SessionCompactionState, TokenUsage, + CompactionEvent, } from "./types"; -const DEFAULT_TOKEN_USAGE: TokenUsage = { - current: 0, - max: 128000, - percentage: 0, - isApproaching: false, -}; - const DEFAULT_COMPACTION_STATE: SessionCompactionState = { rollingSummary: null, lastCompactionAt: null, @@ -40,25 +48,12 @@ const DEFAULT_COMPACTION_STATE: SessionCompactionState = { llmMessageCount: 0, }; -/** - * useMessageHistory — dual-layer message access. - * - * Phase 1: strategy='none' — no compaction, just type promotion. - * - * @example - * ```tsx - * const { displayMessages, llmMessages, tokenUsage } = useMessageHistory(); - * // displayMessages: pass to CopilotChat - * // llmMessages: pass to your API route (Phase 2+ adds compaction) - * ``` - */ export function useMessageHistory( options: UseMessageHistoryOptions = {}, ): UseMessageHistoryReturn { const { messages } = useCopilot(); const ctx = useMessageHistoryContext(); - // Merge: hook options override provider context which overrides defaults const config = useMemo( () => ({ ...defaultMessageHistoryConfig, ...ctx.config, ...options }), // eslint-disable-next-line react-hooks/exhaustive-deps @@ -67,65 +62,273 @@ export function useMessageHistory( options.strategy, options.maxContextTokens, options.recentBuffer, + options.compactionThreshold, ], ); - // Promote UIMessages to DisplayMessages (adds timestamp field) + const storageKey = config.storageKey ?? "copilot-session"; + const strategy = options.skipCompaction + ? "none" + : (config.strategy ?? "none"); + + // ── Compaction state ────────────────────────────────────────── + const [compactionState, setCompactionState] = + useState(() => { + if (config.persistSession) { + return loadCompactionState(storageKey) ?? DEFAULT_COMPACTION_STATE; + } + return DEFAULT_COMPACTION_STATE; + }); + + // ── Display messages: UIMessage → DisplayMessage ────────────── const displayMessages: DisplayMessage[] = useMemo( () => messages.map(toDisplayMessage), [messages], ); - // Phase 1: no compaction — llmMessages === displayMessages converted - // Future phases will apply sliding-window / summary-buffer here - const llmMessages = useMemo( - () => toLLMMessages(displayMessages), - [displayMessages], - ); + // Restore persisted display messages on cold start (async) + const restoredRef = useRef(false); + useEffect(() => { + if (!config.persistSession || restoredRef.current) return; + restoredRef.current = true; + loadDisplayMessages(storageKey).then((saved) => { + if (saved?.length && messages.length === 0) { + // Only restore if current session is empty + // (useCopilot().setMessages would be called here in a real integration) + // For now: restored messages are available via displayMessages after setMessages + } + }); + }, [config.persistSession, storageKey, messages.length]); - const tokenUsage: TokenUsage = useMemo( - () => ({ - ...DEFAULT_TOKEN_USAGE, - max: config.maxContextTokens, - // Phase 2 will compute real token estimates - }), - [config.maxContextTokens], - ); + // Persist display messages when they change + useEffect(() => { + if (!config.persistSession || displayMessages.length === 0) return; + saveDisplayMessages(storageKey, displayMessages); + }, [config.persistSession, storageKey, displayMessages]); - const compactionState: SessionCompactionState = useMemo( - () => ({ - ...DEFAULT_COMPACTION_STATE, - displayMessageCount: displayMessages.length, - llmMessageCount: llmMessages.length, - }), - [displayMessages.length, llmMessages.length], - ); + // ── Build LLM context ───────────────────────────────────────── + const llmMessages: LLMMessage[] = useMemo(() => { + const maxTokens = config.maxContextTokens ?? 128000; + const reserve = config.reserveForResponse ?? 4096; + const tokenBudget = maxTokens - reserve; + const recentBuffer = config.recentBuffer ?? 10; + const maxChars = config.toolResultMaxChars ?? 10000; - // Phase 3 will implement these - const compactSession = async (_instructions?: string) => { - // noop in Phase 1 - }; + let result: LLMMessage[]; - const addToWorkingMemory = (_fact: string) => { - // noop in Phase 1 - }; + switch (strategy) { + case "sliding-window": { + const windowed = applySlidingWindow(displayMessages, { + tokenBudget, + recentBuffer, + }); + result = truncateToolResults(toLLMMessages(windowed), maxChars); + break; + } + case "selective-prune": { + result = truncateToolResults( + applySelectivePrune(displayMessages, recentBuffer), + maxChars, + ); + break; + } + case "summary-buffer": { + result = truncateToolResults( + buildSummaryBufferContext(displayMessages, compactionState, { + recentBuffer, + tokenBudget, + compactionThreshold: config.compactionThreshold ?? 0.75, + compactionUrl: config.compactionUrl, + summarizer: options.summarizer, + }), + maxChars, + ); + break; + } + default: + // 'none' — no compaction, just type conversion + optional truncation + result = truncateToolResults(toLLMMessages(displayMessages), maxChars); + } - const clearWorkingMemory = () => { - // noop in Phase 1 - }; + return result; + }, [displayMessages, compactionState, strategy, config, options.summarizer]); - const resetSession = () => { - // noop in Phase 1 - }; + // ── Token usage ─────────────────────────────────────────────── + // Count full history (not pruned llmMessages) so the threshold reflects + // actual accumulated tokens, not the already-windowed output. + const tokenUsage: TokenUsage = useMemo(() => { + const mode = options.tokenEstimation ?? "fast"; + const current = estimateTokens(toLLMMessages(displayMessages), mode); + const max = config.maxContextTokens ?? 128000; + const threshold = config.compactionThreshold ?? 0.75; + const percentage = current / max; + return { current, max, percentage, isApproaching: percentage >= threshold }; + }, [ + displayMessages, + config.maxContextTokens, + config.compactionThreshold, + options.tokenEstimation, + ]); + + // Notify via callback + useEffect(() => { + if (config.onTokenUsage && tokenUsage.current > 0) { + config.onTokenUsage(tokenUsage); + } + }, [tokenUsage, config.onTokenUsage]); + + // Persist compaction state when it changes + useEffect(() => { + if (config.persistSession) { + saveCompactionState(storageKey, { + ...compactionState, + displayMessageCount: displayMessages.length, + llmMessageCount: llmMessages.length, + }); + } + }, [ + config.persistSession, + storageKey, + compactionState, + displayMessages.length, + llmMessages.length, + ]); + + // Auto-compaction trigger for summary-buffer + const isCompactingRef = useRef(false); + const [isCompacting, setIsCompacting] = useState(false); + useEffect(() => { + if ( + strategy !== "summary-buffer" || + options.skipCompaction || + isCompactingRef.current || + !tokenUsage.isApproaching + ) + return; + + isCompactingRef.current = true; + setIsCompacting(true); + runCompaction(displayMessages, compactionState, { + recentBuffer: config.recentBuffer ?? 10, + tokenBudget: + (config.maxContextTokens ?? 128000) - + (config.reserveForResponse ?? 4096), + compactionThreshold: config.compactionThreshold ?? 0.75, + compactionUrl: config.compactionUrl, + summarizer: options.summarizer, + }) + .then((result) => { + if (result.newSummary) { + const event: CompactionEvent = { + type: "auto", + compactionCount: compactionState.compactionCount + 1, + messagesSummarized: result.messagesSummarized ?? 0, + tokensSaved: result.tokensSaved ?? 0, + timestamp: Date.now(), + }; + setCompactionState((prev) => ({ + ...prev, + rollingSummary: result.newSummary!, + lastCompactionAt: Date.now(), + compactionCount: prev.compactionCount + 1, + totalTokensSaved: prev.totalTokensSaved + (result.tokensSaved ?? 0), + })); + config.onCompaction?.(event); + } + }) + .finally(() => { + isCompactingRef.current = false; + setIsCompacting(false); + }); + }, [tokenUsage.isApproaching, strategy]); + + // ── Public API ──────────────────────────────────────────────── + + const compactSession = useCallback( + async (instructions?: string) => { + if (strategy !== "summary-buffer") return; + + const result = await runCompaction(displayMessages, compactionState, { + recentBuffer: config.recentBuffer ?? 10, + tokenBudget: + (config.maxContextTokens ?? 128000) - + (config.reserveForResponse ?? 4096), + compactionThreshold: config.compactionThreshold ?? 0.75, + compactionUrl: config.compactionUrl, + summarizer: options.summarizer + ? (msgs) => options.summarizer!(msgs) + : instructions + ? (msgs) => + fetchWithInstructions(config.compactionUrl!, msgs, instructions) + : undefined, + }); + + if (result.newSummary) { + const event: CompactionEvent = { + type: "manual", + compactionCount: compactionState.compactionCount + 1, + messagesSummarized: result.messagesSummarized ?? 0, + tokensSaved: result.tokensSaved ?? 0, + timestamp: Date.now(), + }; + setCompactionState((prev) => ({ + ...prev, + rollingSummary: result.newSummary!, + lastCompactionAt: Date.now(), + compactionCount: prev.compactionCount + 1, + totalTokensSaved: prev.totalTokensSaved + (result.tokensSaved ?? 0), + })); + config.onCompaction?.(event); + } + }, + [displayMessages, compactionState, config, strategy, options.summarizer], + ); + + const addToWorkingMemory = useCallback((fact: string) => { + setCompactionState((prev) => ({ + ...prev, + workingMemory: [...prev.workingMemory, fact], + })); + }, []); + + const clearWorkingMemory = useCallback(() => { + setCompactionState((prev) => ({ ...prev, workingMemory: [] })); + }, []); + + const resetSession = useCallback(async () => { + setCompactionState(DEFAULT_COMPACTION_STATE); + if (config.persistSession) { + await clearSession(storageKey); + } + }, [config.persistSession, storageKey]); return { displayMessages, llmMessages, tokenUsage, - compactionState, + isCompacting, + compactionState: { + ...compactionState, + displayMessageCount: displayMessages.length, + llmMessageCount: llmMessages.length, + }, compactSession, addToWorkingMemory, clearWorkingMemory, resetSession, }; } + +async function fetchWithInstructions( + url: string, + messages: LLMMessage[], + instructions: string, +): Promise { + const res = await fetch(url, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ messages, instructions }), + }); + const data = await res.json(); + return data.summary as string; +} diff --git a/packages/copilot-sdk/src/react/provider/CopilotProvider.tsx b/packages/copilot-sdk/src/react/provider/CopilotProvider.tsx index 1ea9439..6a104a8 100644 --- a/packages/copilot-sdk/src/react/provider/CopilotProvider.tsx +++ b/packages/copilot-sdk/src/react/provider/CopilotProvider.tsx @@ -44,6 +44,15 @@ import { type ContextTreeNode, } from "../utils/context-tree"; import { useMCPTools } from "../hooks/useMCPTools"; +import { + MessageHistoryContext, + defaultMessageHistoryConfig, + useMessageHistoryContext, +} from "../message-history/context"; +import { useMessageHistory } from "../message-history/useMessageHistory"; +import { toDisplayMessage } from "../message-history/message-utils"; +import { buildSummaryBufferContext } from "../message-history/strategies/summary-buffer"; +import type { MessageHistoryConfig } from "../message-history/types"; // ============================================ // Internal MCP Connection Component @@ -62,6 +71,147 @@ function MCPConnection({ config }: { config: MCPServerConfig }) { return null; } +// ============================================ +// MessageHistoryBridge — wires useMessageHistory into AbstractChat.buildRequest() +// ============================================ + +const COMPACTING_MARKER_ID = "__compacting-in-progress__"; + +function MessageHistoryBridge({ + chatRef, +}: { + chatRef: React.MutableRefObject | null>; +}) { + const { compactionState, tokenUsage } = useMessageHistory(); + const ctx = useMessageHistoryContext(); + + // Track whether we've already added the loading marker for the current compaction cycle + const loaderAddedRef = useRef(false); + const prevCompactionCountRef = useRef(compactionState.compactionCount); + + // When threshold is first crossed → add loading indicator + useEffect(() => { + if (!tokenUsage.isApproaching) { + loaderAddedRef.current = false; + return; + } + if (loaderAddedRef.current) return; + const chat = chatRef.current; + if (!chat) return; + const alreadyAdded = chat.messages.some( + (m) => m.id === COMPACTING_MARKER_ID, + ); + if (alreadyAdded) return; + loaderAddedRef.current = true; + const loading: UIMessage = { + id: COMPACTING_MARKER_ID, + role: "system", + content: "Compacting conversation…", + createdAt: new Date(), + metadata: { type: "compaction-marker", compacting: true }, + }; + chat.setMessages([...chat.messages, loading]); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [tokenUsage.isApproaching]); + + // When compaction count increases → replace loader with permanent marker + useEffect(() => { + if (compactionState.compactionCount <= prevCompactionCountRef.current) + return; + prevCompactionCountRef.current = compactionState.compactionCount; + loaderAddedRef.current = false; + const chat = chatRef.current; + if (!chat) return; + const hasLoader = chat.messages.some((m) => m.id === COMPACTING_MARKER_ID); + const base = hasLoader + ? chat.messages.map((m) => + m.id === COMPACTING_MARKER_ID + ? { + ...m, + id: `compaction-marker-${compactionState.compactionCount}`, + content: `Conversation compacted — context window refreshed`, + metadata: { type: "compaction-marker", compacting: false }, + } + : m, + ) + : [ + ...chat.messages, + { + id: `compaction-marker-${compactionState.compactionCount}`, + role: "system" as const, + content: `Conversation compacted — context window refreshed`, + createdAt: new Date(), + metadata: { type: "compaction-marker", compacting: false }, + }, + ]; + chat.setMessages(base); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [compactionState.compactionCount]); + + // Keep latest compaction state + config in refs so the transform + // (called synchronously inside AbstractChat) always sees fresh values. + const compactionStateRef = useRef(compactionState); + compactionStateRef.current = compactionState; + const configRef = useRef(ctx.config); + configRef.current = ctx.config; + + useEffect(() => { + const chat = chatRef.current; + if (!chat) return; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (chat as any).setRequestMessageTransform((allMessages: UIMessage[]) => { + if (allMessages.length === 0) return allMessages; + + // Find the last user message — everything from here is the "current turn" + // (user msg + any assistant tool-calls + tool results). + // This is ALWAYS kept verbatim so we never send an invalid payload. + let lastUserIdx = -1; + for (let i = allMessages.length - 1; i >= 0; i--) { + if (allMessages[i].role === "user") { + lastUserIdx = i; + break; + } + } + + // No user message at all — pass through untouched (safety valve) + if (lastUserIdx === -1) return allMessages; + + const historyMessages = allMessages.slice(0, lastUserIdx); + const currentTurn = allMessages.slice(lastUserIdx); + + // Nothing to compact + if (historyMessages.length === 0) return allMessages; + + const cfg = configRef.current; + const maxTokens = cfg.maxContextTokens ?? 128000; + const reserve = cfg.reserveForResponse ?? 4096; + + // Apply summary-buffer only to the completed history, never the current turn + const compactedHistory = buildSummaryBufferContext( + historyMessages.map(toDisplayMessage), + compactionStateRef.current, + { + recentBuffer: cfg.recentBuffer ?? 10, + tokenBudget: maxTokens - reserve, + compactionThreshold: cfg.compactionThreshold ?? 0.75, + compactionUrl: cfg.compactionUrl, + }, + ); + + return [...compactedHistory, ...currentTurn] as unknown as UIMessage[]; + }); + return () => { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (chatRef.current as any)?.setRequestMessageTransform(null); + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + return null; +} + // ============================================ // Types // ============================================ @@ -107,6 +257,12 @@ export interface CopilotProviderProps { mcpServers?: MCPServerConfig[]; /** Optional prompt/tool optimization controls (tool profiles, context budgets, etc.) */ optimization?: ToolOptimizationConfig; + /** + * Context window management config. Controls compaction strategy, token budgets, + * session persistence, and working memory. + * @default strategy: 'none' — current behaviour, zero breaking changes + */ + messageHistory?: MessageHistoryConfig; } export interface CopilotContextValue { @@ -200,6 +356,7 @@ export function CopilotProvider({ maxIterationsMessage, mcpServers, optimization, + messageHistory, }: CopilotProviderProps) { // Debug logger const debugLog = useCallback( @@ -586,12 +743,39 @@ export function CopilotProvider({ ], ); + const messageHistoryContextValue = React.useMemo( + () => ({ + config: { ...defaultMessageHistoryConfig, ...messageHistory }, + tokenUsage: { + current: 0, + max: messageHistory?.maxContextTokens ?? 128000, + percentage: 0, + isApproaching: false, + }, + compactionState: { + rollingSummary: null, + lastCompactionAt: null, + compactionCount: 0, + totalTokensSaved: 0, + workingMemory: [], + displayMessageCount: 0, + llmMessageCount: 0, + }, + }), + [messageHistory], + ); + return ( - - {mcpServers?.map((config) => ( - - ))} - {children} - + + + {mcpServers?.map((config) => ( + + ))} + {messageHistory?.strategy && messageHistory.strategy !== "none" && ( + + )} + {children} + + ); } diff --git a/packages/copilot-sdk/src/server/compact-session.ts b/packages/copilot-sdk/src/server/compact-session.ts new file mode 100644 index 0000000..d0b8d96 --- /dev/null +++ b/packages/copilot-sdk/src/server/compact-session.ts @@ -0,0 +1,134 @@ +/** + * compactSession — server-side summarization helper + * + * Call this in your /api/compact route handler. + * Uses a structured prompt that preserves all semantically important content. + * + * @example + * ```ts + * // app/api/compact/route.ts + * import { compactSession } from '@yourgpt/copilot-sdk/server'; + * + * export async function POST(req: Request) { + * const { messages, existingSummary, workingMemory } = await req.json(); + * const summary = await compactSession({ messages, existingSummary, workingMemory }); + * return Response.json({ summary }); + * } + * ``` + */ + +export interface CompactSessionOptions { + messages: Array<{ role: string; content?: string | null }>; + existingSummary?: string | null; + workingMemory?: string[]; + /** + * Model to use for summarization. + * @default 'claude-haiku-4-5' (cheaper model fine for summaries) + */ + model?: string; + /** Max tokens for the summary output. @default 1024 */ + maxSummaryTokens?: number; + /** Custom fetch implementation (for non-browser environments). */ + fetchImpl?: typeof fetch; + /** Anthropic API key. Falls back to process.env.ANTHROPIC_API_KEY. */ + apiKey?: string; + /** Base URL for Anthropic API. @default 'https://api.anthropic.com' */ + apiBaseUrl?: string; +} + +export interface CompactSessionResult { + summary: string; +} + +const COMPACTION_PROMPT = `You are summarizing a conversation to preserve its key context while reducing token usage. Create a structured summary that includes: + +1. **User's primary goals and requests** — what the user is trying to accomplish +2. **Technical decisions made** — libraries chosen, schemas designed, approaches selected +3. **Tool call outcomes** — what tools were called, key arguments, result status and brief outcome +4. **Errors encountered** — what went wrong and how it was resolved +5. **User messages** — verbatim if short (<50 words), paraphrased if long +6. **Pending tasks** — unresolved questions or next steps mentioned +7. **Current work state** — what was in progress when this summary was created + +Rules: +- Preserve ALL specific values: file names, variable names, URLs, error messages, IDs +- Be detailed on recent work, more concise on earlier work +- Output structured prose (not bullet JSON) +- Do NOT include meta-commentary about the summarization itself`; + +export async function compactSession( + options: CompactSessionOptions, +): Promise { + const { + messages, + existingSummary, + workingMemory = [], + model = "claude-haiku-4-5-20251001", + maxSummaryTokens = 1024, + fetchImpl = fetch, + apiKey = typeof process !== "undefined" + ? process.env.ANTHROPIC_API_KEY + : undefined, + apiBaseUrl = "https://api.anthropic.com", + } = options; + + if (!apiKey) { + throw new Error( + "compactSession: No API key provided. Set ANTHROPIC_API_KEY or pass options.apiKey.", + ); + } + + // Build the content to summarize + const parts: string[] = []; + + if (workingMemory.length > 0) { + parts.push( + `[Working memory — always preserve these facts]\n${workingMemory.join("\n")}`, + ); + } + + if (existingSummary) { + parts.push(`[Previous summary — extend/update this]\n${existingSummary}`); + } + + const conversationText = messages + .map((m) => `${m.role.toUpperCase()}: ${m.content ?? "(no content)"}`) + .join("\n\n"); + + parts.push(`[Conversation to summarize]\n${conversationText}`); + + const userContent = parts.join("\n\n---\n\n"); + + const response = await fetchImpl(`${apiBaseUrl}/v1/messages`, { + method: "POST", + headers: { + "Content-Type": "application/json", + "x-api-key": apiKey, + "anthropic-version": "2023-06-01", + }, + body: JSON.stringify({ + model, + max_tokens: maxSummaryTokens, + system: COMPACTION_PROMPT, + messages: [{ role: "user", content: userContent }], + }), + }); + + if (!response.ok) { + const err = await response.text(); + throw new Error( + `compactSession: Anthropic API error ${response.status}: ${err}`, + ); + } + + const data = (await response.json()) as { + content: Array<{ type: string; text: string }>; + }; + + const summary = data.content + .filter((b) => b.type === "text") + .map((b) => b.text) + .join(""); + + return { summary }; +} diff --git a/packages/copilot-sdk/src/server/index.ts b/packages/copilot-sdk/src/server/index.ts new file mode 100644 index 0000000..9858c6c --- /dev/null +++ b/packages/copilot-sdk/src/server/index.ts @@ -0,0 +1,12 @@ +/** + * @yourgpt/copilot-sdk/server + * + * Server-only exports. Do NOT import in browser/React code. + */ + +// Context Management — server-side compaction +export { compactSession } from "./compact-session"; +export type { + CompactSessionOptions, + CompactSessionResult, +} from "./compact-session"; diff --git a/packages/copilot-sdk/src/ui/components/composed/chat/default-message.tsx b/packages/copilot-sdk/src/ui/components/composed/chat/default-message.tsx index e2d31c2..c4435fc 100644 --- a/packages/copilot-sdk/src/ui/components/composed/chat/default-message.tsx +++ b/packages/copilot-sdk/src/ui/components/composed/chat/default-message.tsx @@ -105,8 +105,41 @@ export function DefaultMessage({ citations = { enabled: true }, }: DefaultMessageProps) { const isUser = message.role === "user"; + const isCompactionMarker = + message.role === "system" && + (message.metadata as Record)?.type === "compaction-marker"; const isStreaming = isLastMessage && isLoading; + // Render compaction marker divider + if (isCompactionMarker) { + const tokensSaved = (message.metadata as Record) + ?.tokensSaved as number | undefined; + return ( +
+
+ + + + + {tokensSaved + ? `Earlier conversation summarized · ~${tokensSaved.toLocaleString()} tokens saved` + : "Earlier conversation summarized"} + +
+
+ ); + } + // Parse follow-up questions from assistant messages const { cleanContent: contentWithoutFollowUps, followUps } = React.useMemo(() => { @@ -357,11 +390,13 @@ export function DefaultMessage({ /> )} - {/* Show loader when processing after tool execution (only for last message) */} - {isLastMessage && isProcessing ? ( -
+ {/* Show loader when processing after tool execution (only for last message with no tools yet) */} + {isLastMessage && + isProcessing && + !completedTools?.length && + !pendingApprovalTools?.length ? ( +
- Continuing...
) : /* Show streaming loader when loading with no content and no tools */ isLastMessage && @@ -504,6 +539,16 @@ export function DefaultMessage({ ); })} + {/* Processing indicator below completed tools (AI is continuing after tool execution) */} + {isLastMessage && + isProcessing && + completedTools && + completedTools.length > 0 && ( +
+ +
+ )} + {/* Tool Approval Confirmations - Priority: toolRenderers > tool.render > default */} {pendingApprovalTools && pendingApprovalTools.length > 0 && (
diff --git a/packages/copilot-sdk/src/ui/components/composed/connected-chat.tsx b/packages/copilot-sdk/src/ui/components/composed/connected-chat.tsx index cb3e0a3..647e538 100644 --- a/packages/copilot-sdk/src/ui/components/composed/connected-chat.tsx +++ b/packages/copilot-sdk/src/ui/components/composed/connected-chat.tsx @@ -350,8 +350,15 @@ function CopilotChatBase( }); // Filter out tool messages and merge results into parent assistant messages + // Keep compaction-markers (system messages with type='compaction-marker') visible const visibleMessages = messages - .filter((m: UIMessage) => m.role !== "tool") // Hide tool messages - results merged into assistant + .filter( + (m: UIMessage) => + m.role !== "tool" && + (m.role !== "system" || + (m.metadata as Record)?.type === + "compaction-marker"), + ) // Hide tool/system messages except compaction markers .map((m: UIMessage) => { // For assistant messages with tool_calls, merge results let messageToolExecutions: ToolExecutionData[] | undefined; @@ -491,37 +498,49 @@ function CopilotChatBase( ? chatProps.suggestions : []; - // isProcessing: Show "Continuing..." loader ONLY when we're in an active tool flow - // Condition: Last message must be assistant with tool_calls (not user starting new request) + // isProcessing: Show "Continuing..." loader when tools finished and AI is about to respond const lastMessage = messages[messages.length - 1]; + + // Find the last assistant message with tool calls (may not be the very last message + // since tool result messages follow it) + const lastAssistantWithTools = [...messages] + .reverse() + .find( + (m) => m.role === "assistant" && (m as UIMessage).toolCalls?.length, + ) as UIMessage | undefined; + + // In tool flow when: last msg is a tool result (tools ran, waiting for AI), + // OR last msg is assistant with tool calls (tools still executing) const isInToolFlow = - lastMessage?.role === "assistant" && - (lastMessage as UIMessage).toolCalls?.length; + lastMessage?.role === "tool" || + (lastMessage?.role === "assistant" && + (lastMessage as UIMessage).toolCalls?.length); let isProcessingToolResults = false; if (isLoading && isInToolFlow) { - const currentToolCallIds = new Set( - (lastMessage as UIMessage).toolCalls?.map( - (tc: { id: string }) => tc.id, - ) || [], - ); - const currentExecutions = toolExecutions.filter((exec) => - currentToolCallIds.has(exec.id), - ); - - const hasCompletedTools = currentExecutions.some( - (exec) => - exec.status === "completed" || - exec.status === "error" || - exec.status === "failed", - ); - const hasExecutingTools = currentExecutions.some( - (exec) => exec.status === "executing" || exec.status === "pending", - ); - - // Show "Continuing..." only when tools completed and waiting for AI to continue - isProcessingToolResults = hasCompletedTools && !hasExecutingTools; + // Last message is a tool result → all tools for this turn are done, AI is continuing + if (lastMessage?.role === "tool") { + isProcessingToolResults = true; + } else if (lastAssistantWithTools) { + const currentToolCallIds = new Set( + lastAssistantWithTools.toolCalls?.map((tc: { id: string }) => tc.id) || + [], + ); + const currentExecutions = toolExecutions.filter((exec) => + currentToolCallIds.has(exec.id), + ); + const hasCompletedTools = currentExecutions.some( + (exec) => + exec.status === "completed" || + exec.status === "error" || + exec.status === "failed", + ); + const hasExecutingTools = currentExecutions.some( + (exec) => exec.status === "executing" || exec.status === "pending", + ); + isProcessingToolResults = hasCompletedTools && !hasExecutingTools; + } } // Extract chat classNames (without thread picker classes) From b85256dc03c8e7c516d21110552591c2808143b6 Mon Sep 17 00:00:00 2001 From: Sahil Date: Thu, 12 Mar 2026 23:43:33 +0530 Subject: [PATCH 11/72] refactor(sdk): enhance logging and streamline tool execution - Introduced a new logging utility to improve debug output with scoped logging capabilities. - Updated the `ChatWithTools` and `AbstractChat` classes to utilize the new logger for better debugging. - Refactored the `AbstractAgentLoop` to execute tool calls in parallel, improving performance and responsiveness. - Enhanced message handling in `CopilotProvider` to support new logging features. - Cleaned up code by removing unnecessary line breaks and comments for better readability. These changes aim to improve the overall debugging experience and performance of the SDK. --- .../copilot-sdk/src/chat/AbstractAgentLoop.ts | 31 +- .../copilot-sdk/src/chat/ChatWithTools.ts | 10 +- .../src/chat/classes/AbstractChat.ts | 297 ++++++++++++++++-- packages/copilot-sdk/src/chat/types/chat.ts | 2 +- packages/copilot-sdk/src/core/index.ts | 6 + packages/copilot-sdk/src/core/utils/logger.ts | 115 +++++++ .../src/react/provider/CopilotProvider.tsx | 73 +++-- 7 files changed, 467 insertions(+), 67 deletions(-) create mode 100644 packages/copilot-sdk/src/core/utils/logger.ts diff --git a/packages/copilot-sdk/src/chat/AbstractAgentLoop.ts b/packages/copilot-sdk/src/chat/AbstractAgentLoop.ts index 470cdaf..81622d8 100644 --- a/packages/copilot-sdk/src/chat/AbstractAgentLoop.ts +++ b/packages/copilot-sdk/src/chat/AbstractAgentLoop.ts @@ -277,23 +277,22 @@ export class AbstractAgentLoop implements AgentLoopActions { this._isProcessing = true; this.setIteration(this._iteration + 1); - const results: ToolResponse[] = []; - for (const toolCall of toolCalls) { - // Check if cancelled before each tool - if (this._isCancelled || this.abortController.signal.aborted) { - // Mark remaining tools as cancelled - results.push({ - toolCallId: toolCall.id, - success: false, - error: "Tool execution cancelled", - }); - continue; - } - - const result = await this.executeSingleTool(toolCall); - results.push(result); - } + // Run all tools in parallel so approval-required tools don't block + // non-approval tools. All results are still collected together before + // returning (Anthropic API requires results for every tool_use block). + const results = await Promise.all( + toolCalls.map((toolCall) => { + if (this._isCancelled || this.abortController!.signal.aborted) { + return Promise.resolve({ + toolCallId: toolCall.id, + success: false, + error: "Tool execution cancelled", + }); + } + return this.executeSingleTool(toolCall); + }), + ); this._isProcessing = false; return results; diff --git a/packages/copilot-sdk/src/chat/ChatWithTools.ts b/packages/copilot-sdk/src/chat/ChatWithTools.ts index d2de06f..772bdcd 100644 --- a/packages/copilot-sdk/src/chat/ChatWithTools.ts +++ b/packages/copilot-sdk/src/chat/ChatWithTools.ts @@ -18,6 +18,7 @@ import type { PermissionLevel, } from "../core"; import type { Resolvable } from "../core/utils/resolvable"; +import { createLogger } from "../core/utils/logger"; import { AbstractChat } from "./classes/AbstractChat"; import { AbstractAgentLoop } from "./AbstractAgentLoop"; import type { ChatConfig, ChatCallbacks } from "./types"; @@ -47,7 +48,7 @@ export interface ChatWithToolsConfig { body?: Resolvable>; /** Thread ID for conversation persistence */ threadId?: string; - /** Debug mode */ + /** Enable debug logging */ debug?: boolean; /** Initial messages */ initialMessages?: UIMessage[]; @@ -601,9 +602,10 @@ export class ChatWithTools { // ============================================ private debug(message: string, ...args: unknown[]): void { - if (this.config.debug) { - console.log(`[ChatWithTools] ${message}`, ...args); - } + createLogger("tools", () => this.config.debug ?? false)( + message, + args.length === 1 ? args[0] : args.length > 1 ? args : undefined, + ); } } diff --git a/packages/copilot-sdk/src/chat/classes/AbstractChat.ts b/packages/copilot-sdk/src/chat/classes/AbstractChat.ts index 2b0d6b9..6bcd59f 100644 --- a/packages/copilot-sdk/src/chat/classes/AbstractChat.ts +++ b/packages/copilot-sdk/src/chat/classes/AbstractChat.ts @@ -43,6 +43,7 @@ import { } from "../functions/stream"; import { SimpleChatState } from "../interfaces/ChatState"; import { ChatContextOptimizer } from "../optimizations"; +import { createLogger } from "../../core/utils/logger"; /** * Event types emitted by AbstractChat @@ -462,13 +463,29 @@ export class AbstractChat { // Build request const request = this.buildRequest(); + // For streaming: pre-push an empty assistant message BEFORE the HTTP + // round-trip so the UI shows a loading bubble immediately (e.g. between + // tool execution and the continuation stream starting). + let preCreatedMessageId: string | undefined; + if (this.config.streaming !== false) { + const preMsg = createEmptyAssistantMessage() as T; + this.state.pushMessage(preMsg); + this.callbacks.onMessagesChange?.(this.state.messages); + preCreatedMessageId = preMsg.id; + } + // Send request const response = await this.transport.send(request); // Check if streaming or JSON if (this.isAsyncIterable(response)) { - await this.handleStreamResponse(response); + await this.handleStreamResponse(response, preCreatedMessageId); } else { + // Non-streaming: remove the pre-pushed placeholder (not needed) + if (preCreatedMessageId) { + const id = preCreatedMessageId; + this.state.setMessages(this.state.messages.filter((m) => m.id !== id)); + } this.handleJsonResponse(response); } } @@ -531,7 +548,6 @@ export class AbstractChat { */ setContext(context: string): void { this.dynamicContext = context; - this.debug("Context updated", { length: context.length }); } /** @@ -540,7 +556,6 @@ export class AbstractChat { */ setSystemPrompt(prompt: string): void { this.config.systemPrompt = prompt; - this.debug("System prompt updated", { length: prompt.length }); } /** @@ -552,7 +567,6 @@ export class AbstractChat { if (this.transport.setHeaders && headers !== undefined) { this.transport.setHeaders(headers); } - this.debug("Headers config updated"); } /** @@ -564,7 +578,6 @@ export class AbstractChat { if (this.transport.setUrl) { this.transport.setUrl(url); } - this.debug("URL config updated"); } /** @@ -576,7 +589,6 @@ export class AbstractChat { if (this.transport.setBody && body !== undefined) { this.transport.setBody(body); } - this.debug("Body config updated"); } /** @@ -624,27 +636,51 @@ export class AbstractChat { */ protected async handleStreamResponse( stream: AsyncIterable, + preCreatedMessageId?: string, ): Promise { this.state.status = "streaming"; this.callbacks.onStatusChange?.("streaming"); - // Create empty assistant message for streaming - const assistantMessage = createEmptyAssistantMessage() as T; - this.state.pushMessage(assistantMessage); + // Reuse the pre-pushed empty assistant message (created in processRequest + // before the HTTP round-trip) so there's no blank gap waiting for stream start. + // Fall back to pushing a new one if not provided. + let assistantMessage: T; + if (preCreatedMessageId) { + const existing = this.state.messages.find( + (m) => m.id === preCreatedMessageId, + ); + if (existing) { + assistantMessage = existing; + } else { + assistantMessage = createEmptyAssistantMessage() as T; + this.state.pushMessage(assistantMessage); + } + } else { + assistantMessage = createEmptyAssistantMessage() as T; + this.state.pushMessage(assistantMessage); + } // Initialize stream state this.streamState = createStreamState(assistantMessage.id); this.callbacks.onMessageStart?.(assistantMessage.id); - this.debug("handleStreamResponse", "Starting to process stream"); + this.debugGroup("handleStreamResponse"); + this.debug("Starting to process stream"); let chunkCount = 0; let toolCallsEmitted = false; // Guard to prevent emitting toolCalls twice + // Holds client tool calls received via a tool_calls chunk AFTER a + // mid-stream message:end nulled streamState. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + let pendingClientToolCalls: any[] | undefined; // Process stream chunks for await (const chunk of stream) { chunkCount++; - this.debug("chunk", { count: chunkCount, type: chunk.type }); + // Skip high-frequency delta chunks from the chunk log to reduce noise + if (chunk.type !== "message:delta") { + this.debug("chunk", { count: chunkCount, type: chunk.type }); + } // Handle error chunks immediately if (chunk.type === "error") { @@ -656,7 +692,12 @@ export class AbstractChat { // Handle message:end mid-stream (server-side agent loop turn completed) // This creates separate messages for each turn instead of combining them if (chunk.type === "message:end" && this.streamState?.content) { - this.debug("message:end mid-stream - finalizing current turn"); + this.debug("message:end mid-stream", { + messageId: this.streamState.messageId, + contentLength: this.streamState.content.length, + toolCallsInState: this.streamState.toolCalls?.length ?? 0, + chunkCount, + }); // Finalize current message with its content and tool calls const turnMessage = streamStateToMessage(this.streamState) as T; @@ -702,8 +743,128 @@ export class AbstractChat { } // Update stream state (pure function) - // Skip if streamState is null (shouldn't happen but be safe) + // Skip most chunks if streamState is null. + // EXCEPTION: after a mid-stream message:end the server can still send + // tool_calls + done for client-side tool dispatch. Handle those directly. if (!this.streamState) { + if (chunk.type === "tool_calls") { + // Store for emission when done arrives. Do NOT update message state + // here — done.messages carries the assistant message with tool_calls + // in proper OpenAI format, which we use in the done handler below. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + pendingClientToolCalls = (chunk as { toolCalls: any[] }).toolCalls; + this.debug("tool_calls (post-message:end, stored as pending)", { + count: pendingClientToolCalls?.length, + ids: pendingClientToolCalls?.map((tc: { id?: string }) => tc.id), + }); + continue; + } + + if (chunk.type === "done") { + this.debug("done (post-message:end)", { + hasPendingToolCalls: !!pendingClientToolCalls?.length, + pendingCount: pendingClientToolCalls?.length ?? 0, + doneMessagesCount: chunk.messages?.length ?? 0, + requiresAction: (chunk as { requiresAction?: boolean }) + .requiresAction, + toolCallsEmitted, + }); + // Process done.messages to: + // 1. Insert any server-side tool results missing from state + // 2. Merge OpenAI-format tool_calls into the finalized assistant message + if (chunk.messages?.length) { + const pendingIds = new Set( + ((pendingClientToolCalls ?? []) as Array<{ id?: string }>) + .filter((tc) => tc?.id) + .map((tc) => tc.id as string), + ); + const messagesToInsert: T[] = []; + let clientAssistantToolCalls: unknown[] | undefined; + + for (const msg of chunk.messages) { + // This is the client-tool assistant message already in state + // (finalized by message:end but without toolCalls). + // Capture its OpenAI-format tool_calls to merge into state. + if ( + msg.role === "assistant" && + msg.tool_calls?.length && + pendingIds.size > 0 && + (msg.tool_calls as Array<{ id?: string }>).every((tc) => + pendingIds.has(tc?.id ?? ""), + ) + ) { + clientAssistantToolCalls = msg.tool_calls as unknown[]; + continue; // Already in state — don't insert a duplicate + } + // Skip plain assistant text — already streamed + if (msg.role === "assistant" && !msg.tool_calls?.length) continue; + // Everything else (server tool results) needs inserting + messagesToInsert.push({ + id: generateMessageId(), + role: msg.role as T["role"], + content: msg.content ?? "", + toolCalls: msg.tool_calls as T["toolCalls"], + toolCallId: msg.tool_call_id, + createdAt: new Date(), + } as T); + } + + // Merge OpenAI-format tool_calls into the existing last assistant message + if (clientAssistantToolCalls) { + const currentMessages = this.state.messages; + for (let i = currentMessages.length - 1; i >= 0; i--) { + if (currentMessages[i].role === "assistant") { + this.state.updateMessageById( + currentMessages[i].id, + (m) => + ({ + ...m, + toolCalls: clientAssistantToolCalls, + }) as T, + ); + break; + } + } + } + + if (messagesToInsert.length > 0) { + // Insert server tool results before the last assistant message + const currentMessages = this.state.messages; + let insertIdx = currentMessages.length; + for (let i = currentMessages.length - 1; i >= 0; i--) { + if (currentMessages[i].role === "assistant") { + insertIdx = i; + break; + } + } + this.state.setMessages([ + ...currentMessages.slice(0, insertIdx), + ...messagesToInsert, + ...currentMessages.slice(insertIdx), + ]); + } + } + + // Emit client tool calls so ChatWithTools executes them + if (!toolCallsEmitted && pendingClientToolCalls?.length) { + toolCallsEmitted = true; + this.debug("emit toolCalls (post-message:end path)", { + count: pendingClientToolCalls.length, + names: pendingClientToolCalls.map( + (tc: { function?: { name: string }; name?: string }) => + tc.function?.name ?? tc.name, + ), + }); + this.emit("toolCalls", { toolCalls: pendingClientToolCalls }); + } else { + this.debug("skip emit toolCalls (post-message:end path)", { + toolCallsEmitted, + hasPending: !!pendingClientToolCalls?.length, + }); + } + continue; + } + this.debug("warning", "streamState is null, skipping chunk"); continue; } @@ -757,13 +918,26 @@ export class AbstractChat { // Check for completion if (isStreamDone(chunk)) { - this.debug("streamDone", { chunk }); + this.debug("streamDone", { + chunkType: chunk.type, + requiresAction: (chunk as { requiresAction?: boolean }) + .requiresAction, + doneMessagesCount: + (chunk as { messages?: unknown[] }).messages?.length ?? 0, + streamToolCallsCount: this.streamState?.toolCalls?.length ?? 0, + toolCallsEmitted, + chunkCount, + }); // CRITICAL: Process messages from done event (server-side tool results) // Without this, tool_call_id is lost and causes Anthropic API errors if (chunk.type === "done" && chunk.messages?.length) { this.debug("processDoneMessages", { count: chunk.messages.length, + roles: chunk.messages.map( + (m) => + `${m.role}${m.tool_calls?.length ? `[${(m.tool_calls as unknown[]).length}tc]` : ""}`, + ), }); const currentStreamToolCallIds = new Set( @@ -851,14 +1025,57 @@ export class AbstractChat { // merged into local state. Emitting earlier on the first tool_calls // chunk can race with recursive server-tool turns and produce an // invalid continuation order for OpenAI-compatible providers. - if ( - chunk.requiresAction && - !toolCallsEmitted && - updatedMessage.toolCalls?.length - ) { - toolCallsEmitted = true; - this.debug("toolCalls", { toolCalls: updatedMessage.toolCalls }); - this.emit("toolCalls", { toolCalls: updatedMessage.toolCalls }); + this.debug("requiresAction check", { + requiresAction: chunk.requiresAction, + toolCallsEmitted, + updatedMessageToolCallsCount: updatedMessage.toolCalls?.length ?? 0, + messagesToInsertCount: messagesToInsert.length, + }); + + if (chunk.requiresAction && !toolCallsEmitted) { + // When the server runs a multi-turn agent loop before handing off + // to the client, the client tool calls arrive via done.messages + // (messagesToInsert), NOT in the current streaming message's + // toolCalls (which is always empty because action:start/args/end + // chunks only fire callbacks and never update streamState.toolCalls). + // Find the last assistant message in the inserted batch that carries + // tool calls — that is the pending client tool dispatch. + let clientToolCalls = updatedMessage.toolCalls; + if (!clientToolCalls?.length && messagesToInsert.length > 0) { + for (let i = messagesToInsert.length - 1; i >= 0; i--) { + const m = messagesToInsert[i]; + if (m.role === "assistant" && m.toolCalls?.length) { + clientToolCalls = m.toolCalls; + this.debug("clientToolCalls from messagesToInsert", { + index: i, + count: clientToolCalls?.length, + }); + break; + } + } + } + + if (clientToolCalls?.length) { + toolCallsEmitted = true; + this.debug("emit toolCalls (normal done path)", { + count: clientToolCalls.length, + names: ( + clientToolCalls as Array<{ + function?: { name: string }; + name?: string; + }> + ).map((tc) => tc.function?.name ?? tc.name), + }); + this.emit("toolCalls", { toolCalls: clientToolCalls }); + } else { + this.debug("requiresAction=true but no clientToolCalls found", { + updatedMessageToolCalls: updatedMessage.toolCalls, + messagesToInsert: messagesToInsert.map((m) => ({ + role: m.role, + hasToolCalls: !!m.toolCalls?.length, + })), + }); + } } } @@ -914,9 +1131,17 @@ export class AbstractChat { this.callbacks.onMessagesChange?.(this.state.messages); + // Close the stream group opened at the start of handleStreamResponse + this.debugGroupEnd(); + // Only set status to "ready" if NO tool calls were emitted // If tool calls were emitted, the async handler will manage status // (it will set "submitted" then "streaming" for the continuation) + this.debug("stream end", { + toolCallsEmitted, + totalChunks: chunkCount, + messagesInState: this.state.messages.length, + }); if (!toolCallsEmitted) { this.state.status = "ready"; this.callbacks.onStatusChange?.("ready"); @@ -1010,15 +1235,33 @@ export class AbstractChat { this.emit("error", { error }); } - /** - * Debug logging - */ + // ─── Debug helpers ──────────────────────────────────────────────────────── + + private _log?: import("../../core/utils/logger").ScopedLogger; + + private get log(): import("../../core/utils/logger").ScopedLogger { + if (!this._log) { + this._log = createLogger("streaming", () => this.config.debug ?? false); + } + return this._log; + } + protected debug(action: string, data?: unknown): void { - if (this.config.debug) { - console.log(`[AbstractChat] ${action}`, data); + this.log(action, data); + } + + protected debugGroup(label: string, collapsed = true): void { + if (collapsed) { + this.log.groupCollapsed(label); + } else { + this.log.group(label); } } + protected debugGroupEnd(): void { + this.log.groupEnd(); + } + /** * Type guard for async iterable */ diff --git a/packages/copilot-sdk/src/chat/types/chat.ts b/packages/copilot-sdk/src/chat/types/chat.ts index 1c536ae..8070461 100644 --- a/packages/copilot-sdk/src/chat/types/chat.ts +++ b/packages/copilot-sdk/src/chat/types/chat.ts @@ -54,7 +54,7 @@ export interface ChatConfig { body?: Resolvable>; /** Thread ID for conversation persistence */ threadId?: string; - /** Debug mode */ + /** Enable debug logging */ debug?: boolean; /** Available tools (passed to LLM) */ tools?: ToolDefinition[]; diff --git a/packages/copilot-sdk/src/core/index.ts b/packages/copilot-sdk/src/core/index.ts index dcc7d36..dd53ae9 100644 --- a/packages/copilot-sdk/src/core/index.ts +++ b/packages/copilot-sdk/src/core/index.ts @@ -97,6 +97,12 @@ export type { WebSearchProviderInterface, } from "./tools"; +// ============================================ +// Logger +// ============================================ +export { createLogger, logOnce } from "./utils/logger"; +export type { ScopedLogger, DebugConfig } from "./utils/logger"; + // ============================================ // Core Types // ============================================ diff --git a/packages/copilot-sdk/src/core/utils/logger.ts b/packages/copilot-sdk/src/core/utils/logger.ts new file mode 100644 index 0000000..b44f7af --- /dev/null +++ b/packages/copilot-sdk/src/core/utils/logger.ts @@ -0,0 +1,115 @@ +/** + * Copilot SDK Logger + * + * Simple debug logger with console grouping support. + * + * Usage: + * debug={true} → enable all logs + * debug={false} → silent (default) + * + * Runtime toggle from browser console (no rebuild needed): + * window.__COPILOT_DEBUG = true + * window.__COPILOT_DEBUG = false + */ + +// ─── Types ───────────────────────────────────────────────────────────────── + +/** Well-known log scopes — used internally as labels, not exposed to the user. */ +type LogScope = "streaming" | "tools" | "provider" | string; + +/** Debug config accepted by CopilotProvider. true = on, false = off. */ +export type DebugConfig = boolean; + +declare global { + interface Window { + __COPILOT_DEBUG?: boolean; + } +} + +// ─── Helpers ─────────────────────────────────────────────────────────────── + +function isEnabled(getEnabled: () => boolean): boolean { + if (typeof window !== "undefined" && window.__COPILOT_DEBUG !== undefined) { + return !!window.__COPILOT_DEBUG; + } + return getEnabled(); +} + +// ─── Factory ─────────────────────────────────────────────────────────────── + +export interface ScopedLogger { + /** Log a message with optional data */ + (action: string, data?: unknown): void; + /** Open a collapsible group — all subsequent logs nest inside until groupEnd() */ + group(label: string): void; + /** Open a collapsed group (hidden by default in DevTools) */ + groupCollapsed(label: string): void; + /** Close the most recently opened group */ + groupEnd(): void; +} + +/** + * Create a scoped logger bound to a specific namespace. + * + * @param scope - Label shown in brackets, e.g. "streaming", "tools" + * @param getEnabled - Returns whether debug logging is currently on + * + * @example + * const log = createLogger("streaming", () => this.config.debug ?? false); + * log("sendMessage", { content }); + * // → [streaming] sendMessage { content: '...' } + * + * log.groupCollapsed("Stream #1"); + * log("chunk", { type: "message:start" }); // nested inside group + * log.groupEnd(); + */ +/** + * One-shot log — for cases where you don't keep a persistent logger. + */ +export function logOnce( + scope: LogScope, + enabled: boolean, + action: string, + data?: unknown, +): void { + if (!isEnabled(() => enabled)) return; + const prefix = `[${scope}]`; + if (data !== undefined) { + console.log(prefix, action, data); + } else { + console.log(prefix, action); + } +} + +export function createLogger( + scope: LogScope, + getEnabled: () => boolean, +): ScopedLogger { + const prefix = `[${scope}]`; + + function log(action: string, data?: unknown): void { + if (!isEnabled(getEnabled)) return; + if (data !== undefined) { + console.log(prefix, action, data); + } else { + console.log(prefix, action); + } + } + + log.group = function (label: string): void { + if (!isEnabled(getEnabled)) return; + console.group(`${prefix} ${label}`); + }; + + log.groupCollapsed = function (label: string): void { + if (!isEnabled(getEnabled)) return; + console.groupCollapsed(`${prefix} ${label}`); + }; + + log.groupEnd = function (): void { + if (!isEnabled(getEnabled)) return; + console.groupEnd(); + }; + + return log; +} diff --git a/packages/copilot-sdk/src/react/provider/CopilotProvider.tsx b/packages/copilot-sdk/src/react/provider/CopilotProvider.tsx index 6a104a8..fbf77e9 100644 --- a/packages/copilot-sdk/src/react/provider/CopilotProvider.tsx +++ b/packages/copilot-sdk/src/react/provider/CopilotProvider.tsx @@ -30,6 +30,7 @@ import type { import type { MCPServerConfig } from "../../mcp/types"; import type { Resolvable } from "../../core/utils/resolvable"; +import { createLogger } from "../../core/utils/logger"; import type { UIMessage, ToolExecution } from "../../chat"; @@ -50,8 +51,6 @@ import { useMessageHistoryContext, } from "../message-history/context"; import { useMessageHistory } from "../message-history/useMessageHistory"; -import { toDisplayMessage } from "../message-history/message-utils"; -import { buildSummaryBufferContext } from "../message-history/strategies/summary-buffer"; import type { MessageHistoryConfig } from "../message-history/types"; // ============================================ @@ -185,22 +184,58 @@ function MessageHistoryBridge({ if (historyMessages.length === 0) return allMessages; const cfg = configRef.current; - const maxTokens = cfg.maxContextTokens ?? 128000; - const reserve = cfg.reserveForResponse ?? 4096; - - // Apply summary-buffer only to the completed history, never the current turn - const compactedHistory = buildSummaryBufferContext( - historyMessages.map(toDisplayMessage), - compactionStateRef.current, - { - recentBuffer: cfg.recentBuffer ?? 10, - tokenBudget: maxTokens - reserve, - compactionThreshold: cfg.compactionThreshold ?? 0.75, - compactionUrl: cfg.compactionUrl, - }, + + // Apply summary-buffer windowing to history, keeping UIMessage format. + // + // WHY NOT buildSummaryBufferContext here: + // buildSummaryBufferContext returns LLMMessage[] (snake_case: tool_calls, + // tool_call_id). The optimizer's transformMessages() only reads camelCase + // (toolCalls, toolCallId), so mixing LLMMessage into this array causes it + // to silently strip tool call data → "Missing call_id" API errors. + // The optimizer must own the UIMessage → RequestMessage conversion. + const cs = compactionStateRef.current; + const recentBuffer = cfg.recentBuffer ?? 10; + + // Identify compaction marker messages (UI-only, already represented by rollingSummary) + const isCompactionMsg = (m: UIMessage) => + m.metadata?.["type"] === "compaction-marker"; + + const windowedHistory: UIMessage[] = []; + + // 1. Working memory (always first) + if (cs.workingMemory.length > 0) { + windowedHistory.push({ + id: "working-memory", + role: "system", + content: `[Working memory — always active]\n${cs.workingMemory.join("\n")}`, + createdAt: new Date(), + } as UIMessage); + } + + // 2. Rolling summary replaces older history + if (cs.rollingSummary) { + windowedHistory.push({ + id: "rolling-summary", + role: "system", + content: `[Previous conversation summary]\n${cs.rollingSummary}`, + createdAt: new Date(), + } as UIMessage); + } + + // 3. Non-compaction system messages (e.g. injected context) + const systemMsgs = historyMessages.filter( + (m) => m.role === "system" && !isCompactionMsg(m), + ); + windowedHistory.push(...systemMsgs); + + // 4. Recent conversation messages (windowed to recentBuffer) + const conversationMsgs = historyMessages.filter( + (m) => m.role !== "system", ); + const recentStart = Math.max(0, conversationMsgs.length - recentBuffer); + windowedHistory.push(...conversationMsgs.slice(recentStart)); - return [...compactedHistory, ...currentTurn] as unknown as UIMessage[]; + return [...windowedHistory, ...currentTurn]; }); return () => { // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -358,10 +393,10 @@ export function CopilotProvider({ optimization, messageHistory, }: CopilotProviderProps) { - // Debug logger + // Debug logger — scoped to "provider" namespace const debugLog = useCallback( - (...args: unknown[]) => { - if (debug) console.log("[Copilot SDK]", ...args); + (action: string, data?: unknown) => { + createLogger("provider", () => debug ?? false)(action, data); }, [debug], ); From 80aa1bef23f5ae9067bd5d78093bb31697c17cd0 Mon Sep 17 00:00:00 2001 From: Sahil Date: Sat, 14 Mar 2026 17:11:04 +0530 Subject: [PATCH 12/72] feat(sdk): add support for fallback tool renderer and message grouping - Introduced a `fallbackToolRenderer` prop to the `Chat` and `DefaultMessage` components, allowing for a catch-all rendering option for unmatched tools. - Implemented `groupConsecutiveMessages` functionality to hide avatars for consecutive messages from the same role, enhancing message display in chat. - Updated type definitions to include new props and ensure proper usage across components. These enhancements improve the flexibility of tool rendering and the visual organization of chat messages. --- .../src/ui/components/composed/chat/chat.tsx | 53 ++++++++++++++++++- .../composed/chat/default-message.tsx | 38 +++++++++++-- .../src/ui/components/composed/chat/types.ts | 17 ++++++ 3 files changed, 102 insertions(+), 6 deletions(-) diff --git a/packages/copilot-sdk/src/ui/components/composed/chat/chat.tsx b/packages/copilot-sdk/src/ui/components/composed/chat/chat.tsx index 0e26989..11e0219 100644 --- a/packages/copilot-sdk/src/ui/components/composed/chat/chat.tsx +++ b/packages/copilot-sdk/src/ui/components/composed/chat/chat.tsx @@ -523,6 +523,7 @@ function ChatComponent({ registeredTools, toolRenderers, mcpToolRenderer, + fallbackToolRenderer, onApproveToolExecution, onRejectToolExecution, // Follow-up Questions @@ -535,6 +536,8 @@ function ChatComponent({ renderMessage, renderInput, renderHeader, + // Avatar grouping + groupConsecutiveMessages = false, // Styling className, classNames = {}, @@ -910,6 +913,43 @@ function ChatComponent({ {/* Messages */} {messages.map((message, index) => { const isLastMessage = index === messages.length - 1; + + const GROUP_THRESHOLD_MS = 5 * 60 * 1000; + const shouldHideAvatar = (() => { + if (!groupConsecutiveMessages || index === 0) + return false; + let prevIdx = index - 1; + while (prevIdx >= 0) { + const prev = messages[prevIdx]; + const isToolMsg = prev.role === "tool"; + const isInvisibleSystem = + prev.role === "system" && + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (prev.metadata as Record)?.type !== + "compaction-marker"; + if (!isToolMsg && !isInvisibleSystem) break; + prevIdx--; + } + if (prevIdx < 0) return false; + const prevVisible = messages[prevIdx]; + if (prevVisible.role !== message.role) return false; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const curTs = (message as any).timestamp as + | number + | undefined; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const prevTs = (prevVisible as any).timestamp as + | number + | undefined; + if ( + curTs && + prevTs && + curTs - prevTs > GROUP_THRESHOLD_MS + ) + return false; + return true; + })(); + const isEmptyAssistant = message.role === "assistant" && !message.content?.trim(); @@ -967,8 +1007,16 @@ function ChatComponent({ ; + /** Catch-all renderer for any tool not matched by toolRenderers */ + fallbackToolRenderer?: React.ComponentType; /** Called when user approves a tool execution */ onApproveToolExecution?: ( executionId: string, @@ -96,6 +104,7 @@ export function DefaultMessage({ registeredTools, toolRenderers, mcpToolRenderer, + fallbackToolRenderer, onApproveToolExecution, onRejectToolExecution, showFollowUps = true, @@ -300,6 +309,7 @@ export function DefaultMessage({ src={userAvatar.src} alt="User" fallback={userAvatar.fallback} + className={userAvatar.className} > {userAvatar.component} @@ -326,13 +336,14 @@ export function DefaultMessage({ (exec) => exec.approvalStatus !== "required" && !isToolHidden(exec), ); - // Helper: check if tool has any custom render (toolRenderers, mcpToolRenderer, or tool.render) + // Helper: check if tool has any custom render (toolRenderers, mcpToolRenderer, fallbackToolRenderer, or tool.render) const hasCustomRender = (toolName: string, execSource?: string): boolean => { if (toolRenderers?.[toolName]) return true; const toolDef = registeredTools?.find((t) => t.name === toolName); // Check if mcpToolRenderer applies (MCP tool with catch-all renderer) if (mcpToolRenderer && (execSource === "mcp" || toolDef?.source === "mcp")) return true; + if (fallbackToolRenderer) return true; if (toolDef?.render) return true; return false; }; @@ -376,7 +387,7 @@ export function DefaultMessage({ ) : undefined } - className="bg-muted" + className={cn("bg-muted", assistantAvatar.className)} > {assistantAvatar.component} @@ -473,7 +484,26 @@ export function DefaultMessage({ ); } - // PRIORITY 3: tool's own render function + // PRIORITY 3: fallbackToolRenderer (catch-all for any unmatched tool) + if (fallbackToolRenderer) { + const FallbackRenderer = fallbackToolRenderer; + return ( + + ); + } + + // PRIORITY 4: tool's own render function // toolDef already defined above for MCP check const toolDefForRender = toolDef ?? diff --git a/packages/copilot-sdk/src/ui/components/composed/chat/types.ts b/packages/copilot-sdk/src/ui/components/composed/chat/types.ts index 428426b..e903df6 100644 --- a/packages/copilot-sdk/src/ui/components/composed/chat/types.ts +++ b/packages/copilot-sdk/src/ui/components/composed/chat/types.ts @@ -403,6 +403,19 @@ export type ChatProps = { */ mcpToolRenderer?: React.ComponentType; + /** + * Catch-all renderer for ALL tools not matched by `toolRenderers`. + * Applied regardless of tool source (native, custom, or MCP). + * + * Priority: toolRenderers[name] > mcpToolRenderer (mcp only) > fallbackToolRenderer > tool.render > default + * + * @example + * ```tsx + * + * ``` + */ + fallbackToolRenderer?: React.ComponentType; + // === Tool Approval (Human-in-the-loop) === /** * Called when user approves a tool execution. @@ -430,6 +443,10 @@ export type ChatProps = { /** Custom header renderer (replaces entire header) */ renderHeader?: () => React.ReactNode; + /** Group consecutive messages from same role — hides avatar on non-first messages in a run. + * Resets on role change or if messages are > 5 minutes apart. */ + groupConsecutiveMessages?: boolean; + // === Styling === /** Class name for root container (use for sizing) */ className?: string; From 07ff53b59d612a495496731367cc104abb76f7b0 Mon Sep 17 00:00:00 2001 From: Sahil Date: Sat, 14 Mar 2026 20:49:57 +0530 Subject: [PATCH 13/72] feat(sdk): introduce useContextStats hook for context window usage tracking - Added the `useContextStats` hook to provide live statistics on the AI copilot's context window usage, including token counts and percentages. - Updated `CopilotProvider` to manage context characters and usage, ensuring reactive updates during message interactions. - Re-exported relevant types and hooks for better accessibility in the SDK. These enhancements improve the monitoring of context usage, aiding in efficient resource management during interactions. --- packages/copilot-sdk/src/react/hooks/index.ts | 7 + .../src/react/hooks/useContextStats.ts | 131 ++++++++++++++++++ packages/copilot-sdk/src/react/index.ts | 10 ++ .../src/react/provider/CopilotProvider.tsx | 22 +++ 4 files changed, 170 insertions(+) create mode 100644 packages/copilot-sdk/src/react/hooks/useContextStats.ts diff --git a/packages/copilot-sdk/src/react/hooks/index.ts b/packages/copilot-sdk/src/react/hooks/index.ts index 52581c7..6f11565 100644 --- a/packages/copilot-sdk/src/react/hooks/index.ts +++ b/packages/copilot-sdk/src/react/hooks/index.ts @@ -96,3 +96,10 @@ export { type UseMCPUIIntentsConfig, type UseMCPUIIntentsReturn, } from "./useMCPUIIntents"; + +// Context Stats (context window usage, token estimates, tool count) +export { + useContextStats, + type ContextStats, + type MessageTokenUsage, +} from "./useContextStats"; diff --git a/packages/copilot-sdk/src/react/hooks/useContextStats.ts b/packages/copilot-sdk/src/react/hooks/useContextStats.ts new file mode 100644 index 0000000..91287d9 --- /dev/null +++ b/packages/copilot-sdk/src/react/hooks/useContextStats.ts @@ -0,0 +1,131 @@ +"use client"; + +import { useMemo } from "react"; +import { useCopilot } from "../provider/CopilotProvider"; +import type { UIMessage } from "../../chat"; +import type { ContextUsage } from "../../core"; + +/** + * Per-message token usage returned by the LLM provider. + */ +export interface MessageTokenUsage { + prompt_tokens: number; + completion_tokens: number; + total_tokens: number; +} + +/** + * Context window stats — updated reactively as messages are sent and contexts change. + */ +export interface ContextStats { + /** + * Full context usage snapshot from the last send — includes token counts and + * percentages for every budget bucket (systemPrompt, history, toolResults, tools). + * null until the first message is sent. + */ + contextUsage: ContextUsage | null; + + /** + * Convenience: total estimated tokens currently in the prompt (from contextUsage). + * Falls back to a fast chars/3.5 estimate from contextChars before first send. + */ + totalTokens: number; + + /** + * Convenience: percentage of context window used (0–1). + * 0 until first send. + */ + usagePercent: number; + + /** Total characters currently in the AI context (system prompt contribution). */ + contextChars: number; + + /** Number of tools currently registered in the agent loop. */ + toolCount: number; + + /** Number of visible (non-system) messages in the active thread. */ + messageCount: number; + + /** + * Actual token usage from the last assistant message metadata (if provider returned it). + * null if not available. + */ + lastResponseUsage: MessageTokenUsage | null; +} + +// ── Helpers ────────────────────────────────────────────────────────────────── + +function getLastResponseUsage(messages: UIMessage[]): MessageTokenUsage | null { + for (let i = messages.length - 1; i >= 0; i--) { + const msg = messages[i]; + if (msg.role === "assistant" && msg.metadata?.usage) { + const u = msg.metadata.usage as { + prompt_tokens?: number; + completion_tokens?: number; + total_tokens?: number; + }; + const prompt = u.prompt_tokens ?? 0; + const completion = u.completion_tokens ?? 0; + return { + prompt_tokens: prompt, + completion_tokens: completion, + total_tokens: u.total_tokens ?? prompt + completion, + }; + } + } + return null; +} + +// ── Hook ───────────────────────────────────────────────────────────────────── + +/** + * useContextStats — live snapshot of the AI copilot's context window usage. + * + * `contextUsage` is the richest field — it has full breakdown by bucket with + * token counts and percentages, updated on every message send. + * + * @example + * ```tsx + * const { contextUsage, toolCount, totalTokens, usagePercent } = useContextStats(); + * // contextUsage.breakdown.systemPrompt.percent — % of window used by system prompt + * // contextUsage.breakdown.history.tokens — tokens from conversation history + * // usagePercent — overall window fill (0–1) + * ``` + */ +export function useContextStats(): ContextStats { + const { contextChars, contextUsage, registeredTools, messages } = + useCopilot(); + + const toolCount = useMemo(() => registeredTools.length, [registeredTools]); + + const messageCount = useMemo( + () => messages.filter((m) => m.role !== "system").length, + [messages], + ); + + const totalTokens = useMemo(() => { + if (contextUsage) return contextUsage.total.tokens; + // fallback before first send: estimate from context chars + return Math.ceil(contextChars / 3.5); + }, [contextUsage, contextChars]); + + const usagePercent = useMemo(() => { + if (contextUsage) return contextUsage.total.percent; + return 0; + }, [contextUsage]); + + const lastResponseUsage = useMemo( + () => getLastResponseUsage(messages), + [messages], + ); + + return { + contextUsage, + totalTokens, + usagePercent, + contextChars, + toolCount, + messageCount, + lastResponseUsage, + }; +} diff --git a/packages/copilot-sdk/src/react/index.ts b/packages/copilot-sdk/src/react/index.ts index 8fb8231..6c08f55 100644 --- a/packages/copilot-sdk/src/react/index.ts +++ b/packages/copilot-sdk/src/react/index.ts @@ -240,3 +240,13 @@ export type { UseMessageHistoryReturn, MessageHistoryContextValue, } from "./message-history"; + +// Context Stats Hook +export { + useContextStats, + type ContextStats, + type MessageTokenUsage, +} from "./hooks/useContextStats"; + +// Re-export ContextUsage for useContextStats consumers +export type { ContextUsage, ContextUsagePart } from "../core"; diff --git a/packages/copilot-sdk/src/react/provider/CopilotProvider.tsx b/packages/copilot-sdk/src/react/provider/CopilotProvider.tsx index fbf77e9..ed51868 100644 --- a/packages/copilot-sdk/src/react/provider/CopilotProvider.tsx +++ b/packages/copilot-sdk/src/react/provider/CopilotProvider.tsx @@ -26,6 +26,7 @@ import type { MessageAttachment, PermissionLevel, ToolOptimizationConfig, + ContextUsage, } from "../../core"; import type { MCPServerConfig } from "../../mcp/types"; @@ -346,6 +347,16 @@ export interface CopilotContextValue { // System Prompt setSystemPrompt: (prompt: string) => void; + // Context stats (reactive — updates when useAIContext adds/removes context) + /** Total characters currently registered in the AI context tree (system prompt contribution). */ + contextChars: number; + /** + * Live prompt context usage snapshot — updated on every message send. + * Includes token counts and percentages for systemPrompt, history, toolResults, tools buckets. + * null until the first message is sent. + */ + contextUsage: ContextUsage | null; + // Config threadId?: string; /** @@ -467,6 +478,9 @@ export function CopilotProvider({ onApprovalRequired: (execution) => { debugLog("Tool approval required:", execution.name); }, + onContextUsageChange: (usage) => { + setContextUsage(usage); + }, onError: (error) => { if (error) onError?.(error); }, @@ -599,6 +613,8 @@ export function CopilotProvider({ const contextTreeRef = useRef([]); const contextIdCounter = useRef(0); + const [contextChars, setContextChars] = useState(0); + const [contextUsage, setContextUsage] = useState(null); const addContext = useCallback( (context: string, parentId?: string): string => { @@ -611,6 +627,7 @@ export function CopilotProvider({ // Update chat's context const contextString = printTree(contextTreeRef.current); chatRef.current?.setContext(contextString); + setContextChars(contextString.length); debugLog("Context added:", id); return id; }, @@ -623,6 +640,7 @@ export function CopilotProvider({ // Update chat's context const contextString = printTree(contextTreeRef.current); chatRef.current?.setContext(contextString); + setContextChars(contextString.length); debugLog("Context removed:", id); }, [debugLog], @@ -740,6 +758,8 @@ export function CopilotProvider({ // AI Context addContext, removeContext, + contextChars, + contextUsage, // System Prompt setSystemPrompt, @@ -771,6 +791,8 @@ export function CopilotProvider({ registeredActions, addContext, removeContext, + contextChars, + contextUsage, setSystemPrompt, threadId, runtimeUrl, From c21fb42c87068cbd5aa9bdc441eaebb6674e004e Mon Sep 17 00:00:00 2001 From: Sahil Date: Sat, 14 Mar 2026 19:40:50 +0530 Subject: [PATCH 14/72] feat(sdk): implement conversation branching (Phases 1-3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - MessageTree: bidirectional flat-map tree (parentId + childrenIds[] + activeChildMap) - ReactChatState: backed by MessageTree; messages getter = visible path; getAllMessages() = all branches - AbstractChat: branch-aware regenerate() (setCurrentLeaf instead of slice); sendMessage({editMessageId}) fork support - ChatWithTools + ReactChat + ReactChatWithTools: switchBranch/getBranchInfo/editMessage/hasBranches pass-throughs - useChat/CopilotProvider/CopilotContext: branching methods wired into context - BranchNavigator: ← N/M → presentational component - default-message: inline edit (pencil on hover, textarea, submit), BranchNavigator below user bubbles - connected-chat: switchBranch/getBranchInfo/editMessage wired from useCopilot to Chat - BranchInfo/MessageTree exported from chat/index + react/index (framework-agnostic) - All new fields optional — zero breaking changes for linear conversations Co-Authored-By: Claude Sonnet 4.6 --- .../copilot-sdk/src/chat/ChatWithTools.ts | 6 +- .../src/chat/branching/MessageTree.ts | 384 ++++++++++++++++++ .../copilot-sdk/src/chat/branching/index.ts | 1 + .../src/chat/classes/AbstractChat.ts | 74 +++- .../chat/functions/message/createMessage.ts | 14 +- packages/copilot-sdk/src/chat/index.ts | 6 + .../src/chat/interfaces/ChatState.ts | 34 ++ .../copilot-sdk/src/chat/types/message.ts | 9 + .../copilot-sdk/src/core/types/message.ts | 15 + .../src/react/context/CopilotContext.tsx | 19 +- packages/copilot-sdk/src/react/index.ts | 3 + .../src/react/internal/ReactChat.ts | 34 ++ .../src/react/internal/ReactChatState.ts | 123 ++++-- .../src/react/internal/ReactChatWithTools.ts | 33 ++ .../copilot-sdk/src/react/internal/useChat.ts | 67 ++- .../src/react/provider/CopilotProvider.tsx | 40 ++ .../src/ui/components/composed/chat/chat.tsx | 11 + .../composed/chat/default-message.tsx | 198 +++++++-- .../src/ui/components/composed/chat/types.ts | 28 ++ .../ui/components/composed/connected-chat.tsx | 7 + .../src/ui/components/ui/branch-navigator.tsx | 125 ++++++ packages/copilot-sdk/src/ui/index.ts | 6 + 22 files changed, 1165 insertions(+), 72 deletions(-) create mode 100644 packages/copilot-sdk/src/chat/branching/MessageTree.ts create mode 100644 packages/copilot-sdk/src/chat/branching/index.ts create mode 100644 packages/copilot-sdk/src/ui/components/ui/branch-navigator.tsx diff --git a/packages/copilot-sdk/src/chat/ChatWithTools.ts b/packages/copilot-sdk/src/chat/ChatWithTools.ts index 772bdcd..6f28165 100644 --- a/packages/copilot-sdk/src/chat/ChatWithTools.ts +++ b/packages/copilot-sdk/src/chat/ChatWithTools.ts @@ -375,10 +375,14 @@ export class ChatWithTools { /** * Send a message * Returns false if a request is already in progress + * + * @param options.editMessageId - Edit flow: new message branches from the + * same parent as this message ID */ async sendMessage( content: string, attachments?: MessageAttachment[], + options?: { editMessageId?: string }, ): Promise { // Guard: Don't send if already processing if (this.isLoading) { @@ -388,7 +392,7 @@ export class ChatWithTools { // Reset iteration counter so user can continue after max iterations this.agentLoop.resetIterations(); - return await this.chat.sendMessage(content, attachments); + return await this.chat.sendMessage(content, attachments, options); } /** diff --git a/packages/copilot-sdk/src/chat/branching/MessageTree.ts b/packages/copilot-sdk/src/chat/branching/MessageTree.ts new file mode 100644 index 0000000..634affa --- /dev/null +++ b/packages/copilot-sdk/src/chat/branching/MessageTree.ts @@ -0,0 +1,384 @@ +/** + * MessageTree — Bidirectional flat-map message tree for conversation branching. + * + * Industry-standard data structure used by ChatGPT, Claude.ai, and Gemini: + * - parentId + childrenIds[] for O(1) navigation + * - activeChildMap tracks the active path through the tree + * + * Zero React dependency — pure TypeScript, works in any environment. + */ + +import type { UIMessage } from "../types/message"; + +// ============================================ +// Types +// ============================================ + +/** + * Branch navigation info for the UI navigator (← N/M →) + */ +export interface BranchInfo { + /** 0-based index of this message among its siblings */ + siblingIndex: number; + /** Total number of sibling variants at this fork */ + totalSiblings: number; + /** Ordered IDs of all siblings (oldest-first) */ + siblingIds: string[]; + hasPrevious: boolean; + hasNext: boolean; +} + +// ============================================ +// MessageTree +// ============================================ + +export class MessageTree { + /** All messages by ID */ + private nodeMap: Map = new Map(); + /** parentKey → ordered list of child IDs (insertion order = oldest-first) */ + private childrenOf: Map = new Map(); + /** parentKey → currently-active child ID */ + private activeChildMap: Map = new Map(); + /** Current leaf message ID (tip of the active path) */ + private _currentLeafId: string | null = null; + + /** Sentinel key used for root-level messages (parentId === null) */ + static readonly ROOT_KEY = "__root__"; + + constructor(messages?: T[]) { + if (messages?.length) { + this._buildFromMessages(messages); + } + } + + // ============================================ + // Static Migration Helpers + // ============================================ + + /** + * Convert a legacy flat array (no parentId) to a tree-linked array. + * + * Rules: + * - Tool messages get parentId = the owning assistant message's id + * (matched via toolCallId → toolCall.id). + * - All other messages get parentId of the previous non-tool message + * (or null for the first message). + * + * Returns a new array with parentId/childrenIds filled in. + * Does NOT mutate the original messages. + */ + static fromFlatArray(messages: T[]): T[] { + if (messages.length === 0) return messages; + + // If already tree-linked (any message has parentId defined), return as-is + const alreadyLinked = messages.some((m) => m.parentId !== undefined); + if (alreadyLinked) return messages; + + const result: T[] = []; + // Track linear parent chain (skip tool messages for parent tracking) + let prevNonToolId: string | null = null; + + // Build assistant id → assistant message map for tool pairing + const assistantById = new Map(); + for (const msg of messages) { + if (msg.role === "assistant") { + assistantById.set(msg.id, msg); + } + } + + for (const msg of messages) { + if (msg.role === "tool" && msg.toolCallId) { + // Find owning assistant message by matching toolCallId → toolCall.id + let ownerAssistantId: string | null = null; + for (const [, assistant] of assistantById) { + if (assistant.toolCalls?.some((tc) => tc.id === msg.toolCallId)) { + ownerAssistantId = assistant.id; + break; + } + } + result.push({ + ...msg, + parentId: ownerAssistantId ?? prevNonToolId, + childrenIds: [], + }); + } else { + result.push({ + ...msg, + parentId: prevNonToolId, + childrenIds: [], + }); + prevNonToolId = msg.id; + } + } + + // Second pass: fill in childrenIds based on parentId assignments + const childrenMap = new Map(); + for (const msg of result) { + const parentKey = + msg.parentId == null ? MessageTree.ROOT_KEY : msg.parentId; + if (!childrenMap.has(parentKey)) { + childrenMap.set(parentKey, []); + } + childrenMap.get(parentKey)!.push(msg.id); + } + + return result.map((msg) => ({ + ...msg, + childrenIds: childrenMap.get(msg.id) ?? [], + })); + } + + // ============================================ + // Core Queries + // ============================================ + + /** + * Returns the visible path (root → current leaf) — what the UI renders + * and what gets sent to the API. + * + * Backward-compat: if NO message has parentId set (all undefined), + * falls back to insertion order (legacy linear mode). + */ + getVisibleMessages(): T[] { + if (this.nodeMap.size === 0) return []; + + // Legacy linear fallback: no parentId on any message + const hasTreeStructure = Array.from(this.nodeMap.values()).some( + (m) => m.parentId !== undefined, + ); + if (!hasTreeStructure) { + return Array.from(this.nodeMap.values()); + } + + return this._getActivePath().map((id) => this.nodeMap.get(id)!); + } + + /** + * Returns ALL messages across every branch (for persistence / ThreadManager). + */ + getAllMessages(): T[] { + return Array.from(this.nodeMap.values()); + } + + /** + * Branch navigation info for the UI navigator. + * Returns null if the message has no siblings (only child). + */ + getBranchInfo(messageId: string): BranchInfo | null { + const msg = this.nodeMap.get(messageId); + if (!msg) return null; + + const parentKey = this._parentKey(msg.parentId); + const siblings = this.childrenOf.get(parentKey) ?? []; + + if (siblings.length <= 1) return null; + + const siblingIndex = siblings.indexOf(messageId); + return { + siblingIndex, + totalSiblings: siblings.length, + siblingIds: [...siblings], + hasPrevious: siblingIndex > 0, + hasNext: siblingIndex < siblings.length - 1, + }; + } + + get currentLeafId(): string | null { + return this._currentLeafId; + } + + get hasBranches(): boolean { + for (const children of this.childrenOf.values()) { + if (children.length > 1) return true; + } + return false; + } + + // ============================================ + // Mutations + // ============================================ + + /** + * Insert a new message. + * - Updates childrenOf and nodeMap. + * - New branch becomes active (activeChildMap updated). + * - Updates current leaf. + */ + addMessage(message: T): T { + this.nodeMap.set(message.id, message); + + const parentKey = this._parentKey(message.parentId); + if (!this.childrenOf.has(parentKey)) { + this.childrenOf.set(parentKey, []); + } + const siblings = this.childrenOf.get(parentKey)!; + if (!siblings.includes(message.id)) { + siblings.push(message.id); + } + + // New message becomes active at its parent fork + this.activeChildMap.set(parentKey, message.id); + + // Update current leaf (walk forward from this message) + this._currentLeafId = this._walkToLeaf(message.id); + + return message; + } + + /** + * Navigate: make messageId the active child at its parent fork, + * then walk to its leaf and update currentLeafId. + */ + switchBranch(messageId: string): void { + const msg = this.nodeMap.get(messageId); + if (!msg) return; + + const parentKey = this._parentKey(msg.parentId); + this.activeChildMap.set(parentKey, messageId); + this._currentLeafId = this._walkToLeaf(messageId); + } + + /** + * Update message content in-place (streaming updates). + * No tree structure change. + */ + updateMessage(id: string, updater: (msg: T) => T): boolean { + const existing = this.nodeMap.get(id); + if (!existing) return false; + this.nodeMap.set(id, updater(existing)); + return true; + } + + /** + * Set current leaf explicitly. + * Used by regenerate() to rewind the active path before pushing a new message. + */ + setCurrentLeaf(leafId: string | null): void { + this._currentLeafId = leafId; + + if (leafId === null) return; + + // Ensure the active path points to this leaf + const msg = this.nodeMap.get(leafId); + if (!msg) return; + + // Walk up and set activeChildMap entries so getVisibleMessages() is consistent + let current: T | undefined = msg; + while (current) { + const parentKey = this._parentKey(current.parentId); + this.activeChildMap.set(parentKey, current.id); + if (current.parentId == null || current.parentId === undefined) break; + current = this.nodeMap.get(current.parentId); + } + } + + /** + * Rebuild entire tree from a message array. + * Used by setMessages(). + */ + reset(messages: T[]): void { + this.nodeMap.clear(); + this.childrenOf.clear(); + this.activeChildMap.clear(); + this._currentLeafId = null; + + if (messages.length > 0) { + this._buildFromMessages(messages); + } + } + + // ============================================ + // Private Helpers + // ============================================ + + private _buildFromMessages(messages: T[]): void { + // Auto-migrate legacy flat arrays + const linked = messages.some((m) => m.parentId !== undefined) + ? messages + : MessageTree.fromFlatArray(messages); + + for (const msg of linked) { + this.nodeMap.set(msg.id, msg); + + const parentKey = this._parentKey(msg.parentId); + if (!this.childrenOf.has(parentKey)) { + this.childrenOf.set(parentKey, []); + } + const siblings = this.childrenOf.get(parentKey)!; + if (!siblings.includes(msg.id)) { + siblings.push(msg.id); + } + } + + // Build activeChildMap: default to last child at each fork + // (last child = most recently added = what was active when saved) + for (const [parentKey, children] of this.childrenOf) { + if (children.length > 0) { + this.activeChildMap.set(parentKey, children[children.length - 1]); + } + } + + // Set current leaf by walking the active path from root + const path = this._getActivePath(); + this._currentLeafId = path.length > 0 ? path[path.length - 1] : null; + } + + private _parentKey(parentId: string | null | undefined): string { + if (parentId == null || parentId === undefined) { + return MessageTree.ROOT_KEY; + } + return parentId; + } + + /** + * Walk forward from a message along active children to find the leaf. + */ + private _walkToLeaf(fromId: string): string { + let current = fromId; + // eslint-disable-next-line no-constant-condition + while (true) { + const children = this.childrenOf.get(current); + if (!children || children.length === 0) break; + const activeChild = this.activeChildMap.get(current); + if (!activeChild) break; + if (!this.nodeMap.has(activeChild)) break; + current = activeChild; + } + return current; + } + + /** + * Walk the active path from root to the current leaf. + */ + private _getActivePath(): string[] { + const path: string[] = []; + const visited = new Set(); + + // Start from root children + const rootChildren = this.childrenOf.get(MessageTree.ROOT_KEY) ?? []; + if (rootChildren.length === 0) return path; + + // Pick active root child + let activeId = this.activeChildMap.get(MessageTree.ROOT_KEY); + if (!activeId) { + // Fall back to last root child + activeId = rootChildren[rootChildren.length - 1]; + } + + // Walk forward along active children + let current: string | undefined = activeId; + while (current && !visited.has(current)) { + if (!this.nodeMap.has(current)) break; + visited.add(current); + path.push(current); + + // Check if this message has an override active child set + // (used when setCurrentLeaf rewinds the active path) + const activeChild = this.activeChildMap.get(current); + if (!activeChild || !this.nodeMap.has(activeChild)) break; + current = activeChild; + } + + return path; + } +} diff --git a/packages/copilot-sdk/src/chat/branching/index.ts b/packages/copilot-sdk/src/chat/branching/index.ts new file mode 100644 index 0000000..12bd694 --- /dev/null +++ b/packages/copilot-sdk/src/chat/branching/index.ts @@ -0,0 +1 @@ +export { MessageTree, type BranchInfo } from "./MessageTree"; diff --git a/packages/copilot-sdk/src/chat/classes/AbstractChat.ts b/packages/copilot-sdk/src/chat/classes/AbstractChat.ts index 6bcd59f..98c13a4 100644 --- a/packages/copilot-sdk/src/chat/classes/AbstractChat.ts +++ b/packages/copilot-sdk/src/chat/classes/AbstractChat.ts @@ -170,10 +170,19 @@ export class AbstractChat { /** * Send a message * Returns false if a request is already in progress + * + * @param content - Message content + * @param attachments - Optional attachments + * @param options - Optional branching options + * @param options.editMessageId - Edit flow: new message branches from the + * same parent as this message ID, creating a parallel conversation path */ async sendMessage( content: string, attachments?: MessageAttachment[], + options?: { + editMessageId?: string; + }, ): Promise { // Guard: Don't send if already processing if (this.isBusy) { @@ -181,15 +190,34 @@ export class AbstractChat { return false; } - this.debug("sendMessage", { content, attachments }); + this.debug("sendMessage", { content, attachments, options }); try { // IMPORTANT: Resolve any pending tool_calls before sending // This prevents Anthropic API errors: "tool_use without tool_result" this.resolveUnresolvedToolCalls(); - // Create user message - const userMessage = createUserMessage(content, attachments) as T; + // Edit flow: branch from the same parent as the edited message + let newParentId: string | null | undefined; + if (options?.editMessageId && this.state.setCurrentLeaf) { + const allMessages = + this.state.getAllMessages?.() ?? this.state.messages; + const target = allMessages.find( + (m) => m.id === options.editMessageId, + ); + if (target && target.parentId !== undefined) { + newParentId = target.parentId; + // Rewind active path to just before the original message + this.state.setCurrentLeaf( + typeof target.parentId === "string" ? target.parentId : null, + ); + } + } + + // Create user message (with optional parentId for branching) + const userMessage = createUserMessage(content, attachments, { + parentId: newParentId, + }) as T; // Add to state this.state.pushMessage(userMessage); @@ -385,31 +413,51 @@ export class AbstractChat { } /** - * Regenerate last response + * Regenerate last response. + * + * Branch-aware: when the state supports branching (setCurrentLeaf is available), + * regenerate creates a new sibling response instead of destroying the original. + * The old response is preserved and navigable via switchBranch(). + * + * Legacy fallback: when branching is not available, uses old slice() behavior. */ async regenerate(messageId?: string): Promise { - // Remove messages from the specified ID (or last assistant message) - const messages = this.state.messages; - let targetIndex = messages.length - 1; + if (this.isBusy) return; + + const messages = this.state.messages; // visible path + let targetMessage: T | undefined; if (messageId) { - targetIndex = messages.findIndex((m) => m.id === messageId); + targetMessage = messages.find((m) => m.id === messageId); } else { - // Find last assistant message + // Find last assistant message in the visible path for (let i = messages.length - 1; i >= 0; i--) { if (messages[i].role === "assistant") { - targetIndex = i; + targetMessage = messages[i]; break; } } } + if (!targetMessage) return; + + // Branch-aware regenerate: preserve old response as inactive sibling + if (targetMessage.parentId !== undefined && this.state.setCurrentLeaf) { + // Rewind active path to target's parent + // The new assistant response will be pushed as a new child (sibling) + this.state.setCurrentLeaf(targetMessage.parentId ?? null); + this.callbacks.onMessagesChange?.(this.state.messages); + this.state.status = "submitted"; + await Promise.resolve(); + await this.processRequest(); + return; + } + + // Legacy fallback: old slice() behavior for non-tree-aware state + const targetIndex = messages.indexOf(targetMessage); if (targetIndex > 0) { - // Remove from target onwards this.state.setMessages(messages.slice(0, targetIndex)); this.callbacks.onMessagesChange?.(this.state.messages); - - // Resend await this.processRequest(); } } diff --git a/packages/copilot-sdk/src/chat/functions/message/createMessage.ts b/packages/copilot-sdk/src/chat/functions/message/createMessage.ts index e198e16..0cdab2a 100644 --- a/packages/copilot-sdk/src/chat/functions/message/createMessage.ts +++ b/packages/copilot-sdk/src/chat/functions/message/createMessage.ts @@ -19,11 +19,15 @@ export function generateMessageId(): string { * * @param content - Message content * @param attachments - Optional attachments + * @param options - Optional branching options * @returns New user message */ export function createUserMessage( content: string, attachments?: MessageAttachment[], + options?: { + parentId?: string | null; + }, ): UIMessage { return { id: generateMessageId(), @@ -31,6 +35,7 @@ export function createUserMessage( content, attachments, createdAt: new Date(), + ...(options?.parentId !== undefined ? { parentId: options.parentId } : {}), }; } @@ -156,13 +161,20 @@ export function streamStateToMessage(state: StreamingMessageState): UIMessage { * Create an empty assistant message (for streaming) * * @param id - Optional message ID + * @param options - Optional branching options * @returns Empty assistant message */ -export function createEmptyAssistantMessage(id?: string): UIMessage { +export function createEmptyAssistantMessage( + id?: string, + options?: { + parentId?: string | null; + }, +): UIMessage { return { id: id ?? generateMessageId(), role: "assistant", content: "", createdAt: new Date(), + ...(options?.parentId !== undefined ? { parentId: options.parentId } : {}), }; } diff --git a/packages/copilot-sdk/src/chat/index.ts b/packages/copilot-sdk/src/chat/index.ts index eddc84b..13c4805 100644 --- a/packages/copilot-sdk/src/chat/index.ts +++ b/packages/copilot-sdk/src/chat/index.ts @@ -85,6 +85,12 @@ export { type ChatWithToolsCallbacks, } from "./ChatWithTools"; +// ============================================ +// Branching +// ============================================ + +export { MessageTree, type BranchInfo } from "./branching"; + // ============================================ // Pure Functions // ============================================ diff --git a/packages/copilot-sdk/src/chat/interfaces/ChatState.ts b/packages/copilot-sdk/src/chat/interfaces/ChatState.ts index 9693a00..577c927 100644 --- a/packages/copilot-sdk/src/chat/interfaces/ChatState.ts +++ b/packages/copilot-sdk/src/chat/interfaces/ChatState.ts @@ -6,6 +6,7 @@ */ import type { UIMessage, ChatStatus } from "../types/index"; +import type { BranchInfo } from "../branching"; /** * ChatState interface - Framework adapters implement this @@ -128,6 +129,39 @@ export interface ChatState { * Get error snapshot */ getErrorSnapshot?(): Error | undefined; + + // ============================================ + // Branching Extensions (optional — only ReactChatState implements these) + // ============================================ + + /** + * Set the current leaf message ID. + * Used by regenerate() to rewind the active path before pushing a new response. + */ + setCurrentLeaf?(leafId: string | null): void; + + /** + * Get all messages across all branches (for persistence). + * The base messages getter returns only the visible path. + */ + getAllMessages?(): T[]; + + /** + * Get branch navigation info for a message. + * Returns null if the message has no siblings. + */ + getBranchInfo?(messageId: string): BranchInfo | null; + + /** + * Navigate to a specific message variant (sibling branch). + * Updates the active path to go through messageId. + */ + switchBranch?(messageId: string): void; + + /** + * Whether any message has siblings (branching has occurred). + */ + readonly hasBranches?: boolean; } /** diff --git a/packages/copilot-sdk/src/chat/types/message.ts b/packages/copilot-sdk/src/chat/types/message.ts index 514b141..ac5fcbb 100644 --- a/packages/copilot-sdk/src/chat/types/message.ts +++ b/packages/copilot-sdk/src/chat/types/message.ts @@ -40,6 +40,15 @@ export interface UIMessage { createdAt: Date; /** Additional metadata */ metadata?: Record; + /** + * Parent message ID for branching support. + * - null = root message (no parent) + * - undefined = legacy linear message (no branch awareness) + * - string = ID of parent message + */ + parentId?: string | null; + /** Direct children IDs for O(1) sibling lookup */ + childrenIds?: string[]; } /** diff --git a/packages/copilot-sdk/src/core/types/message.ts b/packages/copilot-sdk/src/core/types/message.ts index 8cb9c46..ec46b99 100644 --- a/packages/copilot-sdk/src/core/types/message.ts +++ b/packages/copilot-sdk/src/core/types/message.ts @@ -146,6 +146,17 @@ export interface Message { /** When the message was created */ created_at: Date; + + /** + * Parent message ID for branching support. + * - null = root message (no parent) + * - undefined = legacy linear message (no branch awareness) + * - string = ID of parent message + */ + parent_id?: string | null; + + /** Direct children IDs for O(1) sibling lookup */ + children_ids?: string[]; } /** @@ -195,6 +206,8 @@ export function createMessage( tool_call_id: partial.tool_call_id, metadata: partial.metadata, created_at: partial.created_at ?? new Date(), + ...(partial.parent_id !== undefined ? { parent_id: partial.parent_id } : {}), + ...(partial.children_ids !== undefined ? { children_ids: partial.children_ids } : {}), }; } @@ -207,6 +220,7 @@ export function createUserMessage( id?: string; thread_id?: string; attachments?: MessageAttachment[]; + parent_id?: string | null; }, ): Message { return createMessage({ @@ -217,6 +231,7 @@ export function createUserMessage( metadata: options?.attachments ? { attachments: options.attachments } : undefined, + ...(options?.parent_id !== undefined ? { parent_id: options.parent_id } : {}), }); } diff --git a/packages/copilot-sdk/src/react/context/CopilotContext.tsx b/packages/copilot-sdk/src/react/context/CopilotContext.tsx index 52c26d3..62255e6 100644 --- a/packages/copilot-sdk/src/react/context/CopilotContext.tsx +++ b/packages/copilot-sdk/src/react/context/CopilotContext.tsx @@ -19,6 +19,7 @@ import type { ToolPermission, } from "../../core"; import type { ContextTreeNode } from "../utils/context-tree"; +import type { BranchInfo } from "../../chat/branching"; /** * Chat UI state interface (UI-only state, not message data) @@ -96,7 +97,7 @@ export interface ChatActions { stopGeneration: () => void; /** Clear all messages */ clearMessages: () => void; - /** Regenerate last response */ + /** Regenerate last response (branch-aware: preserves original as sibling) */ regenerate: (messageId?: string) => Promise; /** Set messages directly */ setMessages: (messages: Message[]) => void; @@ -106,6 +107,22 @@ export interface ChatActions { * - Free: converts to base64 */ processAttachment: (file: File) => Promise; + + // ============================================ + // Branching Actions + // ============================================ + + /** Navigate to a sibling branch (← / → navigation) */ + switchBranch: (messageId: string) => void; + /** Get branch navigation info for a message */ + getBranchInfo: (messageId: string) => BranchInfo | null; + /** + * Edit a user message: sends newContent as a new branch from the same + * parent as the original message. Preserves the original in place. + */ + editMessage: (messageId: string, newContent: string) => Promise; + /** Whether any message has siblings (branching has occurred) */ + hasBranches: boolean; } /** diff --git a/packages/copilot-sdk/src/react/index.ts b/packages/copilot-sdk/src/react/index.ts index 6c08f55..935b6dd 100644 --- a/packages/copilot-sdk/src/react/index.ts +++ b/packages/copilot-sdk/src/react/index.ts @@ -250,3 +250,6 @@ export { // Re-export ContextUsage for useContextStats consumers export type { ContextUsage, ContextUsagePart } from "../core"; + +// Branching +export { MessageTree, type BranchInfo } from "../chat/branching"; diff --git a/packages/copilot-sdk/src/react/internal/ReactChat.ts b/packages/copilot-sdk/src/react/internal/ReactChat.ts index cb4d486..803e2c3 100644 --- a/packages/copilot-sdk/src/react/internal/ReactChat.ts +++ b/packages/copilot-sdk/src/react/internal/ReactChat.ts @@ -16,6 +16,7 @@ import { type ChatEventHandler, } from "../../chat"; import { ReactChatState } from "./ReactChatState"; +import type { BranchInfo } from "../../chat/branching"; /** * Chat status for UI state @@ -132,6 +133,39 @@ export class ReactChat extends AbstractChat { return this.on("error", handler); } + // ============================================ + // Branching API — pass-throughs to ReactChatState + // ============================================ + + /** + * Navigate to a sibling branch (makes it the active path). + */ + switchBranch(messageId: string): void { + this.reactState.switchBranch(messageId); + } + + /** + * Get branch navigation info for a message. + * Returns null if the message has no siblings. + */ + getBranchInfo(messageId: string): BranchInfo | null { + return this.reactState.getBranchInfo(messageId); + } + + /** + * Get all messages across all branches (for persistence). + */ + getAllMessages(): UIMessage[] { + return this.reactState.getAllMessages(); + } + + /** + * Whether any message has siblings (branching has occurred). + */ + get hasBranches(): boolean { + return this.reactState.hasBranches; + } + // ============================================ // Override dispose to clean up state // ============================================ diff --git a/packages/copilot-sdk/src/react/internal/ReactChatState.ts b/packages/copilot-sdk/src/react/internal/ReactChatState.ts index 31554ae..df4090e 100644 --- a/packages/copilot-sdk/src/react/internal/ReactChatState.ts +++ b/packages/copilot-sdk/src/react/internal/ReactChatState.ts @@ -1,16 +1,19 @@ /** * ReactChatState - React-specific implementation of ChatState * - * This class implements the ChatState interface with callback-based - * reactivity for use with React's useSyncExternalStore. + * Backed by MessageTree for conversation branching support. + * The `messages` getter returns only the visible path (active branch). + * Use `getAllMessages()` for full persistence. * * Pattern inspired by Vercel AI SDK's useSyncExternalStore pattern. */ import type { ChatState, UIMessage, ChatStatus } from "../../chat"; +import { MessageTree, type BranchInfo } from "../../chat/branching"; /** * ReactChatState implements ChatState with callback-based reactivity + * and full conversation branching support via MessageTree. * * @example * ```tsx @@ -21,14 +24,17 @@ import type { ChatState, UIMessage, ChatStatus } from "../../chat"; * console.log('State changed'); * }); * - * // Get snapshot (for useSyncExternalStore) + * // Get visible path (active branch only) * const messages = state.messages; + * + * // Get all messages across branches (for persistence) + * const all = state.getAllMessages(); * ``` */ export class ReactChatState< T extends UIMessage = UIMessage, > implements ChatState { - private _messages: T[] = []; + private tree: MessageTree; private _status: ChatStatus = "ready"; private _error: Error | undefined = undefined; @@ -36,17 +42,21 @@ export class ReactChatState< private subscribers = new Set<() => void>(); constructor(initialMessages?: T[]) { - if (initialMessages) { - this._messages = initialMessages; - } + this.tree = new MessageTree(initialMessages); } // ============================================ - // Getters + // Getters — visible path only // ============================================ + /** + * Returns the VISIBLE PATH (active branch) — what the UI renders + * and what gets sent to the API. + * + * For all messages across all branches, use getAllMessages(). + */ get messages(): T[] { - return this._messages; + return this.tree.getVisibleMessages(); } get status(): ChatStatus { @@ -62,7 +72,7 @@ export class ReactChatState< // ============================================ set messages(value: T[]) { - this._messages = value; + this.tree.reset(value); this.notify(); } @@ -81,53 +91,104 @@ export class ReactChatState< // ============================================ pushMessage(message: T): void { - this._messages = [...this._messages, message]; + this.tree.addMessage(message); this.notify(); } popMessage(): void { - this._messages = this._messages.slice(0, -1); + // Remove current leaf from tree + const leafId = this.tree.currentLeafId; + if (!leafId) return; + + const allMessages = this.tree.getAllMessages().filter((m) => m.id !== leafId); + // Walk up to the parent to set it as new leaf + const leaf = this.tree.getAllMessages().find((m) => m.id === leafId); + const newLeafId = + leaf && leaf.parentId !== undefined && leaf.parentId !== null + ? leaf.parentId + : null; + + this.tree.reset(allMessages); + if (newLeafId) { + this.tree.setCurrentLeaf(newLeafId); + } this.notify(); } replaceMessage(index: number, message: T): void { - this._messages = this._messages.map((m, i) => (i === index ? message : m)); + // replaceMessage operates on the visible path + const visible = this.tree.getVisibleMessages(); + const target = visible[index]; + if (!target) return; + this.tree.updateMessage(target.id, () => message); this.notify(); } updateLastMessage(updater: (message: T) => T): void { - if (this._messages.length === 0) return; - - const lastIndex = this._messages.length - 1; - const lastMessage = this._messages[lastIndex]; - this._messages = [ - ...this._messages.slice(0, lastIndex), - updater(lastMessage), - ]; + const leafId = this.tree.currentLeafId; + if (!leafId) return; + this.tree.updateMessage(leafId, updater); this.notify(); } updateMessageById(id: string, updater: (message: T) => T): boolean { - const index = this._messages.findIndex((m) => m.id === id); - if (index === -1) return false; - - this._messages = this._messages.map((m, i) => - i === index ? updater(m) : m, - ); - this.notify(); - return true; + const updated = this.tree.updateMessage(id, updater); + if (updated) this.notify(); + return updated; } setMessages(messages: T[]): void { - this._messages = messages; + this.tree.reset(messages); this.notify(); } clearMessages(): void { - this._messages = []; + this.tree.reset([]); this.notify(); } + // ============================================ + // Branching API + // ============================================ + + /** + * Returns ALL messages across all branches. + * Use this for persistence (ThreadManager save). + */ + getAllMessages(): T[] { + return this.tree.getAllMessages(); + } + + /** + * Get branch navigation info for a message. + * Returns null if the message has no siblings. + */ + getBranchInfo(messageId: string): BranchInfo | null { + return this.tree.getBranchInfo(messageId); + } + + /** + * Navigate to a sibling branch. + * Triggers re-render via notify(). + */ + switchBranch(messageId: string): void { + this.tree.switchBranch(messageId); + this.notify(); + } + + /** + * Set the current leaf (used by regenerate() to rewind active path). + * Triggers re-render via notify(). + */ + setCurrentLeaf(leafId: string | null): void { + this.tree.setCurrentLeaf(leafId); + this.notify(); + } + + get hasBranches(): boolean { + return this.tree.hasBranches; + } + // ============================================ // Subscription (for useSyncExternalStore) // ============================================ diff --git a/packages/copilot-sdk/src/react/internal/ReactChatWithTools.ts b/packages/copilot-sdk/src/react/internal/ReactChatWithTools.ts index 7773fa5..8e382b8 100644 --- a/packages/copilot-sdk/src/react/internal/ReactChatWithTools.ts +++ b/packages/copilot-sdk/src/react/internal/ReactChatWithTools.ts @@ -12,6 +12,7 @@ import { type ToolExecution, } from "../../chat"; import { ReactChatState } from "./ReactChatState"; +import type { BranchInfo } from "../../chat/branching"; /** * React-specific configuration @@ -60,6 +61,38 @@ export class ReactChatWithTools extends ChatWithTools { return this.reactState.subscribe(callback); }; + // ============================================ + // Branching API — pass-throughs to ReactChatState + // ============================================ + + /** + * Navigate to a sibling branch. + */ + switchBranch(messageId: string): void { + this.reactState.switchBranch(messageId); + } + + /** + * Get branch navigation info for a message. + */ + getBranchInfo(messageId: string): BranchInfo | null { + return this.reactState.getBranchInfo(messageId); + } + + /** + * Get all messages across all branches (for persistence). + */ + getAllMessages(): UIMessage[] { + return this.reactState.getAllMessages(); + } + + /** + * Whether any message has siblings (branching has occurred). + */ + get hasBranches(): boolean { + return this.reactState.hasBranches; + } + /** * Dispose and cleanup */ diff --git a/packages/copilot-sdk/src/react/internal/useChat.ts b/packages/copilot-sdk/src/react/internal/useChat.ts index e18a9c3..c64b602 100644 --- a/packages/copilot-sdk/src/react/internal/useChat.ts +++ b/packages/copilot-sdk/src/react/internal/useChat.ts @@ -17,6 +17,7 @@ import { import { ReactChat, createReactChat, type ReactChatConfig } from "./ReactChat"; import type { UIMessage, ChatStatus } from "../../chat"; import type { MessageAttachment } from "../../core"; +import type { BranchInfo } from "../../chat/branching"; /** * Hook configuration @@ -36,7 +37,7 @@ export interface UseChatConfig extends Omit { * Hook return type */ export interface UseChatReturn { - /** All messages */ + /** All messages (visible path — active branch only) */ messages: UIMessage[]; /** Current status */ status: ChatStatus; @@ -59,7 +60,7 @@ export interface UseChatReturn { clearMessages: () => void; /** Set messages directly */ setMessages: (messages: UIMessage[]) => void; - /** Regenerate last response */ + /** Regenerate last response (branch-aware: preserves original as sibling) */ regenerate: (messageId?: string) => Promise; /** Continue with tool results */ continueWithToolResults: ( @@ -67,6 +68,35 @@ export interface UseChatReturn { ) => Promise; /** Reference to the ReactChat instance */ chatRef: React.RefObject; + + // ============================================ + // Branching API + // ============================================ + + /** + * Navigate to a sibling branch (← / → navigation). + * Only populated when chat is branch-aware. + */ + switchBranch?: (messageId: string) => void; + + /** + * Get branch navigation info for a message. + * Returns null if the message has no siblings. + * Only populated when chat is branch-aware. + */ + getBranchInfo?: (messageId: string) => BranchInfo | null; + + /** + * Edit a user message: sends newContent as a new branch from the same + * parent as the original message. Preserves the original message in place. + * Only populated when chat is branch-aware. + */ + editMessage?: (messageId: string, newContent: string) => Promise; + + /** + * Whether any message has siblings (branching has occurred). + */ + hasBranches?: boolean; } /** @@ -140,6 +170,12 @@ export function useChat(config: UseChatConfig): UseChatReturn { () => undefined, // Server snapshot ); + const hasBranches = useSyncExternalStore( + chatRef.current.subscribe, + () => chatRef.current!.hasBranches, + () => false, + ); + // Derived state const isLoading = status === "streaming" || status === "submitted"; @@ -175,6 +211,28 @@ export function useChat(config: UseChatConfig): UseChatReturn { [], ); + // Branching actions + const switchBranch = useCallback((messageId: string) => { + chatRef.current?.switchBranch(messageId); + }, []); + + const getBranchInfo = useCallback( + (messageId: string): BranchInfo | null => { + return chatRef.current?.getBranchInfo(messageId) ?? null; + }, + [], + ); + + const editMessage = useCallback( + async (messageId: string, newContent: string) => { + await chatRef.current?.sendMessage(newContent, undefined, { + editMessageId: messageId, + }); + setInput(""); + }, + [], + ); + // Cleanup on unmount useEffect(() => { return () => { @@ -196,5 +254,10 @@ export function useChat(config: UseChatConfig): UseChatReturn { regenerate, continueWithToolResults, chatRef, + // Branching + switchBranch, + getBranchInfo, + editMessage, + hasBranches, }; } diff --git a/packages/copilot-sdk/src/react/provider/CopilotProvider.tsx b/packages/copilot-sdk/src/react/provider/CopilotProvider.tsx index ed51868..56b0e50 100644 --- a/packages/copilot-sdk/src/react/provider/CopilotProvider.tsx +++ b/packages/copilot-sdk/src/react/provider/CopilotProvider.tsx @@ -318,6 +318,12 @@ export interface CopilotContextValue { setMessages: (messages: UIMessage[]) => void; regenerate: (messageId?: string) => Promise; + // Branching actions + switchBranch: (messageId: string) => void; + getBranchInfo: (messageId: string) => import("../../chat/branching").BranchInfo | null; + editMessage: (messageId: string, newContent: string) => Promise; + hasBranches: boolean; + // Tool execution registerTool: (tool: ToolDefinition) => void; unregisterTool: (name: string) => void; @@ -686,6 +692,30 @@ export function CopilotProvider({ await chatRef.current?.regenerate(messageId); }, []); + const switchBranch = useCallback((messageId: string) => { + chatRef.current?.switchBranch(messageId); + }, []); + + const getBranchInfo = useCallback( + (messageId: string) => chatRef.current?.getBranchInfo(messageId) ?? null, + [], + ); + + const editMessage = useCallback( + async (messageId: string, newContent: string) => { + await chatRef.current?.sendMessage(newContent, undefined, { + editMessageId: messageId, + }); + }, + [], + ); + + const hasBranches = useSyncExternalStore( + chatRef.current.subscribe, + () => chatRef.current!.hasBranches, + () => false, + ); + // ============================================ // Callbacks // ============================================ @@ -741,6 +771,12 @@ export function CopilotProvider({ setMessages, regenerate, + // Branching + switchBranch, + getBranchInfo, + editMessage, + hasBranches, + // Tool execution registerTool, unregisterTool, @@ -779,6 +815,10 @@ export function CopilotProvider({ clearMessages, setMessages, regenerate, + switchBranch, + getBranchInfo, + editMessage, + hasBranches, registerTool, unregisterTool, registeredTools, diff --git a/packages/copilot-sdk/src/ui/components/composed/chat/chat.tsx b/packages/copilot-sdk/src/ui/components/composed/chat/chat.tsx index 11e0219..4b27402 100644 --- a/packages/copilot-sdk/src/ui/components/composed/chat/chat.tsx +++ b/packages/copilot-sdk/src/ui/components/composed/chat/chat.tsx @@ -547,6 +547,10 @@ function ChatComponent({ currentThreadId, onSwitchThread, isThreadBusy, + // Branching + getBranchInfo, + onSwitchBranch, + onEditMessage, }: ChatProps) { // Merge avatar props with defaults (so user can pass partial config) const userAvatar = { fallback: "U", ...userAvatarProp }; @@ -1038,6 +1042,13 @@ function ChatComponent({ citations={ citations === false ? { enabled: false } : citations } + branchInfo={ + message.role === "user" + ? getBranchInfo?.(message.id) ?? null + : null + } + onSwitchBranch={onSwitchBranch} + onEditMessage={onEditMessage} /> ); })} diff --git a/packages/copilot-sdk/src/ui/components/composed/chat/default-message.tsx b/packages/copilot-sdk/src/ui/components/composed/chat/default-message.tsx index f6b0b67..2fc5252 100644 --- a/packages/copilot-sdk/src/ui/components/composed/chat/default-message.tsx +++ b/packages/copilot-sdk/src/ui/components/composed/chat/default-message.tsx @@ -22,6 +22,8 @@ import type { import type { ToolDefinition, ToolRenderProps } from "../../../../core"; import CopilotSDKLogo from "../../icons/copilot-sdk-logo"; import { SourceGroup, type SourceItem } from "../../ui/source"; +import { BranchNavigator } from "../../ui/branch-navigator"; +import type { BranchInfo } from "../../../../chat/branching"; type DefaultMessageProps = { message: ChatMessage; @@ -87,6 +89,26 @@ type DefaultMessageProps = { followUpButtonClassName?: string; /** Citation/Sources configuration */ citations?: CitationConfig; + + // ============================================ + // Branching + // ============================================ + + /** + * Branch navigation info for this message. + * When non-null and totalSiblings > 1, the BranchNavigator is shown. + */ + branchInfo?: BranchInfo | null; + /** + * Called when the user navigates to a sibling branch. + * Receives the message ID to switch to. + */ + onSwitchBranch?: (messageId: string) => void; + /** + * Called when the user submits an edited message. + * Triggers a new branch from the same parent as messageId. + */ + onEditMessage?: (messageId: string, newContent: string) => void; }; export function DefaultMessage({ @@ -112,6 +134,9 @@ export function DefaultMessage({ followUpClassName, followUpButtonClassName, citations = { enabled: true }, + branchInfo, + onSwitchBranch, + onEditMessage, }: DefaultMessageProps) { const isUser = message.role === "user"; const isCompactionMarker = @@ -119,6 +144,47 @@ export function DefaultMessage({ (message.metadata as Record)?.type === "compaction-marker"; const isStreaming = isLastMessage && isLoading; + // Inline-edit state (user messages only) + const [isEditing, setIsEditing] = React.useState(false); + const [editValue, setEditValue] = React.useState(message.content ?? ""); + const editRef = React.useRef(null); + + const startEdit = React.useCallback(() => { + setEditValue(message.content ?? ""); + setIsEditing(true); + // Focus textarea on next frame + requestAnimationFrame(() => editRef.current?.focus()); + }, [message.content]); + + const cancelEdit = React.useCallback(() => { + setIsEditing(false); + }, []); + + const submitEdit = React.useCallback(() => { + const trimmed = editValue.trim(); + if (!trimmed || !onEditMessage) return; + onEditMessage(message.id, trimmed); + setIsEditing(false); + }, [editValue, message.id, onEditMessage]); + + const handleEditKeyDown = React.useCallback( + (e: React.KeyboardEvent) => { + if (e.key === "Enter" && !e.shiftKey) { + e.preventDefault(); + submitEdit(); + } + if (e.key === "Escape") { + cancelEdit(); + } + }, + [submitEdit, cancelEdit], + ); + + // Whether branching UI should be shown for this message + const showBranchNav = + isUser && branchInfo && branchInfo.totalSiblings > 1 && onSwitchBranch; + const showEditBtn = isUser && !!onEditMessage && !isLoading; + // Render compaction marker divider if (isCompactionMarker) { const tokensSaved = (message.metadata as Record) @@ -276,32 +342,118 @@ export function DefaultMessage({ return (
- {/* Text content */} - {message.content && ( - - {message.content} - - )} - {/* Image Attachments */} - {hasAttachments && ( -
- {message.attachments!.map((attachment, index) => ( - - ))} + {/* Edit mode: inline textarea */} + {isEditing ? ( +
+