From ee42d16f75cf8fd24f8fef2f690eeb2e3d17a28f Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Tue, 5 Aug 2025 03:13:44 +0530 Subject: [PATCH 01/25] refactor: clean up code formatting and improve component structure - Standardized code formatting across multiple files, ensuring consistent style and readability. - Enhanced the SearchResultDisplay component with improved styling and layout for better user experience. - Updated the useStream hook to handle tool chunk processing more effectively, allowing for smoother data handling. - Minor adjustments made to prompt creation functions for clarity and consistency in output formatting. --- convex/langchain/index.ts | 42 ++++++++----- convex/langchain/prompts.ts | 60 ++++++++++++------- services/mcps | 2 +- .../tool-message/search-results.tsx | 14 ++--- src/hooks/chats/use-stream.ts | 47 ++++++++++++--- 5 files changed, 110 insertions(+), 55 deletions(-) diff --git a/convex/langchain/index.ts b/convex/langchain/index.ts index 28b04409..9b1fe2dd 100644 --- a/convex/langchain/index.ts +++ b/convex/langchain/index.ts @@ -29,10 +29,10 @@ export const generateTitle = internalAction({ ctx, [ mapStoredMessageToChatMessage( - JSON.parse(args.message.message) as StoredMessage, + JSON.parse(args.message.message) as StoredMessage ), ], - args.chat.model, + args.chat.model ); const model = await getModel(ctx, "worker", undefined, args.chat.userId); const titleSchema = z.object({ @@ -43,7 +43,7 @@ export const generateTitle = internalAction({ const structuredModel = model.withStructuredOutput(titleSchema); const title = (await structuredModel.invoke([ new SystemMessage( - "You are a title generator that generates a short title for the following user message.", + "You are a title generator that generates a short title for the following user message." ), ...firstMessage, ])) as z.infer; @@ -82,7 +82,7 @@ export const chat = action({ configurable: { ctx, chat, customPrompt, thread_id: chatId }, recursionLimit: 30, signal: abort.signal, - }, + } ); let streamDoc: Doc<"streams"> | null = null; @@ -107,19 +107,19 @@ export const chat = action({ chunks, completedSteps: [ ...(localCheckpoint?.pastSteps?.map( - (pastStep) => pastStep[0], + (pastStep) => pastStep[0] ) ?? []), ...(localCheckpoint?.plan && localCheckpoint.plan.length > 0 ? [ ...(localCheckpoint.plan[0].type === "parallel" ? localCheckpoint.plan[0].data.map( - (step) => step.step, + (step) => step.step ) : [localCheckpoint.plan[0].data.step]), ] : []), ], - }, + } ); } if (streamDoc?.status === "cancelled") { @@ -145,7 +145,7 @@ export const chat = action({ const allowedNodes = ["baseAgent", "simple", "plannerAgent"]; if ( allowedNodes.some((node) => - evt.metadata?.checkpoint_ns?.startsWith(node), + evt.metadata?.checkpoint_ns?.startsWith(node) ) ) { if (evt.event === "on_chat_model_stream") { @@ -155,7 +155,7 @@ export const chat = action({ content: evt.data?.chunk?.content ?? "", reasoning: evt.data?.chunk?.additional_kwargs?.reasoning_content, - } as AIChunkGroup), + } as AIChunkGroup) ); } else if (evt.event === "on_tool_start") { buffer.push( @@ -165,7 +165,17 @@ export const chat = action({ input: evt.data?.input, isComplete: false, toolCallId: evt.run_id, - } as ToolChunkGroup), + } as ToolChunkGroup) + ); + } else if (evt.event === "on_tool_stream") { + buffer.push( + JSON.stringify({ + type: "tool", + toolName: evt.name, + output: evt.data?.chunk, + isComplete: false, + toolCallId: evt.run_id, + } as ToolChunkGroup) ); } else if (evt.event === "on_tool_end") { let output = evt.data?.output.content; @@ -186,7 +196,7 @@ export const chat = action({ }; } return item; - }), + }) ); } @@ -198,7 +208,7 @@ export const chat = action({ output, isComplete: true, toolCallId: evt.run_id, - } as ToolChunkGroup), + } as ToolChunkGroup) ); } } @@ -263,12 +273,12 @@ export const chat = action({ type: "file", key, size: blob.size, - }, + } ); return { type: "file", file: { file_id: docId } }; } return item; - }), + }) ); stored = { ...stored, @@ -347,7 +357,7 @@ export const branchChat = action({ }); const branchFromMessage = allMessages.find( - (m) => m._id === args.branchFrom, + (m) => m._id === args.branchFrom ); if (!branchFromMessage) { throw new Error("Branch message not found"); @@ -370,7 +380,7 @@ export const branchChat = action({ chatId: newChatId, messages: thread.map((m) => ({ message: JSON.stringify( - mapChatMessagesToStoredMessages([m.message])[0], + mapChatMessagesToStoredMessages([m.message])[0] ), })), }); diff --git a/convex/langchain/prompts.ts b/convex/langchain/prompts.ts index accfbd16..6a9348b5 100644 --- a/convex/langchain/prompts.ts +++ b/convex/langchain/prompts.ts @@ -136,7 +136,7 @@ export function createAgentSystemMessage( plannerMode: boolean = false, customPrompt?: string, baseAgentType: boolean = true, - artifacts: boolean = true, + artifacts: boolean = true ): SystemMessage { const baseIdentity = `You are 0bs Chat, an AI assistant powered by the ${model} model.`; @@ -178,35 +178,51 @@ export function createAgentSystemMessage( `- If documents are provided, they are made avilable to in /mnt/data directory.\n`; return new SystemMessage( - `${baseIdentity} ${roleDescription}${communicationGuidelines}${formattingGuidelines}${baseAgentType ? baseAgentGuidelines : ""}${artifacts ? artifactsGuidelines : ""}${customPrompt ? customPrompt : ""}`, + `${baseIdentity} ${roleDescription}${communicationGuidelines}${formattingGuidelines}${baseAgentType ? baseAgentGuidelines : ""}${artifacts ? artifactsGuidelines : ""}${customPrompt ? customPrompt : ""}` ); } // Prompt template for planner export function createPlannerPrompt(availableToolsDescription: string) { - const toolsSection = `\n**Available Tools:**\n${availableToolsDescription}\n\nWhen planning steps, consider which tools are available and how they can be used to accomplish the objective efficiently.`; + const toolsSection = + `\n**Available Tools**\n${availableToolsDescription}\n` + + `When planning, think about which tool each step will need (if any).\n`; + // --- NEW PROMPT --------------------------------------------------------- return ChatPromptTemplate.fromMessages([ [ "system", - `For the given objective, create a step-by-step plan using the planStep and planArray schema conventions.\n\n` + - `**CRITICAL INSTRUCTIONS:**\n` + - `- You are ONLY responsible for creating a plan, NOT executing it\n` + - `- DO NOT call any tools or execute any function calls\n` + - `- DO NOT attempt to carry out the plan yourself\n` + - `- ONLY output valid JSON that conforms to the planArray schema\n` + - `- Your response must be parseable JSON with no additional text, explanations, or markdown formatting\n\n` + - `**Planning Guidelines:**\n` + - `- Each step should be actionable, unambiguous, and provide all information needed for a subagent to execute independently\n` + - `- Use the discriminated union format with nested arrays for parallel execution\n` + - `- Scale the number of steps and parallelism to the complexity of the query\n` + - `- Do not add superfluous steps. The result of the final step should be the final answer\n` + - `- Make the plan technical and specific to the topic\n` + - `- Each planStep object must include both "step" (short instruction) and "context" (detailed explanation) properties\n` + - `- Use the discriminated union format: {{ type: "single", data: planStep }} for single steps or {{ type: "parallel", data: planStep[] }} for parallel execution\n` + - `${toolsSection}\n\n` + - `**Output Format:**\n` + - `Your response must be a valid JSON array following the planArray schema. Do not include any other text, explanations, or formatting.\n`, + String.raw` +You are a task-planner. Your ONLY job is to output a valid JSON object that +matches **exactly** the TypeScript schema shown below. Do NOT execute the plan. + +--------------- REQUIRED SCHEMA ----------------- + +type planStep = { step: string; context: string }; + +type PlanItem = + | { type: "single"; data: planStep } + | { type: "parallel"; data: planStep[] }; + +export type Plan = { plan: PlanItem[] }; + +--------------------------------------------------- + +Important constraints: +1. The top-level key must be "plan". +2. In a "parallel" item, **data is an array of planStep**, NOT wrapped + in {type:"single"} objects. ❌ WRONG: + { "type":"parallel","data":[{ "type":"single", data:{…} }]} + ✅ RIGHT: + { "type":"parallel","data":[ { "step":"...", "context":"..." }, … ] } +3. Every planStep must have both fields: "step" (≤6 words) and "context" + (1-3 sentences with enough detail for an agent to act). +4. No markdown, no extra keys, no comments. +5. Your JSON must pass a strict JSON.parse in JavaScript. + +${toolsSection} + +Respond with the JSON ONLY.`, ], new MessagesPlaceholder("messages"), ]); @@ -266,7 +282,7 @@ export const replannerOutputSchema = (artifacts: boolean) => "coherent, and well-formatted answer. This is the ONLY output the end-user will see. It must fully and directly " + "address the user's original query, leaving no questions unanswered. Do not include any conversational filler, " + "apologies, or meta-commentary about the process; provide only the definitive answer." + - `${artifacts ? ` Adhere to the following additional guidelines and format your response accordingly:\n${artifactsGuidelines}` : ""}`, + `${artifacts ? ` Adhere to the following additional guidelines and format your response accordingly:\n${artifactsGuidelines}` : ""}` ), ]) .describe("The response data - either a plan array or a string response"), diff --git a/services/mcps b/services/mcps index 79728a60..04e22993 160000 --- a/services/mcps +++ b/services/mcps @@ -1 +1 @@ -Subproject commit 79728a601108071447ba015dcaedcbb7ba81e131 +Subproject commit 04e2299328a90aecb6ebcf08819eab35f796ade2 diff --git a/src/components/chat/messages/ai-message/tool-message/search-results.tsx b/src/components/chat/messages/ai-message/tool-message/search-results.tsx index 463456a1..ccb8fd36 100644 --- a/src/components/chat/messages/ai-message/tool-message/search-results.tsx +++ b/src/components/chat/messages/ai-message/tool-message/search-results.tsx @@ -36,7 +36,7 @@ export const SearchResultDisplay = ({ }: SearchResultDisplayProps) => { if (!results || results.length === 0) { return ( -
+
No search results found
); @@ -45,7 +45,7 @@ export const SearchResultDisplay = ({ return ( @@ -53,9 +53,9 @@ export const SearchResultDisplay = ({ className={`flex items-center gap-2 text-sm text-muted-foreground py-0 justify-start`} >
-
+
- + Web Search Results ({results.length})
@@ -73,7 +73,7 @@ export const SearchResultDisplay = ({ window.open( result.metadata.source, "_blank", - "noopener,noreferrer", + "noopener,noreferrer" ) } onKeyDown={(e) => { @@ -82,7 +82,7 @@ export const SearchResultDisplay = ({ window.open( result.metadata.source, "_blank", - "noopener,noreferrer", + "noopener,noreferrer" ); } }} @@ -99,7 +99,7 @@ export const SearchResultDisplay = ({ alt="" className="w-full h-full object-cover" src={`https://api.microlink.io/?url=${encodeURIComponent( - result.metadata.image || result.metadata.source, + result.metadata.image || result.metadata.source )}&screenshot=true&meta=false&embed=screenshot.url`} />
diff --git a/src/hooks/chats/use-stream.ts b/src/hooks/chats/use-stream.ts index d1b52d36..4a956393 100644 --- a/src/hooks/chats/use-stream.ts +++ b/src/hooks/chats/use-stream.ts @@ -17,12 +17,12 @@ export type ChunkGroup = AIChunkGroup | ToolChunkGroup; export function useStream(chatId: Id<"chats"> | "new") { const stream = useQuery( api.streams.queries.get, - chatId !== "new" ? { chatId } : "skip", + chatId !== "new" ? { chatId } : "skip" ); const [groupedChunks, setGroupedChunks] = useState([]); const [lastSeenTime, setLastSeenTime] = useState( - undefined, + undefined ); // Reset state when chat or stream changes @@ -41,7 +41,7 @@ export function useStream(chatId: Id<"chats"> | "new") { lastChunkTime: lastSeenTime, paginationOpts: { numItems: 200, cursor: null }, } - : "skip", + : "skip" ); // Process new chunks when they arrive @@ -51,8 +51,8 @@ export function useStream(chatId: Id<"chats"> | "new") { const newEvents: ChunkGroup[] = chunksResult.chunks.page.flatMap( (chunkDoc: any) => chunkDoc.chunks.map( - (chunkStr: string) => JSON.parse(chunkStr) as ChunkGroup, - ), + (chunkStr: string) => JSON.parse(chunkStr) as ChunkGroup + ) ); if (newEvents.length > 0) { @@ -74,8 +74,34 @@ export function useStream(chatId: Id<"chats"> | "new") { newGroups.push(lastGroup); } } else { - lastGroup = chunk; - newGroups.push(chunk); + // Handle tool chunks (start, stream, end) + if ( + lastGroup?.type === "tool" && + lastGroup.toolCallId === (chunk as ToolChunkGroup).toolCallId && + !lastGroup.isComplete && + !chunk.isComplete + ) { + // Same ongoing tool call → append partial output if present + if (chunk.output !== undefined) { + if ( + typeof lastGroup.output === "string" && + typeof chunk.output === "string" + ) { + lastGroup.output = (lastGroup.output ?? "") + chunk.output; + } else if ( + Array.isArray(lastGroup.output) && + Array.isArray(chunk.output) + ) { + lastGroup.output.push(...chunk.output); + } else { + // Fallback: replace + lastGroup.output = chunk.output; + } + } + } else { + lastGroup = chunk; + newGroups.push(chunk); + } } } return newGroups; @@ -102,7 +128,7 @@ export function useStream(chatId: Id<"chats"> | "new") { const completedIds = new Set( groupedChunks .filter((c) => c.type === "tool" && c.isComplete) - .map((c) => (c as ToolChunkGroup).toolCallId), + .map((c) => (c as ToolChunkGroup).toolCallId) ); return groupedChunks .map((chunk) => { @@ -130,7 +156,10 @@ export function useStream(chatId: Id<"chats"> | "new") { return new LangChainToolMessage({ name: chunk.toolName, tool_call_id: chunk.toolCallId, - content: "", + content: + typeof chunk.output === "string" + ? chunk.output + : JSON.stringify(chunk.output ?? ""), additional_kwargs: { input: JSON.parse(JSON.stringify(chunk.input)), is_complete: false, From a2464bd538ec3a68617bb51f50d3350f76d4c0eb Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Tue, 5 Aug 2025 05:58:43 +0530 Subject: [PATCH 02/25] chore: update subproject commit and enhance ToolMessage component - Updated subproject commit reference for mcps. - Introduced StreamingOutput component for better handling of streaming messages. - Improved ToolMessage component to utilize StreamingOutput for displaying message content and status. - Enhanced SearchResultDisplay styling and adjusted ToolAccordion for better user experience. - Added utility functions for tool display and formatting in tool-utils.ts. --- services/mcps | 2 +- .../ai-message/tool-message/index.tsx | 51 +++++---- .../tool-message/search-results.tsx | 10 +- src/components/ui/streaming-output.tsx | 42 ++++++++ src/components/ui/tool-accoordion.tsx | 25 ++++- src/hooks/chats/use-stream.ts | 3 + src/lib/tool-utils.ts | 102 ++++++++++++++++++ 7 files changed, 200 insertions(+), 35 deletions(-) create mode 100644 src/components/ui/streaming-output.tsx create mode 100644 src/lib/tool-utils.ts diff --git a/services/mcps b/services/mcps index 04e22993..1025baa1 160000 --- a/services/mcps +++ b/services/mcps @@ -1 +1 @@ -Subproject commit 04e2299328a90aecb6ebcf08819eab35f796ade2 +Subproject commit 1025baa1e9ea40402c780cc73e379152ce09f82e diff --git a/src/components/chat/messages/ai-message/tool-message/index.tsx b/src/components/chat/messages/ai-message/tool-message/index.tsx index 1206592c..95179352 100644 --- a/src/components/chat/messages/ai-message/tool-message/index.tsx +++ b/src/components/chat/messages/ai-message/tool-message/index.tsx @@ -4,12 +4,12 @@ import { DocumentResultDisplay, type DocumentResult } from "./document-results"; import type { BaseMessage } from "@langchain/core/messages"; import { FileDisplay } from "./file-result"; import ToolAccordion from "@/components/ui/tool-accoordion"; +import { StreamingOutput } from "@/components/ui/streaming-output"; export const ToolMessage = memo(({ message }: { message: BaseMessage }) => { const parsedContent = useMemo(() => { if (!message) return null; - // 1) Known “searchWeb” → SearchResult[] if (message.name === "searchWeb") { try { return { @@ -21,7 +21,6 @@ export const ToolMessage = memo(({ message }: { message: BaseMessage }) => { } } - // 2) Known “searchProjectDocuments” → DocumentResult[] if (message.name === "searchProjectDocuments") { try { return { @@ -33,14 +32,13 @@ export const ToolMessage = memo(({ message }: { message: BaseMessage }) => { } } - // 3) Mixed [ { type: "file", file: { file_id } } | { type: "text", text } ] try { const maybeArr = JSON.parse(message.content as string); if (Array.isArray(maybeArr)) { const isMixed = maybeArr.some( (i) => (i.type === "file" && i.file?.file_id) || - (i.type === "text" && i.text), + (i.type === "text" && i.text) ); if (isMixed) { return { type: "mixed" as const, content: maybeArr }; @@ -57,15 +55,30 @@ export const ToolMessage = memo(({ message }: { message: BaseMessage }) => { const input = (message.additional_kwargs as any)?.input as | Record | undefined; + const isComplete = (message.additional_kwargs as any)?.is_complete; if (!parsedContent) return null; // Search/Web calls render in their own specialized component if (parsedContent.type === "searchWeb") { + // Show streaming output if not complete + if (isComplete === false || !parsedContent.results.length) { + return ( + + + + ); + } + return ( ); } + if (parsedContent.type === "document") { return ; } @@ -76,7 +89,7 @@ export const ToolMessage = memo(({ message }: { message: BaseMessage }) => {
{parsedContent.content.map((item, idx) => { @@ -85,12 +98,11 @@ export const ToolMessage = memo(({ message }: { message: BaseMessage }) => { } if (item.type === "text" && item.text) { return ( -
-                  {item.text}
-                
+ content={item.text} + isComplete={isComplete} + /> ); } return null; @@ -105,17 +117,16 @@ export const ToolMessage = memo(({ message }: { message: BaseMessage }) => { - {typeof parsedContent.content === "string" ? ( -
-          {parsedContent.content}
-        
- ) : ( -
-          {JSON.stringify(parsedContent.content, null, 2)}
-        
- )} +
); }); diff --git a/src/components/chat/messages/ai-message/tool-message/search-results.tsx b/src/components/chat/messages/ai-message/tool-message/search-results.tsx index ccb8fd36..dff5b7ac 100644 --- a/src/components/chat/messages/ai-message/tool-message/search-results.tsx +++ b/src/components/chat/messages/ai-message/tool-message/search-results.tsx @@ -34,18 +34,10 @@ export const SearchResultDisplay = ({ results, input, }: SearchResultDisplayProps) => { - if (!results || results.length === 0) { - return ( -
- No search results found -
- ); - } - return ( diff --git a/src/components/ui/streaming-output.tsx b/src/components/ui/streaming-output.tsx new file mode 100644 index 00000000..d50ef99c --- /dev/null +++ b/src/components/ui/streaming-output.tsx @@ -0,0 +1,42 @@ +import { Loader2 } from "lucide-react"; + +interface StreamingOutputProps { + content: string; + isComplete?: boolean; + className?: string; +} + +export function StreamingOutput({ + content, + isComplete, + className = "", +}: StreamingOutputProps) { + if (!content && isComplete !== false) { + return ( +
+ No output +
+ ); + } + + if (!content && isComplete === false) { + return ( +
+ + Waiting for output... +
+ ); + } + + const showCursor = isComplete === false; + return ( +
+
+        {content}
+        {showCursor && |}
+      
+
+ ); +} diff --git a/src/components/ui/tool-accoordion.tsx b/src/components/ui/tool-accoordion.tsx index 7c12aeb8..bbb5fbf4 100644 --- a/src/components/ui/tool-accoordion.tsx +++ b/src/components/ui/tool-accoordion.tsx @@ -5,6 +5,7 @@ import { AccordionTrigger, AccordionContent, } from "@/components/ui/accordion"; +import { formatToolInput, getToolStatusText } from "@/lib/tool-utils"; type ToolAccordionProps = { messageName: string; @@ -20,7 +21,11 @@ function ToolAccordion({ isComplete, }: ToolAccordionProps) { return ( - + {isComplete === true ? ( @@ -28,16 +33,26 @@ function ToolAccordion({ ) : isComplete === false ? ( ) : null} - - Tool Call ({messageName}) + + {messageName}{" "} + {getToolStatusText(isComplete) && + `(${getToolStatusText(isComplete)})`} - +

Input

-            {JSON.stringify(input, null, 2)}
+            {formatToolInput(input)}
           

Output diff --git a/src/hooks/chats/use-stream.ts b/src/hooks/chats/use-stream.ts index 4a956393..abded32b 100644 --- a/src/hooks/chats/use-stream.ts +++ b/src/hooks/chats/use-stream.ts @@ -81,6 +81,9 @@ export function useStream(chatId: Id<"chats"> | "new") { !lastGroup.isComplete && !chunk.isComplete ) { + if (!lastGroup.toolName && chunk.toolName) { + lastGroup.toolName = chunk.toolName; + } // Same ongoing tool call → append partial output if present if (chunk.output !== undefined) { if ( diff --git a/src/lib/tool-utils.ts b/src/lib/tool-utils.ts new file mode 100644 index 00000000..d7afb681 --- /dev/null +++ b/src/lib/tool-utils.ts @@ -0,0 +1,102 @@ +/** + * Utility functions for tool display and formatting + */ + +export function cleanToolName( + rawName: string | null | undefined, + isComplete?: boolean +): string { + if (!rawName) { + if (isComplete === false) return "Tool Executing..."; + return "Unknown Tool"; + } + + // Handle MCP tool names like "mcp__browser__read_file" + if (rawName.startsWith("mcp__")) { + const parts = rawName.split("__"); + if (parts.length >= 3) { + const serverName = parts[1]; + const toolName = parts.slice(2).join("_"); + // Capitalize server name and format tool name + const formattedServerName = + serverName.charAt(0).toUpperCase() + serverName.slice(1); + const formattedToolName = toolName + .replace(/_/g, " ") + .replace(/\b\w/g, (l) => l.toUpperCase()); + return `${formattedServerName}: ${formattedToolName}`; + } + } + + if (rawName === "searchWeb") return "Web Search"; + if (rawName === "searchProjectDocuments") return "Project Search"; + if (rawName === "vectorSearch") return "Document Search"; + + return rawName + .replace(/_/g, " ") + .replace(/([A-Z])/g, " $1") + .replace(/\b\w/g, (l) => l.toUpperCase()) + .trim(); +} + +export function formatToolInput(input: any): string { + if (!input) return "No input provided"; + + try { + if (typeof input === "string") { + try { + const parsed = JSON.parse(input); + return JSON.stringify(parsed, null, 2); + } catch { + return input.length > 1 ? input : "No input provided"; + } + } + + return JSON.stringify(input, null, 2); + } catch { + return "Invalid input format"; + } +} + +export function formatToolOutput(output: any, isStreaming?: boolean): string { + if (!output) { + if (isStreaming) return "Waiting for output..."; + return ""; + } + + try { + if (typeof output === "string") { + // Handle malformed JSON inputs like '{"' + if ( + output.length <= 3 && + (output === '{"' || output === "{" || output === '"') + ) { + return isStreaming ? "Receiving data..." : "Invalid output"; + } + + // Try to parse as JSON first + try { + const parsed = JSON.parse(output); + return JSON.stringify(parsed, null, 2); + } catch { + // If not JSON, return as plain text + return output; + } + } + + return JSON.stringify(output, null, 2); + } catch { + return String(output); + } +} + +export function getToolStatusText(isComplete?: boolean): string { + if (isComplete === true) return "Completed"; + if (isComplete === false) return "Running"; + return ""; +} + +export function getToolStatusColor(isComplete?: boolean): string { + if (isComplete === true) return "text-green"; + if (isComplete === false) return "text-yellow-500"; + return "text-muted-foreground"; +} From 9826312837b4f247c8f07da480ac93e7212316d9 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Tue, 5 Aug 2025 06:09:16 +0530 Subject: [PATCH 03/25] refactor: enhance ToolMessage and ToolAccordion components for improved state handling - Introduced a MessageAdditionalKwargs interface in ToolMessage for better type safety. - Updated StreamingOutput to handle content display based on isComplete state more effectively. - Refactored ToolAccordion to utilize a new getContentClassName function for cleaner styling logic based on isComplete status. - Simplified tool name mapping in tool-utils.ts for better maintainability. --- .../ai-message/tool-message/index.tsx | 9 +++++++- src/components/ui/streaming-output.tsx | 2 +- src/components/ui/tool-accoordion.tsx | 23 +++++++++++-------- src/lib/tool-utils.ts | 11 ++++++--- 4 files changed, 31 insertions(+), 14 deletions(-) diff --git a/src/components/chat/messages/ai-message/tool-message/index.tsx b/src/components/chat/messages/ai-message/tool-message/index.tsx index 95179352..ee4e964c 100644 --- a/src/components/chat/messages/ai-message/tool-message/index.tsx +++ b/src/components/chat/messages/ai-message/tool-message/index.tsx @@ -55,7 +55,14 @@ export const ToolMessage = memo(({ message }: { message: BaseMessage }) => { const input = (message.additional_kwargs as any)?.input as | Record | undefined; - const isComplete = (message.additional_kwargs as any)?.is_complete; + + interface MessageAdditionalKwargs { + input?: Record; + is_complete?: boolean; + } + + const isComplete = (message.additional_kwargs as MessageAdditionalKwargs) + ?.is_complete; if (!parsedContent) return null; diff --git a/src/components/ui/streaming-output.tsx b/src/components/ui/streaming-output.tsx index d50ef99c..960ae90c 100644 --- a/src/components/ui/streaming-output.tsx +++ b/src/components/ui/streaming-output.tsx @@ -19,7 +19,7 @@ export function StreamingOutput({ ); } - if (!content && isComplete === false) { + if (!content && (isComplete === true || isComplete === undefined)) { return (
{ + const baseClasses = + "rounded-md p-2 border mt-2 max-h-[38rem] overflow-y-auto"; + + if (isComplete === false) { + return `${baseClasses} bg-yellow-50 dark:bg-yellow-950/20 border-yellow-200 dark:border-yellow-800`; + } + if (isComplete === true) { + return `${baseClasses} bg-green-50 dark:bg-green-950/20 border-green-200 dark:border-green-800`; + } + return `${baseClasses} bg-card`; +}; + function ToolAccordion({ messageName, input, @@ -39,15 +52,7 @@ function ToolAccordion({ `(${getToolStatusText(isComplete)})`} - +

Input

diff --git a/src/lib/tool-utils.ts b/src/lib/tool-utils.ts index d7afb681..4cd44085 100644 --- a/src/lib/tool-utils.ts +++ b/src/lib/tool-utils.ts @@ -1,6 +1,11 @@ /** * Utility functions for tool display and formatting */ +const TOOL_NAME_MAPPINGS = { + searchWeb: "Web Search", + searchProjectDocuments: "Project Search", + vectorSearch: "Document Search", +} as const; export function cleanToolName( rawName: string | null | undefined, @@ -27,9 +32,9 @@ export function cleanToolName( } } - if (rawName === "searchWeb") return "Web Search"; - if (rawName === "searchProjectDocuments") return "Project Search"; - if (rawName === "vectorSearch") return "Document Search"; + if (rawName in TOOL_NAME_MAPPINGS) { + return TOOL_NAME_MAPPINGS[rawName as keyof typeof TOOL_NAME_MAPPINGS]; + } return rawName .replace(/_/g, " ") From 8e06339fb1dfa0c60d0150d4a9cdef3b9f76c94b Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Tue, 12 Aug 2025 16:39:42 +0530 Subject: [PATCH 04/25] refactor: update prompts and toolbar icons for improved clarity and functionality - Replaced string-based system messages with structured SystemMessage objects in createPlannerPrompt and createReplannerPrompt functions for better type safety and readability. - Updated toolbar icon for "Artifacts" from FileIcon to FileCode2 for enhanced visual representation. - Revised todo list in temp.md to reflect current priorities and improvements, including UX enhancements and integration updates. --- convex/langchain/prompts.ts | 59 +++++++++++---------- src/components/chat/input/toolbar/index.tsx | 4 +- temp.md | 40 ++++++-------- 3 files changed, 50 insertions(+), 53 deletions(-) diff --git a/convex/langchain/prompts.ts b/convex/langchain/prompts.ts index 6a9348b5..855b1890 100644 --- a/convex/langchain/prompts.ts +++ b/convex/langchain/prompts.ts @@ -190,8 +190,7 @@ export function createPlannerPrompt(availableToolsDescription: string) { // --- NEW PROMPT --------------------------------------------------------- return ChatPromptTemplate.fromMessages([ - [ - "system", + new SystemMessage( String.raw` You are a task-planner. Your ONLY job is to output a valid JSON object that matches **exactly** the TypeScript schema shown below. Do NOT execute the plan. @@ -222,8 +221,8 @@ Important constraints: ${toolsSection} -Respond with the JSON ONLY.`, - ], +Respond with the JSON ONLY.` + ), new MessagesPlaceholder("messages"), ]); } @@ -233,17 +232,18 @@ export function createReplannerPrompt(availableToolsDescription: string) { const toolsSection = `\n**Available Tools:**\n${availableToolsDescription}\n\nWhen planning remaining steps, consider which tools are available and how they can be used to accomplish the remaining objectives efficiently.`; return ChatPromptTemplate.fromMessages([ - [ - "system", - `## Your Task: Reflect and Re-plan\n\n` + - `For the given objective, update the step-by-step plan using the planStep and planArray schema conventions.\n` + - `- Only include the remaining steps needed to fill the gaps identified in your analysis.\n` + - `- Use the discriminated union format with nested arrays for parallel execution.\n` + - `- Ensure steps are non-overlapping, unambiguous, and context-rich.\n` + - `- The result of the final step should be the final answer.\n` + - `${toolsSection}\n\n` + - `**Message History:**\n`, - ], + new SystemMessage( + String.raw`## Your Task: Reflect and Re-plan + +For the given objective, update the step-by-step plan using the planStep and planArray schema conventions. +- Only include the remaining steps needed to fill the gaps identified in your analysis. +- Use the discriminated union format with nested arrays for parallel execution. +- Ensure steps are non-overlapping, unambiguous, and context-rich. +- The result of the final step should be the final answer. +${toolsSection} + +**Message History:**` + ), new MessagesPlaceholder("messages"), [ "system", @@ -251,18 +251,23 @@ export function createReplannerPrompt(availableToolsDescription: string) { `**Completed steps so far:**\n`, ], new MessagesPlaceholder("pastSteps"), - [ - "system", - `\n\n**MANDATORY ANALYSIS & REPLANNING:**\n\n` + - `1. **Re-evaluate the Original Objective:** Look at the user's first message. What were the core components of their request?\n\n` + - `2. **Conduct a Gap Analysis:** Compare the completed steps' results against the original objective. Have all components been fully addressed? State explicitly what is **still missing**.\n\n` + - `3. **Assess Readiness:** Based on your analysis, decide if you have all the necessary information to synthesize a final, complete answer that satisfies the entire original objective.\n\n` + - `4. **Update Your Plan:**\n` + - ` - **If ready to respond:** Set type to "respond_to_user" and data as the response. Formulate the complete, synthesized response. This is not a draft; it is the complete, polished answer.\n` + - ` - **If not ready:** Set type to "continue_planning" and provide a new plan containing **only the remaining steps needed** to fill ` + - `the gaps you identified.\n\n` + - `**ALWAYS** output valid JSON, do not include any extraneous text or explanations, make sure you understand unions in the output format and respond correctly`, - ], + new SystemMessage( + String.raw` + +**MANDATORY ANALYSIS & REPLANNING:** + +1. **Re-evaluate the Original Objective:** Look at the user's first message. What were the core components of their request? + +2. **Conduct a Gap Analysis:** Compare the completed steps' results against the original objective. Have all components been fully addressed? State explicitly what is **still missing**. + +3. **Assess Readiness:** Based on your analysis, decide if you have all the necessary information to synthesize a final, complete answer that satisfies the entire original objective. + +4. **Update Your Plan:** + - **If ready to respond:** Set type to "respond_to_user" and data as the response. Formulate the complete, synthesized response. This is not a draft; it is the complete, polished answer. + - **If not ready:** Set type to "continue_planning" and provide a new plan containing **only the remaining steps needed** to fill the gaps you identified. + +**ALWAYS** output valid JSON, do not include any extraneous text or explanations, make sure you understand unions in the output format and respond correctly` + ), ]); } diff --git a/src/components/chat/input/toolbar/index.tsx b/src/components/chat/input/toolbar/index.tsx index 58f1f125..2dc9dc55 100644 --- a/src/components/chat/input/toolbar/index.tsx +++ b/src/components/chat/input/toolbar/index.tsx @@ -12,12 +12,12 @@ import { GithubIcon, BrainIcon, Hammer, - FileIcon, Globe2Icon, Network, Binoculars, XIcon, // <-- Add this FoldersIcon, + FileCode2, } from "lucide-react"; import { ProjectsDropdown } from "./projects-dropdown"; import { useUploadDocuments } from "@/hooks/chats/use-documents"; @@ -59,7 +59,7 @@ const TOGGLES = [ { key: "artifacts" as const, label: "Artifacts", - icon: , + icon: , tooltip: undefined, animation: "scale", }, diff --git a/temp.md b/temp.md index 7b64df03..8f1875b3 100644 --- a/temp.md +++ b/temp.md @@ -1,37 +1,29 @@ ### todo -- pricing [imp] -- usage -- improve ux overall with loading states and whatnot. -- google integration (the code is already there just need to setup oauth) -- business related mcp with ability to autofill connection info (like auto fetching api key/oauth key for the headers in mcp using oauth, etc to reduce friction) - -- need message queue system -- immediate send, wait for file to be processed check on the frontend instead of backend - -
- -caching implementation : (tanstack query) - -1. normal queries. -2. normal mutations. (not needed because these are not reactive, so doesn't make any big difference.) -3. paginated and infinite queries. (gotta figure something about it.) +--- keep in mind : move logic to hooks. --------- ---- +- update these + tool-streaming, because of vibz mcp as it one shots the generation. so we need to live stream to the user all the changes. + https://github.com/0bs-chat/zerobs/tree/feat/message-queue : the message queue function. ---- keep in mind : move logic to hooks. --------- +- vibe coding (better auth -> convex cloud migration -> streaming tool calls -> convex oauth integration -> revamp mcp templates to pass along the env vars) + custom ui for vibz mcp. (like artifacts, we will replace the panel content with the ui for vibz)(preview, dashboard (convex dashboard), code (vs code)) +- migrate to better auth (when i get the green light from mantra after better-auth integrates.) - infinite scroll area everywhere. -- convex subscription caching. : will make the ui snappy, can caus too much bandwidth usage. -- preloading. - look into action caching. +- add more integrations on the go allowing to auto fill auth tokens in sse mcp servers like github, nextjs etc. -
+--- -current : we refetch on every visit to a chat. because usequery fetches all the content again and again and again. which results in database bandwidth increase. -magic cache.: using useQuery from convex-helper/react/cache . we get the data from cache for a certain time. we can see the time as much time as we want. no fetching of data again and again. which even if got from cache. it still has to send it over the network which can cause database bandwidth increase. +- pricing [imp] +- usage +- improve ux overall with loading states and whatnot. [done] +- google integration (the code is already there just need to setup oauth) +- business related mcp with ability to autofill connection info (like auto fetching api key/oauth key for the headers in mcp using oauth, etc to reduce friction) -- how about fetching top 20 chats. +- need message queue system +- immediate send, wait for file to be processed check on the frontend instead of backend
From e9e0afb1d31710cdfe4323eca81857dc64e8f089 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Thu, 14 Aug 2025 00:39:49 +0530 Subject: [PATCH 05/25] feat: enhance tool integration and output handling in various components - Updated Google tools to include detailed streaming events and improved error handling for API requests. - Refactored MCP tools to wrap tool invocations with descriptive streaming events for better user feedback. - Enhanced retrieval tools with improved document search capabilities and error messaging. - Improved UI components for displaying search results and streaming outputs, including collapsible sections and better formatting. - Added date formatting for search results and refined tool accordion display for clarity. --- convex/langchain/index.ts | 65 +- convex/langchain/tools/googleTools.ts | 798 ++++++++++++------ convex/langchain/tools/mcpTools.ts | 75 +- convex/langchain/tools/retrievalTools.ts | 233 +++-- .../tool-message/search-results.tsx | 107 ++- src/components/ui/accordion.tsx | 18 +- src/components/ui/streaming-output.tsx | 108 ++- src/components/ui/tool-accoordion.tsx | 71 +- src/hooks/chats/use-stream.ts | 72 +- temp.md | 1 - 10 files changed, 1075 insertions(+), 473 deletions(-) diff --git a/convex/langchain/index.ts b/convex/langchain/index.ts index 9b1fe2dd..3087f265 100644 --- a/convex/langchain/index.ts +++ b/convex/langchain/index.ts @@ -178,7 +178,7 @@ export const chat = action({ } as ToolChunkGroup) ); } else if (evt.event === "on_tool_end") { - let output = evt.data?.output.content; + let output = evt.data?.output?.content ?? evt.data?.output; if (Array.isArray(output)) { output = await Promise.all( @@ -210,6 +210,35 @@ export const chat = action({ toolCallId: evt.run_id, } as ToolChunkGroup) ); + } else if (evt.event === "on_custom_event") { + try { + const eventName = + (evt as any)?.data?.event ?? + (evt as any)?.data?.name ?? + (evt as any)?.name; + const payload = + (evt as any)?.data?.data ?? + (evt as any)?.data?.payload ?? + (evt as any)?.data; + const chunk = + typeof payload?.chunk === "string" + ? payload.chunk + : undefined; + const isComplete = payload?.complete === true; + if (eventName === "tool_stream" && chunk) { + buffer.push( + JSON.stringify({ + type: "tool", + toolName: evt.name, + output: chunk, + isComplete, + toolCallId: evt.run_id, + } as ToolChunkGroup) + ); + } + } catch { + // ignore malformed custom events + } } } } @@ -219,6 +248,40 @@ export const chat = action({ }; await Promise.all([flusher(), streamer()]); + // Final flush in case there are any buffered chunks left + if (buffer.length > 0) { + const chunks = buffer; + buffer = []; + const completedSteps: string[] = []; + const pastSteps = (localCheckpoint as any)?.pastSteps as + | Array<[string, unknown[]]> + | undefined; + if (pastSteps && pastSteps.length > 0) { + completedSteps.push(...pastSteps.map((ps) => ps[0])); + } + const plan = (localCheckpoint as any)?.plan as + | Array< + | { + type: "parallel"; + data: Array<{ step: string; context: string }>; + } + | { type: "single"; data: { step: string; context: string } } + > + | undefined; + if (plan && plan.length > 0) { + const first = plan[0]; + if (first.type === "parallel") { + completedSteps.push(...first.data.map((s) => s.step)); + } else { + completedSteps.push(first.data.step); + } + } + await ctx.runMutation(internal.streams.mutations.flush, { + chatId, + chunks, + completedSteps, + }); + } return localCheckpoint; }; diff --git a/convex/langchain/tools/googleTools.ts b/convex/langchain/tools/googleTools.ts index b66dec81..74552fa6 100644 --- a/convex/langchain/tools/googleTools.ts +++ b/convex/langchain/tools/googleTools.ts @@ -1,4 +1,7 @@ -import { DynamicStructuredTool } from "@langchain/core/tools"; +"use node"; + +import { tool } from "@langchain/core/tools"; +import { dispatchCustomEvent } from "@langchain/core/callbacks/dispatch"; import { z } from "zod"; import type { ExtendedRunnableConfig } from "../helpers"; @@ -12,7 +15,7 @@ async function makeGoogleAPIRequest( endpoint: string, accessToken: string, method: string = "GET", - body?: any, + body?: any ): Promise { const response = await fetch( `https://www.googleapis.com/calendar/v3${endpoint}`, @@ -23,12 +26,12 @@ async function makeGoogleAPIRequest( "Content-Type": "application/json", }, body: body ? JSON.stringify(body) : undefined, - }, + } ); if (!response.ok) { throw new Error( - `Google API request failed: ${response.status} ${response.statusText}`, + `Google API request failed: ${response.status} ${response.statusText}` ); } @@ -40,7 +43,7 @@ async function makeGmailAPIRequest( endpoint: string, accessToken: string, method: string = "GET", - body?: any, + body?: any ): Promise { const response = await fetch( `https://www.googleapis.com/gmail/v1${endpoint}`, @@ -51,12 +54,12 @@ async function makeGmailAPIRequest( "Content-Type": "application/json", }, body: body ? JSON.stringify(body) : undefined, - }, + } ); if (!response.ok) { throw new Error( - `Gmail API request failed: ${response.status} ${response.statusText}`, + `Gmail API request failed: ${response.status} ${response.statusText}` ); } @@ -65,7 +68,7 @@ async function makeGmailAPIRequest( export const getGoogleTools = async ( config: ExtendedRunnableConfig, - returnString: boolean = false, + returnString: boolean = false ) => { const accessToken = await getGoogleAccessToken(config); if (!accessToken) { @@ -73,16 +76,22 @@ export const getGoogleTools = async ( } // Google Calendar Tools - const listCalendarseTool = new DynamicStructuredTool({ - name: "listGoogleCalendars", - description: - "List all Google Calendars accessible to the user. Use this to see available calendars before working with events.", - schema: z.object({}), - func: async () => { + const listCalendarseTool = tool( + async (_args: {}, toolConfig: any) => { try { + await dispatchCustomEvent( + "tool_stream", + { chunk: "Checking Google authentication…" }, + toolConfig + ); + await dispatchCustomEvent( + "tool_stream", + { chunk: "Fetching your Google calendars…" }, + toolConfig + ); const result = await makeGoogleAPIRequest( "/users/me/calendarList", - accessToken, + accessToken ); const calendars = @@ -94,56 +103,64 @@ export const getGoogleTools = async ( accessRole: calendar.accessRole, })) || []; + await dispatchCustomEvent( + "tool_stream", + { chunk: `Found ${calendars.length} calendars. Formatting results…` }, + toolConfig + ); + if (returnString) { return JSON.stringify(calendars, null, 2); } return calendars; } catch (error) { - return `Failed to list calendars: ${error instanceof Error ? error.message : "Unknown error"}`; + const message = + error instanceof Error ? error.message : "Unknown error"; + await dispatchCustomEvent( + "tool_stream", + { chunk: `Failed to list calendars: ${message}`, complete: true }, + toolConfig + ); + return `Failed to list calendars: ${message}`; } }, - }); - - const listCalendarEventsTool = new DynamicStructuredTool({ - name: "listGoogleCalendarEvents", - description: - "List events from a specific Google Calendar. Use this to see upcoming or past events.", - schema: z.object({ - calendarId: z - .string() - .describe("The calendar ID (use 'primary' for default calendar)") - .default("primary"), - timeMin: z - .string() - .optional() - .describe( - "Lower bound (inclusive) for an event's end time (RFC3339 timestamp)", - ), - timeMax: z - .string() - .optional() - .describe( - "Upper bound (exclusive) for an event's start time (RFC3339 timestamp)", - ), - maxResults: z - .number() - .min(1) - .max(2500) - .default(10) - .describe("Maximum number of events to return"), - q: z - .string() - .optional() - .describe("Free text search terms to find events"), - }), - func: async ({ - calendarId = "primary", - timeMin, - timeMax, - maxResults = 10, - q, - }) => { + { + name: "listGoogleCalendars", + description: + "List all Google Calendars accessible to the user. Use this to see available calendars before working with events.", + schema: z.object({}), + } + ); + + const listCalendarEventsTool = tool( + async ( + { + calendarId = "primary", + timeMin, + timeMax, + maxResults = 10, + q, + }: { + calendarId: string; + timeMin?: string; + timeMax?: string; + maxResults?: number; + q?: string; + }, + toolConfig: any + ) => { try { + await dispatchCustomEvent( + "tool_stream", + { + chunk: `Fetching events from calendar '${calendarId}'${ + timeMin || timeMax + ? ` within ${timeMin ?? "-∞"} → ${timeMax ?? "+∞"}` + : "" + }…`, + }, + toolConfig + ); const params = new URLSearchParams({ maxResults: maxResults.toString(), singleEvents: "true", @@ -156,7 +173,7 @@ export const getGoogleTools = async ( const result = await makeGoogleAPIRequest( `/calendars/${encodeURIComponent(calendarId)}/events?${params}`, - accessToken, + accessToken ); const events = @@ -172,56 +189,94 @@ export const getGoogleTools = async ( organizer: event.organizer, })) || []; + await dispatchCustomEvent( + "tool_stream", + { chunk: `Found ${events.length} events. Formatting results…` }, + toolConfig + ); + if (returnString) { return JSON.stringify(events, null, 2); } return events; } catch (error) { - return `Failed to list calendar events: ${error instanceof Error ? error.message : "Unknown error"}`; + const message = + error instanceof Error ? error.message : "Unknown error"; + await dispatchCustomEvent( + "tool_stream", + { + chunk: `Failed to list calendar events: ${message}`, + complete: true, + }, + toolConfig + ); + return `Failed to list calendar events: ${message}`; } }, - }); - - const createCalendarEventTool = new DynamicStructuredTool({ - name: "createGoogleCalendarEvent", - description: - "Create a new event in a Google Calendar. Use this to schedule meetings, appointments, or reminders.", - schema: z.object({ - calendarId: z - .string() - .describe("The calendar ID (use 'primary' for default calendar)") - .default("primary"), - summary: z.string().describe("The title/summary of the event"), - description: z - .string() - .optional() - .describe("The description of the event"), - startDateTime: z - .string() - .describe( - "Start date and time (RFC3339 format, e.g., '2024-01-15T09:00:00-07:00')", - ), - endDateTime: z - .string() - .describe( - "End date and time (RFC3339 format, e.g., '2024-01-15T10:00:00-07:00')", - ), - location: z.string().optional().describe("The location of the event"), - attendees: z - .array(z.string()) - .optional() - .describe("List of email addresses of attendees"), - }), - func: async ({ - calendarId = "primary", - summary, - description, - startDateTime, - endDateTime, - location, - attendees, - }) => { + { + name: "listGoogleCalendarEvents", + description: + "List events from a specific Google Calendar. Use this to see upcoming or past events.", + schema: z.object({ + calendarId: z + .string() + .describe("The calendar ID (use 'primary' for default calendar)") + .default("primary"), + timeMin: z + .string() + .optional() + .describe( + "Lower bound (inclusive) for an event's end time (RFC3339 timestamp)" + ), + timeMax: z + .string() + .optional() + .describe( + "Upper bound (exclusive) for an event's start time (RFC3339 timestamp)" + ), + maxResults: z + .number() + .min(1) + .max(2500) + .default(10) + .describe("Maximum number of events to return"), + q: z + .string() + .optional() + .describe("Free text search terms to find events"), + }), + } + ); + + const createCalendarEventTool = tool( + async ( + { + calendarId = "primary", + summary, + description, + startDateTime, + endDateTime, + location, + attendees, + }: { + calendarId: string; + summary: string; + description?: string; + startDateTime: string; + endDateTime: string; + location?: string; + attendees?: string[]; + }, + toolConfig: any + ) => { try { + await dispatchCustomEvent( + "tool_stream", + { + chunk: `Creating event '${summary}' on '${calendarId}' from ${startDateTime} to ${endDateTime}…`, + }, + toolConfig + ); const event = { summary, description, @@ -239,7 +294,7 @@ export const getGoogleTools = async ( `/calendars/${encodeURIComponent(calendarId)}/events`, accessToken, "POST", - event, + event ); const createdEvent = { @@ -252,61 +307,102 @@ export const getGoogleTools = async ( htmlLink: result.htmlLink, }; + await dispatchCustomEvent( + "tool_stream", + { chunk: "Event created successfully. Preparing output…" }, + toolConfig + ); + if (returnString) { return JSON.stringify(createdEvent, null, 2); } return createdEvent; } catch (error) { - return `Failed to create calendar event: ${error instanceof Error ? error.message : "Unknown error"}`; + const message = + error instanceof Error ? error.message : "Unknown error"; + await dispatchCustomEvent( + "tool_stream", + { + chunk: `Failed to create calendar event: ${message}`, + complete: true, + }, + toolConfig + ); + return `Failed to create calendar event: ${message}`; } }, - }); - - const updateCalendarEventTool = new DynamicStructuredTool({ - name: "updateGoogleCalendarEvent", - description: - "Update an existing event in a Google Calendar. Use this to modify event details.", - schema: z.object({ - calendarId: z - .string() - .describe("The calendar ID (use 'primary' for default calendar)") - .default("primary"), - eventId: z.string().describe("The ID of the event to update"), - summary: z - .string() - .optional() - .describe("The new title/summary of the event"), - description: z - .string() - .optional() - .describe("The new description of the event"), - startDateTime: z - .string() - .optional() - .describe("New start date and time (RFC3339 format)"), - endDateTime: z - .string() - .optional() - .describe("New end date and time (RFC3339 format)"), - location: z.string().optional().describe("The new location of the event"), - }), - func: async ({ - calendarId = "primary", - eventId, - summary, - description, - startDateTime, - endDateTime, - location, - }) => { + { + name: "createGoogleCalendarEvent", + description: + "Create a new event in a Google Calendar. Use this to schedule meetings, appointments, or reminders.", + schema: z.object({ + calendarId: z + .string() + .describe("The calendar ID (use 'primary' for default calendar)") + .default("primary"), + summary: z.string().describe("The title/summary of the event"), + description: z + .string() + .optional() + .describe("The description of the event"), + startDateTime: z + .string() + .describe( + "Start date and time (RFC3339 format, e.g., '2024-01-15T09:00:00-07:00')" + ), + endDateTime: z + .string() + .describe( + "End date and time (RFC3339 format, e.g., '2024-01-15T10:00:00-07:00')" + ), + location: z.string().optional().describe("The location of the event"), + attendees: z + .array(z.string()) + .optional() + .describe("List of email addresses of attendees"), + }), + } + ); + + const updateCalendarEventTool = tool( + async ( + { + calendarId = "primary", + eventId, + summary, + description, + startDateTime, + endDateTime, + location, + }: { + calendarId: string; + eventId: string; + summary?: string; + description?: string; + startDateTime?: string; + endDateTime?: string; + location?: string; + }, + toolConfig: any + ) => { try { - // First get the existing event + await dispatchCustomEvent( + "tool_stream", + { + chunk: `Loading existing event '${eventId}' from '${calendarId}'…`, + }, + toolConfig + ); const existingEvent = await makeGoogleAPIRequest( `/calendars/${encodeURIComponent(calendarId)}/events/${eventId}`, - accessToken, + accessToken ); - // Update only the provided fields + await dispatchCustomEvent( + "tool_stream", + { chunk: "Applying updates to the event…" }, + toolConfig + ); const updatedEvent = { ...existingEvent, ...(summary && { summary }), @@ -320,7 +416,7 @@ export const getGoogleTools = async ( `/calendars/${encodeURIComponent(calendarId)}/events/${eventId}`, accessToken, "PUT", - updatedEvent, + updatedEvent ); const event = { @@ -333,33 +429,82 @@ export const getGoogleTools = async ( htmlLink: result.htmlLink, }; + await dispatchCustomEvent( + "tool_stream", + { chunk: "Event updated successfully. Preparing output…" }, + toolConfig + ); + if (returnString) { return JSON.stringify(event, null, 2); } return event; } catch (error) { - return `Failed to update calendar event: ${error instanceof Error ? error.message : "Unknown error"}`; + const message = + error instanceof Error ? error.message : "Unknown error"; + await dispatchCustomEvent( + "tool_stream", + { + chunk: `Failed to update calendar event: ${message}`, + complete: true, + }, + toolConfig + ); + return `Failed to update calendar event: ${message}`; } }, - }); - - const deleteCalendarEventTool = new DynamicStructuredTool({ - name: "deleteGoogleCalendarEvent", - description: - "Delete an event from a Google Calendar. Use this to cancel or remove events.", - schema: z.object({ - calendarId: z - .string() - .describe("The calendar ID (use 'primary' for default calendar)") - .default("primary"), - eventId: z.string().describe("The ID of the event to delete"), - }), - func: async ({ calendarId = "primary", eventId }) => { + { + name: "updateGoogleCalendarEvent", + description: + "Update an existing event in a Google Calendar. Use this to modify event details.", + schema: z.object({ + calendarId: z + .string() + .describe("The calendar ID (use 'primary' for default calendar)") + .default("primary"), + eventId: z.string().describe("The ID of the event to update"), + summary: z + .string() + .optional() + .describe("The new title/summary of the event"), + description: z + .string() + .optional() + .describe("The new description of the event"), + startDateTime: z + .string() + .optional() + .describe("New start date and time (RFC3339 format)"), + endDateTime: z + .string() + .optional() + .describe("New end date and time (RFC3339 format)"), + location: z + .string() + .optional() + .describe("The new location of the event"), + }), + } + ); + + const deleteCalendarEventTool = tool( + async ( + { + calendarId = "primary", + eventId, + }: { calendarId: string; eventId: string }, + toolConfig: any + ) => { try { + await dispatchCustomEvent( + "tool_stream", + { chunk: `Deleting event '${eventId}' from '${calendarId}'…` }, + toolConfig + ); await makeGoogleAPIRequest( `/calendars/${encodeURIComponent(calendarId)}/events/${eventId}`, accessToken, - "DELETE", + "DELETE" ); const result = { @@ -367,41 +512,60 @@ export const getGoogleTools = async ( message: `Event ${eventId} deleted successfully`, }; + await dispatchCustomEvent( + "tool_stream", + { chunk: "Event deleted successfully." }, + toolConfig + ); + if (returnString) { return JSON.stringify(result, null, 2); } return result; } catch (error) { - return `Failed to delete calendar event: ${error instanceof Error ? error.message : "Unknown error"}`; + const message = + error instanceof Error ? error.message : "Unknown error"; + await dispatchCustomEvent( + "tool_stream", + { + chunk: `Failed to delete calendar event: ${message}`, + complete: true, + }, + toolConfig + ); + return `Failed to delete calendar event: ${message}`; } }, - }); + { + name: "deleteGoogleCalendarEvent", + description: + "Delete an event from a Google Calendar. Use this to cancel or remove events.", + schema: z.object({ + calendarId: z + .string() + .describe("The calendar ID (use 'primary' for default calendar)") + .default("primary"), + eventId: z.string().describe("The ID of the event to delete"), + }), + } + ); // Gmail Tools - const listGmailMessagesTool = new DynamicStructuredTool({ - name: "listGmailMessages", - description: - "List Gmail messages. Use this to search and retrieve email messages.", - schema: z.object({ - q: z - .string() - .optional() - .describe( - "Gmail search query (e.g., 'from:example@gmail.com', 'subject:meeting', 'is:unread')", - ), - maxResults: z - .number() - .min(1) - .max(500) - .default(10) - .describe("Maximum number of messages to return"), - labelIds: z - .array(z.string()) - .optional() - .describe("Only return messages with these label IDs"), - }), - func: async ({ q, maxResults = 10, labelIds }) => { + const listGmailMessagesTool = tool( + async ( + { + q, + maxResults = 10, + labelIds, + }: { q?: string; maxResults?: number; labelIds?: string[] }, + toolConfig: any + ) => { try { + await dispatchCustomEvent( + "tool_stream", + { chunk: "Fetching Gmail messages…" }, + toolConfig + ); const params = new URLSearchParams({ maxResults: maxResults.toString(), }); @@ -413,21 +577,20 @@ export const getGoogleTools = async ( const result = await makeGmailAPIRequest( `/users/me/messages?${params}`, - accessToken, + accessToken ); - // Get full message details for each message const messages = await Promise.all( (result.messages || []).map(async (message: any) => { const fullMessage = await makeGmailAPIRequest( `/users/me/messages/${message.id}`, - accessToken, + accessToken ); const headers = fullMessage.payload?.headers || []; const getHeader = (name: string) => headers.find( - (h: any) => h.name.toLowerCase() === name.toLowerCase(), + (h: any) => h.name.toLowerCase() === name.toLowerCase() )?.value; return { @@ -440,7 +603,13 @@ export const getGoogleTools = async ( date: getHeader("date"), labelIds: fullMessage.labelIds, }; - }), + }) + ); + + await dispatchCustomEvent( + "tool_stream", + { chunk: `Found ${messages.length} messages. Formatting results…` }, + toolConfig ); if (returnString) { @@ -448,27 +617,61 @@ export const getGoogleTools = async ( } return messages; } catch (error) { - return `Failed to list Gmail messages: ${error instanceof Error ? error.message : "Unknown error"}`; + const message = + error instanceof Error ? error.message : "Unknown error"; + await dispatchCustomEvent( + "tool_stream", + { + chunk: `Failed to list Gmail messages: ${message}`, + complete: true, + }, + toolConfig + ); + return `Failed to list Gmail messages: ${message}`; } }, - }); - - const getGmailMessageTool = new DynamicStructuredTool({ - name: "getGmailMessage", - description: - "Get the full content of a specific Gmail message. Use this to read the complete email content.", - schema: z.object({ - messageId: z.string().describe("The ID of the message to retrieve"), - format: z - .enum(["full", "metadata", "minimal"]) - .default("full") - .describe("The format to return the message in"), - }), - func: async ({ messageId, format = "full" }) => { + { + name: "listGmailMessages", + description: + "List Gmail messages. Use this to search and retrieve email messages.", + schema: z.object({ + q: z + .string() + .optional() + .describe( + "Gmail search query (e.g., 'from:example@gmail.com', 'subject:meeting', 'is:unread')" + ), + maxResults: z + .number() + .min(1) + .max(500) + .default(10) + .describe("Maximum number of messages to return"), + labelIds: z + .array(z.string()) + .optional() + .describe("Only return messages with these label IDs"), + }), + } + ); + + const getGmailMessageTool = tool( + async ( + { + messageId, + format = "full", + }: { messageId: string; format?: "full" | "metadata" | "minimal" }, + toolConfig: any + ) => { try { + await dispatchCustomEvent( + "tool_stream", + { chunk: `Fetching Gmail message '${messageId}'…` }, + toolConfig + ); const result = await makeGmailAPIRequest( `/users/me/messages/${messageId}?format=${format}`, - accessToken, + accessToken ); const headers = result.payload?.headers || []; @@ -476,14 +679,12 @@ export const getGoogleTools = async ( headers.find((h: any) => h.name.toLowerCase() === name.toLowerCase()) ?.value; - // Extract body content let body = ""; if (result.payload?.body?.data) { body = Buffer.from(result.payload.body.data, "base64").toString(); } else if (result.payload?.parts) { - // Multi-part message const textPart = result.payload.parts.find( - (part: any) => part.mimeType === "text/plain", + (part: any) => part.mimeType === "text/plain" ); if (textPart?.body?.data) { body = Buffer.from(textPart.body.data, "base64").toString(); @@ -502,48 +703,86 @@ export const getGoogleTools = async ( labelIds: result.labelIds, }; + await dispatchCustomEvent( + "tool_stream", + { chunk: "Message loaded. Preparing output…" }, + toolConfig + ); + if (returnString) { return JSON.stringify(message, null, 2); } return message; } catch (error) { - return `Failed to get Gmail message: ${error instanceof Error ? error.message : "Unknown error"}`; + const message = + error instanceof Error ? error.message : "Unknown error"; + await dispatchCustomEvent( + "tool_stream", + { chunk: `Failed to get Gmail message: ${message}`, complete: true }, + toolConfig + ); + return `Failed to get Gmail message: ${message}`; } }, - }); - - const sendGmailMessageTool = new DynamicStructuredTool({ - name: "sendGmailMessage", - description: - "Send a new Gmail message. Use this to compose and send emails.", - schema: z.object({ - to: z.string().describe("Recipient email address"), - subject: z.string().describe("Email subject line"), - body: z.string().describe("Email body content"), - cc: z.string().optional().describe("CC email address"), - bcc: z.string().optional().describe("BCC email address"), - }), - func: async ({ to, subject, body, cc, bcc }) => { + { + name: "getGmailMessage", + description: + "Get the full content of a specific Gmail message. Use this to read the complete email content.", + schema: z.object({ + messageId: z.string().describe("The ID of the message to retrieve"), + format: z + .enum(["full", "metadata", "minimal"]) + .default("full") + .describe("The format to return the message in"), + }), + } + ); + + const sendGmailMessageTool = tool( + async ( + { + to, + subject, + body, + cc, + bcc, + }: { + to: string; + subject: string; + body: string; + cc?: string; + bcc?: string; + }, + toolConfig: any + ) => { try { - // Create email in RFC 2822 format + await dispatchCustomEvent( + "tool_stream", + { chunk: `Composing email to ${to} with subject '${subject}'…` }, + toolConfig + ); let email = `To: ${to}\r\n`; if (cc) email += `Cc: ${cc}\r\n`; if (bcc) email += `Bcc: ${bcc}\r\n`; email += `Subject: ${subject}\r\n`; email += `\r\n${body}`; - // Encode email in base64url format const encodedEmail = Buffer.from(email) .toString("base64") .replace(/\+/g, "-") .replace(/\//g, "_") .replace(/=+$/, ""); + await dispatchCustomEvent( + "tool_stream", + { chunk: "Sending email via Gmail API…" }, + toolConfig + ); const result = await makeGmailAPIRequest( `/users/me/messages/send`, accessToken, "POST", - { raw: encodedEmail }, + { raw: encodedEmail } ); const sentMessage = { @@ -553,35 +792,52 @@ export const getGoogleTools = async ( message: "Email sent successfully", }; + await dispatchCustomEvent( + "tool_stream", + { chunk: "Email sent successfully. Preparing output…" }, + toolConfig + ); + if (returnString) { return JSON.stringify(sentMessage, null, 2); } return sentMessage; } catch (error) { - return `Failed to send Gmail message: ${error instanceof Error ? error.message : "Unknown error"}`; + const message = + error instanceof Error ? error.message : "Unknown error"; + await dispatchCustomEvent( + "tool_stream", + { chunk: `Failed to send Gmail message: ${message}`, complete: true }, + toolConfig + ); + return `Failed to send Gmail message: ${message}`; } }, - }); - - const searchGmailTool = new DynamicStructuredTool({ - name: "searchGmail", - description: - "Search Gmail messages with advanced query options. Use this for complex email searches.", - schema: z.object({ - query: z - .string() - .describe( - "Gmail search query (supports Gmail search operators like 'from:', 'subject:', 'has:attachment', etc.)", - ), - maxResults: z - .number() - .min(1) - .max(100) - .default(10) - .describe("Maximum number of results to return"), - }), - func: async ({ query, maxResults = 10 }) => { + { + name: "sendGmailMessage", + description: + "Send a new Gmail message. Use this to compose and send emails.", + schema: z.object({ + to: z.string().describe("Recipient email address"), + subject: z.string().describe("Email subject line"), + body: z.string().describe("Email body content"), + cc: z.string().optional().describe("CC email address"), + bcc: z.string().optional().describe("BCC email address"), + }), + } + ); + + const searchGmailTool = tool( + async ( + { query, maxResults = 10 }: { query: string; maxResults?: number }, + toolConfig: any + ) => { try { + await dispatchCustomEvent( + "tool_stream", + { chunk: `Searching Gmail for: ${query}…` }, + toolConfig + ); const params = new URLSearchParams({ q: query, maxResults: maxResults.toString(), @@ -589,23 +845,22 @@ export const getGoogleTools = async ( const result = await makeGmailAPIRequest( `/users/me/messages?${params}`, - accessToken, + accessToken ); - // Get basic info for each message const messages = await Promise.all( (result.messages || []) .slice(0, maxResults) .map(async (message: any) => { const fullMessage = await makeGmailAPIRequest( `/users/me/messages/${message.id}?format=metadata`, - accessToken, + accessToken ); const headers = fullMessage.payload?.headers || []; const getHeader = (name: string) => headers.find( - (h: any) => h.name.toLowerCase() === name.toLowerCase(), + (h: any) => h.name.toLowerCase() === name.toLowerCase() )?.value; return { @@ -617,7 +872,13 @@ export const getGoogleTools = async ( subject: getHeader("subject"), date: getHeader("date"), }; - }), + }) + ); + + await dispatchCustomEvent( + "tool_stream", + { chunk: `Found ${messages.length} matching messages. Formatting…` }, + toolConfig ); if (returnString) { @@ -625,10 +886,35 @@ export const getGoogleTools = async ( } return messages; } catch (error) { - return `Failed to search Gmail: ${error instanceof Error ? error.message : "Unknown error"}`; + const message = + error instanceof Error ? error.message : "Unknown error"; + await dispatchCustomEvent( + "tool_stream", + { chunk: `Failed to search Gmail: ${message}`, complete: true }, + toolConfig + ); + return `Failed to search Gmail: ${message}`; } }, - }); + { + name: "searchGmail", + description: + "Search Gmail messages with advanced query options. Use this for complex email searches.", + schema: z.object({ + query: z + .string() + .describe( + "Gmail search query (supports Gmail search operators like 'from:', 'subject:', 'has:attachment', etc.)" + ), + maxResults: z + .number() + .min(1) + .max(100) + .default(10) + .describe("Maximum number of results to return"), + }), + } + ); return [ listCalendarseTool, diff --git a/convex/langchain/tools/mcpTools.ts b/convex/langchain/tools/mcpTools.ts index 9c241f4a..ba57bf76 100644 --- a/convex/langchain/tools/mcpTools.ts +++ b/convex/langchain/tools/mcpTools.ts @@ -5,6 +5,8 @@ import type { StructuredToolInterface, ToolSchemaBase, } from "@langchain/core/tools"; +import { tool } from "@langchain/core/tools"; +import { dispatchCustomEvent } from "@langchain/core/callbacks/dispatch"; import { api, internal } from "../../_generated/api"; import type { ActionCtx } from "../../_generated/server"; import type { Id } from "../../_generated/dataModel"; @@ -15,7 +17,7 @@ import type { GraphState } from "../state"; export const getMCPTools = async ( ctx: ActionCtx, - state: typeof GraphState.State, + state: typeof GraphState.State ) => { const mcps = await ctx.runQuery(api.mcps.queries.getAll, { paginationOpts: { @@ -42,7 +44,7 @@ export const getMCPTools = async ( mcpId: mcp._id, }); } - }), + }) ); // Wait for all MCPs to transition from 'creating' status to 'running' @@ -68,7 +70,7 @@ export const getMCPTools = async ( // Filter for MCPs that are successfully running and have a URL const readyMcps = currentMcps.filter( - (mcp) => mcp.status === "created" && mcp.url, + (mcp) => mcp.status === "created" && mcp.url ); // Construct connection objects for MultiServerMCPClient @@ -86,7 +88,7 @@ export const getMCPTools = async ( delayMs: 200, }, }, - ]), + ]) ); // Initialize the MultiServerMCPClient @@ -101,7 +103,62 @@ export const getMCPTools = async ( const groupedTools: Map[]> = new Map(); - for (const tool of tools) { + // Wrap MCP tools to emit descriptive streaming events + const wrappedTools: StructuredToolInterface[] = tools.map( + (baseTool) => { + const parts = baseTool.name.split("__"); + const serverName = parts.length >= 2 ? parts[1] : "MCP"; + const prettyName = + parts.length >= 3 ? parts.slice(2).join(": ") : baseTool.name; + + // Preserve original schema/description/name + const wrapped = tool( + async (args: any, toolConfig: any) => { + await dispatchCustomEvent( + "tool_stream", + { + chunk: `Connecting to ${serverName} and invoking ${prettyName}…`, + }, + toolConfig + ); + try { + await dispatchCustomEvent( + "tool_stream", + { chunk: `Executing ${prettyName} with provided parameters…` }, + toolConfig + ); + const result = await (baseTool as any).invoke(args, toolConfig); + await dispatchCustomEvent( + "tool_stream", + { + chunk: `${prettyName} finished. Preparing results for display…`, + }, + toolConfig + ); + return result as any; + } catch (error) { + const message = + error instanceof Error ? error.message : "Unknown error"; + await dispatchCustomEvent( + "tool_stream", + { chunk: `${prettyName} failed: ${message}`, complete: true }, + toolConfig + ); + throw error; + } + }, + { + name: baseTool.name, + description: + (baseTool as any).description ?? `MCP tool from ${serverName}`, + schema: (baseTool as any).schema as any, + } + ); + return wrapped as unknown as StructuredToolInterface; + } + ); + + for (const tool of wrappedTools) { const parts = tool.name.split("__"); if (parts.length >= 2) { const serverName = parts[1]; @@ -128,7 +185,7 @@ export const getMCPTools = async ( internal.documents.crud.read, { id: documentId as Id<"documents">, - }, + } ); if (!documentDoc) return null; // Include various document types for upload (file, image, github, text) @@ -137,7 +194,7 @@ export const getMCPTools = async ( name: `${index}_${documentDoc.name}`, url, }; - }), + }) ) ) // Filter out any null results from documents without URLs @@ -148,13 +205,13 @@ export const getMCPTools = async ( if (["stdio", "docker"].includes(mcp.type) && files.length > 0) { await fly.uploadFileToAllMachines(mcp._id, files); } - }), + }) ); } } return { - tools: tools, + tools: wrappedTools, groupedTools: Object.fromEntries(groupedTools), }; }; diff --git a/convex/langchain/tools/retrievalTools.ts b/convex/langchain/tools/retrievalTools.ts index 0821f9fc..fa28f711 100644 --- a/convex/langchain/tools/retrievalTools.ts +++ b/convex/langchain/tools/retrievalTools.ts @@ -1,6 +1,7 @@ "use node"; -import { DynamicStructuredTool } from "@langchain/core/tools"; +import { tool } from "@langchain/core/tools"; +import { dispatchCustomEvent } from "@langchain/core/callbacks/dispatch"; import { ConvexVectorStore } from "@langchain/community/vectorstores/convex"; import { Document } from "@langchain/core/documents"; import { z } from "zod"; @@ -15,59 +16,71 @@ import { getUrl } from "../../utils/helpers"; export const getRetrievalTools = async ( _state: typeof GraphState.State, config: ExtendedRunnableConfig, - returnString: boolean = false, + returnString: boolean = false ) => { - const vectorSearchTool = new DynamicStructuredTool({ - name: "searchProjectDocuments", - description: - "Search through project documents using vector similarity search. Use this to find relevant information from uploaded project documents." + - "You are always supposed to use this tool if you are asked about something specific to find information but no additional information is provided.", - schema: z.object({ - query: z.string().describe("The search query to find relevant documents"), - limit: z - .number() - .min(1) - .max(256) - .describe("Number of results to return") - .default(10), - }), - func: async ({ query, limit = 10 }: { query: string; limit?: number }) => { - // Initialize ConvexVectorStore with the embedding model + const vectorSearchTool = tool( + async ( + { query, limit = 10 }: { query: string; limit?: number }, + toolConfig: any + ) => { + await dispatchCustomEvent( + "tool_stream", + { chunk: "Initializing vector store..." }, + toolConfig + ); const embeddingModel = await getEmbeddingModel(config.ctx, "embeddings"); const vectorStore = new ConvexVectorStore(embeddingModel, { ctx: config.ctx, table: "documentVectors", }); - // Get selected project documents to filter vector search results + await dispatchCustomEvent( + "tool_stream", + { chunk: "Loading selected project documents…" }, + toolConfig + ); const includedProjectDocuments = await config.ctx.runQuery( internal.projectDocuments.queries.getSelected, { projectId: config.chat.projectId!, selected: true, - }, + } ); if (includedProjectDocuments.length === 0) { - return "No project documents available for retrieval."; + const msg = "No project documents available for retrieval."; + await dispatchCustomEvent( + "tool_stream", + { chunk: msg, complete: true }, + toolConfig + ); + return msg; } - // Perform similarity search, filtering by selected documents + await dispatchCustomEvent( + "tool_stream", + { chunk: "Searching vector index…" }, + toolConfig + ); const results = await vectorStore.similaritySearch(query, limit, { filter: (q) => q.or( - // Assuming documentId is stored in the `source` field of metadata ...includedProjectDocuments.map((document) => q.eq("metadata", { source: document.documentId, - }), - ), + }) + ) ), }); + await dispatchCustomEvent( + "tool_stream", + { chunk: `Found ${results.length} results. Building response…` }, + toolConfig + ); const documentsMap = new Map, Doc<"documents">>(); includedProjectDocuments.forEach((projectDocument) => - documentsMap.set(projectDocument.documentId, projectDocument.document!), + documentsMap.set(projectDocument.documentId, projectDocument.document!) ); const documents = await Promise.all( @@ -86,64 +99,45 @@ export const getRetrievalTools = async ( type: "document", }, }); - }), + }) ); - if (returnString) { - return JSON.stringify(documents, null, 0); - } - - return documents; + await dispatchCustomEvent( + "tool_stream", + { chunk: "Formatting final output…" }, + toolConfig + ); + return returnString ? JSON.stringify(documents, null, 0) : documents; }, - }); + { + name: "searchProjectDocuments", + description: + "Search through project documents using vector similarity search. Use this to find relevant information from uploaded project documents." + + "You are always supposed to use this tool if you are asked about something specific to find information but no additional information is provided.", + schema: z.object({ + query: z + .string() + .describe("The search query to find relevant documents"), + limit: z + .number() + .min(1) + .max(256) + .describe("Number of results to return") + .default(10), + }), + } + ); - const webSearchTool = new DynamicStructuredTool({ - name: "searchWeb", - description: - "Search the web for current information using Exa (if API key is configured) or DuckDuckGo. Use this to find up-to-date information from the internet.", - schema: z.object({ - query: z - .string() - .describe("The search query to find relevant web information"), - topic: z - .union([ - z.literal("company"), - z.literal("research paper"), - z.literal("news"), - z.literal("pdf"), - z.literal("github"), - z.literal("personal site"), - z.literal("linkedin profile"), - z.literal("financial report"), - ]) - .describe( - "The topic of the search query (e.g., 'news', 'finance', ). By default, it will perform a google search." + - "### SEARCH STRATEGY EXAMPLES:\n" + - `- Topic: "AI model performance" → Search: "GPT-4 benchmark results 2024", "LLM performance comparison studies", "AI model evaluation metrics research"` + - `- Topic: "Company financials" → Search: "Tesla Q3 2024 earnings report", "Tesla revenue growth analysis", "electric vehicle market share 2024"` + - `- Topic: "Technical implementation" → Search: "React Server Components best practices", "Next.js performance optimization techniques", "modern web development patterns"` + - `### USAGE GUIDELINES:\n` + - `- Search first, search often, search comprehensively` + - `- Make 1-3 targeted searches per research topic to get different angles and perspectives` + - `- Search queries should be specific and focused` + - `- Follow up initial searches with more targeted queries based on what you learn` + - `- Cross-reference information by searching for the same topic from different angles` + - `- Search for contradictory information to get balanced perspectives` + - `- Include exact metrics, dates, technical terms, and proper nouns in queries` + - `- Make searches progressively more specific as you gather context` + - `- Search for recent developments, trends, and updates on topics` + - `- Always verify information with multiple searches from different sources`, - ) - .nullable() - .optional(), - }), - func: async ({ - query, - topic, - }: { - query: string; - topic?: string | null; - }) => { + const webSearchTool = tool( + async ( + { query, topic }: { query: string; topic?: string | null }, + toolConfig: any + ) => { + await dispatchCustomEvent( + "tool_stream", + { chunk: "Preparing web search…" }, + toolConfig + ); const EXA_API_KEY = ( await config.ctx.runQuery(internal.apiKeys.queries.getFromKey, { @@ -154,23 +148,37 @@ export const getRetrievalTools = async ( ""; try { + await dispatchCustomEvent( + "tool_stream", + { chunk: "Querying Exa…" }, + toolConfig + ); const exa = new Exa(EXA_API_KEY, undefined); - const searchResponse = ( await exa.searchAndContents(query, { numResults: 5, type: "auto", useAutoprompt: false, - topic: topic, + topic: topic ?? undefined, text: true, }) ).results; if (searchResponse.length === 0) { - return "No results found."; + const msg = "No results found."; + await dispatchCustomEvent( + "tool_stream", + { chunk: msg, complete: true }, + toolConfig + ); + return msg; } - // Create LangChain Document objects from Exa search results + await dispatchCustomEvent( + "tool_stream", + { chunk: `Found ${searchResponse.length} results. Formatting…` }, + toolConfig + ); const documents = searchResponse.map((result) => { return new Document({ pageContent: `${result.text}`, @@ -186,18 +194,61 @@ export const getRetrievalTools = async ( }); }); - if (returnString) { - return JSON.stringify(documents, null, 0); - } - - return documents; + return returnString ? JSON.stringify(documents, null, 0) : documents; } catch (error) { - return `Web search failed: ${ + const msg = `Web search failed: ${ error instanceof Error ? error.message : "Unknown error" }`; + await dispatchCustomEvent( + "tool_stream", + { chunk: msg, complete: true }, + toolConfig + ); + return msg; } }, - }); + { + name: "searchWeb", + description: + "Search the web for current information using Exa (if API key is configured) or DuckDuckGo. Use this to find up-to-date information from the internet.", + schema: z.object({ + query: z + .string() + .describe("The search query to find relevant web information"), + topic: z + .union([ + z.literal("company"), + z.literal("research paper"), + z.literal("news"), + z.literal("pdf"), + z.literal("github"), + z.literal("personal site"), + z.literal("linkedin profile"), + z.literal("financial report"), + ]) + .describe( + "The topic of the search query (e.g., 'news', 'finance', ). By default, it will perform a google search." + + "### SEARCH STRATEGY EXAMPLES:\n" + + `- Topic: "AI model performance" → Search: "GPT-4 benchmark results 2024", "LLM performance comparison studies", "AI model evaluation metrics research"` + + `- Topic: "Company financials" → Search: "Tesla Q3 2024 earnings report", "Tesla revenue growth analysis", "electric vehicle market share 2024"` + + `- Topic: "Technical implementation" → Search: "React Server Components best practices", "Next.js performance optimization techniques", "modern web development patterns"` + + `### USAGE GUIDELINES:\n` + + `- Search first, search often, search comprehensively` + + `- Make 1-3 targeted searches per research topic to get different angles and perspectives` + + `- Search queries should be specific and focused` + + `- Follow up initial searches with more targeted queries based on what you learn` + + `- Cross-reference information by searching for the same topic from different angles` + + `- Search for contradictory information to get balanced perspectives` + + `- Include exact metrics, dates, technical terms, and proper nouns in queries` + + `- Make searches progressively more specific as you gather context` + + `- Search for recent developments, trends, and updates on topics` + + `- Always verify information with multiple searches from different sources` + ) + .nullable() + .optional(), + }), + } + ); return { vectorSearch: vectorSearchTool, diff --git a/src/components/chat/messages/ai-message/tool-message/search-results.tsx b/src/components/chat/messages/ai-message/tool-message/search-results.tsx index dff5b7ac..696d4107 100644 --- a/src/components/chat/messages/ai-message/tool-message/search-results.tsx +++ b/src/components/chat/messages/ai-message/tool-message/search-results.tsx @@ -34,6 +34,17 @@ export const SearchResultDisplay = ({ results, input, }: SearchResultDisplayProps) => { + const formatDate = (iso?: string) => { + if (!iso) return ""; + const d = new Date(iso); + if (isNaN(d.getTime())) return ""; + return d.toLocaleDateString(undefined, { + year: "numeric", + month: "short", + day: "2-digit", + }); + }; + return (
- - Web Search Results ({results.length}) + Web Search Results + + {results.length}
@@ -59,54 +72,59 @@ export const SearchResultDisplay = ({
{results.map((result, index) => ( -
- window.open( - result.metadata.source, - "_blank", - "noopener,noreferrer" - ) - } - onKeyDown={(e) => { - if (e.key === "Enter" || e.key === " ") { - e.preventDefault(); - window.open( - result.metadata.source, - "_blank", - "noopener,noreferrer" - ); - } - }} - tabIndex={0} - role="button" + href={result.metadata.source} + target="_blank" + rel="noopener noreferrer" className="group relative flex flex-col flex-shrink-0 rounded-lg border bg-card text-left transition-all - duration-200 hover:shadow-lg hover:border-primary/20 hover:bg-accent/50 w-64 min-w-64 overflow-hidden - [&:hover]:shadow-lg [&:hover]:border-primary/20 [&:hover]:bg-accent/50 h-96 cursor-pointer + duration-200 hover:shadow-lg hover:border-primary/20 hover:bg-accent/50 w-64 min-w-64 overflow-hidden h-96 focus:outline-none focus:ring-2 focus:ring-primary/50" aria-label={`Open ${result.metadata.title} in new tab`} > -
+
{ + // Fallback to source preview if image is invalid + const el = e.currentTarget as HTMLImageElement; + const fallback = `https://api.microlink.io/?url=${encodeURIComponent(result.metadata.source)}&screenshot=true&meta=false&embed=screenshot.url`; + if (el.src !== fallback) el.src = fallback; + }} src={`https://api.microlink.io/?url=${encodeURIComponent( result.metadata.image || result.metadata.source )}&screenshot=true&meta=false&embed=screenshot.url`} /> +
-
-
- {result.metadata.favicon && ( - +
+
+
+ {result.metadata.favicon && ( + + )} + + {extractDomain(result.metadata.source)} + +
+ {result.metadata.publishedDate && ( + + {formatDate(result.metadata.publishedDate)} + )} -

- {result.metadata.title} -

+

+ {result.metadata.title} +

-
- - {extractDomain(result.metadata.source)} +
+ {result.metadata.author && ( + + {result.metadata.author} + + )} + + Visit + -
-
+ ))}
diff --git a/src/components/ui/accordion.tsx b/src/components/ui/accordion.tsx index a213e502..945d041c 100644 --- a/src/components/ui/accordion.tsx +++ b/src/components/ui/accordion.tsx @@ -109,12 +109,14 @@ interface AccordionTriggerProps { className?: string; children?: React.ReactNode; onClick?: React.MouseEventHandler; + showIcon?: boolean; } function AccordionTrigger({ className, children, onClick, + showIcon = true, ...props }: AccordionTriggerProps) { const { expandedItems, toggleItem } = useAccordion(); @@ -143,13 +145,15 @@ function AccordionTrigger({ onClick={handleClick} {...props} > - - - + {showIcon && ( + + + + )} {children}
diff --git a/src/components/ui/streaming-output.tsx b/src/components/ui/streaming-output.tsx index 960ae90c..8f4f99fb 100644 --- a/src/components/ui/streaming-output.tsx +++ b/src/components/ui/streaming-output.tsx @@ -1,42 +1,114 @@ -import { Loader2 } from "lucide-react"; +import { Loader2, Check, Terminal, ChevronDown, ChevronUp } from "lucide-react"; +import { useMemo, useState, useCallback } from "react"; +import { cn } from "@/lib/utils"; interface StreamingOutputProps { content: string; isComplete?: boolean; className?: string; + showHeader?: boolean; + collapsible?: boolean; } export function StreamingOutput({ content, isComplete, className = "", + showHeader = true, + collapsible = false, }: StreamingOutputProps) { - if (!content && isComplete !== false) { - return ( -
- No output -
- ); - } + const [isCollapsed, setIsCollapsed] = useState(false); + + const showCursor = isComplete === false; + + const { lastLine } = useMemo(() => { + const allLines = (content || "").split(/\n+/).filter(Boolean); + return { + lastLine: allLines.length > 0 ? allLines[allLines.length - 1] : "", + }; + }, [content]); + + const toggleCollapsed = useCallback(() => { + setIsCollapsed(!isCollapsed); + }, [isCollapsed]); - if (!content && (isComplete === true || isComplete === undefined)) { + if (!content) { + if (isComplete === false) { + return ( +
+ + Waiting... +
+ ); + } return (
- - Waiting for output... + + No output
); } - const showCursor = isComplete === false; return ( -
-
-        {content}
-        {showCursor && |}
-      
+
+
+ {showHeader && ( +
+
+ +
+
+ {collapsible && ( + + )} + {isComplete === true ? ( + + ) : isComplete === false ? ( + + ) : ( + + )} +
+
+ )} + {!isCollapsed && ( +
+ {lastLine && ( +
+ + {lastLine} + + {showCursor && ( + + )} +
+ )} +
+ )} +
); } diff --git a/src/components/ui/tool-accoordion.tsx b/src/components/ui/tool-accoordion.tsx index e77533e5..1aea6609 100644 --- a/src/components/ui/tool-accoordion.tsx +++ b/src/components/ui/tool-accoordion.tsx @@ -1,11 +1,11 @@ -import { Check, Loader2 } from "lucide-react"; +import { Check, HammerIcon, Loader2 } from "lucide-react"; import { Accordion, AccordionItem, AccordionTrigger, AccordionContent, } from "@/components/ui/accordion"; -import { formatToolInput, getToolStatusText } from "@/lib/tool-utils"; +import { cleanToolName, getToolStatusText } from "@/lib/tool-utils"; type ToolAccordionProps = { messageName: string; @@ -14,55 +14,46 @@ type ToolAccordionProps = { isComplete?: boolean; }; -const getContentClassName = (isComplete?: boolean) => { - const baseClasses = - "rounded-md p-2 border mt-2 max-h-[38rem] overflow-y-auto"; - - if (isComplete === false) { - return `${baseClasses} bg-yellow-50 dark:bg-yellow-950/20 border-yellow-200 dark:border-yellow-800`; - } - if (isComplete === true) { - return `${baseClasses} bg-green-50 dark:bg-green-950/20 border-green-200 dark:border-green-800`; - } - return `${baseClasses} bg-card`; -}; - function ToolAccordion({ messageName, - input, children, isComplete, }: ToolAccordionProps) { + const cleanedName = cleanToolName(messageName, isComplete); + const statusText = getToolStatusText(isComplete); + return ( - - - {isComplete === true ? ( - - ) : isComplete === false ? ( - - ) : null} - - {messageName}{" "} - {getToolStatusText(isComplete) && - `(${getToolStatusText(isComplete)})`} - + + +
+ + {cleanedName} + {statusText && ( + + {statusText} + + )} +
+
+ {isComplete === true ? ( + + ) : isComplete === false ? ( + + ) : ( +
+ )} +
- -

- Input -

-
-            {formatToolInput(input)}
-          
-

- Output -

-
+ +
{children}
diff --git a/src/hooks/chats/use-stream.ts b/src/hooks/chats/use-stream.ts index 77bb9a16..5ad0a2a6 100644 --- a/src/hooks/chats/use-stream.ts +++ b/src/hooks/chats/use-stream.ts @@ -59,10 +59,16 @@ export function useStream(chatId: Id<"chats"> | "new") { chunkDoc._creationTime > lastSeenTime ) .flatMap((chunkDoc: any) => - chunkDoc.chunks.map( - (chunkStr: string) => JSON.parse(chunkStr) as ChunkGroup - ) - ); + chunkDoc.chunks.map((chunkStr: string) => { + try { + return JSON.parse(chunkStr) as ChunkGroup; + } catch (e) { + console.error("Failed to parse stream chunk", { chunkStr, e }); + return undefined as unknown as ChunkGroup; + } + }) + ) + .filter(Boolean) as ChunkGroup[]; if (newEvents.length > 0) { setGroupedChunks((prev) => { const newGroups = [...prev]; @@ -81,8 +87,58 @@ export function useStream(chatId: Id<"chats"> | "new") { newGroups.push(lastGroup); } } else { - lastGroup = chunk; - newGroups.push(chunk); + const toolChunk = chunk as ToolChunkGroup; + // Try to merge with the latest in-flight chunk for the same toolCallId + let mergedIndex = -1; + if ( + lastGroup?.type === "tool" && + (lastGroup as ToolChunkGroup).toolCallId === + toolChunk.toolCallId && + !(lastGroup as ToolChunkGroup).isComplete + ) { + mergedIndex = newGroups.length - 1; + } else { + for (let i = newGroups.length - 1; i >= 0; i--) { + const g = newGroups[i]; + if ( + g.type === "tool" && + (g as ToolChunkGroup).toolCallId === + toolChunk.toolCallId && + !(g as ToolChunkGroup).isComplete + ) { + mergedIndex = i; + break; + } + } + } + + if (mergedIndex >= 0) { + const existing = newGroups[mergedIndex] as ToolChunkGroup; + // Append incremental output if provided + if (typeof toolChunk.output === "string") { + const existingOutput = + typeof existing.output === "string" + ? existing.output + : ""; + existing.output = existingOutput + ? `${existingOutput}\n${toolChunk.output}` + : toolChunk.output; + } else if (toolChunk.output !== undefined) { + existing.output = toolChunk.output as any; + } + // Update input if present + if (toolChunk.input !== undefined) { + existing.input = toolChunk.input as any; + } + // Mark complete if end event + if (toolChunk.isComplete) { + existing.isComplete = true; + } + lastGroup = existing; + } else { + lastGroup = toolChunk; + newGroups.push(toolChunk); + } } } return newGroups; @@ -133,7 +189,7 @@ export function useStream(chatId: Id<"chats"> | "new") { name: chunk.toolName, tool_call_id: chunk.toolCallId, additional_kwargs: { - input: JSON.parse(JSON.stringify(chunk.input)), + input: chunk.input ?? null, is_complete: true, }, }); @@ -147,7 +203,7 @@ export function useStream(chatId: Id<"chats"> | "new") { ? chunk.output : JSON.stringify(chunk.output ?? ""), additional_kwargs: { - input: JSON.parse(JSON.stringify(chunk.input)), + input: chunk.input ?? null, is_complete: false, }, }); diff --git a/temp.md b/temp.md index 8f1875b3..1d353276 100644 --- a/temp.md +++ b/temp.md @@ -9,7 +9,6 @@ - vibe coding (better auth -> convex cloud migration -> streaming tool calls -> convex oauth integration -> revamp mcp templates to pass along the env vars) custom ui for vibz mcp. (like artifacts, we will replace the panel content with the ui for vibz)(preview, dashboard (convex dashboard), code (vs code)) -- migrate to better auth (when i get the green light from mantra after better-auth integrates.) - infinite scroll area everywhere. - look into action caching. - add more integrations on the go allowing to auto fill auth tokens in sse mcp servers like github, nextjs etc. From 167e9f62ac771be925a9fde818a3c03843b93566 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Thu, 14 Aug 2025 04:15:56 +0530 Subject: [PATCH 06/25] refactor: clean up code formatting and improve tool retrieval logic - Removed unnecessary trailing commas and adjusted function signatures for consistency. - Enhanced the getAvailableTools function to return structured tool data more clearly. - Updated the StreamingOutput component to simplify state toggling logic. - Improved the ToolAccordion component's loader display for better visual clarity. --- convex/langchain/helpers.ts | 101 +++++++++++++---------- convex/langchain/tools/mcpTools.ts | 7 +- convex/langchain/tools/retrievalTools.ts | 2 +- src/components/ui/streaming-output.tsx | 4 +- src/components/ui/tool-accoordion.tsx | 2 +- 5 files changed, 65 insertions(+), 51 deletions(-) diff --git a/convex/langchain/helpers.ts b/convex/langchain/helpers.ts index 96eec58d..1f5777fe 100644 --- a/convex/langchain/helpers.ts +++ b/convex/langchain/helpers.ts @@ -35,7 +35,7 @@ export type ExtendedRunnableConfig = RunnableConfig & { export async function createSimpleAgent( _state: typeof GraphState.State, - config: ExtendedRunnableConfig, + config: ExtendedRunnableConfig ) { const chat = config.chat; const model = await getModel(config.ctx, chat.model, chat.reasoningEffort); @@ -45,7 +45,7 @@ export async function createSimpleAgent( undefined, config.customPrompt, false, - chat.artifacts, + chat.artifacts ), new MessagesPlaceholder("messages"), ]); @@ -55,7 +55,7 @@ export async function createSimpleAgent( export async function createAgentWithTools( state: typeof GraphState.State, config: ExtendedRunnableConfig, - plannerMode: boolean = false, + plannerMode: boolean = false ) { const chat = config.chat; const allTools = await getAvailableTools(state, config); @@ -68,7 +68,7 @@ export async function createAgentWithTools( plannerMode, plannerMode ? undefined : config.customPrompt, true, - plannerMode ? false : chat.artifacts, + plannerMode ? false : chat.artifacts ), new MessagesPlaceholder("messages"), ]); @@ -87,7 +87,7 @@ export async function createAgentWithTools( const supervisorLlm = await getModel( config.ctx, chat.model!, - chat.reasoningEffort, + chat.reasoningEffort ); const agents = toolkits.map((toolkit: Toolkit) => createReactAgent({ @@ -95,19 +95,17 @@ export async function createAgentWithTools( tools: toolkit.tools, name: toolkit.name, prompt: `You are a ${toolkit.name} assistant`, - }), + }) ); return createSupervisor({ - agents: [ - ...agents, - ], + agents: [...agents], llm: supervisorLlm, prompt: createAgentSystemMessage( chat.model, plannerMode, plannerMode ? undefined : config.customPrompt, true, - plannerMode ? false : chat.artifacts, + plannerMode ? false : chat.artifacts ), }).compile(); } @@ -116,7 +114,7 @@ export async function createAgentWithTools( export function getPlannerAgentResponse(messages: BaseMessage[]): BaseMessage { // filter and concat all ai messages const aiResponses = messages.filter( - (message) => typeof message === typeof AIMessage, + (message) => typeof message === typeof AIMessage ); const storedAIResponses = mapChatMessagesToStoredMessages(aiResponses); return mapStoredMessagesToChatMessages([ @@ -134,7 +132,7 @@ export function getPlannerAgentResponse(messages: BaseMessage[]): BaseMessage { export function getLastMessage( messages: BaseMessage[], - type: "ai" | "human", + type: "ai" | "human" ): { message: BaseMessage; index: number } | null { for (let i = messages.length - 1; i >= 0; i--) { const message = messages[i]; @@ -156,60 +154,72 @@ export type Toolkit = { // Overloads to provide precise return types based on `groupTools` export async function getAvailableTools( state: typeof GraphState.State, - config: ExtendedRunnableConfig, + config: ExtendedRunnableConfig ): Promise[]>; export async function getAvailableTools( state: typeof GraphState.State, config: ExtendedRunnableConfig, - groupTools: false, + groupTools: false ): Promise[]>; export async function getAvailableTools( state: typeof GraphState.State, config: ExtendedRunnableConfig, - groupTools: true, + groupTools: true ): Promise; export async function getAvailableTools( state: typeof GraphState.State, config: ExtendedRunnableConfig, - groupTools: boolean = false, + groupTools: boolean = false ): Promise[]> { const chat = config.chat; - const [ - mcpTools, - retrievalTools, - googleTools, - githubTools, - ] = await Promise.all([ - getMCPTools(config.ctx, state, config), - getRetrievalTools(state, config, true), - chat.enabledToolkits.includes("google") ? getGoogleTools(config) : Promise.resolve([]), - chat.enabledToolkits.includes("github") ? getGithubTools(config) : Promise.resolve([]), - ]); + const mcpPromise = getMCPTools(config.ctx, state, config); + const retrievalPromise = getRetrievalTools(state, config, true); + const googlePromise: Promise[]> = + chat.enabledToolkits.includes("google") + ? (getGoogleTools(config) as Promise< + StructuredToolInterface[] + >) + : Promise.resolve([] as StructuredToolInterface[]); + const githubPromise: Promise[]> = + chat.enabledToolkits.includes("github") + ? (getGithubTools(config) as Promise< + StructuredToolInterface[] + >) + : Promise.resolve([] as StructuredToolInterface[]); + + const [mcp, retrievalTools, googleTools, githubTools] = await Promise.all([ + mcpPromise, + retrievalPromise, + googlePromise, + githubPromise, + ] as const); if (!groupTools) { - return [ - ...mcpTools, + const pickedRetrievalTools: StructuredToolInterface[] = [ ...(chat.projectId ? [retrievalTools.vectorSearch] : []), ...(chat.webSearch ? [retrievalTools.webSearch] : []), - ...(googleTools.length > 0 ? googleTools : []), - ...(githubTools.length > 0 ? githubTools : []), ]; - } - // Group MCP tools by server name (tool name format: "mcp____") - const mcpGrouped = new Map[]>(); - for (const tool of mcpTools) { - const parts = tool.name.split("__"); - const groupName = parts.length >= 2 ? parts[1] : "MCP"; - if (!mcpGrouped.has(groupName)) mcpGrouped.set(groupName, []); - mcpGrouped.get(groupName)!.push(tool); + return [ + ...mcp.tools, + ...pickedRetrievalTools, + ...googleTools, + ...githubTools, + ]; } const toolkits: Toolkit[] = [ - ...Array.from(mcpGrouped.entries()).map(([name, tools]) => ({ name, tools })), - ...(chat.webSearch ? [{ name: "WebSearch", tools: [retrievalTools.webSearch] }] : []), - ...(chat.projectId ? [{ name: "VectorSearch", tools: [retrievalTools.vectorSearch] }] : []), + ...Object.entries(mcp.groupedTools).map(([name, tools]) => ({ + name, + tools: tools as StructuredToolInterface[], + })), + ...(chat.webSearch + ? [{ name: "WebSearch", tools: [retrievalTools.webSearch] }] + : []), + ...(chat.projectId + ? [{ name: "VectorSearch", tools: [retrievalTools.vectorSearch] }] + : []), ...(googleTools.length > 0 ? [{ name: "Google", tools: googleTools }] : []), ...(githubTools.length > 0 ? [{ name: "GitHub", tools: githubTools }] : []), ]; @@ -219,9 +229,10 @@ export async function getAvailableTools( export async function getAvailableToolsDescription( state: typeof GraphState.State, - config: ExtendedRunnableConfig, + config: ExtendedRunnableConfig ): Promise { - const toolsInfo: StructuredToolInterface[] = await getAvailableTools(state, config); + const toolsInfo: StructuredToolInterface[] = + await getAvailableTools(state, config); if (toolsInfo.length === 0) { return "No tools are currently available."; @@ -233,7 +244,7 @@ export async function getAvailableToolsDescription( } export function extractFileIdsFromMessage( - messageContent: any, + messageContent: any ): Id<"documents">[] { const fileIds: Id<"documents">[] = []; diff --git a/convex/langchain/tools/mcpTools.ts b/convex/langchain/tools/mcpTools.ts index 5b93dbe3..14610917 100644 --- a/convex/langchain/tools/mcpTools.ts +++ b/convex/langchain/tools/mcpTools.ts @@ -23,7 +23,10 @@ export const getMCPTools = async ( ctx: ActionCtx, state: typeof GraphState.State, config: ExtendedRunnableConfig -) => { +): Promise<{ + tools: StructuredToolInterface[]; + groupedTools: Record[]>; +}> => { const mcps = await ctx.runQuery(api.mcps.queries.getAll, { paginationOpts: { numItems: 100, @@ -35,7 +38,7 @@ export const getMCPTools = async ( }); if (mcps.page.length === 0) { - return []; + return { tools: [], groupedTools: {} }; } // Wait for all MCPs to transition from 'creating' status to 'running' diff --git a/convex/langchain/tools/retrievalTools.ts b/convex/langchain/tools/retrievalTools.ts index f654f7ab..f007bd08 100644 --- a/convex/langchain/tools/retrievalTools.ts +++ b/convex/langchain/tools/retrievalTools.ts @@ -159,7 +159,7 @@ export const getRetrievalTools = async ( numResults: 5, type: "auto", useAutoprompt: false, - topic: topic ?? undefined, + topic, text: true, }) ).results; diff --git a/src/components/ui/streaming-output.tsx b/src/components/ui/streaming-output.tsx index 8f4f99fb..a2aeb4df 100644 --- a/src/components/ui/streaming-output.tsx +++ b/src/components/ui/streaming-output.tsx @@ -29,8 +29,8 @@ export function StreamingOutput({ }, [content]); const toggleCollapsed = useCallback(() => { - setIsCollapsed(!isCollapsed); - }, [isCollapsed]); + setIsCollapsed((prev) => !prev); + }, []); if (!content) { if (isComplete === false) { diff --git a/src/components/ui/tool-accoordion.tsx b/src/components/ui/tool-accoordion.tsx index 1aea6609..aa92a86c 100644 --- a/src/components/ui/tool-accoordion.tsx +++ b/src/components/ui/tool-accoordion.tsx @@ -46,7 +46,7 @@ function ToolAccordion({ {isComplete === true ? ( ) : isComplete === false ? ( - + ) : (
)} From c2389758dc3e311858d591c16fa5ecd5b6ba1001 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Thu, 14 Aug 2025 05:11:15 +0530 Subject: [PATCH 07/25] refactor: improve tool handling and response extraction in chat actions - Updated the tool handling logic to utilize a new helper function for appending tool chunk data, enhancing code readability. - Refactored the extraction of completed steps from checkpoints into a dedicated function for better organization and clarity. - Adjusted various tool stream events to use the new helper function for consistency in data formatting. - Fixed minor formatting issues in retrieval tool messages for improved user feedback. --- convex/langchain/helpers.ts | 2 +- convex/langchain/index.ts | 161 ++++++++++++----------- convex/langchain/tools/retrievalTools.ts | 30 +++-- src/components/ui/tool-accoordion.tsx | 4 +- 4 files changed, 108 insertions(+), 89 deletions(-) diff --git a/convex/langchain/helpers.ts b/convex/langchain/helpers.ts index 1f5777fe..83055b41 100644 --- a/convex/langchain/helpers.ts +++ b/convex/langchain/helpers.ts @@ -114,7 +114,7 @@ export async function createAgentWithTools( export function getPlannerAgentResponse(messages: BaseMessage[]): BaseMessage { // filter and concat all ai messages const aiResponses = messages.filter( - (message) => typeof message === typeof AIMessage + (message) => message instanceof AIMessage ); const storedAIResponses = mapChatMessagesToStoredMessages(aiResponses); return mapStoredMessagesToChatMessages([ diff --git a/convex/langchain/index.ts b/convex/langchain/index.ts index 6971bf8a..3adeba11 100644 --- a/convex/langchain/index.ts +++ b/convex/langchain/index.ts @@ -19,6 +19,63 @@ import { getThreadFromMessage } from "../chatMessages/helpers"; import { formatMessages, getModel } from "./models"; import { ChatMessages, Chats } from "../schema"; +// Helper: Append a ToolChunkGroup JSON string to the provided buffer +function appendToolChunk( + buffer: string[], + options: { + toolName: string; + isComplete: boolean; + toolCallId: string; + input?: unknown; + output?: unknown; + } +): void { + buffer.push( + JSON.stringify({ + type: "tool", + toolName: options.toolName, + input: options.input, + output: options.output, + isComplete: options.isComplete, + toolCallId: options.toolCallId, + } as ToolChunkGroup) + ); +} + +// Helper: Extract completed step names from a LangGraph checkpoint +function extractCompletedStepsFromCheckpoint( + checkpoint: typeof GraphState.State | null | undefined +): string[] { + const completedSteps: string[] = []; + + const pastSteps = (checkpoint as any)?.pastSteps as + | Array<[string, unknown[]]> + | undefined; + if (pastSteps && pastSteps.length > 0) { + completedSteps.push(...pastSteps.map((ps) => ps[0])); + } + + const plan = (checkpoint as any)?.plan as + | Array< + | { + type: "parallel"; + data: Array<{ step: string; context: string }>; + } + | { type: "single"; data: { step: string; context: string } } + > + | undefined; + if (plan && plan.length > 0) { + const first = plan[0]; + if (first.type === "parallel") { + completedSteps.push(...first.data.map((s) => s.step)); + } else { + completedSteps.push(first.data.step); + } + } + + return completedSteps; +} + export const generateTitle = internalAction({ args: v.object({ chat: Chats.doc, @@ -105,20 +162,8 @@ export const chat = action({ { chatId, chunks, - completedSteps: [ - ...(localCheckpoint?.pastSteps?.map( - (pastStep) => pastStep[0] - ) ?? []), - ...(localCheckpoint?.plan && localCheckpoint.plan.length > 0 - ? [ - ...(localCheckpoint.plan[0].type === "parallel" - ? localCheckpoint.plan[0].data.map( - (step) => step.step - ) - : [localCheckpoint.plan[0].data.step]), - ] - : []), - ], + completedSteps: + extractCompletedStepsFromCheckpoint(localCheckpoint), } ); } @@ -158,25 +203,19 @@ export const chat = action({ } as AIChunkGroup) ); } else if (evt.event === "on_tool_start") { - buffer.push( - JSON.stringify({ - type: "tool", - toolName: evt.name, - input: evt.data?.input, - isComplete: false, - toolCallId: evt.run_id, - } as ToolChunkGroup) - ); + appendToolChunk(buffer, { + toolName: evt.name, + input: evt.data?.input, + isComplete: false, + toolCallId: evt.run_id, + }); } else if (evt.event === "on_tool_stream") { - buffer.push( - JSON.stringify({ - type: "tool", - toolName: evt.name, - output: evt.data?.chunk, - isComplete: false, - toolCallId: evt.run_id, - } as ToolChunkGroup) - ); + appendToolChunk(buffer, { + toolName: evt.name, + output: evt.data?.chunk, + isComplete: false, + toolCallId: evt.run_id, + }); } else if (evt.event === "on_tool_end") { let output = evt.data?.output?.content ?? evt.data?.output; @@ -200,16 +239,13 @@ export const chat = action({ ); } - buffer.push( - JSON.stringify({ - type: "tool", - toolName: evt.name, - input: evt.data?.input, - output, - isComplete: true, - toolCallId: evt.run_id, - } as ToolChunkGroup) - ); + appendToolChunk(buffer, { + toolName: evt.name, + input: evt.data?.input, + output, + isComplete: true, + toolCallId: evt.run_id, + }); } else if (evt.event === "on_custom_event") { try { const eventName = @@ -226,15 +262,12 @@ export const chat = action({ : undefined; const isComplete = payload?.complete === true; if (eventName === "tool_stream" && chunk) { - buffer.push( - JSON.stringify({ - type: "tool", - toolName: evt.name, - output: chunk, - isComplete, - toolCallId: evt.run_id, - } as ToolChunkGroup) - ); + appendToolChunk(buffer, { + toolName: evt.name, + output: chunk, + isComplete, + toolCallId: evt.run_id, + }); } } catch { // ignore malformed custom events @@ -252,30 +285,8 @@ export const chat = action({ if (buffer.length > 0) { const chunks = buffer; buffer = []; - const completedSteps: string[] = []; - const pastSteps = (localCheckpoint as any)?.pastSteps as - | Array<[string, unknown[]]> - | undefined; - if (pastSteps && pastSteps.length > 0) { - completedSteps.push(...pastSteps.map((ps) => ps[0])); - } - const plan = (localCheckpoint as any)?.plan as - | Array< - | { - type: "parallel"; - data: Array<{ step: string; context: string }>; - } - | { type: "single"; data: { step: string; context: string } } - > - | undefined; - if (plan && plan.length > 0) { - const first = plan[0]; - if (first.type === "parallel") { - completedSteps.push(...first.data.map((s) => s.step)); - } else { - completedSteps.push(first.data.step); - } - } + const completedSteps = + extractCompletedStepsFromCheckpoint(localCheckpoint); await ctx.runMutation(internal.streams.mutations.flush, { chatId, chunks, diff --git a/convex/langchain/tools/retrievalTools.ts b/convex/langchain/tools/retrievalTools.ts index f007bd08..6c94df3d 100644 --- a/convex/langchain/tools/retrievalTools.ts +++ b/convex/langchain/tools/retrievalTools.ts @@ -36,7 +36,7 @@ export const getRetrievalTools = async ( await dispatchCustomEvent( "tool_stream", - { chunk: "Loading selected project documents…" }, + { chunk: "Loading selected project documents..." }, toolConfig ); const includedProjectDocuments = await config.ctx.runQuery( @@ -59,7 +59,7 @@ export const getRetrievalTools = async ( await dispatchCustomEvent( "tool_stream", - { chunk: "Searching vector index…" }, + { chunk: "Searching vector index..." }, toolConfig ); const results = await vectorStore.similaritySearch(query, limit, { @@ -75,13 +75,18 @@ export const getRetrievalTools = async ( await dispatchCustomEvent( "tool_stream", - { chunk: `Found ${results.length} results. Building response…` }, + { chunk: `Found ${results.length} results. Building response...` }, toolConfig ); const documentsMap = new Map, Doc<"documents">>(); - includedProjectDocuments.forEach((projectDocument) => - documentsMap.set(projectDocument.documentId, projectDocument.document!) - ); + includedProjectDocuments.forEach((projectDocument) => { + if (projectDocument.document) { + documentsMap.set( + projectDocument.documentId, + projectDocument.document + ); + } + }); const documents = await Promise.all( results.map(async (doc) => { @@ -104,7 +109,7 @@ export const getRetrievalTools = async ( await dispatchCustomEvent( "tool_stream", - { chunk: "Formatting final output…" }, + { chunk: "Formatting final output...", complete: true }, toolConfig ); return returnString ? JSON.stringify(documents, null, 0) : documents; @@ -135,7 +140,7 @@ export const getRetrievalTools = async ( ) => { await dispatchCustomEvent( "tool_stream", - { chunk: "Preparing web search…" }, + { chunk: "Preparing web search..." }, toolConfig ); const EXA_API_KEY = @@ -150,7 +155,7 @@ export const getRetrievalTools = async ( try { await dispatchCustomEvent( "tool_stream", - { chunk: "Querying Exa…" }, + { chunk: "Querying Exa..." }, toolConfig ); const exa = new Exa(EXA_API_KEY, undefined); @@ -159,7 +164,7 @@ export const getRetrievalTools = async ( numResults: 5, type: "auto", useAutoprompt: false, - topic, + topic: topic ?? undefined, text: true, }) ).results; @@ -176,7 +181,10 @@ export const getRetrievalTools = async ( await dispatchCustomEvent( "tool_stream", - { chunk: `Found ${searchResponse.length} results. Formatting…` }, + { + chunk: `Found ${searchResponse.length} results. Formatting...`, + complete: true, + }, toolConfig ); const documents = searchResponse.map((result) => { diff --git a/src/components/ui/tool-accoordion.tsx b/src/components/ui/tool-accoordion.tsx index aa92a86c..b0692cdf 100644 --- a/src/components/ui/tool-accoordion.tsx +++ b/src/components/ui/tool-accoordion.tsx @@ -1,4 +1,4 @@ -import { Check, HammerIcon, Loader2 } from "lucide-react"; +import { Check, Hammer, Loader2 } from "lucide-react"; import { Accordion, AccordionItem, @@ -34,7 +34,7 @@ function ToolAccordion({ className="px-3 py-2 text-sm items-center justify-start hover:bg-muted transition-colors rounded-md gap-2" >
- + {cleanedName} {statusText && ( From b40b3ad717fb86da96f059d3f5ec16ff055ef1d8 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Thu, 14 Aug 2025 05:42:15 +0530 Subject: [PATCH 08/25] refactor: clean up component structure and improve styling - Removed unnecessary divider from the ToolBar component for a cleaner layout. - Enhanced code readability in ToolkitToggles by formatting and restructuring logic. - Updated text color in ChatMessages for better visual consistency. - Corrected Discord link in feedback settings to point to the appropriate channel. --- src/components/chat/input/toolbar/index.tsx | 1 - .../chat/input/toolbar/toolkit-toggles.tsx | 90 +++++++++++++------ src/components/chat/messages/index.tsx | 4 +- src/routes/settings/feedback.tsx | 2 +- 4 files changed, 66 insertions(+), 31 deletions(-) diff --git a/src/components/chat/input/toolbar/index.tsx b/src/components/chat/input/toolbar/index.tsx index d716468a..4f69e763 100644 --- a/src/components/chat/input/toolbar/index.tsx +++ b/src/components/chat/input/toolbar/index.tsx @@ -147,7 +147,6 @@ export const ToolBar = () => { -
{/* Render project name with X button on hover */} {project && ( diff --git a/src/components/chat/input/toolbar/toolkit-toggles.tsx b/src/components/chat/input/toolbar/toolkit-toggles.tsx index 68c02245..33ee5c54 100644 --- a/src/components/chat/input/toolbar/toolkit-toggles.tsx +++ b/src/components/chat/input/toolbar/toolkit-toggles.tsx @@ -23,18 +23,24 @@ export function ToolkitToggles() { const chat = useAtomValue(chatAtom)!; const setNewChat = useSetAtom(newChatAtom); const existingKeys = useAtomValue(apiKeysAtom); - const existingKeySet = useMemo(() => new Set((existingKeys ?? []).map((k) => k.key)), [existingKeys]); + const existingKeySet = useMemo( + () => new Set((existingKeys ?? []).map((k) => k.key)), + [existingKeys] + ); const navigate = useNavigate(); const connectedProviders = useMemo(() => { const result: ProviderKey[] = []; - (Object.entries(providers) as Array<[ProviderKey, (typeof providers)[ProviderKey]]>).forEach( - ([p, cfg]) => { - const access = cfg.accessKeyKey; - const refresh = cfg.refreshKeyKey; - if (existingKeySet.has(access) || existingKeySet.has(refresh)) result.push(p); - }, - ); + ( + Object.entries(providers) as Array< + [ProviderKey, (typeof providers)[ProviderKey]] + > + ).forEach(([p, cfg]) => { + const access = cfg.accessKeyKey; + const refresh = cfg.refreshKeyKey; + if (existingKeySet.has(access) || existingKeySet.has(refresh)) + result.push(p); + }); return result; }, [existingKeySet]); @@ -42,16 +48,24 @@ export function ToolkitToggles() { mutationFn: useConvexMutation(api.chats.mutations.update), }); - const isEnabled = (p: ProviderKey) => (chat.enabledToolkits || []).includes(p as string); + const isEnabled = (p: ProviderKey) => + (chat.enabledToolkits || []).includes(p as string); const toggleProvider = (p: ProviderKey, enable: boolean) => { const next = new Set(chat.enabledToolkits || []); - if (enable) next.add(p as string); else next.delete(p as string); + if (enable) next.add(p as string); + else next.delete(p as string); if (chatId === "new") { - setNewChat((prev) => ({ ...prev, enabledToolkits: Array.from(next) as string[] })); + setNewChat((prev) => ({ + ...prev, + enabledToolkits: Array.from(next) as string[], + })); } else { - updateChatMutation({ chatId, updates: { enabledToolkits: Array.from(next) } }); + updateChatMutation({ + chatId, + updates: { enabledToolkits: Array.from(next) }, + }); } }; @@ -59,12 +73,19 @@ export function ToolkitToggles() { <> - -
Toolkits
+
+ Toolkits +
{connectedProviders.map((p) => ( - {providers[p].title} + {providers[p].title} {providers[p].title} {isEnabled(p) && ( - + ))} - navigate({ to: "/settings/integrations" })} className="bg-accent/50"> + navigate({ to: "/settings/integrations" })} + className="bg-accent/50" + > Add Integrations @@ -97,16 +129,20 @@ export function ToolkitToggles() { {(chat.enabledToolkits || []) .filter((p) => connectedProviders.includes(p as ProviderKey)) .map((p) => ( - - ))} + + ))} ); } diff --git a/src/components/chat/messages/index.tsx b/src/components/chat/messages/index.tsx index fdf1cd5d..115fe676 100644 --- a/src/components/chat/messages/index.tsx +++ b/src/components/chat/messages/index.tsx @@ -23,13 +23,13 @@ export const ChatMessages = ({ chatId }: { chatId: Id<"chats"> | "new" }) => { return (
how can i help you, - {userName} ? + {userName} ?
); diff --git a/src/routes/settings/feedback.tsx b/src/routes/settings/feedback.tsx index 0889223b..e08680a9 100644 --- a/src/routes/settings/feedback.tsx +++ b/src/routes/settings/feedback.tsx @@ -11,7 +11,7 @@ function RouteComponent() {
Date: Thu, 14 Aug 2025 06:50:03 +0530 Subject: [PATCH 09/25] refactor: enhance event handling and improve component structure - Streamlined custom event handling in chat actions for better data validation and error handling. - Updated ChatMessages component styling for improved layout consistency. - Refactored feedback settings to use constants for URLs, enhancing maintainability and readability. --- convex/langchain/index.ts | 86 +++++++++++++++++++------- src/components/chat/messages/index.tsx | 2 +- src/routes/settings/feedback.tsx | 10 ++- temp.md | 4 +- 4 files changed, 73 insertions(+), 29 deletions(-) diff --git a/convex/langchain/index.ts b/convex/langchain/index.ts index 3adeba11..6adc3303 100644 --- a/convex/langchain/index.ts +++ b/convex/langchain/index.ts @@ -247,30 +247,72 @@ export const chat = action({ toolCallId: evt.run_id, }); } else if (evt.event === "on_custom_event") { - try { - const eventName = - (evt as any)?.data?.event ?? - (evt as any)?.data?.name ?? - (evt as any)?.name; - const payload = - (evt as any)?.data?.data ?? - (evt as any)?.data?.payload ?? - (evt as any)?.data; - const chunk = - typeof payload?.chunk === "string" - ? payload.chunk + const raw: any = evt; + const data = raw?.data; + if (!data || typeof data !== "object") { + console.warn( + "[stream] Ignoring custom event without object data", + { + event: raw?.event, + name: raw?.name, + } + ); + return; + } + + const eventName = + typeof (data as any).event === "string" + ? (data as any).event + : typeof (data as any).name === "string" + ? (data as any).name : undefined; - const isComplete = payload?.complete === true; - if (eventName === "tool_stream" && chunk) { - appendToolChunk(buffer, { - toolName: evt.name, - output: chunk, - isComplete, - toolCallId: evt.run_id, - }); + + if (!eventName) { + console.warn( + "[stream] Ignoring custom event with missing name", + { + name: raw?.name, + } + ); + return; + } + + const payloadCandidate = + (data as any).data ?? (data as any).payload; + const payload = + payloadCandidate && typeof payloadCandidate === "object" + ? payloadCandidate + : undefined; + if (!payload) { + console.warn( + "[stream] Ignoring custom event without object payload", + { + eventName, + } + ); + return; + } + + const chunk = + typeof (payload as any).chunk === "string" + ? (payload as any).chunk + : undefined; + const isComplete = (payload as any).complete === true; + + if (eventName === "tool_stream") { + if (!chunk) { + console.warn( + "[stream] tool_stream custom event missing chunk; skipping" + ); + return; } - } catch { - // ignore malformed custom events + appendToolChunk(buffer, { + toolName: + typeof raw?.name === "string" ? raw.name : "custom_event", + output: chunk, + isComplete, + toolCallId: raw.run_id, + }); } } } diff --git a/src/components/chat/messages/index.tsx b/src/components/chat/messages/index.tsx index 115fe676..3bfd008f 100644 --- a/src/components/chat/messages/index.tsx +++ b/src/components/chat/messages/index.tsx @@ -21,7 +21,7 @@ export const ChatMessages = ({ chatId }: { chatId: Id<"chats"> | "new" }) => { userLoadable.state === "hasData" ? userLoadable.data?.name : ""; return ( -
+
- +
@@ -43,7 +47,7 @@ function RouteComponent() { convex cloud migration -> streaming tool calls -> convex oauth integration -> revamp mcp templates to pass along the env vars) custom ui for vibz mcp. (like artifacts, we will replace the panel content with the ui for vibz)(preview, dashboard (convex dashboard), code (vs code)) From c4f71f2db0bb9971542b3d1c0d879d4d07e47407 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Thu, 14 Aug 2025 15:40:43 +0530 Subject: [PATCH 10/25] refactor: optimize checkpoint handling and improve code clarity in chat actions - Introduced a mechanism to defer checkpoint refresh until necessary, enhancing performance during chat processing. - Updated comments for better clarity on checkpoint management and event handling. - Cleaned up formatting in state validation schemas for improved readability. --- convex/langchain/index.ts | 31 ++++++++++++++++++++++++++----- convex/langchain/state.ts | 6 +++--- src/hooks/chats/use-stream.ts | 1 + src/routes/settings/apiKeys.tsx | 24 ++++++++++++++++++++---- 4 files changed, 50 insertions(+), 12 deletions(-) diff --git a/convex/langchain/index.ts b/convex/langchain/index.ts index 6adc3303..c8fa849e 100644 --- a/convex/langchain/index.ts +++ b/convex/langchain/index.ts @@ -146,6 +146,7 @@ export const chat = action({ let buffer: string[] = []; let checkpoint: typeof GraphState.State | null = null; let finished = false; + let pendingCheckpointRefresh = true; // Track when we actually need a checkpoint refresh const flushAndStream = async (): Promise< typeof GraphState.State | null @@ -155,6 +156,13 @@ export const chat = action({ const flusher = async () => { while (!finished) { if (buffer.length > 0) { + if (pendingCheckpointRefresh) { + // Fetch checkpoint lazily only when we are about to flush + localCheckpoint = ( + await agent.getState({ configurable: { thread_id: chatId } }) + ).values as typeof GraphState.State; + pendingCheckpointRefresh = false; + } const chunks = buffer; buffer = []; streamDoc = await ctx.runMutation( @@ -181,11 +189,7 @@ export const chat = action({ if (abort.signal.aborted) { return; } - localCheckpoint = ( - await agent.getState({ - configurable: { thread_id: chatId }, - }) - ).values as typeof GraphState.State; + // defer checkpoint refresh; will be done just-in-time before flush const allowedNodes = ["baseAgent", "simple", "plannerAgent"]; if ( @@ -246,6 +250,14 @@ export const chat = action({ isComplete: true, toolCallId: evt.run_id, }); + // Mark that the checkpoint should be refreshed soon + pendingCheckpointRefresh = true; + } else if ( + evt.event === "on_chat_model_end" || + evt.event === "on_chain_end" + ) { + // Model or chain finished a unit of work; refresh on next flush + pendingCheckpointRefresh = true; } else if (evt.event === "on_custom_event") { const raw: any = evt; const data = raw?.data; @@ -313,6 +325,10 @@ export const chat = action({ isComplete, toolCallId: raw.run_id, }); + if (isComplete) { + // Custom tool stream finished; refresh on next flush + pendingCheckpointRefresh = true; + } } } } @@ -325,6 +341,11 @@ export const chat = action({ await Promise.all([flusher(), streamer()]); // Final flush in case there are any buffered chunks left if (buffer.length > 0) { + // Always refresh checkpoint before final flush to ensure accuracy + localCheckpoint = ( + await agent.getState({ configurable: { thread_id: chatId } }) + ).values as typeof GraphState.State; + pendingCheckpointRefresh = false; const chunks = buffer; buffer = []; const completedSteps = diff --git a/convex/langchain/state.ts b/convex/langchain/state.ts index 53a4a1a6..ea232608 100644 --- a/convex/langchain/state.ts +++ b/convex/langchain/state.ts @@ -8,7 +8,7 @@ export const planStep = z.object({ .describe( "A short, specific instruction (ideally < 6 words) describing the subtask to be performed " + "by an agent. Should be actionable, unambiguous, and clearly distinct from other steps to ensure effective division " + - "of labor and prevent overlap.", + "of labor and prevent overlap." ), context: z .string() @@ -16,7 +16,7 @@ export const planStep = z.object({ "A concise explanation of the background, objective, and constraints for this step," + "written to help a subagent understand exactly what is needed, what tools or sources to use, and any boundaries or" + " heuristics to follow. Should clarify the subtask's purpose, avoid ambiguity, and prevent duplication or" + - " misinterpretation by other agents.", + " misinterpretation by other agents." ), }); @@ -36,7 +36,7 @@ export const planArray = z "A step-by-step plan for decomposing a complex research objective into clear, non-overlapping subtasks. " + "Each step should be concise, actionable, and include enough context for a subagent to execute independently. " + "The plan should scale in complexity with the query, allocate effort efficiently, and ensure that all necessary aspects of the research are covered without redundancy. " + - "If multiple tasks should be executed in parallel, group them together in a nested list (i.e., use an array of plan steps within the main array) to indicate parallel execution.", + "If multiple tasks should be executed in parallel, group them together in a nested list (i.e., use an array of plan steps within the main array) to indicate parallel execution." ) .min(1) .max(9); diff --git a/src/hooks/chats/use-stream.ts b/src/hooks/chats/use-stream.ts index 5ad0a2a6..75acef19 100644 --- a/src/hooks/chats/use-stream.ts +++ b/src/hooks/chats/use-stream.ts @@ -50,6 +50,7 @@ export function useStream(chatId: Id<"chats"> | "new") { lastChunkTime: lastSeenTime, paginationOpts: { numItems: 200, cursor: null }, }); + console.log("result", result); if (!isMounted || !result?.chunks?.page?.length) return; // Only process chunks newer than lastSeenTime const newEvents: ChunkGroup[] = result.chunks.page diff --git a/src/routes/settings/apiKeys.tsx b/src/routes/settings/apiKeys.tsx index 97dbb5ea..3b5e07c4 100644 --- a/src/routes/settings/apiKeys.tsx +++ b/src/routes/settings/apiKeys.tsx @@ -24,22 +24,38 @@ import { apiKeysAtom } from "@/hooks/use-apikeys"; const Icons = { OpenAI: () => (
- OpenAI + OpenAI
), Google: () => (
- Google + Google
), Exa: () => (
- Exa + Exa
), OpenRouter: () => (
- OpenRouter + OpenRouter
), }; From d1b4bad9fd0aeb9e18b728c47b224eaac80c8ab7 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Fri, 15 Aug 2025 03:57:29 +0530 Subject: [PATCH 11/25] refactor: improve code formatting and enhance response handling in agent functions - Cleaned up function signatures by removing unnecessary trailing commas for consistency. - Enhanced response handling in the replanner function to normalize various output shapes into a single string. - Updated prompt templates to include explicit schema definitions, reducing model output errors. - Improved AI message component to ensure content is a string before parsing, enhancing robustness. --- convex/langchain/agent.ts | 69 ++++++++++++------- convex/langchain/prompts.ts | 41 ++++++++++- .../chat/messages/ai-message/ai-message.tsx | 10 ++- .../chat/messages/ai-message/reasoning.tsx | 6 +- 4 files changed, 91 insertions(+), 35 deletions(-) diff --git a/convex/langchain/agent.ts b/convex/langchain/agent.ts index ad96d37d..e6e1abd5 100644 --- a/convex/langchain/agent.ts +++ b/convex/langchain/agent.ts @@ -25,7 +25,7 @@ import { END, START, StateGraph } from "@langchain/langgraph/web"; async function shouldPlanOrAgentOrSimple( _state: typeof GraphState.State, - config: RunnableConfig, + config: RunnableConfig ) { const formattedConfig = config.configurable as ExtendedRunnableConfig; if (!modelSupportsTools(formattedConfig.chat.model!)) { @@ -46,14 +46,14 @@ async function simple(state: typeof GraphState.State, config: RunnableConfig) { const formattedMessages = await formatMessages( formattedConfig.ctx, state.messages, - formattedConfig.chat.model!, + formattedConfig.chat.model! ); const response = await chain.invoke( { messages: formattedMessages, }, - config, + config ); return { @@ -63,7 +63,7 @@ async function simple(state: typeof GraphState.State, config: RunnableConfig) { async function baseAgent( state: typeof GraphState.State, - config: RunnableConfig, + config: RunnableConfig ) { const formattedConfig = config.configurable as ExtendedRunnableConfig; @@ -71,18 +71,18 @@ async function baseAgent( const formattedMessages = await formatMessages( formattedConfig.ctx, state.messages, - formattedConfig.chat.model!, + formattedConfig.chat.model! ); const response = await chain.invoke( { messages: formattedMessages, }, - config, + config ); let newMessages = response.messages.slice( - formattedMessages.length, + formattedMessages.length ) as BaseMessage[]; return { @@ -94,39 +94,39 @@ async function planner(state: typeof GraphState.State, config: RunnableConfig) { const formattedConfig = config.configurable as ExtendedRunnableConfig; const availableToolsDescription = await getAvailableToolsDescription( state, - formattedConfig, + formattedConfig ); const promptTemplate = createPlannerPrompt(availableToolsDescription); const model = await getModel( formattedConfig.ctx, formattedConfig.chat.model!, - formattedConfig.chat.reasoningEffort, + formattedConfig.chat.reasoningEffort ); // Get model config to check if it's anthropic const modelConfig = models.find( - (m) => m.model_name === formattedConfig.chat.model!, + (m) => m.model_name === formattedConfig.chat.model! ); const isFunctionCallingParser = modelConfig?.parser === "functionCalling"; const modelWithOutputParser = promptTemplate.pipe( isFunctionCallingParser ? model.withStructuredOutput(planSchema, { method: "functionCalling" }) - : model.withStructuredOutput(planSchema), + : model.withStructuredOutput(planSchema) ); const formattedMessages = await formatMessages( formattedConfig.ctx, state.messages, - formattedConfig.chat.model!, + formattedConfig.chat.model! ); const response = (await modelWithOutputParser.invoke( { messages: formattedMessages, }, - config, + config )) as z.infer; return { @@ -136,7 +136,7 @@ async function planner(state: typeof GraphState.State, config: RunnableConfig) { async function plannerAgent( state: typeof GraphState.State, - config: RunnableConfig, + config: RunnableConfig ) { const formattedConfig = config.configurable as ExtendedRunnableConfig; @@ -151,7 +151,7 @@ async function plannerAgent( const plannerAgentChain = await createAgentWithTools( state, formattedConfig, - true, + true ); const invoke = async ({ planItem }: { planItem: typeof currentPlanItem }) => { @@ -160,11 +160,11 @@ async function plannerAgent( { messages: [ new HumanMessage( - `Task: ${planItem.data.step}\nContext: ${planItem.data.context}`, + `Task: ${planItem.data.step}\nContext: ${planItem.data.context}` ), ], }, - config, + config ); const newMessages = response.messages.slice(1, response.messages.length); @@ -189,7 +189,7 @@ async function plannerAgent( return await invoke({ planItem: { type: "single" as const, data: planStep }, }); - }), + }) ); return { @@ -201,37 +201,37 @@ async function plannerAgent( async function replanner( state: typeof GraphState.State, - config: RunnableConfig, + config: RunnableConfig ) { const formattedConfig = config.configurable as ExtendedRunnableConfig; const availableToolsDescription = await getAvailableToolsDescription( state, - formattedConfig, + formattedConfig ); const promptTemplate = createReplannerPrompt(availableToolsDescription); const model = await getModel( formattedConfig.ctx, formattedConfig.chat.model!, - formattedConfig.chat.reasoningEffort, + formattedConfig.chat.reasoningEffort ); // Get model config to check if it's anthropic const outputSchema = replannerOutputSchema(formattedConfig.chat.artifacts); const modelConfig = models.find( - (m) => m.model_name === formattedConfig.chat.model!, + (m) => m.model_name === formattedConfig.chat.model! ); const isFunctionCallingParser = modelConfig?.parser === "functionCalling"; const modelWithOutputParser = promptTemplate.pipe( isFunctionCallingParser ? model.withStructuredOutput(outputSchema, { method: "functionCalling" }) - : model.withStructuredOutput(outputSchema), + : model.withStructuredOutput(outputSchema) ); const formattedMessages = await formatMessages( formattedConfig.ctx, state.messages, - formattedConfig.chat.model!, + formattedConfig.chat.model! ); const response = (await modelWithOutputParser.invoke( @@ -246,13 +246,30 @@ async function replanner( }) .flat(), }, - config, + config )) as z.infer; if (response.type === "respond_to_user") { + // Normalize various shapes into a single string for the final answer. + const raw = response.data; + let finalText: string; + if (typeof raw === "string") { + finalText = raw; + } else if (Array.isArray(raw)) { + const first = raw[0] as any; + finalText = + (first?.data?.context as string | undefined) ?? + (first?.data?.step as string | undefined) ?? + JSON.stringify(first ?? raw); + } else if (raw && typeof raw === "object") { + finalText = raw.context ?? raw.step ?? JSON.stringify(raw); + } else { + finalText = String(raw); + } + const responseMessages = [ new AIMessage({ - content: response.data as string, + content: finalText, additional_kwargs: { pastSteps: state.pastSteps.map((pastStep) => { const [step, messages] = pastStep; diff --git a/convex/langchain/prompts.ts b/convex/langchain/prompts.ts index 855b1890..c0a268a6 100644 --- a/convex/langchain/prompts.ts +++ b/convex/langchain/prompts.ts @@ -229,12 +229,30 @@ Respond with the JSON ONLY.` // Prompt template for replanner export function createReplannerPrompt(availableToolsDescription: string) { + // Explicit schema block to reduce model output errors by clearly + // specifying the exact JSON structure that must be returned. + const schemaSection = String.raw` ++--------------- REQUIRED SCHEMA ----------------- +type planStep = { step: string; context: string }; + +type PlanItem = + | { type: "single"; data: planStep } + | { type: "parallel"; data: planStep[] }; + +// Replanner response must match one of the following shapes exactly +type ReplanResponse = + | { type: "continue_planning"; data: PlanItem[] } + | { type: "respond_to_user"; data: string }; ++---------------------------------------------------`; + const toolsSection = `\n**Available Tools:**\n${availableToolsDescription}\n\nWhen planning remaining steps, consider which tools are available and how they can be used to accomplish the remaining objectives efficiently.`; return ChatPromptTemplate.fromMessages([ new SystemMessage( String.raw`## Your Task: Reflect and Re-plan +${schemaSection} + For the given objective, update the step-by-step plan using the planStep and planArray schema conventions. - Only include the remaining steps needed to fill the gaps identified in your analysis. - Use the discriminated union format with nested arrays for parallel execution. @@ -271,6 +289,19 @@ ${toolsSection} ]); } +// Helper schemas to accept alternative respond_to_user shapes for robustness. +const respondPayloadObject = z.object({ + step: z.string(), + context: z.string(), +}); + +const nestedRespondArray = z.array( + z.object({ + type: z.literal("respond_to_user"), + data: respondPayloadObject, + }) +); + export const replannerOutputSchema = (artifacts: boolean) => z.object({ type: z @@ -284,11 +315,15 @@ export const replannerOutputSchema = (artifacts: boolean) => .describe( "The final, complete, and user-facing response, to be used ONLY when 'type' is 'respond_to_user'. " + "This string MUST synthesize all gathered information and results from the previous steps into a single, " + - "coherent, and well-formatted answer. This is the ONLY output the end-user will see. It must fully and directly " + - "address the user's original query, leaving no questions unanswered. Do not include any conversational filler, " + - "apologies, or meta-commentary about the process; provide only the definitive answer." + + "coherent, and well-formatted answer. Do not include any conversational filler." + `${artifacts ? ` Adhere to the following additional guidelines and format your response accordingly:\n${artifactsGuidelines}` : ""}` ), + respondPayloadObject.describe( + "Alternate object shape: { step, context }. 'context' treated as final answer." + ), + nestedRespondArray.describe( + "Alternate nested array shape: [{ type: 'respond_to_user', data: { step, context } }]. 'context' taken from first entry." + ), ]) .describe("The response data - either a plan array or a string response"), }); diff --git a/src/components/chat/messages/ai-message/ai-message.tsx b/src/components/chat/messages/ai-message/ai-message.tsx index a4adf45d..a216ca93 100644 --- a/src/components/chat/messages/ai-message/ai-message.tsx +++ b/src/components/chat/messages/ai-message/ai-message.tsx @@ -27,7 +27,13 @@ export const AiMessageContent = memo( if (type !== "ai") { return []; } - const parsed = parseContent(content as string); + if (Array.isArray(content)) { + return []; + } + // Ensure content is a string before parsing + const contentString = + typeof content === "string" ? content : String(content); + const parsed = parseContent(contentString); return parsed; }, [content, type]); @@ -84,7 +90,7 @@ export const AiMessageContent = memo( }, [type, reasoning, messageId, renderedContent, message, className]); return <>{messageContent}; - }, + } ); AiMessageContent.displayName = "AiMessageContent"; diff --git a/src/components/chat/messages/ai-message/reasoning.tsx b/src/components/chat/messages/ai-message/reasoning.tsx index 57a96dba..829dadd8 100644 --- a/src/components/chat/messages/ai-message/reasoning.tsx +++ b/src/components/chat/messages/ai-message/reasoning.tsx @@ -22,9 +22,7 @@ export const Reasoning = memo( - - Reasoning - + Reasoning {isStreaming && ( )} @@ -39,7 +37,7 @@ export const Reasoning = memo( ); - }, + } ); Reasoning.displayName = "Reasoning"; From 6cdcab0559ae691c36d4fac3b14bc9a733303894 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Fri, 15 Aug 2025 07:20:41 +0530 Subject: [PATCH 12/25] refactor: enhance replanner error handling and improve prompt schema documentation - Added error handling in the replanner function to manage structured output parsing failures and provide fallback responses. - Updated prompt schema documentation to clarify requirements for "single" and "parallel" plan items, ensuring correct data structures. - Improved AI message component to ensure content is processed correctly, enhancing overall robustness. --- convex/langchain/agent.ts | 185 +++++++++++++++--- convex/langchain/prompts.ts | 13 ++ .../chat/messages/ai-message/ai-message.tsx | 2 - src/hooks/chats/use-stream.ts | 1 - 4 files changed, 176 insertions(+), 25 deletions(-) diff --git a/convex/langchain/agent.ts b/convex/langchain/agent.ts index e6e1abd5..b26f5a20 100644 --- a/convex/langchain/agent.ts +++ b/convex/langchain/agent.ts @@ -234,20 +234,167 @@ async function replanner( formattedConfig.chat.model! ); - const response = (await modelWithOutputParser.invoke( - { - messages: formattedMessages, - plan: state.plan, - pastSteps: state.pastSteps - .map((pastStep) => { - const [step, messages] = pastStep; - const stepMessage = new HumanMessage(step as string); - return [stepMessage, ...messages]; - }) - .flat(), - }, - config - )) as z.infer; + let response: z.infer; + try { + response = (await modelWithOutputParser.invoke( + { + messages: formattedMessages, + plan: state.plan, + pastSteps: state.pastSteps + .map((pastStep) => { + const [step, messages] = pastStep; + const stepMessage = new HumanMessage(step as string); + return [stepMessage, ...messages]; + }) + .flat(), + }, + config + )) as z.infer; + } catch (error) { + // Structured output parsing failed — attempt raw generation and coerce + console.error("Replanner structured output parsing failed:", error); + + try { + const rawChain = promptTemplate.pipe(model); + const rawResponse = await rawChain.invoke( + { + messages: formattedMessages, + plan: state.plan, + pastSteps: state.pastSteps + .map((pastStep) => { + const [step, messages] = pastStep; + const stepMessage = new HumanMessage(step as string); + return [stepMessage, ...messages]; + }) + .flat(), + }, + config + ); + + const rawText = + typeof rawResponse.content === "string" + ? rawResponse.content + : JSON.stringify(rawResponse.content); + + let parsedResponse: any; + try { + parsedResponse = JSON.parse(rawText); + } catch { + // Ultimate fallback: return a graceful error message + return { + messages: [ + new AIMessage({ + content: + "I encountered an error while processing your request. Please try again.", + additional_kwargs: { + pastSteps: state.pastSteps.map((pastStep) => { + const [step, messages] = pastStep; + const storedMessages = + mapChatMessagesToStoredMessages(messages); + return [step, storedMessages]; + }), + }, + }), + ], + plan: [], + pastSteps: [], + }; + } + + // Coerce malformed continue_planning items + if ( + parsedResponse?.type === "continue_planning" && + Array.isArray(parsedResponse.data) + ) { + const fixedData = parsedResponse.data.map((item: any) => { + if (item.type === "single" && Array.isArray(item.data)) { + const stepText = item.data[0] || "Complete task"; + return { + type: "single", + data: { + step: + stepText.length > 50 ? stepText.substring(0, 50) : stepText, + context: stepText, + }, + }; + } else if (item.type === "parallel" && Array.isArray(item.data)) { + const fixedParallelData = item.data.map((parallelItem: any) => { + if (typeof parallelItem === "string") { + return { + step: + parallelItem.length > 50 + ? parallelItem.substring(0, 50) + : parallelItem, + context: parallelItem, + }; + } else if (parallelItem && typeof parallelItem === "object") { + return { + step: + parallelItem.step || + parallelItem.context || + "Complete task", + context: + parallelItem.context || + parallelItem.step || + "Complete the assigned task", + }; + } + return { + step: "Complete task", + context: "Complete the assigned task", + }; + }); + return { type: "parallel", data: fixedParallelData }; + } + return item; + }); + parsedResponse.data = fixedData; + } + + // Coerce malformed respond_to_user arrays (e.g. [{type:'tool_code', data:{step, context}}, ...]) + if ( + parsedResponse?.type === "respond_to_user" && + Array.isArray(parsedResponse.data) + ) { + const pieces = parsedResponse.data + .map( + (d: any) => + d?.data?.context ?? + d?.data?.step ?? + (typeof d === "string" ? d : null) + ) + .filter(Boolean); + parsedResponse = { + type: "respond_to_user", + data: pieces.length + ? pieces.join("\n\n") + : JSON.stringify(parsedResponse.data), + }; + } + + response = parsedResponse as z.infer; + } catch (fallbackError) { + console.error("Fallback response coercion failed:", fallbackError); + return { + messages: [ + new AIMessage({ + content: + "I encountered an error while processing your request. Please try again.", + additional_kwargs: { + pastSteps: state.pastSteps.map((pastStep) => { + const [step, messages] = pastStep; + const storedMessages = + mapChatMessagesToStoredMessages(messages); + return [step, storedMessages]; + }), + }, + }), + ], + plan: [], + pastSteps: [], + }; + } + } if (response.type === "respond_to_user") { // Normalize various shapes into a single string for the final answer. @@ -279,15 +426,9 @@ async function replanner( }, }), ]; - return { - messages: responseMessages, - plan: [], - pastSteps: [], - }; + return { messages: responseMessages, plan: [], pastSteps: [] }; } else if (response.type === "continue_planning") { - return { - plan: response.data as z.infer, - }; + return { plan: response.data as z.infer }; } else { throw new Error("Invalid response from replanner"); } diff --git a/convex/langchain/prompts.ts b/convex/langchain/prompts.ts index c0a268a6..21b64c64 100644 --- a/convex/langchain/prompts.ts +++ b/convex/langchain/prompts.ts @@ -205,6 +205,14 @@ type PlanItem = export type Plan = { plan: PlanItem[] }; +CRITICAL: For "single" items, data must be an OBJECT with step and context fields: +✅ CORRECT: { "type": "single", "data": { "step": "Search docs", "context": "Search for relevant information" } } +❌ WRONG: { "type": "single", "data": ["Search for relevant information"] } + +For "parallel" items, data must be an ARRAY of planStep objects: +✅ CORRECT: { "type": "parallel", "data": [{ "step": "Task A", "context": "Do A" }, { "step": "Task B", "context": "Do B" }] } +❌ WRONG: { "type": "parallel", "data": ["Task A", "Task B"] } + --------------------------------------------------- Important constraints: @@ -243,6 +251,11 @@ type PlanItem = type ReplanResponse = | { type: "continue_planning"; data: PlanItem[] } | { type: "respond_to_user"; data: string }; + +CRITICAL: +- For "single", data MUST be an OBJECT: { step, context } (not an array). +- For "parallel", data MUST be an ARRAY of planStep objects (not strings). +- For "respond_to_user", data MUST be a STRING. Do NOT return arrays or objects. +---------------------------------------------------`; const toolsSection = `\n**Available Tools:**\n${availableToolsDescription}\n\nWhen planning remaining steps, consider which tools are available and how they can be used to accomplish the remaining objectives efficiently.`; diff --git a/src/components/chat/messages/ai-message/ai-message.tsx b/src/components/chat/messages/ai-message/ai-message.tsx index a216ca93..620f366c 100644 --- a/src/components/chat/messages/ai-message/ai-message.tsx +++ b/src/components/chat/messages/ai-message/ai-message.tsx @@ -39,8 +39,6 @@ export const AiMessageContent = memo( // Memoize the content rendering to avoid unnecessary re-renders const renderedContent = useMemo(() => { - if (type !== "ai") return content as string; - return parsedContent.map((part: ContentPart, index: number) => { if (part.type === "text") { // Only render non-empty text content diff --git a/src/hooks/chats/use-stream.ts b/src/hooks/chats/use-stream.ts index 75acef19..5ad0a2a6 100644 --- a/src/hooks/chats/use-stream.ts +++ b/src/hooks/chats/use-stream.ts @@ -50,7 +50,6 @@ export function useStream(chatId: Id<"chats"> | "new") { lastChunkTime: lastSeenTime, paginationOpts: { numItems: 200, cursor: null }, }); - console.log("result", result); if (!isMounted || !result?.chunks?.page?.length) return; // Only process chunks newer than lastSeenTime const newEvents: ChunkGroup[] = result.chunks.page From 0784a7ec5a5c580a0c6515956fbce02001cb355d Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Mon, 18 Aug 2025 14:53:22 +0530 Subject: [PATCH 13/25] refactor: update tool event handling and improve error reporting - Changed event name from "tool_stream" to "tool_progress" for better clarity in event handling. - Enhanced error handling in MCP and Google tools to provide more informative messages during failures. - Updated the getAvailableTools function to handle retrieval tools more robustly with promise settling. - Renamed listCalendarseTool to listCalendarsTool for consistency. --- convex/langchain/helpers.ts | 28 +++++++++-- convex/langchain/index.ts | 38 ++++++-------- convex/langchain/tools/googleTools.ts | 64 ++++++++++++------------ convex/langchain/tools/index.ts | 3 +- convex/langchain/tools/mcpTools.ts | 8 +-- convex/langchain/tools/retrievalTools.ts | 22 ++++---- services/mcps | 2 +- 7 files changed, 88 insertions(+), 77 deletions(-) diff --git a/convex/langchain/helpers.ts b/convex/langchain/helpers.ts index 1234586b..55aec976 100644 --- a/convex/langchain/helpers.ts +++ b/convex/langchain/helpers.ts @@ -171,26 +171,44 @@ export async function getAvailableTools( ): Promise[]> { const chat = config.chat; - const [mcpTools, retrievalTools] = await Promise.all([ + const [mcpResult, retrievalResult] = await Promise.allSettled([ getMCPTools(config.ctx, state, config), getRetrievalTools(state, config, true), ]); + const mcpTools = + mcpResult.status === "fulfilled" + ? mcpResult.value + : { + tools: [], + groupedTools: {} as Record< + string, + StructuredToolInterface[] + >, + }; + const retrievalTools = + retrievalResult.status === "fulfilled" ? retrievalResult.value : null; + if (!groupTools) { const pickedRetrievalTools: StructuredToolInterface[] = [ - ...(chat.projectId ? [retrievalTools.vectorSearch] : []), - ...(chat.webSearch ? [retrievalTools.webSearch] : []), + ...(chat.projectId && retrievalTools?.vectorSearch + ? [retrievalTools.vectorSearch] + : []), + ...(chat.webSearch && retrievalTools?.webSearch + ? [retrievalTools.webSearch] + : []), ]; + return [...mcpTools.tools, ...pickedRetrievalTools]; } const toolkits: Toolkit[] = [ ...Object.entries(mcpTools.groupedTools).map(([name, tools]) => ({ name, tools, })), - ...(chat.webSearch + ...(chat.webSearch && retrievalTools?.webSearch ? [{ name: "WebSearch", tools: [retrievalTools.webSearch] }] : []), - ...(chat.projectId + ...(chat.projectId && retrievalTools?.vectorSearch ? [{ name: "VectorSearch", tools: [retrievalTools.vectorSearch] }] : []), ]; diff --git a/convex/langchain/index.ts b/convex/langchain/index.ts index f32f82f5..076b5da5 100644 --- a/convex/langchain/index.ts +++ b/convex/langchain/index.ts @@ -272,12 +272,15 @@ export const chat = action({ return; } + // Prefer the framework-provided event name const eventName = - typeof (data as any).event === "string" - ? (data as any).event - : typeof (data as any).name === "string" - ? (data as any).name - : undefined; + typeof raw?.name === "string" + ? raw.name + : typeof (data as any).event === "string" + ? (data as any).event + : typeof (data as any).name === "string" + ? (data as any).name + : undefined; if (!eventName) { console.warn( @@ -289,32 +292,21 @@ export const chat = action({ return; } - const payloadCandidate = - (data as any).data ?? (data as any).payload; - const payload = - payloadCandidate && typeof payloadCandidate === "object" - ? payloadCandidate - : undefined; - if (!payload) { - console.warn( - "[stream] Ignoring custom event without object payload", - { - eventName, - } - ); - return; - } - + // For dispatchCustomEvent, the payload is in evt.data + const payload = data; const chunk = typeof (payload as any).chunk === "string" ? (payload as any).chunk : undefined; const isComplete = (payload as any).complete === true; - if (eventName === "tool_stream") { + if ( + eventName === "tool_stream" || + eventName === "tool_progress" + ) { if (!chunk) { console.warn( - "[stream] tool_stream custom event missing chunk; skipping" + "[stream] tool_progress custom event missing chunk; skipping" ); return; } diff --git a/convex/langchain/tools/googleTools.ts b/convex/langchain/tools/googleTools.ts index 0977e1ea..ad90fbc2 100644 --- a/convex/langchain/tools/googleTools.ts +++ b/convex/langchain/tools/googleTools.ts @@ -79,16 +79,16 @@ export const getGoogleTools = async (config: ExtendedRunnableConfig) => { } // Google Calendar Tools - const listCalendarseTool = tool( + const listCalendarsTool = tool( async (_args: {}, toolConfig: any) => { try { await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: "Checking Google authentication…" }, toolConfig ); await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: "Fetching your Google calendars…" }, toolConfig ); @@ -107,7 +107,7 @@ export const getGoogleTools = async (config: ExtendedRunnableConfig) => { })) || []; await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: `Found ${calendars.length} calendars. Formatting results…` }, toolConfig ); @@ -117,7 +117,7 @@ export const getGoogleTools = async (config: ExtendedRunnableConfig) => { const message = error instanceof Error ? error.message : "Unknown error"; await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: `Failed to list calendars: ${message}`, complete: true }, toolConfig ); @@ -151,7 +151,7 @@ export const getGoogleTools = async (config: ExtendedRunnableConfig) => { ) => { try { await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: `Fetching events from calendar '${calendarId}'${ timeMin || timeMax @@ -189,7 +189,7 @@ export const getGoogleTools = async (config: ExtendedRunnableConfig) => { organizer: event.organizer, })) || []; await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: `Found ${events.length} events. Formatting results…` }, toolConfig ); @@ -199,7 +199,7 @@ export const getGoogleTools = async (config: ExtendedRunnableConfig) => { const message = error instanceof Error ? error.message : "Unknown error"; await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: `Failed to list calendar events: ${message}`, complete: true, @@ -267,7 +267,7 @@ export const getGoogleTools = async (config: ExtendedRunnableConfig) => { ) => { try { await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: `Creating event '${summary}' on '${calendarId}' from ${startDateTime} to ${endDateTime}…`, }, @@ -304,7 +304,7 @@ export const getGoogleTools = async (config: ExtendedRunnableConfig) => { }; await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: "Event created successfully. Preparing output…" }, toolConfig ); @@ -314,7 +314,7 @@ export const getGoogleTools = async (config: ExtendedRunnableConfig) => { const message = error instanceof Error ? error.message : "Unknown error"; await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: `Failed to create calendar event: ${message}`, complete: true, @@ -380,7 +380,7 @@ export const getGoogleTools = async (config: ExtendedRunnableConfig) => { ) => { try { await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: `Loading existing event '${eventId}' from '${calendarId}'…`, }, @@ -392,7 +392,7 @@ export const getGoogleTools = async (config: ExtendedRunnableConfig) => { ); await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: "Applying updates to the event…" }, toolConfig ); @@ -423,7 +423,7 @@ export const getGoogleTools = async (config: ExtendedRunnableConfig) => { }; await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: "Event updated successfully. Preparing output…" }, toolConfig ); @@ -433,7 +433,7 @@ export const getGoogleTools = async (config: ExtendedRunnableConfig) => { const message = error instanceof Error ? error.message : "Unknown error"; await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: `Failed to update calendar event: ${message}`, complete: true, @@ -487,7 +487,7 @@ export const getGoogleTools = async (config: ExtendedRunnableConfig) => { ) => { try { await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: `Deleting event '${eventId}' from '${calendarId}'…` }, toolConfig ); @@ -502,7 +502,7 @@ export const getGoogleTools = async (config: ExtendedRunnableConfig) => { message: `Event ${eventId} deleted successfully`, }; await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: "Event deleted successfully." }, toolConfig ); @@ -512,7 +512,7 @@ export const getGoogleTools = async (config: ExtendedRunnableConfig) => { const message = error instanceof Error ? error.message : "Unknown error"; await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: `Failed to delete calendar event: ${message}`, complete: true, @@ -548,7 +548,7 @@ export const getGoogleTools = async (config: ExtendedRunnableConfig) => { ) => { try { await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: "Fetching Gmail messages…" }, toolConfig ); @@ -593,7 +593,7 @@ export const getGoogleTools = async (config: ExtendedRunnableConfig) => { ); await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: `Found ${messages.length} messages. Formatting results…` }, toolConfig ); @@ -603,7 +603,7 @@ export const getGoogleTools = async (config: ExtendedRunnableConfig) => { const message = error instanceof Error ? error.message : "Unknown error"; await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: `Failed to list Gmail messages: ${message}`, complete: true, @@ -648,7 +648,7 @@ export const getGoogleTools = async (config: ExtendedRunnableConfig) => { ) => { try { await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: `Fetching Gmail message '${messageId}'…` }, toolConfig ); @@ -687,7 +687,7 @@ export const getGoogleTools = async (config: ExtendedRunnableConfig) => { }; await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: "Message loaded. Preparing output…" }, toolConfig ); @@ -697,7 +697,7 @@ export const getGoogleTools = async (config: ExtendedRunnableConfig) => { const message = error instanceof Error ? error.message : "Unknown error"; await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: `Failed to get Gmail message: ${message}`, complete: true }, toolConfig ); @@ -737,7 +737,7 @@ export const getGoogleTools = async (config: ExtendedRunnableConfig) => { ) => { try { await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: `Composing email to ${to} with subject '${subject}'…` }, toolConfig ); @@ -754,7 +754,7 @@ export const getGoogleTools = async (config: ExtendedRunnableConfig) => { .replace(/=+$/, ""); await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: "Sending email via Gmail API…" }, toolConfig ); @@ -773,7 +773,7 @@ export const getGoogleTools = async (config: ExtendedRunnableConfig) => { }; await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: "Email sent successfully. Preparing output…" }, toolConfig ); @@ -783,7 +783,7 @@ export const getGoogleTools = async (config: ExtendedRunnableConfig) => { const message = error instanceof Error ? error.message : "Unknown error"; await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: `Failed to send Gmail message: ${message}`, complete: true }, toolConfig ); @@ -811,7 +811,7 @@ export const getGoogleTools = async (config: ExtendedRunnableConfig) => { ) => { try { await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: `Searching Gmail for: ${query}…` }, toolConfig ); @@ -853,7 +853,7 @@ export const getGoogleTools = async (config: ExtendedRunnableConfig) => { ); await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: `Found ${messages.length} matching messages. Formatting…` }, toolConfig ); @@ -863,7 +863,7 @@ export const getGoogleTools = async (config: ExtendedRunnableConfig) => { const message = error instanceof Error ? error.message : "Unknown error"; await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: `Failed to search Gmail: ${message}`, complete: true }, toolConfig ); @@ -891,7 +891,7 @@ export const getGoogleTools = async (config: ExtendedRunnableConfig) => { ); return [ - listCalendarseTool, + listCalendarsTool, listCalendarEventsTool, createCalendarEventTool, updateCalendarEventTool, diff --git a/convex/langchain/tools/index.ts b/convex/langchain/tools/index.ts index 8984d975..19aaab01 100644 --- a/convex/langchain/tools/index.ts +++ b/convex/langchain/tools/index.ts @@ -2,4 +2,5 @@ // Re-export functions from separate files to maintain backward compatibility export { getRetrievalTools } from "./retrievalTools"; -export { getMCPTools } from "./mcpTools"; \ No newline at end of file +export { getMCPTools } from "./mcpTools"; +// export { getGoogleTools } from "./googleTools"; diff --git a/convex/langchain/tools/mcpTools.ts b/convex/langchain/tools/mcpTools.ts index 78eb5a7b..a658f1dd 100644 --- a/convex/langchain/tools/mcpTools.ts +++ b/convex/langchain/tools/mcpTools.ts @@ -304,7 +304,7 @@ export const getMCPTools = async ( const wrapped = tool( async (args: any, toolConfig: any) => { await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: `Connecting to ${serverName} and invoking ${prettyName}…`, }, @@ -312,13 +312,13 @@ export const getMCPTools = async ( ); try { await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: `Executing ${prettyName} with provided parameters…` }, toolConfig ); const result = await (baseTool as any).invoke(args, toolConfig); await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: `${prettyName} finished. Preparing results for display…`, }, @@ -329,7 +329,7 @@ export const getMCPTools = async ( const message = error instanceof Error ? error.message : "Unknown error"; await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: `${prettyName} failed: ${message}`, complete: true }, toolConfig ); diff --git a/convex/langchain/tools/retrievalTools.ts b/convex/langchain/tools/retrievalTools.ts index 6c94df3d..b17f83f7 100644 --- a/convex/langchain/tools/retrievalTools.ts +++ b/convex/langchain/tools/retrievalTools.ts @@ -24,7 +24,7 @@ export const getRetrievalTools = async ( toolConfig: any ) => { await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: "Initializing vector store..." }, toolConfig ); @@ -35,7 +35,7 @@ export const getRetrievalTools = async ( }); await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: "Loading selected project documents..." }, toolConfig ); @@ -50,7 +50,7 @@ export const getRetrievalTools = async ( if (includedProjectDocuments.length === 0) { const msg = "No project documents available for retrieval."; await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: msg, complete: true }, toolConfig ); @@ -58,7 +58,7 @@ export const getRetrievalTools = async ( } await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: "Searching vector index..." }, toolConfig ); @@ -74,7 +74,7 @@ export const getRetrievalTools = async ( }); await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: `Found ${results.length} results. Building response...` }, toolConfig ); @@ -108,7 +108,7 @@ export const getRetrievalTools = async ( ); await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: "Formatting final output...", complete: true }, toolConfig ); @@ -139,7 +139,7 @@ export const getRetrievalTools = async ( toolConfig: any ) => { await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: "Preparing web search..." }, toolConfig ); @@ -154,7 +154,7 @@ export const getRetrievalTools = async ( try { await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: "Querying Exa..." }, toolConfig ); @@ -172,7 +172,7 @@ export const getRetrievalTools = async ( if (searchResponse.length === 0) { const msg = "No results found."; await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: msg, complete: true }, toolConfig ); @@ -180,7 +180,7 @@ export const getRetrievalTools = async ( } await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: `Found ${searchResponse.length} results. Formatting...`, complete: true, @@ -208,7 +208,7 @@ export const getRetrievalTools = async ( error instanceof Error ? error.message : "Unknown error" }`; await dispatchCustomEvent( - "tool_stream", + "tool_progress", { chunk: msg, complete: true }, toolConfig ); diff --git a/services/mcps b/services/mcps index 3c4a7d65..1d83b412 160000 --- a/services/mcps +++ b/services/mcps @@ -1 +1 @@ -Subproject commit 3c4a7d658fd0e80ecc085f3f8f68aef7041c0697 +Subproject commit 1d83b4120aea6ec474f184d2720dc0a507149143 From 3775438f63b04a6c06caf8ca89dfbf6add717783 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Mon, 18 Aug 2025 18:38:00 +0530 Subject: [PATCH 14/25] refactor: streamline tool retrieval logic in getAvailableTools function - Replaced conditional spread syntax with explicit if statements for better readability and maintainability. - Ensured that tools are only added to the toolkit if they are available in the retrieval data. --- convex/langchain/helpers.ts | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/convex/langchain/helpers.ts b/convex/langchain/helpers.ts index cbf944fc..483c962c 100644 --- a/convex/langchain/helpers.ts +++ b/convex/langchain/helpers.ts @@ -219,14 +219,19 @@ export async function getAvailableTools( name, tools, })), - ...(chat.webSearch - ? [{ name: "WebSearch", tools: [retrievalData!.webSearch] }] - : []), - ...(chat.projectId - ? [{ name: "VectorSearch", tools: [retrievalData!.vectorSearch] }] - : []), ]; + if (chat.webSearch && retrievalData?.webSearch) { + toolkits.push({ name: "WebSearch", tools: [retrievalData.webSearch] }); + } + + if (chat.projectId && retrievalData?.vectorSearch) { + toolkits.push({ + name: "VectorSearch", + tools: [retrievalData.vectorSearch], + }); + } + return toolkits; } From d0c11602d0de9e3b36d10417f5436b44d0888b1a Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Sat, 23 Aug 2025 03:11:59 +0530 Subject: [PATCH 15/25] feat: enhance chat streaming functionality - Added "replanner" to the list of allowed nodes to ensure the final AI response is visible in orchestrator mode. - Updated the logic for clearing streamed chunks to retain the last output until persisted messages arrive, improving user experience during chat transitions. - Removed unnecessary line break in ChatMessages component for cleaner code. --- convex/langchain/index.ts | 11 ++++++++++- src/components/chat/messages/index.tsx | 1 - src/hooks/chats/use-stream.ts | 11 +++++++++-- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/convex/langchain/index.ts b/convex/langchain/index.ts index 48931925..43a3b475 100644 --- a/convex/langchain/index.ts +++ b/convex/langchain/index.ts @@ -203,7 +203,16 @@ export const chat = action({ } // defer checkpoint refresh; will be done just-in-time before flush - const allowedNodes = ["baseAgent", "simple", "plannerAgent"]; + // Include nodes whose chat model or tool events we want to forward to the stream. + // "replanner" is where the final AI response is generated in orchestrator mode, + // so we add it here to ensure the user sees the last streamed chunks instead of + // the message disappearing when the planner finishes. + const allowedNodes = [ + "baseAgent", + "simple", + "plannerAgent", + "replanner", + ]; if ( allowedNodes.some((node) => evt.metadata?.checkpoint_ns?.startsWith(node) diff --git a/src/components/chat/messages/index.tsx b/src/components/chat/messages/index.tsx index 3bfd008f..7f19113d 100644 --- a/src/components/chat/messages/index.tsx +++ b/src/components/chat/messages/index.tsx @@ -13,7 +13,6 @@ export const ChatMessages = ({ chatId }: { chatId: Id<"chats"> | "new" }) => { const { isLoading, isEmpty } = useMessages({ chatId }); const streamStatus = useAtomValue(streamStatusAtom); - const mainContent = useMemo(() => { if (chatId === "new") { // Get user name from loadable state diff --git a/src/hooks/chats/use-stream.ts b/src/hooks/chats/use-stream.ts index b37bd39a..d4d111fb 100644 --- a/src/hooks/chats/use-stream.ts +++ b/src/hooks/chats/use-stream.ts @@ -151,9 +151,16 @@ export function useStream(chatId: Id<"chats"> | "new") { }; }, [stream?.status, stream?.chatId, lastSeenTime]); - // Clear chunks when stream is not active + // Clear chunks when the stream is explicitly reset or a new chat is loaded. + // Do NOT clear on "done" so that the UI keeps the last streamed output + // visible until the persisted messages arrive. useEffect(() => { - if (!stream || stream.status !== "streaming") { + if (!stream) { + setGroupedChunks([]); + return; + } + + if (stream.status === "cancelled" || stream.status === "error") { setGroupedChunks([]); } }, [stream?.status]); From 76483e00264f35b8c85d0ebff4ba0ecf7f34c01d Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Sat, 23 Aug 2025 04:51:32 +0530 Subject: [PATCH 16/25] feat: improve error handling and completion logic in chat streaming - Added handling for tool errors to gracefully surface messages to the client and mark tool calls as complete. - Updated completion logic to ignore the `complete` flag on `tool_progress` events to prevent duplicate completion chunks. - Cleaned up comments and removed unnecessary lines for better code clarity. --- convex/langchain/index.ts | 26 ++++++++++++++++++------ convex/langchain/tools/retrievalTools.ts | 3 +-- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/convex/langchain/index.ts b/convex/langchain/index.ts index 43a3b475..49bdf803 100644 --- a/convex/langchain/index.ts +++ b/convex/langchain/index.ts @@ -201,12 +201,7 @@ export const chat = action({ if (abort.signal.aborted) { return; } - // defer checkpoint refresh; will be done just-in-time before flush - // Include nodes whose chat model or tool events we want to forward to the stream. - // "replanner" is where the final AI response is generated in orchestrator mode, - // so we add it here to ensure the user sees the last streamed chunks instead of - // the message disappearing when the planner finishes. const allowedNodes = [ "baseAgent", "simple", @@ -272,6 +267,21 @@ export const chat = action({ }); // Mark that the checkpoint should be refreshed soon pendingCheckpointRefresh = true; + } else if (evt.event === "on_tool_error") { + // Gracefully surface tool errors to the client and mark the tool call as complete + const errorOutput = + (evt.data as any)?.error?.message ?? + (evt.data as any)?.error ?? + "Tool execution failed"; + appendToolChunk(buffer, { + toolName: evt.name, + input: (evt.data as any)?.input, + output: `Error: ${errorOutput}`, + isComplete: true, + toolCallId: evt.run_id, + }); + // Force checkpoint refresh so calling step is marked done/errored + pendingCheckpointRefresh = true; } else if ( evt.event === "on_chat_model_end" || evt.event === "on_chain_end" @@ -318,7 +328,11 @@ export const chat = action({ typeof (payload as any).chunk === "string" ? (payload as any).chunk : undefined; - const isComplete = (payload as any).complete === true; + // Ignore `complete` flag on tool_progress to prevent duplicate completion chunks; let on_tool_end handle completion. + const isComplete = + eventName === "tool_progress" + ? false + : payload.complete === true; if ( eventName === "tool_stream" || diff --git a/convex/langchain/tools/retrievalTools.ts b/convex/langchain/tools/retrievalTools.ts index b17f83f7..083cc380 100644 --- a/convex/langchain/tools/retrievalTools.ts +++ b/convex/langchain/tools/retrievalTools.ts @@ -109,7 +109,7 @@ export const getRetrievalTools = async ( await dispatchCustomEvent( "tool_progress", - { chunk: "Formatting final output...", complete: true }, + { chunk: "Formatting final output..." }, toolConfig ); return returnString ? JSON.stringify(documents, null, 0) : documents; @@ -183,7 +183,6 @@ export const getRetrievalTools = async ( "tool_progress", { chunk: `Found ${searchResponse.length} results. Formatting...`, - complete: true, }, toolConfig ); From 9abbfaba2430d44ae1645ff5aba8087878527847 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Sat, 23 Aug 2025 06:25:01 +0530 Subject: [PATCH 17/25] refactor: streamline error messaging in Google and retrieval tools - Removed the `complete` flag from `tool_progress` events in Google and retrieval tools to simplify error handling. - Updated error dispatch logic to maintain consistency across different tools. - Cleaned up comments and improved code readability. --- convex/langchain/tools/googleTools.ts | 13 +++----- convex/langchain/tools/retrievalTools.ts | 14 ++------ src/hooks/chats/use-stream.ts | 42 +++++++++++------------- 3 files changed, 26 insertions(+), 43 deletions(-) diff --git a/convex/langchain/tools/googleTools.ts b/convex/langchain/tools/googleTools.ts index ad90fbc2..66377d9d 100644 --- a/convex/langchain/tools/googleTools.ts +++ b/convex/langchain/tools/googleTools.ts @@ -118,7 +118,7 @@ export const getGoogleTools = async (config: ExtendedRunnableConfig) => { error instanceof Error ? error.message : "Unknown error"; await dispatchCustomEvent( "tool_progress", - { chunk: `Failed to list calendars: ${message}`, complete: true }, + { chunk: `Failed to list calendars: ${message}` }, toolConfig ); return `Failed to list calendars: ${message}`; @@ -202,7 +202,6 @@ export const getGoogleTools = async (config: ExtendedRunnableConfig) => { "tool_progress", { chunk: `Failed to list calendar events: ${message}`, - complete: true, }, toolConfig ); @@ -317,7 +316,6 @@ export const getGoogleTools = async (config: ExtendedRunnableConfig) => { "tool_progress", { chunk: `Failed to create calendar event: ${message}`, - complete: true, }, toolConfig ); @@ -436,7 +434,6 @@ export const getGoogleTools = async (config: ExtendedRunnableConfig) => { "tool_progress", { chunk: `Failed to update calendar event: ${message}`, - complete: true, }, toolConfig ); @@ -515,7 +512,6 @@ export const getGoogleTools = async (config: ExtendedRunnableConfig) => { "tool_progress", { chunk: `Failed to delete calendar event: ${message}`, - complete: true, }, toolConfig ); @@ -606,7 +602,6 @@ export const getGoogleTools = async (config: ExtendedRunnableConfig) => { "tool_progress", { chunk: `Failed to list Gmail messages: ${message}`, - complete: true, }, toolConfig ); @@ -698,7 +693,7 @@ export const getGoogleTools = async (config: ExtendedRunnableConfig) => { error instanceof Error ? error.message : "Unknown error"; await dispatchCustomEvent( "tool_progress", - { chunk: `Failed to get Gmail message: ${message}`, complete: true }, + { chunk: `Failed to get Gmail message: ${message}` }, toolConfig ); return `Failed to get Gmail message: ${message}`; @@ -784,7 +779,7 @@ export const getGoogleTools = async (config: ExtendedRunnableConfig) => { error instanceof Error ? error.message : "Unknown error"; await dispatchCustomEvent( "tool_progress", - { chunk: `Failed to send Gmail message: ${message}`, complete: true }, + { chunk: `Failed to send Gmail message: ${message}` }, toolConfig ); return `Failed to send Gmail message: ${message}`; @@ -864,7 +859,7 @@ export const getGoogleTools = async (config: ExtendedRunnableConfig) => { error instanceof Error ? error.message : "Unknown error"; await dispatchCustomEvent( "tool_progress", - { chunk: `Failed to search Gmail: ${message}`, complete: true }, + { chunk: `Failed to search Gmail: ${message}` }, toolConfig ); return `Failed to search Gmail: ${message}`; diff --git a/convex/langchain/tools/retrievalTools.ts b/convex/langchain/tools/retrievalTools.ts index 083cc380..10f7e52b 100644 --- a/convex/langchain/tools/retrievalTools.ts +++ b/convex/langchain/tools/retrievalTools.ts @@ -49,11 +49,7 @@ export const getRetrievalTools = async ( if (includedProjectDocuments.length === 0) { const msg = "No project documents available for retrieval."; - await dispatchCustomEvent( - "tool_progress", - { chunk: msg, complete: true }, - toolConfig - ); + await dispatchCustomEvent("tool_progress", { chunk: msg }, toolConfig); return msg; } @@ -173,7 +169,7 @@ export const getRetrievalTools = async ( const msg = "No results found."; await dispatchCustomEvent( "tool_progress", - { chunk: msg, complete: true }, + { chunk: msg }, toolConfig ); return msg; @@ -206,11 +202,7 @@ export const getRetrievalTools = async ( const msg = `Web search failed: ${ error instanceof Error ? error.message : "Unknown error" }`; - await dispatchCustomEvent( - "tool_progress", - { chunk: msg, complete: true }, - toolConfig - ); + await dispatchCustomEvent("tool_progress", { chunk: msg }, toolConfig); return msg; } }, diff --git a/src/hooks/chats/use-stream.ts b/src/hooks/chats/use-stream.ts index d4d111fb..5ad05e47 100644 --- a/src/hooks/chats/use-stream.ts +++ b/src/hooks/chats/use-stream.ts @@ -82,34 +82,26 @@ export function useStream(chatId: Id<"chats"> | "new") { } } else { const toolChunk = chunk as ToolChunkGroup; - // Try to merge with the latest in-flight chunk for the same toolCallId + // Try to merge with an existing group for the same toolCallId (even if it is already marked complete) let mergedIndex = -1; - if ( - lastGroup?.type === "tool" && - (lastGroup as ToolChunkGroup).toolCallId === - toolChunk.toolCallId && - !(lastGroup as ToolChunkGroup).isComplete - ) { - mergedIndex = newGroups.length - 1; - } else { - for (let i = newGroups.length - 1; i >= 0; i--) { - const g = newGroups[i]; - if ( - g.type === "tool" && - (g as ToolChunkGroup).toolCallId === - toolChunk.toolCallId && - !(g as ToolChunkGroup).isComplete - ) { - mergedIndex = i; - break; - } + for (let i = newGroups.length - 1; i >= 0; i--) { + const g = newGroups[i]; + if ( + g.type === "tool" && + (g as ToolChunkGroup).toolCallId === toolChunk.toolCallId + ) { + mergedIndex = i; + break; } } if (mergedIndex >= 0) { const existing = newGroups[mergedIndex] as ToolChunkGroup; // Append incremental output if provided - if (typeof toolChunk.output === "string") { + if ( + typeof toolChunk.output === "string" && + toolChunk.output + ) { const existingOutput = typeof existing.output === "string" ? existing.output @@ -117,19 +109,23 @@ export function useStream(chatId: Id<"chats"> | "new") { existing.output = existingOutput ? `${existingOutput}\n${toolChunk.output}` : toolChunk.output; - } else if (toolChunk.output !== undefined) { + } else if ( + toolChunk.output !== undefined && + toolChunk.output !== null + ) { existing.output = toolChunk.output as any; } // Update input if present if (toolChunk.input !== undefined) { existing.input = toolChunk.input as any; } - // Mark complete if end event + // Mark complete if any chunk indicates completion if (toolChunk.isComplete) { existing.isComplete = true; } lastGroup = existing; } else { + // No existing group, create a new one lastGroup = toolChunk; newGroups.push(toolChunk); } From b85e9c57147d9522cc26152338deabdc4d82d952 Mon Sep 17 00:00:00 2001 From: Shivam Sharma <91240327+shivamhwp@users.noreply.github.com> Date: Tue, 26 Aug 2025 12:08:37 +0530 Subject: [PATCH 18/25] refactor: enhance DocumentDialog and TopNav components - Reformatted import statements for improved consistency and readability. - Added error handling for preview loading in DocumentDialog, providing user feedback on failures. - Updated the handling of document previews and download links to ensure better user experience. - Streamlined key event handling in TopNav for toggling the resizable panel, improving accessibility. - Enhanced user session management by refining user data fetching logic in TopNav. --- .../tool-message/search-results.tsx | 2 +- src/components/document-dialog.tsx | 417 ++++++++++-------- src/components/topnav.tsx | 221 +++++----- 3 files changed, 337 insertions(+), 303 deletions(-) diff --git a/src/components/chat/messages/ai-message/tool-message/search-results.tsx b/src/components/chat/messages/ai-message/tool-message/search-results.tsx index 56f92e1f..cda5cf4d 100644 --- a/src/components/chat/messages/ai-message/tool-message/search-results.tsx +++ b/src/components/chat/messages/ai-message/tool-message/search-results.tsx @@ -7,7 +7,7 @@ import { import { Favicon } from "@/components/ui/favicon"; import { Markdown } from "@/components/ui/markdown"; import { extractDomain } from "@/lib/utils"; -import { ChevronDownIcon, ExternalLinkIcon, GlobeIcon } from "lucide-react"; +import { ExternalLinkIcon, GlobeIcon } from "lucide-react"; // Type definition for search results output export type SearchResultMetadata = { diff --git a/src/components/document-dialog.tsx b/src/components/document-dialog.tsx index 83a97e10..ed4971ab 100644 --- a/src/components/document-dialog.tsx +++ b/src/components/document-dialog.tsx @@ -1,8 +1,8 @@ import { - Dialog, - DialogContent, - DialogHeader, - DialogTitle, + Dialog, + DialogContent, + DialogHeader, + DialogTitle, } from "@/components/ui/dialog"; import { documentDialogOpenAtom } from "@/store/chatStore"; import { api } from "../../convex/_generated/api"; @@ -15,206 +15,233 @@ import { useState, useEffect } from "react"; import { useAtomValue, useSetAtom } from "jotai"; export const DocumentDialog = () => { - const documentDialogOpen = useAtomValue(documentDialogOpenAtom); - const setDocumentDialogOpen = useSetAtom(documentDialogOpenAtom); - const [previewUrl, setPreviewUrl] = useState(null); + const documentDialogOpen = useAtomValue(documentDialogOpenAtom); + const setDocumentDialogOpen = useSetAtom(documentDialogOpenAtom); + const [previewUrl, setPreviewUrl] = useState(null); + const [previewError, setPreviewError] = useState(null); - const { data: document } = useQuery({ - ...convexQuery( - api.documents.queries.get, - documentDialogOpen ? { documentId: documentDialogOpen } : "skip", - ), - enabled: !!documentDialogOpen, - }); + const { data: document } = useQuery({ + ...convexQuery( + api.documents.queries.get, + documentDialogOpen ? { documentId: documentDialogOpen } : "skip", + ), + enabled: !!documentDialogOpen, + }); - const { mutateAsync: generateDownloadUrl } = useMutation({ - mutationFn: useConvexMutation(api.documents.mutations.generateDownloadUrl), - }); + const { mutateAsync: generateDownloadUrl } = useMutation({ + mutationFn: useConvexMutation(api.documents.mutations.generateDownloadUrl), + }); - const documentName = document?.name ?? ""; - const { - icon: Icon, - className: IconClassName, - tag, - } = document - ? getDocTagInfo(document) - : { icon: () => null, className: "", tag: "" }; + const documentName = document?.name ?? ""; + const { + icon: Icon, + className: IconClassName, + tag, + } = document + ? getDocTagInfo(document) + : { icon: () => null, className: "", tag: "" }; - useEffect(() => { - setPreviewUrl(null); - const loadPreviewUrl = async () => { - if (!document) return; - switch (tag) { - case "image": - case "pdf": - case "file": { - // Only files need download URL - const url = await generateDownloadUrl({ - documentId: document._id!, - }); - setPreviewUrl(url); - break; - } - case "url": - case "site": { - setPreviewUrl(document.key as string); - break; - } - case "youtube": { - setPreviewUrl(`https://www.youtube.com/embed/${document.key}`); - break; - } - default: - if (["file", "text", "github"].includes(document.type)) { - const url = await generateDownloadUrl({ - documentId: document._id!, - }); - setPreviewUrl(url); - break; - } else { - setPreviewUrl(document.key as string); - } - break; - } - }; - loadPreviewUrl(); - }, [document, tag, generateDownloadUrl]); + useEffect(() => { + setPreviewUrl(null); + setPreviewError(null); + const loadPreviewUrl = async () => { + if (!document) return; + try { + switch (tag) { + case "image": + case "pdf": + case "file": { + // Only files need download URL + const url = await generateDownloadUrl({ + documentId: document._id!, + }); + setPreviewUrl(url); + break; + } + case "url": + case "site": { + setPreviewUrl(document.key as string); + break; + } + case "youtube": { + setPreviewUrl(`https://www.youtube.com/embed/${document.key}`); + break; + } + default: + if (["file", "text", "github"].includes(document.type)) { + const url = await generateDownloadUrl({ + documentId: document._id!, + }); + setPreviewUrl(url); + break; + } else { + setPreviewUrl(document.key as string); + } + break; + } + } catch (error) { + setPreviewError( + `Failed to load preview: ${error instanceof Error ? error.message : "Unknown error"}`, + ); + } + }; + loadPreviewUrl(); + }, [document, tag, generateDownloadUrl]); - // Early return if dialog is not open - if (!documentDialogOpen) { - return null; - } + // Early return if dialog is not open + if (!documentDialogOpen) { + return null; + } - const handleDownload = async () => { - if (!document || tag !== "file") return; - const url = await generateDownloadUrl({ - documentId: document._id!, - }); - if (url) { - window.open(url, "_blank"); - } - }; + const handleDownload = async () => { + if (!document || tag !== "file") return; + const url = await generateDownloadUrl({ + documentId: document._id!, + }); + if (url) { + const w = window.open(url, "_blank", "noopener,noreferrer"); + if (w) w.opener = null; + } + }; - const handleOpen = () => { - if (!document) return; - if (tag === "url" || tag === "site") { - window.open(document.key as string, "_blank"); - } else if (tag === "youtube") { - window.open(`https://youtube.com/watch?v=${document.key}`, "_blank"); - } - }; + const handleOpen = () => { + if (!document) return; + if (tag === "url" || tag === "site") { + const w = window.open( + document.key as string, + "_blank", + "noopener,noreferrer", + ); + if (w) w.opener = null; + } else if (tag === "youtube") { + const w = window.open( + `https://youtube.com/watch?v=${document.key}`, + "_blank", + "noopener,noreferrer", + ); + if (w) w.opener = null; + } + }; - return ( - setDocumentDialogOpen(undefined)} - > - - - - - {documentName} - - + return ( + setDocumentDialogOpen(undefined)} + > + + + + + {documentName} + + -
-
-
- Type:{" "} - {document?.type && - document.type.charAt(0).toUpperCase() + document.type.slice(1)} -
- {document?.size && ( -
- Size: {formatBytes(document.size)} -
- )} -
+
+
+
+ Type:{" "} + {document?.type && + document.type.charAt(0).toUpperCase() + document.type.slice(1)} +
+ {document?.size && ( +
+ Size: {formatBytes(document.size)} +
+ )} +
- {previewUrl && ( -
- {(() => { - if (tag === "image") { - return ( - {documentName} - ); - } else if (tag === "pdf") { - return ( - -
- PDF preview not supported in your browser. Please - download the file to view it. -
-
- ); - } else if (tag === "youtube") { - return ( -