From 30b09b23ef81a24d8df9d765794b5831d6c04f30 Mon Sep 17 00:00:00 2001 From: Rugved Somwanshi Date: Thu, 5 Feb 2026 12:02:43 -0500 Subject: [PATCH 1/9] Add image chip primitives and rendering --- src/subcommands/chat/react/Chat.tsx | 198 +++++++++--- src/subcommands/chat/react/ChatInput.tsx | 30 +- src/subcommands/chat/react/ChatMessages.tsx | 16 +- .../chat/react/inputReducer.test.ts | 292 ++++++++++++------ src/subcommands/chat/react/inputReducer.ts | 205 ++++++++---- src/subcommands/chat/react/inputRenderer.tsx | 17 +- src/subcommands/chat/react/types.ts | 34 +- src/subcommands/chat/util.ts | 26 +- 8 files changed, 587 insertions(+), 231 deletions(-) diff --git a/src/subcommands/chat/react/Chat.tsx b/src/subcommands/chat/react/Chat.tsx index 833fb6c8..456d673e 100644 --- a/src/subcommands/chat/react/Chat.tsx +++ b/src/subcommands/chat/react/Chat.tsx @@ -1,8 +1,15 @@ import { produce } from "@lmstudio/immer-with-plugins"; -import { type Chat, type LLM, type LLMPredictionStats, type LMStudioClient } from "@lmstudio/sdk"; +import { + type Chat, + type ChatMessagePartFileData, + type ChatMessagePartTextData, + type LLM, + type LLMPredictionStats, + type LMStudioClient, +} from "@lmstudio/sdk"; import { Box, type DOMElement, useApp, Text } from "ink"; import React, { useCallback, useMemo, useRef, useState } from "react"; -import { displayVerboseStats, getLargePastePlaceholderText } from "../util.js"; +import { displayVerboseStats, getChipPreviewText } from "../util.js"; import { ChatInput } from "./ChatInput.js"; import { ChatMessagesList } from "./ChatMessagesList.js"; import { ChatSuggestions } from "./ChatSuggestions.js"; @@ -19,7 +26,12 @@ import { } from "./hooks.js"; import { insertPasteAtCursor } from "./inputReducer.js"; import { createSlashCommands } from "./slashCommands.js"; -import type { ChatUserInputState, InkChatMessage, Suggestion } from "./types.js"; +import type { + ChatUserInputState, + InkChatMessage, + Suggestion, + UserInputContentPart, +} from "./types.js"; // Freezes streaming content into static chunks at natural breaks to reduce re-renders. // Uses multiple boundaries to handle different content (best effort): @@ -181,7 +193,7 @@ export const ChatComponent = React.memo( return []; } const firstSegment = userInputState.segments[0]; - const inputText = firstSegment.content; + const inputText = firstSegment?.type === "text" ? firstSegment.content : ""; return commandHandler.getSuggestions({ input: inputText, shouldShowSuggestions: !isConfirmationActive && !isPredicting && inputText.startsWith("/"), @@ -291,12 +303,8 @@ export const ChatComponent = React.memo( }, [modelLoadingProgress]); const handlePaste = useCallback((content: string) => { - const normalizedContent = content.replaceAll("\r\n", "\n").replaceAll("\r", "\n"); - - if (normalizedContent.length === 0) { - return; - } - + const normalizedContent = content.replaceAll("\r\n", "\n").replaceAll("\r", "\n").trimEnd(); + if (normalizedContent.length === 0) return; setUserInputState(previousState => insertPasteAtCursor({ state: previousState, @@ -307,27 +315,44 @@ export const ChatComponent = React.memo( }, []); const handleSubmit = useCallback(async () => { + const inputStateSnapshot = userInputState; + const inputSegments = userInputState.segments; + const hasImageChip = inputSegments.some( + segment => segment.type === "chip" && segment.data.kind === "image", + ); + // Collect the full text input from the userInputState - const userInputText = userInputState.segments - .map(segment => segment.content) + const userInputText = inputSegments + .map(segment => { + if (segment.type === "text") { + return segment.content; + } + return segment.data.kind === "largePaste" ? segment.data.content : ""; + }) .join("") .trim(); - // Clear the input state - setUserInputState(emptyChatInputState); - const confirmationResponse = await handleConfirmationResponse(userInputText); - if (confirmationResponse === "handled") { - return; - } - if (confirmationResponse === "invalid") { - logInChat("Please answer 'yes' or 'no'"); - return; + + if (isConfirmationActive) { + const confirmationResponse = await handleConfirmationResponse(userInputText); + if (confirmationResponse === "handled") { + setUserInputState(emptyChatInputState); + return; + } + if (confirmationResponse === "invalid") { + logInChat("Please answer 'yes' or 'no'"); + return; + } } - if (userInputText.length === 0) { + if (userInputText.length === 0 && hasImageChip === false) { return; } - if (userInputText.startsWith("/") && userInputState.segments.length === 1) { + if ( + userInputText.startsWith("/") && + inputSegments.length === 1 && + inputSegments[0]?.type === "text" + ) { const selectedSuggestion = normalizedSelectedSuggestionIndex !== null ? suggestions[normalizedSelectedSuggestionIndex] @@ -344,6 +369,7 @@ export const ChatComponent = React.memo( // Check if the command is in exception list, if not, // execute it. if (commandHandler.commandIsIgnored(command) === false) { + setUserInputState(emptyChatInputState); const wasCommandHandled = await commandHandler.execute(command, argumentsText); if (wasCommandHandled === false) { logInChat(`Unknown command: ${userInputText}`); @@ -362,6 +388,11 @@ export const ChatComponent = React.memo( return; } + if (hasImageChip === true && llmRef.current.vision === false) { + logErrorInChat("The current model does not support image input (vision)."); + return; + } + if (isPredicting) { logInChat( "A prediction is already in progress. Please wait for it to finish or press CTRL+C to abort it.", @@ -369,7 +400,23 @@ export const ChatComponent = React.memo( return; } + const userMessageContent = inputSegments.map(segment => { + if (segment.type === "text") { + return { type: "text", text: segment.content }; + } + + const displayText = getChipPreviewText(segment.data); + return { type: "chip", kind: segment.data.kind, displayText }; + }); + // If nothing else, proceed with normal message submission + setUserInputState(emptyChatInputState); + // Render the user message immediately (before any async image preparation) to avoid a + // transient frame where the input clears but the message hasn't appeared yet ("flash"). + addMessage({ + type: "user", + content: userMessageContent, + }); setIsPredicting(true); setPromptProcessingProgress(null); setShowPredictionSpinner(true); @@ -382,20 +429,88 @@ export const ChatComponent = React.memo( const signal = abortControllerRef.current.signal; try { - chatRef.current.append("user", userInputText); - addMessage({ - type: "user", - content: userInputState.segments.map(segment => { - if (segment.type === "largePaste") { - const placeholder = getLargePastePlaceholderText(segment.content); - return { - type: "largePaste", - text: placeholder, - }; - } - return { type: segment.type, text: segment.content }; - }), - }); + const imagesToPrepare = + hasImageChip === true + ? inputSegments.flatMap(segment => { + if (segment.type !== "chip" || segment.data.kind !== "image") { + return []; + } + return [segment.data]; + }) + : []; + + let preparedImages: Awaited>[] = []; + try { + preparedImages = hasImageChip + ? await Promise.all( + imagesToPrepare.map(image => { + return client.files.prepareImageBase64(image.fileName, image.contentBase64); + }), + ) + : []; + } catch (error) { + setMessages( + produce(draftMessages => { + const lastMessage = draftMessages.at(-1); + if (lastMessage?.type === "user") { + draftMessages.pop(); + } + }), + ); + setUserInputState(inputStateSnapshot); + const message = error instanceof Error ? error.message : String(error); + logErrorInChat(`Failed to attach image: ${message}`); + return; + } + if (preparedImages.some(image => image.type !== "image")) { + setMessages( + produce(draftMessages => { + const lastMessage = draftMessages.at(-1); + if (lastMessage?.type === "user") { + draftMessages.pop(); + } + }), + ); + setUserInputState(inputStateSnapshot); + logErrorInChat( + "Failed to attach image: clipboard content was not recognized as an image.", + ); + return; + } + + const parts: Array = []; + let preparedImageIndex = 0; + for (const segment of inputSegments) { + if (segment.type === "text") { + parts.push({ type: "text", text: segment.content }); + continue; + } + if (segment.data.kind === "largePaste") { + parts.push({ type: "text", text: segment.data.content }); + continue; + } + // image chip + const nextImage = preparedImages[preparedImageIndex]; + if (nextImage === undefined) { + continue; + } + preparedImageIndex += 1; + parts.push({ + type: "file", + name: nextImage.name, + identifier: nextImage.identifier, + sizeBytes: nextImage.sizeBytes, + fileType: nextImage.type, + }); + } + + // Ensure there is at least a leading text part. This matches the append(role, content, opts) + // behavior and avoids edge cases with image-only messages. + if (parts.length === 0 || parts[0]?.type !== "text") { + parts.unshift({ type: "text", text: "" }); + } + + chatRef.current.append({ role: "user", content: parts }); const result = await llmRef.current.respond(chatRef.current, { onFirstToken() { setShowPredictionSpinner(false); @@ -577,20 +692,21 @@ export const ChatComponent = React.memo( abortControllerRef.current = null; } }, [ - userInputState.segments, - handleConfirmationResponse, + userInputState, + isConfirmationActive, isPredicting, + addMessage, + handleConfirmationResponse, logInChat, normalizedSelectedSuggestionIndex, suggestions, commandHandler, handleExit, logErrorInChat, - addMessage, + client, stats, promptProcessingProgress, requestConfirmation, - client.llm, ttl, ]); diff --git a/src/subcommands/chat/react/ChatInput.tsx b/src/subcommands/chat/react/ChatInput.tsx index 54ad396b..7307226f 100644 --- a/src/subcommands/chat/react/ChatInput.tsx +++ b/src/subcommands/chat/react/ChatInput.tsx @@ -11,7 +11,9 @@ import { } from "./inputReducer.js"; import { renderInputWithCursor } from "./inputRenderer.js"; import { type ChatUserInputState } from "./types.js"; -import { getLargePastePlaceholderText } from "../util.js"; +import { getChipPreviewText } from "../util.js"; + +type ChipRange = { start: number; end: number; kind: "largePaste" | "image" }; interface ChatInputProps { inputState: ChatUserInputState; @@ -150,10 +152,18 @@ export const ChatInput = ({ return; } - const currentText = inputState.segments.map(segment => segment.content).join(""); + const currentText = inputState.segments + .map(segment => (segment.type === "text" ? segment.content : "")) + .join(""); // Check if input is a slash command without arguments that has suggestions - if (currentText.startsWith("/") && currentText.includes(" ") === false) { + // Only auto-insert space if input is a single text segment (no chips) + if ( + currentText.startsWith("/") && + currentText.includes(" ") === false && + inputState.segments.length === 1 && + inputState.segments[0]?.type === "text" + ) { const commandName = currentText.slice(1); if (commandHasSuggestions(commandName)) { setUserInputState(previousState => @@ -197,16 +207,16 @@ export const ChatInput = ({ } }); - const { fullText, cursorPosition, pasteRanges } = useMemo(() => { + const { fullText, cursorPosition, chipRanges } = useMemo(() => { let fullText = ""; let cursorPosition = 0; - const pasteRanges: Array<{ start: number; end: number }> = []; + const chipRanges: ChipRange[] = []; for (let index = 0; index < inputState.segments.length; index++) { const segment = inputState.segments[index]; - if (segment.type === "largePaste") { - const placeholder = getLargePastePlaceholderText(segment.content); + if (segment.type === "chip") { + const placeholder = getChipPreviewText(segment.data); const startPos = fullText.length; if (index < inputState.cursorOnSegmentIndex) { @@ -216,7 +226,7 @@ export const ChatInput = ({ } fullText += placeholder; - pasteRanges.push({ start: startPos, end: fullText.length }); + chipRanges.push({ start: startPos, end: fullText.length, kind: segment.data.kind }); } else { if (index < inputState.cursorOnSegmentIndex) { cursorPosition += segment.content.length; @@ -228,7 +238,7 @@ export const ChatInput = ({ } } - return { fullText, cursorPosition, pasteRanges }; + return { fullText, cursorPosition, chipRanges }; }, [inputState]); const lines = fullText.split("\n"); @@ -266,7 +276,7 @@ export const ChatInput = ({ {renderInputWithCursor({ fullText: lineText, cursorPosition: isCursorLine ? cursorColumnIndex : -1, - pasteRanges, + chipRanges, lineStartPos, })} diff --git a/src/subcommands/chat/react/ChatMessages.tsx b/src/subcommands/chat/react/ChatMessages.tsx index d9cdd6ae..6963e101 100644 --- a/src/subcommands/chat/react/ChatMessages.tsx +++ b/src/subcommands/chat/react/ChatMessages.tsx @@ -13,7 +13,12 @@ export const ChatMessage = memo(({ message, modelName }: ChatMessageProps) => { const type = message.type; switch (type) { case "user": { - const contentParts = message.content.map(part => ({ ...part })); + const contentParts = message.content.map(part => { + if (part.type === "text") { + return { type: "text" as const, text: part.text }; + } + return { type: "chip" as const, kind: part.kind, text: part.displayText }; + }); // Trim only edge newlines on the first/last non-empty parts to keep internal newlines intact. let startIndex = 0; @@ -38,8 +43,13 @@ export const ChatMessage = memo(({ message, modelName }: ChatMessageProps) => { // After edge trimming, apply large paste coloring and join into a single string. const formattedContent = contentParts .map(contentPart => { - const text = contentPart.text; - return contentPart.type === "largePaste" ? chalk.blue(text) : text; + if (contentPart.type === "text") { + return contentPart.text; + } + + return contentPart.kind === "largePaste" + ? chalk.blue(contentPart.text) + : chalk.cyan(contentPart.text); }) .join(""); diff --git a/src/subcommands/chat/react/inputReducer.test.ts b/src/subcommands/chat/react/inputReducer.test.ts index 83ae8cc5..d14b6559 100644 --- a/src/subcommands/chat/react/inputReducer.test.ts +++ b/src/subcommands/chat/react/inputReducer.test.ts @@ -1,7 +1,9 @@ import { deleteAfterCursor, deleteBeforeCursor, + deleteBeforeCursorCount, insertPasteAtCursor, + insertImageAtCursor, insertSuggestionAtCursor, insertTextAtCursor, moveCursorLeft, @@ -108,10 +110,10 @@ describe("chatInputStateReducers", () => { expect(result.cursorInSegmentOffset).toBe(4); }); - it("removes previous largePaste segment when deleting at start of text segment", () => { + it("removes previous chip segment when deleting at start of text segment", () => { const initialState = createChatUserInputState( [ - { type: "largePaste", content: "pasted" }, + { type: "chip", data: { kind: "largePaste", content: "pasted" } }, { type: "text", content: "text" }, ], 1, @@ -128,8 +130,8 @@ describe("chatInputStateReducers", () => { it("deletes previous largePaste when cursor is at start of largePaste (offset 0)", () => { const initialState = createChatUserInputState( [ - { type: "largePaste", content: "paste1" }, - { type: "largePaste", content: "paste2" }, + { type: "chip", data: { kind: "largePaste", content: "paste1" } }, + { type: "chip", data: { kind: "largePaste", content: "paste2" } }, { type: "text", content: "text" }, ], 1, @@ -139,7 +141,7 @@ describe("chatInputStateReducers", () => { const result = deleteBeforeCursor(initialState); expect(result.segments).toEqual([ - { type: "largePaste", content: "paste2" }, + { type: "chip", data: { kind: "largePaste", content: "paste2" } }, { type: "text", content: "text" }, ]); expect(result.cursorOnSegmentIndex).toBe(0); @@ -150,7 +152,7 @@ describe("chatInputStateReducers", () => { const initialState = createChatUserInputState( [ { type: "text", content: "before" }, - { type: "largePaste", content: "pasted" }, + { type: "chip", data: { kind: "largePaste", content: "pasted" } }, { type: "text", content: "after" }, ], 1, @@ -161,7 +163,7 @@ describe("chatInputStateReducers", () => { expect(result.segments).toEqual([ { type: "text", content: "befor" }, - { type: "largePaste", content: "pasted" }, + { type: "chip", data: { kind: "largePaste", content: "pasted" } }, { type: "text", content: "after" }, ]); expect(result.cursorOnSegmentIndex).toBe(0); @@ -172,7 +174,7 @@ describe("chatInputStateReducers", () => { const initialState = createChatUserInputState( [ { type: "text", content: "" }, - { type: "largePaste", content: "pasted" }, + { type: "chip", data: { kind: "largePaste", content: "pasted" } }, { type: "text", content: "after" }, ], 1, @@ -182,7 +184,7 @@ describe("chatInputStateReducers", () => { const result = deleteBeforeCursor(initialState); expect(result.segments).toEqual([ - { type: "largePaste", content: "pasted" }, + { type: "chip", data: { kind: "largePaste", content: "pasted" } }, { type: "text", content: "after" }, ]); expect(result.cursorOnSegmentIndex).toBe(0); @@ -193,7 +195,7 @@ describe("chatInputStateReducers", () => { const initialState = createChatUserInputState( [ { type: "text", content: "a" }, - { type: "largePaste", content: "pasted" }, + { type: "chip", data: { kind: "largePaste", content: "pasted" } }, ], 1, 0, @@ -201,7 +203,7 @@ describe("chatInputStateReducers", () => { const result = deleteBeforeCursor(initialState); - expect(result.segments).toEqual([{ type: "largePaste", content: "pasted" }]); + expect(result.segments).toEqual([{ type: "chip", data: { kind: "largePaste", content: "pasted" } }]); expect(result.cursorOnSegmentIndex).toBe(0); expect(result.cursorInSegmentOffset).toBe(0); }); @@ -209,7 +211,7 @@ describe("chatInputStateReducers", () => { it("does nothing when cursor is at very start (segment 0, offset 0)", () => { const initialState = createChatUserInputState( [ - { type: "largePaste", content: "paste1" }, + { type: "chip", data: { kind: "largePaste", content: "paste1" } }, { type: "text", content: "text" }, ], 0, @@ -219,7 +221,7 @@ describe("chatInputStateReducers", () => { const result = deleteBeforeCursor(initialState); expect(result.segments).toEqual([ - { type: "largePaste", content: "paste1" }, + { type: "chip", data: { kind: "largePaste", content: "paste1" } }, { type: "text", content: "text" }, ]); expect(result.cursorOnSegmentIndex).toBe(0); @@ -230,9 +232,9 @@ describe("chatInputStateReducers", () => { const initialState = createChatUserInputState( [ { type: "text", content: "abc" }, - { type: "largePaste", content: "pasted1" }, + { type: "chip", data: { kind: "largePaste", content: "pasted1" } }, { type: "text", content: "bb" }, - { type: "largePaste", content: "pasted2" }, + { type: "chip", data: { kind: "largePaste", content: "pasted2" } }, ], 2, 2, @@ -242,9 +244,9 @@ describe("chatInputStateReducers", () => { const afterFirstBackspace = deleteBeforeCursor(initialState); expect(afterFirstBackspace.segments).toEqual([ { type: "text", content: "abc" }, - { type: "largePaste", content: "pasted1" }, + { type: "chip", data: { kind: "largePaste", content: "pasted1" } }, { type: "text", content: "b" }, - { type: "largePaste", content: "pasted2" }, + { type: "chip", data: { kind: "largePaste", content: "pasted2" } }, ]); expect(afterFirstBackspace.cursorOnSegmentIndex).toBe(2); expect(afterFirstBackspace.cursorInSegmentOffset).toBe(1); @@ -253,8 +255,8 @@ describe("chatInputStateReducers", () => { const afterSecondBackspace = deleteBeforeCursor(afterFirstBackspace); expect(afterSecondBackspace.segments).toEqual([ { type: "text", content: "abc" }, - { type: "largePaste", content: "pasted1" }, - { type: "largePaste", content: "pasted2" }, + { type: "chip", data: { kind: "largePaste", content: "pasted1" } }, + { type: "chip", data: { kind: "largePaste", content: "pasted2" } }, ]); expect(afterSecondBackspace.cursorOnSegmentIndex).toBe(2); expect(afterSecondBackspace.cursorInSegmentOffset).toBe(0); @@ -263,12 +265,52 @@ describe("chatInputStateReducers", () => { const afterThirdBackspace = deleteBeforeCursor(afterSecondBackspace); expect(afterThirdBackspace.segments).toEqual([ { type: "text", content: "abc" }, - { type: "largePaste", content: "pasted2" }, + { type: "chip", data: { kind: "largePaste", content: "pasted2" } }, ]); expect(afterThirdBackspace.cursorOnSegmentIndex).toBe(0); expect(afterThirdBackspace.cursorInSegmentOffset).toBe(3); }); }); + describe("deleteBeforeCursorCount", () => { + it("deletes multiple characters within a text segment", () => { + const initialState = createChatUserInputState([{ type: "text", content: "hello" }], 0, 5); + + const result = deleteBeforeCursorCount(initialState, 2); + + expect(result.segments).toEqual([{ type: "text", content: "hel" }]); + expect(result.cursorOnSegmentIndex).toBe(0); + expect(result.cursorInSegmentOffset).toBe(3); + }); + + it("deletes across chip boundaries", () => { + const initialState = createChatUserInputState( + [ + { type: "text", content: "hi" }, + { type: "chip", data: { kind: "largePaste", content: "pasted" } }, + { type: "text", content: "ok" }, + ], + 2, + 0, + ); + + const result = deleteBeforeCursorCount(initialState, 2); + + expect(result.segments).toEqual([ + { type: "text", content: "h" }, + { type: "text", content: "ok" }, + ]); + expect(result.cursorOnSegmentIndex).toBe(0); + expect(result.cursorInSegmentOffset).toBe(1); + }); + + it("does nothing when count is zero", () => { + const initialState = createChatUserInputState([{ type: "text", content: "hello" }], 0, 2); + + const result = deleteBeforeCursorCount(initialState, 0); + + expect(result).toEqual(initialState); + }); + }); describe("deleteAfterCursor", () => { it("deletes character at cursor position within text segment", () => { const initialState = createChatUserInputState([{ type: "text", content: "hello" }], 0, 2); @@ -307,11 +349,11 @@ describe("chatInputStateReducers", () => { expect(result.cursorInSegmentOffset).toBe(5); }); - it("deletes next largePaste segment when cursor is at end of text segment", () => { + it("deletes next chip segment when cursor is at end of text segment", () => { const initialState: ChatUserInputState = { segments: [ { type: "text", content: "hello" }, - { type: "largePaste", content: "x".repeat(300) }, + { type: "chip", data: { kind: "largePaste", content: "x".repeat(300) } }, { type: "text", content: "" }, ], cursorOnSegmentIndex: 0, @@ -325,11 +367,11 @@ describe("chatInputStateReducers", () => { expect(result.cursorInSegmentOffset).toBe(5); }); - it("deletes largePaste segment when cursor is on it at offset 0", () => { + it("deletes chip segment when cursor is on it at offset 0", () => { const initialState: ChatUserInputState = { segments: [ { type: "text", content: "hello" }, - { type: "largePaste", content: "x".repeat(300) }, + { type: "chip", data: { kind: "largePaste", content: "x".repeat(300) } }, { type: "text", content: "" }, ], cursorOnSegmentIndex: 1, @@ -347,7 +389,7 @@ describe("chatInputStateReducers", () => { const initialState: ChatUserInputState = { segments: [ { type: "text", content: "hello" }, - { type: "largePaste", content: "x".repeat(300) }, + { type: "chip", data: { kind: "largePaste", content: "x".repeat(300) } }, { type: "text", content: "" }, ], cursorOnSegmentIndex: 1, @@ -358,7 +400,7 @@ describe("chatInputStateReducers", () => { expect(result.segments).toEqual([ { type: "text", content: "hello" }, - { type: "largePaste", content: "x".repeat(300) }, + { type: "chip", data: { kind: "largePaste", content: "x".repeat(300) } }, { type: "text", content: "" }, ]); expect(result.cursorOnSegmentIndex).toBe(1); @@ -423,7 +465,7 @@ describe("chatInputStateReducers", () => { const initialState: ChatUserInputState = { segments: [ { type: "text", content: "before" }, - { type: "largePaste", content: "x".repeat(300) }, + { type: "chip", data: { kind: "largePaste", content: "x".repeat(300) } }, { type: "text", content: "after" }, ], cursorOnSegmentIndex: 0, @@ -437,12 +479,12 @@ describe("chatInputStateReducers", () => { expect(result.cursorInSegmentOffset).toBe(6); }); - it("deletes multiple largePaste segments in succession", () => { + it("deletes multiple chip segments in succession", () => { const initialState: ChatUserInputState = { segments: [ { type: "text", content: "start" }, - { type: "largePaste", content: "x".repeat(300) }, - { type: "largePaste", content: "y".repeat(300) }, + { type: "chip", data: { kind: "largePaste", content: "x".repeat(300) } }, + { type: "chip", data: { kind: "largePaste", content: "y".repeat(300) } }, { type: "text", content: "" }, ], cursorOnSegmentIndex: 0, @@ -452,7 +494,7 @@ describe("chatInputStateReducers", () => { const afterFirst = deleteAfterCursor(initialState); expect(afterFirst.segments).toEqual([ { type: "text", content: "start" }, - { type: "largePaste", content: "y".repeat(300) }, + { type: "chip", data: { kind: "largePaste", content: "y".repeat(300) } }, { type: "text", content: "" }, ]); @@ -482,7 +524,7 @@ describe("chatInputStateReducers", () => { const initialState: ChatUserInputState = { segments: [ { type: "text", content: "hello" }, - { type: "largePaste", content: "x".repeat(300) }, + { type: "chip", data: { kind: "largePaste", content: "x".repeat(300) } }, { type: "text", content: "" }, ], cursorOnSegmentIndex: 1, @@ -558,8 +600,12 @@ describe("chatInputStateReducers", () => { const result = deleteAfterCursor(initialState); // Note: emoji is multi-byte UTF-16, slice removes first code unit - expect(result.segments[0].type).toBe("text"); - expect(result.segments[0].content.length).toBeLessThan("hello🎉world".length); + const firstSegment = result.segments[0]; + expect(firstSegment?.type).toBe("text"); + if (firstSegment?.type !== "text") { + throw new Error("Expected first segment to be text"); + } + expect(firstSegment.content.length).toBeLessThan("hello🎉world".length); expect(result.cursorOnSegmentIndex).toBe(0); expect(result.cursorInSegmentOffset).toBe(5); }); @@ -590,11 +636,11 @@ describe("chatInputStateReducers", () => { expect(result.cursorInSegmentOffset).toBe(4); }); - it("moves from start of text segment to previous largePaste segment", () => { + it("moves from start of text segment to previous chip segment", () => { const initialState = createChatUserInputState( [ { type: "text", content: "first" }, - { type: "largePaste", content: "pasted" }, + { type: "chip", data: { kind: "largePaste", content: "pasted" } }, { type: "text", content: "third" }, ], 2, @@ -607,11 +653,11 @@ describe("chatInputStateReducers", () => { expect(result.cursorInSegmentOffset).toBe(0); }); - it("moves to previous largePaste segment when current segment is largePaste with offset at start", () => { + it("moves to previous chip segment when current segment is largePaste with offset at start", () => { const initialState = createChatUserInputState( [ - { type: "largePaste", content: "first" }, - { type: "largePaste", content: "second" }, + { type: "chip", data: { kind: "largePaste", content: "first" } }, + { type: "chip", data: { kind: "largePaste", content: "second" } }, { type: "text", content: "tail" }, ], 1, @@ -628,7 +674,7 @@ describe("chatInputStateReducers", () => { const initialState = createChatUserInputState( [ { type: "text", content: "hello" }, - { type: "largePaste", content: "pasted" }, + { type: "chip", data: { kind: "largePaste", content: "pasted" } }, { type: "text", content: "tail" }, ], 1, @@ -641,11 +687,11 @@ describe("chatInputStateReducers", () => { expect(result.cursorInSegmentOffset).toBe(4); }); - it("moves to start of current largePaste segment when cursor offset is greater than zero", () => { + it("moves to start of current chip segment when cursor offset is greater than zero", () => { const initialState = createChatUserInputState( [ { type: "text", content: "text" }, - { type: "largePaste", content: "pasted" }, + { type: "chip", data: { kind: "largePaste", content: "pasted" } }, ], 1, 1, @@ -657,10 +703,10 @@ describe("chatInputStateReducers", () => { expect(result.cursorInSegmentOffset).toBe(0); }); - it("does nothing when at first largePaste segment and moving left", () => { + it("does nothing when at first chip segment and moving left", () => { const initialState = createChatUserInputState( [ - { type: "largePaste", content: "pasted" }, + { type: "chip", data: { kind: "largePaste", content: "pasted" } }, { type: "text", content: "tail" }, ], 0, @@ -700,11 +746,11 @@ describe("chatInputStateReducers", () => { expect(result.cursorInSegmentOffset).toBe(5); }); - it("skips over a single largePaste segment when moving right from text", () => { + it("skips over a single chip segment when moving right from text", () => { const initialState = createChatUserInputState( [ { type: "text", content: "A" }, - { type: "largePaste", content: "P1" }, + { type: "chip", data: { kind: "largePaste", content: "P1" } }, { type: "text", content: "B" }, ], 0, @@ -721,8 +767,8 @@ describe("chatInputStateReducers", () => { const initialState = createChatUserInputState( [ { type: "text", content: "A" }, - { type: "largePaste", content: "P1" }, - { type: "largePaste", content: "P2" }, + { type: "chip", data: { kind: "largePaste", content: "P1" } }, + { type: "chip", data: { kind: "largePaste", content: "P2" } }, ], 0, 1, @@ -737,7 +783,7 @@ describe("chatInputStateReducers", () => { it("moves from largePaste to next text segment", () => { const initialState = createChatUserInputState( [ - { type: "largePaste", content: "P1" }, + { type: "chip", data: { kind: "largePaste", content: "P1" } }, { type: "text", content: "tail" }, ], 0, @@ -753,8 +799,8 @@ describe("chatInputStateReducers", () => { it("moves from one largePaste to the next largePaste in sequence", () => { const initialState = createChatUserInputState( [ - { type: "largePaste", content: "P1" }, - { type: "largePaste", content: "P2" }, + { type: "chip", data: { kind: "largePaste", content: "P1" } }, + { type: "chip", data: { kind: "largePaste", content: "P2" } }, { type: "text", content: "tail" }, ], 0, @@ -767,12 +813,12 @@ describe("chatInputStateReducers", () => { expect(result.cursorInSegmentOffset).toBe(0); }); - it("preserves all segments when moving right between back-to-back largePaste segments", () => { + it("preserves all segments when moving right between back-to-back chip segments", () => { const initialState = createChatUserInputState( [ - { type: "largePaste", content: "Paste1" }, - { type: "largePaste", content: "Paste2" }, - { type: "largePaste", content: "Paste3" }, + { type: "chip", data: { kind: "largePaste", content: "Paste1" } }, + { type: "chip", data: { kind: "largePaste", content: "Paste2" } }, + { type: "chip", data: { kind: "largePaste", content: "Paste3" } }, { type: "text", content: "trailing" }, ], 0, @@ -782,19 +828,19 @@ describe("chatInputStateReducers", () => { const result = moveCursorRight(initialState); expect(result.segments.length).toBe(4); - expect(result.segments[0]).toEqual({ type: "largePaste", content: "Paste1" }); - expect(result.segments[1]).toEqual({ type: "largePaste", content: "Paste2" }); - expect(result.segments[2]).toEqual({ type: "largePaste", content: "Paste3" }); + expect(result.segments[0]).toEqual({ type: "chip", data: { kind: "largePaste", content: "Paste1" } }); + expect(result.segments[1]).toEqual({ type: "chip", data: { kind: "largePaste", content: "Paste2" } }); + expect(result.segments[2]).toEqual({ type: "chip", data: { kind: "largePaste", content: "Paste3" } }); expect(result.segments[3]).toEqual({ type: "text", content: "trailing" }); expect(result.cursorOnSegmentIndex).toBe(1); expect(result.cursorInSegmentOffset).toBe(0); }); - it("does nothing when moving right from last largePaste segment", () => { + it("does nothing when moving right from last chip segment", () => { const initialState = createChatUserInputState( [ { type: "text", content: "head" }, - { type: "largePaste", content: "P1" }, + { type: "chip", data: { kind: "largePaste", content: "P1" } }, ], 1, 0, @@ -823,7 +869,7 @@ describe("chatInputStateReducers", () => { it("creates a new text segment before a leading largePaste when there is no previous text", () => { const initialState = createChatUserInputState( - [{ type: "largePaste", content: "pasted" }], + [{ type: "chip", data: { kind: "largePaste", content: "pasted" } }], 0, 0, ); @@ -835,7 +881,7 @@ describe("chatInputStateReducers", () => { expect(result.segments).toEqual([ { type: "text", content: "typed" }, - { type: "largePaste", content: "pasted" }, + { type: "chip", data: { kind: "largePaste", content: "pasted" } }, ]); expect(result.cursorOnSegmentIndex).toBe(0); expect(result.cursorInSegmentOffset).toBe(5); @@ -845,7 +891,7 @@ describe("chatInputStateReducers", () => { const initialState = createChatUserInputState( [ { type: "text", content: "before" }, - { type: "largePaste", content: "pasted" }, + { type: "chip", data: { kind: "largePaste", content: "pasted" } }, ], 1, 0, @@ -858,7 +904,7 @@ describe("chatInputStateReducers", () => { expect(result.segments).toEqual([ { type: "text", content: "beforeX" }, - { type: "largePaste", content: "pasted" }, + { type: "chip", data: { kind: "largePaste", content: "pasted" } }, ]); expect(result.cursorOnSegmentIndex).toBe(0); expect(result.cursorInSegmentOffset).toBe(7); @@ -909,7 +955,7 @@ describe("chatInputStateReducers", () => { expect(result.segments.length).toBe(3); expect(result.segments[0]).toEqual({ type: "text", content: "before" }); - expect(result.segments[1]).toEqual({ type: "largePaste", content: "PASTE_CONTENT" }); + expect(result.segments[1]).toEqual({ type: "chip", data: { kind: "largePaste", content: "PASTE_CONTENT" } }); expect(result.segments[2]).toEqual({ type: "text", content: "-after" }); expect(result.cursorOnSegmentIndex).toBe(2); expect(result.cursorInSegmentOffset).toBe(0); @@ -917,7 +963,7 @@ describe("chatInputStateReducers", () => { it("inserts small paste as text segment before current largePaste when cursor is at start", () => { const initialState = createChatUserInputState( - [{ type: "largePaste", content: "existing" }], + [{ type: "chip", data: { kind: "largePaste", content: "existing" } }], 0, 0, ); @@ -930,7 +976,7 @@ describe("chatInputStateReducers", () => { expect(result.segments).toEqual([ { type: "text", content: "txt" }, - { type: "largePaste", content: "existing" }, + { type: "chip", data: { kind: "largePaste", content: "existing" } }, ]); expect(result.cursorOnSegmentIndex).toBe(0); expect(result.cursorInSegmentOffset).toBe(3); @@ -940,8 +986,8 @@ describe("chatInputStateReducers", () => { const initialState = createChatUserInputState( [ { type: "text", content: "T1" }, - { type: "largePaste", content: "P1" }, - { type: "largePaste", content: "P2" }, + { type: "chip", data: { kind: "largePaste", content: "P1" } }, + { type: "chip", data: { kind: "largePaste", content: "P2" } }, { type: "text", content: "T2" }, ], 3, @@ -956,9 +1002,9 @@ describe("chatInputStateReducers", () => { expect(result.segments).toEqual([ { type: "text", content: "T1" }, - { type: "largePaste", content: "P1" }, - { type: "largePaste", content: "P2" }, - { type: "largePaste", content: "BIG_PASTE" }, + { type: "chip", data: { kind: "largePaste", content: "P1" } }, + { type: "chip", data: { kind: "largePaste", content: "P2" } }, + { type: "chip", data: { kind: "largePaste", content: "BIG_PASTE" } }, { type: "text", content: "T2" }, ]); expect(result.cursorOnSegmentIndex).toBe(4); @@ -966,6 +1012,70 @@ describe("chatInputStateReducers", () => { }); }); + describe("insertImageAtCursor", () => { + it("inserts an image chip inside a text segment and preserves text after cursor", () => { + const initialState = createChatUserInputState([{ type: "text", content: "hello" }], 0, 2); + + const result = insertImageAtCursor({ + state: initialState, + image: { + source: "base64", + fileName: "cat.png", + contentBase64: "aGVsbG8=", + name: "cat.png", + mime: "image/png", + }, + }); + + expect(result.segments).toEqual([ + { type: "text", content: "he" }, + { + type: "chip", + data: { + kind: "image", + source: "base64", + fileName: "cat.png", + contentBase64: "aGVsbG8=", + name: "cat.png", + mime: "image/png", + }, + }, + { type: "text", content: "llo" }, + ]); + expect(result.cursorOnSegmentIndex).toBe(2); + expect(result.cursorInSegmentOffset).toBe(0); + }); + + it("inserts an image chip before current chip when cursor is on a chip", () => { + const initialState = createChatUserInputState( + [ + { type: "text", content: "head" }, + { type: "chip", data: { kind: "largePaste", content: "PASTE" } }, + { type: "text", content: "tail" }, + ], + 1, + 0, + ); + + const result = insertImageAtCursor({ + state: initialState, + image: { source: "base64", fileName: "cat.png", contentBase64: "aGVsbG8=" }, + }); + + expect(result.segments).toEqual([ + { type: "text", content: "head" }, + { + type: "chip", + data: { kind: "image", source: "base64", fileName: "cat.png", contentBase64: "aGVsbG8=" }, + }, + { type: "chip", data: { kind: "largePaste", content: "PASTE" } }, + { type: "text", content: "tail" }, + ]); + expect(result.cursorOnSegmentIndex).toBe(1); + expect(result.cursorInSegmentOffset).toBe(0); + }); + }); + describe("insertSuggestionAtCursor", () => { it("replaces last text segment content with suggestion text", () => { const initialState = createChatUserInputState([{ type: "text", content: "/mod" }], 0, 4); @@ -985,7 +1095,7 @@ describe("chatInputStateReducers", () => { const initialState = createChatUserInputState( [ { type: "text", content: "hello" }, - { type: "largePaste", content: "x".repeat(1000) }, + { type: "chip", data: { kind: "largePaste", content: "x".repeat(1000) } }, ], 1, 0, @@ -998,7 +1108,7 @@ describe("chatInputStateReducers", () => { expect(result.segments).toHaveLength(3); expect(result.segments[0]).toEqual({ type: "text", content: "hello" }); - expect(result.segments[1]).toEqual({ type: "largePaste", content: "x".repeat(1000) }); + expect(result.segments[1]).toEqual({ type: "chip", data: { kind: "largePaste", content: "x".repeat(1000) } }); expect(result.segments[2]).toEqual({ type: "text", content: "/help " }); expect(result.cursorOnSegmentIndex).toBe(2); expect(result.cursorInSegmentOffset).toBe(6); @@ -1008,7 +1118,7 @@ describe("chatInputStateReducers", () => { const initialState = createChatUserInputState( [ { type: "text", content: "hello" }, - { type: "largePaste", content: "x".repeat(1000) }, + { type: "chip", data: { kind: "largePaste", content: "x".repeat(1000) } }, { type: "text", content: "/co" }, ], 2, @@ -1022,7 +1132,7 @@ describe("chatInputStateReducers", () => { expect(result.segments).toHaveLength(3); expect(result.segments[0]).toEqual({ type: "text", content: "hello" }); - expect(result.segments[1]).toEqual({ type: "largePaste", content: "x".repeat(1000) }); + expect(result.segments[1]).toEqual({ type: "chip", data: { kind: "largePaste", content: "x".repeat(1000) } }); expect(result.segments[2]).toEqual({ type: "text", content: "/context " }); expect(result.cursorOnSegmentIndex).toBe(2); expect(result.cursorInSegmentOffset).toBe(9); @@ -1035,7 +1145,7 @@ describe("chatInputStateReducers", () => { const initialState = createChatUserInputState( [ { type: "text", content: "hello" }, - { type: "largePaste", content: "x".repeat(1000) }, + { type: "chip", data: { kind: "largePaste", content: "x".repeat(1000) } }, { type: "text", content: "" }, ], 2, @@ -1137,9 +1247,9 @@ describe("chatInputStateReducers", () => { expect(result.cursorInSegmentOffset).toBe(0); }); - it("clamps negative cursor offset to zero for largePaste segment", () => { + it("clamps negative cursor offset to zero for chip segment", () => { const initialState = createChatUserInputState( - [{ type: "largePaste", content: "x".repeat(1000) }], + [{ type: "chip", data: { kind: "largePaste", content: "x".repeat(1000) } }], 0, -5, ); @@ -1172,7 +1282,7 @@ describe("chatInputStateReducers", () => { const initialState = createChatUserInputState( [ { type: "text", content: "a" }, - { type: "largePaste", content: "large content" }, + { type: "chip", data: { kind: "largePaste", content: "large content" } }, { type: "text", content: "" }, ], 0, @@ -1184,13 +1294,13 @@ describe("chatInputStateReducers", () => { expect(result.cursorOnSegmentIndex).toBe(0); expect(result.cursorInSegmentOffset).toBe(0); expect(result.segments.length).toBe(2); - expect(result.segments[0]).toEqual({ type: "largePaste", content: "large content" }); + expect(result.segments[0]).toEqual({ type: "chip", data: { kind: "largePaste", content: "large content" } }); }); it("removes single character before largePaste if cursor is in text and keeps cursor at largePaste offset 0", () => { const initialState = createChatUserInputState( [ { type: "text", content: "a" }, - { type: "largePaste", content: "large content" }, + { type: "chip", data: { kind: "largePaste", content: "large content" } }, { type: "text", content: "" }, ], 0, @@ -1202,7 +1312,7 @@ describe("chatInputStateReducers", () => { expect(result.cursorOnSegmentIndex).toBe(0); expect(result.cursorInSegmentOffset).toBe(0); expect(result.segments.length).toBe(2); - expect(result.segments[0]).toEqual({ type: "largePaste", content: "large content" }); + expect(result.segments[0]).toEqual({ type: "chip", data: { kind: "largePaste", content: "large content" } }); }); }); @@ -1229,7 +1339,7 @@ describe("chatInputStateReducers", () => { const initialState = createChatUserInputState( [ { type: "text", content: "hello" }, - { type: "largePaste", content: "x".repeat(1000) }, + { type: "chip", data: { kind: "largePaste", content: "x".repeat(1000) } }, ], 0, 5, @@ -1254,7 +1364,7 @@ describe("chatInputStateReducers", () => { describe("insertTextAtCursor edge cases", () => { it("creates new text segment before largePaste when it is the first segment", () => { const initialState = createChatUserInputState( - [{ type: "largePaste", content: "x".repeat(1000) }], + [{ type: "chip", data: { kind: "largePaste", content: "x".repeat(1000) } }], 0, 0, ); @@ -1266,7 +1376,7 @@ describe("chatInputStateReducers", () => { expect(result.segments.length).toBe(2); expect(result.segments[0]).toEqual({ type: "text", content: "prefix" }); - expect(result.segments[1]).toEqual({ type: "largePaste", content: "x".repeat(1000) }); + expect(result.segments[1]).toEqual({ type: "chip", data: { kind: "largePaste", content: "x".repeat(1000) } }); expect(result.cursorOnSegmentIndex).toBe(0); expect(result.cursorInSegmentOffset).toBe(6); }); @@ -1275,7 +1385,7 @@ describe("chatInputStateReducers", () => { describe("insertPasteAtCursor edge cases", () => { it("inserts large paste before current largePaste when cursor is at start", () => { const initialState = createChatUserInputState( - [{ type: "largePaste", content: "x".repeat(1000) }], + [{ type: "chip", data: { kind: "largePaste", content: "x".repeat(1000) } }], 0, 0, ); @@ -1287,14 +1397,14 @@ describe("chatInputStateReducers", () => { }); expect(result.segments.length).toBe(2); - expect(result.segments[0]).toEqual({ type: "largePaste", content: "y".repeat(1000) }); - expect(result.segments[1]).toEqual({ type: "largePaste", content: "x".repeat(1000) }); + expect(result.segments[0]).toEqual({ type: "chip", data: { kind: "largePaste", content: "y".repeat(1000) } }); + expect(result.segments[1]).toEqual({ type: "chip", data: { kind: "largePaste", content: "x".repeat(1000) } }); expect(result.cursorOnSegmentIndex).toBe(0); }); it("inserts small paste as text before current largePaste when cursor is at start", () => { const initialState = createChatUserInputState( - [{ type: "largePaste", content: "x".repeat(1000) }], + [{ type: "chip", data: { kind: "largePaste", content: "x".repeat(1000) } }], 0, 0, ); @@ -1307,7 +1417,7 @@ describe("chatInputStateReducers", () => { expect(result.segments.length).toBe(2); expect(result.segments[0]).toEqual({ type: "text", content: "small" }); - expect(result.segments[1]).toEqual({ type: "largePaste", content: "x".repeat(1000) }); + expect(result.segments[1]).toEqual({ type: "chip", data: { kind: "largePaste", content: "x".repeat(1000) } }); }); }); }); diff --git a/src/subcommands/chat/react/inputReducer.ts b/src/subcommands/chat/react/inputReducer.ts index 16b7413d..3dfa74ad 100644 --- a/src/subcommands/chat/react/inputReducer.ts +++ b/src/subcommands/chat/react/inputReducer.ts @@ -2,36 +2,35 @@ * Input Reducer for Chat User Input State * * This module manages a multi-segment text input buffer that supports both regular text - * and large paste segments (or chips). The buffer is designed to handle large pastes efficiently - * by treating them as separate segments that can be removed/navigated independently. + * and chip segments. The buffer is designed to handle large pastes efficiently by treating them + * as separate non-text segments that can be removed/navigated independently. * * Segment Model: * - `text`: Regular text segments where the user can type. Cursor can be positioned * anywhere within the text (0 to content.length). - * - `largePaste`: Read-only paste segments for large content. Cursor can only be at - * position 0 (start) and is typically used for navigation. + * - `chip`: Read-only chip segments (e.g. largePaste, image). Cursor can only be at + * position 0 (start) and is typically used for navigation. * - * We do not throw error if in largePaste segment cursorInSegmentOffset > 0, instead + * We do not throw error if in chip segment cursorInSegmentOffset > 0, instead * we sanitize it back to 0. * - * There will be a trailing empty text segment after a largePaste to allow typing - * after the paste. + * There will be a trailing empty text segment after a chip segment to allow typing after it. * * Cursor Semantics: * - `cursorOnSegmentIndex`: Which segment the cursor is currently on * - `cursorInSegmentOffset`: Position within that segment * - For text segments: 0 to content.length (0 = before first char, length = after last char) - * - For largePaste segments: 0 = start, 1 = treated as "inside" for certain operations + * - For chip segments: 0 = start * * Sanitation: * After each mutation, the state is automatically sanitized to ensure: * - At least one segment always exists - * - Empty text segments are removed (except trailing placeholders after largePaste) + * - Empty text segments are removed (except trailing placeholders after chip) * - Cursor indices are within valid bounds */ import { produce } from "@lmstudio/immer-with-plugins"; -import { type ChatUserInputState } from "./types.js"; +import { type ChatInputData, type ChatInputSegment, type ChatUserInputState } from "./types.js"; interface InsertTextAtCursorOpts { state: ChatUserInputState; @@ -49,8 +48,17 @@ interface InsertSuggestionAtCursorOpts { suggestionText: string; } +interface InsertImageAtCursorOpts { + state: ChatUserInputState; + image: InsertableChatImageData; +} + type ChatUserInputStateMutator = (draft: ChatUserInputState) => void; +type ChatImageData = Extract; +type DistributiveOmit = T extends any ? Omit : never; +type InsertableChatImageData = DistributiveOmit; + /** * Wrapper that applies a mutation to the state and automatically sanitizes it afterward. * Uses Immer to create an immutable update. @@ -68,14 +76,14 @@ function produceSanitizedState( /** * Ensures the input state is valid by: * 1. Guaranteeing at least one segment exists - * 2. Removing empty text segments (except trailing placeholders after largePaste) + * 2. Removing empty text segments (except trailing placeholders after chip) * 3. Clamping cursor indices to valid bounds * 4. Adjusting cursor offsets to valid ranges for each segment type * 5. Merging consecutive text segments to prevent navigation issues */ function sanitizeChatUserInputState(state: ChatUserInputState): void { // Remove empty text segments, except "trailing placeholders" - // A trailing placeholder is an empty text segment after a largePaste that allows typing + // A trailing placeholder is an empty text segment after a chip segment that allows typing for (let segmentIndex = state.segments.length - 1; segmentIndex >= 0; segmentIndex -= 1) { const segment = state.segments[segmentIndex]; // Skip non-text segments or non-empty text segments @@ -84,8 +92,8 @@ function sanitizeChatUserInputState(state: ChatUserInputState): void { } const isLastSegment = segmentIndex === state.segments.length - 1; const previousSegment = state.segments[segmentIndex - 1]; - const isTrailingPlaceholder = isLastSegment === true && previousSegment?.type === "largePaste"; - // Keep trailing placeholders - they allow typing after largePaste segments + const isTrailingPlaceholder = isLastSegment === true && previousSegment?.type === "chip"; + // Keep trailing placeholders - they allow typing after chip segments if (isTrailingPlaceholder === true) { continue; } @@ -173,7 +181,7 @@ function sanitizeChatUserInputState(state: ChatUserInputState): void { state.cursorInSegmentOffset = activeSegment.content.length; } } else if (state.cursorInSegmentOffset < 0 || state.cursorInSegmentOffset > 0) { - // For largePaste segments, ensure it is always 0 + // For chip segments, ensure it is always 0 state.cursorInSegmentOffset = 0; } } @@ -182,7 +190,7 @@ function sanitizeChatUserInputState(state: ChatUserInputState): void { * Deletes content before the cursor (backspace behavior). * Handles all segment types and cursor positions: * - On text segment: deletes character or merges with previous segment - * - On largePaste at offset 0: deletes the previous segment + * - On chip at offset 0: deletes the previous segment */ export function deleteBeforeCursor(state: ChatUserInputState): ChatUserInputState { return produceSanitizedState(state, draft => { @@ -226,12 +234,12 @@ export function deleteBeforeCursor(state: ChatUserInputState): ChatUserInputStat draft.cursorInSegmentOffset = 0; return; } else { - // Previous is largePaste - delete the entire segment + // Previous is data - delete the entire segment draft.segments.splice(previousSegmentIndex, 1); draft.cursorOnSegmentIndex = Math.max(0, previousSegmentIndex - 1); if (previousSegmentIndex > 0) { - // Cursor moves to segment before the deleted largePaste + // Cursor moves to segment before the deleted data const currentSegmentAfterDeletion = draft.segments[draft.cursorOnSegmentIndex]; if (currentSegmentAfterDeletion?.type === "text") { draft.cursorInSegmentOffset = currentSegmentAfterDeletion.content.length; @@ -246,8 +254,8 @@ export function deleteBeforeCursor(state: ChatUserInputState): ChatUserInputStat } } - if (currentSegment.type === "largePaste") { - // Error: cursor should not be > 0 on largePaste segments + if (currentSegment.type === "chip") { + // Error: cursor should not be > 0 on chip segments return; } @@ -260,6 +268,23 @@ export function deleteBeforeCursor(state: ChatUserInputState): ChatUserInputStat }); } +/** + * Deletes multiple characters before the cursor by applying deleteBeforeCursor repeatedly. + */ +export function deleteBeforeCursorCount( + state: ChatUserInputState, + count: number, +): ChatUserInputState { + if (count <= 0) { + return state; + } + let nextState = state; + for (let i = 0; i < count; i += 1) { + nextState = deleteBeforeCursor(nextState); + } + return nextState; +} + /** * Deletes the character after the cursor (Delete key behavior). * - Within text: deletes character at cursor position @@ -305,18 +330,18 @@ export function deleteAfterCursor(state: ChatUserInputState): ChatUserInputState draft.segments.splice(nextSegmentIndex, 1); return; } else { - // Next is largePaste - delete it entirely + // Next is data - delete it entirely draft.segments.splice(nextSegmentIndex, 1); return; } } - case "largePaste": { - // Cursor should be at offset 0 on largePaste + case "chip": { + // Cursor should be at offset 0 on chip if (draft.cursorInSegmentOffset !== 0) { return; } - // Delete the largePaste segment + // Delete the chip segment draft.segments.splice(draft.cursorOnSegmentIndex, 1); if (draft.segments.length === 0) { @@ -338,7 +363,7 @@ export function deleteAfterCursor(state: ChatUserInputState): ChatUserInputState /** * Moves the cursor one position to the left. * - Within text: moves one character left - * - On largePaste: moves to start of current largePaste + * - On chip: moves to start of current chip * - At start of anything: moves to previous segment */ export function moveCursorLeft(state: ChatUserInputState): ChatUserInputState { @@ -347,12 +372,12 @@ export function moveCursorLeft(state: ChatUserInputState): ChatUserInputState { if (currentSegment === undefined) { return; } - if (currentSegment.type === "largePaste") { + if (currentSegment.type === "chip") { if (draft.cursorOnSegmentIndex === 0) { return; // Already at first segment } if (draft.cursorInSegmentOffset > 0) { - // Error: Not expected. Move to start of current largePaste + // Error: Not expected. Move to start of current chip draft.cursorInSegmentOffset = 0; return; } @@ -373,7 +398,7 @@ export function moveCursorLeft(state: ChatUserInputState): ChatUserInputState { if (previousSegment === undefined) { return; } - if (previousSegment.type === "largePaste") { + if (previousSegment.type === "chip") { draft.cursorOnSegmentIndex = previousSegmentIndex; draft.cursorInSegmentOffset = 0; return; @@ -388,7 +413,7 @@ export function moveCursorLeft(state: ChatUserInputState): ChatUserInputState { * Moves the cursor one position to the right. * - Within text: moves one character right * - At end of text: moves to next segment - * - On largePaste: moves to the next segment (skipping over the largePaste) + * - On chip: moves to the next segment (skipping over chips when moving from text) */ export function moveCursorRight(state: ChatUserInputState): ChatUserInputState { return produceSanitizedState(state, draft => { @@ -405,7 +430,7 @@ export function moveCursorRight(state: ChatUserInputState): ChatUserInputState { return; } } - // At end of segment or on largePaste - move to next segment, handling largePast skips + // At end of segment or on chip - move to next segment, handling chip skips if (draft.cursorOnSegmentIndex >= draft.segments.length - 1) { return; // Already at last segment } @@ -414,19 +439,19 @@ export function moveCursorRight(state: ChatUserInputState): ChatUserInputState { if (nextSegment === undefined) { return; } - if (currentSegment.type === "largePaste") { - // Move to next segment (whether it's text or largePaste) + if (currentSegment.type === "chip") { + // Move to next segment (whether it's text or chip) draft.cursorOnSegmentIndex = nextSegmentIndex; draft.cursorInSegmentOffset = 0; return; } - if (nextSegment.type === "largePaste") { - // Skip over largePaste to the segment after it - const segmentAfterPaste = nextSegmentIndex + 1; - if (segmentAfterPaste >= draft.segments.length) { - return; // No segment after the largePaste + if (nextSegment.type === "chip") { + // Skip over chip to the segment after it + const segmentAfterChip = nextSegmentIndex + 1; + if (segmentAfterChip >= draft.segments.length) { + return; // No segment after the chip } - draft.cursorOnSegmentIndex = segmentAfterPaste; + draft.cursorOnSegmentIndex = segmentAfterChip; draft.cursorInSegmentOffset = 0; return; } @@ -439,8 +464,8 @@ export function moveCursorRight(state: ChatUserInputState): ChatUserInputState { /** * Inserts text at the cursor position. * - In text segment: inserts text at cursor position - * - At start of largePaste: appends to previous text segment or creates new text segment - * (there is no real "inside" for largePaste) + * - At start of chip: appends to previous text segment or creates new text segment + * (there is no real "inside" for chip) */ export function insertTextAtCursor({ state, text }: InsertTextAtCursorOpts): ChatUserInputState { return produceSanitizedState(state, draft => { @@ -448,23 +473,23 @@ export function insertTextAtCursor({ state, text }: InsertTextAtCursorOpts): Cha if (currentSegment === undefined) { return; } - if (currentSegment.type === "largePaste") { + if (currentSegment.type === "chip") { if (draft.cursorInSegmentOffset !== 0) { - // Something is wrong - we can only insert text at offset 0 of largePaste + // Something is wrong - we can only insert text at offset 0 of chip return; } - const largePasteIndex = draft.cursorOnSegmentIndex; - // At start of largePaste - try to append to previous text segment - const previousSegment = draft.segments[largePasteIndex - 1]; + const chipIndex = draft.cursorOnSegmentIndex; + // At start of chip - try to append to previous text segment + const previousSegment = draft.segments[chipIndex - 1]; if (previousSegment !== undefined && previousSegment.type === "text") { previousSegment.content += text; - draft.cursorOnSegmentIndex = largePasteIndex - 1; + draft.cursorOnSegmentIndex = chipIndex - 1; draft.cursorInSegmentOffset = previousSegment.content.length; return; } - // No previous text segment - create a new one before the largePaste - draft.segments.splice(largePasteIndex, 0, { type: "text", content: text }); - draft.cursorOnSegmentIndex = largePasteIndex; + // No previous text segment - create a new one before the chip + draft.segments.splice(chipIndex, 0, { type: "text", content: text }); + draft.cursorOnSegmentIndex = chipIndex; draft.cursorInSegmentOffset = text.length; return; } @@ -484,9 +509,9 @@ export function insertTextAtCursor({ state, text }: InsertTextAtCursorOpts): Cha * Inserts pasted content at the cursor position. * Content is treated as "large paste" if it exceeds largePasteThreshold. * - Small paste: inserted as regular text - * - Large paste: creates a largePaste segment with a trailing placeholder text segment - * - In text segment: splits the text, inserts largePaste, preserves text after cursor - * - On largePaste at offset 0: inserts before the largePaste segment + * - Large paste: creates a chip segment with a trailing placeholder text segment + * - In text segment: splits the text, inserts chip, preserves text after cursor + * - On chip at offset 0: inserts before the chip segment */ export function insertPasteAtCursor({ state, @@ -502,20 +527,23 @@ export function insertPasteAtCursor({ return; } const isLargePaste = content.length >= largePasteThreshold; - if (currentSegment.type === "largePaste") { - // Inserting paste while on a largePaste segment + if (currentSegment.type === "chip") { + // Inserting paste while on a chip segment if (draft.cursorInSegmentOffset !== 0) { - // Error: cursor should only be at offset 0 on largePaste segments + // Error: cursor should only be at offset 0 on chip segments return; } - const largePasteIndex = draft.cursorOnSegmentIndex; - // Insert before current largePaste - draft.segments.splice(largePasteIndex, 0, { - type: isLargePaste === true ? "largePaste" : "text", - content, - }); - draft.cursorOnSegmentIndex = largePasteIndex; - draft.cursorInSegmentOffset = content.length; + const chipIndex = draft.cursorOnSegmentIndex; + // Insert before current chip + draft.segments.splice( + chipIndex, + 0, + isLargePaste === true + ? { type: "chip", data: { kind: "largePaste", content } } + : { type: "text", content }, + ); + draft.cursorOnSegmentIndex = chipIndex; + draft.cursorInSegmentOffset = isLargePaste === true ? 0 : content.length; return; } if (currentSegment.type !== "text") { @@ -523,22 +551,22 @@ export function insertPasteAtCursor({ } const cursorPosition = draft.cursorInSegmentOffset; if (isLargePaste === true) { - // Split current text segment to insert largePaste + // Split current text segment to insert chip const textBeforeCursor = currentSegment.content.slice(0, cursorPosition); const textAfterCursor = currentSegment.content.slice(cursorPosition); currentSegment.content = textBeforeCursor; const insertIndex = draft.cursorOnSegmentIndex + 1; - // Insert largePaste followed by trailing text segment (preserves text after cursor) + // Insert chip followed by trailing text segment (preserves text after cursor) draft.segments.splice( insertIndex, 0, - { type: "largePaste", content }, + { type: "chip", data: { kind: "largePaste", content } }, { type: "text", content: textAfterCursor, // Empty string if no text after cursor }, ); - // Place cursor in the trailing text segment after largePaste + // Place cursor in the trailing text segment after chip draft.cursorOnSegmentIndex = insertIndex + 1; draft.cursorInSegmentOffset = 0; return; @@ -552,11 +580,52 @@ export function insertPasteAtCursor({ }); } +/** + * Inserts an image chip at the cursor position. + * - In text segment: splits the text, inserts image chip, preserves text after cursor + * - On chip at offset 0: inserts before the chip + * + * Cursor is placed on the inserted chip (when inserting before an existing chip), + * or on the trailing text segment (when inserting inside a text segment). + */ +export function insertImageAtCursor({ state, image }: InsertImageAtCursorOpts): ChatUserInputState { + return produceSanitizedState(state, draft => { + const currentSegment = draft.segments[draft.cursorOnSegmentIndex]; + if (currentSegment === undefined) { + return; + } + + const imageData: ChatImageData = { kind: "image", ...image }; + const chip: ChatInputSegment = { type: "chip", data: imageData }; + + if (currentSegment.type === "chip") { + if (draft.cursorInSegmentOffset !== 0) { + return; + } + const chipIndex = draft.cursorOnSegmentIndex; + draft.segments.splice(chipIndex, 0, chip); + draft.cursorOnSegmentIndex = chipIndex; + draft.cursorInSegmentOffset = 0; + return; + } + + const cursorPosition = draft.cursorInSegmentOffset; + const textBeforeCursor = currentSegment.content.slice(0, cursorPosition); + const textAfterCursor = currentSegment.content.slice(cursorPosition); + currentSegment.content = textBeforeCursor; + + const insertIndex = draft.cursorOnSegmentIndex + 1; + draft.segments.splice(insertIndex, 0, chip, { type: "text", content: textAfterCursor }); + draft.cursorOnSegmentIndex = insertIndex + 1; + draft.cursorInSegmentOffset = 0; + }); +} + /** * Inserts a suggestion at the cursor position by replacing the last segment's content. * Used for autocomplete/suggestion acceptance (e.g., /model or /download suggestions). * - If last segment is text: replaces its content with suggestion - * - If last segment is largePaste: appends a new text segment with suggestion + * - If last segment is chip: appends a new text segment with suggestion * Cursor is placed at the end of the inserted suggestion. */ export function insertSuggestionAtCursor({ diff --git a/src/subcommands/chat/react/inputRenderer.tsx b/src/subcommands/chat/react/inputRenderer.tsx index de4660ae..dd3c2f3a 100644 --- a/src/subcommands/chat/react/inputRenderer.tsx +++ b/src/subcommands/chat/react/inputRenderer.tsx @@ -2,22 +2,17 @@ import { Transform } from "ink"; import { type JSX } from "react"; import chalk from "chalk"; -interface PasteRange { - start: number; - end: number; -} - interface RenderInputWithCursorOpts { fullText: string; cursorPosition: number; - pasteRanges: PasteRange[]; + chipRanges: Array<{ start: number; end: number; kind: "largePaste" | "image" }>; lineStartPos: number; } export function renderInputWithCursor({ fullText, cursorPosition, - pasteRanges, + chipRanges, lineStartPos, }: RenderInputWithCursorOpts): JSX.Element { if (fullText.length === 0 && cursorPosition === 0) { @@ -35,11 +30,13 @@ export function renderInputWithCursor({ if (index === cursorPosition) { result += chalk.inverse(char); } else { - const isInPasteRange = pasteRanges.some( - range => absolutePos >= range.start && absolutePos < range.end, + const range = chipRanges.find( + highlight => absolutePos >= highlight.start && absolutePos < highlight.end, ); - if (isInPasteRange) { + if (range?.kind === "largePaste") { result += chalk.blue(char); + } else if (range?.kind === "image") { + result += chalk.cyan(char); } else { result += char; } diff --git a/src/subcommands/chat/react/types.ts b/src/subcommands/chat/react/types.ts index 56f57794..e9efa71a 100644 --- a/src/subcommands/chat/react/types.ts +++ b/src/subcommands/chat/react/types.ts @@ -6,10 +6,7 @@ export type InkChatMessage = | { type: "user"; - content: Array<{ - type: "text" | "largePaste"; - text: string; - }>; + content: UserInputContentPart[]; } | { type: "assistant"; @@ -49,14 +46,39 @@ export interface Suggestion { priority: number; } +export type ChatInputData = + | { + kind: "largePaste"; + content: string; + } + | { + kind: "image"; + mime?: string; + source: "base64"; + fileName: string; + contentBase64: string; + name?: string; + }; + +export type UserInputContentPart = + | { + type: "text"; + text: string; + } + | { + type: "chip"; + kind: ChatInputData["kind"]; + displayText: string; + }; + export type ChatInputSegment = | { type: "text"; content: string; } | { - type: "largePaste"; - content: string; + type: "chip"; + data: ChatInputData; }; export interface ChatUserInputState { diff --git a/src/subcommands/chat/util.ts b/src/subcommands/chat/util.ts index e0ce4891..8ec50217 100644 --- a/src/subcommands/chat/util.ts +++ b/src/subcommands/chat/util.ts @@ -2,7 +2,7 @@ import type { SimpleLogger } from "@lmstudio/lms-common"; import { type Chat, type LLM, type LLMPredictionStats, type LMStudioClient } from "@lmstudio/sdk"; import chalk from "chalk"; import { Spinner } from "../../Spinner.js"; -import { type InkChatMessage } from "./react/types.js"; +import { type ChatInputData, type InkChatMessage } from "./react/types.js"; export async function loadModelWithProgress( client: LMStudioClient, @@ -173,6 +173,26 @@ export function getLargePastePlaceholderText(content: string, previewLength: num return `[Pasted${spacer}${preview}${ellipsis}]`; } +function getBaseName(filePath: string): string | undefined { + const trimmed = filePath.trim(); + if (trimmed.length === 0) return undefined; + const parts = trimmed.split(/[\\/]/); + return parts[parts.length - 1]; +} + +export function getChipPreviewText(data: ChatInputData): string { + if (data.kind === "largePaste") { + return getLargePastePlaceholderText(data.content); + } + + const imageDisplayName = + data.name ?? + getBaseName(data.fileName ?? "") ?? + data.fileName ?? + "image"; + return `[Image: ${imageDisplayName}]`; +} + export const estimateMessageLinesCount = (message: InkChatMessage): number => { const terminalWidth = process.stdout.columns ?? 80; @@ -196,7 +216,9 @@ export const estimateMessageLinesCount = (message: InkChatMessage): number => { switch (type) { case "user": return countWrappedLines( - message.content.reduce((acc, a) => acc + a.text, ""), + message.content.reduce((acc, part) => { + return acc + (part.type === "text" ? part.text : part.displayText); + }, ""), 5, ); // "You: " prefix case "assistant": { From 5f65b0c92e494dec14a19ee57d79f1c744c02ba7 Mon Sep 17 00:00:00 2001 From: Rugved Somwanshi Date: Thu, 5 Feb 2026 12:14:22 -0500 Subject: [PATCH 2/9] Simplify image chip filename and color --- src/subcommands/chat/react/ChatMessages.tsx | 2 +- src/subcommands/chat/react/inputReducer.test.ts | 2 -- src/subcommands/chat/react/inputRenderer.tsx | 4 +--- src/subcommands/chat/react/types.ts | 1 - src/subcommands/chat/util.ts | 14 ++------------ 5 files changed, 4 insertions(+), 19 deletions(-) diff --git a/src/subcommands/chat/react/ChatMessages.tsx b/src/subcommands/chat/react/ChatMessages.tsx index 6963e101..7e6611e4 100644 --- a/src/subcommands/chat/react/ChatMessages.tsx +++ b/src/subcommands/chat/react/ChatMessages.tsx @@ -49,7 +49,7 @@ export const ChatMessage = memo(({ message, modelName }: ChatMessageProps) => { return contentPart.kind === "largePaste" ? chalk.blue(contentPart.text) - : chalk.cyan(contentPart.text); + : chalk.blue(contentPart.text); }) .join(""); diff --git a/src/subcommands/chat/react/inputReducer.test.ts b/src/subcommands/chat/react/inputReducer.test.ts index d14b6559..56243453 100644 --- a/src/subcommands/chat/react/inputReducer.test.ts +++ b/src/subcommands/chat/react/inputReducer.test.ts @@ -1022,7 +1022,6 @@ describe("chatInputStateReducers", () => { source: "base64", fileName: "cat.png", contentBase64: "aGVsbG8=", - name: "cat.png", mime: "image/png", }, }); @@ -1036,7 +1035,6 @@ describe("chatInputStateReducers", () => { source: "base64", fileName: "cat.png", contentBase64: "aGVsbG8=", - name: "cat.png", mime: "image/png", }, }, diff --git a/src/subcommands/chat/react/inputRenderer.tsx b/src/subcommands/chat/react/inputRenderer.tsx index dd3c2f3a..9f4366ae 100644 --- a/src/subcommands/chat/react/inputRenderer.tsx +++ b/src/subcommands/chat/react/inputRenderer.tsx @@ -33,10 +33,8 @@ export function renderInputWithCursor({ const range = chipRanges.find( highlight => absolutePos >= highlight.start && absolutePos < highlight.end, ); - if (range?.kind === "largePaste") { + if (range?.kind === "largePaste" || range?.kind === "image") { result += chalk.blue(char); - } else if (range?.kind === "image") { - result += chalk.cyan(char); } else { result += char; } diff --git a/src/subcommands/chat/react/types.ts b/src/subcommands/chat/react/types.ts index e9efa71a..553ee09e 100644 --- a/src/subcommands/chat/react/types.ts +++ b/src/subcommands/chat/react/types.ts @@ -57,7 +57,6 @@ export type ChatInputData = source: "base64"; fileName: string; contentBase64: string; - name?: string; }; export type UserInputContentPart = diff --git a/src/subcommands/chat/util.ts b/src/subcommands/chat/util.ts index 8ec50217..22b3f2ab 100644 --- a/src/subcommands/chat/util.ts +++ b/src/subcommands/chat/util.ts @@ -2,6 +2,7 @@ import type { SimpleLogger } from "@lmstudio/lms-common"; import { type Chat, type LLM, type LLMPredictionStats, type LMStudioClient } from "@lmstudio/sdk"; import chalk from "chalk"; import { Spinner } from "../../Spinner.js"; +import { basename } from "path"; import { type ChatInputData, type InkChatMessage } from "./react/types.js"; export async function loadModelWithProgress( @@ -173,23 +174,12 @@ export function getLargePastePlaceholderText(content: string, previewLength: num return `[Pasted${spacer}${preview}${ellipsis}]`; } -function getBaseName(filePath: string): string | undefined { - const trimmed = filePath.trim(); - if (trimmed.length === 0) return undefined; - const parts = trimmed.split(/[\\/]/); - return parts[parts.length - 1]; -} - export function getChipPreviewText(data: ChatInputData): string { if (data.kind === "largePaste") { return getLargePastePlaceholderText(data.content); } - const imageDisplayName = - data.name ?? - getBaseName(data.fileName ?? "") ?? - data.fileName ?? - "image"; + const imageDisplayName = basename(data.fileName ?? "").trim() || data.fileName || "image"; return `[Image: ${imageDisplayName}]`; } From 744c198936337b7b3b44bd1f771bdd92337af404 Mon Sep 17 00:00:00 2001 From: Rugved Somwanshi Date: Thu, 5 Feb 2026 12:15:13 -0500 Subject: [PATCH 3/9] Simplify chip color rendering --- src/subcommands/chat/react/ChatMessages.tsx | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/subcommands/chat/react/ChatMessages.tsx b/src/subcommands/chat/react/ChatMessages.tsx index 7e6611e4..4fb479e8 100644 --- a/src/subcommands/chat/react/ChatMessages.tsx +++ b/src/subcommands/chat/react/ChatMessages.tsx @@ -47,9 +47,7 @@ export const ChatMessage = memo(({ message, modelName }: ChatMessageProps) => { return contentPart.text; } - return contentPart.kind === "largePaste" - ? chalk.blue(contentPart.text) - : chalk.blue(contentPart.text); + return chalk.blue(contentPart.text); }) .join(""); From a7e15f2dba3eb584d91b38e1538c2ad35698c9bd Mon Sep 17 00:00:00 2001 From: Rugved Somwanshi Date: Thu, 5 Feb 2026 12:37:04 -0500 Subject: [PATCH 4/9] Refactor --- src/subcommands/chat/react/Chat.tsx | 106 +++++------------------- src/subcommands/chat/react/buildChat.ts | 89 ++++++++++++++++++++ 2 files changed, 108 insertions(+), 87 deletions(-) create mode 100644 src/subcommands/chat/react/buildChat.ts diff --git a/src/subcommands/chat/react/Chat.tsx b/src/subcommands/chat/react/Chat.tsx index 456d673e..d630ee6e 100644 --- a/src/subcommands/chat/react/Chat.tsx +++ b/src/subcommands/chat/react/Chat.tsx @@ -1,12 +1,5 @@ import { produce } from "@lmstudio/immer-with-plugins"; -import { - type Chat, - type ChatMessagePartFileData, - type ChatMessagePartTextData, - type LLM, - type LLMPredictionStats, - type LMStudioClient, -} from "@lmstudio/sdk"; +import { type Chat, type LLM, type LLMPredictionStats, type LMStudioClient } from "@lmstudio/sdk"; import { Box, type DOMElement, useApp, Text } from "ink"; import React, { useCallback, useMemo, useRef, useState } from "react"; import { displayVerboseStats, getChipPreviewText } from "../util.js"; @@ -32,6 +25,7 @@ import type { Suggestion, UserInputContentPart, } from "./types.js"; +import { buildUserMessageParts, ImagePreparationError } from "./buildChat.js"; // Freezes streaming content into static chunks at natural breaks to reduce re-renders. // Uses multiple boundaries to handle different content (best effort): @@ -303,8 +297,8 @@ export const ChatComponent = React.memo( }, [modelLoadingProgress]); const handlePaste = useCallback((content: string) => { - const normalizedContent = content.replaceAll("\r\n", "\n").replaceAll("\r", "\n").trimEnd(); - if (normalizedContent.length === 0) return; + const normalizedContent = content.replaceAll("\r\n", "\n").replaceAll("\r", "\n"); + if (normalizedContent.trim().length === 0) return; setUserInputState(previousState => insertPasteAtCursor({ state: previousState, @@ -429,87 +423,25 @@ export const ChatComponent = React.memo( const signal = abortControllerRef.current.signal; try { - const imagesToPrepare = - hasImageChip === true - ? inputSegments.flatMap(segment => { - if (segment.type !== "chip" || segment.data.kind !== "image") { - return []; - } - return [segment.data]; - }) - : []; - - let preparedImages: Awaited>[] = []; + let parts: Awaited>; try { - preparedImages = hasImageChip - ? await Promise.all( - imagesToPrepare.map(image => { - return client.files.prepareImageBase64(image.fileName, image.contentBase64); - }), - ) - : []; + parts = await buildUserMessageParts({ client, inputSegments }); } catch (error) { - setMessages( - produce(draftMessages => { - const lastMessage = draftMessages.at(-1); - if (lastMessage?.type === "user") { - draftMessages.pop(); - } - }), - ); - setUserInputState(inputStateSnapshot); - const message = error instanceof Error ? error.message : String(error); - logErrorInChat(`Failed to attach image: ${message}`); - return; - } - if (preparedImages.some(image => image.type !== "image")) { - setMessages( - produce(draftMessages => { - const lastMessage = draftMessages.at(-1); - if (lastMessage?.type === "user") { - draftMessages.pop(); - } - }), - ); - setUserInputState(inputStateSnapshot); - logErrorInChat( - "Failed to attach image: clipboard content was not recognized as an image.", - ); - return; - } - - const parts: Array = []; - let preparedImageIndex = 0; - for (const segment of inputSegments) { - if (segment.type === "text") { - parts.push({ type: "text", text: segment.content }); - continue; - } - if (segment.data.kind === "largePaste") { - parts.push({ type: "text", text: segment.data.content }); - continue; - } - // image chip - const nextImage = preparedImages[preparedImageIndex]; - if (nextImage === undefined) { - continue; + if (error instanceof ImagePreparationError) { + setMessages( + produce(draftMessages => { + const lastMessage = draftMessages.at(-1); + if (lastMessage?.type === "user") { + draftMessages.pop(); + } + }), + ); + setUserInputState(inputStateSnapshot); + logErrorInChat(`Failed to attach image: ${error.message}`); + return; } - preparedImageIndex += 1; - parts.push({ - type: "file", - name: nextImage.name, - identifier: nextImage.identifier, - sizeBytes: nextImage.sizeBytes, - fileType: nextImage.type, - }); - } - - // Ensure there is at least a leading text part. This matches the append(role, content, opts) - // behavior and avoids edge cases with image-only messages. - if (parts.length === 0 || parts[0]?.type !== "text") { - parts.unshift({ type: "text", text: "" }); + throw error; } - chatRef.current.append({ role: "user", content: parts }); const result = await llmRef.current.respond(chatRef.current, { onFirstToken() { diff --git a/src/subcommands/chat/react/buildChat.ts b/src/subcommands/chat/react/buildChat.ts new file mode 100644 index 00000000..4d4fea8a --- /dev/null +++ b/src/subcommands/chat/react/buildChat.ts @@ -0,0 +1,89 @@ +import { + type ChatMessagePartFileData, + type ChatMessagePartTextData, + type LMStudioClient, +} from "@lmstudio/sdk"; +import type { ChatInputSegment } from "./types.js"; + +export type UserMessagePart = ChatMessagePartTextData | ChatMessagePartFileData; + +export class ImagePreparationError extends Error { + code: "not_image" | "prepare_failed"; + + constructor(code: "not_image" | "prepare_failed", message: string) { + super(message); + this.code = code; + this.name = "ImagePreparationError"; + } +} + +export async function buildUserMessageParts({ + client, + inputSegments, +}: { + client: LMStudioClient; + inputSegments: ChatInputSegment[]; +}): Promise { + const imagesToPrepare = inputSegments.flatMap(segment => { + if (segment.type !== "chip" || segment.data.kind !== "image") { + return []; + } + return [segment.data]; + }); + + let preparedImages: Awaited>[] = []; + try { + preparedImages = + imagesToPrepare.length > 0 + ? await Promise.all( + imagesToPrepare.map(image => { + return client.files.prepareImageBase64(image.fileName, image.contentBase64); + }), + ) + : []; + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new ImagePreparationError("prepare_failed", message); + } + + if (preparedImages.some(image => image.type !== "image")) { + throw new ImagePreparationError( + "not_image", + "clipboard content was not recognized as an image.", + ); + } + + const parts: UserMessagePart[] = []; + let preparedImageIndex = 0; + for (const segment of inputSegments) { + if (segment.type === "text") { + parts.push({ type: "text", text: segment.content }); + continue; + } + if (segment.data.kind === "largePaste") { + parts.push({ type: "text", text: segment.data.content }); + continue; + } + // image chip + const nextImage = preparedImages[preparedImageIndex]; + if (nextImage === undefined) { + continue; + } + preparedImageIndex += 1; + parts.push({ + type: "file", + name: nextImage.name, + identifier: nextImage.identifier, + sizeBytes: nextImage.sizeBytes, + fileType: nextImage.type, + }); + } + + // Ensure there is at least a leading text part. This matches the append(role, content, opts) + // behavior and avoids edge cases with image-only messages. + if (parts.length === 0 || parts[0]?.type !== "text") { + parts.unshift({ type: "text", text: "" }); + } + + return parts; +} From 5db1ce9f10d9cf767c571dcab77cb5dc7c24f9e0 Mon Sep 17 00:00:00 2001 From: Rugved Somwanshi Date: Thu, 5 Feb 2026 12:47:46 -0500 Subject: [PATCH 5/9] Refactor --- src/subcommands/chat/react/buildChat.ts | 67 ++++++++++++++----------- 1 file changed, 39 insertions(+), 28 deletions(-) diff --git a/src/subcommands/chat/react/buildChat.ts b/src/subcommands/chat/react/buildChat.ts index 4d4fea8a..24f1fa18 100644 --- a/src/subcommands/chat/react/buildChat.ts +++ b/src/subcommands/chat/react/buildChat.ts @@ -1,4 +1,5 @@ import { + type FileHandle, type ChatMessagePartFileData, type ChatMessagePartTextData, type LMStudioClient, @@ -31,7 +32,7 @@ export async function buildUserMessageParts({ return [segment.data]; }); - let preparedImages: Awaited>[] = []; + let preparedImages: Awaited>[] = []; try { preparedImages = imagesToPrepare.length > 0 @@ -56,34 +57,44 @@ export async function buildUserMessageParts({ const parts: UserMessagePart[] = []; let preparedImageIndex = 0; for (const segment of inputSegments) { - if (segment.type === "text") { - parts.push({ type: "text", text: segment.content }); - continue; + switch (segment.type) { + case "text": { + parts.push({ type: "text", text: segment.content }); + break; + } + case "chip": { + switch (segment.data.kind) { + case "largePaste": { + parts.push({ type: "text", text: segment.data.content }); + break; + } + case "image": { + const nextImage = preparedImages[preparedImageIndex]; + if (nextImage === undefined) { + break; + } + preparedImageIndex += 1; + parts.push({ + type: "file", + name: nextImage.name, + identifier: nextImage.identifier, + sizeBytes: nextImage.sizeBytes, + fileType: nextImage.type, + }); + break; + } + default: { + const exhaustiveCheck: never = segment.data; + throw new Error(`Unhandled chip kind: ${JSON.stringify(exhaustiveCheck)}`); + } + } + break; + } + default: { + const exhaustiveCheck: never = segment; + throw new Error(`Unhandled segment type: ${JSON.stringify(exhaustiveCheck)}`); + } } - if (segment.data.kind === "largePaste") { - parts.push({ type: "text", text: segment.data.content }); - continue; - } - // image chip - const nextImage = preparedImages[preparedImageIndex]; - if (nextImage === undefined) { - continue; - } - preparedImageIndex += 1; - parts.push({ - type: "file", - name: nextImage.name, - identifier: nextImage.identifier, - sizeBytes: nextImage.sizeBytes, - fileType: nextImage.type, - }); } - - // Ensure there is at least a leading text part. This matches the append(role, content, opts) - // behavior and avoids edge cases with image-only messages. - if (parts.length === 0 || parts[0]?.type !== "text") { - parts.unshift({ type: "text", text: "" }); - } - return parts; } From 75ad0f781c8e8c0631752d0992820662915c9ab0 Mon Sep 17 00:00:00 2001 From: Rugved Somwanshi Date: Thu, 5 Feb 2026 13:52:41 -0500 Subject: [PATCH 6/9] Fix tests and refactor --- src/subcommands/chat/react/ChatMessages.tsx | 4 ++-- src/subcommands/chat/react/inputReducer.test.ts | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/subcommands/chat/react/ChatMessages.tsx b/src/subcommands/chat/react/ChatMessages.tsx index 4fb479e8..fdd2bdfa 100644 --- a/src/subcommands/chat/react/ChatMessages.tsx +++ b/src/subcommands/chat/react/ChatMessages.tsx @@ -15,9 +15,9 @@ export const ChatMessage = memo(({ message, modelName }: ChatMessageProps) => { case "user": { const contentParts = message.content.map(part => { if (part.type === "text") { - return { type: "text" as const, text: part.text }; + return { type: "text", text: part.text }; } - return { type: "chip" as const, kind: part.kind, text: part.displayText }; + return { type: "chip", kind: part.kind, text: part.displayText }; }); // Trim only edge newlines on the first/last non-empty parts to keep internal newlines intact. diff --git a/src/subcommands/chat/react/inputReducer.test.ts b/src/subcommands/chat/react/inputReducer.test.ts index 56243453..7bbfeddf 100644 --- a/src/subcommands/chat/react/inputReducer.test.ts +++ b/src/subcommands/chat/react/inputReducer.test.ts @@ -296,8 +296,7 @@ describe("chatInputStateReducers", () => { const result = deleteBeforeCursorCount(initialState, 2); expect(result.segments).toEqual([ - { type: "text", content: "h" }, - { type: "text", content: "ok" }, + { type: "text", content: "hok" }, ]); expect(result.cursorOnSegmentIndex).toBe(0); expect(result.cursorInSegmentOffset).toBe(1); From 7eb52f106e2fc7ef5be2c57e522a0cc2f5b381d8 Mon Sep 17 00:00:00 2001 From: Rugved Somwanshi Date: Thu, 5 Feb 2026 15:01:37 -0500 Subject: [PATCH 7/9] Refactor to use image store --- src/subcommands/chat/react/Chat.tsx | 12 +- src/subcommands/chat/react/buildChat.ts | 53 +++------ .../chat/react/images/imageErrors.ts | 9 ++ .../chat/react/images/imageStore.ts | 109 ++++++++++++++++++ .../chat/react/inputReducer.test.ts | 8 +- src/subcommands/chat/react/types.ts | 2 +- 6 files changed, 148 insertions(+), 45 deletions(-) create mode 100644 src/subcommands/chat/react/images/imageErrors.ts create mode 100644 src/subcommands/chat/react/images/imageStore.ts diff --git a/src/subcommands/chat/react/Chat.tsx b/src/subcommands/chat/react/Chat.tsx index d630ee6e..6f7ba050 100644 --- a/src/subcommands/chat/react/Chat.tsx +++ b/src/subcommands/chat/react/Chat.tsx @@ -25,7 +25,9 @@ import type { Suggestion, UserInputContentPart, } from "./types.js"; -import { buildUserMessageParts, ImagePreparationError } from "./buildChat.js"; +import { buildUserMessageParts } from "./buildChat.js"; +import { ImagePreparationError } from "./images/imageErrors.js"; +import { ImageStore } from "./images/imageStore.js"; // Freezes streaming content into static chunks at natural breaks to reduce re-renders. // Uses multiple boundaries to handle different content (best effort): @@ -54,6 +56,8 @@ export const emptyChatInputState: ChatUserInputState = { cursorInSegmentOffset: 0, }; +const imageStore = new ImageStore(); + export const ChatComponent = React.memo( ({ client, llm, chat, onExit, stats, ttl, shouldFetchModelCatalog }: ChatComponentProps) => { const { exit } = useApp(); @@ -425,7 +429,11 @@ export const ChatComponent = React.memo( try { let parts: Awaited>; try { - parts = await buildUserMessageParts({ client, inputSegments }); + parts = await buildUserMessageParts({ + client, + inputSegments, + imageStore, + }); } catch (error) { if (error instanceof ImagePreparationError) { setMessages( diff --git a/src/subcommands/chat/react/buildChat.ts b/src/subcommands/chat/react/buildChat.ts index 24f1fa18..3f627a5d 100644 --- a/src/subcommands/chat/react/buildChat.ts +++ b/src/subcommands/chat/react/buildChat.ts @@ -1,61 +1,36 @@ import { - type FileHandle, type ChatMessagePartFileData, type ChatMessagePartTextData, + type FileHandle, type LMStudioClient, } from "@lmstudio/sdk"; import type { ChatInputSegment } from "./types.js"; +import type { ImageStore } from "./images/imageStore.js"; +import { ImagePreparationError } from "./images/imageErrors.js"; export type UserMessagePart = ChatMessagePartTextData | ChatMessagePartFileData; -export class ImagePreparationError extends Error { - code: "not_image" | "prepare_failed"; - - constructor(code: "not_image" | "prepare_failed", message: string) { - super(message); - this.code = code; - this.name = "ImagePreparationError"; - } -} - export async function buildUserMessageParts({ client, inputSegments, + imageStore, }: { client: LMStudioClient; inputSegments: ChatInputSegment[]; + imageStore: ImageStore; }): Promise { - const imagesToPrepare = inputSegments.flatMap(segment => { - if (segment.type !== "chip" || segment.data.kind !== "image") { - return []; - } - return [segment.data]; - }); - - let preparedImages: Awaited>[] = []; + let preparedImagesByHash = new Map(); try { - preparedImages = - imagesToPrepare.length > 0 - ? await Promise.all( - imagesToPrepare.map(image => { - return client.files.prepareImageBase64(image.fileName, image.contentBase64); - }), - ) - : []; + preparedImagesByHash = await imageStore.prepareImagesForChat(client, inputSegments); } catch (error) { + if (error instanceof ImagePreparationError) { + throw error; + } const message = error instanceof Error ? error.message : String(error); throw new ImagePreparationError("prepare_failed", message); } - if (preparedImages.some(image => image.type !== "image")) { - throw new ImagePreparationError( - "not_image", - "clipboard content was not recognized as an image.", - ); - } - const parts: UserMessagePart[] = []; - let preparedImageIndex = 0; for (const segment of inputSegments) { switch (segment.type) { case "text": { @@ -69,11 +44,13 @@ export async function buildUserMessageParts({ break; } case "image": { - const nextImage = preparedImages[preparedImageIndex]; + const nextImage = preparedImagesByHash.get(segment.data.imageHash); if (nextImage === undefined) { - break; + throw new ImagePreparationError( + "prepare_failed", + `Missing prepared image for hash ${segment.data.imageHash}.`, + ); } - preparedImageIndex += 1; parts.push({ type: "file", name: nextImage.name, diff --git a/src/subcommands/chat/react/images/imageErrors.ts b/src/subcommands/chat/react/images/imageErrors.ts new file mode 100644 index 00000000..a2afc1e6 --- /dev/null +++ b/src/subcommands/chat/react/images/imageErrors.ts @@ -0,0 +1,9 @@ +export class ImagePreparationError extends Error { + code: "not_image" | "prepare_failed"; + + constructor(code: "not_image" | "prepare_failed", message: string) { + super(message); + this.code = code; + this.name = "ImagePreparationError"; + } +} diff --git a/src/subcommands/chat/react/images/imageStore.ts b/src/subcommands/chat/react/images/imageStore.ts new file mode 100644 index 00000000..03d2d532 --- /dev/null +++ b/src/subcommands/chat/react/images/imageStore.ts @@ -0,0 +1,109 @@ +import { createHash } from "node:crypto"; +import type { FileHandle, LMStudioClient } from "@lmstudio/sdk"; +import type { ChatInputSegment } from "../types.js"; +import { ImagePreparationError } from "./imageErrors.js"; + +type StoredImage = { + contentBase64: string; + mime?: string; + fileName?: string; +}; + +export function hashImageBase64(contentBase64: string): string { + return createHash("sha256").update(contentBase64).digest("hex"); +} + +/** + * Shared image store for chat input images. + * + * - Keeps base64 blobs out of React state to reduce memory overhead. + * - Dedupes identical images by hash and reuses prepared file handles. + * - Uses a small LRU to avoid unbounded growth while still avoiding re-prepare churn. + */ +export class ImageStore { + private readonly images = new Map(); + private readonly preparedByHash = new Map(); + private readonly maxEntries: number; + + constructor(maxEntries: number = 10) { + this.maxEntries = maxEntries; + } + + storeImageBase64(contentBase64: string, mime?: string, fileName?: string): string { + const hash = hashImageBase64(contentBase64); + if (this.images.has(hash)) { + const existing = this.images.get(hash); + this.images.delete(hash); + this.images.set(hash, existing ?? { contentBase64, mime, fileName }); + return hash; + } + this.images.set(hash, { contentBase64, mime, fileName }); + this.enforceLimit(); + return hash; + } + + getImageBase64(hash: string): string | undefined { + return this.images.get(hash)?.contentBase64; + } + + deleteImages(hashes: Iterable): void { + for (const hash of hashes) { + this.preparedByHash.delete(hash); + this.images.delete(hash); + } + } + + private enforceLimit(): void { + while (this.images.size > this.maxEntries) { + const oldestHash = this.images.keys().next().value as string | undefined; + if (oldestHash === undefined) { + break; + } + this.images.delete(oldestHash); + this.preparedByHash.delete(oldestHash); + } + } + + async prepareImagesForChat( + client: LMStudioClient, + segments: ChatInputSegment[], + ): Promise> { + const hashesToPrepare = new Map(); + for (const segment of segments) { + if (segment.type === "chip" && segment.data.kind === "image") { + if (!this.preparedByHash.has(segment.data.imageHash)) { + hashesToPrepare.set(segment.data.imageHash, { fileName: segment.data.fileName }); + } + } + } + + if (hashesToPrepare.size === 0) { + return this.preparedByHash; + } + + const preparedEntries = await Promise.all( + [...hashesToPrepare.entries()].map(async ([hash, { fileName }]) => { + const contentBase64 = this.getImageBase64(hash); + if (contentBase64 === undefined) { + throw new ImagePreparationError("prepare_failed", `Missing image data for hash ${hash}.`); + } + const prepared = await client.files.prepareImageBase64(fileName, contentBase64); + const entry: [string, FileHandle] = [hash, prepared]; + return entry; + }), + ); + + for (const [hash, prepared] of preparedEntries) { + this.preparedByHash.set(hash, prepared); + } + + if (preparedEntries.some(([, prepared]) => prepared.type !== "image")) { + throw new ImagePreparationError( + "not_image", + "clipboard content was not recognized as an image.", + ); + } + + return this.preparedByHash; + } +} diff --git a/src/subcommands/chat/react/inputReducer.test.ts b/src/subcommands/chat/react/inputReducer.test.ts index 7bbfeddf..39237591 100644 --- a/src/subcommands/chat/react/inputReducer.test.ts +++ b/src/subcommands/chat/react/inputReducer.test.ts @@ -1020,7 +1020,7 @@ describe("chatInputStateReducers", () => { image: { source: "base64", fileName: "cat.png", - contentBase64: "aGVsbG8=", + imageHash: "hash-cat", mime: "image/png", }, }); @@ -1033,7 +1033,7 @@ describe("chatInputStateReducers", () => { kind: "image", source: "base64", fileName: "cat.png", - contentBase64: "aGVsbG8=", + imageHash: "hash-cat", mime: "image/png", }, }, @@ -1056,14 +1056,14 @@ describe("chatInputStateReducers", () => { const result = insertImageAtCursor({ state: initialState, - image: { source: "base64", fileName: "cat.png", contentBase64: "aGVsbG8=" }, + image: { source: "base64", fileName: "cat.png", imageHash: "hash-cat" }, }); expect(result.segments).toEqual([ { type: "text", content: "head" }, { type: "chip", - data: { kind: "image", source: "base64", fileName: "cat.png", contentBase64: "aGVsbG8=" }, + data: { kind: "image", source: "base64", fileName: "cat.png", imageHash: "hash-cat" }, }, { type: "chip", data: { kind: "largePaste", content: "PASTE" } }, { type: "text", content: "tail" }, diff --git a/src/subcommands/chat/react/types.ts b/src/subcommands/chat/react/types.ts index 553ee09e..ee2ebc6c 100644 --- a/src/subcommands/chat/react/types.ts +++ b/src/subcommands/chat/react/types.ts @@ -56,7 +56,7 @@ export type ChatInputData = mime?: string; source: "base64"; fileName: string; - contentBase64: string; + imageHash: string; }; export type UserInputContentPart = From 113fa1d358b8dce51024df5b1847ea409444ea3a Mon Sep 17 00:00:00 2001 From: Rugved Somwanshi Date: Thu, 5 Feb 2026 15:10:34 -0500 Subject: [PATCH 8/9] Bump image store max entries --- src/subcommands/chat/react/images/imageStore.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/subcommands/chat/react/images/imageStore.ts b/src/subcommands/chat/react/images/imageStore.ts index 03d2d532..84057837 100644 --- a/src/subcommands/chat/react/images/imageStore.ts +++ b/src/subcommands/chat/react/images/imageStore.ts @@ -25,7 +25,7 @@ export class ImageStore { private readonly preparedByHash = new Map(); private readonly maxEntries: number; - constructor(maxEntries: number = 10) { + constructor(maxEntries: number = 35) { this.maxEntries = maxEntries; } From 8ec1cc58007defc1495387005f15292af5239461 Mon Sep 17 00:00:00 2001 From: Rugved Somwanshi Date: Thu, 5 Feb 2026 15:11:50 -0500 Subject: [PATCH 9/9] Cache images only --- src/subcommands/chat/react/images/imageStore.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/subcommands/chat/react/images/imageStore.ts b/src/subcommands/chat/react/images/imageStore.ts index 84057837..fd29f49b 100644 --- a/src/subcommands/chat/react/images/imageStore.ts +++ b/src/subcommands/chat/react/images/imageStore.ts @@ -93,10 +93,6 @@ export class ImageStore { }), ); - for (const [hash, prepared] of preparedEntries) { - this.preparedByHash.set(hash, prepared); - } - if (preparedEntries.some(([, prepared]) => prepared.type !== "image")) { throw new ImagePreparationError( "not_image", @@ -104,6 +100,10 @@ export class ImageStore { ); } + for (const [hash, prepared] of preparedEntries) { + this.preparedByHash.set(hash, prepared); + } + return this.preparedByHash; } }