diff --git a/src/subcommands/chat/react/Chat.tsx b/src/subcommands/chat/react/Chat.tsx index 833fb6c8..6f7ba050 100644 --- a/src/subcommands/chat/react/Chat.tsx +++ b/src/subcommands/chat/react/Chat.tsx @@ -2,7 +2,7 @@ import { produce } from "@lmstudio/immer-with-plugins"; import { type Chat, type LLM, type LLMPredictionStats, type LMStudioClient } from "@lmstudio/sdk"; import { Box, type DOMElement, useApp, Text } from "ink"; import React, { useCallback, useMemo, useRef, useState } from "react"; -import { displayVerboseStats, getLargePastePlaceholderText } from "../util.js"; +import { displayVerboseStats, getChipPreviewText } from "../util.js"; import { ChatInput } from "./ChatInput.js"; import { ChatMessagesList } from "./ChatMessagesList.js"; import { ChatSuggestions } from "./ChatSuggestions.js"; @@ -19,7 +19,15 @@ import { } from "./hooks.js"; import { insertPasteAtCursor } from "./inputReducer.js"; import { createSlashCommands } from "./slashCommands.js"; -import type { ChatUserInputState, InkChatMessage, Suggestion } from "./types.js"; +import type { + ChatUserInputState, + InkChatMessage, + Suggestion, + UserInputContentPart, +} from "./types.js"; +import { buildUserMessageParts } from "./buildChat.js"; +import { ImagePreparationError } from "./images/imageErrors.js"; +import { ImageStore } from "./images/imageStore.js"; // Freezes streaming content into static chunks at natural breaks to reduce re-renders. // Uses multiple boundaries to handle different content (best effort): @@ -48,6 +56,8 @@ export const emptyChatInputState: ChatUserInputState = { cursorInSegmentOffset: 0, }; +const imageStore = new ImageStore(); + export const ChatComponent = React.memo( ({ client, llm, chat, onExit, stats, ttl, shouldFetchModelCatalog }: ChatComponentProps) => { const { exit } = useApp(); @@ -181,7 +191,7 @@ export const ChatComponent = React.memo( return []; } const firstSegment = userInputState.segments[0]; - const inputText = firstSegment.content; + const inputText = firstSegment?.type === "text" ? firstSegment.content : ""; return commandHandler.getSuggestions({ input: inputText, shouldShowSuggestions: !isConfirmationActive && !isPredicting && inputText.startsWith("/"), @@ -292,11 +302,7 @@ export const ChatComponent = React.memo( const handlePaste = useCallback((content: string) => { const normalizedContent = content.replaceAll("\r\n", "\n").replaceAll("\r", "\n"); - - if (normalizedContent.length === 0) { - return; - } - + if (normalizedContent.trim().length === 0) return; setUserInputState(previousState => insertPasteAtCursor({ state: previousState, @@ -307,27 +313,44 @@ export const ChatComponent = React.memo( }, []); const handleSubmit = useCallback(async () => { + const inputStateSnapshot = userInputState; + const inputSegments = userInputState.segments; + const hasImageChip = inputSegments.some( + segment => segment.type === "chip" && segment.data.kind === "image", + ); + // Collect the full text input from the userInputState - const userInputText = userInputState.segments - .map(segment => segment.content) + const userInputText = inputSegments + .map(segment => { + if (segment.type === "text") { + return segment.content; + } + return segment.data.kind === "largePaste" ? segment.data.content : ""; + }) .join("") .trim(); - // Clear the input state - setUserInputState(emptyChatInputState); - const confirmationResponse = await handleConfirmationResponse(userInputText); - if (confirmationResponse === "handled") { - return; - } - if (confirmationResponse === "invalid") { - logInChat("Please answer 'yes' or 'no'"); - return; + + if (isConfirmationActive) { + const confirmationResponse = await handleConfirmationResponse(userInputText); + if (confirmationResponse === "handled") { + setUserInputState(emptyChatInputState); + return; + } + if (confirmationResponse === "invalid") { + logInChat("Please answer 'yes' or 'no'"); + return; + } } - if (userInputText.length === 0) { + if (userInputText.length === 0 && hasImageChip === false) { return; } - if (userInputText.startsWith("/") && userInputState.segments.length === 1) { + if ( + userInputText.startsWith("/") && + inputSegments.length === 1 && + inputSegments[0]?.type === "text" + ) { const selectedSuggestion = normalizedSelectedSuggestionIndex !== null ? suggestions[normalizedSelectedSuggestionIndex] @@ -344,6 +367,7 @@ export const ChatComponent = React.memo( // Check if the command is in exception list, if not, // execute it. if (commandHandler.commandIsIgnored(command) === false) { + setUserInputState(emptyChatInputState); const wasCommandHandled = await commandHandler.execute(command, argumentsText); if (wasCommandHandled === false) { logInChat(`Unknown command: ${userInputText}`); @@ -362,6 +386,11 @@ export const ChatComponent = React.memo( return; } + if (hasImageChip === true && llmRef.current.vision === false) { + logErrorInChat("The current model does not support image input (vision)."); + return; + } + if (isPredicting) { logInChat( "A prediction is already in progress. Please wait for it to finish or press CTRL+C to abort it.", @@ -369,7 +398,23 @@ export const ChatComponent = React.memo( return; } + const userMessageContent = inputSegments.map(segment => { + if (segment.type === "text") { + return { type: "text", text: segment.content }; + } + + const displayText = getChipPreviewText(segment.data); + return { type: "chip", kind: segment.data.kind, displayText }; + }); + // If nothing else, proceed with normal message submission + setUserInputState(emptyChatInputState); + // Render the user message immediately (before any async image preparation) to avoid a + // transient frame where the input clears but the message hasn't appeared yet ("flash"). + addMessage({ + type: "user", + content: userMessageContent, + }); setIsPredicting(true); setPromptProcessingProgress(null); setShowPredictionSpinner(true); @@ -382,20 +427,30 @@ export const ChatComponent = React.memo( const signal = abortControllerRef.current.signal; try { - chatRef.current.append("user", userInputText); - addMessage({ - type: "user", - content: userInputState.segments.map(segment => { - if (segment.type === "largePaste") { - const placeholder = getLargePastePlaceholderText(segment.content); - return { - type: "largePaste", - text: placeholder, - }; - } - return { type: segment.type, text: segment.content }; - }), - }); + let parts: Awaited>; + try { + parts = await buildUserMessageParts({ + client, + inputSegments, + imageStore, + }); + } catch (error) { + if (error instanceof ImagePreparationError) { + setMessages( + produce(draftMessages => { + const lastMessage = draftMessages.at(-1); + if (lastMessage?.type === "user") { + draftMessages.pop(); + } + }), + ); + setUserInputState(inputStateSnapshot); + logErrorInChat(`Failed to attach image: ${error.message}`); + return; + } + throw error; + } + chatRef.current.append({ role: "user", content: parts }); const result = await llmRef.current.respond(chatRef.current, { onFirstToken() { setShowPredictionSpinner(false); @@ -577,20 +632,21 @@ export const ChatComponent = React.memo( abortControllerRef.current = null; } }, [ - userInputState.segments, - handleConfirmationResponse, + userInputState, + isConfirmationActive, isPredicting, + addMessage, + handleConfirmationResponse, logInChat, normalizedSelectedSuggestionIndex, suggestions, commandHandler, handleExit, logErrorInChat, - addMessage, + client, stats, promptProcessingProgress, requestConfirmation, - client.llm, ttl, ]); diff --git a/src/subcommands/chat/react/ChatInput.tsx b/src/subcommands/chat/react/ChatInput.tsx index 54ad396b..7307226f 100644 --- a/src/subcommands/chat/react/ChatInput.tsx +++ b/src/subcommands/chat/react/ChatInput.tsx @@ -11,7 +11,9 @@ import { } from "./inputReducer.js"; import { renderInputWithCursor } from "./inputRenderer.js"; import { type ChatUserInputState } from "./types.js"; -import { getLargePastePlaceholderText } from "../util.js"; +import { getChipPreviewText } from "../util.js"; + +type ChipRange = { start: number; end: number; kind: "largePaste" | "image" }; interface ChatInputProps { inputState: ChatUserInputState; @@ -150,10 +152,18 @@ export const ChatInput = ({ return; } - const currentText = inputState.segments.map(segment => segment.content).join(""); + const currentText = inputState.segments + .map(segment => (segment.type === "text" ? segment.content : "")) + .join(""); // Check if input is a slash command without arguments that has suggestions - if (currentText.startsWith("/") && currentText.includes(" ") === false) { + // Only auto-insert space if input is a single text segment (no chips) + if ( + currentText.startsWith("/") && + currentText.includes(" ") === false && + inputState.segments.length === 1 && + inputState.segments[0]?.type === "text" + ) { const commandName = currentText.slice(1); if (commandHasSuggestions(commandName)) { setUserInputState(previousState => @@ -197,16 +207,16 @@ export const ChatInput = ({ } }); - const { fullText, cursorPosition, pasteRanges } = useMemo(() => { + const { fullText, cursorPosition, chipRanges } = useMemo(() => { let fullText = ""; let cursorPosition = 0; - const pasteRanges: Array<{ start: number; end: number }> = []; + const chipRanges: ChipRange[] = []; for (let index = 0; index < inputState.segments.length; index++) { const segment = inputState.segments[index]; - if (segment.type === "largePaste") { - const placeholder = getLargePastePlaceholderText(segment.content); + if (segment.type === "chip") { + const placeholder = getChipPreviewText(segment.data); const startPos = fullText.length; if (index < inputState.cursorOnSegmentIndex) { @@ -216,7 +226,7 @@ export const ChatInput = ({ } fullText += placeholder; - pasteRanges.push({ start: startPos, end: fullText.length }); + chipRanges.push({ start: startPos, end: fullText.length, kind: segment.data.kind }); } else { if (index < inputState.cursorOnSegmentIndex) { cursorPosition += segment.content.length; @@ -228,7 +238,7 @@ export const ChatInput = ({ } } - return { fullText, cursorPosition, pasteRanges }; + return { fullText, cursorPosition, chipRanges }; }, [inputState]); const lines = fullText.split("\n"); @@ -266,7 +276,7 @@ export const ChatInput = ({ {renderInputWithCursor({ fullText: lineText, cursorPosition: isCursorLine ? cursorColumnIndex : -1, - pasteRanges, + chipRanges, lineStartPos, })} diff --git a/src/subcommands/chat/react/ChatMessages.tsx b/src/subcommands/chat/react/ChatMessages.tsx index d9cdd6ae..fdd2bdfa 100644 --- a/src/subcommands/chat/react/ChatMessages.tsx +++ b/src/subcommands/chat/react/ChatMessages.tsx @@ -13,7 +13,12 @@ export const ChatMessage = memo(({ message, modelName }: ChatMessageProps) => { const type = message.type; switch (type) { case "user": { - const contentParts = message.content.map(part => ({ ...part })); + const contentParts = message.content.map(part => { + if (part.type === "text") { + return { type: "text", text: part.text }; + } + return { type: "chip", kind: part.kind, text: part.displayText }; + }); // Trim only edge newlines on the first/last non-empty parts to keep internal newlines intact. let startIndex = 0; @@ -38,8 +43,11 @@ export const ChatMessage = memo(({ message, modelName }: ChatMessageProps) => { // After edge trimming, apply large paste coloring and join into a single string. const formattedContent = contentParts .map(contentPart => { - const text = contentPart.text; - return contentPart.type === "largePaste" ? chalk.blue(text) : text; + if (contentPart.type === "text") { + return contentPart.text; + } + + return chalk.blue(contentPart.text); }) .join(""); diff --git a/src/subcommands/chat/react/buildChat.ts b/src/subcommands/chat/react/buildChat.ts new file mode 100644 index 00000000..3f627a5d --- /dev/null +++ b/src/subcommands/chat/react/buildChat.ts @@ -0,0 +1,77 @@ +import { + type ChatMessagePartFileData, + type ChatMessagePartTextData, + type FileHandle, + type LMStudioClient, +} from "@lmstudio/sdk"; +import type { ChatInputSegment } from "./types.js"; +import type { ImageStore } from "./images/imageStore.js"; +import { ImagePreparationError } from "./images/imageErrors.js"; + +export type UserMessagePart = ChatMessagePartTextData | ChatMessagePartFileData; + +export async function buildUserMessageParts({ + client, + inputSegments, + imageStore, +}: { + client: LMStudioClient; + inputSegments: ChatInputSegment[]; + imageStore: ImageStore; +}): Promise { + let preparedImagesByHash = new Map(); + try { + preparedImagesByHash = await imageStore.prepareImagesForChat(client, inputSegments); + } catch (error) { + if (error instanceof ImagePreparationError) { + throw error; + } + const message = error instanceof Error ? error.message : String(error); + throw new ImagePreparationError("prepare_failed", message); + } + + const parts: UserMessagePart[] = []; + for (const segment of inputSegments) { + switch (segment.type) { + case "text": { + parts.push({ type: "text", text: segment.content }); + break; + } + case "chip": { + switch (segment.data.kind) { + case "largePaste": { + parts.push({ type: "text", text: segment.data.content }); + break; + } + case "image": { + const nextImage = preparedImagesByHash.get(segment.data.imageHash); + if (nextImage === undefined) { + throw new ImagePreparationError( + "prepare_failed", + `Missing prepared image for hash ${segment.data.imageHash}.`, + ); + } + parts.push({ + type: "file", + name: nextImage.name, + identifier: nextImage.identifier, + sizeBytes: nextImage.sizeBytes, + fileType: nextImage.type, + }); + break; + } + default: { + const exhaustiveCheck: never = segment.data; + throw new Error(`Unhandled chip kind: ${JSON.stringify(exhaustiveCheck)}`); + } + } + break; + } + default: { + const exhaustiveCheck: never = segment; + throw new Error(`Unhandled segment type: ${JSON.stringify(exhaustiveCheck)}`); + } + } + } + return parts; +} diff --git a/src/subcommands/chat/react/images/imageErrors.ts b/src/subcommands/chat/react/images/imageErrors.ts new file mode 100644 index 00000000..a2afc1e6 --- /dev/null +++ b/src/subcommands/chat/react/images/imageErrors.ts @@ -0,0 +1,9 @@ +export class ImagePreparationError extends Error { + code: "not_image" | "prepare_failed"; + + constructor(code: "not_image" | "prepare_failed", message: string) { + super(message); + this.code = code; + this.name = "ImagePreparationError"; + } +} diff --git a/src/subcommands/chat/react/images/imageStore.ts b/src/subcommands/chat/react/images/imageStore.ts new file mode 100644 index 00000000..fd29f49b --- /dev/null +++ b/src/subcommands/chat/react/images/imageStore.ts @@ -0,0 +1,109 @@ +import { createHash } from "node:crypto"; +import type { FileHandle, LMStudioClient } from "@lmstudio/sdk"; +import type { ChatInputSegment } from "../types.js"; +import { ImagePreparationError } from "./imageErrors.js"; + +type StoredImage = { + contentBase64: string; + mime?: string; + fileName?: string; +}; + +export function hashImageBase64(contentBase64: string): string { + return createHash("sha256").update(contentBase64).digest("hex"); +} + +/** + * Shared image store for chat input images. + * + * - Keeps base64 blobs out of React state to reduce memory overhead. + * - Dedupes identical images by hash and reuses prepared file handles. + * - Uses a small LRU to avoid unbounded growth while still avoiding re-prepare churn. + */ +export class ImageStore { + private readonly images = new Map(); + private readonly preparedByHash = new Map(); + private readonly maxEntries: number; + + constructor(maxEntries: number = 35) { + this.maxEntries = maxEntries; + } + + storeImageBase64(contentBase64: string, mime?: string, fileName?: string): string { + const hash = hashImageBase64(contentBase64); + if (this.images.has(hash)) { + const existing = this.images.get(hash); + this.images.delete(hash); + this.images.set(hash, existing ?? { contentBase64, mime, fileName }); + return hash; + } + this.images.set(hash, { contentBase64, mime, fileName }); + this.enforceLimit(); + return hash; + } + + getImageBase64(hash: string): string | undefined { + return this.images.get(hash)?.contentBase64; + } + + deleteImages(hashes: Iterable): void { + for (const hash of hashes) { + this.preparedByHash.delete(hash); + this.images.delete(hash); + } + } + + private enforceLimit(): void { + while (this.images.size > this.maxEntries) { + const oldestHash = this.images.keys().next().value as string | undefined; + if (oldestHash === undefined) { + break; + } + this.images.delete(oldestHash); + this.preparedByHash.delete(oldestHash); + } + } + + async prepareImagesForChat( + client: LMStudioClient, + segments: ChatInputSegment[], + ): Promise> { + const hashesToPrepare = new Map(); + for (const segment of segments) { + if (segment.type === "chip" && segment.data.kind === "image") { + if (!this.preparedByHash.has(segment.data.imageHash)) { + hashesToPrepare.set(segment.data.imageHash, { fileName: segment.data.fileName }); + } + } + } + + if (hashesToPrepare.size === 0) { + return this.preparedByHash; + } + + const preparedEntries = await Promise.all( + [...hashesToPrepare.entries()].map(async ([hash, { fileName }]) => { + const contentBase64 = this.getImageBase64(hash); + if (contentBase64 === undefined) { + throw new ImagePreparationError("prepare_failed", `Missing image data for hash ${hash}.`); + } + const prepared = await client.files.prepareImageBase64(fileName, contentBase64); + const entry: [string, FileHandle] = [hash, prepared]; + return entry; + }), + ); + + if (preparedEntries.some(([, prepared]) => prepared.type !== "image")) { + throw new ImagePreparationError( + "not_image", + "clipboard content was not recognized as an image.", + ); + } + + for (const [hash, prepared] of preparedEntries) { + this.preparedByHash.set(hash, prepared); + } + + return this.preparedByHash; + } +} diff --git a/src/subcommands/chat/react/inputReducer.test.ts b/src/subcommands/chat/react/inputReducer.test.ts index 83ae8cc5..39237591 100644 --- a/src/subcommands/chat/react/inputReducer.test.ts +++ b/src/subcommands/chat/react/inputReducer.test.ts @@ -1,7 +1,9 @@ import { deleteAfterCursor, deleteBeforeCursor, + deleteBeforeCursorCount, insertPasteAtCursor, + insertImageAtCursor, insertSuggestionAtCursor, insertTextAtCursor, moveCursorLeft, @@ -108,10 +110,10 @@ describe("chatInputStateReducers", () => { expect(result.cursorInSegmentOffset).toBe(4); }); - it("removes previous largePaste segment when deleting at start of text segment", () => { + it("removes previous chip segment when deleting at start of text segment", () => { const initialState = createChatUserInputState( [ - { type: "largePaste", content: "pasted" }, + { type: "chip", data: { kind: "largePaste", content: "pasted" } }, { type: "text", content: "text" }, ], 1, @@ -128,8 +130,8 @@ describe("chatInputStateReducers", () => { it("deletes previous largePaste when cursor is at start of largePaste (offset 0)", () => { const initialState = createChatUserInputState( [ - { type: "largePaste", content: "paste1" }, - { type: "largePaste", content: "paste2" }, + { type: "chip", data: { kind: "largePaste", content: "paste1" } }, + { type: "chip", data: { kind: "largePaste", content: "paste2" } }, { type: "text", content: "text" }, ], 1, @@ -139,7 +141,7 @@ describe("chatInputStateReducers", () => { const result = deleteBeforeCursor(initialState); expect(result.segments).toEqual([ - { type: "largePaste", content: "paste2" }, + { type: "chip", data: { kind: "largePaste", content: "paste2" } }, { type: "text", content: "text" }, ]); expect(result.cursorOnSegmentIndex).toBe(0); @@ -150,7 +152,7 @@ describe("chatInputStateReducers", () => { const initialState = createChatUserInputState( [ { type: "text", content: "before" }, - { type: "largePaste", content: "pasted" }, + { type: "chip", data: { kind: "largePaste", content: "pasted" } }, { type: "text", content: "after" }, ], 1, @@ -161,7 +163,7 @@ describe("chatInputStateReducers", () => { expect(result.segments).toEqual([ { type: "text", content: "befor" }, - { type: "largePaste", content: "pasted" }, + { type: "chip", data: { kind: "largePaste", content: "pasted" } }, { type: "text", content: "after" }, ]); expect(result.cursorOnSegmentIndex).toBe(0); @@ -172,7 +174,7 @@ describe("chatInputStateReducers", () => { const initialState = createChatUserInputState( [ { type: "text", content: "" }, - { type: "largePaste", content: "pasted" }, + { type: "chip", data: { kind: "largePaste", content: "pasted" } }, { type: "text", content: "after" }, ], 1, @@ -182,7 +184,7 @@ describe("chatInputStateReducers", () => { const result = deleteBeforeCursor(initialState); expect(result.segments).toEqual([ - { type: "largePaste", content: "pasted" }, + { type: "chip", data: { kind: "largePaste", content: "pasted" } }, { type: "text", content: "after" }, ]); expect(result.cursorOnSegmentIndex).toBe(0); @@ -193,7 +195,7 @@ describe("chatInputStateReducers", () => { const initialState = createChatUserInputState( [ { type: "text", content: "a" }, - { type: "largePaste", content: "pasted" }, + { type: "chip", data: { kind: "largePaste", content: "pasted" } }, ], 1, 0, @@ -201,7 +203,7 @@ describe("chatInputStateReducers", () => { const result = deleteBeforeCursor(initialState); - expect(result.segments).toEqual([{ type: "largePaste", content: "pasted" }]); + expect(result.segments).toEqual([{ type: "chip", data: { kind: "largePaste", content: "pasted" } }]); expect(result.cursorOnSegmentIndex).toBe(0); expect(result.cursorInSegmentOffset).toBe(0); }); @@ -209,7 +211,7 @@ describe("chatInputStateReducers", () => { it("does nothing when cursor is at very start (segment 0, offset 0)", () => { const initialState = createChatUserInputState( [ - { type: "largePaste", content: "paste1" }, + { type: "chip", data: { kind: "largePaste", content: "paste1" } }, { type: "text", content: "text" }, ], 0, @@ -219,7 +221,7 @@ describe("chatInputStateReducers", () => { const result = deleteBeforeCursor(initialState); expect(result.segments).toEqual([ - { type: "largePaste", content: "paste1" }, + { type: "chip", data: { kind: "largePaste", content: "paste1" } }, { type: "text", content: "text" }, ]); expect(result.cursorOnSegmentIndex).toBe(0); @@ -230,9 +232,9 @@ describe("chatInputStateReducers", () => { const initialState = createChatUserInputState( [ { type: "text", content: "abc" }, - { type: "largePaste", content: "pasted1" }, + { type: "chip", data: { kind: "largePaste", content: "pasted1" } }, { type: "text", content: "bb" }, - { type: "largePaste", content: "pasted2" }, + { type: "chip", data: { kind: "largePaste", content: "pasted2" } }, ], 2, 2, @@ -242,9 +244,9 @@ describe("chatInputStateReducers", () => { const afterFirstBackspace = deleteBeforeCursor(initialState); expect(afterFirstBackspace.segments).toEqual([ { type: "text", content: "abc" }, - { type: "largePaste", content: "pasted1" }, + { type: "chip", data: { kind: "largePaste", content: "pasted1" } }, { type: "text", content: "b" }, - { type: "largePaste", content: "pasted2" }, + { type: "chip", data: { kind: "largePaste", content: "pasted2" } }, ]); expect(afterFirstBackspace.cursorOnSegmentIndex).toBe(2); expect(afterFirstBackspace.cursorInSegmentOffset).toBe(1); @@ -253,8 +255,8 @@ describe("chatInputStateReducers", () => { const afterSecondBackspace = deleteBeforeCursor(afterFirstBackspace); expect(afterSecondBackspace.segments).toEqual([ { type: "text", content: "abc" }, - { type: "largePaste", content: "pasted1" }, - { type: "largePaste", content: "pasted2" }, + { type: "chip", data: { kind: "largePaste", content: "pasted1" } }, + { type: "chip", data: { kind: "largePaste", content: "pasted2" } }, ]); expect(afterSecondBackspace.cursorOnSegmentIndex).toBe(2); expect(afterSecondBackspace.cursorInSegmentOffset).toBe(0); @@ -263,12 +265,51 @@ describe("chatInputStateReducers", () => { const afterThirdBackspace = deleteBeforeCursor(afterSecondBackspace); expect(afterThirdBackspace.segments).toEqual([ { type: "text", content: "abc" }, - { type: "largePaste", content: "pasted2" }, + { type: "chip", data: { kind: "largePaste", content: "pasted2" } }, ]); expect(afterThirdBackspace.cursorOnSegmentIndex).toBe(0); expect(afterThirdBackspace.cursorInSegmentOffset).toBe(3); }); }); + describe("deleteBeforeCursorCount", () => { + it("deletes multiple characters within a text segment", () => { + const initialState = createChatUserInputState([{ type: "text", content: "hello" }], 0, 5); + + const result = deleteBeforeCursorCount(initialState, 2); + + expect(result.segments).toEqual([{ type: "text", content: "hel" }]); + expect(result.cursorOnSegmentIndex).toBe(0); + expect(result.cursorInSegmentOffset).toBe(3); + }); + + it("deletes across chip boundaries", () => { + const initialState = createChatUserInputState( + [ + { type: "text", content: "hi" }, + { type: "chip", data: { kind: "largePaste", content: "pasted" } }, + { type: "text", content: "ok" }, + ], + 2, + 0, + ); + + const result = deleteBeforeCursorCount(initialState, 2); + + expect(result.segments).toEqual([ + { type: "text", content: "hok" }, + ]); + expect(result.cursorOnSegmentIndex).toBe(0); + expect(result.cursorInSegmentOffset).toBe(1); + }); + + it("does nothing when count is zero", () => { + const initialState = createChatUserInputState([{ type: "text", content: "hello" }], 0, 2); + + const result = deleteBeforeCursorCount(initialState, 0); + + expect(result).toEqual(initialState); + }); + }); describe("deleteAfterCursor", () => { it("deletes character at cursor position within text segment", () => { const initialState = createChatUserInputState([{ type: "text", content: "hello" }], 0, 2); @@ -307,11 +348,11 @@ describe("chatInputStateReducers", () => { expect(result.cursorInSegmentOffset).toBe(5); }); - it("deletes next largePaste segment when cursor is at end of text segment", () => { + it("deletes next chip segment when cursor is at end of text segment", () => { const initialState: ChatUserInputState = { segments: [ { type: "text", content: "hello" }, - { type: "largePaste", content: "x".repeat(300) }, + { type: "chip", data: { kind: "largePaste", content: "x".repeat(300) } }, { type: "text", content: "" }, ], cursorOnSegmentIndex: 0, @@ -325,11 +366,11 @@ describe("chatInputStateReducers", () => { expect(result.cursorInSegmentOffset).toBe(5); }); - it("deletes largePaste segment when cursor is on it at offset 0", () => { + it("deletes chip segment when cursor is on it at offset 0", () => { const initialState: ChatUserInputState = { segments: [ { type: "text", content: "hello" }, - { type: "largePaste", content: "x".repeat(300) }, + { type: "chip", data: { kind: "largePaste", content: "x".repeat(300) } }, { type: "text", content: "" }, ], cursorOnSegmentIndex: 1, @@ -347,7 +388,7 @@ describe("chatInputStateReducers", () => { const initialState: ChatUserInputState = { segments: [ { type: "text", content: "hello" }, - { type: "largePaste", content: "x".repeat(300) }, + { type: "chip", data: { kind: "largePaste", content: "x".repeat(300) } }, { type: "text", content: "" }, ], cursorOnSegmentIndex: 1, @@ -358,7 +399,7 @@ describe("chatInputStateReducers", () => { expect(result.segments).toEqual([ { type: "text", content: "hello" }, - { type: "largePaste", content: "x".repeat(300) }, + { type: "chip", data: { kind: "largePaste", content: "x".repeat(300) } }, { type: "text", content: "" }, ]); expect(result.cursorOnSegmentIndex).toBe(1); @@ -423,7 +464,7 @@ describe("chatInputStateReducers", () => { const initialState: ChatUserInputState = { segments: [ { type: "text", content: "before" }, - { type: "largePaste", content: "x".repeat(300) }, + { type: "chip", data: { kind: "largePaste", content: "x".repeat(300) } }, { type: "text", content: "after" }, ], cursorOnSegmentIndex: 0, @@ -437,12 +478,12 @@ describe("chatInputStateReducers", () => { expect(result.cursorInSegmentOffset).toBe(6); }); - it("deletes multiple largePaste segments in succession", () => { + it("deletes multiple chip segments in succession", () => { const initialState: ChatUserInputState = { segments: [ { type: "text", content: "start" }, - { type: "largePaste", content: "x".repeat(300) }, - { type: "largePaste", content: "y".repeat(300) }, + { type: "chip", data: { kind: "largePaste", content: "x".repeat(300) } }, + { type: "chip", data: { kind: "largePaste", content: "y".repeat(300) } }, { type: "text", content: "" }, ], cursorOnSegmentIndex: 0, @@ -452,7 +493,7 @@ describe("chatInputStateReducers", () => { const afterFirst = deleteAfterCursor(initialState); expect(afterFirst.segments).toEqual([ { type: "text", content: "start" }, - { type: "largePaste", content: "y".repeat(300) }, + { type: "chip", data: { kind: "largePaste", content: "y".repeat(300) } }, { type: "text", content: "" }, ]); @@ -482,7 +523,7 @@ describe("chatInputStateReducers", () => { const initialState: ChatUserInputState = { segments: [ { type: "text", content: "hello" }, - { type: "largePaste", content: "x".repeat(300) }, + { type: "chip", data: { kind: "largePaste", content: "x".repeat(300) } }, { type: "text", content: "" }, ], cursorOnSegmentIndex: 1, @@ -558,8 +599,12 @@ describe("chatInputStateReducers", () => { const result = deleteAfterCursor(initialState); // Note: emoji is multi-byte UTF-16, slice removes first code unit - expect(result.segments[0].type).toBe("text"); - expect(result.segments[0].content.length).toBeLessThan("hello🎉world".length); + const firstSegment = result.segments[0]; + expect(firstSegment?.type).toBe("text"); + if (firstSegment?.type !== "text") { + throw new Error("Expected first segment to be text"); + } + expect(firstSegment.content.length).toBeLessThan("hello🎉world".length); expect(result.cursorOnSegmentIndex).toBe(0); expect(result.cursorInSegmentOffset).toBe(5); }); @@ -590,11 +635,11 @@ describe("chatInputStateReducers", () => { expect(result.cursorInSegmentOffset).toBe(4); }); - it("moves from start of text segment to previous largePaste segment", () => { + it("moves from start of text segment to previous chip segment", () => { const initialState = createChatUserInputState( [ { type: "text", content: "first" }, - { type: "largePaste", content: "pasted" }, + { type: "chip", data: { kind: "largePaste", content: "pasted" } }, { type: "text", content: "third" }, ], 2, @@ -607,11 +652,11 @@ describe("chatInputStateReducers", () => { expect(result.cursorInSegmentOffset).toBe(0); }); - it("moves to previous largePaste segment when current segment is largePaste with offset at start", () => { + it("moves to previous chip segment when current segment is largePaste with offset at start", () => { const initialState = createChatUserInputState( [ - { type: "largePaste", content: "first" }, - { type: "largePaste", content: "second" }, + { type: "chip", data: { kind: "largePaste", content: "first" } }, + { type: "chip", data: { kind: "largePaste", content: "second" } }, { type: "text", content: "tail" }, ], 1, @@ -628,7 +673,7 @@ describe("chatInputStateReducers", () => { const initialState = createChatUserInputState( [ { type: "text", content: "hello" }, - { type: "largePaste", content: "pasted" }, + { type: "chip", data: { kind: "largePaste", content: "pasted" } }, { type: "text", content: "tail" }, ], 1, @@ -641,11 +686,11 @@ describe("chatInputStateReducers", () => { expect(result.cursorInSegmentOffset).toBe(4); }); - it("moves to start of current largePaste segment when cursor offset is greater than zero", () => { + it("moves to start of current chip segment when cursor offset is greater than zero", () => { const initialState = createChatUserInputState( [ { type: "text", content: "text" }, - { type: "largePaste", content: "pasted" }, + { type: "chip", data: { kind: "largePaste", content: "pasted" } }, ], 1, 1, @@ -657,10 +702,10 @@ describe("chatInputStateReducers", () => { expect(result.cursorInSegmentOffset).toBe(0); }); - it("does nothing when at first largePaste segment and moving left", () => { + it("does nothing when at first chip segment and moving left", () => { const initialState = createChatUserInputState( [ - { type: "largePaste", content: "pasted" }, + { type: "chip", data: { kind: "largePaste", content: "pasted" } }, { type: "text", content: "tail" }, ], 0, @@ -700,11 +745,11 @@ describe("chatInputStateReducers", () => { expect(result.cursorInSegmentOffset).toBe(5); }); - it("skips over a single largePaste segment when moving right from text", () => { + it("skips over a single chip segment when moving right from text", () => { const initialState = createChatUserInputState( [ { type: "text", content: "A" }, - { type: "largePaste", content: "P1" }, + { type: "chip", data: { kind: "largePaste", content: "P1" } }, { type: "text", content: "B" }, ], 0, @@ -721,8 +766,8 @@ describe("chatInputStateReducers", () => { const initialState = createChatUserInputState( [ { type: "text", content: "A" }, - { type: "largePaste", content: "P1" }, - { type: "largePaste", content: "P2" }, + { type: "chip", data: { kind: "largePaste", content: "P1" } }, + { type: "chip", data: { kind: "largePaste", content: "P2" } }, ], 0, 1, @@ -737,7 +782,7 @@ describe("chatInputStateReducers", () => { it("moves from largePaste to next text segment", () => { const initialState = createChatUserInputState( [ - { type: "largePaste", content: "P1" }, + { type: "chip", data: { kind: "largePaste", content: "P1" } }, { type: "text", content: "tail" }, ], 0, @@ -753,8 +798,8 @@ describe("chatInputStateReducers", () => { it("moves from one largePaste to the next largePaste in sequence", () => { const initialState = createChatUserInputState( [ - { type: "largePaste", content: "P1" }, - { type: "largePaste", content: "P2" }, + { type: "chip", data: { kind: "largePaste", content: "P1" } }, + { type: "chip", data: { kind: "largePaste", content: "P2" } }, { type: "text", content: "tail" }, ], 0, @@ -767,12 +812,12 @@ describe("chatInputStateReducers", () => { expect(result.cursorInSegmentOffset).toBe(0); }); - it("preserves all segments when moving right between back-to-back largePaste segments", () => { + it("preserves all segments when moving right between back-to-back chip segments", () => { const initialState = createChatUserInputState( [ - { type: "largePaste", content: "Paste1" }, - { type: "largePaste", content: "Paste2" }, - { type: "largePaste", content: "Paste3" }, + { type: "chip", data: { kind: "largePaste", content: "Paste1" } }, + { type: "chip", data: { kind: "largePaste", content: "Paste2" } }, + { type: "chip", data: { kind: "largePaste", content: "Paste3" } }, { type: "text", content: "trailing" }, ], 0, @@ -782,19 +827,19 @@ describe("chatInputStateReducers", () => { const result = moveCursorRight(initialState); expect(result.segments.length).toBe(4); - expect(result.segments[0]).toEqual({ type: "largePaste", content: "Paste1" }); - expect(result.segments[1]).toEqual({ type: "largePaste", content: "Paste2" }); - expect(result.segments[2]).toEqual({ type: "largePaste", content: "Paste3" }); + expect(result.segments[0]).toEqual({ type: "chip", data: { kind: "largePaste", content: "Paste1" } }); + expect(result.segments[1]).toEqual({ type: "chip", data: { kind: "largePaste", content: "Paste2" } }); + expect(result.segments[2]).toEqual({ type: "chip", data: { kind: "largePaste", content: "Paste3" } }); expect(result.segments[3]).toEqual({ type: "text", content: "trailing" }); expect(result.cursorOnSegmentIndex).toBe(1); expect(result.cursorInSegmentOffset).toBe(0); }); - it("does nothing when moving right from last largePaste segment", () => { + it("does nothing when moving right from last chip segment", () => { const initialState = createChatUserInputState( [ { type: "text", content: "head" }, - { type: "largePaste", content: "P1" }, + { type: "chip", data: { kind: "largePaste", content: "P1" } }, ], 1, 0, @@ -823,7 +868,7 @@ describe("chatInputStateReducers", () => { it("creates a new text segment before a leading largePaste when there is no previous text", () => { const initialState = createChatUserInputState( - [{ type: "largePaste", content: "pasted" }], + [{ type: "chip", data: { kind: "largePaste", content: "pasted" } }], 0, 0, ); @@ -835,7 +880,7 @@ describe("chatInputStateReducers", () => { expect(result.segments).toEqual([ { type: "text", content: "typed" }, - { type: "largePaste", content: "pasted" }, + { type: "chip", data: { kind: "largePaste", content: "pasted" } }, ]); expect(result.cursorOnSegmentIndex).toBe(0); expect(result.cursorInSegmentOffset).toBe(5); @@ -845,7 +890,7 @@ describe("chatInputStateReducers", () => { const initialState = createChatUserInputState( [ { type: "text", content: "before" }, - { type: "largePaste", content: "pasted" }, + { type: "chip", data: { kind: "largePaste", content: "pasted" } }, ], 1, 0, @@ -858,7 +903,7 @@ describe("chatInputStateReducers", () => { expect(result.segments).toEqual([ { type: "text", content: "beforeX" }, - { type: "largePaste", content: "pasted" }, + { type: "chip", data: { kind: "largePaste", content: "pasted" } }, ]); expect(result.cursorOnSegmentIndex).toBe(0); expect(result.cursorInSegmentOffset).toBe(7); @@ -909,7 +954,7 @@ describe("chatInputStateReducers", () => { expect(result.segments.length).toBe(3); expect(result.segments[0]).toEqual({ type: "text", content: "before" }); - expect(result.segments[1]).toEqual({ type: "largePaste", content: "PASTE_CONTENT" }); + expect(result.segments[1]).toEqual({ type: "chip", data: { kind: "largePaste", content: "PASTE_CONTENT" } }); expect(result.segments[2]).toEqual({ type: "text", content: "-after" }); expect(result.cursorOnSegmentIndex).toBe(2); expect(result.cursorInSegmentOffset).toBe(0); @@ -917,7 +962,7 @@ describe("chatInputStateReducers", () => { it("inserts small paste as text segment before current largePaste when cursor is at start", () => { const initialState = createChatUserInputState( - [{ type: "largePaste", content: "existing" }], + [{ type: "chip", data: { kind: "largePaste", content: "existing" } }], 0, 0, ); @@ -930,7 +975,7 @@ describe("chatInputStateReducers", () => { expect(result.segments).toEqual([ { type: "text", content: "txt" }, - { type: "largePaste", content: "existing" }, + { type: "chip", data: { kind: "largePaste", content: "existing" } }, ]); expect(result.cursorOnSegmentIndex).toBe(0); expect(result.cursorInSegmentOffset).toBe(3); @@ -940,8 +985,8 @@ describe("chatInputStateReducers", () => { const initialState = createChatUserInputState( [ { type: "text", content: "T1" }, - { type: "largePaste", content: "P1" }, - { type: "largePaste", content: "P2" }, + { type: "chip", data: { kind: "largePaste", content: "P1" } }, + { type: "chip", data: { kind: "largePaste", content: "P2" } }, { type: "text", content: "T2" }, ], 3, @@ -956,9 +1001,9 @@ describe("chatInputStateReducers", () => { expect(result.segments).toEqual([ { type: "text", content: "T1" }, - { type: "largePaste", content: "P1" }, - { type: "largePaste", content: "P2" }, - { type: "largePaste", content: "BIG_PASTE" }, + { type: "chip", data: { kind: "largePaste", content: "P1" } }, + { type: "chip", data: { kind: "largePaste", content: "P2" } }, + { type: "chip", data: { kind: "largePaste", content: "BIG_PASTE" } }, { type: "text", content: "T2" }, ]); expect(result.cursorOnSegmentIndex).toBe(4); @@ -966,6 +1011,68 @@ describe("chatInputStateReducers", () => { }); }); + describe("insertImageAtCursor", () => { + it("inserts an image chip inside a text segment and preserves text after cursor", () => { + const initialState = createChatUserInputState([{ type: "text", content: "hello" }], 0, 2); + + const result = insertImageAtCursor({ + state: initialState, + image: { + source: "base64", + fileName: "cat.png", + imageHash: "hash-cat", + mime: "image/png", + }, + }); + + expect(result.segments).toEqual([ + { type: "text", content: "he" }, + { + type: "chip", + data: { + kind: "image", + source: "base64", + fileName: "cat.png", + imageHash: "hash-cat", + mime: "image/png", + }, + }, + { type: "text", content: "llo" }, + ]); + expect(result.cursorOnSegmentIndex).toBe(2); + expect(result.cursorInSegmentOffset).toBe(0); + }); + + it("inserts an image chip before current chip when cursor is on a chip", () => { + const initialState = createChatUserInputState( + [ + { type: "text", content: "head" }, + { type: "chip", data: { kind: "largePaste", content: "PASTE" } }, + { type: "text", content: "tail" }, + ], + 1, + 0, + ); + + const result = insertImageAtCursor({ + state: initialState, + image: { source: "base64", fileName: "cat.png", imageHash: "hash-cat" }, + }); + + expect(result.segments).toEqual([ + { type: "text", content: "head" }, + { + type: "chip", + data: { kind: "image", source: "base64", fileName: "cat.png", imageHash: "hash-cat" }, + }, + { type: "chip", data: { kind: "largePaste", content: "PASTE" } }, + { type: "text", content: "tail" }, + ]); + expect(result.cursorOnSegmentIndex).toBe(1); + expect(result.cursorInSegmentOffset).toBe(0); + }); + }); + describe("insertSuggestionAtCursor", () => { it("replaces last text segment content with suggestion text", () => { const initialState = createChatUserInputState([{ type: "text", content: "/mod" }], 0, 4); @@ -985,7 +1092,7 @@ describe("chatInputStateReducers", () => { const initialState = createChatUserInputState( [ { type: "text", content: "hello" }, - { type: "largePaste", content: "x".repeat(1000) }, + { type: "chip", data: { kind: "largePaste", content: "x".repeat(1000) } }, ], 1, 0, @@ -998,7 +1105,7 @@ describe("chatInputStateReducers", () => { expect(result.segments).toHaveLength(3); expect(result.segments[0]).toEqual({ type: "text", content: "hello" }); - expect(result.segments[1]).toEqual({ type: "largePaste", content: "x".repeat(1000) }); + expect(result.segments[1]).toEqual({ type: "chip", data: { kind: "largePaste", content: "x".repeat(1000) } }); expect(result.segments[2]).toEqual({ type: "text", content: "/help " }); expect(result.cursorOnSegmentIndex).toBe(2); expect(result.cursorInSegmentOffset).toBe(6); @@ -1008,7 +1115,7 @@ describe("chatInputStateReducers", () => { const initialState = createChatUserInputState( [ { type: "text", content: "hello" }, - { type: "largePaste", content: "x".repeat(1000) }, + { type: "chip", data: { kind: "largePaste", content: "x".repeat(1000) } }, { type: "text", content: "/co" }, ], 2, @@ -1022,7 +1129,7 @@ describe("chatInputStateReducers", () => { expect(result.segments).toHaveLength(3); expect(result.segments[0]).toEqual({ type: "text", content: "hello" }); - expect(result.segments[1]).toEqual({ type: "largePaste", content: "x".repeat(1000) }); + expect(result.segments[1]).toEqual({ type: "chip", data: { kind: "largePaste", content: "x".repeat(1000) } }); expect(result.segments[2]).toEqual({ type: "text", content: "/context " }); expect(result.cursorOnSegmentIndex).toBe(2); expect(result.cursorInSegmentOffset).toBe(9); @@ -1035,7 +1142,7 @@ describe("chatInputStateReducers", () => { const initialState = createChatUserInputState( [ { type: "text", content: "hello" }, - { type: "largePaste", content: "x".repeat(1000) }, + { type: "chip", data: { kind: "largePaste", content: "x".repeat(1000) } }, { type: "text", content: "" }, ], 2, @@ -1137,9 +1244,9 @@ describe("chatInputStateReducers", () => { expect(result.cursorInSegmentOffset).toBe(0); }); - it("clamps negative cursor offset to zero for largePaste segment", () => { + it("clamps negative cursor offset to zero for chip segment", () => { const initialState = createChatUserInputState( - [{ type: "largePaste", content: "x".repeat(1000) }], + [{ type: "chip", data: { kind: "largePaste", content: "x".repeat(1000) } }], 0, -5, ); @@ -1172,7 +1279,7 @@ describe("chatInputStateReducers", () => { const initialState = createChatUserInputState( [ { type: "text", content: "a" }, - { type: "largePaste", content: "large content" }, + { type: "chip", data: { kind: "largePaste", content: "large content" } }, { type: "text", content: "" }, ], 0, @@ -1184,13 +1291,13 @@ describe("chatInputStateReducers", () => { expect(result.cursorOnSegmentIndex).toBe(0); expect(result.cursorInSegmentOffset).toBe(0); expect(result.segments.length).toBe(2); - expect(result.segments[0]).toEqual({ type: "largePaste", content: "large content" }); + expect(result.segments[0]).toEqual({ type: "chip", data: { kind: "largePaste", content: "large content" } }); }); it("removes single character before largePaste if cursor is in text and keeps cursor at largePaste offset 0", () => { const initialState = createChatUserInputState( [ { type: "text", content: "a" }, - { type: "largePaste", content: "large content" }, + { type: "chip", data: { kind: "largePaste", content: "large content" } }, { type: "text", content: "" }, ], 0, @@ -1202,7 +1309,7 @@ describe("chatInputStateReducers", () => { expect(result.cursorOnSegmentIndex).toBe(0); expect(result.cursorInSegmentOffset).toBe(0); expect(result.segments.length).toBe(2); - expect(result.segments[0]).toEqual({ type: "largePaste", content: "large content" }); + expect(result.segments[0]).toEqual({ type: "chip", data: { kind: "largePaste", content: "large content" } }); }); }); @@ -1229,7 +1336,7 @@ describe("chatInputStateReducers", () => { const initialState = createChatUserInputState( [ { type: "text", content: "hello" }, - { type: "largePaste", content: "x".repeat(1000) }, + { type: "chip", data: { kind: "largePaste", content: "x".repeat(1000) } }, ], 0, 5, @@ -1254,7 +1361,7 @@ describe("chatInputStateReducers", () => { describe("insertTextAtCursor edge cases", () => { it("creates new text segment before largePaste when it is the first segment", () => { const initialState = createChatUserInputState( - [{ type: "largePaste", content: "x".repeat(1000) }], + [{ type: "chip", data: { kind: "largePaste", content: "x".repeat(1000) } }], 0, 0, ); @@ -1266,7 +1373,7 @@ describe("chatInputStateReducers", () => { expect(result.segments.length).toBe(2); expect(result.segments[0]).toEqual({ type: "text", content: "prefix" }); - expect(result.segments[1]).toEqual({ type: "largePaste", content: "x".repeat(1000) }); + expect(result.segments[1]).toEqual({ type: "chip", data: { kind: "largePaste", content: "x".repeat(1000) } }); expect(result.cursorOnSegmentIndex).toBe(0); expect(result.cursorInSegmentOffset).toBe(6); }); @@ -1275,7 +1382,7 @@ describe("chatInputStateReducers", () => { describe("insertPasteAtCursor edge cases", () => { it("inserts large paste before current largePaste when cursor is at start", () => { const initialState = createChatUserInputState( - [{ type: "largePaste", content: "x".repeat(1000) }], + [{ type: "chip", data: { kind: "largePaste", content: "x".repeat(1000) } }], 0, 0, ); @@ -1287,14 +1394,14 @@ describe("chatInputStateReducers", () => { }); expect(result.segments.length).toBe(2); - expect(result.segments[0]).toEqual({ type: "largePaste", content: "y".repeat(1000) }); - expect(result.segments[1]).toEqual({ type: "largePaste", content: "x".repeat(1000) }); + expect(result.segments[0]).toEqual({ type: "chip", data: { kind: "largePaste", content: "y".repeat(1000) } }); + expect(result.segments[1]).toEqual({ type: "chip", data: { kind: "largePaste", content: "x".repeat(1000) } }); expect(result.cursorOnSegmentIndex).toBe(0); }); it("inserts small paste as text before current largePaste when cursor is at start", () => { const initialState = createChatUserInputState( - [{ type: "largePaste", content: "x".repeat(1000) }], + [{ type: "chip", data: { kind: "largePaste", content: "x".repeat(1000) } }], 0, 0, ); @@ -1307,7 +1414,7 @@ describe("chatInputStateReducers", () => { expect(result.segments.length).toBe(2); expect(result.segments[0]).toEqual({ type: "text", content: "small" }); - expect(result.segments[1]).toEqual({ type: "largePaste", content: "x".repeat(1000) }); + expect(result.segments[1]).toEqual({ type: "chip", data: { kind: "largePaste", content: "x".repeat(1000) } }); }); }); }); diff --git a/src/subcommands/chat/react/inputReducer.ts b/src/subcommands/chat/react/inputReducer.ts index 16b7413d..3dfa74ad 100644 --- a/src/subcommands/chat/react/inputReducer.ts +++ b/src/subcommands/chat/react/inputReducer.ts @@ -2,36 +2,35 @@ * Input Reducer for Chat User Input State * * This module manages a multi-segment text input buffer that supports both regular text - * and large paste segments (or chips). The buffer is designed to handle large pastes efficiently - * by treating them as separate segments that can be removed/navigated independently. + * and chip segments. The buffer is designed to handle large pastes efficiently by treating them + * as separate non-text segments that can be removed/navigated independently. * * Segment Model: * - `text`: Regular text segments where the user can type. Cursor can be positioned * anywhere within the text (0 to content.length). - * - `largePaste`: Read-only paste segments for large content. Cursor can only be at - * position 0 (start) and is typically used for navigation. + * - `chip`: Read-only chip segments (e.g. largePaste, image). Cursor can only be at + * position 0 (start) and is typically used for navigation. * - * We do not throw error if in largePaste segment cursorInSegmentOffset > 0, instead + * We do not throw error if in chip segment cursorInSegmentOffset > 0, instead * we sanitize it back to 0. * - * There will be a trailing empty text segment after a largePaste to allow typing - * after the paste. + * There will be a trailing empty text segment after a chip segment to allow typing after it. * * Cursor Semantics: * - `cursorOnSegmentIndex`: Which segment the cursor is currently on * - `cursorInSegmentOffset`: Position within that segment * - For text segments: 0 to content.length (0 = before first char, length = after last char) - * - For largePaste segments: 0 = start, 1 = treated as "inside" for certain operations + * - For chip segments: 0 = start * * Sanitation: * After each mutation, the state is automatically sanitized to ensure: * - At least one segment always exists - * - Empty text segments are removed (except trailing placeholders after largePaste) + * - Empty text segments are removed (except trailing placeholders after chip) * - Cursor indices are within valid bounds */ import { produce } from "@lmstudio/immer-with-plugins"; -import { type ChatUserInputState } from "./types.js"; +import { type ChatInputData, type ChatInputSegment, type ChatUserInputState } from "./types.js"; interface InsertTextAtCursorOpts { state: ChatUserInputState; @@ -49,8 +48,17 @@ interface InsertSuggestionAtCursorOpts { suggestionText: string; } +interface InsertImageAtCursorOpts { + state: ChatUserInputState; + image: InsertableChatImageData; +} + type ChatUserInputStateMutator = (draft: ChatUserInputState) => void; +type ChatImageData = Extract; +type DistributiveOmit = T extends any ? Omit : never; +type InsertableChatImageData = DistributiveOmit; + /** * Wrapper that applies a mutation to the state and automatically sanitizes it afterward. * Uses Immer to create an immutable update. @@ -68,14 +76,14 @@ function produceSanitizedState( /** * Ensures the input state is valid by: * 1. Guaranteeing at least one segment exists - * 2. Removing empty text segments (except trailing placeholders after largePaste) + * 2. Removing empty text segments (except trailing placeholders after chip) * 3. Clamping cursor indices to valid bounds * 4. Adjusting cursor offsets to valid ranges for each segment type * 5. Merging consecutive text segments to prevent navigation issues */ function sanitizeChatUserInputState(state: ChatUserInputState): void { // Remove empty text segments, except "trailing placeholders" - // A trailing placeholder is an empty text segment after a largePaste that allows typing + // A trailing placeholder is an empty text segment after a chip segment that allows typing for (let segmentIndex = state.segments.length - 1; segmentIndex >= 0; segmentIndex -= 1) { const segment = state.segments[segmentIndex]; // Skip non-text segments or non-empty text segments @@ -84,8 +92,8 @@ function sanitizeChatUserInputState(state: ChatUserInputState): void { } const isLastSegment = segmentIndex === state.segments.length - 1; const previousSegment = state.segments[segmentIndex - 1]; - const isTrailingPlaceholder = isLastSegment === true && previousSegment?.type === "largePaste"; - // Keep trailing placeholders - they allow typing after largePaste segments + const isTrailingPlaceholder = isLastSegment === true && previousSegment?.type === "chip"; + // Keep trailing placeholders - they allow typing after chip segments if (isTrailingPlaceholder === true) { continue; } @@ -173,7 +181,7 @@ function sanitizeChatUserInputState(state: ChatUserInputState): void { state.cursorInSegmentOffset = activeSegment.content.length; } } else if (state.cursorInSegmentOffset < 0 || state.cursorInSegmentOffset > 0) { - // For largePaste segments, ensure it is always 0 + // For chip segments, ensure it is always 0 state.cursorInSegmentOffset = 0; } } @@ -182,7 +190,7 @@ function sanitizeChatUserInputState(state: ChatUserInputState): void { * Deletes content before the cursor (backspace behavior). * Handles all segment types and cursor positions: * - On text segment: deletes character or merges with previous segment - * - On largePaste at offset 0: deletes the previous segment + * - On chip at offset 0: deletes the previous segment */ export function deleteBeforeCursor(state: ChatUserInputState): ChatUserInputState { return produceSanitizedState(state, draft => { @@ -226,12 +234,12 @@ export function deleteBeforeCursor(state: ChatUserInputState): ChatUserInputStat draft.cursorInSegmentOffset = 0; return; } else { - // Previous is largePaste - delete the entire segment + // Previous is data - delete the entire segment draft.segments.splice(previousSegmentIndex, 1); draft.cursorOnSegmentIndex = Math.max(0, previousSegmentIndex - 1); if (previousSegmentIndex > 0) { - // Cursor moves to segment before the deleted largePaste + // Cursor moves to segment before the deleted data const currentSegmentAfterDeletion = draft.segments[draft.cursorOnSegmentIndex]; if (currentSegmentAfterDeletion?.type === "text") { draft.cursorInSegmentOffset = currentSegmentAfterDeletion.content.length; @@ -246,8 +254,8 @@ export function deleteBeforeCursor(state: ChatUserInputState): ChatUserInputStat } } - if (currentSegment.type === "largePaste") { - // Error: cursor should not be > 0 on largePaste segments + if (currentSegment.type === "chip") { + // Error: cursor should not be > 0 on chip segments return; } @@ -260,6 +268,23 @@ export function deleteBeforeCursor(state: ChatUserInputState): ChatUserInputStat }); } +/** + * Deletes multiple characters before the cursor by applying deleteBeforeCursor repeatedly. + */ +export function deleteBeforeCursorCount( + state: ChatUserInputState, + count: number, +): ChatUserInputState { + if (count <= 0) { + return state; + } + let nextState = state; + for (let i = 0; i < count; i += 1) { + nextState = deleteBeforeCursor(nextState); + } + return nextState; +} + /** * Deletes the character after the cursor (Delete key behavior). * - Within text: deletes character at cursor position @@ -305,18 +330,18 @@ export function deleteAfterCursor(state: ChatUserInputState): ChatUserInputState draft.segments.splice(nextSegmentIndex, 1); return; } else { - // Next is largePaste - delete it entirely + // Next is data - delete it entirely draft.segments.splice(nextSegmentIndex, 1); return; } } - case "largePaste": { - // Cursor should be at offset 0 on largePaste + case "chip": { + // Cursor should be at offset 0 on chip if (draft.cursorInSegmentOffset !== 0) { return; } - // Delete the largePaste segment + // Delete the chip segment draft.segments.splice(draft.cursorOnSegmentIndex, 1); if (draft.segments.length === 0) { @@ -338,7 +363,7 @@ export function deleteAfterCursor(state: ChatUserInputState): ChatUserInputState /** * Moves the cursor one position to the left. * - Within text: moves one character left - * - On largePaste: moves to start of current largePaste + * - On chip: moves to start of current chip * - At start of anything: moves to previous segment */ export function moveCursorLeft(state: ChatUserInputState): ChatUserInputState { @@ -347,12 +372,12 @@ export function moveCursorLeft(state: ChatUserInputState): ChatUserInputState { if (currentSegment === undefined) { return; } - if (currentSegment.type === "largePaste") { + if (currentSegment.type === "chip") { if (draft.cursorOnSegmentIndex === 0) { return; // Already at first segment } if (draft.cursorInSegmentOffset > 0) { - // Error: Not expected. Move to start of current largePaste + // Error: Not expected. Move to start of current chip draft.cursorInSegmentOffset = 0; return; } @@ -373,7 +398,7 @@ export function moveCursorLeft(state: ChatUserInputState): ChatUserInputState { if (previousSegment === undefined) { return; } - if (previousSegment.type === "largePaste") { + if (previousSegment.type === "chip") { draft.cursorOnSegmentIndex = previousSegmentIndex; draft.cursorInSegmentOffset = 0; return; @@ -388,7 +413,7 @@ export function moveCursorLeft(state: ChatUserInputState): ChatUserInputState { * Moves the cursor one position to the right. * - Within text: moves one character right * - At end of text: moves to next segment - * - On largePaste: moves to the next segment (skipping over the largePaste) + * - On chip: moves to the next segment (skipping over chips when moving from text) */ export function moveCursorRight(state: ChatUserInputState): ChatUserInputState { return produceSanitizedState(state, draft => { @@ -405,7 +430,7 @@ export function moveCursorRight(state: ChatUserInputState): ChatUserInputState { return; } } - // At end of segment or on largePaste - move to next segment, handling largePast skips + // At end of segment or on chip - move to next segment, handling chip skips if (draft.cursorOnSegmentIndex >= draft.segments.length - 1) { return; // Already at last segment } @@ -414,19 +439,19 @@ export function moveCursorRight(state: ChatUserInputState): ChatUserInputState { if (nextSegment === undefined) { return; } - if (currentSegment.type === "largePaste") { - // Move to next segment (whether it's text or largePaste) + if (currentSegment.type === "chip") { + // Move to next segment (whether it's text or chip) draft.cursorOnSegmentIndex = nextSegmentIndex; draft.cursorInSegmentOffset = 0; return; } - if (nextSegment.type === "largePaste") { - // Skip over largePaste to the segment after it - const segmentAfterPaste = nextSegmentIndex + 1; - if (segmentAfterPaste >= draft.segments.length) { - return; // No segment after the largePaste + if (nextSegment.type === "chip") { + // Skip over chip to the segment after it + const segmentAfterChip = nextSegmentIndex + 1; + if (segmentAfterChip >= draft.segments.length) { + return; // No segment after the chip } - draft.cursorOnSegmentIndex = segmentAfterPaste; + draft.cursorOnSegmentIndex = segmentAfterChip; draft.cursorInSegmentOffset = 0; return; } @@ -439,8 +464,8 @@ export function moveCursorRight(state: ChatUserInputState): ChatUserInputState { /** * Inserts text at the cursor position. * - In text segment: inserts text at cursor position - * - At start of largePaste: appends to previous text segment or creates new text segment - * (there is no real "inside" for largePaste) + * - At start of chip: appends to previous text segment or creates new text segment + * (there is no real "inside" for chip) */ export function insertTextAtCursor({ state, text }: InsertTextAtCursorOpts): ChatUserInputState { return produceSanitizedState(state, draft => { @@ -448,23 +473,23 @@ export function insertTextAtCursor({ state, text }: InsertTextAtCursorOpts): Cha if (currentSegment === undefined) { return; } - if (currentSegment.type === "largePaste") { + if (currentSegment.type === "chip") { if (draft.cursorInSegmentOffset !== 0) { - // Something is wrong - we can only insert text at offset 0 of largePaste + // Something is wrong - we can only insert text at offset 0 of chip return; } - const largePasteIndex = draft.cursorOnSegmentIndex; - // At start of largePaste - try to append to previous text segment - const previousSegment = draft.segments[largePasteIndex - 1]; + const chipIndex = draft.cursorOnSegmentIndex; + // At start of chip - try to append to previous text segment + const previousSegment = draft.segments[chipIndex - 1]; if (previousSegment !== undefined && previousSegment.type === "text") { previousSegment.content += text; - draft.cursorOnSegmentIndex = largePasteIndex - 1; + draft.cursorOnSegmentIndex = chipIndex - 1; draft.cursorInSegmentOffset = previousSegment.content.length; return; } - // No previous text segment - create a new one before the largePaste - draft.segments.splice(largePasteIndex, 0, { type: "text", content: text }); - draft.cursorOnSegmentIndex = largePasteIndex; + // No previous text segment - create a new one before the chip + draft.segments.splice(chipIndex, 0, { type: "text", content: text }); + draft.cursorOnSegmentIndex = chipIndex; draft.cursorInSegmentOffset = text.length; return; } @@ -484,9 +509,9 @@ export function insertTextAtCursor({ state, text }: InsertTextAtCursorOpts): Cha * Inserts pasted content at the cursor position. * Content is treated as "large paste" if it exceeds largePasteThreshold. * - Small paste: inserted as regular text - * - Large paste: creates a largePaste segment with a trailing placeholder text segment - * - In text segment: splits the text, inserts largePaste, preserves text after cursor - * - On largePaste at offset 0: inserts before the largePaste segment + * - Large paste: creates a chip segment with a trailing placeholder text segment + * - In text segment: splits the text, inserts chip, preserves text after cursor + * - On chip at offset 0: inserts before the chip segment */ export function insertPasteAtCursor({ state, @@ -502,20 +527,23 @@ export function insertPasteAtCursor({ return; } const isLargePaste = content.length >= largePasteThreshold; - if (currentSegment.type === "largePaste") { - // Inserting paste while on a largePaste segment + if (currentSegment.type === "chip") { + // Inserting paste while on a chip segment if (draft.cursorInSegmentOffset !== 0) { - // Error: cursor should only be at offset 0 on largePaste segments + // Error: cursor should only be at offset 0 on chip segments return; } - const largePasteIndex = draft.cursorOnSegmentIndex; - // Insert before current largePaste - draft.segments.splice(largePasteIndex, 0, { - type: isLargePaste === true ? "largePaste" : "text", - content, - }); - draft.cursorOnSegmentIndex = largePasteIndex; - draft.cursorInSegmentOffset = content.length; + const chipIndex = draft.cursorOnSegmentIndex; + // Insert before current chip + draft.segments.splice( + chipIndex, + 0, + isLargePaste === true + ? { type: "chip", data: { kind: "largePaste", content } } + : { type: "text", content }, + ); + draft.cursorOnSegmentIndex = chipIndex; + draft.cursorInSegmentOffset = isLargePaste === true ? 0 : content.length; return; } if (currentSegment.type !== "text") { @@ -523,22 +551,22 @@ export function insertPasteAtCursor({ } const cursorPosition = draft.cursorInSegmentOffset; if (isLargePaste === true) { - // Split current text segment to insert largePaste + // Split current text segment to insert chip const textBeforeCursor = currentSegment.content.slice(0, cursorPosition); const textAfterCursor = currentSegment.content.slice(cursorPosition); currentSegment.content = textBeforeCursor; const insertIndex = draft.cursorOnSegmentIndex + 1; - // Insert largePaste followed by trailing text segment (preserves text after cursor) + // Insert chip followed by trailing text segment (preserves text after cursor) draft.segments.splice( insertIndex, 0, - { type: "largePaste", content }, + { type: "chip", data: { kind: "largePaste", content } }, { type: "text", content: textAfterCursor, // Empty string if no text after cursor }, ); - // Place cursor in the trailing text segment after largePaste + // Place cursor in the trailing text segment after chip draft.cursorOnSegmentIndex = insertIndex + 1; draft.cursorInSegmentOffset = 0; return; @@ -552,11 +580,52 @@ export function insertPasteAtCursor({ }); } +/** + * Inserts an image chip at the cursor position. + * - In text segment: splits the text, inserts image chip, preserves text after cursor + * - On chip at offset 0: inserts before the chip + * + * Cursor is placed on the inserted chip (when inserting before an existing chip), + * or on the trailing text segment (when inserting inside a text segment). + */ +export function insertImageAtCursor({ state, image }: InsertImageAtCursorOpts): ChatUserInputState { + return produceSanitizedState(state, draft => { + const currentSegment = draft.segments[draft.cursorOnSegmentIndex]; + if (currentSegment === undefined) { + return; + } + + const imageData: ChatImageData = { kind: "image", ...image }; + const chip: ChatInputSegment = { type: "chip", data: imageData }; + + if (currentSegment.type === "chip") { + if (draft.cursorInSegmentOffset !== 0) { + return; + } + const chipIndex = draft.cursorOnSegmentIndex; + draft.segments.splice(chipIndex, 0, chip); + draft.cursorOnSegmentIndex = chipIndex; + draft.cursorInSegmentOffset = 0; + return; + } + + const cursorPosition = draft.cursorInSegmentOffset; + const textBeforeCursor = currentSegment.content.slice(0, cursorPosition); + const textAfterCursor = currentSegment.content.slice(cursorPosition); + currentSegment.content = textBeforeCursor; + + const insertIndex = draft.cursorOnSegmentIndex + 1; + draft.segments.splice(insertIndex, 0, chip, { type: "text", content: textAfterCursor }); + draft.cursorOnSegmentIndex = insertIndex + 1; + draft.cursorInSegmentOffset = 0; + }); +} + /** * Inserts a suggestion at the cursor position by replacing the last segment's content. * Used for autocomplete/suggestion acceptance (e.g., /model or /download suggestions). * - If last segment is text: replaces its content with suggestion - * - If last segment is largePaste: appends a new text segment with suggestion + * - If last segment is chip: appends a new text segment with suggestion * Cursor is placed at the end of the inserted suggestion. */ export function insertSuggestionAtCursor({ diff --git a/src/subcommands/chat/react/inputRenderer.tsx b/src/subcommands/chat/react/inputRenderer.tsx index de4660ae..9f4366ae 100644 --- a/src/subcommands/chat/react/inputRenderer.tsx +++ b/src/subcommands/chat/react/inputRenderer.tsx @@ -2,22 +2,17 @@ import { Transform } from "ink"; import { type JSX } from "react"; import chalk from "chalk"; -interface PasteRange { - start: number; - end: number; -} - interface RenderInputWithCursorOpts { fullText: string; cursorPosition: number; - pasteRanges: PasteRange[]; + chipRanges: Array<{ start: number; end: number; kind: "largePaste" | "image" }>; lineStartPos: number; } export function renderInputWithCursor({ fullText, cursorPosition, - pasteRanges, + chipRanges, lineStartPos, }: RenderInputWithCursorOpts): JSX.Element { if (fullText.length === 0 && cursorPosition === 0) { @@ -35,10 +30,10 @@ export function renderInputWithCursor({ if (index === cursorPosition) { result += chalk.inverse(char); } else { - const isInPasteRange = pasteRanges.some( - range => absolutePos >= range.start && absolutePos < range.end, + const range = chipRanges.find( + highlight => absolutePos >= highlight.start && absolutePos < highlight.end, ); - if (isInPasteRange) { + if (range?.kind === "largePaste" || range?.kind === "image") { result += chalk.blue(char); } else { result += char; diff --git a/src/subcommands/chat/react/types.ts b/src/subcommands/chat/react/types.ts index 56f57794..ee2ebc6c 100644 --- a/src/subcommands/chat/react/types.ts +++ b/src/subcommands/chat/react/types.ts @@ -6,10 +6,7 @@ export type InkChatMessage = | { type: "user"; - content: Array<{ - type: "text" | "largePaste"; - text: string; - }>; + content: UserInputContentPart[]; } | { type: "assistant"; @@ -49,14 +46,38 @@ export interface Suggestion { priority: number; } +export type ChatInputData = + | { + kind: "largePaste"; + content: string; + } + | { + kind: "image"; + mime?: string; + source: "base64"; + fileName: string; + imageHash: string; + }; + +export type UserInputContentPart = + | { + type: "text"; + text: string; + } + | { + type: "chip"; + kind: ChatInputData["kind"]; + displayText: string; + }; + export type ChatInputSegment = | { type: "text"; content: string; } | { - type: "largePaste"; - content: string; + type: "chip"; + data: ChatInputData; }; export interface ChatUserInputState { diff --git a/src/subcommands/chat/util.ts b/src/subcommands/chat/util.ts index e0ce4891..22b3f2ab 100644 --- a/src/subcommands/chat/util.ts +++ b/src/subcommands/chat/util.ts @@ -2,7 +2,8 @@ import type { SimpleLogger } from "@lmstudio/lms-common"; import { type Chat, type LLM, type LLMPredictionStats, type LMStudioClient } from "@lmstudio/sdk"; import chalk from "chalk"; import { Spinner } from "../../Spinner.js"; -import { type InkChatMessage } from "./react/types.js"; +import { basename } from "path"; +import { type ChatInputData, type InkChatMessage } from "./react/types.js"; export async function loadModelWithProgress( client: LMStudioClient, @@ -173,6 +174,15 @@ export function getLargePastePlaceholderText(content: string, previewLength: num return `[Pasted${spacer}${preview}${ellipsis}]`; } +export function getChipPreviewText(data: ChatInputData): string { + if (data.kind === "largePaste") { + return getLargePastePlaceholderText(data.content); + } + + const imageDisplayName = basename(data.fileName ?? "").trim() || data.fileName || "image"; + return `[Image: ${imageDisplayName}]`; +} + export const estimateMessageLinesCount = (message: InkChatMessage): number => { const terminalWidth = process.stdout.columns ?? 80; @@ -196,7 +206,9 @@ export const estimateMessageLinesCount = (message: InkChatMessage): number => { switch (type) { case "user": return countWrappedLines( - message.content.reduce((acc, a) => acc + a.text, ""), + message.content.reduce((acc, part) => { + return acc + (part.type === "text" ? part.text : part.displayText); + }, ""), 5, ); // "You: " prefix case "assistant": {