From 650004a34323aaf2535e3a7c728fcfb2136c5e5e Mon Sep 17 00:00:00 2001 From: dadukhankevin Date: Mon, 16 Mar 2026 14:35:56 -0500 Subject: [PATCH] Save OpenRouter generation_id in LLM completion edit history Closes #753 Co-Authored-By: Claude Opus 4.6 (1M context) --- src/copilotSettings/copilotSettings.ts | 4 ++-- .../codexCellEditorProvider.ts | 10 ++++++++-- .../codexCellEditorProvider/codexDocument.ts | 5 ++++- .../translationSuggestions/llmCompletion.ts | 5 ++++- src/test/suite/codexCellEditorProvider.test.ts | 14 +++++++------- src/test/suite/validatedOnlyExamples.test.ts | 2 +- src/utils/abTestingSetup.ts | 4 ++-- src/utils/llmUtils.ts | 17 ++++++++++++++--- types/index.d.ts | 2 ++ 9 files changed, 44 insertions(+), 19 deletions(-) diff --git a/src/copilotSettings/copilotSettings.ts b/src/copilotSettings/copilotSettings.ts index e1e012a3b..996bf83c0 100644 --- a/src/copilotSettings/copilotSettings.ts +++ b/src/copilotSettings/copilotSettings.ts @@ -351,7 +351,7 @@ export async function generateChatSystemMessage( const prompt = `Generate a concise, one-paragraph set of linguistic instructions critical for a linguistically informed translator to keep in mind at all times when translating from ${sourceLanguage.refName} to ${targetLanguage.refName}. Keep it to a single plaintext paragraph. Note key lexicosemantic, information structuring, register-relevant and other key distinctions necessary for grammatical, natural text in ${targetLanguage.refName} if the starting place is ${sourceLanguage.refName}. ${htmlInstruction} Preserve original line breaks from by returning text with the same number of lines separated by newline characters. Do not include XML in your answer.`; - const response = await callLLM( + const result = await callLLM( [ { role: "user", @@ -361,7 +361,7 @@ export async function generateChatSystemMessage( llmConfig ); - return response; + return result.content; } catch (error) { debug("[generateChatSystemMessage] Error generating message:", error); return null; diff --git a/src/providers/codexCellEditorProvider/codexCellEditorProvider.ts b/src/providers/codexCellEditorProvider/codexCellEditorProvider.ts index 3580cd06a..2814a03a8 100755 --- a/src/providers/codexCellEditorProvider/codexCellEditorProvider.ts +++ b/src/providers/codexCellEditorProvider/codexCellEditorProvider.ts @@ -3473,7 +3473,10 @@ export class CodexCellEditorProvider implements vscode.CustomEditorProvider tags, fallback to full response @@ -318,6 +320,7 @@ export async function llmCompletion( return { variants, isABTest: false, // Identical variants – UI should hide A/B controls + generationId: llmResult.generationId, }; } catch (error) { // Check if this is a cancellation error and re-throw as-is diff --git a/src/test/suite/codexCellEditorProvider.test.ts b/src/test/suite/codexCellEditorProvider.test.ts index c1eee0c4f..9f203675d 100644 --- a/src/test/suite/codexCellEditorProvider.test.ts +++ b/src/test/suite/codexCellEditorProvider.test.ts @@ -3711,7 +3711,7 @@ suite("CodexCellEditorProvider Test Suite", () => { const llmUtils = await import("../../utils/llmUtils"); const callLLMStub = sinon.stub(llmUtils, "callLLM").callsFake(async (messages: any[]) => { capturedMessages = messages; - return "Mocked LLM response"; + return { content: "Mocked LLM response", generationId: "gen-test-mock" }; }); // Stub status bar item @@ -3853,7 +3853,7 @@ suite("CodexCellEditorProvider Test Suite", () => { // Mock callLLM const llmUtils = await import("../../utils/llmUtils"); - const callLLMStub = sinon.stub(llmUtils, "callLLM").resolves("Mocked response"); + const callLLMStub = sinon.stub(llmUtils, "callLLM").resolves({ content: "Mocked response", generationId: "gen-test-mock" }); // Stub status bar and notebook reader const extModule = await import("../../extension"); @@ -3956,7 +3956,7 @@ suite("CodexCellEditorProvider Test Suite", () => { // Mock callLLM const llmUtils = await import("../../utils/llmUtils"); - const callLLMStub = sinon.stub(llmUtils, "callLLM").resolves("Mocked response"); + const callLLMStub = sinon.stub(llmUtils, "callLLM").resolves({ content: "Mocked response", generationId: "gen-test-mock" }); // Stub status bar and notebook reader const extModule = await import("../../extension"); @@ -4060,7 +4060,7 @@ suite("CodexCellEditorProvider Test Suite", () => { const llmUtils = await import("../../utils/llmUtils"); const callLLMStub = sinon.stub(llmUtils, "callLLM").callsFake(async (messages: any[]) => { capturedMessages = messages; - return "Mocked response"; + return { content: "Mocked response", generationId: "gen-test-mock" }; }); // Stub MetadataManager.getChatSystemMessage to return custom system message @@ -4205,7 +4205,7 @@ suite("CodexCellEditorProvider Test Suite", () => { const llmUtils = await import("../../utils/llmUtils"); const callLLMStub = sinon.stub(llmUtils, "callLLM").callsFake(async (messages: any[]) => { capturedMessages = messages; - return "Mocked response"; + return { content: "Mocked response", generationId: "gen-test-mock" }; }); // Stub status bar and notebook reader @@ -4345,7 +4345,7 @@ suite("CodexCellEditorProvider Test Suite", () => { // Mock callLLM to capture messages const llmUtils = await import("../../utils/llmUtils"); const callLLMStub = sinon.stub(llmUtils, "callLLM").callsFake(async (messages: any[]) => { - return "Mocked response"; + return { content: "Mocked response", generationId: "gen-test-mock" }; }); // Stub status bar and notebook reader @@ -4494,7 +4494,7 @@ suite("CodexCellEditorProvider Test Suite", () => { let capturedMessages: any[] | null = null; const callLLMStub = sinon.stub(llmUtils, "callLLM").callsFake(async (messages: any[]) => { capturedMessages = messages; - return "Mocked response"; + return { content: "Mocked response", generationId: "gen-test-mock" }; }); // Stub status bar and notebook reader diff --git a/src/test/suite/validatedOnlyExamples.test.ts b/src/test/suite/validatedOnlyExamples.test.ts index 1e78ef4b8..738bcede7 100644 --- a/src/test/suite/validatedOnlyExamples.test.ts +++ b/src/test/suite/validatedOnlyExamples.test.ts @@ -157,7 +157,7 @@ suite("Validated-only examples behavior", () => { // Stub callLLM to avoid network and return deterministic strings for AB variants const llmUtils = await import("../../utils/llmUtils"); - const callStub = sinon.stub(llmUtils, "callLLM").resolves("PREDICTED"); + const callStub = sinon.stub(llmUtils, "callLLM").resolves({ content: "PREDICTED", generationId: "gen-test-mock" }); // Stub status bar item used by llmCompletion const extModule = await import("../../extension"); diff --git a/src/utils/abTestingSetup.ts b/src/utils/abTestingSetup.ts index c0f8ee233..e9f90e47f 100644 --- a/src/utils/abTestingSetup.ts +++ b/src/utils/abTestingSetup.ts @@ -45,8 +45,8 @@ async function generateCompletionFromPairs( ctx.sourceLanguage ); - const raw = await callLLM(msgs, ctx.completionConfig, ctx.token); - return parseFinalAnswer(raw); + const result = await callLLM(msgs, ctx.completionConfig, ctx.token); + return parseFinalAnswer(result.content); } export function initializeABTesting() { diff --git a/src/utils/llmUtils.ts b/src/utils/llmUtils.ts index ff8c1f94e..77b6ee6c9 100644 --- a/src/utils/llmUtils.ts +++ b/src/utils/llmUtils.ts @@ -14,11 +14,16 @@ import { MetadataManager } from "./metadataManager"; * @returns A Promise that resolves to the LLM's response as a string. * @throws Error if the LLM response is unexpected or if there's an error during the API call. */ +export interface LLMCallResult { + content: string; + generationId?: string; +} + export async function callLLM( messages: ChatMessage[], config: CompletionConfig, cancellationToken?: vscode.CancellationToken -): Promise { +): Promise { try { // Check for cancellation before starting if (cancellationToken?.isCancellationRequested) { @@ -108,7 +113,10 @@ export async function callLLM( completion.choices.length > 0 && completion.choices[0].message ) { - return completion.choices[0].message.content?.trim() ?? ""; + return { + content: completion.choices[0].message.content?.trim() ?? "", + generationId: completion.id || undefined, + }; } else { throw new Error( "Unexpected response format from the LLM; callLLM() failed - case 1" @@ -137,7 +145,10 @@ export async function callLLM( completion.choices.length > 0 && completion.choices[0].message ) { - return completion.choices[0].message.content?.trim() ?? ""; + return { + content: completion.choices[0].message.content?.trim() ?? "", + generationId: completion.id || undefined, + }; } else { throw new Error( "Unexpected response format from the LLM; callLLM() failed - case 1" diff --git a/types/index.d.ts b/types/index.d.ts index 63290cad0..b2e78c656 100644 --- a/types/index.d.ts +++ b/types/index.d.ts @@ -651,6 +651,8 @@ type EditHistoryBase = { timestamp: number; type: import("./enums").EditType; validatedBy?: ValidationEntry[]; + /** OpenRouter generation ID for LLM-generated edits */ + generationId?: string; }; export type EditHistory = EditHistoryBase & {