Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/copilotSettings/copilotSettings.ts
Original file line number Diff line number Diff line change
Expand Up @@ -351,7 +351,7 @@ export async function generateChatSystemMessage(

const prompt = `Generate a concise, one-paragraph set of linguistic instructions critical for a linguistically informed translator to keep in mind at all times when translating from ${sourceLanguage.refName} to ${targetLanguage.refName}. Keep it to a single plaintext paragraph. Note key lexicosemantic, information structuring, register-relevant and other key distinctions necessary for grammatical, natural text in ${targetLanguage.refName} if the starting place is ${sourceLanguage.refName}. ${htmlInstruction} Preserve original line breaks from <currentTask><source> by returning text with the same number of lines separated by newline characters. Do not include XML in your answer.`;

const response = await callLLM(
const result = await callLLM(
[
{
role: "user",
Expand All @@ -361,7 +361,7 @@ export async function generateChatSystemMessage(
llmConfig
);

return response;
return result.content;
} catch (error) {
debug("[generateChatSystemMessage] Error generating message:", error);
return null;
Expand Down
10 changes: 8 additions & 2 deletions src/providers/codexCellEditorProvider/codexCellEditorProvider.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3473,7 +3473,10 @@ export class CodexCellEditorProvider implements vscode.CustomEditorProvider<Code
currentCellId,
singleCompletion,
EditType.LLM_GENERATION,
shouldUpdateValue
shouldUpdateValue,
false,
false,
completionResult.generationId
);
this.updateSingleCellTranslation(1.0);
debug("LLM completion result (identical variants)", { completion: singleCompletion?.slice?.(0, 80) });
Expand Down Expand Up @@ -3532,7 +3535,10 @@ export class CodexCellEditorProvider implements vscode.CustomEditorProvider<Code
currentCellId,
singleCompletion,
EditType.LLM_GENERATION,
shouldUpdateValue
shouldUpdateValue,
false,
false,
completionResult.generationId
);

// If this was a preview-only update, persist the edit to disk immediately so edit history is saved
Expand Down
5 changes: 4 additions & 1 deletion src/providers/codexCellEditorProvider/codexDocument.ts
Original file line number Diff line number Diff line change
Expand Up @@ -299,7 +299,8 @@ export class CodexCellDocument implements vscode.CustomDocument {
editType: EditType,
shouldUpdateValue = true,
retainValidations = false,
skipAutoValidation = false
skipAutoValidation = false,
generationId?: string
) {
debug("trace 124 updateCellContent", cellId, newContent, editType, shouldUpdateValue);

Expand Down Expand Up @@ -353,6 +354,7 @@ export class CodexCellDocument implements vscode.CustomDocument {
author: this._author,
validatedBy: [],
preview: true,
...(generationId ? { generationId } : {}),
};
cellToUpdate.metadata.edits.push(previewEdit);

Expand Down Expand Up @@ -465,6 +467,7 @@ export class CodexCellDocument implements vscode.CustomDocument {
type: editType,
author: this._author,
validatedBy,
...(generationId ? { generationId } : {}),
});

// Record the edit
Expand Down
5 changes: 4 additions & 1 deletion src/providers/translationSuggestions/llmCompletion.ts
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ export interface LLMCompletionResult {
isAttentionCheck?: boolean;
correctIndex?: number;
decoyCellId?: string;
generationId?: string;
}

export async function llmCompletion(
Expand Down Expand Up @@ -296,7 +297,8 @@ export async function llmCompletion(
}

// A/B testing not triggered (or failed): call LLM once, return two identical variants
const completion = await callLLM(messages, completionConfig, token);
const llmResult = await callLLM(messages, completionConfig, token);
const completion = llmResult.content;
const allowHtml = Boolean(completionConfig.allowHtmlPredictions);

// Extract translation from <final_answer> tags, fallback to full response
Expand All @@ -318,6 +320,7 @@ export async function llmCompletion(
return {
variants,
isABTest: false, // Identical variants – UI should hide A/B controls
generationId: llmResult.generationId,
};
} catch (error) {
// Check if this is a cancellation error and re-throw as-is
Expand Down
14 changes: 7 additions & 7 deletions src/test/suite/codexCellEditorProvider.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3711,7 +3711,7 @@ suite("CodexCellEditorProvider Test Suite", () => {
const llmUtils = await import("../../utils/llmUtils");
const callLLMStub = sinon.stub(llmUtils, "callLLM").callsFake(async (messages: any[]) => {
capturedMessages = messages;
return "Mocked LLM response";
return { content: "Mocked LLM response", generationId: "gen-test-mock" };
});

// Stub status bar item
Expand Down Expand Up @@ -3853,7 +3853,7 @@ suite("CodexCellEditorProvider Test Suite", () => {

// Mock callLLM
const llmUtils = await import("../../utils/llmUtils");
const callLLMStub = sinon.stub(llmUtils, "callLLM").resolves("Mocked response");
const callLLMStub = sinon.stub(llmUtils, "callLLM").resolves({ content: "Mocked response", generationId: "gen-test-mock" });

// Stub status bar and notebook reader
const extModule = await import("../../extension");
Expand Down Expand Up @@ -3956,7 +3956,7 @@ suite("CodexCellEditorProvider Test Suite", () => {

// Mock callLLM
const llmUtils = await import("../../utils/llmUtils");
const callLLMStub = sinon.stub(llmUtils, "callLLM").resolves("Mocked response");
const callLLMStub = sinon.stub(llmUtils, "callLLM").resolves({ content: "Mocked response", generationId: "gen-test-mock" });

// Stub status bar and notebook reader
const extModule = await import("../../extension");
Expand Down Expand Up @@ -4060,7 +4060,7 @@ suite("CodexCellEditorProvider Test Suite", () => {
const llmUtils = await import("../../utils/llmUtils");
const callLLMStub = sinon.stub(llmUtils, "callLLM").callsFake(async (messages: any[]) => {
capturedMessages = messages;
return "Mocked response";
return { content: "Mocked response", generationId: "gen-test-mock" };
});

// Stub MetadataManager.getChatSystemMessage to return custom system message
Expand Down Expand Up @@ -4205,7 +4205,7 @@ suite("CodexCellEditorProvider Test Suite", () => {
const llmUtils = await import("../../utils/llmUtils");
const callLLMStub = sinon.stub(llmUtils, "callLLM").callsFake(async (messages: any[]) => {
capturedMessages = messages;
return "Mocked response";
return { content: "Mocked response", generationId: "gen-test-mock" };
});

// Stub status bar and notebook reader
Expand Down Expand Up @@ -4345,7 +4345,7 @@ suite("CodexCellEditorProvider Test Suite", () => {
// Mock callLLM to capture messages
const llmUtils = await import("../../utils/llmUtils");
const callLLMStub = sinon.stub(llmUtils, "callLLM").callsFake(async (messages: any[]) => {
return "Mocked response";
return { content: "Mocked response", generationId: "gen-test-mock" };
});

// Stub status bar and notebook reader
Expand Down Expand Up @@ -4494,7 +4494,7 @@ suite("CodexCellEditorProvider Test Suite", () => {
let capturedMessages: any[] | null = null;
const callLLMStub = sinon.stub(llmUtils, "callLLM").callsFake(async (messages: any[]) => {
capturedMessages = messages;
return "Mocked response";
return { content: "Mocked response", generationId: "gen-test-mock" };
});

// Stub status bar and notebook reader
Expand Down
2 changes: 1 addition & 1 deletion src/test/suite/validatedOnlyExamples.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ suite("Validated-only examples behavior", () => {

// Stub callLLM to avoid network and return deterministic strings for AB variants
const llmUtils = await import("../../utils/llmUtils");
const callStub = sinon.stub(llmUtils, "callLLM").resolves("PREDICTED");
const callStub = sinon.stub(llmUtils, "callLLM").resolves({ content: "PREDICTED", generationId: "gen-test-mock" });

// Stub status bar item used by llmCompletion
const extModule = await import("../../extension");
Expand Down
4 changes: 2 additions & 2 deletions src/utils/abTestingSetup.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,8 @@ async function generateCompletionFromPairs(
ctx.sourceLanguage
);

const raw = await callLLM(msgs, ctx.completionConfig, ctx.token);
return parseFinalAnswer(raw);
const result = await callLLM(msgs, ctx.completionConfig, ctx.token);
return parseFinalAnswer(result.content);
}

export function initializeABTesting() {
Expand Down
17 changes: 14 additions & 3 deletions src/utils/llmUtils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,11 +14,16 @@ import { MetadataManager } from "./metadataManager";
* @returns A Promise that resolves to the LLM's response as a string.
* @throws Error if the LLM response is unexpected or if there's an error during the API call.
*/
export interface LLMCallResult {
content: string;
generationId?: string;
}

export async function callLLM(
messages: ChatMessage[],
config: CompletionConfig,
cancellationToken?: vscode.CancellationToken
): Promise<string> {
): Promise<LLMCallResult> {
try {
// Check for cancellation before starting
if (cancellationToken?.isCancellationRequested) {
Expand Down Expand Up @@ -108,7 +113,10 @@ export async function callLLM(
completion.choices.length > 0 &&
completion.choices[0].message
) {
return completion.choices[0].message.content?.trim() ?? "";
return {
content: completion.choices[0].message.content?.trim() ?? "",
generationId: completion.id || undefined,
};
} else {
throw new Error(
"Unexpected response format from the LLM; callLLM() failed - case 1"
Expand Down Expand Up @@ -137,7 +145,10 @@ export async function callLLM(
completion.choices.length > 0 &&
completion.choices[0].message
) {
return completion.choices[0].message.content?.trim() ?? "";
return {
content: completion.choices[0].message.content?.trim() ?? "",
generationId: completion.id || undefined,
};
} else {
throw new Error(
"Unexpected response format from the LLM; callLLM() failed - case 1"
Expand Down
2 changes: 2 additions & 0 deletions types/index.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -651,6 +651,8 @@ type EditHistoryBase = {
timestamp: number;
type: import("./enums").EditType;
validatedBy?: ValidationEntry[];
/** OpenRouter generation ID for LLM-generated edits */
generationId?: string;
};

export type EditHistory<TEditMap extends readonly string[] = readonly string[]> = EditHistoryBase & {
Expand Down