From 343ba52ae3902bf4a5f480d6e3989356e7aa482d Mon Sep 17 00:00:00 2001 From: Jeremy Tuloup Date: Wed, 15 Oct 2025 12:54:55 +0200 Subject: [PATCH 1/6] Support discovering commands with a query --- src/tools/commands.ts | 85 +++++++++++++++++++++++-------------------- 1 file changed, 45 insertions(+), 40 deletions(-) diff --git a/src/tools/commands.ts b/src/tools/commands.ts index 310b23b7..85535676 100644 --- a/src/tools/commands.ts +++ b/src/tools/commands.ts @@ -20,51 +20,56 @@ export function createDiscoverCommandsTool(commands: CommandRegistry): ITool { .nullable() .describe('Optional search query to filter commands') }), - execute: async () => { - try { - const commandList: Array<{ - id: string; - label?: string; - caption?: string; - description?: string; - args?: any; - }> = []; + execute: async (input: { query?: string | null }) => { + const { query } = input; + const commandList: Array<{ + id: string; + label?: string; + caption?: string; + description?: string; + args?: any; + }> = []; - // Get all command IDs - const commandIds = commands.listCommands(); + // Get all command IDs + const commandIds = commands.listCommands(); - for (const id of commandIds) { - try { - // Get command metadata using various CommandRegistry methods - const description = await commands.describedBy(id); - const label = commands.label(id); - const caption = commands.caption(id); - const usage = commands.usage(id); + for (const id of commandIds) { + // Get command metadata using various CommandRegistry methods + const description = await commands.describedBy(id); + const label = commands.label(id); + const caption = commands.caption(id); + const usage = commands.usage(id); + + const command = { + id, + label: label || undefined, + caption: caption || undefined, + description: usage || undefined, + args: description?.args || undefined + }; - commandList.push({ - id, - label: label || undefined, - caption: caption || undefined, - description: usage || undefined, - args: description?.args || undefined - }); - } catch (error) { - // Some commands might not have descriptions, skip them - commandList.push({ id }); + // Filter by query if provided + if (query) { + const searchTerm = query.toLowerCase(); + const matchesQuery = + id.toLowerCase().includes(searchTerm) || + label?.toLowerCase().includes(searchTerm) || + caption?.toLowerCase().includes(searchTerm) || + usage?.toLowerCase().includes(searchTerm); + + if (matchesQuery) { + commandList.push(command); } + } else { + commandList.push(command); } - - return { - success: true, - commandCount: commandList.length, - commands: commandList - }; - } catch (error) { - return { - success: false, - error: `Failed to discover commands: ${error instanceof Error ? error.message : String(error)}` - }; } + + return { + success: true, + commandCount: commandList.length, + commands: commandList + }; } }); } @@ -87,7 +92,7 @@ export function createExecuteCommandTool( .optional() .describe('Optional arguments to pass to the command') }), - needsApproval: async (_context, { commandId }) => { + needsApproval: async (context, { commandId }) => { // Use configurable list of commands requiring approval const commandsRequiringApproval = settingsModel.config.commandsRequiringApproval; From 21b3ce68b9c34348cda3dac3efc29f60f387346d Mon Sep 17 00:00:00 2001 From: Jeremy Tuloup Date: Wed, 15 Oct 2025 12:56:54 +0200 Subject: [PATCH 2/6] improve prompt --- src/agent.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/agent.ts b/src/agent.ts index f1b54699..ab4db6ce 100644 --- a/src/agent.ts +++ b/src/agent.ts @@ -921,7 +921,9 @@ Guidelines: - Use natural, conversational tone throughout COMMAND DISCOVERY: -- When you want to execute JupyterLab commands, ALWAYS use the 'discover_commands' tool first to find available commands and their metadata. +- When you want to execute JupyterLab commands, ALWAYS use the 'discover_commands' tool first to find available commands and their metadata, with the optional query parameter. +- The query should typically be a single word, e.g., 'terminal', 'notebook', 'cell', 'file', 'edit', 'view', 'run', etc, to find relevant commands. +- If searching with a query does not yield the desired command, try again with a different query or use an empty query to list all commands. - This ensures you have complete information about command IDs, descriptions, and required arguments before attempting to execute them. Only after discovering the available commands should you use the 'execute_command' tool with the correct command ID and arguments. TOOL SELECTION GUIDELINES: From 3474f3979e4f878efbea4b8e522973416854f8e7 Mon Sep 17 00:00:00 2001 From: Jeremy Tuloup Date: Wed, 15 Oct 2025 14:28:04 +0200 Subject: [PATCH 3/6] add UI tests --- ui-tests/tests/commands-tool.spec.ts | 121 +++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 ui-tests/tests/commands-tool.spec.ts diff --git a/ui-tests/tests/commands-tool.spec.ts b/ui-tests/tests/commands-tool.spec.ts new file mode 100644 index 00000000..21e3b9fd --- /dev/null +++ b/ui-tests/tests/commands-tool.spec.ts @@ -0,0 +1,121 @@ +/* + * Copyright (c) Jupyter Development Team. + * Distributed under the terms of the Modified BSD License. + */ + +import { expect, galata, test } from '@jupyterlab/galata'; +import { DEFAULT_SETTINGS_MODEL_SETTINGS, openChatPanel } from './test-utils'; + +test.use({ + mockSettings: { + ...galata.DEFAULT_SETTINGS, + '@jupyterlab/apputils-extension:notification': { + checkForUpdates: false, + fetchNews: 'false', + doNotDisturbMode: true + }, + '@jupyterlite/ai:settings-model': { + ...DEFAULT_SETTINGS_MODEL_SETTINGS['@jupyterlite/ai:settings-model'], + toolsEnabled: true, + // To nudge the model to call the tool with specific parameters + systemPrompt: + 'When asked to discover commands, call the discover_commands tool with the exact query parameter provided in the user message. Always use the query parameter exactly as specified.' + } + } +}); + +test.describe('#commandsTool', () => { + test.beforeEach(async () => { + test.setTimeout(120 * 1000); + }); + + test('should filter commands using query parameter', async ({ page }) => { + const panel = await openChatPanel(page); + const input = panel + .locator('.jp-chat-input-container') + .getByRole('combobox'); + const sendButton = panel.locator( + '.jp-chat-input-container .jp-chat-send-button' + ); + + // Very specific prompt to ensure the query parameter is used + const PROMPT = + 'Use the discover_commands tool with query parameter set to "notebook" to find notebook-related commands'; + + await input.pressSequentially(PROMPT); + await sendButton.click(); + + // Wait for AI response + await expect( + panel.locator('.jp-chat-message-header:has-text("Jupyternaut")') + ).toHaveCount(1, { timeout: 60000 }); + + // Wait for tool call to appear + const toolCall = panel.locator('.jp-ai-tool-call'); + await expect(toolCall).toHaveCount(1, { timeout: 30000 }); + + // Verify the tool was called + await expect(toolCall).toContainText('discover_commands'); + + // Click to expand the tool call + await toolCall.click(); + + // Get the tool call result to check the command count + const toolResultText = await toolCall.textContent(); + + // Parse the commandCount from the JSON response + const countMatch = toolResultText?.match(/"commandCount":\s*(\d+)/); + expect(countMatch).toBeTruthy(); + const count = parseInt(countMatch![1], 10); + + // The filtered results should have significantly fewer than 300 commands + // (JupyterLab typically has 300+ total commands, but only a subset contain "notebook") + expect(count).toBeLessThan(300); + expect(count).toBeGreaterThan(0); + }); + + test('should return all commands without query parameter', async ({ + page + }) => { + const panel = await openChatPanel(page); + const input = panel + .locator('.jp-chat-input-container') + .getByRole('combobox'); + const sendButton = panel.locator( + '.jp-chat-input-container .jp-chat-send-button' + ); + + // Prompt without specifying a query parameter + const PROMPT = + 'Use the discover_commands tool without any query parameter to list all available commands'; + + await input.pressSequentially(PROMPT); + await sendButton.click(); + + // Wait for AI response + await expect( + panel.locator('.jp-chat-message-header:has-text("Jupyternaut")') + ).toHaveCount(1, { timeout: 60000 }); + + // Wait for tool call to appear + const toolCall = panel.locator('.jp-ai-tool-call'); + await expect(toolCall).toHaveCount(1, { timeout: 30000 }); + + // Verify the tool was called + await expect(toolCall).toContainText('discover_commands'); + + // Click to expand the tool call + await toolCall.click(); + + // Get the tool call result to check the command count + const toolResultText = await toolCall.textContent(); + + // Parse the commandCount from the JSON response + const countMatch = toolResultText?.match(/"commandCount":\s*(\d+)/); + expect(countMatch).toBeTruthy(); + const count = parseInt(countMatch![1], 10); + + // Should have many commands (typically 400+) + expect(count).toBeGreaterThan(400); + }); +}); From 96ad4cf6b663e61967f2b17e786611a522be4be3 Mon Sep 17 00:00:00 2001 From: Jeremy Tuloup Date: Wed, 15 Oct 2025 14:51:24 +0200 Subject: [PATCH 4/6] fix timeouts --- ui-tests/tests/commands-tool.spec.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ui-tests/tests/commands-tool.spec.ts b/ui-tests/tests/commands-tool.spec.ts index 21e3b9fd..89f59595 100644 --- a/ui-tests/tests/commands-tool.spec.ts +++ b/ui-tests/tests/commands-tool.spec.ts @@ -25,11 +25,9 @@ test.use({ }); test.describe('#commandsTool', () => { - test.beforeEach(async () => { + test('should filter commands using query parameter', async ({ page }) => { test.setTimeout(120 * 1000); - }); - test('should filter commands using query parameter', async ({ page }) => { const panel = await openChatPanel(page); const input = panel .locator('.jp-chat-input-container') @@ -77,6 +75,8 @@ test.describe('#commandsTool', () => { test('should return all commands without query parameter', async ({ page }) => { + test.setTimeout(120 * 1000); + const panel = await openChatPanel(page); const input = panel .locator('.jp-chat-input-container') From 960be1ad84de98ac0fbf81b91abfaef472375e93 Mon Sep 17 00:00:00 2001 From: Jeremy Tuloup Date: Wed, 15 Oct 2025 15:17:14 +0200 Subject: [PATCH 5/6] fix timeouts --- ui-tests/tests/commands-tool.spec.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ui-tests/tests/commands-tool.spec.ts b/ui-tests/tests/commands-tool.spec.ts index 89f59595..c99561f4 100644 --- a/ui-tests/tests/commands-tool.spec.ts +++ b/ui-tests/tests/commands-tool.spec.ts @@ -46,11 +46,11 @@ test.describe('#commandsTool', () => { // Wait for AI response await expect( panel.locator('.jp-chat-message-header:has-text("Jupyternaut")') - ).toHaveCount(1, { timeout: 60000 }); + ).toHaveCount(1); // Wait for tool call to appear const toolCall = panel.locator('.jp-ai-tool-call'); - await expect(toolCall).toHaveCount(1, { timeout: 30000 }); + await expect(toolCall).toHaveCount(1); // Verify the tool was called await expect(toolCall).toContainText('discover_commands'); @@ -95,11 +95,11 @@ test.describe('#commandsTool', () => { // Wait for AI response await expect( panel.locator('.jp-chat-message-header:has-text("Jupyternaut")') - ).toHaveCount(1, { timeout: 60000 }); + ).toHaveCount(1); // Wait for tool call to appear const toolCall = panel.locator('.jp-ai-tool-call'); - await expect(toolCall).toHaveCount(1, { timeout: 30000 }); + await expect(toolCall).toHaveCount(1); // Verify the tool was called await expect(toolCall).toContainText('discover_commands'); From f3bb6602f89fe91043c048cef6d58515b39faa80 Mon Sep 17 00:00:00 2001 From: Jeremy Tuloup Date: Wed, 15 Oct 2025 15:38:54 +0200 Subject: [PATCH 6/6] timeouts --- ui-tests/tests/commands-tool.spec.ts | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/ui-tests/tests/commands-tool.spec.ts b/ui-tests/tests/commands-tool.spec.ts index c99561f4..a1b62b8e 100644 --- a/ui-tests/tests/commands-tool.spec.ts +++ b/ui-tests/tests/commands-tool.spec.ts @@ -6,6 +6,8 @@ import { expect, galata, test } from '@jupyterlab/galata'; import { DEFAULT_SETTINGS_MODEL_SETTINGS, openChatPanel } from './test-utils'; +const EXPECT_TIMEOUT = 120000; + test.use({ mockSettings: { ...galata.DEFAULT_SETTINGS, @@ -46,14 +48,16 @@ test.describe('#commandsTool', () => { // Wait for AI response await expect( panel.locator('.jp-chat-message-header:has-text("Jupyternaut")') - ).toHaveCount(1); + ).toHaveCount(1, { timeout: EXPECT_TIMEOUT }); // Wait for tool call to appear const toolCall = panel.locator('.jp-ai-tool-call'); - await expect(toolCall).toHaveCount(1); + await expect(toolCall).toHaveCount(1, { timeout: EXPECT_TIMEOUT }); // Verify the tool was called - await expect(toolCall).toContainText('discover_commands'); + await expect(toolCall).toContainText('discover_commands', { + timeout: EXPECT_TIMEOUT + }); // Click to expand the tool call await toolCall.click(); @@ -95,14 +99,16 @@ test.describe('#commandsTool', () => { // Wait for AI response await expect( panel.locator('.jp-chat-message-header:has-text("Jupyternaut")') - ).toHaveCount(1); + ).toHaveCount(1, { timeout: EXPECT_TIMEOUT }); // Wait for tool call to appear const toolCall = panel.locator('.jp-ai-tool-call'); - await expect(toolCall).toHaveCount(1); + await expect(toolCall).toHaveCount(1, { timeout: EXPECT_TIMEOUT }); // Verify the tool was called - await expect(toolCall).toContainText('discover_commands'); + await expect(toolCall).toContainText('discover_commands', { + timeout: EXPECT_TIMEOUT + }); // Click to expand the tool call await toolCall.click();