From d082ba8506079eb67dcdebf775e29ca5db06d6d5 Mon Sep 17 00:00:00 2001 From: Julia Silge Date: Thu, 20 Nov 2025 21:22:55 -0700 Subject: [PATCH 1/5] First draft of semantic token support for Quarto files --- apps/lsp/src/middleware.ts | 17 +- apps/vscode/src/lsp/client.ts | 4 +- apps/vscode/src/providers/semantic-tokens.ts | 233 ++++++++++++++++++ apps/vscode/src/vdoc/vdoc.ts | 42 +++- packages/quarto-core/src/index.ts | 1 + .../quarto-core/src/semantic-tokens-legend.ts | 39 +++ 6 files changed, 327 insertions(+), 9 deletions(-) create mode 100644 apps/vscode/src/providers/semantic-tokens.ts create mode 100644 packages/quarto-core/src/semantic-tokens-legend.ts diff --git a/apps/lsp/src/middleware.ts b/apps/lsp/src/middleware.ts index f61088a8..946ecfe3 100644 --- a/apps/lsp/src/middleware.ts +++ b/apps/lsp/src/middleware.ts @@ -1,7 +1,7 @@ /* * middleware.ts * - * Copyright (C) 2023 by Posit Software, PBC + * Copyright (C) 2023-2025 by Posit Software, PBC * Copyright (c) Microsoft Corporation. All rights reserved. * * Unless you have received this program directly from Posit Software pursuant @@ -14,7 +14,8 @@ * */ -import { Connection, ServerCapabilities } from "vscode-languageserver" +import { Connection, ServerCapabilities } from "vscode-languageserver"; +import { QUARTO_SEMANTIC_TOKEN_LEGEND } from "quarto-core"; // capabilities provided just so we can intercept them w/ middleware on the client @@ -28,8 +29,12 @@ export function middlewareCapabilities(): ServerCapabilities { }, documentFormattingProvider: true, documentRangeFormattingProvider: true, - definitionProvider: true - } + definitionProvider: true, + semanticTokensProvider: { + legend: QUARTO_SEMANTIC_TOKEN_LEGEND, + full: true + } + }; }; // methods provided just so we can intercept them w/ middleware on the client @@ -51,4 +56,8 @@ export function middlewareRegister(connection: Connection) { return null; }); + connection.languages.semanticTokens.on(async () => { + return { data: [] }; + }); + } diff --git a/apps/vscode/src/lsp/client.ts b/apps/vscode/src/lsp/client.ts index 07f92e9f..c38e124a 100644 --- a/apps/vscode/src/lsp/client.ts +++ b/apps/vscode/src/lsp/client.ts @@ -1,7 +1,7 @@ /* * client.ts * - * Copyright (C) 2022 by Posit Software, PBC + * Copyright (C) 2022-2025 by Posit Software, PBC * * Unless you have received this program directly from Posit Software pursuant * to the terms of a commercial license agreement with Posit Software, then @@ -64,6 +64,7 @@ import { embeddedDocumentFormattingProvider, embeddedDocumentRangeFormattingProvider, } from "../providers/format"; +import { embeddedSemanticTokensProvider } from "../providers/semantic-tokens"; import { getHover, getSignatureHelpHover } from "../core/hover"; import { imageHover } from "../providers/hover-image"; import { LspInitializationOptions, QuartoContext } from "quarto-core"; @@ -109,6 +110,7 @@ export async function activateLsp( provideDocumentRangeFormattingEdits: embeddedDocumentRangeFormattingProvider( engine ), + provideDocumentSemanticTokens: embeddedSemanticTokensProvider(engine), }; if (config.get("cells.hoverHelp.enabled", true)) { middleware.provideHover = embeddedHoverProvider(engine); diff --git a/apps/vscode/src/providers/semantic-tokens.ts b/apps/vscode/src/providers/semantic-tokens.ts new file mode 100644 index 00000000..6834b125 --- /dev/null +++ b/apps/vscode/src/providers/semantic-tokens.ts @@ -0,0 +1,233 @@ +/* + * semantic-tokens.ts + * + * Copyright (C) 2025 by Posit Software, PBC + * + * Unless you have received this program directly from Posit Software pursuant + * to the terms of a commercial license agreement with Posit Software, then + * this program is licensed to you under the terms of version 3 of the + * GNU Affero General Public License. This program is distributed WITHOUT + * ANY EXPRESS OR IMPLIED WARRANTY, INCLUDING THOSE OF NON-INFRINGEMENT, + * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Please refer to the + * AGPL (http://www.gnu.org/licenses/agpl-3.0.txt) for more details. + * + */ + +import { + CancellationToken, + commands, + Position, + SemanticTokens, + SemanticTokensBuilder, + TextDocument, + Uri, + window, +} from "vscode"; +import { DocumentSemanticsTokensSignature } from "vscode-languageclient"; +import { MarkdownEngine } from "../markdown/engine"; +import { isQuartoDoc } from "../core/doc"; +import { unadjustedSemanticTokens, virtualDoc, withVirtualDocUri } from "../vdoc/vdoc"; +import { QUARTO_SEMANTIC_TOKEN_LEGEND } from "quarto-core"; + +/** + * Decode semantic tokens from delta-encoded format to absolute positions + * + * Semantic tokens are encoded as [deltaLine, deltaStartChar, length, tokenType, tokenModifiers, ...] + * This function converts them to absolute line/character positions for easier manipulation. + */ +export function decodeSemanticTokens(tokens: SemanticTokens): Array<{ + line: number; + startChar: number; + length: number; + tokenType: number; + tokenModifiers: number; +}> { + const decoded: Array<{ + line: number; + startChar: number; + length: number; + tokenType: number; + tokenModifiers: number; + }> = []; + + let currentLine = 0; + let currentChar = 0; + + for (let i = 0; i < tokens.data.length; i += 5) { + const deltaLine = tokens.data[i]; + const deltaStartChar = tokens.data[i + 1]; + const length = tokens.data[i + 2]; + const tokenType = tokens.data[i + 3]; + const tokenModifiers = tokens.data[i + 4]; + + // Update absolute position + currentLine += deltaLine; + if (deltaLine > 0) { + currentChar = deltaStartChar; + } else { + currentChar += deltaStartChar; + } + + decoded.push({ + line: currentLine, + startChar: currentChar, + length, + tokenType, + tokenModifiers + }); + } + + return decoded; +} + +/** + * Encode semantic tokens from absolute positions to delta-encoded format + * + * Uses VS Code's built-in SemanticTokensBuilder for proper delta encoding. + */ +export function encodeSemanticTokens( + tokens: Array<{ + line: number; + startChar: number; + length: number; + tokenType: number; + tokenModifiers: number; + }>, + resultId?: string +): SemanticTokens { + const builder = new SemanticTokensBuilder(); + + for (const token of tokens) { + builder.push( + token.line, + token.startChar, + token.length, + token.tokenType, + token.tokenModifiers + ); + } + + return builder.build(resultId); +} + +/** + * Build a map from source type/modifier names to target indices + */ +function buildLegendMap( + sourceNames: string[], + targetNames: string[] +): Map { + const map = new Map(); + + for (let i = 0; i < sourceNames.length; i++) { + const targetIndex = targetNames.indexOf(sourceNames[i]); + if (targetIndex >= 0) { + map.set(i, targetIndex); + } + } + + return map; +} + +/** + * Remap a modifier bitfield from source indices to target indices + */ +function remapModifierBitfield( + sourceModifiers: number, + modifierMap: Map +): number { + let targetModifiers = 0; + + // Check each bit in the source bitfield + for (const [sourceBit, targetBit] of modifierMap) { + if (sourceModifiers & (1 << sourceBit)) { + targetModifiers |= (1 << targetBit); + } + } + + return targetModifiers; +} + +/** + * Remap token type/modifier indices from source legend to target legend + * Only maps types that exist in both legends (standard types only) + */ +function remapTokenIndices( + tokens: SemanticTokens, + sourceLegend: { tokenTypes: string[]; tokenModifiers: string[]; }, + targetLegend: { tokenTypes: string[]; tokenModifiers: string[]; } +): SemanticTokens { + // Build mappings once + const typeMap = buildLegendMap(sourceLegend.tokenTypes, targetLegend.tokenTypes); + const modifierMap = buildLegendMap(sourceLegend.tokenModifiers, targetLegend.tokenModifiers); + + // Decode, filter, and remap tokens + const decoded = decodeSemanticTokens(tokens); + const remapped = decoded + .filter(token => typeMap.has(token.tokenType)) + .map(token => ({ + ...token, + tokenType: typeMap.get(token.tokenType)!, + tokenModifiers: remapModifierBitfield(token.tokenModifiers, modifierMap) + })); + + return encodeSemanticTokens(remapped, tokens.resultId); +} + +export function embeddedSemanticTokensProvider(engine: MarkdownEngine) { + return async ( + document: TextDocument, + token: CancellationToken, + next: DocumentSemanticsTokensSignature + ): Promise => { + // Only handle Quarto documents + if (!isQuartoDoc(document, true)) { + return await next(document, token); + } + + const editor = window.activeTextEditor; + const activeDocument = editor?.document; + if (!editor || activeDocument?.uri.toString() !== document.uri.toString()) { + // Not the active document, delegate to default + return await next(document, token); + } + + const line = editor.selection.active.line; + const position = new Position(line, 0); + const vdoc = await virtualDoc(document, position, engine); + + if (!vdoc) { + return await next(document, token); + } + + return await withVirtualDocUri(vdoc, document.uri, "semanticTokens", async (uri: Uri) => { + try { + // Get the legend from the embedded language provider + const legend = await commands.executeCommand( + "vscode.provideDocumentSemanticTokensLegend", + uri + ); + + const tokens = await commands.executeCommand( + "vscode.provideDocumentSemanticTokens", + uri + ); + + if (!tokens || tokens.data.length === 0) { + return tokens; + } + + // Remap token indices from embedded provider's legend to our universal legend + let remappedTokens = tokens; + if (legend) { + remappedTokens = remapTokenIndices(tokens, legend, QUARTO_SEMANTIC_TOKEN_LEGEND); + } + + // Adjust token positions from virtual doc to real doc coordinates + return unadjustedSemanticTokens(vdoc.language, remappedTokens); + } catch (error) { + return undefined; + } + }); + }; +} diff --git a/apps/vscode/src/vdoc/vdoc.ts b/apps/vscode/src/vdoc/vdoc.ts index f1441fe8..03a69786 100644 --- a/apps/vscode/src/vdoc/vdoc.ts +++ b/apps/vscode/src/vdoc/vdoc.ts @@ -1,7 +1,7 @@ /* * vdoc.ts * - * Copyright (C) 2022 by Posit Software, PBC + * Copyright (C) 2022-2025 by Posit Software, PBC * * Unless you have received this program directly from Posit Software pursuant * to the terms of a commercial license agreement with Posit Software, then @@ -13,7 +13,7 @@ * */ -import { Position, TextDocument, Uri, Range } from "vscode"; +import { Position, TextDocument, Uri, Range, SemanticTokens } from "vscode"; import { Token, isExecutableLanguageBlock, languageBlockAtPosition, languageNameFromBlock } from "quarto-core"; import { isQuartoDoc } from "../core/doc"; @@ -21,6 +21,7 @@ import { MarkdownEngine } from "../markdown/engine"; import { embeddedLanguage, EmbeddedLanguage } from "./languages"; import { virtualDocUriFromEmbeddedContent } from "./vdoc-content"; import { virtualDocUriFromTempFile } from "./vdoc-tempfile"; +import { decodeSemanticTokens, encodeSemanticTokens } from "../providers/semantic-tokens"; export interface VirtualDoc { language: EmbeddedLanguage; @@ -118,9 +119,10 @@ export type VirtualDocAction = "definition" | "format" | "statementRange" | - "helpTopic"; + "helpTopic" | + "semanticTokens"; -export type VirtualDocUri = { uri: Uri, cleanup?: () => Promise }; +export type VirtualDocUri = { uri: Uri, cleanup?: () => Promise; }; /** * Execute a callback on a virtual document's temporary URI @@ -232,3 +234,35 @@ export function unadjustedRange(language: EmbeddedLanguage, range: Range) { unadjustedPosition(language, range.end) ); } + +/** + * Adjust semantic tokens from virtual document coordinates to real document coordinates + * + * This function decodes the tokens, adjusts each token's position using unadjustedRange, + * and re-encodes them back to delta format. + */ +export function unadjustedSemanticTokens( + language: EmbeddedLanguage, + tokens: SemanticTokens +): SemanticTokens { + // Decode tokens to absolute positions + const decoded = decodeSemanticTokens(tokens); + + // Adjust each token's position + const adjusted = decoded.map(t => { + const range = unadjustedRange(language, new Range( + new Position(t.line, t.startChar), + new Position(t.line, t.startChar + t.length) + )); + return { + line: range.start.line, + startChar: range.start.character, + length: range.end.character - range.start.character, + tokenType: t.tokenType, + tokenModifiers: t.tokenModifiers + }; + }); + + // Re-encode to delta format + return encodeSemanticTokens(adjusted, tokens.resultId); +} diff --git a/packages/quarto-core/src/index.ts b/packages/quarto-core/src/index.ts index 64f2418c..8aeb4a3f 100644 --- a/packages/quarto-core/src/index.ts +++ b/packages/quarto-core/src/index.ts @@ -24,3 +24,4 @@ export * from './position'; export * from './range'; export * from './document'; export * from './lsp'; +export * from './semantic-tokens-legend'; diff --git a/packages/quarto-core/src/semantic-tokens-legend.ts b/packages/quarto-core/src/semantic-tokens-legend.ts new file mode 100644 index 00000000..9a450863 --- /dev/null +++ b/packages/quarto-core/src/semantic-tokens-legend.ts @@ -0,0 +1,39 @@ +/* + * semantic-tokens-legend.ts + * + * Copyright (C) 2025 by Posit Software, PBC + * + * Unless you have received this program directly from Posit Software pursuant + * to the terms of a commercial license agreement with Posit Software, then + * this program is licensed to you under the terms of version 3 of the + * GNU Affero General Public License. This program is distributed WITHOUT + * ANY EXPRESS OR IMPLIED WARRANTY, INCLUDING THOSE OF NON-INFRINGEMENT, + * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. Please refer to the + * AGPL (http://www.gnu.org/licenses/agpl-3.0.txt) for more details. + * + */ + +/** + * Semantic token legend for Quarto documents + * + * Based on standard VS Code semantic token types and modifiers: + * https://code.visualstudio.com/api/language-extensions/semantic-highlight-guide#standard-token-types-and-modifiers + * + * This legend is used by both the LSP server (to advertise capabilities) + * and the VS Code extension (to remap tokens from embedded language providers) + */ +export const QUARTO_SEMANTIC_TOKEN_LEGEND = { + tokenTypes: [ + 'namespace', 'class', 'enum', 'interface', 'struct', + 'typeParameter', 'type', 'parameter', 'variable', 'property', + 'enumMember', 'decorator', 'event', 'function', 'method', + 'macro', 'label', 'comment', 'string', 'keyword', + 'number', 'regexp', 'operator', + // Commonly used by language servers, widely supported by themes + 'module' + ], + tokenModifiers: [ + 'declaration', 'definition', 'readonly', 'static', 'deprecated', + 'abstract', 'async', 'modification', 'documentation', 'defaultLibrary' + ] +}; From 347d8f990b4845a455ed47b4360c2a83f8045177 Mon Sep 17 00:00:00 2001 From: Julia Silge Date: Thu, 20 Nov 2025 21:31:10 -0700 Subject: [PATCH 2/5] Actually, let's put this in the `quarto-utils` package --- apps/lsp/src/middleware.ts | 2 +- apps/quarto-utils/src/index.ts | 1 + .../quarto-utils}/src/semantic-tokens-legend.ts | 0 apps/vscode/src/providers/semantic-tokens.ts | 2 +- packages/quarto-core/src/index.ts | 1 - 5 files changed, 3 insertions(+), 3 deletions(-) rename {packages/quarto-core => apps/quarto-utils}/src/semantic-tokens-legend.ts (100%) diff --git a/apps/lsp/src/middleware.ts b/apps/lsp/src/middleware.ts index 946ecfe3..aa48889a 100644 --- a/apps/lsp/src/middleware.ts +++ b/apps/lsp/src/middleware.ts @@ -15,7 +15,7 @@ */ import { Connection, ServerCapabilities } from "vscode-languageserver"; -import { QUARTO_SEMANTIC_TOKEN_LEGEND } from "quarto-core"; +import { QUARTO_SEMANTIC_TOKEN_LEGEND } from "quarto-utils"; // capabilities provided just so we can intercept them w/ middleware on the client diff --git a/apps/quarto-utils/src/index.ts b/apps/quarto-utils/src/index.ts index ecd26f7f..25c73708 100644 --- a/apps/quarto-utils/src/index.ts +++ b/apps/quarto-utils/src/index.ts @@ -14,3 +14,4 @@ */ export * from './r-utils'; +export * from './semantic-tokens-legend'; diff --git a/packages/quarto-core/src/semantic-tokens-legend.ts b/apps/quarto-utils/src/semantic-tokens-legend.ts similarity index 100% rename from packages/quarto-core/src/semantic-tokens-legend.ts rename to apps/quarto-utils/src/semantic-tokens-legend.ts diff --git a/apps/vscode/src/providers/semantic-tokens.ts b/apps/vscode/src/providers/semantic-tokens.ts index 6834b125..aa149b84 100644 --- a/apps/vscode/src/providers/semantic-tokens.ts +++ b/apps/vscode/src/providers/semantic-tokens.ts @@ -27,7 +27,7 @@ import { DocumentSemanticsTokensSignature } from "vscode-languageclient"; import { MarkdownEngine } from "../markdown/engine"; import { isQuartoDoc } from "../core/doc"; import { unadjustedSemanticTokens, virtualDoc, withVirtualDocUri } from "../vdoc/vdoc"; -import { QUARTO_SEMANTIC_TOKEN_LEGEND } from "quarto-core"; +import { QUARTO_SEMANTIC_TOKEN_LEGEND } from "quarto-utils"; /** * Decode semantic tokens from delta-encoded format to absolute positions diff --git a/packages/quarto-core/src/index.ts b/packages/quarto-core/src/index.ts index 8aeb4a3f..64f2418c 100644 --- a/packages/quarto-core/src/index.ts +++ b/packages/quarto-core/src/index.ts @@ -24,4 +24,3 @@ export * from './position'; export * from './range'; export * from './document'; export * from './lsp'; -export * from './semantic-tokens-legend'; From 3fc179bf2ecdf1225579939a223114f1dcb5d30c Mon Sep 17 00:00:00 2001 From: Julia Silge Date: Fri, 21 Nov 2025 17:14:45 -0700 Subject: [PATCH 3/5] Better to use `virtualDocForLanguage()` for this provider --- apps/vscode/src/providers/semantic-tokens.ts | 25 +++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/apps/vscode/src/providers/semantic-tokens.ts b/apps/vscode/src/providers/semantic-tokens.ts index aa149b84..7c6ff92f 100644 --- a/apps/vscode/src/providers/semantic-tokens.ts +++ b/apps/vscode/src/providers/semantic-tokens.ts @@ -26,7 +26,14 @@ import { import { DocumentSemanticsTokensSignature } from "vscode-languageclient"; import { MarkdownEngine } from "../markdown/engine"; import { isQuartoDoc } from "../core/doc"; -import { unadjustedSemanticTokens, virtualDoc, withVirtualDocUri } from "../vdoc/vdoc"; +import { + unadjustedSemanticTokens, + virtualDocForLanguage, + withVirtualDocUri, + languageAtPosition, + mainLanguage +} from "../vdoc/vdoc"; +import { EmbeddedLanguage } from "../vdoc/languages"; import { QUARTO_SEMANTIC_TOKEN_LEGEND } from "quarto-utils"; /** @@ -185,6 +192,7 @@ export function embeddedSemanticTokensProvider(engine: MarkdownEngine) { return await next(document, token); } + // Ensure we are dealing with the active document const editor = window.activeTextEditor; const activeDocument = editor?.document; if (!editor || activeDocument?.uri.toString() !== document.uri.toString()) { @@ -192,14 +200,25 @@ export function embeddedSemanticTokensProvider(engine: MarkdownEngine) { return await next(document, token); } + // Parse the document to get all tokens + const tokens = engine.parse(document); + + // Try to find language at cursor position, otherwise use main language const line = editor.selection.active.line; const position = new Position(line, 0); - const vdoc = await virtualDoc(document, position, engine); + let language = languageAtPosition(tokens, position); + if (!language) { + language = mainLanguage(tokens); + } - if (!vdoc) { + if (!language) { + // No language found, delegate to default return await next(document, token); } + // Create virtual doc for all blocks of this language + const vdoc = virtualDocForLanguage(document, tokens, language); + return await withVirtualDocUri(vdoc, document.uri, "semanticTokens", async (uri: Uri) => { try { // Get the legend from the embedded language provider From 739e525eb115e02ae95e8d84643f3be760091dce Mon Sep 17 00:00:00 2001 From: Julia Silge Date: Fri, 21 Nov 2025 17:29:24 -0700 Subject: [PATCH 4/5] Add some tests --- apps/vscode/src/providers/semantic-tokens.ts | 2 +- apps/vscode/src/test/semanticTokens.test.ts | 167 +++++++++++++++++++ 2 files changed, 168 insertions(+), 1 deletion(-) create mode 100644 apps/vscode/src/test/semanticTokens.test.ts diff --git a/apps/vscode/src/providers/semantic-tokens.ts b/apps/vscode/src/providers/semantic-tokens.ts index 7c6ff92f..fee7f83c 100644 --- a/apps/vscode/src/providers/semantic-tokens.ts +++ b/apps/vscode/src/providers/semantic-tokens.ts @@ -159,7 +159,7 @@ function remapModifierBitfield( * Remap token type/modifier indices from source legend to target legend * Only maps types that exist in both legends (standard types only) */ -function remapTokenIndices( +export function remapTokenIndices( tokens: SemanticTokens, sourceLegend: { tokenTypes: string[]; tokenModifiers: string[]; }, targetLegend: { tokenTypes: string[]; tokenModifiers: string[]; } diff --git a/apps/vscode/src/test/semanticTokens.test.ts b/apps/vscode/src/test/semanticTokens.test.ts new file mode 100644 index 00000000..f2e0e8cb --- /dev/null +++ b/apps/vscode/src/test/semanticTokens.test.ts @@ -0,0 +1,167 @@ +import * as vscode from "vscode"; +import * as assert from "assert"; +import { decodeSemanticTokens, encodeSemanticTokens, remapTokenIndices } from "../providers/semantic-tokens"; + +suite("Semantic Tokens", function () { + + test("Encode and decode semantic tokens roundtrip", function () { + // Create a set of semantic tokens in absolute format + const tokens = [ + { line: 0, startChar: 0, length: 3, tokenType: 1, tokenModifiers: 0 }, + { line: 0, startChar: 4, length: 4, tokenType: 2, tokenModifiers: 1 }, + { line: 2, startChar: 2, length: 5, tokenType: 3, tokenModifiers: 2 }, + ]; + + // Encode to delta format + const encoded = encodeSemanticTokens(tokens); + + // Verify the encoded data has the expected structure + assert.ok(encoded.data, "Encoded tokens should have data property"); + assert.strictEqual(encoded.data.length, 15, "Encoded data should have 15 elements (3 tokens x 5 fields)"); + + // Decode back to absolute format + const decoded = decodeSemanticTokens(encoded); + + // Verify roundtrip produces original tokens + assert.deepStrictEqual(decoded, tokens, "Decoded tokens should match original tokens"); + }); + + test("Decode semantic tokens with delta encoding", function () { + // Create semantic tokens in delta-encoded format + // Format: [deltaLine, deltaStartChar, length, tokenType, tokenModifiers, ...] + const deltaEncoded: vscode.SemanticTokens = { + data: new Uint32Array([ + 0, 5, 3, 1, 0, // line 0, char 5, length 3 + 0, 4, 4, 2, 1, // line 0 (0+0), char 9 (5+4), length 4 + 2, 0, 5, 3, 2, // line 2 (0+2), char 0 (reset), length 5 + ]), + resultId: undefined + }; + + const decoded = decodeSemanticTokens(deltaEncoded); + + // Verify the decoded tokens have correct absolute positions + assert.strictEqual(decoded.length, 3, "Should decode 3 tokens"); + assert.deepStrictEqual(decoded[0], { line: 0, startChar: 5, length: 3, tokenType: 1, tokenModifiers: 0 }); + assert.deepStrictEqual(decoded[1], { line: 0, startChar: 9, length: 4, tokenType: 2, tokenModifiers: 1 }); + assert.deepStrictEqual(decoded[2], { line: 2, startChar: 0, length: 5, tokenType: 3, tokenModifiers: 2 }); + }); + + test("Legend mapping remaps matching token types", function () { + // Source legend has types at different indices than target + const sourceLegend = { + tokenTypes: ["class", "function", "variable"], + tokenModifiers: ["readonly", "static"] + }; + + const targetLegend = { + tokenTypes: ["variable", "function", "class"], // Different order + tokenModifiers: ["static", "readonly"] // Different order + }; + + // Create tokens using source indices + const sourceTokens = encodeSemanticTokens([ + { line: 0, startChar: 0, length: 3, tokenType: 0, tokenModifiers: 0 }, // "class" in source + { line: 1, startChar: 0, length: 4, tokenType: 1, tokenModifiers: 0 }, // "function" in source + { line: 2, startChar: 0, length: 5, tokenType: 2, tokenModifiers: 0 }, // "variable" in source + ]); + + // Remap to target legend + const remapped = remapTokenIndices(sourceTokens, sourceLegend, targetLegend); + const decoded = decodeSemanticTokens(remapped); + + // Verify types were remapped to target indices + assert.strictEqual(decoded.length, 3, "Should have 3 tokens"); + assert.strictEqual(decoded[0].tokenType, 2, "class should map to index 2 in target"); + assert.strictEqual(decoded[1].tokenType, 1, "function should map to index 1 in target"); + assert.strictEqual(decoded[2].tokenType, 0, "variable should map to index 0 in target"); + }); + + test("Legend mapping filters out unmapped token types", function () { + // Source legend has types that don't exist in target + const sourceLegend = { + tokenTypes: ["class", "customType", "function", "anotherCustomType"], + tokenModifiers: [] + }; + + const targetLegend = { + tokenTypes: ["class", "function"], // Only has class and function + tokenModifiers: [] + }; + + // Create tokens including unmapped types + const sourceTokens = encodeSemanticTokens([ + { line: 0, startChar: 0, length: 3, tokenType: 0, tokenModifiers: 0 }, // "class" - should be kept + { line: 1, startChar: 0, length: 4, tokenType: 1, tokenModifiers: 0 }, // "customType" - should be filtered + { line: 2, startChar: 0, length: 5, tokenType: 2, tokenModifiers: 0 }, // "function" - should be kept + { line: 3, startChar: 0, length: 6, tokenType: 3, tokenModifiers: 0 }, // "anotherCustomType" - should be filtered + ]); + + // Remap to target legend + const remapped = remapTokenIndices(sourceTokens, sourceLegend, targetLegend); + const decoded = decodeSemanticTokens(remapped); + + // Verify unmapped types were filtered out + assert.strictEqual(decoded.length, 2, "Should have 2 tokens after filtering"); + assert.strictEqual(decoded[0].tokenType, 0, "class should map to index 0"); + assert.strictEqual(decoded[0].line, 0, "First token should be from line 0"); + assert.strictEqual(decoded[1].tokenType, 1, "function should map to index 1"); + assert.strictEqual(decoded[1].line, 2, "Second token should be from line 2"); + }); + + test("Legend mapping remaps modifier bitfields correctly", function () { + // Source and target have modifiers in different positions + const sourceLegend = { + tokenTypes: ["function"], + tokenModifiers: ["readonly", "static", "async"] // indices 0, 1, 2 + }; + + const targetLegend = { + tokenTypes: ["function"], + tokenModifiers: ["async", "readonly", "static"] // indices 0, 1, 2 (reordered) + }; + + // Create token with multiple modifiers set + // In source: readonly=bit 0, static=bit 1, async=bit 2 + const modifierBits = (1 << 0) | (1 << 1); // readonly + static in source + const sourceTokens = encodeSemanticTokens([ + { line: 0, startChar: 0, length: 3, tokenType: 0, tokenModifiers: modifierBits } + ]); + + // Remap to target legend + const remapped = remapTokenIndices(sourceTokens, sourceLegend, targetLegend); + const decoded = decodeSemanticTokens(remapped); + + // In target: async=bit 0, readonly=bit 1, static=bit 2 + const expectedModifiers = (1 << 1) | (1 << 2); // readonly + static in target + assert.strictEqual(decoded[0].tokenModifiers, expectedModifiers, "Modifiers should be remapped to target positions"); + }); + + test("Legend mapping handles modifiers not in target legend", function () { + // Source has modifiers that don't exist in target + const sourceLegend = { + tokenTypes: ["function"], + tokenModifiers: ["readonly", "customModifier", "static"] + }; + + const targetLegend = { + tokenTypes: ["function"], + tokenModifiers: ["readonly", "static"] // Missing "customModifier" + }; + + // Set all three modifiers in source + const modifierBits = (1 << 0) | (1 << 1) | (1 << 2); // all three modifiers + const sourceTokens = encodeSemanticTokens([ + { line: 0, startChar: 0, length: 3, tokenType: 0, tokenModifiers: modifierBits } + ]); + + // Remap to target legend + const remapped = remapTokenIndices(sourceTokens, sourceLegend, targetLegend); + const decoded = decodeSemanticTokens(remapped); + + // Only readonly and static should be mapped; customModifier should be dropped + const expectedModifiers = (1 << 0) | (1 << 1); // readonly + static in target + assert.strictEqual(decoded[0].tokenModifiers, expectedModifiers, "Unmapped modifiers should be filtered out"); + }); + +}); From 05d70a4dfe54b10a0ec0274572bfa7b3b9702dad Mon Sep 17 00:00:00 2001 From: Julia Silge Date: Fri, 21 Nov 2025 17:37:08 -0700 Subject: [PATCH 5/5] Update CHANGELOG --- apps/vscode/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/vscode/CHANGELOG.md b/apps/vscode/CHANGELOG.md index 623dbfef..0caafddc 100644 --- a/apps/vscode/CHANGELOG.md +++ b/apps/vscode/CHANGELOG.md @@ -6,6 +6,7 @@ - Added a new setting `quarto.useBundledQuartoInPositron` to prefer the Quarto CLI bundled with Positron when available. This setting has precedence _between_ `quarto.path` and `quarto.usePipQuarto`, and has no effect outside of Positron (). - Visual Editor: uses a text box for alternative text and captions in callouts, images, and tables interface. () - Fixed a bug where previewing showed "Not Found" on Quarto files with spaces in the name in subfolders of projects (). +- Added support for semantic highlighting in Quarto documents, when using an LSP that supports it (for example, Pylance) (). ## 1.126.0 (Release on 2025-10-08)