From 58259ec0b18214c5adcee3a67b98b8792409a9c1 Mon Sep 17 00:00:00 2001 From: Martin Leduc <31558169+DecimalTurn@users.noreply.github.com> Date: Sat, 18 Oct 2025 04:27:28 +0000 Subject: [PATCH 1/2] Add test for pre compiler issues with parentheses --- .gitignore | 3 +- package.json | 1 + server/src/test/antlr-parser-pre.test.ts | 131 +++++++++++++++++++++++ test/parser/pre/ParsingParentheses.bas | 3 + test/parser/pre/TwoFunctionCalls.bas | 2 + 5 files changed, 139 insertions(+), 1 deletion(-) create mode 100644 server/src/test/antlr-parser-pre.test.ts create mode 100644 test/parser/pre/ParsingParentheses.bas create mode 100644 test/parser/pre/TwoFunctionCalls.bas diff --git a/.gitignore b/.gitignore index 730ef66..6ab6f06 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,5 @@ template* # Workspace settings .vscode-test -.vscode/settings.json \ No newline at end of file +.vscode/settings.json +server/out/ \ No newline at end of file diff --git a/package.json b/package.json index 8c446ca..af97a1c 100644 --- a/package.json +++ b/package.json @@ -162,6 +162,7 @@ "test:textMate:unit": "vscode-tmgrammar-test ./test/textmate/**/*.vba", "test:textMate:snap": "vscode-tmgrammar-snap ./test/textmate/snapshot/*.??s", "test:vsc:unit": "vscode-test", + "test:antlr:unit": "tsc --project server/tsconfig.json && npx mocha server/out/test/**/*.test.js", "testsh": "sh ./scripts/e2e.sh", "testps": "powershell ./scripts/e2e.ps1" }, diff --git a/server/src/test/antlr-parser-pre.test.ts b/server/src/test/antlr-parser-pre.test.ts new file mode 100644 index 0000000..5e0f58d --- /dev/null +++ b/server/src/test/antlr-parser-pre.test.ts @@ -0,0 +1,131 @@ +/** + * Direct ANTLR parser test for VBA preprocessor grammar. + * + * This test directly uses the ANTLR parser to catch syntax errors and undesired implicit tokens (T__1, T__2, etc.) + * without going through the VS Code diagnostics layer. + */ + +import { describe, it } from 'mocha'; +import * as assert from 'assert'; +import * as fs from 'fs'; +import * as path from 'path'; +import { VbaPreParser, VbaPreLexer } from '../project/parser/vbaAntlr'; +import { CharStream, CommonTokenStream } from 'antlr4ng'; + +describe('ANTLR VBA Preprocessor Parser', () => { + + /** + * Helper function to check and report implicit tokens + */ + function checkImplicitTokens(result: ReturnType): Array<{type: number, text: string, typeName: string}> { + const implicitTokens = result.tokenInfo.filter(t => t.typeName.startsWith('T__')); + if (implicitTokens.length > 0) { + console.log(` ❌ Found ${implicitTokens.length} implicit token(s): ${implicitTokens.map(t => t.typeName).join(', ')}`); + } else { + console.log(' ✅ No implicit tokens found'); + } + return implicitTokens; + } + + /** + * Helper function to log parsing results consistently + */ + function logParsingResults(input: string, result: ReturnType) { + console.log('\n 📝 Input:'); + const inputLines = input.split('\n'); + inputLines.forEach((line, index) => { + // Show line numbers and preserve exact whitespace + if (line.trim() || index < inputLines.length - 1) { // Show non-empty lines and all but the last empty line + console.log(` ${(index + 1).toString().padStart(2)}: ${line}`); + } + }); + console.log(' 🔤 Tokens:'); + result.tokenInfo.forEach((t, i) => { + const displayText = t.text.replace(/\n/g, '\\n').replace(/\r/g, '\\r'); + console.log(` ${i.toString().padStart(2)}: ${t.typeName.padEnd(12)} = "${displayText}"`); + }); + if (result.lexerErrors.length > 0) { + console.log(' ❌ Lexer errors:', result.lexerErrors); + } + if (result.errors.length > 0) { + console.log(' ❌ Parser errors:', result.errors); + } + console.log(` 📊 Syntax errors: ${result.syntaxErrors}`); + } + + /** + * Test helper to parse input and collect syntax errors + */ + function parseAndGetErrors(input: string) { + const lexer = VbaPreLexer.create(input); + const tokens = new CommonTokenStream(lexer); + const parser = new VbaPreParser(tokens); + + // Collect all error information + const errors: string[] = []; + const lexerErrors: string[] = []; + const tokenInfo: Array<{type: number, text: string, typeName: string}> = []; + + lexer.removeErrorListeners(); + parser.removeErrorListeners(); + + // Get tokens for inspection + tokens.fill(); + const allTokens = tokens.getTokens(); + for (const token of allTokens) { + if (token.type !== -1) { // Skip EOF + const typeName = lexer.vocabulary.getSymbolicName(token.type) || `T__${token.type - 1}`; + tokenInfo.push({ + type: token.type, + text: token.text || '', + typeName: typeName + }); + } + } + + // Try to parse + let parseTree = null; + try { + parseTree = parser.startRule(); + } catch (error) { + errors.push(`Parse exception: ${error}`); + } + + return { + errors, + lexerErrors, + tokenInfo, + syntaxErrors: parser.numberOfSyntaxErrors, + parseTree + }; + } + + it('should parse function call with string literal and parentheses', () => { + const testFilePath = path.join(__dirname, '../../../test/parser/pre/ParsingParentheses.bas'); + const input = fs.readFileSync(testFilePath, 'utf8'); + + const result = parseAndGetErrors(input); + + logParsingResults(input, result); + const implicitTokens = checkImplicitTokens(result); + + // The test should fail if there are implicit T__ tokens for parentheses + assert.strictEqual(result.syntaxErrors, 0, `Expected no syntax errors, but found: ${result.errors.join(', ')}`); + assert.strictEqual(implicitTokens.length, 0, `Found implicit tokens: ${implicitTokens.map(t => t.typeName).join(', ')}`); + }); + + it('should parse multiple function calls correctly', () => { + const testFilePath = path.join(__dirname, '../../../test/parser/pre/TwoFunctionCalls.bas'); + const input = fs.readFileSync(testFilePath, 'utf8'); + + const result = parseAndGetErrors(input); + + logParsingResults(input, result); + const implicitTokens = checkImplicitTokens(result); + + assert.strictEqual(result.syntaxErrors, 0); + assert.strictEqual(implicitTokens.length, 0); + }); + + +}); \ No newline at end of file diff --git a/test/parser/pre/ParsingParentheses.bas b/test/parser/pre/ParsingParentheses.bas new file mode 100644 index 0000000..d0d125c --- /dev/null +++ b/test/parser/pre/ParsingParentheses.bas @@ -0,0 +1,3 @@ +y = Format( "Test '<'") + + diff --git a/test/parser/pre/TwoFunctionCalls.bas b/test/parser/pre/TwoFunctionCalls.bas new file mode 100644 index 0000000..68e9856 --- /dev/null +++ b/test/parser/pre/TwoFunctionCalls.bas @@ -0,0 +1,2 @@ +result = Trim("hello") +val = Left("test", 2) From c5d072876617e3e7e9afb8d4ab29dce7437e2d23 Mon Sep 17 00:00:00 2001 From: Martin Leduc <31558169+DecimalTurn@users.noreply.github.com> Date: Tue, 21 Oct 2025 00:26:28 +0000 Subject: [PATCH 2/2] Rename to parenthesis --- server/src/test/antlr-parser-pre.test.ts | 2 +- .../pre/{ParsingParentheses.bas => ParsingParenthesis.bas} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename test/parser/pre/{ParsingParentheses.bas => ParsingParenthesis.bas} (100%) diff --git a/server/src/test/antlr-parser-pre.test.ts b/server/src/test/antlr-parser-pre.test.ts index 5e0f58d..471df42 100644 --- a/server/src/test/antlr-parser-pre.test.ts +++ b/server/src/test/antlr-parser-pre.test.ts @@ -101,7 +101,7 @@ describe('ANTLR VBA Preprocessor Parser', () => { } it('should parse function call with string literal and parentheses', () => { - const testFilePath = path.join(__dirname, '../../../test/parser/pre/ParsingParentheses.bas'); + const testFilePath = path.join(__dirname, '../../../test/parser/pre/ParsingParenthesis.bas'); const input = fs.readFileSync(testFilePath, 'utf8'); const result = parseAndGetErrors(input); diff --git a/test/parser/pre/ParsingParentheses.bas b/test/parser/pre/ParsingParenthesis.bas similarity index 100% rename from test/parser/pre/ParsingParentheses.bas rename to test/parser/pre/ParsingParenthesis.bas