fix: split whiteSpace into multiple tokens to fix findCaretTokenIndex may undefined (#385)

JackWang032 · web-flow · commit 8e110120de52 · 2025-05-08T14:26:00.000+08:00
+1
diff --git a/src/grammar/flink/FlinkSqlLexer.g4 b/src/grammar/flink/FlinkSqlLexer.g4
@@ -11,7 +11,7 @@ options {
 
 // SKIP
 
-SPACE         : [ \t\r\n]+                                                         -> channel(HIDDEN);
+SPACE         : (' ' | '\t' | '\r' | '\n')                                         -> channel(HIDDEN);
 COMMENT_INPUT : '/*' .*? '*/'                                                      -> channel(HIDDEN);
 LINE_COMMENT: (('--' | '#') ~[\r\n]* ('\r'? '\n' | EOF) | '--' ('\r'? '\n' | EOF)) -> channel(HIDDEN);
 
diff --git a/src/grammar/impala/ImpalaSqlLexer.g4 b/src/grammar/impala/ImpalaSqlLexer.g4
@@ -350,4 +350,4 @@ SIMPLE_COMMENT: '--' ~[\r\n]* '\r'? '\n'? -> channel(HIDDEN);
 
 BRACKETED_COMMENT: '/*' .*? '*/' -> channel(HIDDEN);
 
-WS: [ \r\n\t]+ -> channel(HIDDEN);
+WS: (' ' | '\t' | '\r' | '\n') -> channel(HIDDEN);
diff --git a/src/grammar/mysql/MySqlLexer.g4 b/src/grammar/mysql/MySqlLexer.g4
@@ -41,7 +41,7 @@ channels {
     ERRORCHANNEL
 }
 
-SPACE              : [ \t\r\n]+                                                           -> channel(HIDDEN);
+SPACE              : (' ' | '\t' | '\r' | '\n')                                           -> channel(HIDDEN);
 SPEC_MYSQL_COMMENT : '/*!' .+? '*/'                                                       -> channel(MYSQLCOMMENT);
 COMMENT_INPUT      : '/*' .*? '*/'                                                        -> channel(HIDDEN);
 LINE_COMMENT: (('--' [ \t]* | '#') ~[\r\n]* ('\r'? '\n' | EOF) | '--' ('\r'? '\n' | EOF)) -> channel(HIDDEN);
diff --git a/src/grammar/spark/SparkSqlLexer.g4 b/src/grammar/spark/SparkSqlLexer.g4
@@ -481,7 +481,7 @@ SIMPLE_COMMENT: '--' ('\\\n' | ~[\r\n])* '\r'? '\n'? -> channel(HIDDEN);
 BRACKETED_COMMENT:
     '/*' (BRACKETED_COMMENT | .)*? ('*/' | {this.markUnclosedComment();} EOF) -> channel(HIDDEN);
 
-WS: [ \r\n\t]+ -> channel(HIDDEN);
+WS: (' ' | '\t' | '\r' | '\n') -> channel(HIDDEN);
 
 // Catch-all for anything we can't recognize.
 // We use this to be able to ignore and recover all the text
diff --git a/src/grammar/trino/TrinoSql.g4 b/src/grammar/trino/TrinoSql.g4
@@ -1648,7 +1648,7 @@ SIMPLE_COMMENT: '--' ~[\r\n]* '\r'? '\n'? -> channel(HIDDEN);
 
 BRACKETED_COMMENT: '/*' .*? '*/' -> channel(HIDDEN);
 
-WS: [ \r\n\t]+ -> channel(HIDDEN);
+WS: (' ' | '\t' | '\r' | '\n') -> channel(HIDDEN);
 
 // Catch-all for anything we can't recognize.
 // We use this to be able to ignore and recover all the text
diff --git a/src/lib/flink/FlinkSqlLexer.interp b/src/lib/flink/FlinkSqlLexer.interp
diff --git a/src/lib/flink/FlinkSqlLexer.ts b/src/lib/flink/FlinkSqlLexer.ts
diff --git a/src/lib/impala/ImpalaSqlLexer.interp b/src/lib/impala/ImpalaSqlLexer.interp
diff --git a/src/lib/impala/ImpalaSqlLexer.ts b/src/lib/impala/ImpalaSqlLexer.ts
diff --git a/src/lib/mysql/MySqlLexer.interp b/src/lib/mysql/MySqlLexer.interp
diff --git a/src/lib/mysql/MySqlLexer.ts b/src/lib/mysql/MySqlLexer.ts
diff --git a/src/lib/spark/SparkSqlLexer.interp b/src/lib/spark/SparkSqlLexer.interp
diff --git a/src/lib/spark/SparkSqlLexer.ts b/src/lib/spark/SparkSqlLexer.ts
diff --git a/src/lib/trino/TrinoSqlLexer.interp b/src/lib/trino/TrinoSqlLexer.interp
diff --git a/src/lib/trino/TrinoSqlLexer.ts b/src/lib/trino/TrinoSqlLexer.ts
diff --git a/test/helper.ts b/test/helper.ts
@@ -32,9 +32,15 @@ export const readSQL = (dirname: string, fileName: string) => {
     return result;
 };
 
-export function commentOtherLine(sqlContent: string, line: number) {
+export function commentOtherLine(
+    sqlContent: string,
+    line: number | [startLine: number, endLine: number]
+) {
     const slices = sqlContent.split('\n').map((item, index) => {
-        if (index !== line - 1) {
+        if (
+            (Array.isArray(line) && (index + 1 < line[0] || index + 1 > line[1])) ||
+            (typeof line === 'number' && index + 1 !== line)
+        ) {
             return '-- ' + item;
         } else {
             return item;
diff --git a/test/parser/flink/suggestion/fixtures/tokenSuggestion.sql b/test/parser/flink/suggestion/fixtures/tokenSuggestion.sql
@@ -4,4 +4,8 @@ USE
 ;
 CREATE 
 ;
-SHOW 
+SHOW 
+
+CREATE TABLE tb (id 
+ 
+);
diff --git a/test/parser/flink/suggestion/tokenSuggestion.test.ts b/test/parser/flink/suggestion/tokenSuggestion.test.ts
@@ -67,4 +67,16 @@ describe('Flink SQL Token Suggestion', () => {
             'JARS',
         ]);
     });
+
+    test('Suggestion in new line', () => {
+        const pos: CaretPosition = {
+            lineNumber: 10,
+            column: 2,
+        };
+        const suggestion = flink.getSuggestionAtCaretPosition(
+            commentOtherLine(tokenSql, [9, 11]),
+            pos
+        )?.keywords;
+        expect(suggestion.length).not.toBe(0);
+    });
 });
diff --git a/test/parser/hive/suggestion/fixtures/tokenSuggestion.sql b/test/parser/hive/suggestion/fixtures/tokenSuggestion.sql
@@ -18,3 +18,7 @@ LOAD
 ;
 SHOW 
 ;
+
+CREATE TABLE tb (id 
+ 
+);
diff --git a/test/parser/hive/suggestion/tokenSuggestion.test.ts b/test/parser/hive/suggestion/tokenSuggestion.test.ts
@@ -219,4 +219,16 @@ describe('Hive SQL Token Suggestion', () => {
             'SCHEMAS',
         ]);
     });
+
+    test('Suggestion in new line', () => {
+        const pos: CaretPosition = {
+            lineNumber: 23,
+            column: 2,
+        };
+        const suggestion = hive.getSuggestionAtCaretPosition(
+            commentOtherLine(tokenSql, [21, 23]),
+            pos
+        )?.keywords;
+        expect(suggestion.length).not.toBe(0);
+    });
 });
diff --git a/test/parser/impala/suggestion/fixtures/tokenSuggestion.sql b/test/parser/impala/suggestion/fixtures/tokenSuggestion.sql
@@ -9,3 +9,7 @@ INSERT ;
 SHOW ;
 
 CREATE TABLE t1 (id );
+
+CREATE TABLE tb (id 
+ 
+);
diff --git a/test/parser/impala/suggestion/tokenSuggestion.test.ts b/test/parser/impala/suggestion/tokenSuggestion.test.ts
@@ -143,4 +143,16 @@ describe('Impala SQL Token Suggestion', () => {
 
         expect(dataTypes.every((dataType) => suggestion.includes(dataType))).toBe(true);
     });
+
+    test('Suggestion in new line', () => {
+        const pos: CaretPosition = {
+            lineNumber: 14,
+            column: 2,
+        };
+        const suggestion = impala.getSuggestionAtCaretPosition(
+            commentOtherLine(tokenSql, [13, 15]),
+            pos
+        )?.keywords;
+        expect(suggestion.length).not.toBe(0);
+    });
 });
diff --git a/test/parser/mysql/suggestion/fixtures/tokenSuggestion.sql b/test/parser/mysql/suggestion/fixtures/tokenSuggestion.sql
@@ -14,3 +14,7 @@ LOAD
 ;
 SHOW 
 ;
+
+CREATE TABLE tb (id 
+ 
+);
diff --git a/test/parser/mysql/suggestion/tokenSuggestion.test.ts b/test/parser/mysql/suggestion/tokenSuggestion.test.ts
@@ -242,4 +242,16 @@ describe('MySQL Token Suggestion', () => {
             'BINARY',
         ]);
     });
+
+    test('Suggestion in new line', () => {
+        const pos: CaretPosition = {
+            lineNumber: 19,
+            column: 2,
+        };
+        const suggestion = mysql.getSuggestionAtCaretPosition(
+            commentOtherLine(tokenSql, [18, 20]),
+            pos
+        )?.keywords;
+        expect(suggestion.length).not.toBe(0);
+    });
 });
diff --git a/test/parser/postgresql/suggestion/fixtures/tokenSuggestion.sql b/test/parser/postgresql/suggestion/fixtures/tokenSuggestion.sql
@@ -8,5 +8,6 @@ DELETE ;
 
 CREATE ;
 
-
-
+CREATE TABLE tb (id 
+ 
+);
diff --git a/test/parser/postgresql/suggestion/tokenSuggestion.test.ts b/test/parser/postgresql/suggestion/tokenSuggestion.test.ts
@@ -190,4 +190,16 @@ describe('Postgres SQL Token Suggestion', () => {
         )?.keywords;
         expect(suggestion).toMatchUnorderedArray(['INTO']);
     });
+
+    test('Suggestion in new line', () => {
+        const pos: CaretPosition = {
+            lineNumber: 12,
+            column: 2,
+        };
+        const suggestion = postgresql.getSuggestionAtCaretPosition(
+            commentOtherLine(tokenSql, [11, 13]),
+            pos
+        )?.keywords;
+        expect(suggestion.length).not.toBe(0);
+    });
 });
diff --git a/test/parser/spark/suggestion/fixtures/tokenSuggestion.sql b/test/parser/spark/suggestion/fixtures/tokenSuggestion.sql
@@ -16,3 +16,7 @@ SHOW
 ;
 EXPORT 
 ;
+
+CREATE TABLE tb (id 
+ 
+);
diff --git a/test/parser/spark/suggestion/tokenSuggestion.test.ts b/test/parser/spark/suggestion/tokenSuggestion.test.ts
@@ -197,4 +197,16 @@ describe('Spark SQL Token Suggestion', () => {
 
         expect(suggestion).toMatchUnorderedArray(['TABLE']);
     });
+
+    test('Suggestion in new line', () => {
+        const pos: CaretPosition = {
+            lineNumber: 21,
+            column: 2,
+        };
+        const suggestion = spark.getSuggestionAtCaretPosition(
+            commentOtherLine(tokenSql, [20, 22]),
+            pos
+        )?.keywords;
+        expect(suggestion.length).not.toBe(0);
+    });
 });
diff --git a/test/parser/trino/suggestion/fixtures/tokenSuggestion.sql b/test/parser/trino/suggestion/fixtures/tokenSuggestion.sql
@@ -11,3 +11,7 @@ DESCRIBE ;
 DROP ;
 
 INSERT ;
+
+CREATE TABLE tb (id 
+ 
+);
diff --git a/test/parser/trino/suggestion/tokenSuggestion.test.ts b/test/parser/trino/suggestion/tokenSuggestion.test.ts
@@ -124,4 +124,16 @@ describe('Trino SQL Token Suggestion', () => {
 
         expect(suggestion).toMatchUnorderedArray(['INTO']);
     });
+
+    test('Suggestion in new line', () => {
+        const pos: CaretPosition = {
+            lineNumber: 16,
+            column: 2,
+        };
+        const suggestion = trino.getSuggestionAtCaretPosition(
+            commentOtherLine(tokenSql, [15, 17]),
+            pos
+        )?.keywords;
+        expect(suggestion.length).not.toBe(0);
+    });
 });

Original file line number	Diff line number	Diff line change
`@@ -350,4 +350,4 @@ SIMPLE_COMMENT: '--' ~[\r\n]* '\r'? '\n'? -> channel(HIDDEN);`
`350`	`350`
`351`	`351`	`BRACKETED_COMMENT: '/' .? '*/' -> channel(HIDDEN);`
`352`	`352`
`353`		`-WS: [ \r\n\t]+ -> channel(HIDDEN);`
	`353`	`+WS: (' ' \| '\t' \| '\r' \| '\n') -> channel(HIDDEN);`
Original file line number	Diff line number	Diff line change
`@@ -41,7 +41,7 @@ channels {`
`41`	`41`	`ERRORCHANNEL`
`42`	`42`	`}`
`43`	`43`
`44`		`-SPACE : [ \t\r\n]+ -> channel(HIDDEN);`
	`44`	`+SPACE : (' ' \| '\t' \| '\r' \| '\n') -> channel(HIDDEN);`
`45`	`45`	`SPEC_MYSQL_COMMENT : '/!' .+? '/' -> channel(MYSQLCOMMENT);`
`46`	`46`	`COMMENT_INPUT : '/' .? '*/' -> channel(HIDDEN);`
`47`	`47`	`LINE_COMMENT: (('--' [ \t]* \| '#') ~[\r\n]* ('\r'? '\n' \| EOF) \| '--' ('\r'? '\n' \| EOF)) -> channel(HIDDEN);`