feat(langserver): rework "Analysing", "Hover", "Document Highlight", "Goto" and other things to make them faster, simpler, and easier to extend

Author: d-biehl

packages/language_server/src/robotcode/language_server/robotframework/diagnostics/analyzer.py

@@ -41,6 +41,7 @@
     LibraryEntry,
     ResourceEntry,
     VariableDefinition,
+    VariableDefinitionType,
     VariableNotFoundDefinition,
 )
 from .library_doc import KeywordDoc, KeywordMatcher, is_embedded_keyword
@@ -245,16 +246,36 @@ async def visit(self, node: ast.AST) -> None:
                                         var_range = range_from_token(var_token)
                                     else:
                                         var_range = range_from_token(var_token)
+
+                                    suite_var = None
+                                    if isinstance(var, CommandLineVariableDefinition):
+                                        suite_var = await self.namespace.find_variable(
+                                            var.name,
+                                            skip_commandline_variables=True,
+                                            ignore_error=True,
+                                        )
+                                        if suite_var is not None and suite_var.type not in [
+                                            VariableDefinitionType.VARIABLE
+                                        ]:
+                                            suite_var = None
+
                                     if var.name_range != var_range:
                                         self._variable_references[var].add(
                                             Location(self.namespace.document.document_uri, var_range)
                                         )
+                                        if suite_var is not None:
+                                            self._variable_references[suite_var].add(
+                                                Location(self.namespace.document.document_uri, var_range)
+                                            )
+
                                     elif var not in self._variable_references and token1.type in [
                                         RobotToken.ASSIGN,
                                         RobotToken.ARGUMENT,
                                         RobotToken.VARIABLE,
                                     ]:
                                         self._variable_references[var] = set()
+                                        if suite_var is not None:
+                                            self._variable_references[suite_var] = set()
 
             if (
                 isinstance(node, Statement)
@@ -280,11 +301,23 @@ async def visit(self, node: ast.AST) -> None:
                     else:
                         if self.namespace.document is not None:
                             var_range = range_from_token(var_token)
+
                             if var.name_range != var_range:
                                 self._variable_references[var].add(
                                     Location(self.namespace.document.document_uri, range_from_token(var_token))
                                 )
 
+                                if isinstance(var, CommandLineVariableDefinition):
+                                    suite_var = await self.namespace.find_variable(
+                                        var.name,
+                                        skip_commandline_variables=True,
+                                        ignore_error=True,
+                                    )
+                                    if suite_var is not None and suite_var.type in [VariableDefinitionType.VARIABLE]:
+                                        self._variable_references[suite_var].add(
+                                            Location(self.namespace.document.document_uri, range_from_token(var_token))
+                                        )
+
             await super().visit(node)
         finally:
             self.node_stack = self.node_stack[:-1]
@@ -377,6 +410,37 @@ async def _analyze_keyword_call(
 
             result = self.finder.find_keyword(keyword)
 
+            if (
+                result is not None
+                and lib_entry is not None
+                and kw_namespace
+                and result.parent is not None
+                and result.parent != lib_entry.library_doc.digest
+            ):
+                entry = next(
+                    (
+                        v
+                        for v in (await self.namespace.get_libraries()).values()
+                        if v.library_doc.digest == result.parent
+                    ),
+                    None,
+                )
+                if entry is None:
+                    entry = next(
+                        (
+                            v
+                            for v in (await self.namespace.get_resources()).values()
+                            if v.library_doc.digest == result.parent
+                        ),
+                        None,
+                    )
+                if entry is not None:
+                    lib_entry = entry
+
+            if kw_namespace and lib_entry is not None and lib_range is not None:
+                if self.namespace.document is not None:
+                    self._namespace_references[lib_entry].add(Location(self.namespace.document.document_uri, lib_range))
+
             if not ignore_errors_if_contains_variables or is_not_variable_token(keyword_token):
                 for e in self.finder.diagnostics:
                     self.append_diagnostics(
@@ -386,11 +450,11 @@ async def _analyze_keyword_call(
                         code=e.code,
                     )
 
-            if kw_namespace and lib_entry is not None and lib_range is not None:
-                if self.namespace.document is not None:
-                    self._namespace_references[lib_entry].add(Location(self.namespace.document.document_uri, lib_range))
-
-            if result is not None:
+            if result is None:
+                if self.namespace.document is not None and self.finder.multiple_keywords_result is not None:
+                    for lib_entry, kw_doc in self.finder.multiple_keywords_result:
+                        self._keyword_references[kw_doc].add(Location(self.namespace.document.document_uri, kw_range))
+            else:
                 if self.namespace.document is not None:
                     self._keyword_references[result].add(Location(self.namespace.document.document_uri, kw_range))
 
@@ -538,6 +602,17 @@ async def _analyze_keyword_call(
                                 self._variable_references[var].add(
                                     Location(self.namespace.document.document_uri, range_from_token(var_token))
                                 )
+
+                                if isinstance(var, CommandLineVariableDefinition):
+                                    suite_var = await self.namespace.find_variable(
+                                        var.name,
+                                        skip_commandline_variables=True,
+                                        ignore_error=True,
+                                    )
+                                    if suite_var is not None and suite_var.type in [VariableDefinitionType.VARIABLE]:
+                                        self._variable_references[suite_var].add(
+                                            Location(self.namespace.document.document_uri, range_from_token(var_token))
+                                        )
             if result.argument_definitions:
                 for arg in argument_tokens:
                     name, value = split_from_equals(arg.value)
@@ -964,22 +1039,61 @@ def _check_import_name(self, value: Optional[str], node: ast.AST, type: str) ->
             )
 
     async def visit_VariablesImport(self, node: ast.AST) -> None:  # noqa: N802
-        if get_robot_version() >= (6, 1):
-            from robot.parsing.model.statements import VariablesImport
+        from robot.parsing.lexer.tokens import Token as RobotToken
+        from robot.parsing.model.statements import VariablesImport
 
+        if get_robot_version() >= (6, 1):
             import_node = cast(VariablesImport, node)
             self._check_import_name(import_node.name, node, "Variables")
 
+        n = cast(VariablesImport, node)
+        name_token = cast(RobotToken, n.get_token(RobotToken.NAME))
+        if name_token is None:
+            return
+
+        entries = await self.namespace.get_imported_variables()
+        if entries and self.namespace.document:
+            for v in entries.values():
+                if v.import_source == self.namespace.source and v.import_range == range_from_token(name_token):
+                    if v not in self._namespace_references:
+                        self._namespace_references[v] = set()
+
     async def visit_ResourceImport(self, node: ast.AST) -> None:  # noqa: N802
-        if get_robot_version() >= (6, 1):
-            from robot.parsing.model.statements import ResourceImport
+        from robot.parsing.lexer.tokens import Token as RobotToken
+        from robot.parsing.model.statements import ResourceImport
 
+        if get_robot_version() >= (6, 1):
             import_node = cast(ResourceImport, node)
             self._check_import_name(import_node.name, node, "Resource")
 
+        n = cast(ResourceImport, node)
+        name_token = cast(RobotToken, n.get_token(RobotToken.NAME))
+        if name_token is None:
+            return
+
+        entries = await self.namespace.get_resources()
+        if entries and self.namespace.document:
+            for v in entries.values():
+                if v.import_source == self.namespace.source and v.import_range == range_from_token(name_token):
+                    if v not in self._namespace_references:
+                        self._namespace_references[v] = set()
+
     async def visit_LibraryImport(self, node: ast.AST) -> None:  # noqa: N802
-        if get_robot_version() >= (6, 1):
-            from robot.parsing.model.statements import LibraryImport
+        from robot.parsing.lexer.tokens import Token as RobotToken
+        from robot.parsing.model.statements import LibraryImport
 
+        if get_robot_version() >= (6, 1):
             import_node = cast(LibraryImport, node)
             self._check_import_name(import_node.name, node, "Library")
+
+        n = cast(LibraryImport, node)
+        name_token = cast(RobotToken, n.get_token(RobotToken.NAME))
+        if name_token is None:
+            return
+
+        entries = await self.namespace.get_libraries()
+        if entries and self.namespace.document:
+            for v in entries.values():
+                if v.import_source == self.namespace.source and v.import_range == range_from_token(name_token):
+                    if v not in self._namespace_references:
+                        self._namespace_references[v] = set()

packages/language_server/src/robotcode/language_server/robotframework/diagnostics/entities.py

@@ -171,12 +171,12 @@ def __repr__(self) -> str:
 
 
 class VariableDefinitionType(Enum):
-    VARIABLE = "variable"
+    VARIABLE = "suite variable"
     LOCAL_VARIABLE = "local variable"
     ARGUMENT = "argument"
-    COMMAND_LINE_VARIABLE = "command line variable"
+    COMMAND_LINE_VARIABLE = "global variable [command line]"
     BUILTIN_VARIABLE = "builtin variable"
-    IMPORTED_VARIABLE = "imported variable"
+    IMPORTED_VARIABLE = "suite variable [imported]"
     ENVIRONMENT_VARIABLE = "environment variable"
     VARIABLE_NOT_FOUND = "variable not found"
 
@@ -310,7 +310,7 @@ def __hash__(self) -> int:
 class LibraryEntry:
     name: str
     import_name: str
-    library_doc: "LibraryDoc"
+    library_doc: "LibraryDoc" = field(compare=False)
     args: Tuple[Any, ...] = ()
     alias: Optional[str] = None
     import_range: Range = field(default_factory=lambda: Range.zero())
@@ -328,22 +328,33 @@ def __str__(self) -> str:
     @single_call
     def __hash__(self) -> int:
         return hash(
-            (type(self), self.name, self.import_name, self.args, self.alias, self.import_range, self.import_source)
+            (
+                type(self),
+                self.name,
+                self.import_name,
+                self.args,
+                self.alias,
+                self.import_range,
+                self.import_source,
+                self.alias_range,
+            )
         )
 
 
 @dataclass
 class ResourceEntry(LibraryEntry):
-    imports: List[Import] = field(default_factory=list)
-    variables: List[VariableDefinition] = field(default_factory=list)
+    imports: List[Import] = field(default_factory=list, compare=False)
+    variables: List[VariableDefinition] = field(default_factory=list, compare=False)
 
     @single_call
     def __hash__(self) -> int:
-        return hash(
-            (type(self), self.name, self.import_name, self.args, self.alias, self.import_range, self.import_source)
-        )
+        return hash((type(self), self.name, self.import_name, self.import_range, self.import_source))
 
 
 @dataclass
 class VariablesEntry(LibraryEntry):
-    variables: List[ImportedVariableDefinition] = field(default_factory=list)
+    variables: List[ImportedVariableDefinition] = field(default_factory=list, compare=False)
+
+    @single_call
+    def __hash__(self) -> int:
+        return hash((type(self), self.name, self.import_name, self.args, self.import_range, self.import_source))

packages/language_server/src/robotcode/language_server/robotframework/diagnostics/imports_manager.py

@@ -571,18 +571,24 @@ def clear_cache(self) -> None:
 
     @_logger.call
     async def get_command_line_variables(self) -> List[VariableDefinition]:
-        from robot.errors import DataError
         from robot.utils.text import split_args_from_name_or_path
 
         async with self._command_line_variables_lock:
             if self._command_line_variables is None:
                 command_line_vars: List[VariableDefinition] = []
+
                 command_line_vars += [
                     CommandLineVariableDefinition(0, 0, 0, 0, "", f"${{{k}}}", None, has_value=True, value=v)
-                    for k, v in (self.parent_protocol.profile.variables or {}).items()
+                    for k, v in {
+                        **{k1: v1 for k1, v1 in (self.parent_protocol.profile.variables or {}).items()},
+                        **self.config.robot.variables,
+                    }.items()
                 ]
 
-                for variable_file in self.parent_protocol.profile.variable_files or []:
+                for variable_file in [
+                    *(self.parent_protocol.profile.variable_files or []),
+                    *self.config.robot.variable_files,
+                ]:
                     name, args = split_args_from_name_or_path(str(variable_file))
                     try:
                         lib_doc = await self.get_libdoc_for_variables_import(
@@ -594,49 +600,31 @@ async def get_command_line_variables(self) -> List[VariableDefinition]:
                             resolve_command_line_vars=False,
                         )
                         if lib_doc is not None:
-                            command_line_vars += lib_doc.variables
-
-                            if lib_doc.errors:
-                                # TODO add diagnostics
-                                for error in lib_doc.errors:
-                                    self._logger.error(
-                                        lambda: f"{error.type_name}: {error.message} in {error.source}:{error.line_no}"
-                                    )
-                    except (SystemExit, KeyboardInterrupt, asyncio.CancelledError):
-                        raise
-                    except BaseException as e:
-                        # TODO add diagnostics
-                        self._logger.exception(e)
-
-                command_line_vars += [
-                    CommandLineVariableDefinition(0, 0, 0, 0, "", f"${{{k}}}", None, has_value=True, value=v)
-                    for k, v in self.config.robot.variables.items()
-                ]
-                for variable_file in self.config.robot.variable_files:
-                    name, args = split_args_from_name_or_path(variable_file)
-                    try:
-                        lib_doc = await self.get_libdoc_for_variables_import(
-                            name,
-                            tuple(args),
-                            str(self.folder.to_path()),
-                            self,
-                            resolve_variables=False,
-                            resolve_command_line_vars=False,
-                        )
-                        if lib_doc is not None:
-                            command_line_vars += lib_doc.variables
+                            command_line_vars += [
+                                CommandLineVariableDefinition(
+                                    line_no=e.line_no,
+                                    col_offset=e.col_offset,
+                                    end_line_no=e.end_line_no,
+                                    end_col_offset=e.end_col_offset,
+                                    source=e.source,
+                                    name=e.name,
+                                    name_token=e.name_token,
+                                    has_value=e.has_value,
+                                    resolvable=e.resolvable,
+                                    value=e.value,
+                                    value_is_native=e.value_is_native,
+                                )
+                                for e in lib_doc.variables
+                            ]
 
                             if lib_doc.errors:
                                 # TODO add diagnostics
                                 for error in lib_doc.errors:
                                     self._logger.error(
                                         lambda: f"{error.type_name}: {error.message} in {error.source}:{error.line_no}"
                                     )
-
                     except (SystemExit, KeyboardInterrupt, asyncio.CancelledError):
                         raise
-                    except DataError:
-                        pass
                     except BaseException as e:
                         # TODO add diagnostics
                         self._logger.exception(e)

packages/language_server/src/robotcode/language_server/robotframework/diagnostics/library_doc.py

@@ -1385,9 +1385,11 @@ def get_test_library(
                 create_handlers=False,
                 variables=robot_variables,
             )
+            lib.get_instance()
         except (SystemExit, KeyboardInterrupt):
             raise
         except BaseException as e:
+            lib = None
             errors.append(
                 error_from_exception(
                     e,
@@ -1399,9 +1401,11 @@ def get_test_library(
             if args:
                 try:
                     lib = get_test_library(libcode, source, library_name, (), create_handlers=False)
+                    lib.get_instance()
                 except (SystemExit, KeyboardInterrupt):
                     raise
                 except BaseException:
+                    lib = None
                     pass
 
         real_source = lib.source if lib is not None else source