From 159f5a2139d5fcefaa54c66c0747d31ae6c29ff7 Mon Sep 17 00:00:00 2001 From: Arsam Date: Fri, 16 Aug 2024 20:42:17 +0200 Subject: [PATCH 01/25] Various fixes, new tests and replacements of raises with logger warnings --- .../api_analyzer/_ast_visitor.py | 77 +++++++++++++++++-- .../api_analyzer/_ast_walker.py | 10 ++- src/safeds_stubgen/api_analyzer/_get_api.py | 11 ++- .../api_analyzer/_mypy_helpers.py | 7 +- .../docstring_parsing/_docstring_parser.py | 8 +- .../stubs_generator/_generate_stubs.py | 4 + .../various_modules_package/class_module.py | 16 +++- .../various_modules_package/enum_module.py | 17 ++++ .../function_module.py | 14 ++++ .../__snapshots__/test__get_api.ambr | 52 +++++++++++++ ...n.test_stub_creation[class_module].sdsstub | 16 ++++ ...on.test_stub_creation[enum_module].sdsstub | 6 ++ ...est_stub_creation[function_module].sdsstub | 16 ++++ 13 files changed, 237 insertions(+), 17 deletions(-) diff --git a/src/safeds_stubgen/api_analyzer/_ast_visitor.py b/src/safeds_stubgen/api_analyzer/_ast_visitor.py index 192e8e22..1d826a4c 100644 --- a/src/safeds_stubgen/api_analyzer/_ast_visitor.py +++ b/src/safeds_stubgen/api_analyzer/_ast_visitor.py @@ -202,8 +202,8 @@ def enter_classdef(self, node: mp_nodes.ClassDef) -> None: ): inherits_from_exception = True - if hasattr(superclass, "fullname"): - superclass_qname = superclass.fullname + if hasattr(superclass, "fullname") or hasattr(superclass, "name"): + superclass_qname = getattr(superclass, "fullname", "") or getattr(superclass, "name", "") superclass_name = superclass_qname.split(".")[-1] # Check if the superclass name is an alias and find the real name @@ -323,7 +323,7 @@ def enter_funcdef(self, node: mp_nodes.FuncDef) -> None: if result_doc_type is not None: if result_type is None: # Add missing returns - result_name = result_doc.name if result_doc.name else f"result_{i}" + result_name = result_doc.name if result_doc.name else f"result_{i + 1}" new_result = Result(type=result_doc_type, name=result_name, id=f"{function_id}/{result_name}") results_code.append(new_result) @@ -408,6 +408,10 @@ def enter_assignmentstmt(self, node: mp_nodes.AssignmentStmt) -> None: assignments: list[Attribute | EnumInstance] = [] for lvalue in node.lvalues: + if isinstance(lvalue, mp_nodes.IndexExpr): + # e.g.: `self.obj.ob_dict['index'] = "some value"` + continue + if isinstance(parent, Class): for assignment in self._parse_attributes(lvalue, node.unanalyzed_type, is_static=True): assignments.append(assignment) @@ -427,7 +431,7 @@ def enter_assignmentstmt(self, node: mp_nodes.AssignmentStmt) -> None: else: if not hasattr(lvalue, "name"): # pragma: no cover raise AttributeError("Expected lvalue to have attribtue 'name'.") - names.append(lvalue.name) + names.append(getattr(lvalue, "name", "")) for name in names: assignments.append( @@ -577,7 +581,60 @@ def _parse_results( return all_results @staticmethod - def _infer_type_from_return_stmts(func_node: mp_nodes.FuncDef) -> sds_types.TupleType | None: + def _remove_assignments(func_defn: list, type_: AbstractType) -> AbstractType: + """ + Check if the expression comes from an `AssignmentStmt`. + + If the return value of a function consists of variables we have to check if those variables are defined + in the function itself (assignment). If this is not the case, we can assume that they are imported or from + outside the funciton. + """ + actual_types: list[AbstractType] = [] + if isinstance(type_, sds_types.NamedType | sds_types.TupleType): + if isinstance(type_, sds_types.TupleType): + found_types = type_.types + else: + found_types = [type_] + + for found_type in found_types: + if isinstance(found_type, sds_types.NamedType): + is_assignment = False + found_type_name = found_type.name + + for defn in func_defn: + if not isinstance(defn, mp_nodes.AssignmentStmt): + continue + + for lvalue in defn.lvalues: + if isinstance(lvalue, mp_nodes.TupleExpr): + name_expressions = lvalue.items + else: + name_expressions = [lvalue] + + for expr in name_expressions: + if isinstance(expr, mp_nodes.NameExpr) and found_type_name == expr.name: + is_assignment = True + break + if is_assignment: + break + + if is_assignment: + break + + if is_assignment: + actual_types.append(sds_types.UnknownType()) + else: + actual_types.append(found_type) + + if len(actual_types) > 1: + type_ = sds_types.TupleType(types=actual_types) + elif len(actual_types) == 1: + type_ = actual_types[0] + else: + type_ = sds_types.UnknownType() + return type_ + + def _infer_type_from_return_stmts(self, func_node: mp_nodes.FuncDef) -> sds_types.TupleType | None: # To infer the type, we iterate through all return statements we find in the function func_defn = get_funcdef_definitions(func_node) return_stmts = find_return_stmts_recursive(func_defn) @@ -604,6 +661,8 @@ def _infer_type_from_return_stmts(func_node: mp_nodes.FuncDef) -> sds_types.Tupl types.add(sds_types.NamedType(name=expr_type.name, qname=expr_type.fullname)) else: type_ = mypy_expression_to_sds_type(return_stmt.expr) + type_ = self._remove_assignments(func_defn, type_) + if isinstance(type_, sds_types.NamedType | sds_types.TupleType): types.add(type_) @@ -1197,12 +1256,14 @@ def _is_public(self, name: str, qname: str) -> bool: parent = self.__declaration_stack[-1] - if not isinstance(parent, Module | Class) and not (isinstance(parent, Function) and parent.name == "__init__"): + if not isinstance(parent, Module | Class | Enum) and not ( + isinstance(parent, Function) and parent.name == "__init__" + ): raise TypeError( f"Expected parent for {name} in module {self.mypy_file.fullname} to be a class or a module.", ) # pragma: no cover - if not isinstance(parent, Function): + if not isinstance(parent, Function | Enum): _check_publicity_with_reexports: bool | None = self._check_publicity_in_reexports(name, qname, parent) if _check_publicity_with_reexports is not None: @@ -1326,7 +1387,7 @@ def _check_publicity_in_reexports(self, name: str, qname: str, parent: Module | def result_name_generator() -> Generator: - """Generate a name for callable type parameters starting from 'a' until 'zz'.""" + """Generate a name for callable type parameters starting from 'result_1' until 'result_1000'.""" while True: for x in range(1, 1000): yield f"result_{x}" diff --git a/src/safeds_stubgen/api_analyzer/_ast_walker.py b/src/safeds_stubgen/api_analyzer/_ast_walker.py index 684e6a61..df630335 100644 --- a/src/safeds_stubgen/api_analyzer/_ast_walker.py +++ b/src/safeds_stubgen/api_analyzer/_ast_walker.py @@ -68,7 +68,15 @@ def __walk(self, node: MypyFile | ClassDef | Decorator | FuncDef | AssignmentStm if getattr(child_node, "name", "") == "__mypy-replace": # pragma: no cover continue - self.__walk(child_node, visited_nodes) + # Overloaded Functions can either have one implementation (impl) or can have multiple (items) + if isinstance(child_node, OverloadedFuncDef): + if child_node.impl is not None: + self.__walk(child_node.impl, visited_nodes) + else: + for child_node_impl in child_node.items: + self.__walk(child_node_impl, visited_nodes) + else: + self.__walk(child_node, visited_nodes) self.__leave(node) def __enter(self, node: MypyFile | ClassDef | FuncDef | AssignmentStmt) -> None: diff --git a/src/safeds_stubgen/api_analyzer/_get_api.py b/src/safeds_stubgen/api_analyzer/_get_api.py index d14ecacc..12800645 100644 --- a/src/safeds_stubgen/api_analyzer/_get_api.py +++ b/src/safeds_stubgen/api_analyzer/_get_api.py @@ -185,8 +185,13 @@ def _get_aliases(result_types: dict, package_name: str) -> dict[str, set[str]]: else: continue + # Try to find the original qname (fullname) of the alias if in_package: - if isinstance(type_value, mypy_types.CallableType) and hasattr(type_value.bound_args[0], "type"): + if ( + isinstance(type_value, mypy_types.CallableType) + and type_value.bound_args + and hasattr(type_value.bound_args[0], "type") + ): fullname = type_value.bound_args[0].type.fullname # type: ignore[union-attr] elif isinstance(type_value, mypy_types.Instance): fullname = type_value.type.fullname @@ -195,7 +200,9 @@ def _get_aliases(result_types: dict, package_name: str) -> dict[str, set[str]]: elif isinstance(key, mypy_nodes.NameExpr) and isinstance(key.node, mypy_nodes.Var): fullname = key.node.fullname else: # pragma: no cover - raise TypeError("Received unexpected type while searching for aliases.") + msg = f"Received unexpected type while searching for aliases. Skipping for '{name}'." + logging.warning(msg) + continue aliases[name].add(fullname) diff --git a/src/safeds_stubgen/api_analyzer/_mypy_helpers.py b/src/safeds_stubgen/api_analyzer/_mypy_helpers.py index cfeb102a..4b827446 100644 --- a/src/safeds_stubgen/api_analyzer/_mypy_helpers.py +++ b/src/safeds_stubgen/api_analyzer/_mypy_helpers.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging from typing import TYPE_CHECKING, Literal import mypy.types as mp_types @@ -103,7 +104,11 @@ def mypy_expression_to_sds_type(expr: mp_nodes.Expression) -> sds_types.Abstract elif isinstance(expr, mp_nodes.UnaryExpr): return mypy_expression_to_sds_type(expr.expr) - raise TypeError("Unexpected expression type.") # pragma: no cover + logging.warning( + "Could not parse a parameter or return type for a function: Safe-DS does not support " + "types such as call expressions. Added 'unknown' instead.", + ) + return sds_types.UnknownType() def mypy_expression_to_python_value( diff --git a/src/safeds_stubgen/docstring_parsing/_docstring_parser.py b/src/safeds_stubgen/docstring_parsing/_docstring_parser.py index 2a2f95cc..99660e6f 100644 --- a/src/safeds_stubgen/docstring_parsing/_docstring_parser.py +++ b/src/safeds_stubgen/docstring_parsing/_docstring_parser.py @@ -385,9 +385,6 @@ def _get_griffe_node(self, qname: str) -> Object | None: node_qname_parts = qname.split(".") griffe_node = self.griffe_build for part in node_qname_parts: - if griffe_node.name == part: - continue - if part in griffe_node.modules: griffe_node = griffe_node.modules[part] elif part in griffe_node.classes: @@ -398,11 +395,14 @@ def _get_griffe_node(self, qname: str) -> Object | None: griffe_node = griffe_node.attributes[part] elif part == "__init__" and griffe_node.is_class: return None + elif griffe_node.name == part: + continue else: # pragma: no cover - raise ValueError( + msg = ( f"Something went wrong while searching for the docstring for {qname}. Please make sure" " that all directories with python files have an __init__.py file.", ) + logging.warning(msg) return griffe_node diff --git a/src/safeds_stubgen/stubs_generator/_generate_stubs.py b/src/safeds_stubgen/stubs_generator/_generate_stubs.py index b7e76c0a..09dc213b 100644 --- a/src/safeds_stubgen/stubs_generator/_generate_stubs.py +++ b/src/safeds_stubgen/stubs_generator/_generate_stubs.py @@ -124,6 +124,10 @@ def _create_outside_package_class( created_module_paths: set[str], ) -> set[str]: path_parts = class_path.split(".") + + if len(path_parts) == 1: + return created_module_paths + class_name = path_parts.pop(-1) module_name = path_parts[-1] module_path = "/".join(path_parts) diff --git a/tests/data/various_modules_package/class_module.py b/tests/data/various_modules_package/class_module.py index c53a8966..534f620f 100644 --- a/tests/data/various_modules_package/class_module.py +++ b/tests/data/various_modules_package/class_module.py @@ -8,7 +8,11 @@ class ClassModuleEmptyClassA: class ClassModuleClassB(ClassModuleEmptyClassA): - def __init__(self, a: int, b: ClassModuleEmptyClassA | None): ... + b_attr_1: int + b_attr_2: dict = {} + + def __init__(self, a: int, b: ClassModuleEmptyClassA | None): + self.b_attr_1 = self.b_attr_2['index'] = 0 def f(self): ... @@ -88,3 +92,13 @@ def overloaded_function( parameter_2: bool = True, ) -> bool | None: return None + + +class ClassWithOverloadedFunction2: + @property + def stale(self): + return self._stale + + @stale.setter + def stale(self, val): + self._stale = val diff --git a/tests/data/various_modules_package/enum_module.py b/tests/data/various_modules_package/enum_module.py index 5600555e..c8b8232d 100644 --- a/tests/data/various_modules_package/enum_module.py +++ b/tests/data/various_modules_package/enum_module.py @@ -31,3 +31,20 @@ class EnumTest3(IntEnum): class EmptyEnum(Enum, IntEnum): ... + + +class EnumWithFunctions(str, Enum): + A = "a" + B = "b" + C = "c" + + def __str__(self): + return self.value + + @staticmethod + def enum_function(): + return 1 + 2 + + @property + def _give_b(self): + return EnumWithFunctions.B diff --git a/tests/data/various_modules_package/function_module.py b/tests/data/various_modules_package/function_module.py index 2bc30e9c..8357b76b 100644 --- a/tests/data/various_modules_package/function_module.py +++ b/tests/data/various_modules_package/function_module.py @@ -213,3 +213,17 @@ def property_function_infer(self): def ret_conditional_statement(): return 1 if True else False + + +def ignore_assignment(a: int, b: int): + def _f(x: int, y: int) -> int: + return x + y + + g, f = _f(a, b) + Cxy = _f(g, f)**2 + return Cxy, f + + +def ignore_assignment2(a: int, b: int): + Cxy = 3**2 + return Cxy diff --git a/tests/safeds_stubgen/api_analyzer/__snapshots__/test__get_api.ambr b/tests/safeds_stubgen/api_analyzer/__snapshots__/test__get_api.ambr index c63c4f8f..14b5962c 100644 --- a/tests/safeds_stubgen/api_analyzer/__snapshots__/test__get_api.ambr +++ b/tests/safeds_stubgen/api_analyzer/__snapshots__/test__get_api.ambr @@ -2077,6 +2077,8 @@ # name: test_classes[ClassModuleClassB] dict({ 'attributes': list([ + 'tests/data/various_modules_package/class_module/ClassModuleClassB/b_attr_1', + 'tests/data/various_modules_package/class_module/ClassModuleClassB/b_attr_2', ]), 'classes': list([ ]), @@ -6068,6 +6070,52 @@ 'tests/data/various_modules_package/function_module/float_result/result_1', ]), }), + dict({ + 'docstring': dict({ + 'description': '', + 'examples': list([ + ]), + 'full_docstring': '', + }), + 'id': 'tests/data/various_modules_package/function_module/ignore_assignment', + 'is_class_method': False, + 'is_property': False, + 'is_public': True, + 'is_static': False, + 'name': 'ignore_assignment', + 'parameters': list([ + 'tests/data/various_modules_package/function_module/ignore_assignment/a', + 'tests/data/various_modules_package/function_module/ignore_assignment/b', + ]), + 'reexported_by': list([ + ]), + 'results': list([ + 'tests/data/various_modules_package/function_module/ignore_assignment/result_1', + 'tests/data/various_modules_package/function_module/ignore_assignment/result_2', + ]), + }), + dict({ + 'docstring': dict({ + 'description': '', + 'examples': list([ + ]), + 'full_docstring': '', + }), + 'id': 'tests/data/various_modules_package/function_module/ignore_assignment2', + 'is_class_method': False, + 'is_property': False, + 'is_public': True, + 'is_static': False, + 'name': 'ignore_assignment2', + 'parameters': list([ + 'tests/data/various_modules_package/function_module/ignore_assignment2/a', + 'tests/data/various_modules_package/function_module/ignore_assignment2/b', + ]), + 'reexported_by': list([ + ]), + 'results': list([ + ]), + }), dict({ 'docstring': dict({ 'description': '', @@ -7219,6 +7267,7 @@ 'tests/data/various_modules_package/class_module/SelfTypes1', 'tests/data/various_modules_package/class_module/SelfTypes2', 'tests/data/various_modules_package/class_module/ClassWithOverloadedFunction', + 'tests/data/various_modules_package/class_module/ClassWithOverloadedFunction2', ]), 'docstring': '', 'enums': list([ @@ -7281,6 +7330,7 @@ 'tests/data/various_modules_package/enum_module/EnumTest2', 'tests/data/various_modules_package/enum_module/EnumTest3', 'tests/data/various_modules_package/enum_module/EmptyEnum', + 'tests/data/various_modules_package/enum_module/EnumWithFunctions', ]), 'functions': list([ ]), @@ -7542,6 +7592,8 @@ 'tests/data/various_modules_package/function_module/param_from_outside_the_package', 'tests/data/various_modules_package/function_module/result_from_outside_the_package', 'tests/data/various_modules_package/function_module/ret_conditional_statement', + 'tests/data/various_modules_package/function_module/ignore_assignment', + 'tests/data/various_modules_package/function_module/ignore_assignment2', ]), 'id': 'tests/data/various_modules_package/function_module', 'name': 'function_module', diff --git a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[class_module].sdsstub b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[class_module].sdsstub index 33c78c4d..6fd58651 100644 --- a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[class_module].sdsstub +++ b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[class_module].sdsstub @@ -9,6 +9,11 @@ class ClassModuleClassB( a: Int, b: ClassModuleEmptyClassA? ) sub ClassModuleEmptyClassA { + @PythonName("b_attr_1") + static attr bAttr1: Int + @PythonName("b_attr_2") + static attr bAttr2: Map + // TODO Result type information missing. @Pure fun f() @@ -50,3 +55,14 @@ class ClassWithOverloadedFunction() { @PythonName("parameter_2") parameter2: Boolean = true ) -> result1: Boolean? } + +class ClassWithOverloadedFunction2() { + attr stale + + // TODO Result type information missing. + // TODO Some parameter have no type information. + @Pure + fun stale( + `val` + ) +} diff --git a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[enum_module].sdsstub b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[enum_module].sdsstub index 6a9a9afd..7859f633 100644 --- a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[enum_module].sdsstub +++ b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[enum_module].sdsstub @@ -32,3 +32,9 @@ enum EnumTest3 { } enum EmptyEnum + +enum EnumWithFunctions { + A + B + C +} diff --git a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[function_module].sdsstub b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[function_module].sdsstub index 990e244c..13562faa 100644 --- a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[function_module].sdsstub +++ b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[function_module].sdsstub @@ -264,6 +264,22 @@ fun resultFromOutsideThePackage() -> result1: AnotherClass @PythonName("ret_conditional_statement") fun retConditionalStatement() -> result1: union +// TODO Unknown type - Type could not be parsed. +@Pure +@PythonName("ignore_assignment") +fun ignoreAssignment( + a: Int, + b: Int +) -> (result1: unknown, result2: unknown) + +// TODO Result type information missing. +@Pure +@PythonName("ignore_assignment2") +fun ignoreAssignment2( + a: Int, + b: Int +) + class FunctionModuleClassA() // TODO Some parameter have no type information. From 56c0a3907d16015e60791aebc10efe38769f814d Mon Sep 17 00:00:00 2001 From: Arsam Date: Sat, 17 Aug 2024 21:57:38 +0200 Subject: [PATCH 02/25] Fixing several bugs where return variables cant be parsed correctly --- .../api_analyzer/_ast_visitor.py | 191 ++++++++------ src/safeds_stubgen/api_analyzer/_get_api.py | 1 + .../api_analyzer/_mypy_helpers.py | 29 +- .../stubs_generator/_generate_stubs.py | 6 +- .../various_modules_package/class_module.py | 8 + .../function_module.py | 34 +++ .../__snapshots__/test__get_api.ambr | 247 ++++++++++++++++-- ...n.test_stub_creation[class_module].sdsstub | 7 + ...est_stub_creation[function_module].sdsstub | 62 ++++- ..._stub_creation[infer_types_module].sdsstub | 2 +- 10 files changed, 474 insertions(+), 113 deletions(-) diff --git a/src/safeds_stubgen/api_analyzer/_ast_visitor.py b/src/safeds_stubgen/api_analyzer/_ast_visitor.py index 1d826a4c..fd8f0629 100644 --- a/src/safeds_stubgen/api_analyzer/_ast_visitor.py +++ b/src/safeds_stubgen/api_analyzer/_ast_visitor.py @@ -32,7 +32,7 @@ WildcardImport, ) from ._mypy_helpers import ( - find_return_stmts_recursive, + find_stmts_recursive, get_argument_kind, get_classdef_definitions, get_funcdef_definitions, @@ -58,6 +58,7 @@ def __init__( aliases: dict[str, set[str]], type_source_preference: TypeSourcePreference, type_source_warning: TypeSourceWarning, + is_test_run: bool = False, ) -> None: self.docstring_parser: AbstractDocstringParser = docstring_parser self.type_source_preference = type_source_preference @@ -68,6 +69,7 @@ def __init__( self.mypy_file: mp_nodes.MypyFile | None = None # We gather type var types used as a parameter type in a function self.type_var_types: set[sds_types.TypeVarType] = set() + self.is_test_run = is_test_run def enter_moduledef(self, node: mp_nodes.MypyFile) -> None: self.mypy_file = node @@ -202,8 +204,14 @@ def enter_classdef(self, node: mp_nodes.ClassDef) -> None: ): inherits_from_exception = True - if hasattr(superclass, "fullname") or hasattr(superclass, "name"): - superclass_qname = getattr(superclass, "fullname", "") or getattr(superclass, "name", "") + if hasattr(superclass, "fullname"): + superclass_qname = superclass.fullname + + if not superclass_qname and hasattr(superclass, "name"): + superclass_qname = superclass.name + if hasattr(superclass, "expr") and isinstance(superclass.expr, mp_nodes.NameExpr): + superclass_qname = f"{superclass.expr.name}.{superclass_qname}" + superclass_name = superclass_qname.split(".")[-1] # Check if the superclass name is an alias and find the real name @@ -428,10 +436,10 @@ def enter_assignmentstmt(self, node: mp_nodes.AssignmentStmt) -> None: if hasattr(lvalue, "items"): for item in lvalue.items: names.append(item.name) - else: - if not hasattr(lvalue, "name"): # pragma: no cover - raise AttributeError("Expected lvalue to have attribtue 'name'.") - names.append(getattr(lvalue, "name", "")) + elif hasattr(lvalue, "name"): + names.append(lvalue.name) + else: # pragma: no cover + raise AttributeError("Expected lvalue to have attribtue 'name'.") for name in names: assignments.append( @@ -589,91 +597,122 @@ def _remove_assignments(func_defn: list, type_: AbstractType) -> AbstractType: in the function itself (assignment). If this is not the case, we can assume that they are imported or from outside the funciton. """ + if not isinstance(type_, sds_types.NamedType | sds_types.TupleType): + return type_ + + found_types = type_.types if isinstance(type_, sds_types.TupleType) else [type_] actual_types: list[AbstractType] = [] - if isinstance(type_, sds_types.NamedType | sds_types.TupleType): - if isinstance(type_, sds_types.TupleType): - found_types = type_.types - else: - found_types = [type_] + assignment_stmts = find_stmts_recursive(stmt_type=mp_nodes.AssignmentStmt, stmts=func_defn) - for found_type in found_types: - if isinstance(found_type, sds_types.NamedType): - is_assignment = False - found_type_name = found_type.name + for found_type in found_types: + if not isinstance(found_type, sds_types.NamedType): # pragma: no cover + continue - for defn in func_defn: - if not isinstance(defn, mp_nodes.AssignmentStmt): - continue + is_assignment = False + found_type_name = found_type.name - for lvalue in defn.lvalues: - if isinstance(lvalue, mp_nodes.TupleExpr): - name_expressions = lvalue.items - else: - name_expressions = [lvalue] + for stmt in assignment_stmts: + if not isinstance(stmt, mp_nodes.AssignmentStmt): # pragma: no cover + continue - for expr in name_expressions: - if isinstance(expr, mp_nodes.NameExpr) and found_type_name == expr.name: - is_assignment = True - break - if is_assignment: - break + for lvalue in stmt.lvalues: + name_expressions = lvalue.items if isinstance(lvalue, mp_nodes.TupleExpr) else [lvalue] - if is_assignment: + for expr in name_expressions: + if isinstance(expr, mp_nodes.NameExpr) and found_type_name == expr.name: + is_assignment = True break - if is_assignment: - actual_types.append(sds_types.UnknownType()) - else: - actual_types.append(found_type) + break - if len(actual_types) > 1: - type_ = sds_types.TupleType(types=actual_types) - elif len(actual_types) == 1: - type_ = actual_types[0] + if is_assignment: + break + + if is_assignment: + actual_types.append(sds_types.UnknownType()) else: - type_ = sds_types.UnknownType() - return type_ + actual_types.append(found_type) + + if len(actual_types) > 1: + return sds_types.TupleType(types=actual_types) + elif len(actual_types) == 1: + return actual_types[0] + return sds_types.UnknownType() def _infer_type_from_return_stmts(self, func_node: mp_nodes.FuncDef) -> sds_types.TupleType | None: - # To infer the type, we iterate through all return statements we find in the function + # To infer the possible result types, we iterate through all return statements we find in the function func_defn = get_funcdef_definitions(func_node) - return_stmts = find_return_stmts_recursive(func_defn) - if return_stmts: - types = set() - for return_stmt in return_stmts: - if return_stmt.expr is None: # pragma: no cover + return_stmts = find_stmts_recursive(mp_nodes.ReturnStmt, func_defn) + if not return_stmts: + return None + + types = [] + for return_stmt in return_stmts: + if not isinstance(return_stmt, mp_nodes.ReturnStmt): # pragma: no cover + continue + + if return_stmt.expr is not None and hasattr(return_stmt.expr, "node"): + if isinstance(return_stmt.expr.node, mp_nodes.FuncDef | mp_nodes.Decorator): + # In this case we have an inner function which the outer function returns. + continue + if ( + isinstance(return_stmt.expr.node, mp_nodes.Var) + and hasattr(return_stmt.expr, "name") + and return_stmt.expr.name in func_node.arg_names + and return_stmt.expr.node.type is not None + ): + # In this case the return value is a parameter of the function + type_ = self.mypy_type_to_abstract_type(return_stmt.expr.node.type) + types.append(type_) continue - if not isinstance(return_stmt.expr, mp_nodes.CallExpr | mp_nodes.MemberExpr): + if not isinstance(return_stmt.expr, mp_nodes.CallExpr | mp_nodes.MemberExpr): + if isinstance(return_stmt.expr, mp_nodes.ConditionalExpr): # If the return statement is a conditional expression we parse the "if" and "else" branches - if isinstance(return_stmt.expr, mp_nodes.ConditionalExpr): - for conditional_branch in [return_stmt.expr.if_expr, return_stmt.expr.else_expr]: - if conditional_branch is None: # pragma: no cover - continue - - if not isinstance(conditional_branch, mp_nodes.CallExpr | mp_nodes.MemberExpr): - type_ = mypy_expression_to_sds_type(conditional_branch) - if isinstance(type_, sds_types.NamedType | sds_types.TupleType): - types.add(type_) - elif hasattr(return_stmt.expr, "node") and getattr(return_stmt.expr.node, "is_self", False): - # The result type is an instance of the parent class - expr_type = return_stmt.expr.node.type.type - types.add(sds_types.NamedType(name=expr_type.name, qname=expr_type.fullname)) - else: - type_ = mypy_expression_to_sds_type(return_stmt.expr) - type_ = self._remove_assignments(func_defn, type_) + for cond_branch in [return_stmt.expr.if_expr, return_stmt.expr.else_expr]: + if cond_branch is None: # pragma: no cover + continue - if isinstance(type_, sds_types.NamedType | sds_types.TupleType): - types.add(type_) + if not isinstance(cond_branch, mp_nodes.CallExpr | mp_nodes.MemberExpr): + if ( + hasattr(cond_branch, "node") + and isinstance(cond_branch.node, mp_nodes.Var) + and cond_branch.node.type is not None + ): + # In this case the return value is a parameter of the function + type_ = self.mypy_type_to_abstract_type(cond_branch.node.type) + else: + type_ = mypy_expression_to_sds_type(cond_branch) + types.append(type_) + elif ( + return_stmt.expr is not None + and hasattr(return_stmt.expr, "node") + and getattr(return_stmt.expr.node, "is_self", False) + ): + # The result type is an instance of the parent class + expr_type = return_stmt.expr.node.type.type + types.append(sds_types.NamedType(name=expr_type.name, qname=expr_type.fullname)) + elif isinstance(return_stmt.expr, mp_nodes.TupleExpr): + all_types = [] + for item in return_stmt.expr.items: + if hasattr(item, "node") and isinstance(item.node, mp_nodes.Var) and item.node.type is not None: + # In this case the return value is a parameter of the function + type_ = self.mypy_type_to_abstract_type(item.node.type) + else: + type_ = mypy_expression_to_sds_type(item) + type_ = self._remove_assignments(func_defn, type_) + all_types.append(type_) + types.append(sds_types.TupleType(types=all_types)) + else: + # Lastly, we have a mypy expression object, which we have to parse + if return_stmt.expr is None: # pragma: no cover + continue - # We have to sort the list for the snapshot tests - return_stmt_types = list(types) - return_stmt_types.sort( - key=lambda x: (x.name if isinstance(x, sds_types.NamedType) else str(len(x.types))), - ) + type_ = mypy_expression_to_sds_type(return_stmt.expr) + type_ = self._remove_assignments(func_defn, type_) + types.append(type_) - return sds_types.TupleType(types=return_stmt_types) - return None + return sds_types.TupleType(types=types) @staticmethod def _create_inferred_results( @@ -705,12 +744,12 @@ def _create_inferred_results( result_array: list[list[AbstractType]] = [] longest_inner_list = 1 for type_ in results.types: - if isinstance(type_, sds_types.NamedType): + if not isinstance(type_, sds_types.TupleType): if result_array: result_array[0].append(type_) else: result_array.append([type_]) - elif isinstance(type_, sds_types.TupleType): + else: for i, type__ in enumerate(type_.types): if len(result_array) > i: if type__ not in result_array[i]: @@ -720,8 +759,6 @@ def _create_inferred_results( longest_inner_list = len(result_array[i]) else: result_array.append([type__]) - else: # pragma: no cover - raise TypeError(f"Expected NamedType or TupleType, received {type(type_)}") # If there are any arrays longer than others, these "others" are optional types and can be None none_element = sds_types.NamedType(name="None", qname="builtins.None") diff --git a/src/safeds_stubgen/api_analyzer/_get_api.py b/src/safeds_stubgen/api_analyzer/_get_api.py index 12800645..f03c6abd 100644 --- a/src/safeds_stubgen/api_analyzer/_get_api.py +++ b/src/safeds_stubgen/api_analyzer/_get_api.py @@ -77,6 +77,7 @@ def get_api( aliases=aliases, type_source_preference=type_source_preference, type_source_warning=type_source_warning, + is_test_run=is_test_run, ) walker = ASTWalker(handler=callable_visitor) diff --git a/src/safeds_stubgen/api_analyzer/_mypy_helpers.py b/src/safeds_stubgen/api_analyzer/_mypy_helpers.py index 4b827446..b3d0dc8b 100644 --- a/src/safeds_stubgen/api_analyzer/_mypy_helpers.py +++ b/src/safeds_stubgen/api_analyzer/_mypy_helpers.py @@ -44,26 +44,29 @@ def get_argument_kind(arg: mp_nodes.Argument) -> ParameterAssignment: raise ValueError("Could not find an appropriate parameter assignment.") -def find_return_stmts_recursive(stmts: list[mp_nodes.Statement] | list[mp_nodes.Block]) -> list[mp_nodes.ReturnStmt]: - return_stmts = [] +def find_stmts_recursive( + stmt_type: type[mp_nodes.Statement], + stmts: list[mp_nodes.Statement] | list[mp_nodes.Block], +) -> list[mp_nodes.Statement]: + found_stmts = [] for stmt in stmts: - if isinstance(stmt, mp_nodes.IfStmt): - return_stmts += find_return_stmts_recursive(stmt.body) + if isinstance(stmt, stmt_type): + found_stmts.append(stmt) + elif isinstance(stmt, mp_nodes.IfStmt): + found_stmts += find_stmts_recursive(stmt_type, stmt.body) if stmt.else_body: - return_stmts += find_return_stmts_recursive(stmt.else_body.body) + found_stmts += find_stmts_recursive(stmt_type, stmt.else_body.body) elif isinstance(stmt, mp_nodes.Block): - return_stmts += find_return_stmts_recursive(stmt.body) + found_stmts += find_stmts_recursive(stmt_type, stmt.body) elif isinstance(stmt, mp_nodes.TryStmt): - return_stmts += find_return_stmts_recursive([stmt.body]) - return_stmts += find_return_stmts_recursive(stmt.handlers) + found_stmts += find_stmts_recursive(stmt_type, [stmt.body]) + found_stmts += find_stmts_recursive(stmt_type, stmt.handlers) elif isinstance(stmt, mp_nodes.MatchStmt): - return_stmts += find_return_stmts_recursive(stmt.bodies) + found_stmts += find_stmts_recursive(stmt_type, stmt.bodies) elif isinstance(stmt, mp_nodes.WhileStmt | mp_nodes.WithStmt | mp_nodes.ForStmt): - return_stmts += find_return_stmts_recursive(stmt.body.body) - elif isinstance(stmt, mp_nodes.ReturnStmt): - return_stmts.append(stmt) + found_stmts += find_stmts_recursive(stmt_type, stmt.body.body) - return return_stmts + return found_stmts def mypy_variance_parser(mypy_variance_type: Literal[0, 1, 2]) -> VarianceKind: diff --git a/src/safeds_stubgen/stubs_generator/_generate_stubs.py b/src/safeds_stubgen/stubs_generator/_generate_stubs.py index 09dc213b..a5e7f35d 100644 --- a/src/safeds_stubgen/stubs_generator/_generate_stubs.py +++ b/src/safeds_stubgen/stubs_generator/_generate_stubs.py @@ -125,7 +125,11 @@ def _create_outside_package_class( ) -> set[str]: path_parts = class_path.split(".") - if len(path_parts) == 1: + # There are cases where we could not correctly parse or find the origin of a variable, which is then put into + # the imports. But since these variables have no qname and only consist of a name we cannot create seperate files + # for them. + # E.g.: `x: numpy.some_class; ...; return x` would have the result type parsed as just "numpy" + if len(path_parts) == 1: # pragma: no cover return created_module_paths class_name = path_parts.pop(-1) diff --git a/tests/data/various_modules_package/class_module.py b/tests/data/various_modules_package/class_module.py index 534f620f..ff62a509 100644 --- a/tests/data/various_modules_package/class_module.py +++ b/tests/data/various_modules_package/class_module.py @@ -1,4 +1,5 @@ from typing import Self, overload +from . import unknown_source from tests.data.main_package.another_path.another_module import yetAnotherClass @@ -14,6 +15,9 @@ class ClassModuleClassB(ClassModuleEmptyClassA): def __init__(self, a: int, b: ClassModuleEmptyClassA | None): self.b_attr_1 = self.b_attr_2['index'] = 0 + def __enter__(self): + return self + def f(self): ... @@ -102,3 +106,7 @@ def stale(self): @stale.setter def stale(self, val): self._stale = val + + +class ClassWithImportedSuperclasses(unknown_source.UnknownClass): + pass diff --git a/tests/data/various_modules_package/function_module.py b/tests/data/various_modules_package/function_module.py index 8357b76b..85c5b9eb 100644 --- a/tests/data/various_modules_package/function_module.py +++ b/tests/data/various_modules_package/function_module.py @@ -227,3 +227,37 @@ def _f(x: int, y: int) -> int: def ignore_assignment2(a: int, b: int): Cxy = 3**2 return Cxy + + +def ignore_assignment3(xys, p): + return ((p - xys[0]) ** 2).sum(), xys[0], (0, 0) + + +def ignore_assignment4(a, b, c): + return ignore_assignment3(a, a), (ignore_assignment2(b, a), ignore_assignment(c, a)) + + +def return_inner_function(): + def return_me(): + return 123 + + return return_me + + +def return_param1(a): + return a + + +def return_param2(a: int): + return a + + +def return_param3(a: int, b, c: bool): + return a if b else c + + +def return_param4(a: int, b, x): + if x == 0: + return a, b, a, b + + return True diff --git a/tests/safeds_stubgen/api_analyzer/__snapshots__/test__get_api.ambr b/tests/safeds_stubgen/api_analyzer/__snapshots__/test__get_api.ambr index 14b5962c..e06f14de 100644 --- a/tests/safeds_stubgen/api_analyzer/__snapshots__/test__get_api.ambr +++ b/tests/safeds_stubgen/api_analyzer/__snapshots__/test__get_api.ambr @@ -1215,6 +1215,28 @@ # --- # name: test_class_methods[ClassModuleClassB] list([ + dict({ + 'docstring': dict({ + 'description': '', + 'examples': list([ + ]), + 'full_docstring': '', + }), + 'id': 'tests/data/various_modules_package/class_module/ClassModuleClassB/__enter__', + 'is_class_method': False, + 'is_property': False, + 'is_public': True, + 'is_static': False, + 'name': '__enter__', + 'parameters': list([ + 'tests/data/various_modules_package/class_module/ClassModuleClassB/__enter__/self', + ]), + 'reexported_by': list([ + ]), + 'results': list([ + 'tests/data/various_modules_package/class_module/ClassModuleClassB/__enter__/result_1', + ]), + }), dict({ 'docstring': dict({ 'description': '', @@ -2115,6 +2137,7 @@ 'inherits_from_exception': False, 'is_public': True, 'methods': list([ + 'tests/data/various_modules_package/class_module/ClassModuleClassB/__enter__', 'tests/data/various_modules_package/class_module/ClassModuleClassB/f', ]), 'name': 'ClassModuleClassB', @@ -5216,48 +5239,53 @@ }), dict({ 'kind': 'NamedType', - 'name': 'float', - 'qname': 'builtins.float', + 'name': 'int', + 'qname': 'builtins.int', }), dict({ 'kind': 'NamedType', - 'name': 'InferMe', - 'qname': 'tests.data.various_modules_package.infer_types_module.InferMe', + 'name': 'bool', + 'qname': 'builtins.bool', }), dict({ 'kind': 'NamedType', - 'name': 'InferMe2', - 'qname': 'tests.data.various_modules_package.infer_types_module.InferMe2', + 'name': 'InferMyTypes', + 'qname': 'tests.data.various_modules_package.infer_types_module.InferMyTypes', }), dict({ 'kind': 'NamedType', - 'name': 'InferMe3', - 'qname': 'tests.data.various_modules_package.infer_types_module.InferMe3', + 'name': 'None', + 'qname': 'builtins.None', }), dict({ 'kind': 'NamedType', - 'name': 'InferMyTypes', - 'qname': 'tests.data.various_modules_package.infer_types_module.InferMyTypes', + 'name': 'float', + 'qname': 'builtins.float', }), dict({ 'kind': 'NamedType', - 'name': 'None', - 'qname': 'builtins.None', + 'name': 'str', + 'qname': 'builtins.str', }), dict({ 'kind': 'NamedType', - 'name': 'bool', - 'qname': 'builtins.bool', + 'name': 'InferMe', + 'qname': 'tests.data.various_modules_package.infer_types_module.InferMe', }), dict({ 'kind': 'NamedType', - 'name': 'int', - 'qname': 'builtins.int', + 'name': 'InferMe2', + 'qname': 'tests.data.various_modules_package.infer_types_module.InferMe2', }), dict({ 'kind': 'NamedType', - 'name': 'str', - 'qname': 'builtins.str', + 'name': 'InferMe3', + 'qname': 'tests.data.various_modules_package.infer_types_module.InferMe3', + }), + dict({ + 'kind': 'NamedType', + 'name': 'int', + 'qname': 'builtins.int', }), ]), }), @@ -5278,6 +5306,11 @@ 'name': 'float', 'qname': 'builtins.float', }), + dict({ + 'kind': 'NamedType', + 'name': 'None', + 'qname': 'builtins.None', + }), ]), }), }), @@ -6114,6 +6147,57 @@ 'reexported_by': list([ ]), 'results': list([ + 'tests/data/various_modules_package/function_module/ignore_assignment2/result_1', + ]), + }), + dict({ + 'docstring': dict({ + 'description': '', + 'examples': list([ + ]), + 'full_docstring': '', + }), + 'id': 'tests/data/various_modules_package/function_module/ignore_assignment3', + 'is_class_method': False, + 'is_property': False, + 'is_public': True, + 'is_static': False, + 'name': 'ignore_assignment3', + 'parameters': list([ + 'tests/data/various_modules_package/function_module/ignore_assignment3/xys', + 'tests/data/various_modules_package/function_module/ignore_assignment3/p', + ]), + 'reexported_by': list([ + ]), + 'results': list([ + 'tests/data/various_modules_package/function_module/ignore_assignment3/result_1', + 'tests/data/various_modules_package/function_module/ignore_assignment3/result_2', + 'tests/data/various_modules_package/function_module/ignore_assignment3/result_3', + ]), + }), + dict({ + 'docstring': dict({ + 'description': '', + 'examples': list([ + ]), + 'full_docstring': '', + }), + 'id': 'tests/data/various_modules_package/function_module/ignore_assignment4', + 'is_class_method': False, + 'is_property': False, + 'is_public': True, + 'is_static': False, + 'name': 'ignore_assignment4', + 'parameters': list([ + 'tests/data/various_modules_package/function_module/ignore_assignment4/a', + 'tests/data/various_modules_package/function_module/ignore_assignment4/b', + 'tests/data/various_modules_package/function_module/ignore_assignment4/c', + ]), + 'reexported_by': list([ + ]), + 'results': list([ + 'tests/data/various_modules_package/function_module/ignore_assignment4/result_1', + 'tests/data/various_modules_package/function_module/ignore_assignment4/result_2', ]), }), dict({ @@ -6568,6 +6652,121 @@ 'tests/data/various_modules_package/function_module/ret_conditional_statement/result_1', ]), }), + dict({ + 'docstring': dict({ + 'description': '', + 'examples': list([ + ]), + 'full_docstring': '', + }), + 'id': 'tests/data/various_modules_package/function_module/return_inner_function', + 'is_class_method': False, + 'is_property': False, + 'is_public': True, + 'is_static': False, + 'name': 'return_inner_function', + 'parameters': list([ + ]), + 'reexported_by': list([ + ]), + 'results': list([ + ]), + }), + dict({ + 'docstring': dict({ + 'description': '', + 'examples': list([ + ]), + 'full_docstring': '', + }), + 'id': 'tests/data/various_modules_package/function_module/return_param1', + 'is_class_method': False, + 'is_property': False, + 'is_public': True, + 'is_static': False, + 'name': 'return_param1', + 'parameters': list([ + 'tests/data/various_modules_package/function_module/return_param1/a', + ]), + 'reexported_by': list([ + ]), + 'results': list([ + 'tests/data/various_modules_package/function_module/return_param1/result_1', + ]), + }), + dict({ + 'docstring': dict({ + 'description': '', + 'examples': list([ + ]), + 'full_docstring': '', + }), + 'id': 'tests/data/various_modules_package/function_module/return_param2', + 'is_class_method': False, + 'is_property': False, + 'is_public': True, + 'is_static': False, + 'name': 'return_param2', + 'parameters': list([ + 'tests/data/various_modules_package/function_module/return_param2/a', + ]), + 'reexported_by': list([ + ]), + 'results': list([ + 'tests/data/various_modules_package/function_module/return_param2/result_1', + ]), + }), + dict({ + 'docstring': dict({ + 'description': '', + 'examples': list([ + ]), + 'full_docstring': '', + }), + 'id': 'tests/data/various_modules_package/function_module/return_param3', + 'is_class_method': False, + 'is_property': False, + 'is_public': True, + 'is_static': False, + 'name': 'return_param3', + 'parameters': list([ + 'tests/data/various_modules_package/function_module/return_param3/a', + 'tests/data/various_modules_package/function_module/return_param3/b', + 'tests/data/various_modules_package/function_module/return_param3/c', + ]), + 'reexported_by': list([ + ]), + 'results': list([ + 'tests/data/various_modules_package/function_module/return_param3/result_1', + ]), + }), + dict({ + 'docstring': dict({ + 'description': '', + 'examples': list([ + ]), + 'full_docstring': '', + }), + 'id': 'tests/data/various_modules_package/function_module/return_param4', + 'is_class_method': False, + 'is_property': False, + 'is_public': True, + 'is_static': False, + 'name': 'return_param4', + 'parameters': list([ + 'tests/data/various_modules_package/function_module/return_param4/a', + 'tests/data/various_modules_package/function_module/return_param4/b', + 'tests/data/various_modules_package/function_module/return_param4/x', + ]), + 'reexported_by': list([ + ]), + 'results': list([ + 'tests/data/various_modules_package/function_module/return_param4/result_1', + 'tests/data/various_modules_package/function_module/return_param4/result_2', + 'tests/data/various_modules_package/function_module/return_param4/result_3', + 'tests/data/various_modules_package/function_module/return_param4/result_4', + ]), + }), dict({ 'docstring': dict({ 'description': '', @@ -7268,6 +7467,7 @@ 'tests/data/various_modules_package/class_module/SelfTypes2', 'tests/data/various_modules_package/class_module/ClassWithOverloadedFunction', 'tests/data/various_modules_package/class_module/ClassWithOverloadedFunction2', + 'tests/data/various_modules_package/class_module/ClassWithImportedSuperclasses', ]), 'docstring': '', 'enums': list([ @@ -7285,6 +7485,10 @@ 'alias': None, 'qualified_name': 'typing.overload', }), + dict({ + 'alias': None, + 'qualified_name': 'unknown_source', + }), dict({ 'alias': None, 'qualified_name': 'tests.data.main_package.another_path.another_module.yetAnotherClass', @@ -7594,6 +7798,13 @@ 'tests/data/various_modules_package/function_module/ret_conditional_statement', 'tests/data/various_modules_package/function_module/ignore_assignment', 'tests/data/various_modules_package/function_module/ignore_assignment2', + 'tests/data/various_modules_package/function_module/ignore_assignment3', + 'tests/data/various_modules_package/function_module/ignore_assignment4', + 'tests/data/various_modules_package/function_module/return_inner_function', + 'tests/data/various_modules_package/function_module/return_param1', + 'tests/data/various_modules_package/function_module/return_param2', + 'tests/data/various_modules_package/function_module/return_param3', + 'tests/data/various_modules_package/function_module/return_param4', ]), 'id': 'tests/data/various_modules_package/function_module', 'name': 'function_module', diff --git a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[class_module].sdsstub b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[class_module].sdsstub index 6fd58651..23a94f24 100644 --- a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[class_module].sdsstub +++ b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[class_module].sdsstub @@ -2,6 +2,7 @@ package tests.data.variousModulesPackage.classModule from tests.data.mainPackage.anotherPath.anotherModule import yetAnotherClass +from unknownSource import UnknownClass class ClassModuleEmptyClassA() @@ -14,6 +15,10 @@ class ClassModuleClassB( @PythonName("b_attr_2") static attr bAttr2: Map + @Pure + @PythonName("__enter__") + fun enter() -> result1: ClassModuleClassB + // TODO Result type information missing. @Pure fun f() @@ -66,3 +71,5 @@ class ClassWithOverloadedFunction2() { `val` ) } + +class ClassWithImportedSuperclasses() sub UnknownClass diff --git a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[function_module].sdsstub b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[function_module].sdsstub index 13562faa..9c71f348 100644 --- a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[function_module].sdsstub +++ b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[function_module].sdsstub @@ -270,15 +270,71 @@ fun retConditionalStatement() -> result1: union fun ignoreAssignment( a: Int, b: Int -) -> (result1: unknown, result2: unknown) +) -> (result1: Int, result2: unknown) -// TODO Result type information missing. +// TODO Unknown type - Type could not be parsed. @Pure @PythonName("ignore_assignment2") fun ignoreAssignment2( a: Int, b: Int -) +) -> result1: unknown + +// TODO Safe-DS does not support tuple types. +// TODO Some parameter have no type information. +// TODO Unknown type - Type could not be parsed. +@Pure +@PythonName("ignore_assignment3") +fun ignoreAssignment3( + xys, + p +) -> (result1: unknown, result2: unknown, result3: Tuple) + +// TODO Some parameter have no type information. +// TODO Unknown type - Type could not be parsed. +@Pure +@PythonName("ignore_assignment4") +fun ignoreAssignment4( + a, + b, + c +) -> (result1: unknown, result2: unknown) + +// TODO Result type information missing. +@Pure +@PythonName("return_inner_function") +fun returnInnerFunction() + +// TODO Some parameter have no type information. +@Pure +@PythonName("return_param1") +fun returnParam1( + a +) -> result1: Any + +@Pure +@PythonName("return_param2") +fun returnParam2( + a: Int +) -> result1: Int + +// TODO Some parameter have no type information. +@Pure +@PythonName("return_param3") +fun returnParam3( + a: Int, + b, + c: Boolean +) -> result1: union + +// TODO Some parameter have no type information. +@Pure +@PythonName("return_param4") +fun returnParam4( + a: Int, + b, + x +) -> (result1: union, result2: Any, result3: Int, result4: Any) class FunctionModuleClassA() diff --git a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[infer_types_module].sdsstub b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[infer_types_module].sdsstub index 07ac05f5..e3a67c14 100644 --- a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[infer_types_module].sdsstub +++ b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[infer_types_module].sdsstub @@ -51,7 +51,7 @@ class InferMyTypes( static fun inferFunction( @PythonName("infer_param") inferParam: Int = 1, @PythonName("infer_param_2") inferParam2: Int = "Something" - ) -> (result1: union, result2: union, result3: Float?) + ) -> (result1: union, result2: union, result3: Float?) /** * Test for inferring results with just one possible result, and not a tuple of results. From f53d93b9d53a0a531c2c070ea8ef785e86a734a1 Mon Sep 17 00:00:00 2001 From: Arsam Date: Sun, 18 Aug 2024 17:42:36 +0200 Subject: [PATCH 03/25] Removed unused EnumType and BoundaryType; Added short Docstring description to a lot of functions and methods. --- src/safeds_stubgen/_helpers.py | 1 + src/safeds_stubgen/api_analyzer/__init__.py | 4 - .../api_analyzer/_ast_visitor.py | 32 +- src/safeds_stubgen/api_analyzer/_get_api.py | 15 + .../api_analyzer/_mypy_helpers.py | 13 +- src/safeds_stubgen/api_analyzer/_types.py | 355 +----------------- .../docstring_parsing/_docstring_parser.py | 1 + .../stubs_generator/_generate_stubs.py | 5 + .../safeds_stubgen/api_analyzer/test_types.py | 334 +--------------- 9 files changed, 62 insertions(+), 698 deletions(-) diff --git a/src/safeds_stubgen/_helpers.py b/src/safeds_stubgen/_helpers.py index 7ec8ff05..087d9ba7 100644 --- a/src/safeds_stubgen/_helpers.py +++ b/src/safeds_stubgen/_helpers.py @@ -1,2 +1,3 @@ def is_internal(name: str) -> bool: + """Check if a function / method / class name indicate if it's internal.""" return name.startswith("_") diff --git a/src/safeds_stubgen/api_analyzer/__init__.py b/src/safeds_stubgen/api_analyzer/__init__.py index ed98497d..9dce02ba 100644 --- a/src/safeds_stubgen/api_analyzer/__init__.py +++ b/src/safeds_stubgen/api_analyzer/__init__.py @@ -24,10 +24,8 @@ from ._type_source_enums import TypeSourcePreference, TypeSourceWarning from ._types import ( AbstractType, - BoundaryType, CallableType, DictType, - EnumType, FinalType, ListType, LiteralType, @@ -44,14 +42,12 @@ "AbstractType", "API", "Attribute", - "BoundaryType", "CallableType", "Class", "DictType", "distribution", "distribution_version", "Enum", - "EnumType", "FinalType", "Function", "get_api", diff --git a/src/safeds_stubgen/api_analyzer/_ast_visitor.py b/src/safeds_stubgen/api_analyzer/_ast_visitor.py index fd8f0629..3c017c33 100644 --- a/src/safeds_stubgen/api_analyzer/_ast_visitor.py +++ b/src/safeds_stubgen/api_analyzer/_ast_visitor.py @@ -495,6 +495,7 @@ def _parse_results( function_id: str, result_docstrings: list[ResultDocstring], ) -> list[Result]: + """Parse the results from given Mypy function nodes and return our own Result objects.""" # __init__ functions aren't supposed to have returns, so we can ignore them if node.name == "__init__": return [] @@ -640,6 +641,7 @@ def _remove_assignments(func_defn: list, type_: AbstractType) -> AbstractType: return sds_types.UnknownType() def _infer_type_from_return_stmts(self, func_node: mp_nodes.FuncDef) -> sds_types.TupleType | None: + """Infer the type of the return statements.""" # To infer the possible result types, we iterate through all return statements we find in the function func_defn = get_funcdef_definitions(func_node) return_stmts = find_stmts_recursive(mp_nodes.ReturnStmt, func_defn) @@ -823,6 +825,7 @@ def _parse_attributes( unanalyzed_type: mp_types.Type | None, is_static: bool = True, ) -> list[Attribute]: + """Parse the attributes from given Mypy expressions and return our own Attribute objects.""" assert isinstance(lvalue, mp_nodes.NameExpr | mp_nodes.MemberExpr | mp_nodes.TupleExpr) attributes: list[Attribute] = [] @@ -850,6 +853,7 @@ def _parse_attributes( return attributes def _is_attribute_already_defined(self, value_name: str) -> bool: + """Check our already created Attribute objects if we already defined the given attribute name.""" # If node is None, it's possible that the attribute was already defined once parent = self.__declaration_stack[-1] if isinstance(parent, Function): @@ -866,6 +870,7 @@ def _create_attribute( unanalyzed_type: mp_types.Type | None, is_static: bool, ) -> Attribute: + """Create an Attribute object from a Mypy expression.""" # Get node information type_: sds_types.AbstractType | None = None node = None @@ -950,6 +955,7 @@ def _create_attribute( # #### Parameter utilities def _parse_parameter_data(self, node: mp_nodes.FuncDef, function_id: str) -> list[Parameter]: + """Parse the parameter from a Mypy Function node and return our own Parameter objects.""" arguments: list[Parameter] = [] for argument in node.arguments: @@ -1018,6 +1024,7 @@ def _get_parameter_type_and_default_value( initializer: mp_nodes.Expression, function_id: str, ) -> tuple[str | None | int | float | UnknownValue, bool]: + """Parse the parameter type and default value from a Mypy node expression.""" default_value: str | None | int | float = None default_is_none = False if initializer is not None: @@ -1067,6 +1074,7 @@ def _get_parameter_type_and_default_value( # #### Reexport utilities def _get_reexported_by(self, qname: str) -> list[Module]: + """Get all __init__ modules where a given function / class / enum was reexported.""" path = qname.split(".") # Check if there is a reexport entry for each item in the path to the current module @@ -1074,22 +1082,23 @@ def _get_reexported_by(self, qname: str) -> list[Module]: for i in range(len(path)): reexport_name_forward = ".".join(path[: i + 1]) if reexport_name_forward in self.api.reexport_map: - for mod in self.api.reexport_map[reexport_name_forward]: - reexported_by.add(mod) + for module in self.api.reexport_map[reexport_name_forward]: + reexported_by.add(module) reexport_name_backward = ".".join(path[-i - 1 :]) if reexport_name_backward in self.api.reexport_map: - for mod in self.api.reexport_map[reexport_name_backward]: - reexported_by.add(mod) + for module in self.api.reexport_map[reexport_name_backward]: + reexported_by.add(module) reexport_name_backward_whitelist = f"{'.'.join(path[-2 - i:-1])}.*" if reexport_name_backward_whitelist in self.api.reexport_map: - for mod in self.api.reexport_map[reexport_name_backward_whitelist]: - reexported_by.add(mod) + for module in self.api.reexport_map[reexport_name_backward_whitelist]: + reexported_by.add(module) return list(reexported_by) def _add_reexports(self, module: Module) -> None: + """Add all reexports of an __init__ module to the reexport_map.""" for qualified_import in module.qualified_imports: name = qualified_import.qualified_name self.api.reexport_map[name].add(module) @@ -1104,7 +1113,7 @@ def mypy_type_to_abstract_type( mypy_type: mp_types.Instance | mp_types.ProperType | mp_types.Type, unanalyzed_type: mp_types.Type | None = None, ) -> AbstractType: - + """Convert Mypy types to our AbstractType objects.""" # Special cases where we need the unanalyzed_type to get the type information we need if unanalyzed_type is not None and hasattr(unanalyzed_type, "name"): unanalyzed_type_name = unanalyzed_type.name @@ -1242,6 +1251,7 @@ def mypy_type_to_abstract_type( return sds_types.UnknownType() # pragma: no cover def _find_alias(self, type_name: str) -> tuple[str, str]: + """Try to resolve the alias name by searching for it.""" module = self.__declaration_stack[0] # At this point, the first item of the stack can only ever be a module @@ -1280,6 +1290,7 @@ def _search_alias_in_qualified_imports( qualified_imports: list[QualifiedImport], alias_name: str, ) -> tuple[str, str]: + """Try to resolve the alias name by searching for it in the qualified imports.""" for qualified_import in qualified_imports: if alias_name in {qualified_import.alias, qualified_import.qualified_name.split(".")[-1]}: qname = qualified_import.qualified_name @@ -1288,6 +1299,7 @@ def _search_alias_in_qualified_imports( return "", "" def _is_public(self, name: str, qname: str) -> bool: + """Check if a function / method / class / enum is public.""" if self.mypy_file is None: # pragma: no cover raise ValueError("A Mypy file (module) should be defined.") @@ -1341,12 +1353,18 @@ def _create_id_from_stack(self, name: str) -> str: return "/".join(segments) def _inherits_from_exception(self, node: mp_nodes.TypeInfo) -> bool: + """Check if a class inherits from the Exception class.""" if node.fullname == "builtins.Exception": return True return any(self._inherits_from_exception(base.type) for base in node.bases) def _check_publicity_in_reexports(self, name: str, qname: str, parent: Module | Class) -> bool | None: + """Check if an internal function was made public. + + This can happen if either the function was reexported with a public alias or by its internal parent being + reexported and being made public. + """ not_internal = not is_internal(name) module_qname = getattr(self.mypy_file, "fullname", "") module_name = getattr(self.mypy_file, "name", "") diff --git a/src/safeds_stubgen/api_analyzer/_get_api.py b/src/safeds_stubgen/api_analyzer/_get_api.py index f03c6abd..28522eba 100644 --- a/src/safeds_stubgen/api_analyzer/_get_api.py +++ b/src/safeds_stubgen/api_analyzer/_get_api.py @@ -28,6 +28,7 @@ def get_api( type_source_preference: TypeSourcePreference = TypeSourcePreference.CODE, type_source_warning: TypeSourceWarning = TypeSourceWarning.WARN, ) -> API: + """Parse a given code package with Mypy, walk the Mypy AST and create an API object.""" init_roots = _get_nearest_init_dirs(root) if len(init_roots) == 1: root = init_roots[0] @@ -88,6 +89,11 @@ def get_api( def _get_nearest_init_dirs(root: Path) -> list[Path]: + """Check for the nearest directory with an __init__.py file. + + For the Mypy parser we need to start at a directory with an __init__.py file. Directories without __init__.py files + will be skipped py Mypy. + """ all_inits = list(root.glob("./**/__init__.py")) shortest_init_paths = [] shortest_len = -1 @@ -125,6 +131,10 @@ def _get_mypy_asts( files: list[str], package_paths: list[str], ) -> list[mypy_nodes.MypyFile]: + """Get all module ASTs from Mypy. + + We have to return the package ASTs first though, b/c we need to parse all reexports first. + """ package_ast = [] module_ast = [] for graph_key in build_result.graph: @@ -145,6 +155,11 @@ def _get_mypy_asts( def _get_aliases(result_types: dict, package_name: str) -> dict[str, set[str]]: + """Get the needed aliases from Mypy. + + Mypy has a long list of all aliases it has found. We have to parse the list and get only the aliases we need for our + package we analyze. + """ aliases: dict[str, set[str]] = defaultdict(set) for key in result_types: if isinstance(key, mypy_nodes.NameExpr | mypy_nodes.MemberExpr | mypy_nodes.TypeVarExpr): diff --git a/src/safeds_stubgen/api_analyzer/_mypy_helpers.py b/src/safeds_stubgen/api_analyzer/_mypy_helpers.py index b3d0dc8b..7e19bf96 100644 --- a/src/safeds_stubgen/api_analyzer/_mypy_helpers.py +++ b/src/safeds_stubgen/api_analyzer/_mypy_helpers.py @@ -16,18 +16,22 @@ def get_classdef_definitions(node: ClassDef) -> list: + """Return the objects withhin a Mypy class node.""" return node.defs.body def get_funcdef_definitions(node: FuncDef) -> list: + """Return the objects withhin a Mypy function node.""" return node.body.body def get_mypyfile_definitions(node: MypyFile) -> list: + """Return the objects withhin a Mypy module node.""" return node.defs def get_argument_kind(arg: mp_nodes.Argument) -> ParameterAssignment: + """Translate a Mypy argument kind.""" if arg.variable.is_self or arg.variable.is_cls: return ParameterAssignment.IMPLICIT elif arg.kind in {ArgKind.ARG_POS, ArgKind.ARG_OPT} and arg.pos_only: @@ -48,6 +52,7 @@ def find_stmts_recursive( stmt_type: type[mp_nodes.Statement], stmts: list[mp_nodes.Statement] | list[mp_nodes.Block], ) -> list[mp_nodes.Statement]: + """Try to find all statements of a specific type in a Mypy node.""" found_stmts = [] for stmt in stmts: if isinstance(stmt, stmt_type): @@ -70,6 +75,7 @@ def find_stmts_recursive( def mypy_variance_parser(mypy_variance_type: Literal[0, 1, 2]) -> VarianceKind: + """Translate the Mypy variance ID to our VarianceKind object.""" match mypy_variance_type: case 0: return VarianceKind.INVARIANT @@ -82,7 +88,10 @@ def mypy_variance_parser(mypy_variance_type: Literal[0, 1, 2]) -> VarianceKind: def has_correct_type_of_any(type_of_any: int) -> bool: - # In Mypy AnyType can be set as type because of different reasons (see TypeOfAny class-documentation) + """Check if a given type of the Mypy AnyType is actually an "Any" type for our use-case. + + In Mypy AnyType can be set as type because of different reasons (see Mypy TypeOfAny class-documentation). + """ return type_of_any in { mp_types.TypeOfAny.explicit, mp_types.TypeOfAny.from_another_any, @@ -91,6 +100,7 @@ def has_correct_type_of_any(type_of_any: int) -> bool: def mypy_expression_to_sds_type(expr: mp_nodes.Expression) -> sds_types.AbstractType: + """Translate a Mypy expression to a Safe-DS type.""" if isinstance(expr, mp_nodes.NameExpr): if expr.name in {"False", "True"}: return sds_types.NamedType(name="bool", qname="builtins.bool") @@ -117,6 +127,7 @@ def mypy_expression_to_sds_type(expr: mp_nodes.Expression) -> sds_types.Abstract def mypy_expression_to_python_value( expr: mp_nodes.IntExpr | mp_nodes.FloatExpr | mp_nodes.StrExpr | mp_nodes.NameExpr, ) -> str | None | int | float: + """Translate a Mypy expression to a Python value.""" if isinstance(expr, mp_nodes.NameExpr): match expr.name: case "None": diff --git a/src/safeds_stubgen/api_analyzer/_types.py b/src/safeds_stubgen/api_analyzer/_types.py index 57595af4..655e6530 100644 --- a/src/safeds_stubgen/api_analyzer/_types.py +++ b/src/safeds_stubgen/api_analyzer/_types.py @@ -1,10 +1,9 @@ from __future__ import annotations -import re from abc import ABCMeta, abstractmethod from collections import Counter -from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any, ClassVar +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any if TYPE_CHECKING: from collections.abc import Sequence @@ -20,10 +19,6 @@ def from_dict(cls, d: dict[str, Any]) -> AbstractType: return NamedType.from_dict(d) case NamedSequenceType.__name__: return NamedSequenceType.from_dict(d) - case EnumType.__name__: - return EnumType.from_dict(d) - case BoundaryType.__name__: - return BoundaryType.from_dict(d) case ListType.__name__: return ListType.from_dict(d) case DictType.__name__: @@ -117,162 +112,6 @@ def __hash__(self) -> int: return hash(frozenset([self.name, self.qname, *self.types])) -@dataclass(frozen=True) -class EnumType(AbstractType): - values: frozenset[str] = field(default_factory=frozenset) - full_match: str = field(default="", compare=False) - - @classmethod - def from_dict(cls, d: dict[str, Any]) -> EnumType: - return EnumType(d["values"]) - - @classmethod - def from_string(cls, string: str) -> EnumType | None: - def remove_backslash(e: str) -> str: - e = e.replace(r"\"", '"') - return e.replace(r"\'", "'") - - enum_match = re.search(r"{(.*?)}", string) - if enum_match: - quotes = "'\"" - values = set() - enum_str = enum_match.group(1) - value = "" - inside_value = False - curr_quote = None - for i, char in enumerate(enum_str): - if char in quotes and (i == 0 or (i > 0 and enum_str[i - 1] != "\\")): - if not inside_value: - inside_value = True - curr_quote = char - elif inside_value: - if curr_quote == char: - inside_value = False - curr_quote = None - values.add(remove_backslash(value)) - value = "" - else: - value += char - elif inside_value: - value += char - - return EnumType(frozenset(values), enum_match.group(0)) - - return None - - def update(self, enum: EnumType) -> EnumType: - values = set(self.values) - values.update(enum.values) - return EnumType(frozenset(values)) - - def to_dict(self) -> dict[str, Any]: - return {"kind": self.__class__.__name__, "values": set(self.values)} - - -@dataclass(frozen=True) -class BoundaryType(AbstractType): - NEGATIVE_INFINITY: ClassVar = "NegativeInfinity" - INFINITY: ClassVar = "Infinity" - - base_type: str - min: float | int | str - max: float | int | str - min_inclusive: bool - max_inclusive: bool - - full_match: str = field(default="", compare=False) - - @classmethod - def _is_inclusive(cls, bracket: str) -> bool: - if bracket in ("(", ")"): - return False - if bracket in ("[", "]"): - return True - raise ValueError(f"{bracket} is not one of []()") - - @classmethod - def from_dict(cls, d: dict[str, Any]) -> BoundaryType: - return BoundaryType( - d["base_type"], - d["min"], - d["max"], - d["min_inclusive"], - d["max_inclusive"], - ) - - @classmethod - def from_string(cls, string: str) -> BoundaryType | None: - pattern = r"""(?Pfloat|int)?[ ] # optional base type of either float or int - (in|of)[ ](the[ ])?(range|interval)[ ](of[ ])? - # 'in' or 'of', optional 'the', 'range' or 'interval', optional 'of' - `?(?P[\[(])(?P[-+]?\d+(.\d*)?|negative_infinity),[ ] # left side of the range - (?P[-+]?\d+(.\d*)?|infinity)(?P[\])])`?""" # right side of the range - match = re.search(pattern, string, re.VERBOSE) - - if match is not None: - base_type = match.group("base_type") - if base_type is None: - base_type = "float" - - min_value: str | int | float = match.group("min") - if min_value != "negative_infinity": - if base_type == "int": - min_value = int(min_value) - else: - min_value = float(min_value) - else: - min_value = BoundaryType.NEGATIVE_INFINITY - - max_value: str | int | float = match.group("max") - if max_value != "infinity": - if base_type == "int": - max_value = int(max_value) - else: - max_value = float(max_value) - else: - max_value = BoundaryType.INFINITY - - min_bracket = match.group("min_bracket") - max_bracket = match.group("max_bracket") - min_inclusive = BoundaryType._is_inclusive(min_bracket) - max_inclusive = BoundaryType._is_inclusive(max_bracket) - - return BoundaryType( - base_type=base_type, - min=min_value, - max=max_value, - min_inclusive=min_inclusive, - max_inclusive=max_inclusive, - full_match=match.group(0), - ) - - return None - - def __eq__(self, __o: object) -> bool: - if isinstance(__o, BoundaryType): - eq = ( - self.base_type == __o.base_type - and self.min == __o.min - and self.min_inclusive == __o.min_inclusive - and self.max == __o.max - ) - if eq: - if self.max == BoundaryType.INFINITY: - return True - return self.max_inclusive == __o.max_inclusive - return False - - def to_dict(self) -> dict[str, Any]: - return { - "kind": self.__class__.__name__, - "base_type": self.base_type, - "min": self.min, - "max": self.max, - "min_inclusive": self.min_inclusive, - "max_inclusive": self.max_inclusive, - } - - @dataclass(frozen=True) class UnionType(AbstractType): types: Sequence[AbstractType] @@ -488,193 +327,3 @@ def to_dict(self) -> dict[str, Any]: def __hash__(self) -> int: return hash(frozenset([self.name, self.upper_bound])) - - -# ############################## Utilities ############################## # -# def _dismantel_type_string_structure(type_structure: str) -> list: -# current_type = "" -# result = [] -# -# while True: -# i = 0 -# for i, char in enumerate(type_structure): -# if char == "[": -# try: -# brackets_content, remaining_content = _parse_type_string_bracket_content(type_structure[i + 1:]) -# except TypeParsingError as parsing_error: -# raise TypeParsingError( -# f"Missing brackets in the following string: \n{type_structure}") from parsing_error -# -# result.append(current_type + "[" + brackets_content + "]") -# type_structure = remaining_content -# current_type = "" -# break -# elif char == ",": -# if current_type: -# result.append(current_type) -# current_type = "" -# else: -# current_type += char -# -# if len(type_structure) == 0 or i + 1 == len(type_structure): -# break -# -# if current_type: -# result.append(current_type) -# -# return result -# -# -# def _parse_type_string_bracket_content(substring: str) -> tuple[str, str]: -# brackets_content = "" -# bracket_count = 0 -# for i, char in enumerate(substring): -# if char == "[": -# bracket_count += 1 -# elif char == "]" and bracket_count: -# bracket_count -= 1 -# elif char == "]" and not bracket_count: -# return brackets_content, substring[i + 1:] -# -# brackets_content += char -# raise TypeParsingError("") -# -# -# # T0do Return mypy\types -> Type class -# def create_type(type_string: str, description: str) -> AbstractType: -# if not type_string: -# return NamedType("None", "builtins.None") -# -# type_string = type_string.replace(" ", "") -# -# # t0do Replace pipes with Union -# # if "|" in type_string: -# # type_string = _replace_pipes_with_union(type_string) -# -# # Structures, which only take one type argument -# one_arg_structures = {"Final": FinalType, "Optional": OptionalType} -# for key in one_arg_structures: -# regex = r"^" + key + r"\[(.*)]$" -# match = re.match(regex, type_string) -# if match: -# content = match.group(1) -# return one_arg_structures[key](create_type(content, description)) -# -# # List-like structures, which take multiple type arguments -# mult_arg_structures = {"List": ListType, "Set": SetType, "Tuple": TupleType, "Union": UnionType} -# for key in mult_arg_structures: -# regex = r"^" + key + r"\[(.*)]$" -# match = re.match(regex, type_string) -# if match: -# content = match.group(1) -# content_elements = _dismantel_type_string_structure(content) -# return mult_arg_structures[key]([ -# create_type(element, description) -# for element in content_elements -# ]) -# -# match = re.match(r"^Literal\[(.*)]$", type_string) -# if match: -# content = match.group(1) -# contents = content.replace(" ", "").split(",") -# literals = [] -# for element in contents: -# try: -# value = ast.literal_eval(element) -# except (SyntaxError, ValueError): -# value = element[1:-1] -# literals.append(value) -# return LiteralType(literals) -# -# # Misc. special structures -# match = re.match(r"^Dict\[(.*)]$", type_string) -# if match: -# content = match.group(1) -# content_elements = _dismantel_type_string_structure(content) -# if len(content_elements) != 2: -# raise TypeParsingError(f"Could not parse Dict from the following string: \n{type_string}") -# return DictType( -# create_type(content_elements[0], description), -# create_type(content_elements[1], description), -# ) -# -# # raise TypeParsingError(f"Could not parse type for the following type string:\n{type_string}") -# type_ = _create_enum_boundry_type(type_string, description) -# if type_ is not None: -# return type_ -# return NamedType(name=type_string, qname=) -# -# -# # t0do übernehmen in create_type -> Tests schlagen nun fehl -# def _create_enum_boundry_type(type_string: str, description: str) -> AbstractType | None: -# types: list[AbstractType] = [] -# -# # Collapse whitespaces -# type_string = re.sub(r"\s+", " ", type_string) -# -# # Get boundary from description -# boundary = BoundaryType.from_string(description) -# if boundary is not None: -# types.append(boundary) -# -# # Find all enums and remove them from doc_string -# enum_array_matches = re.findall(r"\{.*?}", type_string) -# type_string = re.sub(r"\{.*?}", " ", type_string) -# for enum in enum_array_matches: -# enum_type = EnumType.from_string(enum) -# if enum_type is not None: -# types.append(enum_type) -# -# # Remove default value from doc_string -# type_string = re.sub("default=.*", " ", type_string) -# -# # Create a list with all values and types -# # ") or (" must be replaced by a very unlikely string ("&%&") so that it is not removed when filtering out. -# # The string will be replaced by ") or (" again after filtering out. -# type_string = re.sub(r"\) or \(", "&%&", type_string) -# type_string = re.sub(r" ?, ?or ", ", ", type_string) -# type_string = re.sub(r" or ", ", ", type_string) -# type_string = re.sub("&%&", ") or (", type_string) -# -# brackets = 0 -# build_string = "" -# for c in type_string: -# if c == "(": -# brackets += 1 -# elif c == ")": -# brackets -= 1 -# -# if brackets > 0: -# build_string += c -# continue -# -# if brackets == 0 and c != ",": -# build_string += c -# elif brackets == 0 and c == ",": -# # remove leading and trailing whitespaces -# build_string = build_string.strip() -# if build_string != "": -# named = NamedType.from_string(build_string) -# types.append(named) -# build_string = "" -# -# build_string = build_string.strip() -# -# # Append the last remaining entry -# if build_string != "": -# named = NamedType.from_string(build_string) -# types.append(named) -# -# if len(types) == 1: -# return types[0] -# if len(types) == 0: -# return None -# return UnionType(types) -# -# -# class TypeParsingError(Exception): -# def __init__(self, message: str): -# self.message = message -# -# def __str__(self) -> str: -# return f"TypeParsingException: {self.message}" diff --git a/src/safeds_stubgen/docstring_parsing/_docstring_parser.py b/src/safeds_stubgen/docstring_parsing/_docstring_parser.py index 99660e6f..14c098f5 100644 --- a/src/safeds_stubgen/docstring_parsing/_docstring_parser.py +++ b/src/safeds_stubgen/docstring_parsing/_docstring_parser.py @@ -32,6 +32,7 @@ class DocstringParser(AbstractDocstringParser): def __init__(self, parser: Parser, package_path: Path): while True: + # If a package has no __init__.py file Griffe can't parse it, therefore we check the parent try: self.griffe_build = load(package_path, docstring_parser=parser) break diff --git a/src/safeds_stubgen/stubs_generator/_generate_stubs.py b/src/safeds_stubgen/stubs_generator/_generate_stubs.py index a5e7f35d..06602fba 100644 --- a/src/safeds_stubgen/stubs_generator/_generate_stubs.py +++ b/src/safeds_stubgen/stubs_generator/_generate_stubs.py @@ -123,6 +123,11 @@ def _create_outside_package_class( naming_convention: NamingConvention, created_module_paths: set[str], ) -> set[str]: + """Create imported classes from outside the package. + + If classes of functions from outside the analyzed package are used, like e.g. `import math`, these classes and + functions will be created as stubs outside the actual package we analyze. + """ path_parts = class_path.split(".") # There are cases where we could not correctly parse or find the origin of a variable, which is then put into diff --git a/tests/safeds_stubgen/api_analyzer/test_types.py b/tests/safeds_stubgen/api_analyzer/test_types.py index d0bce5e5..e5c9735b 100644 --- a/tests/safeds_stubgen/api_analyzer/test_types.py +++ b/tests/safeds_stubgen/api_analyzer/test_types.py @@ -4,10 +4,8 @@ from safeds_stubgen.api_analyzer import ( AbstractType, Attribute, - BoundaryType, CallableType, DictType, - EnumType, FinalType, ListType, LiteralType, @@ -35,14 +33,6 @@ def test_correct_hash() -> None: type=NamedType(name="str", qname=""), ) assert hash(parameter) == hash(deepcopy(parameter)) - enum_values = frozenset({"a", "b", "c"}) - enum_type = EnumType(enum_values, "full_match") - assert enum_type == deepcopy(enum_type) - assert hash(enum_type) == hash(deepcopy(enum_type)) - assert enum_type == EnumType(deepcopy(enum_values), "full_match") - assert hash(enum_type) == hash(EnumType(deepcopy(enum_values), "full_match")) - assert enum_type != EnumType(frozenset({"a", "b"}), "full_match") - assert hash(enum_type) != hash(EnumType(frozenset({"a", "b"}), "full_match")) assert NamedType("a", "") == NamedType("a", "") assert hash(NamedType("a", "")) == hash(NamedType("a", "")) assert NamedType("a", "") != NamedType("b", "") @@ -50,13 +40,7 @@ def test_correct_hash() -> None: attribute = Attribute( id="boundary", name="boundary", - type=BoundaryType( - base_type="int", - min=0, - max=1, - min_inclusive=True, - max_inclusive=True, - ), + type=NamedType("a", ""), is_public=True, is_static=True, docstring=AttributeDocstring(), @@ -77,38 +61,6 @@ def test_named_type() -> None: assert named_type.to_dict() == named_type_dict -def test_enum_type() -> None: - value = frozenset({"a", "b"}) - type_ = EnumType(value, "a, b") - type_dict = {"kind": "EnumType", "values": {"a", "b"}} - - assert AbstractType.from_dict(type_dict) == type_ - assert EnumType.from_dict(type_dict) == type_ - assert type_.to_dict() == type_dict - - -def test_boundary_type() -> None: - type_ = BoundaryType( - base_type="int", - min=1, - max="b", - min_inclusive=True, - max_inclusive=True, - ) - type_dict = { - "kind": "BoundaryType", - "base_type": "int", - "min": 1, - "max": "b", - "min_inclusive": True, - "max_inclusive": True, - } - - assert AbstractType.from_dict(type_dict) == type_ - assert BoundaryType.from_dict(type_dict) == type_ - assert type_.to_dict() == type_dict - - def test_union_type() -> None: union_type = UnionType([NamedType("str", ""), NamedType("int", "")]) union_type_dict = { @@ -398,287 +350,3 @@ def test_tuple_type() -> None: def test_abstract_type_from_dict_exception() -> None: with pytest.raises(ValueError, match="Cannot parse unknown_type value."): AbstractType.from_dict({"kind": "unknown_type"}) - - -@pytest.mark.parametrize( - ("string", "expected"), - [ - ( - ( - "float, default=0.0 Tolerance for singular values computed by svd_solver == 'arpack'.\nMust be of range" - " [0.0, infinity).\n\n.. versionadded:: 0.18.0" - ), - BoundaryType( - base_type="float", - min=0, - max="Infinity", - min_inclusive=True, - max_inclusive=True, - ), - ), - ( - """If bootstrap is True, the number of samples to draw from X\nto train each base estimator.\n\n - - If None (default), then draw `X.shape[0]` samples.\n- If int, then draw `max_samples` samples.\n - - If float, then draw `max_samples * X.shape[0]` samples. Thus,\n `max_samples` should be in the interval `(0.0, 1.0]`.\n\n.. - versionadded:: 0.22""", - BoundaryType( - base_type="float", - min=0, - max=1, - min_inclusive=False, - max_inclusive=True, - ), - ), - ( - """When building the vocabulary ignore terms that have a document\nfrequency strictly lower than the given threshold. This value is also\n - called cut-off in the literature.\nIf float in range of [0.0, 1.0], the parameter represents a proportion\nof documents, integer absolute counts.\n - This parameter is ignored if vocabulary is not None.""", - BoundaryType( - base_type="float", - min=0, - max=1, - min_inclusive=True, - max_inclusive=True, - ), - ), - ( - """float in range [0.0, 1.0] or int, default=1.0 When building the vocabulary ignore terms that have a document\n - frequency strictly higher than the given threshold (corpus-specific\nstop words).\nIf float, the parameter represents a proportion of documents, integer\n - absolute counts.\nThis parameter is ignored if vocabulary is not None.""", - BoundaryType( - base_type="float", - min=0, - max=1, - min_inclusive=True, - max_inclusive=True, - ), - ), - ( - ( - "Tolerance for singular values computed by svd_solver == 'arpack'.\nMust be of range [-2, -1].\n\n.." - " versionadded:: 0.18.0" - ), - BoundaryType( - base_type="float", - min=-2, - max=-1, - min_inclusive=True, - max_inclusive=True, - ), - ), - ( - "Damping factor in the range (-1, -0.5)", - BoundaryType( - base_type="float", - min=-1, - max=-0.5, - min_inclusive=False, - max_inclusive=False, - ), - ), - ( - "'max_samples' should be in the interval (-1.0, -0.5]", - BoundaryType( - base_type="float", - min=-1.0, - max=-0.5, - min_inclusive=False, - max_inclusive=True, - ), - ), - ], -) -def test_boundaries_from_string(string: str, expected: BoundaryType) -> None: - ref_type = BoundaryType.from_string(string) - assert ref_type == expected - - -@pytest.mark.parametrize( - ("docstring_type", "expected"), - [ - ("", ""), - ('{"frobenius", "spectral"}, default="frobenius"', {"frobenius", "spectral"}), - ( - "{'strict', 'ignore', 'replace'}, default='strict'", - {"strict", "ignore", "replace"}, - ), - ( - "{'linear', 'poly', 'rbf', 'sigmoid', 'cosine', 'precomputed'}, default='linear'", - {"linear", "poly", "rbf", "sigmoid", "cosine", "precomputed"}, - ), - # https://github.com/lars-reimann/sem21/pull/30#discussion_r771288528 - (r"{\"frobenius\", \'spectral\'}", set()), - (r"""{"frobenius'}""", set()), - (r"""{'spectral"}""", set()), - (r"""{'text\", \"that'}""", {'text", "that'}), - (r"""{'text", "that'}""", {'text", "that'}), - (r"{'text\', \'that'}", {"text', 'that"}), - (r"{'text', 'that'}", {"text", "that"}), - (r"""{"text\', \'that"}""", {"text', 'that"}), - (r"""{"text', 'that"}""", {"text', 'that"}), - (r"""{"text\", \"that"}""", {'text", "that'}), - (r'{"text", "that"}', {"text", "that"}), - (r"""{\"not', 'be', 'matched'}""", {", "}), - ("""{"gini\\", \\"entropy"}""", {'gini", "entropy'}), - ("""{'best\\', \\'random'}""", {"best', 'random"}), - ], -) -def test_enum_from_string(docstring_type: str, expected: set[str] | None) -> None: - result = EnumType.from_string(docstring_type) - if result is not None: - assert result.values == expected - - -# Todo create_type Tests deactivated since create_type is not in use yet -# @pytest.mark.parametrize( -# ("docstring_type", "expected"), -# [ -# ( -# "", -# {"kind": "NamedType", "name": "None", "qname": ""} -# ), -# ( -# "int, or None, 'manual', {'auto', 'sqrt', 'log2'}, default='auto'", -# { -# "kind": "UnionType", -# "types": [ -# {"kind": "EnumType", "values": {"auto", "log2", "sqrt"}}, -# {"kind": "NamedType", "name": "int", "qname": ""}, -# {"kind": "NamedType", "name": "None", "qname": ""}, -# {"kind": "NamedType", "name": "'manual'", "qname": ""}, -# ], -# }, -# ), -# ( -# "tuple of slice, AUTO or array of shape (12,2), default=(slice(70, 195), slice(78, 172))", -# { -# "kind": "UnionType", -# "types": [ -# {"kind": "NamedType", "name": "tuple of slice", "qname": ""}, -# {"kind": "NamedType", "name": "AUTO", "qname": ""}, -# {"kind": "NamedType", "name": "array of shape (12,2)", "qname": ""}, -# ], -# }, -# ), -# ("object", {"kind": "NamedType", "name": "object", "qname": ""}), -# ( -# "ndarray, shape (n_samples,), default=None", -# { -# "kind": "UnionType", -# "types": [ -# {"kind": "NamedType", "name": "ndarray", "qname": ""}, -# {"kind": "NamedType", "name": "shape (n_samples,)", "qname": ""}, -# ], -# }, -# ), -# ( -# "estor adventus or None", -# { -# "kind": "UnionType", -# "types": [ -# {"kind": "NamedType", "name": "estor adventus", "qname": ""}, -# {"kind": "NamedType", "name": "None", "qname": ""}, -# ], -# }, -# ), -# ( -# "int or array-like, shape (n_samples, n_classes) or (n_samples, 1) when binary.", -# { -# "kind": "UnionType", -# "types": [ -# {"kind": "NamedType", "name": "int", "qname": ""}, -# {"kind": "NamedType", "name": "array-like", "qname": ""}, -# { -# "kind": "NamedType", -# "name": "shape (n_samples, n_classes) or (n_samples, 1) when binary.", "qname": "" -# }, -# ], -# }, -# ), -# ], -# ) -# def test_union_from_string(docstring_type: str, expected: dict[str, Any]) -> None: -# result = create_type(docstring_type, docstring_type) -# if result is None: -# assert expected == {} -# else: -# assert result.to_dict() == expected - - -# @pytest.mark.parametrize( -# ("description", "expected"), -# [ -# ( -# "Scale factor between inner and outer circle in the range `[0, 1)`", -# { -# "base_type": "float", -# "kind": "BoundaryType", -# "max": 1.0, -# "max_inclusive": False, -# "min": 0.0, -# "min_inclusive": True, -# }, -# ), -# ( -# ( -# "Tolerance for singular values computed by svd_solver == 'arpack'.\nMust be of range [1," -# " infinity].\n\n.. versionadded:: 0.18.0" -# ), -# { -# "base_type": "float", -# "kind": "BoundaryType", -# "max": "Infinity", -# "max_inclusive": True, -# "min": 1.0, -# "min_inclusive": True, -# }, -# ), -# ("", {}), -# ], -# ) -# def test_boundary_from_string(description: str, expected: dict[str, Any]) -> None: -# result = create_type(ParameterDocstring("", "", description)) -# if result is None: -# assert expected == {} -# else: -# assert result.to_dict() == expected - - -# @pytest.mark.parametrize( -# ("docstring_type", "docstring_description", "expected"), -# [ -# ( -# "int or 'Auto', or {'today', 'yesterday'}", -# "int in the range `[0, 10]`", -# { -# "kind": "UnionType", -# "types": [ -# { -# "base_type": "int", -# "kind": "BoundaryType", -# "max": 10.0, -# "max_inclusive": True, -# "min": 0.0, -# "min_inclusive": True, -# }, -# {"kind": "EnumType", "values": ["today", "yesterday"]}, -# {"kind": "NamedType", "name": "int", "qname": ""}, -# {"kind": "NamedType", "name": "'Auto'", "qname": ""}, -# ], -# }, -# ), -# ], -# ) -# def test_boundary_and_union_from_string( -# docstring_type: str, -# docstring_description: str, -# expected: dict[str, Any], -# ) -> None: -# result = create_type( -# ParameterDocstring(type=docstring_type, default_value="", description=docstring_description), -# ) -# -# if result is None: -# assert expected == {} -# else: -# assert result.to_dict() == expected From 84c723747066f67c34d55d46a582eeeb1282883b Mon Sep 17 00:00:00 2001 From: Arsam Date: Sun, 18 Aug 2024 22:09:08 +0200 Subject: [PATCH 04/25] Fixing a few bugs and replacing some raises with warnings --- .../api_analyzer/_ast_visitor.py | 48 +++++++++++--- .../docstring_parsing/_docstring_parser.py | 65 ++++++++++++------- .../stubs_generator/_stub_string_generator.py | 15 +++-- .../attribute_module.py | 2 + .../various_modules_package/class_module.py | 6 +- .../__snapshots__/test__get_api.ambr | 61 +++++++++++++++++ ...st_stub_creation[attribute_module].sdsstub | 5 ++ 7 files changed, 164 insertions(+), 38 deletions(-) diff --git a/src/safeds_stubgen/api_analyzer/_ast_visitor.py b/src/safeds_stubgen/api_analyzer/_ast_visitor.py index 3c017c33..095b2160 100644 --- a/src/safeds_stubgen/api_analyzer/_ast_visitor.py +++ b/src/safeds_stubgen/api_analyzer/_ast_visitor.py @@ -174,7 +174,10 @@ def enter_classdef(self, node: mp_nodes.ClassDef) -> None: variance_type = mypy_variance_parser(generic_type.variance) variance_values: sds_types.AbstractType | None = None if variance_type == VarianceKind.INVARIANT: - values = [self.mypy_type_to_abstract_type(value) for value in generic_type.values] + values = [] + if hasattr(generic_type, "values"): + values = [self.mypy_type_to_abstract_type(value) for value in generic_type.values] + if values: variance_values = sds_types.UnionType( [self.mypy_type_to_abstract_type(value) for value in generic_type.values], @@ -840,10 +843,11 @@ def _parse_attributes( elif hasattr(lvalue, "items"): lvalues = list(lvalue.items) for lvalue_ in lvalues: - if not hasattr(lvalue_, "name"): # pragma: no cover - raise AttributeError("Expected value to have attribute 'name'.") - - if self._is_attribute_already_defined(lvalue_.name): + if ( + hasattr(lvalue_, "name") + and self._is_attribute_already_defined(lvalue_.name) + or isinstance(lvalue_, mp_nodes.IndexExpr) + ): continue attributes.append( @@ -920,7 +924,9 @@ def _create_attribute( if unanalyzed_type is not None and hasattr(unanalyzed_type, "args"): attribute_type.args = unanalyzed_type.args else: # pragma: no cover - raise AttributeError("Could not get argument information for attribute.") + logging.warning("Could not get argument information for attribute.") + attribute_type = None + type_ = sds_types.UnknownType() # Ignore types that are special mypy any types. The Any type "from_unimported_type" could appear for aliase if ( @@ -966,8 +972,10 @@ def _parse_parameter_data(self, node: mp_nodes.FuncDef, function_id: str) -> lis default_is_none = False # Get type information for parameter - if mypy_type is None: # pragma: no cover - raise ValueError("Argument has no type.") + if mypy_type is None: + msg = f"Could not parse the type for parameter {argument.variable.name} of function {node.fullname}." + logging.warning(msg) + arg_type = sds_types.UnknownType() elif isinstance(mypy_type, mp_types.AnyType) and not has_correct_type_of_any(mypy_type.type_of_any): # We try to infer the type through the default value later, if possible pass @@ -1122,8 +1130,24 @@ def mypy_type_to_abstract_type( types = [self.mypy_type_to_abstract_type(arg) for arg in getattr(unanalyzed_type, "args", [])] if len(types) == 1: return sds_types.FinalType(type_=types[0]) - elif len(types) == 0: # pragma: no cover - raise ValueError("Final type has no type arguments.") + elif len(types) == 0: + if hasattr(mypy_type, "items"): + literals = [ + self.mypy_type_to_abstract_type(item.last_known_value) + for item in mypy_type.items + if isinstance(item.last_known_value, mp_types.LiteralType) + ] + + if literals: + all_literals = [] + for literal_type in literals: + if isinstance(literal_type, sds_types.LiteralType): + all_literals += literal_type.literals + + return sds_types.FinalType(type_=sds_types.LiteralType(literals=all_literals)) + + logging.warning("Final type has no type arguments.") # pragma: no cover + return sds_types.FinalType(type_=sds_types.UnknownType()) # pragma: no cover return sds_types.FinalType(type_=sds_types.UnionType(types=types)) elif unanalyzed_type_name in {"list", "set"}: type_args = getattr(mypy_type, "args", []) @@ -1168,6 +1192,10 @@ def mypy_type_to_abstract_type( if mypy_type.type_of_any == mp_types.TypeOfAny.from_unimported_type: # If the Any type is generated b/c of from_unimported_type, then we can parse the type # from the import information + if mypy_type.missing_import_name is None: # pragma: no cover + logging.warning("Could not parse a type, added unknown type instead.") + return sds_types.UnknownType() + missing_import_name = mypy_type.missing_import_name.split(".")[-1] # type: ignore[union-attr] name, qname = self._find_alias(missing_import_name) diff --git a/src/safeds_stubgen/docstring_parsing/_docstring_parser.py b/src/safeds_stubgen/docstring_parsing/_docstring_parser.py index 14c098f5..092daabd 100644 --- a/src/safeds_stubgen/docstring_parsing/_docstring_parser.py +++ b/src/safeds_stubgen/docstring_parsing/_docstring_parser.py @@ -55,12 +55,16 @@ def get_class_documentation(self, class_node: nodes.ClassDef) -> ClassDocstring: if griffe_node.docstring is not None: docstring = griffe_node.docstring.value.strip("\n") - for docstring_section in griffe_node.docstring.parsed: - if docstring_section.kind == DocstringSectionKind.text: - description = docstring_section.value.strip("\n") - elif docstring_section.kind == DocstringSectionKind.examples: - for example_data in docstring_section.value: - examples.append(example_data[1].strip("\n")) + try: + for docstring_section in griffe_node.docstring.parsed: + if docstring_section.kind == DocstringSectionKind.text: + description = docstring_section.value.strip("\n") + elif docstring_section.kind == DocstringSectionKind.examples: + for example_data in docstring_section.value: + examples.append(example_data[1].strip("\n")) + except IndexError as _: # pragma: no cover + msg = f"There was an error while parsing the following docstring:\n{docstring}." + logging.warning(msg) return ClassDocstring( description=description, @@ -75,12 +79,17 @@ def get_function_documentation(self, function_node: nodes.FuncDef) -> FunctionDo griffe_docstring = self.__get_cached_docstring(function_node.fullname) if griffe_docstring is not None: docstring = griffe_docstring.value.strip("\n") - for docstring_section in griffe_docstring.parsed: - if docstring_section.kind == DocstringSectionKind.text: - description = docstring_section.value.strip("\n") - elif docstring_section.kind == DocstringSectionKind.examples: - for example_data in docstring_section.value: - examples.append(example_data[1].strip("\n")) + + try: + for docstring_section in griffe_docstring.parsed: + if docstring_section.kind == DocstringSectionKind.text: + description = docstring_section.value.strip("\n") + elif docstring_section.kind == DocstringSectionKind.examples: + for example_data in docstring_section.value: + examples.append(example_data[1].strip("\n")) + except IndexError as _: # pragma: no cover + msg = f"There was an error while parsing the following docstring:\n{docstring}." + logging.warning(msg) return FunctionDocstring( description=description, @@ -193,10 +202,15 @@ def get_result_documentation(self, function_qname: str) -> list[ResultDocstring] return [] all_returns = None - for docstring_section in griffe_docstring.parsed: - if docstring_section.kind == DocstringSectionKind.returns: - all_returns = docstring_section - break + try: + for docstring_section in griffe_docstring.parsed: + if docstring_section.kind == DocstringSectionKind.returns: + all_returns = docstring_section + break + except IndexError as _: # pragma: no cover + msg = f"There was an error while parsing the following docstring:\n{griffe_docstring.value}." + logging.warning(msg) + return [] if not all_returns: return [] @@ -240,13 +254,18 @@ def _get_matching_docstrings( type_: Literal["attr", "param"], ) -> list[DocstringAttribute | DocstringParameter]: all_docstrings = None - for docstring_section in function_doc.parsed: - section_kind = docstring_section.kind - if (type_ == "attr" and section_kind == DocstringSectionKind.attributes) or ( - type_ == "param" and section_kind == DocstringSectionKind.parameters - ): - all_docstrings = docstring_section - break + try: + for docstring_section in function_doc.parsed: + section_kind = docstring_section.kind + if (type_ == "attr" and section_kind == DocstringSectionKind.attributes) or ( + type_ == "param" and section_kind == DocstringSectionKind.parameters + ): + all_docstrings = docstring_section + break + except IndexError as _: # pragma: no cover + msg = f"There was an error while parsing the following docstring:\n{function_doc.value}." + logging.warning(msg) + return [] if all_docstrings: name = name.lstrip("*") diff --git a/src/safeds_stubgen/stubs_generator/_stub_string_generator.py b/src/safeds_stubgen/stubs_generator/_stub_string_generator.py index 537e4075..d0f2fce2 100644 --- a/src/safeds_stubgen/stubs_generator/_stub_string_generator.py +++ b/src/safeds_stubgen/stubs_generator/_stub_string_generator.py @@ -860,6 +860,9 @@ def _create_internal_class_string( ) -> str: superclass_class = self._get_class_in_package(superclass) + if superclass_class is None: # pragma: no cover + return "" + # Methods superclass_methods_text, existing_names = self._create_class_method_string( superclass_class.methods, @@ -881,6 +884,10 @@ def _create_internal_class_string( already_defined_names = already_defined_names.union(existing_names) for superclass_superclass in superclass_class.superclasses: + if superclass_superclass == superclass: # pragma: no cover + # If the class somehow has itself as a superclass + continue + name = superclass_superclass.split(".")[-1] if is_internal(name): superclass_methods_text += self._create_internal_class_string( @@ -1107,7 +1114,7 @@ def _create_todo_msg(self, indentations: str) -> str: return indentations + f"\n{indentations}".join(todo_msgs) + "\n" - def _get_class_in_package(self, class_qname: str) -> Class: + def _get_class_in_package(self, class_qname: str) -> Class | None: class_qname = class_qname.replace(".", "/") class_path = "/".join(class_qname.split("/")[:-1]) class_name = class_qname.split("/")[-1] @@ -1122,9 +1129,9 @@ def _get_class_in_package(self, class_qname: str) -> Class: ): return self.api.classes[class_] - raise LookupError( - f"Expected finding class '{class_name}' in module '{self._get_module_id(get_actual_id=True)}'.", - ) # pragma: no cover + msg = f"Expected finding class '{class_name}' in module '{self._get_module_id(get_actual_id=True)}'." # pragma: no cover + logging.warning(msg) # pragma: no cover + return None # pragma: no cover @staticmethod def _create_docstring_description_part(description: str, indentations: str) -> str: diff --git a/tests/data/various_modules_package/attribute_module.py b/tests/data/various_modules_package/attribute_module.py index d8432018..ead7c834 100644 --- a/tests/data/various_modules_package/attribute_module.py +++ b/tests/data/various_modules_package/attribute_module.py @@ -51,6 +51,8 @@ def some_func() -> bool: str_attr_with_none_value: str = None optional: Optional[int] + final_int: Final = (101, 32741, 2147483621) + no_final_type: Final final: Final[str] = "Value" finals: Final[str, int] = "Value" final_union: Final[str | int] = "Value" diff --git a/tests/data/various_modules_package/class_module.py b/tests/data/various_modules_package/class_module.py index ff62a509..cb80586a 100644 --- a/tests/data/various_modules_package/class_module.py +++ b/tests/data/various_modules_package/class_module.py @@ -1,4 +1,4 @@ -from typing import Self, overload +from typing import Self, overload, no_type_check from . import unknown_source from tests.data.main_package.another_path.another_module import yetAnotherClass @@ -18,6 +18,10 @@ def __init__(self, a: int, b: ClassModuleEmptyClassA | None): def __enter__(self): return self + @no_type_check + def _apply(self, f, *args, **kwargs): + pass + def f(self): ... diff --git a/tests/safeds_stubgen/api_analyzer/__snapshots__/test__get_api.ambr b/tests/safeds_stubgen/api_analyzer/__snapshots__/test__get_api.ambr index e06f14de..876ed4f7 100644 --- a/tests/safeds_stubgen/api_analyzer/__snapshots__/test__get_api.ambr +++ b/tests/safeds_stubgen/api_analyzer/__snapshots__/test__get_api.ambr @@ -270,6 +270,27 @@ }), }), }), + dict({ + 'docstring': dict({ + 'description': '', + 'type': None, + }), + 'id': 'tests/data/various_modules_package/attribute_module/AttributesClassB/final_int', + 'is_public': True, + 'is_static': True, + 'name': 'final_int', + 'type': dict({ + 'kind': 'FinalType', + 'type': dict({ + 'kind': 'LiteralType', + 'literals': list([ + 101, + 32741, + 2147483621, + ]), + }), + }), + }), dict({ 'docstring': dict({ 'description': '', @@ -685,6 +706,17 @@ ]), }), }), + dict({ + 'docstring': dict({ + 'description': '', + 'type': None, + }), + 'id': 'tests/data/various_modules_package/attribute_module/AttributesClassB/no_final_type', + 'is_public': True, + 'is_static': True, + 'name': 'no_final_type', + 'type': None, + }), dict({ 'docstring': dict({ 'description': '', @@ -1237,6 +1269,30 @@ 'tests/data/various_modules_package/class_module/ClassModuleClassB/__enter__/result_1', ]), }), + dict({ + 'docstring': dict({ + 'description': '', + 'examples': list([ + ]), + 'full_docstring': '', + }), + 'id': 'tests/data/various_modules_package/class_module/ClassModuleClassB/_apply', + 'is_class_method': False, + 'is_property': False, + 'is_public': False, + 'is_static': False, + 'name': '_apply', + 'parameters': list([ + 'tests/data/various_modules_package/class_module/ClassModuleClassB/_apply/self', + 'tests/data/various_modules_package/class_module/ClassModuleClassB/_apply/f', + 'tests/data/various_modules_package/class_module/ClassModuleClassB/_apply/args', + 'tests/data/various_modules_package/class_module/ClassModuleClassB/_apply/kwargs', + ]), + 'reexported_by': list([ + ]), + 'results': list([ + ]), + }), dict({ 'docstring': dict({ 'description': '', @@ -2138,6 +2194,7 @@ 'is_public': True, 'methods': list([ 'tests/data/various_modules_package/class_module/ClassModuleClassB/__enter__', + 'tests/data/various_modules_package/class_module/ClassModuleClassB/_apply', 'tests/data/various_modules_package/class_module/ClassModuleClassB/f', ]), 'name': 'ClassModuleClassB', @@ -7485,6 +7542,10 @@ 'alias': None, 'qualified_name': 'typing.overload', }), + dict({ + 'alias': None, + 'qualified_name': 'typing.no_type_check', + }), dict({ 'alias': None, 'qualified_name': 'unknown_source', diff --git a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[attribute_module].sdsstub b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[attribute_module].sdsstub index e39e7027..a1ff3bde 100644 --- a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[attribute_module].sdsstub +++ b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[attribute_module].sdsstub @@ -70,6 +70,11 @@ class AttributesClassB() { @PythonName("str_attr_with_none_value") static attr strAttrWithNoneValue: String static attr optional: Int? + @PythonName("final_int") + static attr finalInt: literal<101, 32741, 2147483621> + // TODO Attribute has no type information. + @PythonName("no_final_type") + static attr noFinalType static attr final: String static attr finals: union @PythonName("final_union") From 091868b5c8b65153edfad285efef4a584972c011 Mon Sep 17 00:00:00 2001 From: Arsam Date: Wed, 21 Aug 2024 17:14:57 +0200 Subject: [PATCH 05/25] added analysis of "not" statements as bool type --- .../api_analyzer/_ast_visitor.py | 2 ++ .../function_module.py | 4 ++++ .../__snapshots__/test__get_api.ambr | 22 +++++++++++++++++++ ...est_stub_creation[function_module].sdsstub | 4 ++++ 4 files changed, 32 insertions(+) diff --git a/src/safeds_stubgen/api_analyzer/_ast_visitor.py b/src/safeds_stubgen/api_analyzer/_ast_visitor.py index 095b2160..2b01a74f 100644 --- a/src/safeds_stubgen/api_analyzer/_ast_visitor.py +++ b/src/safeds_stubgen/api_analyzer/_ast_visitor.py @@ -689,6 +689,8 @@ def _infer_type_from_return_stmts(self, func_node: mp_nodes.FuncDef) -> sds_type else: type_ = mypy_expression_to_sds_type(cond_branch) types.append(type_) + elif isinstance(return_stmt.expr, mp_nodes.UnaryExpr) and return_stmt.expr.op == "not": + types.append(sds_types.NamedType(name="bool", qname="builtins.bool")) elif ( return_stmt.expr is not None and hasattr(return_stmt.expr, "node") diff --git a/tests/data/various_modules_package/function_module.py b/tests/data/various_modules_package/function_module.py index 85c5b9eb..5979e883 100644 --- a/tests/data/various_modules_package/function_module.py +++ b/tests/data/various_modules_package/function_module.py @@ -261,3 +261,7 @@ def return_param4(a: int, b, x): return a, b, a, b return True + + +def return_not_statement(): + return not (0 or "...") diff --git a/tests/safeds_stubgen/api_analyzer/__snapshots__/test__get_api.ambr b/tests/safeds_stubgen/api_analyzer/__snapshots__/test__get_api.ambr index 876ed4f7..dc7db518 100644 --- a/tests/safeds_stubgen/api_analyzer/__snapshots__/test__get_api.ambr +++ b/tests/safeds_stubgen/api_analyzer/__snapshots__/test__get_api.ambr @@ -6729,6 +6729,27 @@ 'results': list([ ]), }), + dict({ + 'docstring': dict({ + 'description': '', + 'examples': list([ + ]), + 'full_docstring': '', + }), + 'id': 'tests/data/various_modules_package/function_module/return_not_statement', + 'is_class_method': False, + 'is_property': False, + 'is_public': True, + 'is_static': False, + 'name': 'return_not_statement', + 'parameters': list([ + ]), + 'reexported_by': list([ + ]), + 'results': list([ + 'tests/data/various_modules_package/function_module/return_not_statement/result_1', + ]), + }), dict({ 'docstring': dict({ 'description': '', @@ -7866,6 +7887,7 @@ 'tests/data/various_modules_package/function_module/return_param2', 'tests/data/various_modules_package/function_module/return_param3', 'tests/data/various_modules_package/function_module/return_param4', + 'tests/data/various_modules_package/function_module/return_not_statement', ]), 'id': 'tests/data/various_modules_package/function_module', 'name': 'function_module', diff --git a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[function_module].sdsstub b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[function_module].sdsstub index 9c71f348..7bc2d613 100644 --- a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[function_module].sdsstub +++ b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[function_module].sdsstub @@ -336,6 +336,10 @@ fun returnParam4( x ) -> (result1: union, result2: Any, result3: Int, result4: Any) +@Pure +@PythonName("return_not_statement") +fun returnNotStatement() -> result1: Boolean + class FunctionModuleClassA() // TODO Some parameter have no type information. From 32036c7359349031093c726b4a5c02a523d476e8 Mon Sep 17 00:00:00 2001 From: Arsam Date: Wed, 21 Aug 2024 18:57:35 +0200 Subject: [PATCH 06/25] fixed a bug where, if stub files already existed, content that was already in there would be appended to the file once more --- .../stubs_generator/_generate_stubs.py | 23 ++++++++++--------- .../stubs_generator/test_generate_stubs.py | 6 ++++- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/src/safeds_stubgen/stubs_generator/_generate_stubs.py b/src/safeds_stubgen/stubs_generator/_generate_stubs.py index 06602fba..89da4b8b 100644 --- a/src/safeds_stubgen/stubs_generator/_generate_stubs.py +++ b/src/safeds_stubgen/stubs_generator/_generate_stubs.py @@ -105,9 +105,8 @@ def create_stub_files( # Create and open module file public_module_name = module_name.lstrip("_") file_path = Path(corrected_module_dir / f"{public_module_name}.sdsstub") - Path(file_path).touch() - with file_path.open("w", encoding="utf-8") as f: + with file_path.open("w+", encoding="utf-8") as f: f.write(module_text) created_module_paths: set[str] = set() @@ -141,20 +140,22 @@ def _create_outside_package_class( module_name = path_parts[-1] module_path = "/".join(path_parts) - first_creation = False - if module_path not in created_module_paths: - created_module_paths.add(module_path) - first_creation = True - module_dir = Path(out_path / module_path) module_dir.mkdir(parents=True, exist_ok=True) file_path = Path(module_dir / f"{module_name}.sdsstub") - if Path.exists(file_path) and not first_creation: - with file_path.open("a", encoding="utf-8") as f: - f.write(_create_outside_package_class_text(class_name, naming_convention)) + + if Path.exists(file_path): + text = _create_outside_package_class_text(class_name, naming_convention) + + with file_path.open("r", encoding="utf-8") as f: + content = f.read() + + if text not in content: + with file_path.open("a", encoding="utf-8") as f: + f.write(text) else: - with file_path.open("w", encoding="utf-8") as f: + with file_path.open("w+", encoding="utf-8") as f: module_text = "" # package name & annotation diff --git a/tests/safeds_stubgen/stubs_generator/test_generate_stubs.py b/tests/safeds_stubgen/stubs_generator/test_generate_stubs.py index 397b688c..9f68979b 100644 --- a/tests/safeds_stubgen/stubs_generator/test_generate_stubs.py +++ b/tests/safeds_stubgen/stubs_generator/test_generate_stubs.py @@ -70,9 +70,13 @@ def test_file_creation() -> None: def test_file_creation_limited_stubs_outside_package(snapshot_sds_stub: SnapshotAssertion) -> None: + path = Path(_out_dir / "tests/data/main_package/another_path/another_module/another_module.sdsstub") + + if path.exists(): + path.unlink() + create_stub_files(stubs_generator=stubs_generator, stubs_data=stubs_data, out_path=_out_dir) - path = Path(_out_dir / "tests/data/main_package/another_path/another_module/another_module.sdsstub") assert path.is_file() with path.open("r", encoding="utf-8") as f: From 16a589c12533e312689768a401417302b780180a Mon Sep 17 00:00:00 2001 From: Arsam Date: Fri, 23 Aug 2024 14:13:42 +0200 Subject: [PATCH 07/25] fixed a bug where some functions would not have "None" types even though having a "return" without any type; fixed a bug where attributes would not have a type even though a type is defined the docstrings --- .../api_analyzer/_ast_visitor.py | 5 ++--- .../stubs_generator/_stub_string_generator.py | 6 ++++- .../function_module.py | 7 ++++++ .../__snapshots__/test__get_api.ambr | 22 +++++++++++++++++++ ...est_stub_creation[function_module].sdsstub | 4 ++++ ...cstring_creation[googledoc-GOOGLE].sdsstub | 3 +-- ...string_creation[numpydoc-NUMPYDOC].sdsstub | 3 +-- 7 files changed, 42 insertions(+), 8 deletions(-) diff --git a/src/safeds_stubgen/api_analyzer/_ast_visitor.py b/src/safeds_stubgen/api_analyzer/_ast_visitor.py index 2b01a74f..120c4bc4 100644 --- a/src/safeds_stubgen/api_analyzer/_ast_visitor.py +++ b/src/safeds_stubgen/api_analyzer/_ast_visitor.py @@ -710,11 +710,10 @@ def _infer_type_from_return_stmts(self, func_node: mp_nodes.FuncDef) -> sds_type type_ = self._remove_assignments(func_defn, type_) all_types.append(type_) types.append(sds_types.TupleType(types=all_types)) + elif return_stmt.expr is None: + types.append(sds_types.NamedType(name="None", qname="builtins.None")) else: # Lastly, we have a mypy expression object, which we have to parse - if return_stmt.expr is None: # pragma: no cover - continue - type_ = mypy_expression_to_sds_type(return_stmt.expr) type_ = self._remove_assignments(func_defn, type_) types.append(type_) diff --git a/src/safeds_stubgen/stubs_generator/_stub_string_generator.py b/src/safeds_stubgen/stubs_generator/_stub_string_generator.py index d0f2fce2..e590b184 100644 --- a/src/safeds_stubgen/stubs_generator/_stub_string_generator.py +++ b/src/safeds_stubgen/stubs_generator/_stub_string_generator.py @@ -405,13 +405,17 @@ def _create_class_attribute_string( attr_name_camel_case = _replace_if_safeds_keyword(attr_name_camel_case) # Create type information + attr_docstring: AttributeDocstring = attribute.docstring + if attribute_type is None and attr_docstring and attr_docstring.type: + attribute_type = attr_docstring.type.to_dict() + attr_type = self._create_type_string(attribute_type) type_string = f": {attr_type}" if attr_type else "" if not type_string: self._current_todo_msgs.add("attr without type") # Create docstring text - docstring = self._create_sds_docstring(attribute.docstring, inner_indentations) + docstring = self._create_sds_docstring(attr_docstring, inner_indentations) # Create attribute string class_attributes.append( diff --git a/tests/data/various_modules_package/function_module.py b/tests/data/various_modules_package/function_module.py index 5979e883..2ef44dab 100644 --- a/tests/data/various_modules_package/function_module.py +++ b/tests/data/various_modules_package/function_module.py @@ -265,3 +265,10 @@ def return_param4(a: int, b, x): def return_not_statement(): return not (0 or "...") + + +def return_without_result(): + if 1: + return + else: + return diff --git a/tests/safeds_stubgen/api_analyzer/__snapshots__/test__get_api.ambr b/tests/safeds_stubgen/api_analyzer/__snapshots__/test__get_api.ambr index dc7db518..4e574cdc 100644 --- a/tests/safeds_stubgen/api_analyzer/__snapshots__/test__get_api.ambr +++ b/tests/safeds_stubgen/api_analyzer/__snapshots__/test__get_api.ambr @@ -6845,6 +6845,27 @@ 'tests/data/various_modules_package/function_module/return_param4/result_4', ]), }), + dict({ + 'docstring': dict({ + 'description': '', + 'examples': list([ + ]), + 'full_docstring': '', + }), + 'id': 'tests/data/various_modules_package/function_module/return_without_result', + 'is_class_method': False, + 'is_property': False, + 'is_public': True, + 'is_static': False, + 'name': 'return_without_result', + 'parameters': list([ + ]), + 'reexported_by': list([ + ]), + 'results': list([ + 'tests/data/various_modules_package/function_module/return_without_result/result_1', + ]), + }), dict({ 'docstring': dict({ 'description': '', @@ -7888,6 +7909,7 @@ 'tests/data/various_modules_package/function_module/return_param3', 'tests/data/various_modules_package/function_module/return_param4', 'tests/data/various_modules_package/function_module/return_not_statement', + 'tests/data/various_modules_package/function_module/return_without_result', ]), 'id': 'tests/data/various_modules_package/function_module', 'name': 'function_module', diff --git a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[function_module].sdsstub b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[function_module].sdsstub index 7bc2d613..c73ea6bd 100644 --- a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[function_module].sdsstub +++ b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[function_module].sdsstub @@ -340,6 +340,10 @@ fun returnParam4( @PythonName("return_not_statement") fun returnNotStatement() -> result1: Boolean +@Pure +@PythonName("return_without_result") +fun returnWithoutResult() -> result1: Nothing? + class FunctionModuleClassA() // TODO Some parameter have no type information. diff --git a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/test_stub_docstring_creation[googledoc-GOOGLE].sdsstub b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/test_stub_docstring_creation[googledoc-GOOGLE].sdsstub index f8d8f751..3bd09c71 100644 --- a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/test_stub_docstring_creation[googledoc-GOOGLE].sdsstub +++ b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/test_stub_docstring_creation[googledoc-GOOGLE].sdsstub @@ -295,9 +295,8 @@ class ClassWithVariousAttributeTypes() { static attr classType: ClassWithAttributes @PythonName("imported_type") static attr importedType: AnotherClass - // TODO Attribute has no type information. @PythonName("callable_type") - static attr callableType + static attr callableType: (param1: Int) -> result1: String @PythonName("mapping_type") static attr mappingType: Map @PythonName("bool_op_type") diff --git a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/test_stub_docstring_creation[numpydoc-NUMPYDOC].sdsstub b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/test_stub_docstring_creation[numpydoc-NUMPYDOC].sdsstub index 0e716a19..94d6886a 100644 --- a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/test_stub_docstring_creation[numpydoc-NUMPYDOC].sdsstub +++ b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/test_stub_docstring_creation[numpydoc-NUMPYDOC].sdsstub @@ -447,9 +447,8 @@ class ClassWithVariousAttributeTypes() { static attr classType: ClassWithAttributes @PythonName("imported_type") static attr importedType: AnotherClass - // TODO Attribute has no type information. @PythonName("callable_type") - static attr callableType + static attr callableType: (param1: Int) -> result1: String @PythonName("mapping_type") static attr mappingType: Map @PythonName("bool_op_type") From 702344fc68fb48075d2f2958c0cbbf3e31c5296e Mon Sep 17 00:00:00 2001 From: Arsam Date: Sun, 25 Aug 2024 14:01:20 +0200 Subject: [PATCH 08/25] Added TypeAliasType parsing for parameters --- .../api_analyzer/_ast_visitor.py | 4 +++ .../function_module.py | 7 ++++- .../__snapshots__/test__get_api.ambr | 27 +++++++++++++++++++ ...est_stub_creation[function_module].sdsstub | 6 +++++ 4 files changed, 43 insertions(+), 1 deletion(-) diff --git a/src/safeds_stubgen/api_analyzer/_ast_visitor.py b/src/safeds_stubgen/api_analyzer/_ast_visitor.py index 120c4bc4..a2b87447 100644 --- a/src/safeds_stubgen/api_analyzer/_ast_visitor.py +++ b/src/safeds_stubgen/api_analyzer/_ast_visitor.py @@ -1171,6 +1171,10 @@ def mypy_type_to_abstract_type( return sds_types.UnionType(types=[self.mypy_type_to_abstract_type(item) for item in mypy_type.items]) # Special Cases + elif isinstance(mypy_type, mp_types.TypeAliasType): + fullname = mypy_type.alias.fullname + name = getattr(mypy_type.alias, "name", fullname.split(".")[-1]) + return sds_types.NamedType(name=name, qname=fullname) elif isinstance(mypy_type, mp_types.TypeVarType): upper_bound = mypy_type.upper_bound type_ = None diff --git a/tests/data/various_modules_package/function_module.py b/tests/data/various_modules_package/function_module.py index 2ef44dab..c4ff2cfa 100644 --- a/tests/data/various_modules_package/function_module.py +++ b/tests/data/various_modules_package/function_module.py @@ -1,4 +1,4 @@ -from typing import Callable, Optional, Literal, Any +from typing import Callable, Optional, Literal, Any, Union from tests.data.main_package.another_path.another_module import AnotherClass @@ -272,3 +272,8 @@ def return_without_result(): return else: return + + +ArrayLike = Union["ExtensionArray", Any] +def type_alias_param(values: ArrayLike) -> ArrayLike: + ... diff --git a/tests/safeds_stubgen/api_analyzer/__snapshots__/test__get_api.ambr b/tests/safeds_stubgen/api_analyzer/__snapshots__/test__get_api.ambr index 4e574cdc..22e64da6 100644 --- a/tests/safeds_stubgen/api_analyzer/__snapshots__/test__get_api.ambr +++ b/tests/safeds_stubgen/api_analyzer/__snapshots__/test__get_api.ambr @@ -6960,6 +6960,28 @@ 'tests/data/various_modules_package/function_module/tuple_results/result_2', ]), }), + dict({ + 'docstring': dict({ + 'description': '', + 'examples': list([ + ]), + 'full_docstring': '', + }), + 'id': 'tests/data/various_modules_package/function_module/type_alias_param', + 'is_class_method': False, + 'is_property': False, + 'is_public': True, + 'is_static': False, + 'name': 'type_alias_param', + 'parameters': list([ + 'tests/data/various_modules_package/function_module/type_alias_param/values', + ]), + 'reexported_by': list([ + ]), + 'results': list([ + 'tests/data/various_modules_package/function_module/type_alias_param/result_1', + ]), + }), dict({ 'docstring': dict({ 'description': '', @@ -7910,6 +7932,7 @@ 'tests/data/various_modules_package/function_module/return_param4', 'tests/data/various_modules_package/function_module/return_not_statement', 'tests/data/various_modules_package/function_module/return_without_result', + 'tests/data/various_modules_package/function_module/type_alias_param', ]), 'id': 'tests/data/various_modules_package/function_module', 'name': 'function_module', @@ -7930,6 +7953,10 @@ 'alias': None, 'qualified_name': 'typing.Any', }), + dict({ + 'alias': None, + 'qualified_name': 'typing.Union', + }), dict({ 'alias': None, 'qualified_name': 'tests.data.main_package.another_path.another_module.AnotherClass', diff --git a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[function_module].sdsstub b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[function_module].sdsstub index c73ea6bd..dd7ff706 100644 --- a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[function_module].sdsstub +++ b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[function_module].sdsstub @@ -344,6 +344,12 @@ fun returnNotStatement() -> result1: Boolean @PythonName("return_without_result") fun returnWithoutResult() -> result1: Nothing? +@Pure +@PythonName("type_alias_param") +fun typeAliasParam( + values: ArrayLike +) -> result1: ArrayLike + class FunctionModuleClassA() // TODO Some parameter have no type information. From b384f99d301d4d0e9e869d83901f8509972af5da Mon Sep 17 00:00:00 2001 From: Arsam Date: Sun, 25 Aug 2024 15:06:35 +0200 Subject: [PATCH 09/25] fixed a bug for type parsing where subclasses of aliases couldn't be parsed --- .../api_analyzer/_ast_visitor.py | 11 ++++++++ .../function_module.py | 5 ++++ .../__snapshots__/test__get_api.ambr | 26 +++++++++++++++++++ ...est_stub_creation[function_module].sdsstub | 5 ++++ 4 files changed, 47 insertions(+) diff --git a/src/safeds_stubgen/api_analyzer/_ast_visitor.py b/src/safeds_stubgen/api_analyzer/_ast_visitor.py index a2b87447..e97ebee0 100644 --- a/src/safeds_stubgen/api_analyzer/_ast_visitor.py +++ b/src/safeds_stubgen/api_analyzer/_ast_visitor.py @@ -1168,6 +1168,13 @@ def mypy_type_to_abstract_type( if isinstance(mypy_type, mp_types.TupleType): return sds_types.TupleType(types=[self.mypy_type_to_abstract_type(item) for item in mypy_type.items]) elif isinstance(mypy_type, mp_types.UnionType): + if hasattr(unanalyzed_type, "items") and unanalyzed_type and len(getattr(unanalyzed_type, "items", [])) == len(mypy_type.items): + return sds_types.UnionType( + types=[ + self.mypy_type_to_abstract_type(mypy_type.items[i], unanalyzed_type.items[i]) + for i in range(len(mypy_type.items)) + ] + ) return sds_types.UnionType(types=[self.mypy_type_to_abstract_type(item) for item in mypy_type.items]) # Special Cases @@ -1204,6 +1211,10 @@ def mypy_type_to_abstract_type( missing_import_name = mypy_type.missing_import_name.split(".")[-1] # type: ignore[union-attr] name, qname = self._find_alias(missing_import_name) + if unanalyzed_type and hasattr(unanalyzed_type, "name") and "." in unanalyzed_type.name and unanalyzed_type.name.startswith(missing_import_name): + name = unanalyzed_type.name.split(".")[-1] + qname = unanalyzed_type.name.replace(missing_import_name, qname) + if not qname: # pragma: no cover logging.warning("Could not parse a type, added unknown type instead.") return sds_types.UnknownType() diff --git a/tests/data/various_modules_package/function_module.py b/tests/data/various_modules_package/function_module.py index c4ff2cfa..62e5307f 100644 --- a/tests/data/various_modules_package/function_module.py +++ b/tests/data/various_modules_package/function_module.py @@ -1,5 +1,6 @@ from typing import Callable, Optional, Literal, Any, Union from tests.data.main_package.another_path.another_module import AnotherClass +import numpy as np class FunctionModuleClassA: @@ -277,3 +278,7 @@ def return_without_result(): ArrayLike = Union["ExtensionArray", Any] def type_alias_param(values: ArrayLike) -> ArrayLike: ... + + +def alias_subclass_result_type() -> ArrayLike | np.ndarray: + ... diff --git a/tests/safeds_stubgen/api_analyzer/__snapshots__/test__get_api.ambr b/tests/safeds_stubgen/api_analyzer/__snapshots__/test__get_api.ambr index 22e64da6..e61ca3b0 100644 --- a/tests/safeds_stubgen/api_analyzer/__snapshots__/test__get_api.ambr +++ b/tests/safeds_stubgen/api_analyzer/__snapshots__/test__get_api.ambr @@ -5949,6 +5949,27 @@ 'results': list([ ]), }), + dict({ + 'docstring': dict({ + 'description': '', + 'examples': list([ + ]), + 'full_docstring': '', + }), + 'id': 'tests/data/various_modules_package/function_module/alias_subclass_result_type', + 'is_class_method': False, + 'is_property': False, + 'is_public': True, + 'is_static': False, + 'name': 'alias_subclass_result_type', + 'parameters': list([ + ]), + 'reexported_by': list([ + ]), + 'results': list([ + 'tests/data/various_modules_package/function_module/alias_subclass_result_type/result_1', + ]), + }), dict({ 'docstring': dict({ 'description': '', @@ -7933,6 +7954,7 @@ 'tests/data/various_modules_package/function_module/return_not_statement', 'tests/data/various_modules_package/function_module/return_without_result', 'tests/data/various_modules_package/function_module/type_alias_param', + 'tests/data/various_modules_package/function_module/alias_subclass_result_type', ]), 'id': 'tests/data/various_modules_package/function_module', 'name': 'function_module', @@ -7961,6 +7983,10 @@ 'alias': None, 'qualified_name': 'tests.data.main_package.another_path.another_module.AnotherClass', }), + dict({ + 'alias': 'np', + 'qualified_name': 'numpy', + }), ]), 'wildcard_imports': list([ ]), diff --git a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[function_module].sdsstub b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[function_module].sdsstub index dd7ff706..5839ae88 100644 --- a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[function_module].sdsstub +++ b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[function_module].sdsstub @@ -1,6 +1,7 @@ @PythonModule("tests.data.various_modules_package.function_module") package tests.data.variousModulesPackage.functionModule +from numpy import ndarray from tests.data.mainPackage.anotherPath.anotherModule import AnotherClass // TODO Result type information missing. @@ -350,6 +351,10 @@ fun typeAliasParam( values: ArrayLike ) -> result1: ArrayLike +@Pure +@PythonName("alias_subclass_result_type") +fun aliasSubclassResultType() -> result1: union + class FunctionModuleClassA() // TODO Some parameter have no type information. From fe3ef130a5544dde537b271b23dd90225e55ff57 Mon Sep 17 00:00:00 2001 From: Arsam Date: Sun, 25 Aug 2024 16:01:40 +0200 Subject: [PATCH 10/25] fixed a bug where some class attributes defined in the __init__ function would wrongly be labeled as TypeVarType --- src/safeds_stubgen/api_analyzer/_ast_visitor.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/safeds_stubgen/api_analyzer/_ast_visitor.py b/src/safeds_stubgen/api_analyzer/_ast_visitor.py index e97ebee0..01c562b9 100644 --- a/src/safeds_stubgen/api_analyzer/_ast_visitor.py +++ b/src/safeds_stubgen/api_analyzer/_ast_visitor.py @@ -880,7 +880,7 @@ def _create_attribute( type_: sds_types.AbstractType | None = None node = None if hasattr(attribute, "node"): - if not isinstance(attribute.node, mp_nodes.Var): + if not isinstance(attribute.node, mp_nodes.Var) and not isinstance(attribute, mp_nodes.MemberExpr): # In this case we have a TypeVar attribute attr_name = getattr(attribute, "name", "") @@ -904,10 +904,16 @@ def _create_attribute( attribute_type = None # MemberExpr are constructor (__init__) attributes - if node is not None and isinstance(attribute, mp_nodes.MemberExpr): - attribute_type = node.type - if isinstance(attribute_type, mp_types.AnyType) and not has_correct_type_of_any(attribute_type.type_of_any): - attribute_type = None + if isinstance(attribute, mp_nodes.MemberExpr): + if node is not None: + attribute_type = node.type + if isinstance(attribute_type, mp_types.AnyType) and not has_correct_type_of_any(attribute_type.type_of_any): + attribute_type = None + else: # pragma: no cover + # There seems to be a case where MemberExpr objects don't have node information (e.g. the + # SingleBlockManager.blocks attribute of the Pandas library (a7a14108)) but I couldn't recreate this + # case + type_ = sds_types.UnknownType() # NameExpr are class attributes elif node is not None and isinstance(attribute, mp_nodes.NameExpr) and not node.explicit_self_type: From 3921da7869a41cf172d5a22de0a69a90fbf83643 Mon Sep 17 00:00:00 2001 From: Arsam Date: Sun, 25 Aug 2024 18:55:43 +0200 Subject: [PATCH 11/25] fixed a bug where some class attributes would wrongly be labeled as TypeVarType --- .../api_analyzer/_ast_visitor.py | 76 ++++++++++--------- 1 file changed, 42 insertions(+), 34 deletions(-) diff --git a/src/safeds_stubgen/api_analyzer/_ast_visitor.py b/src/safeds_stubgen/api_analyzer/_ast_visitor.py index 01c562b9..027384e9 100644 --- a/src/safeds_stubgen/api_analyzer/_ast_visitor.py +++ b/src/safeds_stubgen/api_analyzer/_ast_visitor.py @@ -424,7 +424,8 @@ def enter_assignmentstmt(self, node: mp_nodes.AssignmentStmt) -> None: continue if isinstance(parent, Class): - for assignment in self._parse_attributes(lvalue, node.unanalyzed_type, is_static=True): + is_type_var = hasattr(node, "rvalue") and hasattr(node.rvalue, "analyzed") and isinstance(node.rvalue.analyzed, mp_nodes.TypeVarExpr) + for assignment in self._parse_attributes(lvalue, node.unanalyzed_type, is_static=True, is_type_var=is_type_var): assignments.append(assignment) elif isinstance(parent, Function) and parent.name == "__init__": grand_parent = self.__declaration_stack[-2] @@ -828,6 +829,7 @@ def _parse_attributes( lvalue: mp_nodes.Expression, unanalyzed_type: mp_types.Type | None, is_static: bool = True, + is_type_var: bool = False ) -> list[Attribute]: """Parse the attributes from given Mypy expressions and return our own Attribute objects.""" assert isinstance(lvalue, mp_nodes.NameExpr | mp_nodes.MemberExpr | mp_nodes.TupleExpr) @@ -838,7 +840,7 @@ def _parse_attributes( return attributes attributes.append( - self._create_attribute(lvalue, unanalyzed_type, is_static), + self._create_attribute(lvalue, unanalyzed_type, is_static, is_type_var), ) elif hasattr(lvalue, "items"): @@ -852,7 +854,7 @@ def _parse_attributes( continue attributes.append( - self._create_attribute(lvalue_, unanalyzed_type, is_static), + self._create_attribute(lvalue_, unanalyzed_type, is_static, is_type_var), ) return attributes @@ -874,24 +876,23 @@ def _create_attribute( attribute: mp_nodes.Expression, unanalyzed_type: mp_types.Type | None, is_static: bool, + is_type_var: bool = False ) -> Attribute: """Create an Attribute object from a Mypy expression.""" # Get node information type_: sds_types.AbstractType | None = None node = None if hasattr(attribute, "node"): - if not isinstance(attribute.node, mp_nodes.Var) and not isinstance(attribute, mp_nodes.MemberExpr): - # In this case we have a TypeVar attribute - attr_name = getattr(attribute, "name", "") + node = attribute.node - if not attr_name: # pragma: no cover - raise AttributeError("Expected TypeVar to have attribute 'name'.") + if is_type_var: + # In this case we have a TypeVar attribute + attr_name = getattr(attribute, "name", "") - type_ = sds_types.TypeVarType(attr_name) - else: - node = attribute.node - else: # pragma: no cover - raise AttributeError("Expected attribute to have attribute 'node'.") + if not attr_name: # pragma: no cover + raise AttributeError("Expected TypeVar to have attribute 'name'.") + + type_ = sds_types.TypeVarType(attr_name) # Get name and qname name = getattr(attribute, "name", "") @@ -904,7 +905,7 @@ def _create_attribute( attribute_type = None # MemberExpr are constructor (__init__) attributes - if isinstance(attribute, mp_nodes.MemberExpr): + if not is_type_var and isinstance(attribute, mp_nodes.MemberExpr): if node is not None: attribute_type = node.type if isinstance(attribute_type, mp_types.AnyType) and not has_correct_type_of_any(attribute_type.type_of_any): @@ -916,24 +917,29 @@ def _create_attribute( type_ = sds_types.UnknownType() # NameExpr are class attributes - elif node is not None and isinstance(attribute, mp_nodes.NameExpr) and not node.explicit_self_type: - attribute_type = node.type + elif not is_type_var and isinstance(attribute, mp_nodes.NameExpr): + if node is not None and not node.explicit_self_type: + attribute_type = node.type - # We need to get the unanalyzed_type for lists, since mypy is not able to check type hint information - # regarding list item types - if ( - attribute_type is not None - and hasattr(attribute_type, "type") - and hasattr(attribute_type, "args") - and attribute_type.type.fullname == "builtins.list" - and not node.is_inferred - ): - if unanalyzed_type is not None and hasattr(unanalyzed_type, "args"): - attribute_type.args = unanalyzed_type.args - else: # pragma: no cover - logging.warning("Could not get argument information for attribute.") - attribute_type = None - type_ = sds_types.UnknownType() + # We need to get the unanalyzed_type for lists, since mypy is not able to check type hint information + # regarding list item types + if ( + attribute_type is not None + and hasattr(attribute_type, "type") + and hasattr(attribute_type, "args") + and attribute_type.type.fullname == "builtins.list" + and not node.is_inferred + ): + if unanalyzed_type is not None and hasattr(unanalyzed_type, "args"): + attribute_type.args = unanalyzed_type.args + else: # pragma: no cover + logging.warning("Could not get argument information for attribute.") + attribute_type = None + type_ = sds_types.UnknownType() + elif not unanalyzed_type: # pragma: no cover + type_ = sds_types.UnknownType() + else: # pragma: no cover + type_ = self.mypy_type_to_abstract_type(unanalyzed_type) # Ignore types that are special mypy any types. The Any type "from_unimported_type" could appear for aliase if ( @@ -1096,7 +1102,8 @@ def _get_reexported_by(self, qname: str) -> list[Module]: reexported_by = set() for i in range(len(path)): reexport_name_forward = ".".join(path[: i + 1]) - if reexport_name_forward in self.api.reexport_map: + # We ignore i = 0, b/c some inner package could import the whole upper package + if i != 0 and reexport_name_forward in self.api.reexport_map: for module in self.api.reexport_map[reexport_name_forward]: reexported_by.add(module) @@ -1127,7 +1134,7 @@ def mypy_type_to_abstract_type( self, mypy_type: mp_types.Instance | mp_types.ProperType | mp_types.Type, unanalyzed_type: mp_types.Type | None = None, - ) -> AbstractType: + ) -> sds_types.AbstractType: """Convert Mypy types to our AbstractType objects.""" # Special cases where we need the unanalyzed_type to get the type information we need if unanalyzed_type is not None and hasattr(unanalyzed_type, "name"): @@ -1233,10 +1240,11 @@ def mypy_type_to_abstract_type( elif isinstance(mypy_type, mp_types.LiteralType): return sds_types.LiteralType(literals=[mypy_type.value]) elif isinstance(mypy_type, mp_types.UnboundType): - if mypy_type.name in {"list", "set"}: + if mypy_type.name in {"list", "set", "tuple"}: return { "list": sds_types.ListType, "set": sds_types.SetType, + "tuple": sds_types.TupleType, }[ mypy_type.name ](types=[self.mypy_type_to_abstract_type(arg) for arg in mypy_type.args]) From fc467246ef51319eda25599710d917620ff9e9f4 Mon Sep 17 00:00:00 2001 From: Arsam Date: Sun, 25 Aug 2024 23:30:28 +0200 Subject: [PATCH 12/25] Added handling for results with operations (boolean results) and fixed a few bugs regarding the reexport check --- .../api_analyzer/_mypy_helpers.py | 5 ++++ .../stubs_generator/_generate_stubs.py | 2 +- src/safeds_stubgen/stubs_generator/_helper.py | 21 +++++++++++---- .../stubs_generator/_stub_string_generator.py | 2 +- .../function_module.py | 16 +++++++++++ .../__snapshots__/test__get_api.ambr | 27 +++++++++++++++++-- ...est_stub_creation[function_module].sdsstub | 9 ++++++- 7 files changed, 72 insertions(+), 10 deletions(-) diff --git a/src/safeds_stubgen/api_analyzer/_mypy_helpers.py b/src/safeds_stubgen/api_analyzer/_mypy_helpers.py index 7e19bf96..865b3f27 100644 --- a/src/safeds_stubgen/api_analyzer/_mypy_helpers.py +++ b/src/safeds_stubgen/api_analyzer/_mypy_helpers.py @@ -115,7 +115,12 @@ def mypy_expression_to_sds_type(expr: mp_nodes.Expression) -> sds_types.Abstract elif isinstance(expr, mp_nodes.TupleExpr): return sds_types.TupleType(types=[mypy_expression_to_sds_type(item) for item in expr.items]) elif isinstance(expr, mp_nodes.UnaryExpr): + if expr.op == "not": + return sds_types.NamedType(name="bool", qname="builtins.bool") return mypy_expression_to_sds_type(expr.expr) + elif ((isinstance(expr, mp_nodes.OpExpr) and expr.op in {"or", "and"}) or + (isinstance(expr, mp_nodes.ComparisonExpr) and ("is not" in expr.operators or "is" in expr.operators))): + return sds_types.NamedType(name="bool", qname="builtins.bool") logging.warning( "Could not parse a parameter or return type for a function: Safe-DS does not support " diff --git a/src/safeds_stubgen/stubs_generator/_generate_stubs.py b/src/safeds_stubgen/stubs_generator/_generate_stubs.py index 89da4b8b..e321be24 100644 --- a/src/safeds_stubgen/stubs_generator/_generate_stubs.py +++ b/src/safeds_stubgen/stubs_generator/_generate_stubs.py @@ -64,7 +64,7 @@ def generate_stub_data( shortest_path, alias = _get_shortest_public_reexport( reexport_map=api.reexport_map, name=module.name, - qname="", + qname=module.id, is_module=True, ) if shortest_path: diff --git a/src/safeds_stubgen/stubs_generator/_helper.py b/src/safeds_stubgen/stubs_generator/_helper.py index ccc6c051..320b26c6 100644 --- a/src/safeds_stubgen/stubs_generator/_helper.py +++ b/src/safeds_stubgen/stubs_generator/_helper.py @@ -49,10 +49,10 @@ def _get_shortest_public_reexport( is_module: bool, ) -> tuple[str, str]: parent_name = "" - if not is_module and qname: - qname_parts = qname.split(".") - if len(qname_parts) > 2: - parent_name = qname_parts[-2] + qname = qname.replace("/", ".") + qname_parts = qname.split(".") + if not is_module and qname and len(qname_parts) > 2: + parent_name = qname_parts[-2] def _module_name_check(text: str, is_wildcard: bool = False) -> bool: if is_module: @@ -66,7 +66,18 @@ def _module_name_check(text: str, is_wildcard: bool = False) -> bool: or (parent_name != "" and text.endswith(f"{parent_name}.*")) ) - keys = [reexport_key for reexport_key in reexport_map if _module_name_check(reexport_key)] + found_keys = [reexport_key for reexport_key in reexport_map if _module_name_check(reexport_key)] + + # Make sure we didn't get similar modules + keys = [] + for key in found_keys: + if key.split(".")[0] == qname_parts[0] and key.split(".")[-1] == qname_parts[-1]: + if qname == key: + keys.append(key) + else: + pass + else: + keys.append(key) module_ids = set() for key in keys: diff --git a/src/safeds_stubgen/stubs_generator/_stub_string_generator.py b/src/safeds_stubgen/stubs_generator/_stub_string_generator.py index e590b184..bd959477 100644 --- a/src/safeds_stubgen/stubs_generator/_stub_string_generator.py +++ b/src/safeds_stubgen/stubs_generator/_stub_string_generator.py @@ -117,7 +117,7 @@ def _create_module_string(self, module: Module) -> tuple[str, str]: package_info, _ = _get_shortest_public_reexport( reexport_map=self.api.reexport_map, name=module.name, - qname="", + qname=module.id, is_module=True, ) diff --git a/tests/data/various_modules_package/function_module.py b/tests/data/various_modules_package/function_module.py index 62e5307f..51c11556 100644 --- a/tests/data/various_modules_package/function_module.py +++ b/tests/data/various_modules_package/function_module.py @@ -282,3 +282,19 @@ def type_alias_param(values: ArrayLike) -> ArrayLike: def alias_subclass_result_type() -> ArrayLike | np.ndarray: ... + + +def different_result_operants(y): + if y: + return False + elif y - 1: + return y or y - 1 + elif y - 2: + return y and y + 1 + elif y - 3: + return y or y - 1 and y - 2 + elif y - 4: + return y is not None + elif y - 5: + return y is None + return not y diff --git a/tests/safeds_stubgen/api_analyzer/__snapshots__/test__get_api.ambr b/tests/safeds_stubgen/api_analyzer/__snapshots__/test__get_api.ambr index e61ca3b0..109ee28b 100644 --- a/tests/safeds_stubgen/api_analyzer/__snapshots__/test__get_api.ambr +++ b/tests/safeds_stubgen/api_analyzer/__snapshots__/test__get_api.ambr @@ -4973,8 +4973,8 @@ 'name': 'not_true', 'type': dict({ 'kind': 'NamedType', - 'name': 'int', - 'qname': 'builtins.int', + 'name': 'bool', + 'qname': 'builtins.bool', }), }), ]) @@ -6160,6 +6160,28 @@ 'tests/data/various_modules_package/function_module/dictionary_results_no_key_no_value/result_1', ]), }), + dict({ + 'docstring': dict({ + 'description': '', + 'examples': list([ + ]), + 'full_docstring': '', + }), + 'id': 'tests/data/various_modules_package/function_module/different_result_operants', + 'is_class_method': False, + 'is_property': False, + 'is_public': True, + 'is_static': False, + 'name': 'different_result_operants', + 'parameters': list([ + 'tests/data/various_modules_package/function_module/different_result_operants/y', + ]), + 'reexported_by': list([ + ]), + 'results': list([ + 'tests/data/various_modules_package/function_module/different_result_operants/result_1', + ]), + }), dict({ 'docstring': dict({ 'description': '', @@ -7955,6 +7977,7 @@ 'tests/data/various_modules_package/function_module/return_without_result', 'tests/data/various_modules_package/function_module/type_alias_param', 'tests/data/various_modules_package/function_module/alias_subclass_result_type', + 'tests/data/various_modules_package/function_module/different_result_operants', ]), 'id': 'tests/data/various_modules_package/function_module', 'name': 'function_module', diff --git a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[function_module].sdsstub b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[function_module].sdsstub index 5839ae88..cb3a2f6b 100644 --- a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[function_module].sdsstub +++ b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[function_module].sdsstub @@ -97,7 +97,7 @@ fun specialParams( @PythonName("none_none_bool_none_union") noneNoneBoolNoneUnion: Boolean?, @PythonName("none_list_union_none_none") noneListUnionNoneNone: List?, none: Nothing?, - @PythonName("not_true") notTrue: Int = unknown + @PythonName("not_true") notTrue: Boolean = unknown ) // TODO Result type information missing. @@ -355,6 +355,13 @@ fun typeAliasParam( @PythonName("alias_subclass_result_type") fun aliasSubclassResultType() -> result1: union +// TODO Some parameter have no type information. +@Pure +@PythonName("different_result_operants") +fun differentResultOperants( + y +) -> result1: Boolean + class FunctionModuleClassA() // TODO Some parameter have no type information. From 2fd3e5df2adb07baa58c1ba95d8740c8f984c176 Mon Sep 17 00:00:00 2001 From: Arsam Date: Mon, 26 Aug 2024 17:56:11 +0200 Subject: [PATCH 13/25] fixed a bug where type names would not be checked for naming convention and keywords --- src/safeds_stubgen/stubs_generator/_helper.py | 5 ++++ .../stubs_generator/_stub_string_generator.py | 24 +++++++------------ ...t_stub_creation[aliasing_module_1].sdsstub | 4 ++-- 3 files changed, 15 insertions(+), 18 deletions(-) diff --git a/src/safeds_stubgen/stubs_generator/_helper.py b/src/safeds_stubgen/stubs_generator/_helper.py index 320b26c6..01cedd3f 100644 --- a/src/safeds_stubgen/stubs_generator/_helper.py +++ b/src/safeds_stubgen/stubs_generator/_helper.py @@ -148,3 +148,8 @@ def _replace_if_safeds_keyword(keyword: str) -> str: }: return f"`{keyword}`" return keyword + + +def _name_convention_and_keyword_check(name: str, naming_convention: NamingConvention) -> str: + name = _convert_name_to_convention(name=name, naming_convention=naming_convention) + return _replace_if_safeds_keyword(keyword=name) diff --git a/src/safeds_stubgen/stubs_generator/_stub_string_generator.py b/src/safeds_stubgen/stubs_generator/_stub_string_generator.py index bd959477..31f6f2f8 100644 --- a/src/safeds_stubgen/stubs_generator/_stub_string_generator.py +++ b/src/safeds_stubgen/stubs_generator/_stub_string_generator.py @@ -30,6 +30,7 @@ _convert_name_to_convention, _create_name_annotation, _get_shortest_public_reexport, + _name_convention_and_keyword_check, _replace_if_safeds_keyword, ) @@ -174,13 +175,8 @@ def _create_imports_string(self) -> str: import_parts = import_.split(".") from_ = ".".join(import_parts[0:-1]) - from_ = _convert_name_to_convention(from_, self.naming_convention) - from_ = _replace_if_safeds_keyword(from_) - - name = import_parts[-1] - name = _convert_name_to_convention(name, self.naming_convention) - name = _replace_if_safeds_keyword(name) - + from_ = _name_convention_and_keyword_check(from_, self.naming_convention) + name = _name_convention_and_keyword_check(import_parts[-1], self.naming_convention) import_strings.append(f"from {from_} import {name}") # We have to sort for the snapshot tests @@ -231,8 +227,7 @@ def _create_class_string(self, class_: Class, class_indentation: str = "", in_re }[variance.variance.name] # Convert name to camelCase and check for keywords - variance_name_camel_case = _convert_name_to_convention(variance.name, self.naming_convention) - variance_name_camel_case = _replace_if_safeds_keyword(variance_name_camel_case) + variance_name_camel_case = _name_convention_and_keyword_check(variance.name, self.naming_convention) variance_item = f"{variance_direction}{variance_name_camel_case}" if variance.type is not None: @@ -465,8 +460,7 @@ def _create_function_string( if function.type_var_types: type_var_names = [] for type_var in function.type_var_types: - type_var_name = _convert_name_to_convention(type_var.name, self.naming_convention) - type_var_name = _replace_if_safeds_keyword(type_var_name) + type_var_name = _name_convention_and_keyword_check(type_var.name, self.naming_convention) # We don't have to display generic types in methods if they were already displayed in the class if not is_method or (is_method and type_var_name not in self.class_generics): @@ -547,8 +541,7 @@ def _create_result_string(self, function_results: list[Result]) -> str: ret_type = self._create_type_string(result_type) type_string = f": {ret_type}" if ret_type else "" - result_name = _convert_name_to_convention(result.name, self.naming_convention) - result_name = _replace_if_safeds_keyword(result_name) + result_name = _name_convention_and_keyword_check(result.name, self.naming_convention) if type_string: results.append( f"{result_name}{type_string}", @@ -712,7 +705,7 @@ def _create_type_string(self, type_data: dict | None) -> str: if name[0] == "_" and type_data["qname"] not in self.module_imports: self._current_todo_msgs.add("internal class as type") - return name + return _name_convention_and_keyword_check(name, self.naming_convention) elif kind == "FinalType": return self._create_type_string(type_data["type"]) elif kind == "CallableType": @@ -851,8 +844,7 @@ def _create_type_string(self, type_data: dict | None) -> str: types.append(f"{literal_type}") return f"literal<{', '.join(types)}>" elif kind == "TypeVarType": - name = _convert_name_to_convention(type_data["name"], self.naming_convention) - return _replace_if_safeds_keyword(name) + return _name_convention_and_keyword_check(type_data["name"], self.naming_convention) raise ValueError(f"Unexpected type: {kind}") # pragma: no cover diff --git a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[aliasing_module_1].sdsstub b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[aliasing_module_1].sdsstub index cd9c6a53..3155c50b 100644 --- a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[aliasing_module_1].sdsstub +++ b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[aliasing_module_1].sdsstub @@ -11,7 +11,7 @@ class AliasingModuleClassC() { static attr typedAliasAttr: AliasingModuleClassB // TODO An internal class must not be used as a type in a public class. @PythonName("infer_alias_attr") - static attr inferAliasAttr: _AliasingModuleClassA + static attr inferAliasAttr: AliasingModuleClassA @PythonName("typed_alias_attr2") static attr typedAliasAttr2: AliasingModule2ClassA @PythonName("infer_alias_attr2") @@ -22,5 +22,5 @@ class AliasingModuleClassC() { // TODO An internal class must not be used as a type in a public class. // TODO List type has to many type arguments. @PythonName("alias_list") - static attr aliasList: List, AliasingModule2ClassA, ImportMeAliasingModuleClass> + static attr aliasList: List, AliasingModule2ClassA, ImportMeAliasingModuleClass> } From 54ae8514e8b5f998498300ca35cfafc469c24894 Mon Sep 17 00:00:00 2001 From: Arsam Date: Tue, 27 Aug 2024 14:58:24 +0200 Subject: [PATCH 14/25] fixed the way reexported paths are searched and found; multiple text sections of docstrings will be merged now --- src/safeds_stubgen/_helpers.py | 64 +++++++++++++++++++ .../api_analyzer/_ast_visitor.py | 30 +-------- .../docstring_parsing/_docstring_parser.py | 4 +- .../stubs_generator/_generate_stubs.py | 4 +- src/safeds_stubgen/stubs_generator/_helper.py | 41 +++++------- .../stubs_generator/_stub_string_generator.py | 6 +- .../docstring_parser_package/googledoc.py | 20 ++++++ ...cstring_creation[googledoc-GOOGLE].sdsstub | 25 ++++++++ 8 files changed, 135 insertions(+), 59 deletions(-) diff --git a/src/safeds_stubgen/_helpers.py b/src/safeds_stubgen/_helpers.py index 087d9ba7..1ee8a684 100644 --- a/src/safeds_stubgen/_helpers.py +++ b/src/safeds_stubgen/_helpers.py @@ -1,3 +1,67 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from safeds_stubgen.api_analyzer import Module + + def is_internal(name: str) -> bool: """Check if a function / method / class name indicate if it's internal.""" return name.startswith("_") + + +def get_reexported_by(qname: str, reexport_map: dict[str, set[Module]]) -> list[Module]: + """Get all __init__ modules where a given function / class / enum was reexported.""" + path = qname.split(".") + + # Check if there is a reexport entry for each item in the path to the current module + reexported_by = set() + for i in range(len(path)): + reexport_name_forward = ".".join(path[: i + 1]) + # We ignore i = 0, b/c some inner package could import the whole upper package + if i != 0 and reexport_name_forward in reexport_map: + for module in reexport_map[reexport_name_forward]: + + # Check if the module or the class/function itself are beeing reexported. If not, it means a + # subpackage is beeing reexported. + last_forward_part = reexport_name_forward.split(".")[-1] + part_index = path.index(last_forward_part) + 1 + if len(path) - part_index > 1: + continue + + reexported_by.add(module) + + reexport_name_backward = ".".join(path[-i - 1 :]) + if reexport_name_backward in reexport_map: + for module in reexport_map[reexport_name_backward]: + + # Check if the module or the class/function itself are beeing reexported. If not, it means a + # subpackage is beeing reexported. + if module.name == "__init__" and i < len(path)-1: + zipped = list(zip(module.id.split("/"), path, strict=False)) + # Check if there is a part of the paths that differs + if not all(m == p for m, p in zipped): + continue + + reexported_by.add(module) + + reexport_name_backward_whitelist = f"{'.'.join(path[-2 - i:-1])}.*" + if reexport_name_backward_whitelist in reexport_map: + for module in reexport_map[reexport_name_backward_whitelist]: + + if len(path) > i + 2: + # Check if the found module actually references our object. E.g.: It could be the case that the + # file at `path/to/__init__.py` has `from .api import *` and we find this entry, even though we + # are searching for reexports of a class/function in the `path\from\api.py` file. + is_wrong_module_path = False + for j, module_part in enumerate(module.id.split("/")): + if module_part != path[j]: + is_wrong_module_path = True + break + if is_wrong_module_path: + continue + + reexported_by.add(module) + + return list(reexported_by) diff --git a/src/safeds_stubgen/api_analyzer/_ast_visitor.py b/src/safeds_stubgen/api_analyzer/_ast_visitor.py index 027384e9..62a7aed4 100644 --- a/src/safeds_stubgen/api_analyzer/_ast_visitor.py +++ b/src/safeds_stubgen/api_analyzer/_ast_visitor.py @@ -42,6 +42,7 @@ mypy_expression_to_sds_type, mypy_variance_parser, ) +from safeds_stubgen._helpers import get_reexported_by if TYPE_CHECKING: from collections.abc import Generator @@ -225,7 +226,7 @@ def enter_classdef(self, node: mp_nodes.ClassDef) -> None: superclasses.append(superclass_qname) # Get reexported data - reexported_by = self._get_reexported_by(node.fullname) + reexported_by = get_reexported_by(qname=node.fullname, reexport_map=self.api.reexport_map) # Sort for snapshot tests reexported_by.sort(key=lambda x: x.id) @@ -345,7 +346,7 @@ def enter_funcdef(self, node: mp_nodes.FuncDef) -> None: i += 1 # Get reexported data - reexported_by = self._get_reexported_by(node.fullname) + reexported_by = get_reexported_by(qname=node.fullname, reexport_map=self.api.reexport_map) # Sort for snapshot tests reexported_by.sort(key=lambda x: x.id) @@ -1094,31 +1095,6 @@ def _get_parameter_type_and_default_value( # #### Reexport utilities - def _get_reexported_by(self, qname: str) -> list[Module]: - """Get all __init__ modules where a given function / class / enum was reexported.""" - path = qname.split(".") - - # Check if there is a reexport entry for each item in the path to the current module - reexported_by = set() - for i in range(len(path)): - reexport_name_forward = ".".join(path[: i + 1]) - # We ignore i = 0, b/c some inner package could import the whole upper package - if i != 0 and reexport_name_forward in self.api.reexport_map: - for module in self.api.reexport_map[reexport_name_forward]: - reexported_by.add(module) - - reexport_name_backward = ".".join(path[-i - 1 :]) - if reexport_name_backward in self.api.reexport_map: - for module in self.api.reexport_map[reexport_name_backward]: - reexported_by.add(module) - - reexport_name_backward_whitelist = f"{'.'.join(path[-2 - i:-1])}.*" - if reexport_name_backward_whitelist in self.api.reexport_map: - for module in self.api.reexport_map[reexport_name_backward_whitelist]: - reexported_by.add(module) - - return list(reexported_by) - def _add_reexports(self, module: Module) -> None: """Add all reexports of an __init__ module to the reexport_map.""" for qualified_import in module.qualified_imports: diff --git a/src/safeds_stubgen/docstring_parsing/_docstring_parser.py b/src/safeds_stubgen/docstring_parsing/_docstring_parser.py index 092daabd..87a06b8e 100644 --- a/src/safeds_stubgen/docstring_parsing/_docstring_parser.py +++ b/src/safeds_stubgen/docstring_parsing/_docstring_parser.py @@ -83,7 +83,9 @@ def get_function_documentation(self, function_node: nodes.FuncDef) -> FunctionDo try: for docstring_section in griffe_docstring.parsed: if docstring_section.kind == DocstringSectionKind.text: - description = docstring_section.value.strip("\n") + if description: + description += "\n\n" + description += docstring_section.value.strip("\n") elif docstring_section.kind == DocstringSectionKind.examples: for example_data in docstring_section.value: examples.append(example_data[1].strip("\n")) diff --git a/src/safeds_stubgen/stubs_generator/_generate_stubs.py b/src/safeds_stubgen/stubs_generator/_generate_stubs.py index e321be24..adbbf336 100644 --- a/src/safeds_stubgen/stubs_generator/_generate_stubs.py +++ b/src/safeds_stubgen/stubs_generator/_generate_stubs.py @@ -8,7 +8,7 @@ NamingConvention, _convert_name_to_convention, _create_name_annotation, - _get_shortest_public_reexport, + _get_shortest_public_reexport_and_alias, _replace_if_safeds_keyword, ) @@ -61,7 +61,7 @@ def generate_stub_data( if len(splitted_text) <= 2 or (len(splitted_text) == 3 and splitted_text[1].startswith("package ")): continue - shortest_path, alias = _get_shortest_public_reexport( + shortest_path, alias = _get_shortest_public_reexport_and_alias( reexport_map=api.reexport_map, name=module.name, qname=module.id, diff --git a/src/safeds_stubgen/stubs_generator/_helper.py b/src/safeds_stubgen/stubs_generator/_helper.py index 01cedd3f..032d3a6b 100644 --- a/src/safeds_stubgen/stubs_generator/_helper.py +++ b/src/safeds_stubgen/stubs_generator/_helper.py @@ -3,6 +3,8 @@ from enum import IntEnum from typing import TYPE_CHECKING +from safeds_stubgen._helpers import get_reexported_by + if TYPE_CHECKING: from safeds_stubgen.api_analyzer import Module @@ -42,7 +44,7 @@ def _convert_name_to_convention( return name_parts[0] + "".join(part[0].upper() + part[1:] for part in name_parts[1:] if part) -def _get_shortest_public_reexport( +def _get_shortest_public_reexport_and_alias( reexport_map: dict[str, set[Module]], name: str, qname: str, @@ -54,7 +56,7 @@ def _get_shortest_public_reexport( if not is_module and qname and len(qname_parts) > 2: parent_name = qname_parts[-2] - def _module_name_check(text: str, is_wildcard: bool = False) -> bool: + def _import_check(text: str, is_wildcard: bool = False) -> bool: if is_module: return text.endswith(f".{name}") or text == name elif is_wildcard: @@ -66,32 +68,19 @@ def _module_name_check(text: str, is_wildcard: bool = False) -> bool: or (parent_name != "" and text.endswith(f"{parent_name}.*")) ) - found_keys = [reexport_key for reexport_key in reexport_map if _module_name_check(reexport_key)] - - # Make sure we didn't get similar modules - keys = [] - for key in found_keys: - if key.split(".")[0] == qname_parts[0] and key.split(".")[-1] == qname_parts[-1]: - if qname == key: - keys.append(key) - else: - pass - else: - keys.append(key) + found_modules = get_reexported_by(qname=qname, reexport_map=reexport_map) module_ids = set() - for key in keys: - for module in reexport_map[key]: - - for qualified_import in module.qualified_imports: - if _module_name_check(qualified_import.qualified_name): - module_ids.add((module.id, qualified_import.alias)) - break - - for wildcard_import in module.wildcard_imports: - if _module_name_check(wildcard_import.module_name, is_wildcard=True): - module_ids.add((module.id, None)) - break + for module in found_modules: + for qualified_import in module.qualified_imports: + if _import_check(qualified_import.qualified_name): + module_ids.add((module.id, qualified_import.alias)) + break + + for wildcard_import in module.wildcard_imports: + if _import_check(wildcard_import.module_name, is_wildcard=True): + module_ids.add((module.id, None)) + break shortest_id = None alias = None diff --git a/src/safeds_stubgen/stubs_generator/_stub_string_generator.py b/src/safeds_stubgen/stubs_generator/_stub_string_generator.py index 31f6f2f8..133582fe 100644 --- a/src/safeds_stubgen/stubs_generator/_stub_string_generator.py +++ b/src/safeds_stubgen/stubs_generator/_stub_string_generator.py @@ -29,7 +29,7 @@ NamingConvention, _convert_name_to_convention, _create_name_annotation, - _get_shortest_public_reexport, + _get_shortest_public_reexport_and_alias, _name_convention_and_keyword_check, _replace_if_safeds_keyword, ) @@ -115,7 +115,7 @@ def _create_module_string(self, module: Module) -> tuple[str, str]: module_text = "" # Create package info - package_info, _ = _get_shortest_public_reexport( + package_info, _ = _get_shortest_public_reexport_and_alias( reexport_map=self.api.reexport_map, name=module.name, qname=module.id, @@ -1057,7 +1057,7 @@ def _add_to_imports(self, import_qname: str) -> None: qname = class_id.replace("/", ".") name = qname.split(".")[-1] - shortest_qname, _ = _get_shortest_public_reexport( + shortest_qname, _ = _get_shortest_public_reexport_and_alias( reexport_map=self.api.reexport_map, name=name, qname=qname, diff --git a/tests/data/docstring_parser_package/googledoc.py b/tests/data/docstring_parser_package/googledoc.py index 0f39017b..70781cc7 100644 --- a/tests/data/docstring_parser_package/googledoc.py +++ b/tests/data/docstring_parser_package/googledoc.py @@ -276,6 +276,26 @@ def uninferable_return_doc(): """ +def google_multiple_text_parts(a, b): + """ + Nihil possimus iusto autem aut. Laboriosam ut ipsum veritatis. + Excepturi voluptatem beatae nam voluptas. + + Est aliquid numquam at error quis laborum et perferendis. + + Args: + a: First arg + b: Second arg + Throws: + RuntimeError + Returns: + Nothing + + Illum amet velit et qui. + """ + ... + + def infer_types(): """ property_method_with_docstring. diff --git a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/test_stub_docstring_creation[googledoc-GOOGLE].sdsstub b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/test_stub_docstring_creation[googledoc-GOOGLE].sdsstub index 3bd09c71..3846e8fe 100644 --- a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/test_stub_docstring_creation[googledoc-GOOGLE].sdsstub +++ b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/test_stub_docstring_creation[googledoc-GOOGLE].sdsstub @@ -117,6 +117,31 @@ fun functionWithoutReturnValue() @PythonName("uninferable_return_doc") fun uninferableReturnDoc() +// TODO Result type information missing. +// TODO Some parameter have no type information. +/** + * Nihil possimus iusto autem aut. Laboriosam ut ipsum veritatis. + * Excepturi voluptatem beatae nam voluptas. + * + * Est aliquid numquam at error quis laborum et perferendis. + * + * Throws: + * RuntimeError + * Returns: + * Nothing + * + * Illum amet velit et qui. + * + * @param a First arg + * @param b Second arg + */ +@Pure +@PythonName("google_multiple_text_parts") +fun googleMultipleTextParts( + a, + b +) + /** * property_method_with_docstring. * From 881a1b67cd6c06c1cf2d6cd6df7e7b5133cd9835 Mon Sep 17 00:00:00 2001 From: Arsam Date: Tue, 27 Aug 2024 15:16:55 +0200 Subject: [PATCH 15/25] reversed the change concerning None from commit #16a589c1 --- .../api_analyzer/_ast_visitor.py | 9 ++++++-- .../function_module.py | 23 +++++++++++-------- ...est_stub_creation[function_module].sdsstub | 20 +++++++++------- 3 files changed, 32 insertions(+), 20 deletions(-) diff --git a/src/safeds_stubgen/api_analyzer/_ast_visitor.py b/src/safeds_stubgen/api_analyzer/_ast_visitor.py index 62a7aed4..9c073426 100644 --- a/src/safeds_stubgen/api_analyzer/_ast_visitor.py +++ b/src/safeds_stubgen/api_analyzer/_ast_visitor.py @@ -672,6 +672,10 @@ def _infer_type_from_return_stmts(self, func_node: mp_nodes.FuncDef) -> sds_type type_ = self.mypy_type_to_abstract_type(return_stmt.expr.node.type) types.append(type_) continue + elif return_stmt.expr is None: + # In this case we have an impliciz None return + types.append(sds_types.NamedType(name="None", qname="builtins.None")) + continue if not isinstance(return_stmt.expr, mp_nodes.CallExpr | mp_nodes.MemberExpr): if isinstance(return_stmt.expr, mp_nodes.ConditionalExpr): @@ -712,10 +716,11 @@ def _infer_type_from_return_stmts(self, func_node: mp_nodes.FuncDef) -> sds_type type_ = self._remove_assignments(func_defn, type_) all_types.append(type_) types.append(sds_types.TupleType(types=all_types)) - elif return_stmt.expr is None: - types.append(sds_types.NamedType(name="None", qname="builtins.None")) else: # Lastly, we have a mypy expression object, which we have to parse + if return_stmt.expr is None: # pragma: no cover + continue + type_ = mypy_expression_to_sds_type(return_stmt.expr) type_ = self._remove_assignments(func_defn, type_) types.append(type_) diff --git a/tests/data/various_modules_package/function_module.py b/tests/data/various_modules_package/function_module.py index 51c11556..c99d4b0f 100644 --- a/tests/data/various_modules_package/function_module.py +++ b/tests/data/various_modules_package/function_module.py @@ -140,9 +140,6 @@ def bool_result() -> bool: ... def float_result() -> float: ... -def none_result() -> None: ... - - def obj_result() -> FunctionModuleClassA: ... @@ -268,13 +265,6 @@ def return_not_statement(): return not (0 or "...") -def return_without_result(): - if 1: - return - else: - return - - ArrayLike = Union["ExtensionArray", Any] def type_alias_param(values: ArrayLike) -> ArrayLike: ... @@ -298,3 +288,16 @@ def different_result_operants(y): elif y - 5: return y is None return not y + + +def none_result_1() -> None: + ... + + +def none_result_2(): + return + + +def none_result_3(): + return None + diff --git a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[function_module].sdsstub b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[function_module].sdsstub index cb3a2f6b..ec930c2e 100644 --- a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[function_module].sdsstub +++ b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[function_module].sdsstub @@ -168,10 +168,6 @@ fun boolResult() -> result1: Boolean @PythonName("float_result") fun floatResult() -> result1: Float -@Pure -@PythonName("none_result") -fun noneResult() - @Pure @PythonName("obj_result") fun objResult() -> result1: FunctionModuleClassA @@ -341,10 +337,6 @@ fun returnParam4( @PythonName("return_not_statement") fun returnNotStatement() -> result1: Boolean -@Pure -@PythonName("return_without_result") -fun returnWithoutResult() -> result1: Nothing? - @Pure @PythonName("type_alias_param") fun typeAliasParam( @@ -362,6 +354,18 @@ fun differentResultOperants( y ) -> result1: Boolean +@Pure +@PythonName("none_result_1") +fun noneResult1() + +@Pure +@PythonName("none_result_2") +fun noneResult2() + +@Pure +@PythonName("none_result_3") +fun noneResult3() + class FunctionModuleClassA() // TODO Some parameter have no type information. From ad2a11e3a3d55220ed53e60bc34f3099d18d9d6e Mon Sep 17 00:00:00 2001 From: Arsam Date: Tue, 27 Aug 2024 15:33:58 +0200 Subject: [PATCH 16/25] fixed a bug for parsing parameter types; replacing some logging.warning's with logging.info; various fixes --- src/safeds_stubgen/_helpers.py | 2 +- .../api_analyzer/_ast_visitor.py | 81 ++++-- src/safeds_stubgen/api_analyzer/_get_api.py | 2 +- .../api_analyzer/_mypy_helpers.py | 7 +- .../data/various_modules_package/__init__.py | 1 + .../another_path/__init__.py | 5 + .../another_path/not_reexported.py | 0 .../file_creation/package_1/not_reexported.py | 2 + .../function_module.py | 5 + .../reexport_test/__init__.py | 0 .../reexport_test/reexport_test_2/__init__.py | 0 .../reexport_test_3/__init__.py | 0 .../reexport_test_3/reexport_test.py | 4 + .../__snapshots__/test__get_api.ambr | 246 ++++++++++++++++-- .../api_analyzer/test__get_api.py | 8 +- ...est_stub_creation[function_module].sdsstub | 10 +- ...test_stub_creation[not_reexported].sdsstub | 4 + ....test_stub_creation[reexport_test].sdsstub | 4 + .../stubs_generator/test_generate_stubs.py | 1 + 19 files changed, 318 insertions(+), 64 deletions(-) create mode 100644 tests/data/various_modules_package/another_path/not_reexported.py create mode 100644 tests/data/various_modules_package/file_creation/package_1/not_reexported.py create mode 100644 tests/data/various_modules_package/reexport_test/__init__.py create mode 100644 tests/data/various_modules_package/reexport_test/reexport_test_2/__init__.py create mode 100644 tests/data/various_modules_package/reexport_test/reexport_test_2/reexport_test_3/__init__.py create mode 100644 tests/data/various_modules_package/reexport_test/reexport_test_2/reexport_test_3/reexport_test.py create mode 100644 tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[not_reexported].sdsstub create mode 100644 tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[reexport_test].sdsstub diff --git a/src/safeds_stubgen/_helpers.py b/src/safeds_stubgen/_helpers.py index 1ee8a684..35e144b8 100644 --- a/src/safeds_stubgen/_helpers.py +++ b/src/safeds_stubgen/_helpers.py @@ -38,7 +38,7 @@ def get_reexported_by(qname: str, reexport_map: dict[str, set[Module]]) -> list[ # Check if the module or the class/function itself are beeing reexported. If not, it means a # subpackage is beeing reexported. - if module.name == "__init__" and i < len(path)-1: + if module.name == "__init__" and i < len(path) - 1: zipped = list(zip(module.id.split("/"), path, strict=False)) # Check if there is a part of the paths that differs if not all(m == p for m, p in zipped): diff --git a/src/safeds_stubgen/api_analyzer/_ast_visitor.py b/src/safeds_stubgen/api_analyzer/_ast_visitor.py index 9c073426..6fd3567b 100644 --- a/src/safeds_stubgen/api_analyzer/_ast_visitor.py +++ b/src/safeds_stubgen/api_analyzer/_ast_visitor.py @@ -12,6 +12,7 @@ import safeds_stubgen.api_analyzer._types as sds_types from safeds_stubgen import is_internal +from safeds_stubgen._helpers import get_reexported_by from safeds_stubgen.api_analyzer._type_source_enums import TypeSourcePreference, TypeSourceWarning from safeds_stubgen.docstring_parsing import ResultDocstring @@ -42,7 +43,6 @@ mypy_expression_to_sds_type, mypy_variance_parser, ) -from safeds_stubgen._helpers import get_reexported_by if TYPE_CHECKING: from collections.abc import Generator @@ -299,7 +299,7 @@ def enter_funcdef(self, node: mp_nodes.FuncDef) -> None: and self.type_source_warning == TypeSourceWarning.WARN ): msg = f"Different type hint and docstring types for '{function_id}'." - logging.warning(msg) + logging.info(msg) if doc_type is not None and ( code_type is None or self.type_source_preference == TypeSourcePreference.DOCSTRING @@ -330,7 +330,7 @@ def enter_funcdef(self, node: mp_nodes.FuncDef) -> None: and self.type_source_warning == TypeSourceWarning.WARN ): msg = f"Different type hint and docstring types for the result of '{function_id}'." - logging.warning(msg) + logging.info(msg) if result_doc_type is not None: if result_type is None: @@ -425,8 +425,17 @@ def enter_assignmentstmt(self, node: mp_nodes.AssignmentStmt) -> None: continue if isinstance(parent, Class): - is_type_var = hasattr(node, "rvalue") and hasattr(node.rvalue, "analyzed") and isinstance(node.rvalue.analyzed, mp_nodes.TypeVarExpr) - for assignment in self._parse_attributes(lvalue, node.unanalyzed_type, is_static=True, is_type_var=is_type_var): + is_type_var = ( + hasattr(node, "rvalue") + and hasattr(node.rvalue, "analyzed") + and isinstance(node.rvalue.analyzed, mp_nodes.TypeVarExpr) + ) + for assignment in self._parse_attributes( + lvalue, + node.unanalyzed_type, + is_static=True, + is_type_var=is_type_var, + ): assignments.append(assignment) elif isinstance(parent, Function) and parent.name == "__init__": grand_parent = self.__declaration_stack[-2] @@ -835,7 +844,7 @@ def _parse_attributes( lvalue: mp_nodes.Expression, unanalyzed_type: mp_types.Type | None, is_static: bool = True, - is_type_var: bool = False + is_type_var: bool = False, ) -> list[Attribute]: """Parse the attributes from given Mypy expressions and return our own Attribute objects.""" assert isinstance(lvalue, mp_nodes.NameExpr | mp_nodes.MemberExpr | mp_nodes.TupleExpr) @@ -882,7 +891,7 @@ def _create_attribute( attribute: mp_nodes.Expression, unanalyzed_type: mp_types.Type | None, is_static: bool, - is_type_var: bool = False + is_type_var: bool = False, ) -> Attribute: """Create an Attribute object from a Mypy expression.""" # Get node information @@ -914,7 +923,9 @@ def _create_attribute( if not is_type_var and isinstance(attribute, mp_nodes.MemberExpr): if node is not None: attribute_type = node.type - if isinstance(attribute_type, mp_types.AnyType) and not has_correct_type_of_any(attribute_type.type_of_any): + if isinstance(attribute_type, mp_types.AnyType) and not has_correct_type_of_any( + attribute_type.type_of_any, + ): attribute_type = None else: # pragma: no cover # There seems to be a case where MemberExpr objects don't have node information (e.g. the @@ -924,7 +935,9 @@ def _create_attribute( # NameExpr are class attributes elif not is_type_var and isinstance(attribute, mp_nodes.NameExpr): - if node is not None and not node.explicit_self_type: + if node is not None and not hasattr(node, "explicit_self_type"): # pragma: no cover + pass + elif node is not None and not node.explicit_self_type: attribute_type = node.type # We need to get the unanalyzed_type for lists, since mypy is not able to check type hint information @@ -939,7 +952,7 @@ def _create_attribute( if unanalyzed_type is not None and hasattr(unanalyzed_type, "args"): attribute_type.args = unanalyzed_type.args else: # pragma: no cover - logging.warning("Could not get argument information for attribute.") + logging.info("Could not get argument information for attribute.") attribute_type = None type_ = sds_types.UnknownType() elif not unanalyzed_type: # pragma: no cover @@ -993,7 +1006,7 @@ def _parse_parameter_data(self, node: mp_nodes.FuncDef, function_id: str) -> lis # Get type information for parameter if mypy_type is None: msg = f"Could not parse the type for parameter {argument.variable.name} of function {node.fullname}." - logging.warning(msg) + logging.info(msg) arg_type = sds_types.UnknownType() elif isinstance(mypy_type, mp_types.AnyType) and not has_correct_type_of_any(mypy_type.type_of_any): # We try to infer the type through the default value later, if possible @@ -1008,7 +1021,7 @@ def _parse_parameter_data(self, node: mp_nodes.FuncDef, function_id: str) -> lis # way in Mypy. arg_type = self.mypy_type_to_abstract_type(type_annotation) elif type_annotation is not None: - arg_type = self.mypy_type_to_abstract_type(mypy_type) + arg_type = self.mypy_type_to_abstract_type(mypy_type, type_annotation) # Get default value and infer type information initializer = argument.initializer @@ -1031,6 +1044,10 @@ def _parse_parameter_data(self, node: mp_nodes.FuncDef, function_id: str) -> lis if isinstance(default_value, type): # pragma: no cover raise TypeError("default_value has the unexpected type 'type'.") + # Special case + if default_value == '"""': + default_value = '"\\""' + # Create parameter object arguments.append( Parameter( @@ -1064,7 +1081,7 @@ def _get_parameter_type_and_default_value( f"Could not parse parameter type for function {function_id}: Safe-DS does not support call " f"expressions as types." ) - logging.warning(msg) + logging.info(msg) # Safe-DS does not support call expressions as types return default_value, default_is_none elif isinstance(initializer, mp_nodes.UnaryExpr): @@ -1080,7 +1097,7 @@ def _get_parameter_type_and_default_value( f"Received the parameter {value} with an unexpected operator {initializer.op} for function " f"{function_id}. This parameter could not be parsed." ) - logging.warning(msg) + logging.info(msg) return UnknownValue(), default_is_none elif isinstance( initializer, @@ -1141,7 +1158,7 @@ def mypy_type_to_abstract_type( return sds_types.FinalType(type_=sds_types.LiteralType(literals=all_literals)) - logging.warning("Final type has no type arguments.") # pragma: no cover + logging.info("Final type has no type arguments.") # pragma: no cover return sds_types.FinalType(type_=sds_types.UnknownType()) # pragma: no cover return sds_types.FinalType(type_=sds_types.UnionType(types=types)) elif unanalyzed_type_name in {"list", "set"}: @@ -1162,18 +1179,23 @@ def mypy_type_to_abstract_type( if isinstance(mypy_type, mp_types.TupleType): return sds_types.TupleType(types=[self.mypy_type_to_abstract_type(item) for item in mypy_type.items]) elif isinstance(mypy_type, mp_types.UnionType): - if hasattr(unanalyzed_type, "items") and unanalyzed_type and len(getattr(unanalyzed_type, "items", [])) == len(mypy_type.items): + unanalyzed_type_items = getattr(unanalyzed_type, "items", []) + if ( + hasattr(unanalyzed_type, "items") + and unanalyzed_type + and len(unanalyzed_type_items) == len(mypy_type.items) + ): return sds_types.UnionType( types=[ - self.mypy_type_to_abstract_type(mypy_type.items[i], unanalyzed_type.items[i]) + self.mypy_type_to_abstract_type(mypy_type.items[i], unanalyzed_type_items[i]) for i in range(len(mypy_type.items)) - ] + ], ) return sds_types.UnionType(types=[self.mypy_type_to_abstract_type(item) for item in mypy_type.items]) # Special Cases elif isinstance(mypy_type, mp_types.TypeAliasType): - fullname = mypy_type.alias.fullname + fullname = getattr(mypy_type.alias, "fullname", "") name = getattr(mypy_type.alias, "name", fullname.split(".")[-1]) return sds_types.NamedType(name=name, qname=fullname) elif isinstance(mypy_type, mp_types.TypeVarType): @@ -1199,18 +1221,23 @@ def mypy_type_to_abstract_type( # If the Any type is generated b/c of from_unimported_type, then we can parse the type # from the import information if mypy_type.missing_import_name is None: # pragma: no cover - logging.warning("Could not parse a type, added unknown type instead.") + logging.info("Could not parse a type, added unknown type instead.") return sds_types.UnknownType() missing_import_name = mypy_type.missing_import_name.split(".")[-1] # type: ignore[union-attr] name, qname = self._find_alias(missing_import_name) - if unanalyzed_type and hasattr(unanalyzed_type, "name") and "." in unanalyzed_type.name and unanalyzed_type.name.startswith(missing_import_name): + if ( + unanalyzed_type + and hasattr(unanalyzed_type, "name") + and "." in unanalyzed_type.name + and unanalyzed_type.name.startswith(missing_import_name) + ): name = unanalyzed_type.name.split(".")[-1] qname = unanalyzed_type.name.replace(missing_import_name, qname) if not qname: # pragma: no cover - logging.warning("Could not parse a type, added unknown type instead.") + logging.info("Could not parse a type, added unknown type instead.") return sds_types.UnknownType() return sds_types.NamedType(name=name, qname=qname) @@ -1222,13 +1249,11 @@ def mypy_type_to_abstract_type( return sds_types.LiteralType(literals=[mypy_type.value]) elif isinstance(mypy_type, mp_types.UnboundType): if mypy_type.name in {"list", "set", "tuple"}: - return { + return { # type: ignore[abstract] "list": sds_types.ListType, "set": sds_types.SetType, "tuple": sds_types.TupleType, - }[ - mypy_type.name - ](types=[self.mypy_type_to_abstract_type(arg) for arg in mypy_type.args]) + }[mypy_type.name](types=[self.mypy_type_to_abstract_type(arg) for arg in mypy_type.args]) # Get qname if mypy_type.name in {"Any", "str", "int", "bool", "float", "None"}: @@ -1249,7 +1274,7 @@ def mypy_type_to_abstract_type( name, qname = self._find_alias(mypy_type.name) if not qname: # pragma: no cover - logging.warning("Could not parse a type, added unknown type instead.") + logging.info("Could not parse a type, added unknown type instead.") return sds_types.UnknownType() return sds_types.NamedType(name=name, qname=qname) @@ -1286,7 +1311,7 @@ def mypy_type_to_abstract_type( return sds_types.NamedSequenceType(name=type_name, qname=mypy_type.type.fullname, types=types) return sds_types.NamedType(name=type_name, qname=mypy_type.type.fullname) - logging.warning("Could not parse a type, added unknown type instead.") # pragma: no cover + logging.info("Could not parse a type, added unknown type instead.") # pragma: no cover return sds_types.UnknownType() # pragma: no cover def _find_alias(self, type_name: str) -> tuple[str, str]: diff --git a/src/safeds_stubgen/api_analyzer/_get_api.py b/src/safeds_stubgen/api_analyzer/_get_api.py index 28522eba..6b3fd50c 100644 --- a/src/safeds_stubgen/api_analyzer/_get_api.py +++ b/src/safeds_stubgen/api_analyzer/_get_api.py @@ -217,7 +217,7 @@ def _get_aliases(result_types: dict, package_name: str) -> dict[str, set[str]]: fullname = key.node.fullname else: # pragma: no cover msg = f"Received unexpected type while searching for aliases. Skipping for '{name}'." - logging.warning(msg) + logging.info(msg) continue aliases[name].add(fullname) diff --git a/src/safeds_stubgen/api_analyzer/_mypy_helpers.py b/src/safeds_stubgen/api_analyzer/_mypy_helpers.py index 865b3f27..675376dd 100644 --- a/src/safeds_stubgen/api_analyzer/_mypy_helpers.py +++ b/src/safeds_stubgen/api_analyzer/_mypy_helpers.py @@ -118,11 +118,12 @@ def mypy_expression_to_sds_type(expr: mp_nodes.Expression) -> sds_types.Abstract if expr.op == "not": return sds_types.NamedType(name="bool", qname="builtins.bool") return mypy_expression_to_sds_type(expr.expr) - elif ((isinstance(expr, mp_nodes.OpExpr) and expr.op in {"or", "and"}) or - (isinstance(expr, mp_nodes.ComparisonExpr) and ("is not" in expr.operators or "is" in expr.operators))): + elif (isinstance(expr, mp_nodes.OpExpr) and expr.op in {"or", "and"}) or ( + isinstance(expr, mp_nodes.ComparisonExpr) and ("is not" in expr.operators or "is" in expr.operators) + ): return sds_types.NamedType(name="bool", qname="builtins.bool") - logging.warning( + logging.info( "Could not parse a parameter or return type for a function: Safe-DS does not support " "types such as call expressions. Added 'unknown' instead.", ) diff --git a/tests/data/various_modules_package/__init__.py b/tests/data/various_modules_package/__init__.py index 25bd9a4e..9503dce6 100644 --- a/tests/data/various_modules_package/__init__.py +++ b/tests/data/various_modules_package/__init__.py @@ -11,6 +11,7 @@ from file_creation._module_3 import Reexported from .class_module import ClassModuleClassD as ClMCD from file_creation.module_1 import Lv2 +from tests.data.various_modules_package import reexport_test __all__ = [ "reex_1", diff --git a/tests/data/various_modules_package/another_path/__init__.py b/tests/data/various_modules_package/another_path/__init__.py index 9811f6ff..76089c80 100644 --- a/tests/data/various_modules_package/another_path/__init__.py +++ b/tests/data/various_modules_package/another_path/__init__.py @@ -1,5 +1,10 @@ from typing import TYPE_CHECKING +# Testing if the "not_reexported" module is beeing wrongly reexported b/c of the existence +# of file_creation/package_1/not_reexported.py +from not_reexported import * +import not_reexported + if TYPE_CHECKING: from another_module import _YetAnotherPrivateClass diff --git a/tests/data/various_modules_package/another_path/not_reexported.py b/tests/data/various_modules_package/another_path/not_reexported.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/various_modules_package/file_creation/package_1/not_reexported.py b/tests/data/various_modules_package/file_creation/package_1/not_reexported.py new file mode 100644 index 00000000..04e82710 --- /dev/null +++ b/tests/data/various_modules_package/file_creation/package_1/not_reexported.py @@ -0,0 +1,2 @@ +class NotReexported(): + ... diff --git a/tests/data/various_modules_package/function_module.py b/tests/data/various_modules_package/function_module.py index c99d4b0f..4c61de77 100644 --- a/tests/data/various_modules_package/function_module.py +++ b/tests/data/various_modules_package/function_module.py @@ -87,6 +87,7 @@ def params_with_default_value( tuple_: tuple[int, str, bool] = (1, "2", True), literal: Literal["Some String"] = "Some String", any_: Any = False, + single_quote: str = '"' ): ... @@ -274,6 +275,10 @@ def alias_subclass_result_type() -> ArrayLike | np.ndarray: ... +def alias_subclass_param_type(x: ArrayLike | np.ndarray): + ... + + def different_result_operants(y): if y: return False diff --git a/tests/data/various_modules_package/reexport_test/__init__.py b/tests/data/various_modules_package/reexport_test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/various_modules_package/reexport_test/reexport_test_2/__init__.py b/tests/data/various_modules_package/reexport_test/reexport_test_2/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/various_modules_package/reexport_test/reexport_test_2/reexport_test_3/__init__.py b/tests/data/various_modules_package/reexport_test/reexport_test_2/reexport_test_3/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/various_modules_package/reexport_test/reexport_test_2/reexport_test_3/reexport_test.py b/tests/data/various_modules_package/reexport_test/reexport_test_2/reexport_test_3/reexport_test.py new file mode 100644 index 00000000..2e2c0eb1 --- /dev/null +++ b/tests/data/various_modules_package/reexport_test/reexport_test_2/reexport_test_3/reexport_test.py @@ -0,0 +1,4 @@ +# With this directory we test the reexportation of this file if the root directory imports reexport_test. + +class NotReexportedTest: + ... diff --git a/tests/safeds_stubgen/api_analyzer/__snapshots__/test__get_api.ambr b/tests/safeds_stubgen/api_analyzer/__snapshots__/test__get_api.ambr index 109ee28b..9b81e2a7 100644 --- a/tests/safeds_stubgen/api_analyzer/__snapshots__/test__get_api.ambr +++ b/tests/safeds_stubgen/api_analyzer/__snapshots__/test__get_api.ambr @@ -5491,10 +5491,36 @@ }), ]) # --- -# name: test_function_results[none_result] +# name: test_function_results[none_result_1] list([ dict({ - 'id': 'tests/data/various_modules_package/function_module/none_result/result_1', + 'id': 'tests/data/various_modules_package/function_module/none_result_1/result_1', + 'name': 'result_1', + 'type': dict({ + 'kind': 'NamedType', + 'name': 'None', + 'qname': 'builtins.None', + }), + }), + ]) +# --- +# name: test_function_results[none_result_2] + list([ + dict({ + 'id': 'tests/data/various_modules_package/function_module/none_result_2/result_1', + 'name': 'result_1', + 'type': dict({ + 'kind': 'NamedType', + 'name': 'None', + 'qname': 'builtins.None', + }), + }), + ]) +# --- +# name: test_function_results[none_result_3] + list([ + dict({ + 'id': 'tests/data/various_modules_package/function_module/none_result_3/result_1', 'name': 'result_1', 'type': dict({ 'kind': 'NamedType', @@ -5949,6 +5975,27 @@ 'results': list([ ]), }), + dict({ + 'docstring': dict({ + 'description': '', + 'examples': list([ + ]), + 'full_docstring': '', + }), + 'id': 'tests/data/various_modules_package/function_module/alias_subclass_param_type', + 'is_class_method': False, + 'is_property': False, + 'is_public': True, + 'is_static': False, + 'name': 'alias_subclass_param_type', + 'parameters': list([ + 'tests/data/various_modules_package/function_module/alias_subclass_param_type/x', + ]), + 'reexported_by': list([ + ]), + 'results': list([ + ]), + }), dict({ 'docstring': dict({ 'description': '', @@ -6458,18 +6505,60 @@ ]), 'full_docstring': '', }), - 'id': 'tests/data/various_modules_package/function_module/none_result', + 'id': 'tests/data/various_modules_package/function_module/none_result_1', + 'is_class_method': False, + 'is_property': False, + 'is_public': True, + 'is_static': False, + 'name': 'none_result_1', + 'parameters': list([ + ]), + 'reexported_by': list([ + ]), + 'results': list([ + 'tests/data/various_modules_package/function_module/none_result_1/result_1', + ]), + }), + dict({ + 'docstring': dict({ + 'description': '', + 'examples': list([ + ]), + 'full_docstring': '', + }), + 'id': 'tests/data/various_modules_package/function_module/none_result_2', + 'is_class_method': False, + 'is_property': False, + 'is_public': True, + 'is_static': False, + 'name': 'none_result_2', + 'parameters': list([ + ]), + 'reexported_by': list([ + ]), + 'results': list([ + 'tests/data/various_modules_package/function_module/none_result_2/result_1', + ]), + }), + dict({ + 'docstring': dict({ + 'description': '', + 'examples': list([ + ]), + 'full_docstring': '', + }), + 'id': 'tests/data/various_modules_package/function_module/none_result_3', 'is_class_method': False, 'is_property': False, 'is_public': True, 'is_static': False, - 'name': 'none_result', + 'name': 'none_result_3', 'parameters': list([ ]), 'reexported_by': list([ ]), 'results': list([ - 'tests/data/various_modules_package/function_module/none_result/result_1', + 'tests/data/various_modules_package/function_module/none_result_3/result_1', ]), }), dict({ @@ -6662,6 +6751,7 @@ 'tests/data/various_modules_package/function_module/params_with_default_value/tuple_', 'tests/data/various_modules_package/function_module/params_with_default_value/literal', 'tests/data/various_modules_package/function_module/params_with_default_value/any_', + 'tests/data/various_modules_package/function_module/params_with_default_value/single_quote', ]), 'reexported_by': list([ ]), @@ -6888,27 +6978,6 @@ 'tests/data/various_modules_package/function_module/return_param4/result_4', ]), }), - dict({ - 'docstring': dict({ - 'description': '', - 'examples': list([ - ]), - 'full_docstring': '', - }), - 'id': 'tests/data/various_modules_package/function_module/return_without_result', - 'is_class_method': False, - 'is_property': False, - 'is_public': True, - 'is_static': False, - 'name': 'return_without_result', - 'parameters': list([ - ]), - 'reexported_by': list([ - ]), - 'results': list([ - 'tests/data/various_modules_package/function_module/return_without_result/result_1', - ]), - }), dict({ 'docstring': dict({ 'description': '', @@ -7119,6 +7188,10 @@ 'alias': None, 'qualified_name': 'file_creation.module_1.Lv2', }), + dict({ + 'alias': None, + 'qualified_name': 'tests.data.various_modules_package.reexport_test', + }), ]) # --- # name: test_imports[__init__ (wildcard_imports)] @@ -7240,6 +7313,10 @@ 'alias': None, 'qualified_name': 'file_creation.module_1.Lv2', }), + dict({ + 'alias': None, + 'qualified_name': 'tests.data.various_modules_package.reexport_test', + }), ]), 'wildcard_imports': list([ dict({ @@ -7480,12 +7557,19 @@ 'alias': None, 'qualified_name': 'typing.TYPE_CHECKING', }), + dict({ + 'alias': None, + 'qualified_name': 'not_reexported', + }), dict({ 'alias': None, 'qualified_name': 'another_module._YetAnotherPrivateClass', }), ]), 'wildcard_imports': list([ + dict({ + 'module_name': 'not_reexported', + }), ]), }) # --- @@ -7582,6 +7666,23 @@ ]), }) # --- +# name: test_modules[another_path/not_reexported] + dict({ + 'classes': list([ + ]), + 'docstring': '', + 'enums': list([ + ]), + 'functions': list([ + ]), + 'id': 'tests/data/various_modules_package/another_path/not_reexported', + 'name': 'not_reexported', + 'qualified_imports': list([ + ]), + 'wildcard_imports': list([ + ]), + }) +# --- # name: test_modules[attribute_module] dict({ 'classes': list([ @@ -7917,6 +8018,24 @@ ]), }) # --- +# name: test_modules[file_creation/package_1/not_reexported] + dict({ + 'classes': list([ + 'tests/data/various_modules_package/file_creation/package_1/not_reexported/NotReexported', + ]), + 'docstring': '', + 'enums': list([ + ]), + 'functions': list([ + ]), + 'id': 'tests/data/various_modules_package/file_creation/package_1/not_reexported', + 'name': 'not_reexported', + 'qualified_imports': list([ + ]), + 'wildcard_imports': list([ + ]), + }) +# --- # name: test_modules[function_module] dict({ 'classes': list([ @@ -7943,7 +8062,6 @@ 'tests/data/various_modules_package/function_module/str_result', 'tests/data/various_modules_package/function_module/bool_result', 'tests/data/various_modules_package/function_module/float_result', - 'tests/data/various_modules_package/function_module/none_result', 'tests/data/various_modules_package/function_module/obj_result', 'tests/data/various_modules_package/function_module/callexr_result_class', 'tests/data/various_modules_package/function_module/callexr_result_function', @@ -7974,10 +8092,13 @@ 'tests/data/various_modules_package/function_module/return_param3', 'tests/data/various_modules_package/function_module/return_param4', 'tests/data/various_modules_package/function_module/return_not_statement', - 'tests/data/various_modules_package/function_module/return_without_result', 'tests/data/various_modules_package/function_module/type_alias_param', 'tests/data/various_modules_package/function_module/alias_subclass_result_type', + 'tests/data/various_modules_package/function_module/alias_subclass_param_type', 'tests/data/various_modules_package/function_module/different_result_operants', + 'tests/data/various_modules_package/function_module/none_result_1', + 'tests/data/various_modules_package/function_module/none_result_2', + 'tests/data/various_modules_package/function_module/none_result_3', ]), 'id': 'tests/data/various_modules_package/function_module', 'name': 'function_module', @@ -8099,6 +8220,75 @@ ]), }) # --- +# name: test_modules[reexport_test/__init__] + dict({ + 'classes': list([ + ]), + 'docstring': '', + 'enums': list([ + ]), + 'functions': list([ + ]), + 'id': 'tests/data/various_modules_package/reexport_test', + 'name': '__init__', + 'qualified_imports': list([ + ]), + 'wildcard_imports': list([ + ]), + }) +# --- +# name: test_modules[reexport_test/reexport_test_2/__init__] + dict({ + 'classes': list([ + ]), + 'docstring': '', + 'enums': list([ + ]), + 'functions': list([ + ]), + 'id': 'tests/data/various_modules_package/reexport_test/reexport_test_2', + 'name': '__init__', + 'qualified_imports': list([ + ]), + 'wildcard_imports': list([ + ]), + }) +# --- +# name: test_modules[reexport_test/reexport_test_2/reexport_test_3/__init__] + dict({ + 'classes': list([ + ]), + 'docstring': '', + 'enums': list([ + ]), + 'functions': list([ + ]), + 'id': 'tests/data/various_modules_package/reexport_test/reexport_test_2/reexport_test_3', + 'name': '__init__', + 'qualified_imports': list([ + ]), + 'wildcard_imports': list([ + ]), + }) +# --- +# name: test_modules[reexport_test/reexport_test_2/reexport_test_3/reexport_test] + dict({ + 'classes': list([ + 'tests/data/various_modules_package/reexport_test/reexport_test_2/reexport_test_3/reexport_test/NotReexportedTest', + ]), + 'docstring': '', + 'enums': list([ + ]), + 'functions': list([ + ]), + 'id': 'tests/data/various_modules_package/reexport_test/reexport_test_2/reexport_test_3/reexport_test', + 'name': 'reexport_test', + 'qualified_imports': list([ + ]), + 'wildcard_imports': list([ + ]), + }) +# --- # name: test_modules[type_var_module] dict({ 'classes': list([ diff --git a/tests/safeds_stubgen/api_analyzer/test__get_api.py b/tests/safeds_stubgen/api_analyzer/test__get_api.py index 25f4edfd..c8f6285a 100644 --- a/tests/safeds_stubgen/api_analyzer/test__get_api.py +++ b/tests/safeds_stubgen/api_analyzer/test__get_api.py @@ -470,7 +470,9 @@ def test_function_parameters( ("int_result", _function_module_name, "", "plaintext"), ("str_result", _function_module_name, "", "plaintext"), ("float_result", _function_module_name, "", "plaintext"), - ("none_result", _function_module_name, "", "plaintext"), + ("none_result_1", _function_module_name, "", "plaintext"), + ("none_result_2", _function_module_name, "", "plaintext"), + ("none_result_3", _function_module_name, "", "plaintext"), ("obj_result", _function_module_name, "", "plaintext"), ("callexr_result_class", _function_module_name, "", "plaintext"), ("callexr_result_function", _function_module_name, "", "plaintext"), @@ -510,7 +512,9 @@ def test_function_parameters( "int_result", "str_result", "float_result", - "none_result", + "none_result_1", + "none_result_2", + "none_result_3", "obj_result", "callexr_result_class", "callexr_result_function", diff --git a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[function_module].sdsstub b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[function_module].sdsstub index ec930c2e..319c2b00 100644 --- a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[function_module].sdsstub +++ b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[function_module].sdsstub @@ -67,7 +67,8 @@ fun paramsWithDefaultValue( optional: Int? = null, @PythonName("tuple_") tuple: Tuple, `literal`: literal<"Some String"> = "Some String", - @PythonName("any_") any: Any = false + @PythonName("any_") any: Any = false, + @PythonName("single_quote") singleQuote: String = "\"" ) // TODO List type has to many type arguments. @@ -347,6 +348,13 @@ fun typeAliasParam( @PythonName("alias_subclass_result_type") fun aliasSubclassResultType() -> result1: union +// TODO Result type information missing. +@Pure +@PythonName("alias_subclass_param_type") +fun aliasSubclassParamType( + x: union +) + // TODO Some parameter have no type information. @Pure @PythonName("different_result_operants") diff --git a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[not_reexported].sdsstub b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[not_reexported].sdsstub new file mode 100644 index 00000000..ba0b3870 --- /dev/null +++ b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[not_reexported].sdsstub @@ -0,0 +1,4 @@ +@PythonModule("tests.data.various_modules_package.file_creation.package_1.not_reexported") +package tests.data.variousModulesPackage.fileCreation.package1.notReexported + +class NotReexported() diff --git a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[reexport_test].sdsstub b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[reexport_test].sdsstub new file mode 100644 index 00000000..c9889e99 --- /dev/null +++ b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/TestStubFileGeneration.test_stub_creation[reexport_test].sdsstub @@ -0,0 +1,4 @@ +@PythonModule("tests.data.various_modules_package.reexport_test.reexport_test_2.reexport_test_3.reexport_test") +package tests.data.variousModulesPackage.reexportTest.reexportTest2.reexportTest3.reexportTest + +class NotReexportedTest() diff --git a/tests/safeds_stubgen/stubs_generator/test_generate_stubs.py b/tests/safeds_stubgen/stubs_generator/test_generate_stubs.py index 9f68979b..a32278bc 100644 --- a/tests/safeds_stubgen/stubs_generator/test_generate_stubs.py +++ b/tests/safeds_stubgen/stubs_generator/test_generate_stubs.py @@ -50,6 +50,7 @@ def test_file_creation() -> None: ), ("tests/data/various_modules_package/file_creation/module_1", "module_1"), ("tests/data/various_modules_package/file_creation/package_1/module_5", "module_5"), + ("tests/data/various_modules_package/file_creation/package_1/not_reexported", "not_reexported"), ("tests/data/various_modules_package/file_creation/public_reexported", "public_reexported"), ("tests/data/various_modules_package/file_creation", "reexported_from_another_package_3"), ( From a6e640bc66d2a49e87d94443dadf273634b1cfb4 Mon Sep 17 00:00:00 2001 From: Arsam Date: Thu, 3 Oct 2024 22:33:10 +0200 Subject: [PATCH 17/25] Big performance fix for the docstring parser by creating an indexer for the loaded griffe data instead of parsing through it every time we search for a specific docstring --- .../docstring_parsing/_docstring_parser.py | 118 +++++++----------- 1 file changed, 48 insertions(+), 70 deletions(-) diff --git a/src/safeds_stubgen/docstring_parsing/_docstring_parser.py b/src/safeds_stubgen/docstring_parsing/_docstring_parser.py index 87a06b8e..3dc5ab35 100644 --- a/src/safeds_stubgen/docstring_parsing/_docstring_parser.py +++ b/src/safeds_stubgen/docstring_parsing/_docstring_parser.py @@ -3,8 +3,8 @@ import logging from typing import TYPE_CHECKING, Literal +from _griffe import models as griffe_models from griffe import load -from griffe.dataclasses import Docstring from griffe.docstrings.dataclasses import DocstringAttribute, DocstringParameter from griffe.docstrings.utils import parse_annotation from griffe.enumerations import DocstringSectionKind, Parser @@ -24,35 +24,49 @@ if TYPE_CHECKING: from pathlib import Path - - from griffe.dataclasses import Object from mypy import nodes class DocstringParser(AbstractDocstringParser): def __init__(self, parser: Parser, package_path: Path): + self.parser = parser + while True: # If a package has no __init__.py file Griffe can't parse it, therefore we check the parent try: - self.griffe_build = load(package_path, docstring_parser=parser) + griffe_build = load(package_path, docstring_parser=parser) break except KeyError: package_path = package_path.parent - self.parser = parser - self.__cached_node: str | None = None - self.__cached_docstring: Docstring | None = None + self.griffe_index: dict[str, griffe_models.Object] = {} + self._recursive_griffe_indexer(griffe_build) + + def _recursive_griffe_indexer(self, griffe_build: griffe_models.Object | griffe_models.Alias) -> None: + for member in griffe_build.all_members.values(): + if isinstance( + member, + griffe_models.Class | griffe_models.Function | griffe_models.Attribute | griffe_models.Alias + ): + self.griffe_index[member.path] = member + + if isinstance(member, griffe_models.Module | griffe_models.Class): + self._recursive_griffe_indexer(member) def get_class_documentation(self, class_node: nodes.ClassDef) -> ClassDocstring: - griffe_node = self._get_griffe_node(class_node.fullname) + griffe_node = self.griffe_index[class_node.fullname] if class_node.fullname in self.griffe_index else None if griffe_node is None: # pragma: no cover - raise TypeError(f"Expected a griffe node for {class_node.fullname}, got None.") + msg = ( + f"Something went wrong while searching for the docstring for {class_node.fullname}. Please make sure" + " that all directories with python files have an __init__.py file.", + ) + logging.warning(msg) description = "" docstring = "" examples = [] - if griffe_node.docstring is not None: + if griffe_node is not None and griffe_node.docstring is not None: docstring = griffe_node.docstring.value.strip("\n") try: @@ -76,7 +90,7 @@ def get_function_documentation(self, function_node: nodes.FuncDef) -> FunctionDo docstring = "" description = "" examples = [] - griffe_docstring = self.__get_cached_docstring(function_node.fullname) + griffe_docstring = self._get_griffe_docstring(function_node.fullname) if griffe_docstring is not None: docstring = griffe_docstring.value.strip("\n") @@ -110,9 +124,9 @@ def get_parameter_documentation( # For constructors (__init__ functions) the parameters are described on the class if function_name == "__init__" and parent_class_qname: parent_qname = parent_class_qname.replace("/", ".") - griffe_docstring = self.__get_cached_docstring(parent_qname) + griffe_docstring = self._get_griffe_docstring(parent_qname) else: - griffe_docstring = self.__get_cached_docstring(function_qname) + griffe_docstring = self._get_griffe_docstring(function_qname) # Find matching parameter docstrings matching_parameters = [] @@ -123,7 +137,7 @@ def get_parameter_documentation( # https://github.com/Safe-DS/Library-Analyzer/issues/10) if self.parser == Parser.numpy and len(matching_parameters) == 0 and function_name == "__init__": # Get constructor docstring & find matching parameter docstrings - constructor_docstring = self.__get_cached_docstring(function_qname) + constructor_docstring = self._get_griffe_docstring(function_qname) if constructor_docstring is not None: matching_parameters = self._get_matching_docstrings(constructor_docstring, parameter_name, "param") @@ -136,7 +150,7 @@ def get_parameter_documentation( raise TypeError(f"Expected parameter docstring, got {type(last_parameter)}.") if griffe_docstring is None: # pragma: no cover - griffe_docstring = Docstring("") + griffe_docstring = griffe_models.Docstring("") annotation = last_parameter.annotation if annotation is None: @@ -154,19 +168,15 @@ def get_parameter_documentation( description=last_parameter.description.strip("\n") or "", ) - def get_attribute_documentation( - self, - parent_class_qname: str, - attribute_name: str, - ) -> AttributeDocstring: + def get_attribute_documentation(self, parent_class_qname: str, attribute_name: str) -> AttributeDocstring: parent_class_qname = parent_class_qname.replace("/", ".") # Find matching attribute docstrings parent_qname = parent_class_qname - griffe_docstring = self.__get_cached_docstring(parent_qname) + griffe_docstring = self._get_griffe_docstring(parent_qname) if griffe_docstring is None: matching_attributes = [] - griffe_docstring = Docstring("") + griffe_docstring = griffe_models.Docstring("") else: matching_attributes = self._get_matching_docstrings(griffe_docstring, attribute_name, "attr") @@ -174,7 +184,7 @@ def get_attribute_documentation( # (see issue https://github.com/Safe-DS/Library-Analyzer/issues/10) if self.parser == Parser.numpy and len(matching_attributes) == 0: constructor_qname = f"{parent_class_qname}.__init__" - constructor_docstring = self.__get_cached_docstring(constructor_qname) + constructor_docstring = self._get_griffe_docstring(constructor_qname) # Find matching parameter docstrings if constructor_docstring is not None: @@ -198,7 +208,7 @@ def get_attribute_documentation( def get_result_documentation(self, function_qname: str) -> list[ResultDocstring]: # Find matching parameter docstrings - griffe_docstring = self.__get_cached_docstring(function_qname) + griffe_docstring = self._get_griffe_docstring(function_qname) if griffe_docstring is None: return [] @@ -251,7 +261,7 @@ def get_result_documentation(self, function_qname: str) -> list[ResultDocstring] @staticmethod def _get_matching_docstrings( - function_doc: Docstring, + function_doc: griffe_models.Docstring, name: str, type_: Literal["attr", "param"], ) -> list[DocstringAttribute | DocstringParameter]: @@ -278,7 +288,7 @@ def _get_matching_docstrings( def _griffe_annotation_to_api_type( self, annotation: Expr | str, - docstring: Docstring, + docstring: griffe_models.Docstring, ) -> sds_types.AbstractType | None: if isinstance(annotation, ExprName | ExprAttribute): if annotation.canonical_path == "typing.Any": @@ -291,6 +301,8 @@ def _griffe_annotation_to_api_type( return sds_types.NamedType(name="float", qname="builtins.float") elif annotation.canonical_path == "str": return sds_types.NamedType(name="str", qname="builtins.str") + elif annotation.canonical_path == "bytes": + return sds_types.NamedType(name="bytes", qname="builtins.bytes") elif annotation.canonical_path == "list": return sds_types.ListType(types=[]) elif annotation.canonical_path == "tuple": @@ -403,49 +415,15 @@ def _remove_default_from_griffe_annotation(self, annotation: str) -> str: return annotation.split(", default")[0] return annotation - def _get_griffe_node(self, qname: str) -> Object | None: - node_qname_parts = qname.split(".") - griffe_node = self.griffe_build - for part in node_qname_parts: - if part in griffe_node.modules: - griffe_node = griffe_node.modules[part] - elif part in griffe_node.classes: - griffe_node = griffe_node.classes[part] - elif part in griffe_node.functions: - griffe_node = griffe_node.functions[part] - elif part in griffe_node.attributes: - griffe_node = griffe_node.attributes[part] - elif part == "__init__" and griffe_node.is_class: - return None - elif griffe_node.name == part: - continue - else: # pragma: no cover - msg = ( - f"Something went wrong while searching for the docstring for {qname}. Please make sure" - " that all directories with python files have an __init__.py file.", - ) - logging.warning(msg) - - return griffe_node - - def __get_cached_docstring(self, qname: str) -> Docstring | None: - """ - Return the Docstring for the given function node. + def _get_griffe_docstring(self, qname: str) -> griffe_models.Docstring | None: + griffe_node = self.griffe_index[qname] if qname in self.griffe_index else None - It is only recomputed when the function node differs from the previous one that was passed to this function. - This avoids reparsing the docstring for the function itself and all of its parameters. + if griffe_node is not None: + return griffe_node.docstring - On Lars's system this caused a significant performance improvement: Previously, 8.382s were spent inside the - function get_parameter_documentation when parsing sklearn. Afterward, it was only 2.113s. - """ - if self.__cached_node != qname or qname.endswith("__init__"): - self.__cached_node = qname - - griffe_node = self._get_griffe_node(qname) - if griffe_node is not None: - griffe_docstring = griffe_node.docstring - self.__cached_docstring = griffe_docstring - else: - self.__cached_docstring = None - - return self.__cached_docstring + msg = ( + f"Something went wrong while searching for the docstring for {qname}. Please make sure" + " that all directories with python files have an __init__.py file.", + ) + logging.warning(msg) + return None From adc0a08ca87035f494f21081a1c824a19dc0fd57 Mon Sep 17 00:00:00 2001 From: Arsam Date: Fri, 4 Oct 2024 14:30:46 +0200 Subject: [PATCH 18/25] Trying to reduce the runtime for the "_add_to_imports" function in the stub generator, which probably takes about O(n^3) right now. --- .../stubs_generator/_stub_string_generator.py | 64 ++++++++++++------- 1 file changed, 42 insertions(+), 22 deletions(-) diff --git a/src/safeds_stubgen/stubs_generator/_stub_string_generator.py b/src/safeds_stubgen/stubs_generator/_stub_string_generator.py index 133582fe..e07cd1d6 100644 --- a/src/safeds_stubgen/stubs_generator/_stub_string_generator.py +++ b/src/safeds_stubgen/stubs_generator/_stub_string_generator.py @@ -50,8 +50,9 @@ def __init__(self, api: API, convert_identifiers: bool) -> None: self.class_generics: list = [] self.module_imports: set[str] = set() self.currently_creating_reexport_data: bool = False + self.import_index: dict[str, str] = {} - self.api = api + self.api: API = api self.naming_convention = NamingConvention.SAFE_DS if convert_identifiers else NamingConvention.PYTHON self.classes_outside_package: set[str] = set() self.reexport_modules: dict[str, list[Class | Function]] = defaultdict(list) @@ -1012,13 +1013,13 @@ def _has_node_shorter_reexport(self, node: Class | Function) -> bool: return False def _is_path_connected_to_class(self, path: str, class_path: str) -> bool: - if class_path.endswith(path): + if class_path.endswith(f".{path}") or class_path == path: return True name = path.split("/")[-1] class_name = class_path.split("/")[-1] for reexport in self.api.reexport_map: - if reexport.endswith(name): + if reexport.endswith(f".{name}") or reexport == name: for module in self.api.reexport_map[reexport]: # Added "no cover" since I can't recreate this in the tests if ( @@ -1047,28 +1048,47 @@ def _add_to_imports(self, import_qname: str) -> None: module_id = self._get_module_id(get_actual_id=True).replace("/", ".") if module_id not in import_qname: - # We need the full path for an import from the same package, but we sometimes don't get enough information, - # therefore we have to search for the class and get its id - import_qname_path = import_qname.replace(".", "/") - in_package = False qname = "" - for class_id in self.api.classes: - if self._is_path_connected_to_class(import_qname_path, class_id): - qname = class_id.replace("/", ".") - - name = qname.split(".")[-1] - shortest_qname, _ = _get_shortest_public_reexport_and_alias( - reexport_map=self.api.reexport_map, - name=name, - qname=qname, - is_module=False, - ) + module_id_parts = module_id.split(".") + + # First we hope that we already found and indexed the type we are searching + if import_qname in self.import_index: + qname = self.import_index[import_qname] + + # To save performance we next try to build the possible paths the type could originate from + if not qname: + for i in range(1, len(module_id_parts)): + test_id = ".".join(module_id_parts[:-i]) + "." + import_qname + if test_id.replace(".", "/") in self.api.classes: + qname = test_id + break + + # If the tries above did not work we have to use this performance heavy way. + # We need the full path for an import from the same package, but we sometimes don't get enough + # information, therefore we have to search for the class and get its id + if not qname: + import_qname_path = import_qname.replace(".", "/") + for class_id in self.api.classes: + if self._is_path_connected_to_class(import_qname_path, class_id): + qname = class_id.replace("/", ".") + break + + in_package = False + if qname: + self.import_index[import_qname] = qname + + name = qname.split(".")[-1] + shortest_qname, _ = _get_shortest_public_reexport_and_alias( + reexport_map=self.api.reexport_map, + name=name, + qname=qname, + is_module=False, + ) - if shortest_qname: - qname = f"{shortest_qname}.{name}" + if shortest_qname: + qname = f"{shortest_qname}.{name}" - in_package = True - break + in_package = True qname = qname or import_qname From 5a47d30fca8f89d0b134a3f3e3cdce5aed7078e0 Mon Sep 17 00:00:00 2001 From: Arsam Islami Date: Thu, 17 Oct 2024 02:41:53 +0200 Subject: [PATCH 19/25] Runtime fix for the O(n^3) bug --- src/safeds_stubgen/stubs_generator/_stub_string_generator.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/safeds_stubgen/stubs_generator/_stub_string_generator.py b/src/safeds_stubgen/stubs_generator/_stub_string_generator.py index e07cd1d6..db518e6c 100644 --- a/src/safeds_stubgen/stubs_generator/_stub_string_generator.py +++ b/src/safeds_stubgen/stubs_generator/_stub_string_generator.py @@ -1068,8 +1068,10 @@ def _add_to_imports(self, import_qname: str) -> None: # information, therefore we have to search for the class and get its id if not qname: import_qname_path = import_qname.replace(".", "/") + import_path_name = import_qname_path.split("/")[-1] for class_id in self.api.classes: - if self._is_path_connected_to_class(import_qname_path, class_id): + if (import_path_name == class_id.split("/")[-1] and + self._is_path_connected_to_class(import_qname_path, class_id)): qname = class_id.replace("/", ".") break From 26577a1c40ba399df8a03c082295d0a57024845b Mon Sep 17 00:00:00 2001 From: Arsam Date: Thu, 17 Oct 2024 12:35:46 +0200 Subject: [PATCH 20/25] Optimizing the _check_publicity_in_reexports function --- src/safeds_stubgen/api_analyzer/_ast_visitor.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/src/safeds_stubgen/api_analyzer/_ast_visitor.py b/src/safeds_stubgen/api_analyzer/_ast_visitor.py index 6fd3567b..92373cf4 100644 --- a/src/safeds_stubgen/api_analyzer/_ast_visitor.py +++ b/src/safeds_stubgen/api_analyzer/_ast_visitor.py @@ -1455,18 +1455,12 @@ def _check_publicity_in_reexports(self, name: str, qname: str, parent: Module | continue # If the whole module was reexported we have to check if the name or alias is intern - if module_is_reexported: + if module_is_reexported and not_internal and (isinstance(parent, Module) or parent.is_public): # Check the wildcard imports of the source for wildcard_import in reexport_source.wildcard_imports: - if ( - ( - (is_from_same_package and wildcard_import.module_name == module_name) - or (is_from_another_package and wildcard_import.module_name == module_qname) - ) - and not_internal - and (isinstance(parent, Module) or parent.is_public) - ): + if ((is_from_same_package and wildcard_import.module_name == module_name) + or (is_from_another_package and wildcard_import.module_name == module_qname)): return True # Check the qualified imports of the source @@ -1477,11 +1471,9 @@ def _check_publicity_in_reexports(self, name: str, qname: str, parent: Module | if ( qualified_import.qualified_name in {module_name, module_qname} and ( - (qualified_import.alias is None and not_internal) + qualified_import.alias is None or (qualified_import.alias is not None and not is_internal(qualified_import.alias)) ) - and not_internal - and (isinstance(parent, Module) or parent.is_public) ): # If the module name or alias is not internal, check if the parent is public return True From ee06dc2cfe57bfb980652828c75e4cf7ac255875 Mon Sep 17 00:00:00 2001 From: Arsam Date: Sun, 10 Nov 2024 13:42:50 +0100 Subject: [PATCH 21/25] Removed unused code --- src/safeds_stubgen/api_analyzer/_ast_visitor.py | 2 -- src/safeds_stubgen/api_analyzer/_get_api.py | 1 - 2 files changed, 3 deletions(-) diff --git a/src/safeds_stubgen/api_analyzer/_ast_visitor.py b/src/safeds_stubgen/api_analyzer/_ast_visitor.py index 6fd3567b..a597552d 100644 --- a/src/safeds_stubgen/api_analyzer/_ast_visitor.py +++ b/src/safeds_stubgen/api_analyzer/_ast_visitor.py @@ -59,7 +59,6 @@ def __init__( aliases: dict[str, set[str]], type_source_preference: TypeSourcePreference, type_source_warning: TypeSourceWarning, - is_test_run: bool = False, ) -> None: self.docstring_parser: AbstractDocstringParser = docstring_parser self.type_source_preference = type_source_preference @@ -70,7 +69,6 @@ def __init__( self.mypy_file: mp_nodes.MypyFile | None = None # We gather type var types used as a parameter type in a function self.type_var_types: set[sds_types.TypeVarType] = set() - self.is_test_run = is_test_run def enter_moduledef(self, node: mp_nodes.MypyFile) -> None: self.mypy_file = node diff --git a/src/safeds_stubgen/api_analyzer/_get_api.py b/src/safeds_stubgen/api_analyzer/_get_api.py index 6b3fd50c..cb10edf8 100644 --- a/src/safeds_stubgen/api_analyzer/_get_api.py +++ b/src/safeds_stubgen/api_analyzer/_get_api.py @@ -78,7 +78,6 @@ def get_api( aliases=aliases, type_source_preference=type_source_preference, type_source_warning=type_source_warning, - is_test_run=is_test_run, ) walker = ASTWalker(handler=callable_visitor) From bcbc7d02f1815db4fbf31a5327f2fbec16b4c659 Mon Sep 17 00:00:00 2001 From: Lars Reimann Date: Fri, 7 Mar 2025 17:25:01 +0100 Subject: [PATCH 22/25] style: fix ruff errors --- src/safeds_stubgen/api_analyzer/_ast_visitor.py | 11 +++++------ .../docstring_parsing/_docstring_parser.py | 7 ++++--- .../stubs_generator/_stub_string_generator.py | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/safeds_stubgen/api_analyzer/_ast_visitor.py b/src/safeds_stubgen/api_analyzer/_ast_visitor.py index a6f2ee35..2be70811 100644 --- a/src/safeds_stubgen/api_analyzer/_ast_visitor.py +++ b/src/safeds_stubgen/api_analyzer/_ast_visitor.py @@ -775,8 +775,7 @@ def _create_inferred_results( if type__ not in result_array[i]: result_array[i].append(type__) - if len(result_array[i]) > longest_inner_list: - longest_inner_list = len(result_array[i]) + longest_inner_list = max(longest_inner_list, len(result_array[i])) else: result_array.append([type__]) @@ -860,8 +859,8 @@ def _parse_attributes( lvalues = list(lvalue.items) for lvalue_ in lvalues: if ( - hasattr(lvalue_, "name") - and self._is_attribute_already_defined(lvalue_.name) + (hasattr(lvalue_, "name") + and self._is_attribute_already_defined(lvalue_.name)) or isinstance(lvalue_, mp_nodes.IndexExpr) ): continue @@ -1483,8 +1482,8 @@ def _check_publicity_in_reexports(self, name: str, qname: str, parent: Module | for qualified_import in reexport_source.qualified_imports: if qname.endswith(qualified_import.qualified_name) and ( - qualified_import.alias is not None - and not is_internal(qualified_import.alias) + (qualified_import.alias is not None + and not is_internal(qualified_import.alias)) or (qualified_import.alias is None and not_internal) ): # First we check if we've found the right import then do the following: diff --git a/src/safeds_stubgen/docstring_parsing/_docstring_parser.py b/src/safeds_stubgen/docstring_parsing/_docstring_parser.py index 3dc5ab35..f55dabb6 100644 --- a/src/safeds_stubgen/docstring_parsing/_docstring_parser.py +++ b/src/safeds_stubgen/docstring_parsing/_docstring_parser.py @@ -24,6 +24,7 @@ if TYPE_CHECKING: from pathlib import Path + from mypy import nodes @@ -46,7 +47,7 @@ def _recursive_griffe_indexer(self, griffe_build: griffe_models.Object | griffe_ for member in griffe_build.all_members.values(): if isinstance( member, - griffe_models.Class | griffe_models.Function | griffe_models.Attribute | griffe_models.Alias + griffe_models.Class | griffe_models.Function | griffe_models.Attribute | griffe_models.Alias, ): self.griffe_index[member.path] = member @@ -54,7 +55,7 @@ def _recursive_griffe_indexer(self, griffe_build: griffe_models.Object | griffe_ self._recursive_griffe_indexer(member) def get_class_documentation(self, class_node: nodes.ClassDef) -> ClassDocstring: - griffe_node = self.griffe_index[class_node.fullname] if class_node.fullname in self.griffe_index else None + griffe_node = self.griffe_index.get(class_node.fullname, None) if griffe_node is None: # pragma: no cover msg = ( @@ -416,7 +417,7 @@ def _remove_default_from_griffe_annotation(self, annotation: str) -> str: return annotation def _get_griffe_docstring(self, qname: str) -> griffe_models.Docstring | None: - griffe_node = self.griffe_index[qname] if qname in self.griffe_index else None + griffe_node = self.griffe_index.get(qname, None) if griffe_node is not None: return griffe_node.docstring diff --git a/src/safeds_stubgen/stubs_generator/_stub_string_generator.py b/src/safeds_stubgen/stubs_generator/_stub_string_generator.py index db518e6c..fac601cb 100644 --- a/src/safeds_stubgen/stubs_generator/_stub_string_generator.py +++ b/src/safeds_stubgen/stubs_generator/_stub_string_generator.py @@ -341,8 +341,8 @@ def _create_class_method_string( for method in methods: # Add methods of internal classes that are inherited if the methods themselfe are public if ( - not method.is_public - and (not is_internal_class or (is_internal_class and is_internal(method.name))) + (not method.is_public + and (not is_internal_class or (is_internal_class and is_internal(method.name)))) or method.name in already_defined_names ): continue From 806b7e996571a0ff4a94de563d1cad98c797f1ce Mon Sep 17 00:00:00 2001 From: Lars Reimann Date: Fri, 7 Mar 2025 17:31:06 +0100 Subject: [PATCH 23/25] fix: mypy error --- src/safeds_stubgen/docstring_parsing/_docstring_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/safeds_stubgen/docstring_parsing/_docstring_parser.py b/src/safeds_stubgen/docstring_parsing/_docstring_parser.py index f55dabb6..0dde2aff 100644 --- a/src/safeds_stubgen/docstring_parsing/_docstring_parser.py +++ b/src/safeds_stubgen/docstring_parsing/_docstring_parser.py @@ -60,7 +60,7 @@ def get_class_documentation(self, class_node: nodes.ClassDef) -> ClassDocstring: if griffe_node is None: # pragma: no cover msg = ( f"Something went wrong while searching for the docstring for {class_node.fullname}. Please make sure" - " that all directories with python files have an __init__.py file.", + " that all directories with python files have an __init__.py file." ) logging.warning(msg) From a7db44ca32cfc7515e9d0ae4e24d66c25a4a988d Mon Sep 17 00:00:00 2001 From: Arsam Date: Fri, 4 Apr 2025 00:37:01 +0200 Subject: [PATCH 24/25] Fix for the import generation --- src/safeds_stubgen/stubs_generator/_stub_string_generator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/safeds_stubgen/stubs_generator/_stub_string_generator.py b/src/safeds_stubgen/stubs_generator/_stub_string_generator.py index fac601cb..f1dd9b9e 100644 --- a/src/safeds_stubgen/stubs_generator/_stub_string_generator.py +++ b/src/safeds_stubgen/stubs_generator/_stub_string_generator.py @@ -1013,13 +1013,13 @@ def _has_node_shorter_reexport(self, node: Class | Function) -> bool: return False def _is_path_connected_to_class(self, path: str, class_path: str) -> bool: - if class_path.endswith(f".{path}") or class_path == path: + if class_path.endswith(f"/{path}") or class_path == path: return True name = path.split("/")[-1] class_name = class_path.split("/")[-1] for reexport in self.api.reexport_map: - if reexport.endswith(f".{name}") or reexport == name: + if reexport.endswith(f"/{name}") or reexport == name: for module in self.api.reexport_map[reexport]: # Added "no cover" since I can't recreate this in the tests if ( From 2a3cb0955f6121dbcadce6d71f600a870c0586ed Mon Sep 17 00:00:00 2001 From: Arsam Date: Fri, 4 Apr 2025 00:45:32 +0200 Subject: [PATCH 25/25] Added test for bytes types conversion to Safe-DS stubs from docstrings --- .../stubs_generator/_stub_string_generator.py | 4 ++-- tests/data/docstring_parser_package/googledoc.py | 7 ++++--- tests/data/docstring_parser_package/numpydoc.py | 7 ++++--- tests/data/docstring_parser_package/restdoc.py | 8 +++++--- ...test_stub_docstring_creation[googledoc-GOOGLE].sdsstub | 1 + ...est_stub_docstring_creation[numpydoc-NUMPYDOC].sdsstub | 1 + .../test_stub_docstring_creation[restdoc-REST].sdsstub | 1 + 7 files changed, 18 insertions(+), 11 deletions(-) diff --git a/src/safeds_stubgen/stubs_generator/_stub_string_generator.py b/src/safeds_stubgen/stubs_generator/_stub_string_generator.py index f1dd9b9e..d6072439 100644 --- a/src/safeds_stubgen/stubs_generator/_stub_string_generator.py +++ b/src/safeds_stubgen/stubs_generator/_stub_string_generator.py @@ -1020,13 +1020,13 @@ def _is_path_connected_to_class(self, path: str, class_path: str) -> bool: class_name = class_path.split("/")[-1] for reexport in self.api.reexport_map: if reexport.endswith(f"/{name}") or reexport == name: - for module in self.api.reexport_map[reexport]: + for module in self.api.reexport_map[reexport]: # pragma: no cover # Added "no cover" since I can't recreate this in the tests if ( path.startswith(module.id) and class_path.startswith(module.id) and path.lstrip(module.id).lstrip("/") == name == class_name - ): # pragma: no cover + ): return True return False diff --git a/tests/data/docstring_parser_package/googledoc.py b/tests/data/docstring_parser_package/googledoc.py index 70781cc7..e06aba66 100644 --- a/tests/data/docstring_parser_package/googledoc.py +++ b/tests/data/docstring_parser_package/googledoc.py @@ -175,6 +175,7 @@ class ClassWithVariousParameterTypes: bool_type (bool): str_type (str): float_type (float): + byte_type (bytes): multiple_types (int, bool): list_type_1 (list): list_type_2 (list[str]): @@ -195,9 +196,9 @@ class ClassWithVariousParameterTypes: """ def __init__( - self, no_type, optional_type, none_type, int_type, bool_type, str_type, float_type, multiple_types, list_type_1, - list_type_2, list_type_3, list_type_4, list_type_5, set_type_1, set_type_2, set_type_3, set_type_4, set_type_5, - tuple_type_1, tuple_type_2, tuple_type_3, tuple_type_4, any_type: Any, + self, no_type, optional_type, none_type, int_type, bool_type, str_type, float_type, byte_type, multiple_types, + list_type_1, list_type_2, list_type_3, list_type_4, list_type_5, set_type_1, set_type_2, set_type_3, set_type_4, + set_type_5, tuple_type_1, tuple_type_2, tuple_type_3, tuple_type_4, any_type: Any, optional_type_2: Optional[int], class_type, imported_type ) -> None: pass diff --git a/tests/data/docstring_parser_package/numpydoc.py b/tests/data/docstring_parser_package/numpydoc.py index b066ff52..9a7a1386 100644 --- a/tests/data/docstring_parser_package/numpydoc.py +++ b/tests/data/docstring_parser_package/numpydoc.py @@ -308,6 +308,7 @@ class ClassWithVariousParameterTypes: bool_type : bool str_type : str float_type : float + byte_type : bytes multiple_types : int, bool list_type_1 : list list_type_2 : list[str] @@ -328,9 +329,9 @@ class ClassWithVariousParameterTypes: """ def __init__( - self, no_type, optional_type, none_type, int_type, bool_type, str_type, float_type, multiple_types, list_type_1, - list_type_2, list_type_3, list_type_4, list_type_5, set_type_1, set_type_2, set_type_3, set_type_4, set_type_5, - tuple_type_1, tuple_type_2, tuple_type_3, tuple_type_4, any_type: Any, + self, no_type, optional_type, none_type, int_type, bool_type, str_type, float_type, byte_type, multiple_types, + list_type_1, list_type_2, list_type_3, list_type_4, list_type_5, set_type_1, set_type_2, set_type_3, set_type_4, + set_type_5, tuple_type_1, tuple_type_2, tuple_type_3, tuple_type_4, any_type: Any, optional_type_2: Optional[int], class_type, imported_type ) -> None: pass diff --git a/tests/data/docstring_parser_package/restdoc.py b/tests/data/docstring_parser_package/restdoc.py index 94e34a5b..9ae96883 100644 --- a/tests/data/docstring_parser_package/restdoc.py +++ b/tests/data/docstring_parser_package/restdoc.py @@ -164,6 +164,8 @@ class ClassWithVariousParameterTypes: :type str_type: str :param float_type: :type float_type: float + :param byte_type: + :type byte_type: bytes :param multiple_types: :type multiple_types: int, bool :param list_type_1: @@ -201,9 +203,9 @@ class ClassWithVariousParameterTypes: """ def __init__( - self, no_type, optional_type, none_type, int_type, bool_type, str_type, float_type, multiple_types, list_type_1, - list_type_2, list_type_3, list_type_4, list_type_5, set_type_1, set_type_2, set_type_3, set_type_4, set_type_5, - tuple_type_1, tuple_type_2, tuple_type_3, tuple_type_4, any_type: Any, + self, no_type, optional_type, none_type, int_type, bool_type, str_type, float_type, byte_type, multiple_types, + list_type_1, list_type_2, list_type_3, list_type_4, list_type_5, set_type_1, set_type_2, set_type_3, set_type_4, + set_type_5, tuple_type_1, tuple_type_2, tuple_type_3, tuple_type_4, any_type: Any, optional_type_2: Optional[int], class_type, imported_type ) -> None: pass diff --git a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/test_stub_docstring_creation[googledoc-GOOGLE].sdsstub b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/test_stub_docstring_creation[googledoc-GOOGLE].sdsstub index 3846e8fe..b1d7a11d 100644 --- a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/test_stub_docstring_creation[googledoc-GOOGLE].sdsstub +++ b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/test_stub_docstring_creation[googledoc-GOOGLE].sdsstub @@ -250,6 +250,7 @@ class ClassWithVariousParameterTypes( @PythonName("bool_type") boolType: Boolean, @PythonName("str_type") strType: String, @PythonName("float_type") floatType: Float, + @PythonName("byte_type") byteType: bytes, @PythonName("multiple_types") multipleTypes: Tuple, @PythonName("list_type_1") listType1: List, @PythonName("list_type_2") listType2: List, diff --git a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/test_stub_docstring_creation[numpydoc-NUMPYDOC].sdsstub b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/test_stub_docstring_creation[numpydoc-NUMPYDOC].sdsstub index 94d6886a..69da1e9a 100644 --- a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/test_stub_docstring_creation[numpydoc-NUMPYDOC].sdsstub +++ b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/test_stub_docstring_creation[numpydoc-NUMPYDOC].sdsstub @@ -377,6 +377,7 @@ class ClassWithVariousParameterTypes( @PythonName("bool_type") boolType: Boolean, @PythonName("str_type") strType: String, @PythonName("float_type") floatType: Float, + @PythonName("byte_type") byteType: bytes, @PythonName("multiple_types") multipleTypes: Tuple, @PythonName("list_type_1") listType1: List, @PythonName("list_type_2") listType2: List, diff --git a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/test_stub_docstring_creation[restdoc-REST].sdsstub b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/test_stub_docstring_creation[restdoc-REST].sdsstub index aa8939ca..748762da 100644 --- a/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/test_stub_docstring_creation[restdoc-REST].sdsstub +++ b/tests/safeds_stubgen/stubs_generator/__snapshots__/test_generate_stubs/test_stub_docstring_creation[restdoc-REST].sdsstub @@ -191,6 +191,7 @@ class ClassWithVariousParameterTypes( @PythonName("bool_type") boolType: Boolean, @PythonName("str_type") strType: String, @PythonName("float_type") floatType: Float, + @PythonName("byte_type") byteType: bytes, @PythonName("multiple_types") multipleTypes: Tuple, @PythonName("list_type_1") listType1: List, @PythonName("list_type_2") listType2: List,