KeyError in Autowig with StatisKit.Core

[jpeyhardi]

Hi, I have some KeyError when using Autowig on StatisKit.Core.
Using the Sublime Text build system (Ctrl + Shift + B then choosing "StatisKit"), I have the following error message:

scons: *** [/home/jean/.miniconda/envs/statiskit/lib/python3.6/site-packages/scons_tools/site_autowig/ASG/statiskit_core.pkl] KeyError : Path("'enum ::std::io_errc' parameter")
Traceback (most recent call last):
  File "/home/jean/.miniconda/envs/statiskit/lib/python3.6/site-packages/SCons/Action.py", line 1192, in execute
    result = self.execfunction(target=target, source=rsources, env=env)
  File "/home/jean/.miniconda/envs/statiskit/lib/python3.6/site-packages/scons_tools/site_tools/wig.py", line 109, in boost_python_builder
    **kwargs)
  File "/home/jean/.miniconda/envs/statiskit/lib/python3.6/site-packages/scons_tools/site_autowig/controller/statiskit_core.py", line 4, in controller
    return stl_controller(asg, library=False)
  File "/home/jean/.miniconda/envs/statiskit/lib/python3.6/site-packages/scons_tools/site_autowig/controller/statiskit_stl.py", line 68, in controller
    template = template.desugared_type.unqualified_type
  File "/home/jean/.miniconda/envs/statiskit/lib/python3.6/site-packages/autowig/asg.py", line 805, in desugared_type
    desugared_type = self._asg[self._target]
  File "/home/jean/.miniconda/envs/statiskit/lib/python3.6/site-packages/autowig/asg.py", line 1802, in __getitem__
    raise KeyError('\'' + node + '\' parameter')
KeyError: Path("'enum ::std::io_errc' parameter")
scons: building terminated because of errors.
[Finished in 384.8s with exit code 2]
[cmd: bash -c 'source /home/jean/.miniconda/bin/activate statiskit && scons autowig --diagnostics-color=never && scons --diagnostics-color=never --with-nose-debug=none']
[dir: /home/jean/Bureau/developpement/Core]
[path: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin]

[pfernique]

Looking at the line

  File "/home/jean/.miniconda/envs/statiskit/lib/python3.6/site-packages/scons_tools/site_autowig/controller/statiskit_stl.py", line 68, in controller

the problem seems to be when the StatisKit.STL controller is called.
Can you show the content of this file (all the file, please) in order to see what is happening on line 68.

[jpeyhardi]

import autowig
from autowig.asg import TemplateSpecializationProxy
import itertools

def controller(asg, library=True, **kwargs):
    autowig.controller.plugin = 'default'
    asg = autowig.controller(asg, clean=False, **kwargs)
    if library:
        for function in asg['::statiskit::stl'].functions():
            if function.localname in ['generator', 'insert']:
                parameter = function.parameters[0].qualified_type.desugared_type
                if parameter.is_class:
                    function.parent = parameter.unqualified_type
        for mtd in asg['::statiskit::stl::String'].qualified_type.desugared_type.unqualified_type.methods():
            mtd.boost_python_export = False
    if 'class ::std::vector' in asg:
        for cls in asg['class ::std::vector'].specializations(partial=False):
            for method in cls.methods():
                if method.localname in ['resize', 'shrink_to_fit', 'operator[]']:
                    if isinstance(method.boost_python_export, bool):
                        method.boost_python_export = False
            for constructor in cls.constructors():
                if not(constructor.nb_parameters == 0 or constructor.nb_parameters == 1 and constructor.parameters[0].qualified_type.unqualified_type == cls):
                    if isinstance(constructor.boost_python_export, bool):
                        constructor.boost_python_export = False
    if 'class ::std::set' in asg:
        for cls in asg['class ::std::set'].specializations(partial=False):
            for method in cls.methods():
                if method.localname in ['swap', 'key_comp', 'value_comp', 'get_allocator']:
                    if isinstance(method.boost_python_export, bool):
                        method.boost_python_export = False
            for constructor in cls.constructors():
                if not(constructor.nb_parameters == 0 or constructor.nb_parameters == 1 and constructor.parameters[0].qualified_type.unqualified_type == cls):
                    if isinstance(constructor.boost_python_export, bool):
                        constructor.boost_python_export = False
    if 'class ::std::less' in asg:
        for cls in asg['class ::std::less'].specializations(partial=False):
            cls.boost_python_export = False
    if 'class ::std::hash' in asg:
        for cls in asg['class ::std::hash'].specializations(partial=False):
            cls.boost_python_export = False
    if 'class ::std::char_traits' in asg:
        for cls in asg['class ::std::char_traits'].specializations(partial=False):
            for mtd in cls.methods(access='public'):
                cls.boost_python_export = False
    if 'class ::std::allocator' in asg:
        for cls in asg['class ::std::allocator'].specializations(partial=False):
            cls.boost_python_export = False
    if 'class ::std::reverse_iterator' in asg:
        for cls in asg['class ::std::reverse_iterator'].specializations(partial=False):
            cls.boost_python_export = False
    if 'class ::std::initializer_list' in asg:
        for cls in asg['class ::std::initializer_list'].specializations(partial=False):
            cls.boost_python_export = False
    if 'class ::std::default_delete' in asg:
        for cls in asg['class ::std::default_delete'].specializations(partial=False):
            cls.boost_python_export = False
    if 'class ::std::move_iterator' in asg:
       for cls in asg['class ::std::move_iterator'].specializations(partial=False):
        cls.boost_python_export = False
    for supercls in ['class ::std::_Rb_tree_const_iterator', 'class ::std::_Rb_tree_iterator',
                     'class ::std::__detail::_Node_const_iterator', 'class ::std::__detail::_Node_iterator']:
        if supercls in asg:
            for cls in asg[supercls].specializations(partial=False):
                cls.boost_python_export = False
            for cls in asg.classes(templated=False, specialized=True):
                for template in cls.templates:
                    template = template.desugared_type.unqualified_type
                    if isinstance(template, TemplateSpecializationProxy) and template.specialize.globalname == supercls:
                        cls.boost_python_export = False 
    return asg

def generator(asg, module, decorator):
    autowig.generator.plugin = 'boost_python'
    nodes = [typedef.qualified_type.unqualified_type for typedef in asg['::statiskit::stl'].typedefs()] + asg['class ::std::basic_string'].specializations(partial=False)
    nodes = list(itertools.chain(*[node.bases(inherited=True) for node in nodes])) + nodes + asg['::statiskit::stl'].declarations()
    return autowig.generator(asg, nodes, module=module,
                             decorator=decorator,
                             closure=False,
                             helder='std::shared_ptr')

[pfernique]

This seems to be a bug in AutoWIG.
Replace the controller function by:

def controller(asg, library=True, **kwargs):
    autowig.controller.plugin = 'default'
    asg = autowig.controller(asg, clean=False, **kwargs)
    if library:
        for function in asg['::statiskit::stl'].functions():
            if function.localname in ['generator', 'insert']:
                parameter = function.parameters[0].qualified_type.desugared_type
                if parameter.is_class:
                    function.parent = parameter.unqualified_type
        for mtd in asg['::statiskit::stl::String'].qualified_type.desugared_type.unqualified_type.methods():
            mtd.boost_python_export = False
    if 'class ::std::vector' in asg:
        for cls in asg['class ::std::vector'].specializations(partial=False):
            for method in cls.methods():
                if method.localname in ['resize', 'shrink_to_fit', 'operator[]']:
                    if isinstance(method.boost_python_export, bool):
                        method.boost_python_export = False
            for constructor in cls.constructors():
                if not(constructor.nb_parameters == 0 or constructor.nb_parameters == 1 and constructor.parameters[0].qualified_type.unqualified_type == cls):
                    if isinstance(constructor.boost_python_export, bool):
                        constructor.boost_python_export = False
    if 'class ::std::set' in asg:
        for cls in asg['class ::std::set'].specializations(partial=False):
            for method in cls.methods():
                if method.localname in ['swap', 'key_comp', 'value_comp', 'get_allocator']:
                    if isinstance(method.boost_python_export, bool):
                        method.boost_python_export = False
            for constructor in cls.constructors():
                if not(constructor.nb_parameters == 0 or constructor.nb_parameters == 1 and constructor.parameters[0].qualified_type.unqualified_type == cls):
                    if isinstance(constructor.boost_python_export, bool):
                        constructor.boost_python_export = False
    if 'class ::std::less' in asg:
        for cls in asg['class ::std::less'].specializations(partial=False):
            cls.boost_python_export = False
    if 'class ::std::hash' in asg:
        for cls in asg['class ::std::hash'].specializations(partial=False):
            cls.boost_python_export = False
    if 'class ::std::char_traits' in asg:
        for cls in asg['class ::std::char_traits'].specializations(partial=False):
            for mtd in cls.methods(access='public'):
                cls.boost_python_export = False
    if 'class ::std::allocator' in asg:
        for cls in asg['class ::std::allocator'].specializations(partial=False):
            cls.boost_python_export = False
    if 'class ::std::reverse_iterator' in asg:
        for cls in asg['class ::std::reverse_iterator'].specializations(partial=False):
            cls.boost_python_export = False
    if 'class ::std::initializer_list' in asg:
        for cls in asg['class ::std::initializer_list'].specializations(partial=False):
            cls.boost_python_export = False
    if 'class ::std::default_delete' in asg:
        for cls in asg['class ::std::default_delete'].specializations(partial=False):
            cls.boost_python_export = False
    if 'class ::std::move_iterator' in asg:
       for cls in asg['class ::std::move_iterator'].specializations(partial=False):
        cls.boost_python_export = False
    for supercls in ['class ::std::_Rb_tree_const_iterator', 'class ::std::_Rb_tree_iterator',
                     'class ::std::__detail::_Node_const_iterator', 'class ::std::__detail::_Node_iterator']:
        if supercls in asg:
            for cls in asg[supercls].specializations(partial=False):
                cls.boost_python_export = False
            for cls in asg.classes(templated=False, specialized=True):
                for template in cls.templates:
                    try:
                        template = template.desugared_type.unqualified_type
                        if isinstance(template, TemplateSpecializationProxy) and template.specialize.globalname == supercls:
                            cls.boost_python_export = False 
                    except:
                        print("Error on template: " + template.globalname)
                        pass
    return asg

This will bypass the error and print a message "Error on template: (.*)".
Can you given the complete message printed ?

[jpeyhardi]

scons: *** [/home/jean/.miniconda/envs/statiskit/lib/python3.6/site-packages/scons_tools/site_autowig/ASG/statiskit_core.pkl] KeyError : Path("'enum ::std::io_errc' parameter")
Traceback (most recent call last):
  File "/home/jean/.miniconda/envs/statiskit/lib/python3.6/site-packages/scons_tools/site_autowig/controller/statiskit_stl.py", line 69, in controller
    template = template.desugared_type.unqualified_type
  File "/home/jean/.miniconda/envs/statiskit/lib/python3.6/site-packages/autowig/asg.py", line 805, in desugared_type
    desugared_type = self._asg[self._target]
  File "/home/jean/.miniconda/envs/statiskit/lib/python3.6/site-packages/autowig/asg.py", line 1802, in __getitem__
    raise KeyError('\'' + node + '\' parameter')
KeyError: Path("'enum ::std::io_errc' parameter")

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/jean/.miniconda/envs/statiskit/lib/python3.6/site-packages/SCons/Action.py", line 1192, in execute
    result = self.execfunction(target=target, source=rsources, env=env)
  File "/home/jean/.miniconda/envs/statiskit/lib/python3.6/site-packages/scons_tools/site_tools/wig.py", line 109, in boost_python_builder
    **kwargs)
  File "/home/jean/.miniconda/envs/statiskit/lib/python3.6/site-packages/scons_tools/site_autowig/controller/statiskit_core.py", line 4, in controller
    return stl_controller(asg, library=False)
  File "/home/jean/.miniconda/envs/statiskit/lib/python3.6/site-packages/scons_tools/site_autowig/controller/statiskit_stl.py", line 73, in controller
    print("Error on template: " + template.globalname)
  File "/home/jean/.miniconda/envs/statiskit/lib/python3.6/site-packages/autowig/asg.py", line 793, in globalname
    return self.unqualified_type.globalname + ' ' + self.qualifiers
  File "/home/jean/.miniconda/envs/statiskit/lib/python3.6/site-packages/autowig/asg.py", line 785, in unqualified_type
    return self._asg[self._target]
  File "/home/jean/.miniconda/envs/statiskit/lib/python3.6/site-packages/autowig/asg.py", line 1802, in __getitem__
    raise KeyError('\'' + node + '\' parameter')
KeyError: Path("'enum ::std::io_errc' parameter")
scons: building terminated because of errors.
[Finished in 396.4s with exit code 2]
[cmd: bash -c 'source /home/jean/.miniconda/bin/activate statiskit && scons autowig --diagnostics-color=never && scons --diagnostics-color=never --with-nose-debug=none']
[dir: /home/jean/Bureau/developpement/Core]
[path: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin]

[pfernique]

It seems that template.globalname is raising an error, replace it with template._node

[jpeyhardi]

scons: *** [/home/jean/.miniconda/envs/statiskit/lib/python3.6/site-packages/scons_tools/site_autowig/ASG/statiskit_core.pkl] AttributeError : 'QualifiedTypeProxy' object has no attribute '_node'
Traceback (most recent call last):
  File "/home/jean/.miniconda/envs/statiskit/lib/python3.6/site-packages/scons_tools/site_autowig/controller/statiskit_stl.py", line 69, in controller
    template = template.desugared_type.unqualified_type
  File "/home/jean/.miniconda/envs/statiskit/lib/python3.6/site-packages/autowig/asg.py", line 805, in desugared_type
    desugared_type = self._asg[self._target]
  File "/home/jean/.miniconda/envs/statiskit/lib/python3.6/site-packages/autowig/asg.py", line 1802, in __getitem__
    raise KeyError('\'' + node + '\' parameter')
KeyError: Path("'enum ::std::io_errc' parameter")

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/jean/.miniconda/envs/statiskit/lib/python3.6/site-packages/SCons/Action.py", line 1192, in execute
    result = self.execfunction(target=target, source=rsources, env=env)
  File "/home/jean/.miniconda/envs/statiskit/lib/python3.6/site-packages/scons_tools/site_tools/wig.py", line 109, in boost_python_builder
    **kwargs)
  File "/home/jean/.miniconda/envs/statiskit/lib/python3.6/site-packages/scons_tools/site_autowig/controller/statiskit_core.py", line 4, in controller
    return stl_controller(asg, library=False)
  File "/home/jean/.miniconda/envs/statiskit/lib/python3.6/site-packages/scons_tools/site_autowig/controller/statiskit_stl.py", line 73, in controller
    print("Error on template: " + template._node)
AttributeError: 'QualifiedTypeProxy' object has no attribute '_node'
scons: building terminated because of errors.
[Finished in 391.9s with exit code 2]
[cmd: bash -c 'source /home/jean/.miniconda/bin/activate statiskit && scons autowig --diagnostics-color=never && scons --diagnostics-color=never --with-nose-debug=none']
[dir: /home/jean/Bureau/developpement/Core]
[path: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin]

[pfernique]

My bad, use template._source and template._target in place of template._node.
Note that if you are in a hurry, remove the line print("Error on template: " + template._node).
But don't forget to do the modification afterward in order to be able to correct the bug.

[jpeyhardi]

Ok thanks it works ! Now I have test errors to debug :)

[pfernique]

Don't forget, at some point, to add the line print("Error on template: " + template._source + " -> " + template._target) and to post the result.
I need this in order to debug AutoWIG (This is only a hack)

[jpeyhardi]

I used the line

print("Error on template: " + template._source + " -> " + template._target)

and obtained the following result

[Build error - file "src/py/wrapper/wrapper_b3aefb8f8c96565c95d583848719e5b2.cpp" at line 43, see build results]
build/src/py/wrapper/wrapper_b3aefb8f8c96565c95d583848719e5b2.cpp:43:300: error: template argument 2 is invalid
         boost::python::implicitly_convertible< autowig::Held< class ::statiskit::UniformDistribution >::Type, autowig::Held< struct ::statiskit::PolymorphicCopy< struct ::statiskit::UnivariateDistribution, class ::statiskit::UniformDistribution, struct ::statiskit::ContinuousUnivariateDistribution > >::Type >();
                                                                                                                                                                                                                                                                                                            ^
[Build error - file "src/py/wrapper/wrapper_b3aefb8f8c96565c95d583848719e5b2.cpp" at line 43, see build results]
build/src/py/wrapper/wrapper_b3aefb8f8c96565c95d583848719e5b2.cpp:43:302: error: template argument 1 is invalid
         boost::python::implicitly_convertible< autowig::Held< class ::statiskit::UniformDistribution >::Type, autowig::Held< struct ::statiskit::PolymorphicCopy< struct ::statiskit::UnivariateDistribution, class ::statiskit::UniformDistribution, struct ::statiskit::ContinuousUnivariateDistribution > >::Type >();
                                                                                                                                                                                                                                                                                                              ^
[Build error - file "src/py/wrapper/wrapper_b3aefb8f8c96565c95d583848719e5b2.cpp" at line 43, see build results]
build/src/py/wrapper/wrapper_b3aefb8f8c96565c95d583848719e5b2.cpp:43:312: error: no matching function for call to 'implicitly_convertible<<expression error>, <expression error> >()'
         boost::python::implicitly_convertible< autowig::Held< class ::statiskit::UniformDistribution >::Type, autowig::Held< struct ::statiskit::PolymorphicCopy< struct ::statiskit::UnivariateDistribution, class ::statiskit::UniformDistribution, struct ::statiskit::ContinuousUnivariateDistribution > >::Type >();
                                                                                                                                                                                                                                                                                                                        ^
In file included from /home/jean/.miniconda/envs/statiskit/include/boost/python.hpp:33:0,
                 from build/src/py/wrapper/_core.h:4:
/home/jean/.miniconda/envs/statiskit/include/boost/python/implicit.hpp:20:6: note: candidate: template<class Source, class Target> void boost::python::implicitly_convertible(boost::type<Target>*, boost::type<U>*)
 void implicitly_convertible(boost::type<Source>* = 0, boost::type<Target>* = 0)
      ^~~~~~~~~~~~~~~~~~~~~~
/home/jean/.miniconda/envs/statiskit/include/boost/python/implicit.hpp:20:6: note:   template argument deduction/substitution failed:
[Build error - file "src/py/wrapper/wrapper_b3aefb8f8c96565c95d583848719e5b2.cpp" at line 43, see build results]
build/src/py/wrapper/wrapper_b3aefb8f8c96565c95d583848719e5b2.cpp:43:312: error: template argument 1 is invalid
         boost::python::implicitly_convertible< autowig::Held< class ::statiskit::UniformDistribution >::Type, autowig::Held< struct ::statiskit::PolymorphicCopy< struct ::statiskit::UnivariateDistribution, class ::statiskit::UniformDistribution, struct ::statiskit::ContinuousUnivariateDistribution > >::Type >();
                                                                                                                                                                                                                                                                                                                        ^
[Build error - file "src/py/wrapper/wrapper_b3aefb8f8c96565c95d583848719e5b2.cpp" at line 43, see build results]
build/src/py/wrapper/wrapper_b3aefb8f8c96565c95d583848719e5b2.cpp:43:312: error: template argument 2 is invalid
/home/jean/.miniconda/envs/statiskit/bin/x86_64-conda_cos6-linux-gnu-g++ -o build/src/py/wrapper/wrapper_9d7f0f97517952029268e1fd35ac8843.os -c -std=c++11 -fvisibility=hidden -Wl,--no-undefined -fdiagnostics-color=never -Wno-attributes -Wno-ignored-attributes -Winvalid-pch -fPIC -DBOOST_PYTHON_DYNAMIC_LIB -DBOOST_ALL_NO_LIB -I/home/jean/.miniconda/envs/statiskit/include -I/home/jean/.miniconda/envs/statiskit/include/python3.6m build/src/py/wrapper/wrapper_9d7f0f97517952029268e1fd35ac8843.cpp
scons: *** [build/src/py/wrapper/wrapper_b3aefb8f8c96565c95d583848719e5b2.os] Error 1
scons: building terminated because of errors.
[Finished in 800.1s with exit code 2]
[cmd: bash -c 'source /home/jean/.miniconda/bin/activate statiskit && scons autowig --diagnostics-color=never && scons --diagnostics-color=never --with-nose-debug=none']
[dir: /home/jean/Bureau/developpement/Core]
[path: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin]

[pfernique]

This is another issue to open since it concerns the compilation of generated wrappers.
This current issue concerns an error on wrapper generation.

With the hack, wrappers were generated using the Sublime Text build system (Ctrl + Shift + B then choosing StatisKit) which is equivalent to the following command lines:

scons autowig
scons install

Hence, to see the error raised when generating wrappers you need to remove the wrappers and then re-generate them.
To do so:

1. Use the Sublime Text build system (Ctrl + Shift + B then choose the Clean variant of StatisKit) which is equivalent to the following command lines:

   scons autowig -c
   scons install -c

2. Re-use the Sublime Text build system (Ctrl + Shift + B then choose StatisKit) .

[jpeyhardi]

Ok
I add two print in the python file

                    except:
                        print("########################################################")
                        print("Error on template: " + template.globalname)
                        print("########################################################")
                        pass

after 1 and 2 instructions, the result is

#########################################################
Error on template: struct ::std::is_error_code_enum< enum ::std::io_errc >enum ::std::io_errc
#########################################################
#########################################################
Error on template: struct ::std::is_error_condition_enum< enum ::std::errc >enum ::std::errc
#########################################################
#########################################################
Error on template: struct ::std::is_error_code_enum< enum ::std::io_errc >enum ::std::io_errc
#########################################################
#########################################################
Error on template: struct ::std::is_error_condition_enum< enum ::std::errc >enum ::std::errc
#########################################################
#########################################################
Error on template: struct ::std::is_error_code_enum< enum ::std::io_errc >enum ::std::io_errc
#########################################################
#########################################################
Error on template: struct ::std::is_error_condition_enum< enum ::std::errc >enum ::std::errc
#########################################################
#########################################################
Error on template: struct ::std::is_error_code_enum< enum ::std::io_errc >enum ::std::io_errc
#########################################################
#########################################################
Error on template: struct ::std::is_error_condition_enum< enum ::std::errc >enum ::std::errc
#########################################################

[pfernique]

Ok, looks like at some point the enum ::std::errc is deleted from the ASG but not all template classes using it as template parameter.
Will look further to resolve this.