Python coverage improvements

plugins/python/src/configwrap.cpp

@@ -15,17 +15,12 @@ struct phlex::experimental::py_config_map {
 };
 // clang-format on
 
-PyObject* phlex::experimental::wrap_configuration(configuration const* config)
+PyObject* phlex::experimental::wrap_configuration(configuration const& config)
 {
-  if (!config) {
-    PyErr_SetString(PyExc_ValueError, "provided configuration is null");
-    return nullptr;
-  }
-
   py_config_map* pyconfig =
     (py_config_map*)PhlexConfig_Type.tp_new(&PhlexConfig_Type, nullptr, nullptr);
 
-  pyconfig->ph_config = config;
+  pyconfig->ph_config = &config;
 
   return (PyObject*)pyconfig;
 }
@@ -73,6 +68,11 @@ static PyObject* pcm_subscript(py_config_map* pycmap, PyObject* pykey)
 
   std::string ckey = PyUnicode_AsUTF8(pykey);
 
+  // Note: Python3.14 adds PyLong_FromInt64/PyLong_FromUInt64 to replace the
+  // long long variants
+  static_assert(sizeof(long long) >= sizeof(int64_t));
+  static_assert(sizeof(unsigned long long) >= sizeof(uint64_t));
+
   try {
     auto k = pycmap->ph_config->prototype_internal_kind(ckey);
     if (k.second /* is array */) {
@@ -87,14 +87,16 @@ static PyObject* pcm_subscript(py_config_map* pycmap, PyObject* pykey)
         auto const& cvalue = pycmap->ph_config->get<std::vector<std::int64_t>>(ckey);
         pyvalue = PyTuple_New(cvalue.size());
         for (Py_ssize_t i = 0; i < (Py_ssize_t)cvalue.size(); ++i) {
-          PyObject* item = PyLong_FromLong(cvalue[i]);
+          // Note Python3.14 is expected to add PyLong_FromInt64
+          PyObject* item = PyLong_FromLongLong(cvalue[i]);
           PyTuple_SetItem(pyvalue, i, item);
         }
       } else if (k.first == boost::json::kind::uint64) {
         auto const& cvalue = pycmap->ph_config->get<std::vector<std::uint64_t>>(ckey);
         pyvalue = PyTuple_New(cvalue.size());
         for (Py_ssize_t i = 0; i < (Py_ssize_t)cvalue.size(); ++i) {
-          PyObject* item = PyLong_FromUnsignedLong(cvalue[i]);
+          // Note Python3.14 is expected to add PyLong_FromUInt64
+          PyObject* item = PyLong_FromUnsignedLongLong(cvalue[i]);
           PyTuple_SetItem(pyvalue, i, item);
         }
       } else if (k.first == boost::json::kind::double_) {
@@ -111,17 +113,22 @@ static PyObject* pcm_subscript(py_config_map* pycmap, PyObject* pykey)
           PyObject* item = PyUnicode_FromStringAndSize(cvalue[i].c_str(), cvalue[i].size());
           PyTuple_SetItem(pyvalue, i, item);
         }
+      } else if (k.first == boost::json::kind::null) {
+        // special case: empty array
+        pyvalue = PyTuple_New(0);
       }
     } else {
       if (k.first == boost::json::kind::bool_) {
         auto cvalue = pycmap->ph_config->get<bool>(ckey);
         pyvalue = PyBool_FromLong((long)cvalue);
       } else if (k.first == boost::json::kind::int64) {
         auto cvalue = pycmap->ph_config->get<std::int64_t>(ckey);
-        pyvalue = PyLong_FromLong(cvalue);
+        // Note Python3.14 is expected to add PyLong_FromInt64
+        pyvalue = PyLong_FromLongLong(cvalue);
       } else if (k.first == boost::json::kind::uint64) {
         auto cvalue = pycmap->ph_config->get<std::uint64_t>(ckey);
-        pyvalue = PyLong_FromUnsignedLong(cvalue);
+        // Note Python3.14 is expected to add PyLong_FromUInt64
+        pyvalue = PyLong_FromUnsignedLongLong(cvalue);
       } else if (k.first == boost::json::kind::double_) {
         auto cvalue = pycmap->ph_config->get<double>(ckey);
         pyvalue = PyFloat_FromDouble(cvalue);

plugins/python/src/errorwrap.cpp

@@ -2,6 +2,12 @@
 
 #include <string>
 
+// This code has excluded several error checking paths from code coverage,
+// because the conditions to create the errors (trace formatting problems)
+// are rare and too hard to recreate in a test, while the resolution (fall
+// back to a generic error messsage) is rather straightforward and thus
+// does not need testing.
+
 using namespace phlex::experimental;
 
 static bool format_traceback(std::string& msg,
@@ -25,20 +31,24 @@ static bool format_traceback(std::string& msg,
 #endif
   Py_DECREF(format_exception);
 
+  // LCOV_EXCL_START
   if (!formatted_tb) {
     PyErr_Clear();
     return false;
   }
+  // LCOV_EXCL_STOP
 
   PyObject* py_msg_empty = PyUnicode_FromString("");
   PyObject* py_msg = PyUnicode_Join(py_msg_empty, formatted_tb);
   Py_DECREF(py_msg_empty);
   Py_DECREF(formatted_tb);
 
+  // LCOV_EXCL_START
   if (!py_msg) {
     PyErr_Clear();
     return false;
   }
+  // LCOV_EXCL_STOP
 
   char const* c_msg = PyUnicode_AsUTF8(py_msg);
   if (c_msg) {
@@ -47,9 +57,11 @@ static bool format_traceback(std::string& msg,
     return true;
   }
 
+  // LCOV_EXCL_START
   PyErr_Clear();
   Py_DECREF(py_msg);
   return false;
+  // LCOV_EXCL_STOP
 }
 
 bool phlex::experimental::msg_from_py_error(std::string& msg, bool check_error)
@@ -66,11 +78,13 @@ bool phlex::experimental::msg_from_py_error(std::string& msg, bool check_error)
   PyErr_Fetch(&type, &value, &traceback);
   if (value) {
     bool tb_ok = format_traceback(msg, type, value, traceback);
+    // LCOV_EXCL_START
     if (!tb_ok) {
       PyObject* pymsg = PyObject_Str(value);
       msg = PyUnicode_AsUTF8(pymsg);
       Py_DECREF(pymsg);
     }
+    // LCOV_EXCL_STOP
   } else {
     msg = "unknown Python error occurred";
   }

plugins/python/src/modulewrap.cpp

@@ -29,15 +29,10 @@ struct phlex::experimental::py_phlex_module {
 };
 // clang-format on
 
-PyObject* phlex::experimental::wrap_module(phlex_module_t* module_)
+PyObject* phlex::experimental::wrap_module(phlex_module_t& module_)
 {
-  if (!module_) {
-    PyErr_SetString(PyExc_ValueError, "provided module is null");
-    return nullptr;
-  }
-
   py_phlex_module* pymod = PyObject_New(py_phlex_module, &PhlexModule_Type);
-  pymod->ph_module = module_;
+  pymod->ph_module = &module_;
 
   return (PyObject*)pymod;
 }
@@ -518,8 +513,16 @@ static PyObject* parse_args(PyObject* args,
     return nullptr;
   }
 
+  // special case of Phlex Variant wrapper
+  PyObject* wrapped_callable = PyObject_GetAttrString(callable, "phlex_callable");
+  if (wrapped_callable) {
+    callable = wrapped_callable;
+  } else {
+    PyErr_Clear();
+    Py_INCREF(callable);
+  }
+
   // no common errors detected; actual registration may have more checks
-  Py_INCREF(callable);
   return callable;
 }
 

plugins/python/src/pymodule.cpp

@@ -27,8 +27,8 @@ PHLEX_REGISTER_ALGORITHMS(m, config)
   if (mod) {
     PyObject* reg = PyObject_GetAttrString(mod, "PHLEX_REGISTER_ALGORITHMS");
     if (reg) {
-      PyObject* pym = wrap_module(&m);
-      PyObject* pyconfig = wrap_configuration(&config);
+      PyObject* pym = wrap_module(m);
+      PyObject* pyconfig = wrap_configuration(config);
       if (pym && pyconfig) {
         PyObject* res = PyObject_CallFunctionObjArgs(reg, pym, pyconfig, nullptr);
         Py_XDECREF(res);

plugins/python/src/wrap.hpp

@@ -28,7 +28,7 @@ namespace phlex::experimental {
 
   // Create dict-like access to the configuration from Python.
   // Returns a new reference.
-  PyObject* wrap_configuration(configuration const* config);
+  PyObject* wrap_configuration(configuration const& config);
 
   // Python wrapper for Phlex configuration
   extern PyTypeObject PhlexConfig_Type;
@@ -37,7 +37,7 @@ namespace phlex::experimental {
   // Phlex' Module wrapper to register algorithms
   typedef module_graph_proxy<void_tag> phlex_module_t;
   // Returns a new reference.
-  PyObject* wrap_module(phlex_module_t* mod);
+  PyObject* wrap_module(phlex_module_t& mod);
 
   // Python wrapper for Phlex modules
   extern PyTypeObject PhlexModule_Type;

test/python/adder.py

@@ -4,18 +4,30 @@
 real. It serves as a "Hello, World" equivalent for running Python code.
 """
 
-def add(i: int, j: int) -> int:
+from typing import Protocol, TypeVar
+
+from variant import Variant
+
+
+class AddableProtocol[T](Protocol):
+    """Typer bound for any types that can be added."""
+    def __add__(self, other: T) -> T: ...  # noqa: D105
+
+Addable = TypeVar('Addable', bound=AddableProtocol)
+
+
+def add(i: Addable, j: Addable) -> Addable:
     """Add the inputs together and return the sum total.
 
     Use the standard `+` operator to add the two inputs together
     to arrive at their total.
 
     Args:
-        i (int): First input.
-        j (int): Second input.
+        i (Number): First input.
+        j (Number): Second input.
 
     Returns:
-        int: Sum of the two inputs.
+        Number: Sum of the two inputs.
 
     Examples:
         >>> add(1, 2)
@@ -39,7 +51,8 @@
     Returns:
         None
     """
-    m.transform(add,
+    int_adder = Variant(add, {"i": int, "j": int, "return": int}, "iadd")
+    m.transform(int_adder,
                 input_family = config["input"],
                 output_products = config["output"])
 

test/python/all_config.py

@@ -30,17 +30,28 @@
         # builtin types
         assert config['a_bool'] == False # noqa: E712  # we really want to check False
         assert config['an_int'] == -37
-        assert config['a_uint'] ==  18446744073709551616
+        assert type(config['a_uint']) is int
+        assert config['a_uint'] == 18446744073709549568
         assert config['a_float'] == 3.1415
         assert config['a_string'] == 'foo'
 
         # collection types
         assert config['some_bools'] == (False, True)
         assert config['some_ints'] == (-1, 42, -55)
-        assert config['some_uints'] == (18446744073709551616, 29, 137)
+        assert type(config['some_uints'][0]) is int
+        assert config['some_uints'] == (18446744073709549568, 29, 137)
         assert config['some_floats'] == (3.1415, 2.71828)
         assert config['some_strings'] == ('aap', 'noot', 'mies')
 
+        # special case of empty collection
+        assert config['empty'] == ()
+
+        try:
+            config[42]   # should raise
+            assert not "did not raise TypeError"
+        except TypeError:
+            pass
+
     def __call__(self, i: int, j: int) -> None:
         """Dummy routine to do something.
 

test/python/pyconfig.jsonnet

@@ -16,14 +16,15 @@
       input: ['i', 'j'],
       a_bool: false,
       an_int: -37,
-      a_uint: 18446744073709551616,
+      a_uint: 18446744073709549568,
       a_float: 3.1415,
       a_string: 'foo',
       some_bools: [false, true],
       some_ints: [-1, 42, -55],
-      some_uints: [18446744073709551616, 29, 137],
+      some_uints: [18446744073709549568, 29, 137],
       some_floats: [3.1415, 2.71828],
       some_strings: ['aap', 'noot', 'mies'],
+      empty: [],
     },
   },
 }

test/python/variant.py

@@ -0,0 +1,79 @@
+"""Annotation helper for C++ typing variants.
+
+Python algorithms are generic, like C++ templates, but the Phlex registration
+process requires a single unique signature. These helpers generate annotated
+functions for registration with the proper C++ types.
+"""
+
+import copy
+from typing import Any, Callable
+
+
+class Variant:
+    """Wrapper to associate custom annotations with a callable.
+
+    This class wraps a callable and provides custom ``__annotations__`` and
+    ``__name__`` attributes, allowing the same underlying function or callable
+    object to be registered multiple times with different type annotations.
+
+    By default, the provided callable is kept by reference, but can be cloned
+    (e.g. for callable instances) if requested.
+
+    Phlex will recognize the "phlex_callable" data member, allowing an unwrap
+    and thus saving an indirection. To detect performance degradation, the
+    wrapper is not callable by default.
+
+    Attributes:
+        phlex_callable (Callable): The underlying callable (public).
+        __annotations__ (dict): Type information of arguments and return product.
+        __name__ (str): The name associated with this variant.
+
+    Examples:
+        >>> def add(i: Number, j: Number) -> Number:
+        ...     return i + j
+        ...
+        >>> int_adder = variant(add, {"i": int, "j": int, "return": int}, "iadd")
+    """
+    def __init__(
+        self,
+        f: Callable,
+        annotations: dict[str, str | type | Any],
+        name: str,
+        clone: bool | str = False,
+        allow_call: bool = False,
+    ):
+        """Annotate the callable F.
+
+        Args:
+            f (Callable): Annotable function.
+            annotations (dict): Type information of arguments and return product.
+            name (str): Name to assign to this variant.
+            clone (bool|str): If True (or "deep"), creates a shallow (deep) copy
+                of the callable.
+            allow_call (bool): Allow this wrapper to forward to the callable.
+        """
+        if clone == 'deep':
+            self.phlex_callable = copy.deepcopy(f)
+        elif clone:
+            self.phlex_callable = copy.copy(f)
+        else:
+            self.phlex_callable = f
+        self.__annotations__ = annotations
+        self.__name__ = name
+        self._allow_call = allow_call
+
+    def __call__(self, *args, **kwargs):
+        """Raises an error if called directly.
+
+        Variant instances should not be called directly. The framework should
+        extract ``phlex_callable`` instead and call that.
+
+        Raises:
+            AssertionError: To indicate incorrect usage, unless overridden.
+        """
+        assert self._allow_call, (
+            f"TypedVariant '{self.__name__}' was called directly. "
+            f"The framework should extract phlex_callable instead."
+        )
+        return self.phlex_callable(*args, **kwargs)  # type: ignore
+