From b9baa3538ecd911d3a18bf93db1237a241e1f551 Mon Sep 17 00:00:00 2001 From: Wim Lavrijsen Date: Thu, 8 Jan 2026 12:47:50 -0800 Subject: [PATCH 1/7] support for empty arrays in configuration to python mapping --- plugins/python/src/configwrap.cpp | 3 +++ test/python/all_config.py | 3 +++ test/python/pyconfig.jsonnet | 1 + 3 files changed, 7 insertions(+) diff --git a/plugins/python/src/configwrap.cpp b/plugins/python/src/configwrap.cpp index d019725f..d7ac2303 100644 --- a/plugins/python/src/configwrap.cpp +++ b/plugins/python/src/configwrap.cpp @@ -111,6 +111,9 @@ static PyObject* pcm_subscript(py_config_map* pycmap, PyObject* pykey) PyObject* item = PyUnicode_FromStringAndSize(cvalue[i].c_str(), cvalue[i].size()); PyTuple_SetItem(pyvalue, i, item); } + } else if (k.first == boost::json::kind::null) { + // special case: empty array + pyvalue = PyTuple_New(0); } } else { if (k.first == boost::json::kind::bool_) { diff --git a/test/python/all_config.py b/test/python/all_config.py index 10b4f5df..0acec0c3 100644 --- a/test/python/all_config.py +++ b/test/python/all_config.py @@ -41,6 +41,9 @@ def __init__(self, config): assert config['some_floats'] == (3.1415, 2.71828) assert config['some_strings'] == ('aap', 'noot', 'mies') + # special case of empty collection + assert config['empty'] == () + def __call__(self, i: int, j: int) -> None: """Dummy routine to do something. diff --git a/test/python/pyconfig.jsonnet b/test/python/pyconfig.jsonnet index 2effc8c4..0ebbc892 100644 --- a/test/python/pyconfig.jsonnet +++ b/test/python/pyconfig.jsonnet @@ -24,6 +24,7 @@ some_uints: [18446744073709551616, 29, 137], some_floats: [3.1415, 2.71828], some_strings: ['aap', 'noot', 'mies'], + empty: [], }, }, } From f10587168b73ba2184a3bc244c96fa42960ca1f6 Mon Sep 17 00:00:00 2001 From: Wim Lavrijsen Date: Thu, 8 Jan 2026 13:06:21 -0800 Subject: [PATCH 2/7] use references instead of pointers for wrapping as pointers should never be null --- plugins/python/src/configwrap.cpp | 9 ++------- plugins/python/src/modulewrap.cpp | 9 ++------- plugins/python/src/pymodule.cpp | 4 ++-- plugins/python/src/wrap.hpp | 4 ++-- 4 files changed, 8 insertions(+), 18 deletions(-) diff --git a/plugins/python/src/configwrap.cpp b/plugins/python/src/configwrap.cpp index d7ac2303..a1ec5f40 100644 --- a/plugins/python/src/configwrap.cpp +++ b/plugins/python/src/configwrap.cpp @@ -15,17 +15,12 @@ struct phlex::experimental::py_config_map { }; // clang-format on -PyObject* phlex::experimental::wrap_configuration(configuration const* config) +PyObject* phlex::experimental::wrap_configuration(configuration const& config) { - if (!config) { - PyErr_SetString(PyExc_ValueError, "provided configuration is null"); - return nullptr; - } - py_config_map* pyconfig = (py_config_map*)PhlexConfig_Type.tp_new(&PhlexConfig_Type, nullptr, nullptr); - pyconfig->ph_config = config; + pyconfig->ph_config = &config; return (PyObject*)pyconfig; } diff --git a/plugins/python/src/modulewrap.cpp b/plugins/python/src/modulewrap.cpp index e1525a42..aa202f92 100644 --- a/plugins/python/src/modulewrap.cpp +++ b/plugins/python/src/modulewrap.cpp @@ -29,15 +29,10 @@ struct phlex::experimental::py_phlex_module { }; // clang-format on -PyObject* phlex::experimental::wrap_module(phlex_module_t* module_) +PyObject* phlex::experimental::wrap_module(phlex_module_t& module_) { - if (!module_) { - PyErr_SetString(PyExc_ValueError, "provided module is null"); - return nullptr; - } - py_phlex_module* pymod = PyObject_New(py_phlex_module, &PhlexModule_Type); - pymod->ph_module = module_; + pymod->ph_module = &module_; return (PyObject*)pymod; } diff --git a/plugins/python/src/pymodule.cpp b/plugins/python/src/pymodule.cpp index ef86de63..86e9a816 100644 --- a/plugins/python/src/pymodule.cpp +++ b/plugins/python/src/pymodule.cpp @@ -27,8 +27,8 @@ PHLEX_REGISTER_ALGORITHMS(m, config) if (mod) { PyObject* reg = PyObject_GetAttrString(mod, "PHLEX_REGISTER_ALGORITHMS"); if (reg) { - PyObject* pym = wrap_module(&m); - PyObject* pyconfig = wrap_configuration(&config); + PyObject* pym = wrap_module(m); + PyObject* pyconfig = wrap_configuration(config); if (pym && pyconfig) { PyObject* res = PyObject_CallFunctionObjArgs(reg, pym, pyconfig, nullptr); Py_XDECREF(res); diff --git a/plugins/python/src/wrap.hpp b/plugins/python/src/wrap.hpp index f0818dd1..0f5e9385 100644 --- a/plugins/python/src/wrap.hpp +++ b/plugins/python/src/wrap.hpp @@ -28,7 +28,7 @@ namespace phlex::experimental { // Create dict-like access to the configuration from Python. // Returns a new reference. - PyObject* wrap_configuration(configuration const* config); + PyObject* wrap_configuration(configuration const& config); // Python wrapper for Phlex configuration extern PyTypeObject PhlexConfig_Type; @@ -37,7 +37,7 @@ namespace phlex::experimental { // Phlex' Module wrapper to register algorithms typedef module_graph_proxy phlex_module_t; // Returns a new reference. - PyObject* wrap_module(phlex_module_t* mod); + PyObject* wrap_module(phlex_module_t& mod); // Python wrapper for Phlex modules extern PyTypeObject PhlexModule_Type; From aa96f29a4153c8477788c404d46fe07a33bfcd6b Mon Sep 17 00:00:00 2001 From: Wim Lavrijsen Date: Thu, 8 Jan 2026 14:58:44 -0800 Subject: [PATCH 3/7] correctly test unsigned integer configuration values --- plugins/python/src/configwrap.cpp | 17 +++++++++++++---- test/python/all_config.py | 12 ++++++++++-- test/python/pyconfig.jsonnet | 4 ++-- 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/plugins/python/src/configwrap.cpp b/plugins/python/src/configwrap.cpp index a1ec5f40..34c44168 100644 --- a/plugins/python/src/configwrap.cpp +++ b/plugins/python/src/configwrap.cpp @@ -68,6 +68,11 @@ static PyObject* pcm_subscript(py_config_map* pycmap, PyObject* pykey) std::string ckey = PyUnicode_AsUTF8(pykey); + // Note: Python3.14 adds PyLong_FromInt64/PyLong_FromUInt64 to replace the + // long long variants + static_assert(sizeof(long long) >= sizeof(int64_t)); + static_assert(sizeof(unsigned long long) >= sizeof(uint64_t)); + try { auto k = pycmap->ph_config->prototype_internal_kind(ckey); if (k.second /* is array */) { @@ -82,14 +87,16 @@ static PyObject* pcm_subscript(py_config_map* pycmap, PyObject* pykey) auto const& cvalue = pycmap->ph_config->get>(ckey); pyvalue = PyTuple_New(cvalue.size()); for (Py_ssize_t i = 0; i < (Py_ssize_t)cvalue.size(); ++i) { - PyObject* item = PyLong_FromLong(cvalue[i]); + // Note Python3.14 is expected to add PyLong_FromInt64 + PyObject* item = PyLong_FromLongLong(cvalue[i]); PyTuple_SetItem(pyvalue, i, item); } } else if (k.first == boost::json::kind::uint64) { auto const& cvalue = pycmap->ph_config->get>(ckey); pyvalue = PyTuple_New(cvalue.size()); for (Py_ssize_t i = 0; i < (Py_ssize_t)cvalue.size(); ++i) { - PyObject* item = PyLong_FromUnsignedLong(cvalue[i]); + // Note Python3.14 is expected to add PyLong_FromUInt64 + PyObject* item = PyLong_FromUnsignedLongLong(cvalue[i]); PyTuple_SetItem(pyvalue, i, item); } } else if (k.first == boost::json::kind::double_) { @@ -116,10 +123,12 @@ static PyObject* pcm_subscript(py_config_map* pycmap, PyObject* pykey) pyvalue = PyBool_FromLong((long)cvalue); } else if (k.first == boost::json::kind::int64) { auto cvalue = pycmap->ph_config->get(ckey); - pyvalue = PyLong_FromLong(cvalue); + // Note Python3.14 is expected to add PyLong_FromInt64 + pyvalue = PyLong_FromLongLong(cvalue); } else if (k.first == boost::json::kind::uint64) { auto cvalue = pycmap->ph_config->get(ckey); - pyvalue = PyLong_FromUnsignedLong(cvalue); + // Note Python3.14 is expected to add PyLong_FromUInt64 + pyvalue = PyLong_FromUnsignedLongLong(cvalue); } else if (k.first == boost::json::kind::double_) { auto cvalue = pycmap->ph_config->get(ckey); pyvalue = PyFloat_FromDouble(cvalue); diff --git a/test/python/all_config.py b/test/python/all_config.py index 0acec0c3..7255290c 100644 --- a/test/python/all_config.py +++ b/test/python/all_config.py @@ -30,20 +30,28 @@ def __init__(self, config): # builtin types assert config['a_bool'] == False # noqa: E712 # we really want to check False assert config['an_int'] == -37 - assert config['a_uint'] == 18446744073709551616 + assert type(config['a_uint']) == int + assert config['a_uint'] == 18446744073709549568 assert config['a_float'] == 3.1415 assert config['a_string'] == 'foo' # collection types assert config['some_bools'] == (False, True) assert config['some_ints'] == (-1, 42, -55) - assert config['some_uints'] == (18446744073709551616, 29, 137) + assert type(config['some_uints'][0]) == int + assert config['some_uints'] == (18446744073709549568, 29, 137) assert config['some_floats'] == (3.1415, 2.71828) assert config['some_strings'] == ('aap', 'noot', 'mies') # special case of empty collection assert config['empty'] == () + try: + config[42] # should raise + assert not "did not raise TypeError" + except TypeError: + pass + def __call__(self, i: int, j: int) -> None: """Dummy routine to do something. diff --git a/test/python/pyconfig.jsonnet b/test/python/pyconfig.jsonnet index 0ebbc892..7b8b0350 100644 --- a/test/python/pyconfig.jsonnet +++ b/test/python/pyconfig.jsonnet @@ -16,12 +16,12 @@ input: ['i', 'j'], a_bool: false, an_int: -37, - a_uint: 18446744073709551616, + a_uint: 18446744073709549568, a_float: 3.1415, a_string: 'foo', some_bools: [false, true], some_ints: [-1, 42, -55], - some_uints: [18446744073709551616, 29, 137], + some_uints: [18446744073709549568, 29, 137], some_floats: [3.1415, 2.71828], some_strings: ['aap', 'noot', 'mies'], empty: [], From a482934b9c480c0772cf9355f2cb3044e144f3d2 Mon Sep 17 00:00:00 2001 From: Wim Lavrijsen Date: Fri, 9 Jan 2026 11:38:48 -0800 Subject: [PATCH 4/7] add variant helper to simplify typing --- plugins/python/src/modulewrap.cpp | 10 +++- test/python/adder.py | 16 +++++-- test/python/variant.py | 80 +++++++++++++++++++++++++++++++ 3 files changed, 100 insertions(+), 6 deletions(-) create mode 100644 test/python/variant.py diff --git a/plugins/python/src/modulewrap.cpp b/plugins/python/src/modulewrap.cpp index aa202f92..040eddff 100644 --- a/plugins/python/src/modulewrap.cpp +++ b/plugins/python/src/modulewrap.cpp @@ -513,8 +513,16 @@ static PyObject* parse_args(PyObject* args, return nullptr; } + // special case of Phlex Variant wrapper + PyObject* wrapped_callable = PyObject_GetAttrString(callable, "phlex_callable"); + if (wrapped_callable) { + callable = wrapped_callable; + } else { + PyErr_Clear(); + Py_INCREF(callable); + } + // no common errors detected; actual registration may have more checks - Py_INCREF(callable); return callable; } diff --git a/test/python/adder.py b/test/python/adder.py index d4889447..973f6fb1 100644 --- a/test/python/adder.py +++ b/test/python/adder.py @@ -4,18 +4,23 @@ real. It serves as a "Hello, World" equivalent for running Python code. """ -def add(i: int, j: int) -> int: +from numbers import Number + +from variant import Variant + + +def add(i: Number, j: Number) -> Number: """Add the inputs together and return the sum total. Use the standard `+` operator to add the two inputs together to arrive at their total. Args: - i (int): First input. - j (int): Second input. + i (Number): First input. + j (Number): Second input. Returns: - int: Sum of the two inputs. + Number: Sum of the two inputs. Examples: >>> add(1, 2) @@ -39,7 +44,8 @@ def PHLEX_REGISTER_ALGORITHMS(m, config): Returns: None """ - m.transform(add, + int_adder = Variant(add, {"i": int, "j": int, "return": int}, "iadd") + m.transform(int_adder, input_family = config["input"], output_products = config["output"]) diff --git a/test/python/variant.py b/test/python/variant.py new file mode 100644 index 00000000..d09fe434 --- /dev/null +++ b/test/python/variant.py @@ -0,0 +1,80 @@ +"""Annotation helper for C++ typing variants. + +Python algorithms are generic, like C++ templates, but the Phlex registration +process requires a single unique signature. These helpers generate annotated +functions for registration with the proper C++ types. +""" + +import copy +import types +from typing import Any, Callable + + +class Variant: + """Wrapper to associate custom annotations with a callable. + + This class wraps a callable and provides custom ``__annotations__`` and + ``__name__`` attributes, allowing the same underlying function or callable + object to be registered multiple times with different type annotations. + + By default, the provided callable is kept by reference, but can be cloned + (e.g. for callable instances) if requested. + + Phlex will recognize the "phlex_callable" data member, allowing an unwrap + and thus saving an indirection. To detect performance degradation, the + wrapper is not callable by default. + + Attributes: + phlex_callable (Callable): The underlying callable (public). + __annotations__ (dict): Type information of arguments and return product. + __name__ (str): The name associated with this variant. + + Examples: + >>> def add(i: Number, j: Number) -> Number: + ... return i + j + ... + >>> int_adder = variant(add, {"i": int, "j": int, "return": int}, "iadd") + """ + def __init__( + self, + f: Callable, + annotations: dict[str, str | type | Any], + name: str, + clone: bool | str = False, + allow_call: bool = False, + ): + """Annotate the callable F. + + Args: + f (Callable): Annotable function. + annotations (dict): Type information of arguments and return product. + name (str): Name to assign to this variant. + clone (bool|str): If True (or "deep"), creates a shallow (deep) copy + of the callable. + allow_call (bool): Allow this wrapper to forward to the callable. + """ + if clone == 'deep': + self.phlex_callable = copy.deepcopy(f) + elif clone: + self.phlex_callable = copy.copy(f) + else: + self.phlex_callable = f + self.__annotations__ = annotations + self.__name__ = name + self._allow_call = allow_call + + def __call__(self, *args, **kwargs): + """Raises an error if called directly. + + Variant instances should not be called directly. The framework should + extract ``phlex_callable`` instead and call that. + + Raises: + AssertionError: To indicate incorrect usage, unless overridden. + """ + assert self._allow_call, ( + f"TypedVariant '{self.__name__}' was called directly. " + f"The framework should extract phlex_callable instead." + ) + return self.phlex_callable(*args, **kwargs) # type: ignore + From 13bcbdc2210adddd84da1a2ff56228a86e3261bc Mon Sep 17 00:00:00 2001 From: Wim Lavrijsen Date: Fri, 9 Jan 2026 11:50:38 -0800 Subject: [PATCH 5/7] add coverage ignores for unlikely error resolution paths --- plugins/python/src/errorwrap.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/plugins/python/src/errorwrap.cpp b/plugins/python/src/errorwrap.cpp index c8a073ee..3ccaa8ee 100644 --- a/plugins/python/src/errorwrap.cpp +++ b/plugins/python/src/errorwrap.cpp @@ -2,6 +2,12 @@ #include +// This code has excluded several error checking paths from code coverage, +// because the conditions to create the errors (trace formatting problems) +// are rare and too hard to recreate in a test, while the resolution (fall +// back to a generic error messsage) is rather straightforward and thus +// does not need testing. + using namespace phlex::experimental; static bool format_traceback(std::string& msg, @@ -25,20 +31,24 @@ static bool format_traceback(std::string& msg, #endif Py_DECREF(format_exception); + // LCOV_EXCL_START if (!formatted_tb) { PyErr_Clear(); return false; } + // LCOV_EXCL_STOP PyObject* py_msg_empty = PyUnicode_FromString(""); PyObject* py_msg = PyUnicode_Join(py_msg_empty, formatted_tb); Py_DECREF(py_msg_empty); Py_DECREF(formatted_tb); + // LCOV_EXCL_START if (!py_msg) { PyErr_Clear(); return false; } + // LCOV_EXCL_STOP char const* c_msg = PyUnicode_AsUTF8(py_msg); if (c_msg) { @@ -47,9 +57,11 @@ static bool format_traceback(std::string& msg, return true; } + // LCOV_EXCL_START PyErr_Clear(); Py_DECREF(py_msg); return false; + // LCOV_EXCL_STOP } bool phlex::experimental::msg_from_py_error(std::string& msg, bool check_error) @@ -66,11 +78,13 @@ bool phlex::experimental::msg_from_py_error(std::string& msg, bool check_error) PyErr_Fetch(&type, &value, &traceback); if (value) { bool tb_ok = format_traceback(msg, type, value, traceback); + // LCOV_EXCL_START if (!tb_ok) { PyObject* pymsg = PyObject_Str(value); msg = PyUnicode_AsUTF8(pymsg); Py_DECREF(pymsg); } + // LCOV_EXCL_STOP } else { msg = "unknown Python error occurred"; } From cb85a36452bc22b94c8ae1f27f724020e3e513f7 Mon Sep 17 00:00:00 2001 From: Wim Lavrijsen Date: Fri, 9 Jan 2026 12:50:04 -0800 Subject: [PATCH 6/7] add an adder protocol to satisfy mypy (which doesn't handle Number) --- test/python/adder.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/test/python/adder.py b/test/python/adder.py index 973f6fb1..4b4203ce 100644 --- a/test/python/adder.py +++ b/test/python/adder.py @@ -4,12 +4,19 @@ real. It serves as a "Hello, World" equivalent for running Python code. """ -from numbers import Number +from typing import Protocol, TypeVar from variant import Variant -def add(i: Number, j: Number) -> Number: +class AddableProtocol[T](Protocol): + """Typer bound for any types that can be added.""" + def __add__(self, other: T) -> T: ... # noqa: D105 + +Addable = TypeVar('Addable', bound=AddableProtocol) + + +def add(i: Addable, j: Addable) -> Addable: """Add the inputs together and return the sum total. Use the standard `+` operator to add the two inputs together From b18aa9a4f09e30b3394d0bd2909f7ffee5241aa7 Mon Sep 17 00:00:00 2001 From: Wim Lavrijsen Date: Fri, 9 Jan 2026 14:57:30 -0800 Subject: [PATCH 7/7] fix ruff/mypy issues --- test/python/all_config.py | 4 ++-- test/python/variant.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/test/python/all_config.py b/test/python/all_config.py index 7255290c..5d8132c7 100644 --- a/test/python/all_config.py +++ b/test/python/all_config.py @@ -30,7 +30,7 @@ def __init__(self, config): # builtin types assert config['a_bool'] == False # noqa: E712 # we really want to check False assert config['an_int'] == -37 - assert type(config['a_uint']) == int + assert type(config['a_uint']) is int assert config['a_uint'] == 18446744073709549568 assert config['a_float'] == 3.1415 assert config['a_string'] == 'foo' @@ -38,7 +38,7 @@ def __init__(self, config): # collection types assert config['some_bools'] == (False, True) assert config['some_ints'] == (-1, 42, -55) - assert type(config['some_uints'][0]) == int + assert type(config['some_uints'][0]) is int assert config['some_uints'] == (18446744073709549568, 29, 137) assert config['some_floats'] == (3.1415, 2.71828) assert config['some_strings'] == ('aap', 'noot', 'mies') diff --git a/test/python/variant.py b/test/python/variant.py index d09fe434..2e4a7ac4 100644 --- a/test/python/variant.py +++ b/test/python/variant.py @@ -6,7 +6,6 @@ """ import copy -import types from typing import Any, Callable